from __future__ import annotations from ariadne.services import hermes_autotriage as module from tests.hermes_autotriage_harness import ( INCIDENT_ID, INFRA_MARKER, JOB, _build, _counter, _events, _gauge, _prepare, _retry_output, _run, _seed_incident, _settings, _statuses, ) OTHER_JOB = "titan-iac" BOTH_ACTIONS = ["repair_demo_fixture", "retry_transient_infra"] def _retry_settings(**overrides): # type: ignore[no-untyped-def] values = {"hermes_allowed_actions": BOTH_ACTIONS} values.update(overrides) return _settings(**values) def _retry_env(monkeypatch, **overrides): # type: ignore[no-untyped-def] kwargs = {"cfg": _retry_settings(), "run": _run(output=_retry_output())} kwargs.update(overrides) return _prepare(monkeypatch, **kwargs) def test_transient_infra_failure_is_retried(monkeypatch) -> None: success_before = _counter("retry_transient_infra", "success") env = _retry_env(monkeypatch) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB] == { "status": "awaiting_rebuild", "incident_id": INCIDENT_ID, "action": "retry_transient_infra", "evidence_marker": INFRA_MARKER, } assert _statuses(env.storage) == ["detected", "diagnosed", "awaiting_rebuild"] assert env.calls["retries"] == [(JOB, True)] assert env.calls["repairs"] == [] assert env.calls["rebuilds"] == [] assert _counter("retry_transient_infra", "success") == success_before + 1.0 assert _gauge("12", "awaiting_rebuild") == 1.0 def test_retry_records_the_matched_marker_on_every_event(monkeypatch) -> None: env = _retry_env(monkeypatch) module.run_hermes_autotriage(env.storage) actions = _events(env.storage, module.ACTION_EVENT_TYPE) assert [action["result"] for action in actions] == ["requested", "accepted", "executed"] assert all(action["action"] == "retry_transient_infra" for action in actions) assert actions[0]["detail"] == {"evidence_marker": INFRA_MARKER} assert actions[2]["detail"] == {"evidence_marker": INFRA_MARKER} diagnosis = _events(env.storage, module.DIAGNOSIS_EVENT_TYPE)[0] assert diagnosis["authorized"] is True assert diagnosis["authorize_reason"] == "authorized" assert diagnosis["evidence_marker"] == INFRA_MARKER assert diagnosis["outcome"]["classification"] == "transient_infra_failure" incident = _events(env.storage, module.INCIDENT_EVENT_TYPE)[-1] assert incident["phase"] == {"action": "retry_transient_infra", "evidence_marker": INFRA_MARKER} def test_retry_resolves_on_the_next_green_build(monkeypatch) -> None: env = _retry_env(monkeypatch) module.run_hermes_autotriage(env.storage) green = _prepare( monkeypatch, cfg=_retry_settings(), last_build=_build(13, "SUCCESS"), storage=env.storage, ) summary = module.run_hermes_autotriage(green.storage) assert summary["jobs"][JOB] == {"status": "healthy", "resolved": [INCIDENT_ID]} assert _statuses(env.storage)[-1] == "resolved" assert _gauge("12", "resolved") == 1.0 def test_unparameterized_job_is_retried_without_parameters(monkeypatch) -> None: cfg = _retry_settings( hermes_autotriage_job_allowlist=[OTHER_JOB], hermes_parameterized_jobs=[JOB], ) env = _prepare( monkeypatch, cfg=cfg, last_build=_build(12, "FAILURE", job=OTHER_JOB), run=_run(output=_retry_output(job=OTHER_JOB)), ) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][OTHER_JOB]["status"] == "awaiting_rebuild" assert env.calls["retries"] == [(OTHER_JOB, False)] def test_retry_without_infra_signature_requires_a_human(monkeypatch) -> None: env = _retry_env(monkeypatch, infra=(False, None)) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"] == "evidence_signature_missing" assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"] assert env.calls["retries"] == [] assert _events(env.storage, module.DIAGNOSIS_EVENT_TYPE)[0]["evidence_marker"] is None def test_retry_ignores_the_fixture_signature(monkeypatch) -> None: env = _retry_env(monkeypatch, signature=False) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["status"] == "awaiting_rebuild" assert env.calls["retries"] == [(JOB, True)] def test_fixture_action_ignores_the_infra_signature(monkeypatch) -> None: env = _prepare(monkeypatch, cfg=_retry_settings(), signature=False, infra=(True, INFRA_MARKER)) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"] == "evidence_signature_missing" assert env.calls["retries"] == [] def test_retry_is_capped_at_one_action_per_incident(monkeypatch) -> None: env = _retry_env(monkeypatch) _seed_incident(env.storage, "detected") env.storage.record_event( module.ACTION_EVENT_TYPE, {"incident_id": INCIDENT_ID, "action": "retry_transient_infra", "result": "requested"}, ) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"] == "max_actions_reached" assert env.calls["retries"] == [] def test_retry_trigger_failure_marks_failed_and_human(monkeypatch) -> None: failed_before = _counter("retry_transient_infra", "failed") env = _retry_env(monkeypatch, retry={"requested": False, "error": "retry http 500"}) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB] == { "status": "failed", "incident_id": INCIDENT_ID, "reason": "retry http 500", } assert _statuses(env.storage) == ["detected", "diagnosed", "failed"] actions = _events(env.storage, module.ACTION_EVENT_TYPE) assert [action["result"] for action in actions] == ["requested", "accepted", "failed"] assert _counter("retry_transient_infra", "failed") == failed_before + 1.0 assert _gauge("12", "human_required") == 1.0 def test_retry_action_is_rejected_while_not_allowlisted(monkeypatch) -> None: rejected_before = _counter("retry_transient_infra", "rejected") env = _prepare(monkeypatch, run=_run(output=_retry_output())) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"] == "action_not_allowlisted: 'retry_transient_infra'" assert env.calls["retries"] == [] assert _counter("retry_transient_infra", "rejected") == rejected_before + 1.0 def test_action_must_match_its_classification(monkeypatch) -> None: output = _retry_output(requested_action={"type": "run_ariadne_job", "id": "repair_demo_fixture"}) env = _retry_env(monkeypatch, run=_run(output=output)) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"] == ( "action_does_not_match_classification: 'repair_demo_fixture' expected 'retry_transient_infra'" ) assert env.calls["retries"] == [] assert env.calls["repairs"] == [] def test_unregistered_classification_requires_a_human(monkeypatch) -> None: env = _retry_env(monkeypatch, run=_run(output=_retry_output(classification="flaky_network_guess"))) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"] == "classification_not_supported: 'flaky_network_guess'" assert env.calls["retries"] == [] def test_prompt_documents_the_retry_action(monkeypatch) -> None: env = _retry_env(monkeypatch) module.run_hermes_autotriage(env.storage) _, prompt = env.calls["triage"][0] assert "Use transient_infra_failure with requested_action" in prompt assert '{"type": "run_ariadne_job", "id": "retry_transient_infra"}' in prompt assert "unrelated to the repository's code or tests" in prompt assert "Otherwise leave requested_action null" in prompt # Every allowlisted action must be described, or Hermes can never request # it and the allowlist would claim a capability that cannot fire. assert '"id": "reclaim_workspace_storage"' in prompt assert '"id": "clear_stuck_agent_pods"' in prompt assert "lands on the same full volume" in prompt