from __future__ import annotations import pytest from ariadne.services import hermes_autotriage as module from tests.hermes_autotriage_harness import ( INCIDENT_ID, JOB, _build, _counter, _events, _gauge, _model_output, _prepare, _run, _seed_incident, _settings, _statuses, ) def test_disabled_tick(monkeypatch) -> None: env = _prepare(monkeypatch, cfg=_settings(hermes_autotriage_enabled=False)) assert module.run_hermes_autotriage(env.storage) == {"status": "disabled"} assert env.storage.events == [] assert env.calls["gets"] == [] def test_healthy_tick_without_incidents(monkeypatch) -> None: env = _prepare(monkeypatch, last_build=_build(13, "SUCCESS")) summary = module.run_hermes_autotriage(env.storage) assert summary["status"] == "ok" assert summary["jobs"][JOB] == {"status": "healthy", "resolved": []} assert env.storage.events == [] assert env.calls["triage"] == [] def test_success_resolves_only_older_awaiting_rebuild(monkeypatch) -> None: env = _prepare(monkeypatch, last_build=_build(13, "SUCCESS")) _seed_incident(env.storage, "awaiting_rebuild", build_number=12) _seed_incident(env.storage, "awaiting_rebuild", build_number=13) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["resolved"] == [INCIDENT_ID] resolved = _events(env.storage, module.INCIDENT_EVENT_TYPE)[-1] assert resolved["incident_id"] == INCIDENT_ID assert resolved["status"] == "resolved" assert resolved["phase"] == {"resolved_by_build": 13} assert _gauge("12", "resolved") == 1.0 assert _gauge("12", "human_required") == 0.0 assert module.HERMES_TRIAGE_LAST_SUCCESS_TS._value.get() > 0 def test_new_failure_full_happy_path(monkeypatch) -> None: success_before = _counter("repair_demo_fixture", "success") env = _prepare(monkeypatch) summary = module.run_hermes_autotriage(env.storage) job_summary = summary["jobs"][JOB] assert job_summary["status"] == "awaiting_rebuild" assert job_summary["repair_job"] == "hermes-demo-repair-12" assert _statuses(env.storage) == ["detected", "diagnosed", "repairing", "awaiting_rebuild"] actions = _events(env.storage, module.ACTION_EVENT_TYPE) assert [action["result"] for action in actions] == ["requested", "accepted", "executed"] assert all(action["action"] == "repair_demo_fixture" for action in actions) diagnosis = _events(env.storage, module.DIAGNOSIS_EVENT_TYPE)[0] assert diagnosis["authorized"] is True assert diagnosis["authorize_reason"] == "authorized" assert diagnosis["run"] == { "status": "completed", "run_id": "run-1", "session_id": "sess-1", "error": None, "duration_seconds": 1.5, "denied_approvals": 0, } assert diagnosis["outcome"]["classification"] == "known_demo_fixture_failure" assert env.calls["repairs"] == [ ( { "namespace": "hermes-triage-demo", "fixture_configmap": "hermes-triage-demo-fixture", "image": "busybox:1.37", }, INCIDENT_ID, 12, ) ] assert env.calls["rebuilds"] == [JOB] assert _counter("repair_demo_fixture", "success") == success_before + 1.0 assert _gauge("12", "awaiting_rebuild") == 1.0 assert _gauge("12", "detected") == 0.0 for phase in ("evidence", "diagnosis", "repair", "total"): assert module.HERMES_TRIAGE_DURATION_SECONDS.labels(phase=phase)._value.get() >= 0.0 def test_prompt_is_frozen_shape(monkeypatch) -> None: env = _prepare(monkeypatch) module.run_hermes_autotriage(env.storage) config, prompt = env.calls["triage"][0] assert config == { "base_url": "http://hermes:8642", "api_key": "key", "total_timeout_seconds": 420.0, } assert prompt.startswith("Use $triage-titan-test-failures.\n") assert f"Analyze incident {INCIDENT_ID}." in prompt assert f'""' in prompt assert "You are diagnosing only; you do not execute anything." in prompt assert "Set human_required to false when the evidence matches" in prompt assert "Do not perform mutations.\n\nBundle:\n" in prompt assert prompt.rstrip().endswith('"log_evidence":{"records":[]}}') def test_observe_mode_requires_human_without_actions(monkeypatch) -> None: rejected_before = _counter("repair_demo_fixture", "rejected") env = _prepare(monkeypatch, cfg=_settings(hermes_autoremediation_enabled=False)) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB] == { "status": "human_required", "incident_id": INCIDENT_ID, "reason": "autoremediation_disabled", } assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"] assert _events(env.storage, module.ACTION_EVENT_TYPE) == [] assert env.calls["repairs"] == [] assert env.calls["rebuilds"] == [] assert _counter("repair_demo_fixture", "rejected") == rejected_before + 1.0 def test_known_incident_is_deduped(monkeypatch) -> None: env = _prepare(monkeypatch) _seed_incident(env.storage, "human_required") summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB] == {"status": "deduped", "incident_id": INCIDENT_ID} assert len(env.storage.events) == 1 assert env.calls["triage"] == [] def test_incident_state_reads_json_string_detail(monkeypatch) -> None: env = _prepare(monkeypatch) _seed_incident(env.storage, "resolved", as_json=True) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["status"] == "deduped" def test_failed_rebuild_marks_both_incidents(monkeypatch) -> None: env = _prepare(monkeypatch, last_build=_build(13, "FAILURE")) _seed_incident(env.storage, "awaiting_rebuild", build_number=12) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB] == { "status": "rebuild_failed", "incident_id": f"{JOB}/13", "failed_incident": INCIDENT_ID, } details = _events(env.storage, module.INCIDENT_EVENT_TYPE)[1:] assert [(d["incident_id"], d["status"]) for d in details] == [ (INCIDENT_ID, "failed"), (f"{JOB}/13", "human_required"), ] assert details[1]["phase"] == {"reason": "repair rebuild failed"} assert env.calls["triage"] == [] assert env.calls["repairs"] == [] @pytest.mark.parametrize("status", ["timeout", "lost", "error", "failed", "cancelled"]) def test_unfinished_hermes_run_requires_human(monkeypatch, status) -> None: env = _prepare(monkeypatch, run=_run(status=status, error="boom")) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"] == f"hermes_run_{status}" assert _statuses(env.storage) == ["detected", "human_required"] diagnosis = _events(env.storage, module.DIAGNOSIS_EVENT_TYPE)[0] assert diagnosis["run"]["status"] == status assert diagnosis["outcome"] is None assert diagnosis["authorized"] is False assert env.calls["repairs"] == [] def test_invalid_response_requires_human(monkeypatch) -> None: rejected_before = _counter("unknown", "rejected") env = _prepare(monkeypatch, run=_run(output="no json here")) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"].startswith("response_invalid") assert _statuses(env.storage) == ["detected", "human_required"] assert _counter("unknown", "rejected") == rejected_before + 1.0 def test_model_human_required_is_rejected(monkeypatch) -> None: env = _prepare(monkeypatch, run=_run(output=_model_output(human_required=True))) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"] == "human_required" assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"] assert env.calls["repairs"] == [] def test_missing_signature_is_rejected(monkeypatch) -> None: env = _prepare(monkeypatch, signature=False) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"] == "evidence_signature_missing" assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"] assert env.calls["repairs"] == [] def test_non_allowlisted_action_is_rejected(monkeypatch) -> None: rejected_before = _counter("unknown", "rejected") output = _model_output(requested_action={"type": "run_ariadne_job", "id": "other_action"}) env = _prepare(monkeypatch, run=_run(output=output)) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"].startswith("action_not_allowlisted") assert _counter("unknown", "rejected") == rejected_before + 1.0 def test_prior_action_blocks_second_action(monkeypatch) -> None: env = _prepare(monkeypatch) _seed_incident(env.storage, "detected") env.storage.record_event( module.ACTION_EVENT_TYPE, {"incident_id": INCIDENT_ID, "action": "repair_demo_fixture", "result": "requested"}, ) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"] == "max_actions_reached" assert env.calls["repairs"] == [] def test_repair_failure_marks_failed_and_human(monkeypatch) -> None: failed_before = _counter("repair_demo_fixture", "failed") env = _prepare( monkeypatch, repair={"job_name": "hermes-demo-repair-12", "succeeded": False, "error": "repair job failed"}, ) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB] == { "status": "failed", "incident_id": INCIDENT_ID, "reason": "repair job failed", } assert _statuses(env.storage) == ["detected", "diagnosed", "repairing", "failed"] actions = _events(env.storage, module.ACTION_EVENT_TYPE) assert [action["result"] for action in actions] == ["requested", "accepted", "failed"] assert env.calls["rebuilds"] == [] assert _counter("repair_demo_fixture", "failed") == failed_before + 1.0 assert _gauge("12", "failed") == 1.0 assert _gauge("12", "human_required") == 1.0 def test_rebuild_trigger_failure_marks_failed(monkeypatch) -> None: env = _prepare(monkeypatch, rebuild={"requested": False, "error": "rebuild http 500"}) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["status"] == "failed" assert summary["jobs"][JOB]["reason"] == "rebuild http 500" actions = _events(env.storage, module.ACTION_EVENT_TYPE) assert [action["result"] for action in actions] == ["requested", "accepted", "failed"] def test_jenkins_fetch_failure_skips_job(monkeypatch) -> None: env = _prepare(monkeypatch, jenkins_exc=RuntimeError("boom")) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB] == {"status": "skipped"} assert env.storage.events == [] def test_building_build_is_skipped(monkeypatch) -> None: env = _prepare(monkeypatch, last_build=_build(12, None, building=True)) assert module.run_hermes_autotriage(env.storage)["jobs"][JOB] == {"status": "skipped"} assert env.storage.events == [] def test_empty_jenkins_base_url_skips(monkeypatch) -> None: env = _prepare(monkeypatch, cfg=_settings(jenkins_base_url="")) assert module.run_hermes_autotriage(env.storage)["jobs"][JOB] == {"status": "skipped"} assert env.calls["gets"] == [] def test_non_terminal_result_is_ignored(monkeypatch) -> None: env = _prepare(monkeypatch, last_build=_build(12, "ABORTED")) assert module.run_hermes_autotriage(env.storage)["jobs"][JOB] == { "status": "ignored", "result": "ABORTED", } assert env.storage.events == [] def test_event_detail_tolerates_bad_payloads() -> None: assert module.hermes_events.event_detail({"detail": "not-json"}) is None assert module.hermes_events.event_detail({"detail": "[1,2]"}) is None assert module.hermes_events.event_detail({"detail": 5}) is None assert module.hermes_events.event_detail("not-a-row") is None assert module._int_value("not-a-number") == 0 def test_jenkins_request_uses_basic_auth_and_tree(monkeypatch) -> None: env = _prepare(monkeypatch, last_build=_build(13, "SUCCESS")) module.run_hermes_autotriage(env.storage) assert env.calls["client_kwargs"]["auth"] == ("user", "token") url, params = env.calls["gets"][0] assert url == f"https://ci.example/job/{JOB}/api/json" assert params == {"tree": "lastBuild[number,result,building,timestamp,duration,url]"}