from __future__ import annotations import time import pytest from ariadne.services import hermes_autotriage as module from tests.hermes_autotriage_harness import ( INCIDENT_ID, JOB, _build, _counter, _events, _gauge, _model_output, _prepare, _run, _seed_incident, _settings, _statuses, ) def test_disabled_tick(monkeypatch) -> None: env = _prepare(monkeypatch, cfg=_settings(hermes_autotriage_enabled=False)) assert module.run_hermes_autotriage(env.storage) == {"status": "disabled"} assert env.storage.events == [] assert env.calls["gets"] == [] def test_healthy_tick_without_incidents(monkeypatch) -> None: env = _prepare(monkeypatch, last_build=_build(13, "SUCCESS")) summary = module.run_hermes_autotriage(env.storage) assert summary["status"] == "ok" assert summary["jobs"][JOB] == {"status": "healthy", "resolved": []} assert env.storage.events == [] assert env.calls["triage"] == [] def test_success_resolves_only_older_awaiting_rebuild(monkeypatch) -> None: env = _prepare(monkeypatch, last_build=_build(13, "SUCCESS")) _seed_incident(env.storage, "awaiting_rebuild", build_number=12) _seed_incident(env.storage, "awaiting_rebuild", build_number=13) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["resolved"] == [INCIDENT_ID] resolved = _events(env.storage, module.INCIDENT_EVENT_TYPE)[-1] assert resolved["incident_id"] == INCIDENT_ID assert resolved["status"] == "resolved" assert resolved["phase"] == {"resolved_by_build": 13} assert _gauge("12", "resolved") == 1.0 assert _gauge("12", "human_required") == 0.0 assert module.HERMES_TRIAGE_LAST_SUCCESS_TS._value.get() > 0 def test_new_failure_full_happy_path(monkeypatch) -> None: success_before = _counter("repair_demo_fixture", "success") env = _prepare(monkeypatch) summary = module.run_hermes_autotriage(env.storage) job_summary = summary["jobs"][JOB] assert job_summary["status"] == "awaiting_rebuild" assert job_summary["repair"] == "configmap_patch" assert job_summary["target"] == "hermes-triage-demo/hermes-triage-demo-fixture" assert _statuses(env.storage) == ["detected", "diagnosed", "repairing", "awaiting_rebuild"] actions = _events(env.storage, module.ACTION_EVENT_TYPE) assert [action["result"] for action in actions] == ["requested", "accepted", "executed"] assert all(action["action"] == "repair_demo_fixture" for action in actions) diagnosis = _events(env.storage, module.DIAGNOSIS_EVENT_TYPE)[0] assert diagnosis["authorized"] is True assert diagnosis["authorize_reason"] == "authorized" assert diagnosis["run"] == { "status": "completed", "run_id": "run-1", "session_id": "sess-1", "error": None, "duration_seconds": 1.5, "denied_approvals": 0, } assert diagnosis["outcome"]["classification"] == "known_demo_fixture_failure" assert env.calls["repairs"] == [ ( { "namespace": "hermes-triage-demo", "fixture_configmap": "hermes-triage-demo-fixture", }, INCIDENT_ID, 12, ) ] assert actions[-1]["detail"] == { "repair": "configmap_patch", "target": "hermes-triage-demo/hermes-triage-demo-fixture", } awaiting = _events(env.storage, module.INCIDENT_EVENT_TYPE)[-1] assert awaiting["phase"] == { "action": "repair_demo_fixture", "repair": "configmap_patch", "target": "hermes-triage-demo/hermes-triage-demo-fixture", } assert env.calls["rebuilds"] == [JOB] assert _counter("repair_demo_fixture", "success") == success_before + 1.0 assert _gauge("12", "awaiting_rebuild") == 1.0 assert _gauge("12", "detected") == 0.0 for phase in ("evidence", "diagnosis", "repair", "total"): assert module.HERMES_TRIAGE_DURATION_SECONDS.labels(phase=phase)._value.get() >= 0.0 def test_prompt_is_frozen_shape(monkeypatch) -> None: env = _prepare(monkeypatch) module.run_hermes_autotriage(env.storage) config, prompt = env.calls["triage"][0] assert config == { "base_url": "http://hermes:8642", "api_key": "key", "total_timeout_seconds": 420.0, } assert prompt.startswith("Use $triage-titan-test-failures.\n") assert f"Analyze incident {INCIDENT_ID} for the Jenkins job" in prompt assert f'""' in prompt assert "You are diagnosing only; you do not execute anything." in prompt assert "for the Jenkins job hermes-triage-demo" in prompt assert "repair_demo_fixture" in prompt assert "Do not perform mutations." in prompt assert "\n\nBundle:\n" in prompt assert prompt.rstrip().endswith('"log_evidence":{"records":[]}}') def test_observe_mode_requires_human_without_actions(monkeypatch) -> None: rejected_before = _counter("repair_demo_fixture", "rejected") env = _prepare(monkeypatch, cfg=_settings(hermes_autoremediation_enabled=False)) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB] == { "status": "human_required", "incident_id": INCIDENT_ID, "reason": "autoremediation_disabled", } assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"] assert _events(env.storage, module.ACTION_EVENT_TYPE) == [] assert env.calls["repairs"] == [] assert env.calls["rebuilds"] == [] assert _counter("repair_demo_fixture", "rejected") == rejected_before + 1.0 def test_known_incident_is_deduped(monkeypatch) -> None: env = _prepare(monkeypatch) _seed_incident(env.storage, "human_required") summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB] == {"status": "deduped", "incident_id": INCIDENT_ID} assert len(env.storage.events) == 1 assert env.calls["triage"] == [] def test_incident_state_reads_json_string_detail(monkeypatch) -> None: env = _prepare(monkeypatch) _seed_incident(env.storage, "resolved", as_json=True) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["status"] == "deduped" def test_failed_rebuild_marks_both_incidents(monkeypatch) -> None: env = _prepare(monkeypatch, last_build=_build(13, "FAILURE")) _seed_incident(env.storage, "awaiting_rebuild", build_number=12) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB] == { "status": "rebuild_failed", "incident_id": f"{JOB}/13", "failed_incident": INCIDENT_ID, } details = _events(env.storage, module.INCIDENT_EVENT_TYPE)[1:] assert [(d["incident_id"], d["status"]) for d in details] == [ (INCIDENT_ID, "failed"), (f"{JOB}/13", "human_required"), ] assert details[1]["phase"] == {"reason": "repair rebuild failed"} assert env.calls["triage"] == [] assert env.calls["repairs"] == [] @pytest.mark.parametrize("status", ["timeout", "lost", "error", "failed", "cancelled"]) def test_unfinished_hermes_run_requires_human(monkeypatch, status) -> None: env = _prepare(monkeypatch, run=_run(status=status, error="boom")) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"] == f"hermes_run_{status}" assert _statuses(env.storage) == ["detected", "human_required"] diagnosis = _events(env.storage, module.DIAGNOSIS_EVENT_TYPE)[0] assert diagnosis["run"]["status"] == status assert diagnosis["outcome"] is None assert diagnosis["authorized"] is False assert env.calls["repairs"] == [] def test_invalid_response_requires_human(monkeypatch) -> None: rejected_before = _counter("unknown", "rejected") env = _prepare(monkeypatch, run=_run(output="no json here")) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"].startswith("response_invalid") assert _statuses(env.storage) == ["detected", "human_required"] assert _counter("unknown", "rejected") == rejected_before + 1.0 def test_model_human_required_is_rejected(monkeypatch) -> None: env = _prepare(monkeypatch, run=_run(output=_model_output(human_required=True))) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"] == "human_required" assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"] assert env.calls["repairs"] == [] def test_missing_signature_is_rejected(monkeypatch) -> None: env = _prepare(monkeypatch, signature=False) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"] == "evidence_signature_missing" assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"] assert env.calls["repairs"] == [] def test_non_allowlisted_action_is_rejected(monkeypatch) -> None: rejected_before = _counter("unknown", "rejected") output = _model_output(requested_action={"type": "run_ariadne_job", "id": "other_action"}) env = _prepare(monkeypatch, run=_run(output=output)) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"].startswith("action_not_allowlisted") assert _counter("unknown", "rejected") == rejected_before + 1.0 def test_prior_action_blocks_second_action(monkeypatch) -> None: env = _prepare(monkeypatch) _seed_incident(env.storage, "detected") env.storage.record_event( module.ACTION_EVENT_TYPE, {"incident_id": INCIDENT_ID, "action": "repair_demo_fixture", "result": "requested"}, ) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["reason"] == "max_actions_reached" assert env.calls["repairs"] == [] def test_repair_failure_marks_failed_and_human(monkeypatch) -> None: failed_before = _counter("repair_demo_fixture", "failed") env = _prepare( monkeypatch, repair={ "action": "configmap_patch", "target": "hermes-triage-demo/hermes-triage-demo-fixture", "succeeded": False, "error": "configmap patch http 403", }, ) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB] == { "status": "failed", "incident_id": INCIDENT_ID, "reason": "configmap patch http 403", } assert _statuses(env.storage) == ["detected", "diagnosed", "repairing", "failed"] actions = _events(env.storage, module.ACTION_EVENT_TYPE) assert [action["result"] for action in actions] == ["requested", "accepted", "failed"] assert env.calls["rebuilds"] == [] assert _counter("repair_demo_fixture", "failed") == failed_before + 1.0 assert _gauge("12", "failed") == 1.0 assert _gauge("12", "human_required") == 1.0 def test_rebuild_trigger_failure_marks_failed(monkeypatch) -> None: env = _prepare(monkeypatch, rebuild={"requested": False, "error": "rebuild http 500"}) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["status"] == "failed" assert summary["jobs"][JOB]["reason"] == "rebuild http 500" actions = _events(env.storage, module.ACTION_EVENT_TYPE) assert [action["result"] for action in actions] == ["requested", "accepted", "failed"] def test_jenkins_fetch_failure_skips_job(monkeypatch) -> None: env = _prepare(monkeypatch, jenkins_exc=RuntimeError("boom")) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB] == {"status": "skipped"} assert env.storage.events == [] def test_building_build_within_the_cap_is_skipped(monkeypatch) -> None: """A build that is merely still running is not triage's business.""" fresh = int(time.time() * 1000) env = _prepare(monkeypatch, last_build=_build(12, None, building=True, timestamp=fresh)) assert module.run_hermes_autotriage(env.storage)["jobs"][JOB] == {"status": "skipped"} assert env.storage.events == [] def test_building_build_past_the_cap_escalates(monkeypatch) -> None: """A build that never terminates must not stay invisible. It files no incident, raises no alert, and holds a Jenkins agent slot the whole time, so the only signal is that it has run too long. """ env = _prepare(monkeypatch, last_build=_build(12, None, building=True)) summary = module.run_hermes_autotriage(env.storage)["jobs"][JOB] assert summary == { "status": "human_required", "incident_id": INCIDENT_ID, "reason": module.hermes_hung_builds.HUNG_REASON, } assert env.storage.events != [] def test_hung_build_is_not_escalated_twice(monkeypatch) -> None: """Every tick sees the same running build; only the first may escalate.""" env = _prepare(monkeypatch, last_build=_build(12, None, building=True)) module.run_hermes_autotriage(env.storage) before = len(env.storage.events) second = module.run_hermes_autotriage(env.storage)["jobs"][JOB] assert second == {"status": "deduped", "incident_id": INCIDENT_ID} assert len(env.storage.events) == before def test_empty_jenkins_base_url_skips(monkeypatch) -> None: env = _prepare(monkeypatch, cfg=_settings(jenkins_base_url="")) assert module.run_hermes_autotriage(env.storage)["jobs"][JOB] == {"status": "skipped"} assert env.calls["gets"] == [] def test_non_terminal_result_is_ignored(monkeypatch) -> None: env = _prepare(monkeypatch, last_build=_build(12, "ABORTED")) assert module.run_hermes_autotriage(env.storage)["jobs"][JOB] == { "status": "ignored", "result": "ABORTED", } assert env.storage.events == [] def test_event_detail_tolerates_bad_payloads() -> None: assert module.hermes_events.event_detail({"detail": "not-json"}) is None assert module.hermes_events.event_detail({"detail": "[1,2]"}) is None assert module.hermes_events.event_detail({"detail": 5}) is None assert module.hermes_events.event_detail("not-a-row") is None assert module._int_value("not-a-number") == 0 def test_jenkins_request_uses_basic_auth_and_tree(monkeypatch) -> None: env = _prepare(monkeypatch, last_build=_build(13, "SUCCESS")) module.run_hermes_autotriage(env.storage) assert env.calls["client_kwargs"]["auth"] == ("user", "token") url, params = env.calls["gets"][0] assert url == f"https://ci.example/job/{JOB}/api/json" tree = params["tree"] assert "lastBuild[number,result,building,timestamp,duration,url]" in tree # jobs[name] rides along on the same request so a multibranch folder, which # carries no lastBuild, is recognisable without a second call per tick. assert "jobs[name]" in tree def _issue_settings(**overrides): # type: ignore[no-untyped-def] values = { "hermes_issues_enabled": True, "hermes_issue_repos": {JOB: ("bstein", JOB)}, } values.update(overrides) return _settings(**values) def _install_issue_tracker(monkeypatch, calls) -> None: # type: ignore[no-untyped-def] filed: dict = {} def fake_find(cfg, job, classification, incident_id): # type: ignore[no-untyped-def] calls["lookups"].append((cfg["owner"], cfg["repo"], job, classification, incident_id)) existing = filed.get((job, classification)) if existing is None: return {"found": False, "issue_number": None, "url": None, "error": None} return {"found": True, "issue_number": existing, "url": f"https://scm/issues/{existing}", "error": None} def fake_create(cfg, context): # type: ignore[no-untyped-def] calls["creates"].append(context) number = 40 + len(calls["creates"]) filed[(context["job"], context["classification"])] = number return {"issue_number": number, "url": f"https://scm/issues/{number}", "error": None} monkeypatch.setattr(module.hermes_incident_issue, "find_open_incident_issue", fake_find) monkeypatch.setattr(module.hermes_incident_issue, "create_incident_issue", fake_create) def _issue_env(monkeypatch, cfg=None, **kwargs): # type: ignore[no-untyped-def] calls: dict = {"lookups": [], "creates": []} env = _prepare(monkeypatch, cfg=cfg if cfg is not None else _issue_settings(), **kwargs) _install_issue_tracker(monkeypatch, calls) env.calls.update(calls) return env def _human_required_output(build_number: int = 12) -> str: return _model_output( incident_id=f"{JOB}/{build_number}", classification="unknown_build_failure", requested_action=None, human_required=True, reason="the failure matches no known signature", ) def _issue_events(storage): # type: ignore[no-untyped-def] return _events(storage, module.hermes_incident_issue.ISSUE_EVENT_TYPE) def test_human_required_incident_files_one_issue(monkeypatch) -> None: env = _issue_env(monkeypatch, run=_run(output=_human_required_output())) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["status"] == "human_required" assert env.calls["lookups"] == [("bstein", JOB, JOB, "unknown_build_failure", INCIDENT_ID)] context = env.calls["creates"][0] assert context["incident_id"] == INCIDENT_ID assert context["classification"] == "unknown_build_failure" assert context["reason"] == "the failure matches no known signature" assert context["run_id"] == "run-1" assert _issue_events(env.storage) == [ { "incident_id": INCIDENT_ID, "job": JOB, "build_number": 12, "classification": "unknown_build_failure", "issue_number": 41, "url": "https://scm/issues/41", "skipped": False, "error": None, } ] assert _statuses(env.storage)[-1] == "human_required" def test_second_incident_with_the_same_classification_skips(monkeypatch) -> None: env = _issue_env(monkeypatch, run=_run(output=_human_required_output())) module.run_hermes_autotriage(env.storage) _prepare( monkeypatch, cfg=_issue_settings(), last_build=_build(13, "FAILURE"), run=_run(output=_human_required_output(13)), storage=env.storage, ) module.run_hermes_autotriage(env.storage) assert len(env.calls["creates"]) == 1 events = _issue_events(env.storage) assert [event["skipped"] for event in events] == [False, True] assert events[1]["incident_id"] == f"{JOB}/13" assert events[1]["issue_number"] == 41 def test_unmapped_job_and_disabled_switch_file_nothing(monkeypatch) -> None: for cfg in (_issue_settings(hermes_issue_repos={}), _issue_settings(hermes_issues_enabled=False)): env = _issue_env(monkeypatch, cfg=cfg, run=_run(output=_human_required_output())) module.run_hermes_autotriage(env.storage) assert env.calls["lookups"] == [] assert env.calls["creates"] == [] assert _issue_events(env.storage) == [] def test_remediated_incident_files_no_issue(monkeypatch) -> None: env = _issue_env(monkeypatch) summary = module.run_hermes_autotriage(env.storage) assert summary["jobs"][JOB]["status"] == "awaiting_rebuild" assert env.calls["creates"] == [] assert _issue_events(env.storage) == [] def test_issue_filing_failure_never_breaks_the_tick(monkeypatch) -> None: env = _issue_env(monkeypatch, run=_run(output=_human_required_output())) def explode(*args, **kwargs): # type: ignore[no-untyped-def] raise RuntimeError("gitea is unreachable") monkeypatch.setattr(module.hermes_incident_issue, "maybe_file_issue", explode) summary = module.run_hermes_autotriage(env.storage) assert summary["status"] == "ok" assert summary["jobs"][JOB]["status"] == "human_required" assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"]