c8d9ab2015
Adds the automated failure-to-repair loop for the hermes-triage-demo Jenkins job: - hermes_autotriage_logs: bounded kube-* OpenSearch evidence (fixed query, incident-ID correlation with window fallback, sanitization, byte caps) - hermes_agent_client: async /v1/runs client (bearer auth, poll, auto-deny approvals, lost-run and timeout handling) - hermes_autotriage_decision: frozen response schema parser + nine-gate action authorization (allowlist, confidence, idempotency, kill switch) - hermes_autotriage_evidence: Jenkins wfapi/testReport/console bundle assembly + failure-signature detection - hermes_autotriage_repair: hardcoded repair Job executor + one rebuild with SEED_FAILURE=false - hermes_autotriage: orchestrator state machine (detected -> diagnosed -> repairing -> awaiting_rebuild -> resolved | human_required | failed), bounded-label triage metrics, incident dedupe via storage events - settings/app: ARIADNE_HERMES_* config (default off) + 1m schedule task 126 new tests; all quality gates pass locally. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
480 lines
19 KiB
Python
480 lines
19 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from ariadne.services import hermes_autotriage as module
|
|
from ariadne.services.hermes_agent_client import HermesRunResult
|
|
|
|
|
|
JOB = "hermes-triage-demo"
|
|
INCIDENT_ID = f"{JOB}/12"
|
|
|
|
|
|
class FakeStorage:
|
|
def __init__(self) -> None:
|
|
self.events: list[dict] = []
|
|
|
|
def record_event(self, event_type, detail) -> None: # type: ignore[no-untyped-def]
|
|
self.events.append({"event_type": event_type, "detail": detail})
|
|
|
|
def list_events(self, limit=200, event_type=None): # type: ignore[no-untyped-def]
|
|
rows = [
|
|
dict(row)
|
|
for row in reversed(self.events)
|
|
if event_type is None or row["event_type"] == event_type
|
|
]
|
|
return rows[:limit]
|
|
|
|
|
|
class FakeResponse:
|
|
def __init__(self, payload) -> None: # type: ignore[no-untyped-def]
|
|
self.payload = payload
|
|
|
|
def raise_for_status(self) -> None:
|
|
return None
|
|
|
|
def json(self): # type: ignore[no-untyped-def]
|
|
return self.payload
|
|
|
|
|
|
def _settings(**overrides) -> SimpleNamespace: # type: ignore[no-untyped-def]
|
|
values = {
|
|
"hermes_autotriage_enabled": True,
|
|
"hermes_autotriage_job_allowlist": [JOB],
|
|
"hermes_autoremediation_enabled": True,
|
|
"hermes_allowed_actions": ["repair_demo_fixture"],
|
|
"hermes_min_confidence": 0.85,
|
|
"hermes_max_actions_per_incident": 1,
|
|
"hermes_api_url": "http://hermes:8642",
|
|
"hermes_api_key": "key",
|
|
"hermes_run_timeout_seconds": 420.0,
|
|
"hermes_demo_namespace": "hermes-triage-demo",
|
|
"hermes_demo_fixture_pvc": "hermes-triage-demo-fixture",
|
|
"hermes_repair_image": "busybox:1.37",
|
|
"jenkins_base_url": "https://ci.example",
|
|
"jenkins_api_user": "user",
|
|
"jenkins_api_token": "token",
|
|
"jenkins_api_timeout_sec": 5.0,
|
|
}
|
|
values.update(overrides)
|
|
return SimpleNamespace(**values)
|
|
|
|
|
|
def _build(number: int, result, **overrides): # type: ignore[no-untyped-def]
|
|
payload = {
|
|
"number": number,
|
|
"result": result,
|
|
"building": False,
|
|
"timestamp": 1720000000000,
|
|
"duration": 60000,
|
|
"url": f"https://ci.example/job/{JOB}/{number}/",
|
|
}
|
|
payload.update(overrides)
|
|
return payload
|
|
|
|
|
|
def _model_output(**overrides) -> str: # type: ignore[no-untyped-def]
|
|
payload = {
|
|
"incident_id": INCIDENT_ID,
|
|
"classification": "known_demo_fixture_failure",
|
|
"confidence": 0.95,
|
|
"facts": [
|
|
{"statement": "fixture-state-check failed", "source": "jenkins", "reference": "console"}
|
|
],
|
|
"inferences": [],
|
|
"first_failed_gate": "fixture-state-check",
|
|
"requested_action": {"type": "run_ariadne_job", "id": "repair_demo_fixture"},
|
|
"human_required": False,
|
|
"reason": "known fixture failure",
|
|
}
|
|
payload.update(overrides)
|
|
return json.dumps(payload)
|
|
|
|
|
|
def _run(status: str = "completed", output=None, error=None) -> HermesRunResult: # type: ignore[no-untyped-def]
|
|
return HermesRunResult(
|
|
status=status,
|
|
output=output,
|
|
run_id="run-1",
|
|
session_id="sess-1",
|
|
error=error,
|
|
duration_seconds=1.5,
|
|
denied_approvals=0,
|
|
)
|
|
|
|
|
|
def _install_jenkins(monkeypatch, calls, last_build, exc) -> None: # type: ignore[no-untyped-def]
|
|
class FakeClient:
|
|
def __init__(self, **kwargs) -> None: # type: ignore[no-untyped-def]
|
|
calls["client_kwargs"] = kwargs
|
|
|
|
def __enter__(self): # type: ignore[no-untyped-def]
|
|
return self
|
|
|
|
def __exit__(self, *args) -> None: # type: ignore[no-untyped-def]
|
|
return None
|
|
|
|
def get(self, url, params=None): # type: ignore[no-untyped-def]
|
|
calls["gets"].append((url, params))
|
|
if exc is not None:
|
|
raise exc
|
|
return FakeResponse({"lastBuild": last_build})
|
|
|
|
monkeypatch.setattr(module.httpx, "Client", FakeClient)
|
|
|
|
|
|
def _prepare( # type: ignore[no-untyped-def]
|
|
monkeypatch,
|
|
*,
|
|
cfg=None,
|
|
last_build=None,
|
|
jenkins_exc=None,
|
|
run=None,
|
|
repair=None,
|
|
rebuild=None,
|
|
signature=True,
|
|
):
|
|
storage = FakeStorage()
|
|
calls: dict = {"gets": [], "triage": [], "repairs": [], "rebuilds": []}
|
|
monkeypatch.setattr(module, "settings", cfg if cfg is not None else _settings())
|
|
_install_jenkins(
|
|
monkeypatch, calls, last_build if last_build is not None else _build(12, "FAILURE"), jenkins_exc
|
|
)
|
|
monkeypatch.setattr(
|
|
module.hermes_evidence,
|
|
"collect_evidence",
|
|
lambda incident_id, job, build: {
|
|
"incident_id": incident_id,
|
|
"jenkins": {"job": job},
|
|
"log_evidence": {"records": []},
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
module.hermes_evidence, "evidence_has_signature", lambda bundle, incident_id: signature
|
|
)
|
|
|
|
def fake_run_triage(config, prompt): # type: ignore[no-untyped-def]
|
|
calls["triage"].append((config, prompt))
|
|
return run if run is not None else _run(output=_model_output())
|
|
|
|
def fake_execute_repair(repair_cfg, incident_id, build_number): # type: ignore[no-untyped-def]
|
|
calls["repairs"].append((repair_cfg, incident_id, build_number))
|
|
if repair is not None:
|
|
return repair
|
|
return {"job_name": "hermes-demo-repair-12", "succeeded": True, "error": None}
|
|
|
|
def fake_trigger_rebuild(rebuild_cfg, job): # type: ignore[no-untyped-def]
|
|
calls["rebuilds"].append(job)
|
|
return rebuild if rebuild is not None else {"requested": True, "error": None}
|
|
|
|
monkeypatch.setattr(module.hermes_agent_client, "run_triage", fake_run_triage)
|
|
monkeypatch.setattr(module.hermes_autotriage_repair, "execute_repair", fake_execute_repair)
|
|
monkeypatch.setattr(module.hermes_autotriage_repair, "trigger_rebuild", fake_trigger_rebuild)
|
|
return SimpleNamespace(storage=storage, calls=calls)
|
|
|
|
|
|
def _events(storage: FakeStorage, event_type: str) -> list[dict]:
|
|
return [row["detail"] for row in storage.events if row["event_type"] == event_type]
|
|
|
|
|
|
def _statuses(storage: FakeStorage) -> list[str]:
|
|
return [detail["status"] for detail in _events(storage, module.INCIDENT_EVENT_TYPE)]
|
|
|
|
|
|
def _seed_incident(storage: FakeStorage, status: str, build_number: int = 12, as_json: bool = False) -> None:
|
|
detail = {
|
|
"incident_id": f"{JOB}/{build_number}",
|
|
"job": JOB,
|
|
"build_number": build_number,
|
|
"status": status,
|
|
"phase": {},
|
|
}
|
|
storage.record_event(module.INCIDENT_EVENT_TYPE, json.dumps(detail) if as_json else detail)
|
|
|
|
|
|
def _gauge(build: str, status: str) -> float:
|
|
return module.HERMES_TRIAGE_INCIDENT.labels(jenkins_job=JOB, build=build, status=status)._value.get()
|
|
|
|
|
|
def _counter(action: str, result: str) -> float:
|
|
return module.HERMES_TRIAGE_ACTION_TOTAL.labels(action=action, result=result)._value.get()
|
|
|
|
|
|
def test_disabled_tick(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch, cfg=_settings(hermes_autotriage_enabled=False))
|
|
assert module.run_hermes_autotriage(env.storage) == {"status": "disabled"}
|
|
assert env.storage.events == []
|
|
assert env.calls["gets"] == []
|
|
|
|
|
|
def test_healthy_tick_without_incidents(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch, last_build=_build(13, "SUCCESS"))
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
assert summary["status"] == "ok"
|
|
assert summary["jobs"][JOB] == {"status": "healthy", "resolved": []}
|
|
assert env.storage.events == []
|
|
assert env.calls["triage"] == []
|
|
|
|
|
|
def test_success_resolves_only_older_awaiting_rebuild(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch, last_build=_build(13, "SUCCESS"))
|
|
_seed_incident(env.storage, "awaiting_rebuild", build_number=12)
|
|
_seed_incident(env.storage, "awaiting_rebuild", build_number=13)
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
assert summary["jobs"][JOB]["resolved"] == [INCIDENT_ID]
|
|
resolved = _events(env.storage, module.INCIDENT_EVENT_TYPE)[-1]
|
|
assert resolved["incident_id"] == INCIDENT_ID
|
|
assert resolved["status"] == "resolved"
|
|
assert resolved["phase"] == {"resolved_by_build": 13}
|
|
assert _gauge("12", "resolved") == 1.0
|
|
assert _gauge("12", "human_required") == 0.0
|
|
assert module.HERMES_TRIAGE_LAST_SUCCESS_TS._value.get() > 0
|
|
|
|
|
|
def test_new_failure_full_happy_path(monkeypatch) -> None:
|
|
success_before = _counter("repair_demo_fixture", "success")
|
|
env = _prepare(monkeypatch)
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
job_summary = summary["jobs"][JOB]
|
|
assert job_summary["status"] == "awaiting_rebuild"
|
|
assert job_summary["repair_job"] == "hermes-demo-repair-12"
|
|
assert _statuses(env.storage) == ["detected", "diagnosed", "repairing", "awaiting_rebuild"]
|
|
actions = _events(env.storage, module.ACTION_EVENT_TYPE)
|
|
assert [action["result"] for action in actions] == ["requested", "accepted", "executed"]
|
|
assert all(action["action"] == "repair_demo_fixture" for action in actions)
|
|
diagnosis = _events(env.storage, module.DIAGNOSIS_EVENT_TYPE)[0]
|
|
assert diagnosis["authorized"] is True
|
|
assert diagnosis["authorize_reason"] == "authorized"
|
|
assert diagnosis["run"] == {
|
|
"status": "completed",
|
|
"run_id": "run-1",
|
|
"session_id": "sess-1",
|
|
"error": None,
|
|
"duration_seconds": 1.5,
|
|
"denied_approvals": 0,
|
|
}
|
|
assert diagnosis["outcome"]["classification"] == "known_demo_fixture_failure"
|
|
assert env.calls["repairs"] == [
|
|
(
|
|
{
|
|
"namespace": "hermes-triage-demo",
|
|
"fixture_pvc": "hermes-triage-demo-fixture",
|
|
"image": "busybox:1.37",
|
|
},
|
|
INCIDENT_ID,
|
|
12,
|
|
)
|
|
]
|
|
assert env.calls["rebuilds"] == [JOB]
|
|
assert _counter("repair_demo_fixture", "success") == success_before + 1.0
|
|
assert _gauge("12", "awaiting_rebuild") == 1.0
|
|
assert _gauge("12", "detected") == 0.0
|
|
for phase in ("evidence", "diagnosis", "repair", "total"):
|
|
assert module.HERMES_TRIAGE_DURATION_SECONDS.labels(phase=phase)._value.get() >= 0.0
|
|
|
|
|
|
def test_prompt_is_frozen_shape(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch)
|
|
module.run_hermes_autotriage(env.storage)
|
|
config, prompt = env.calls["triage"][0]
|
|
assert config == {
|
|
"base_url": "http://hermes:8642",
|
|
"api_key": "key",
|
|
"total_timeout_seconds": 420.0,
|
|
}
|
|
assert prompt.startswith("Use $triage-titan-test-failures.\n")
|
|
assert f"Analyze incident {INCIDENT_ID}." in prompt
|
|
assert f'"<must equal {INCIDENT_ID}>"' in prompt
|
|
assert "You are diagnosing only; you do not execute anything." in prompt
|
|
assert "Set human_required to false when the evidence matches" in prompt
|
|
assert "Do not perform mutations.\n\nBundle:\n" in prompt
|
|
assert prompt.rstrip().endswith('"log_evidence":{"records":[]}}')
|
|
|
|
|
|
def test_observe_mode_requires_human_without_actions(monkeypatch) -> None:
|
|
rejected_before = _counter("repair_demo_fixture", "rejected")
|
|
env = _prepare(monkeypatch, cfg=_settings(hermes_autoremediation_enabled=False))
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
assert summary["jobs"][JOB] == {
|
|
"status": "human_required",
|
|
"incident_id": INCIDENT_ID,
|
|
"reason": "autoremediation_disabled",
|
|
}
|
|
assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"]
|
|
assert _events(env.storage, module.ACTION_EVENT_TYPE) == []
|
|
assert env.calls["repairs"] == []
|
|
assert env.calls["rebuilds"] == []
|
|
assert _counter("repair_demo_fixture", "rejected") == rejected_before + 1.0
|
|
|
|
|
|
def test_known_incident_is_deduped(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch)
|
|
_seed_incident(env.storage, "human_required")
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
assert summary["jobs"][JOB] == {"status": "deduped", "incident_id": INCIDENT_ID}
|
|
assert len(env.storage.events) == 1
|
|
assert env.calls["triage"] == []
|
|
|
|
|
|
def test_incident_state_reads_json_string_detail(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch)
|
|
_seed_incident(env.storage, "resolved", as_json=True)
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
assert summary["jobs"][JOB]["status"] == "deduped"
|
|
|
|
|
|
def test_failed_rebuild_marks_both_incidents(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch, last_build=_build(13, "FAILURE"))
|
|
_seed_incident(env.storage, "awaiting_rebuild", build_number=12)
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
assert summary["jobs"][JOB] == {
|
|
"status": "rebuild_failed",
|
|
"incident_id": f"{JOB}/13",
|
|
"failed_incident": INCIDENT_ID,
|
|
}
|
|
details = _events(env.storage, module.INCIDENT_EVENT_TYPE)[1:]
|
|
assert [(d["incident_id"], d["status"]) for d in details] == [
|
|
(INCIDENT_ID, "failed"),
|
|
(f"{JOB}/13", "human_required"),
|
|
]
|
|
assert details[1]["phase"] == {"reason": "repair rebuild failed"}
|
|
assert env.calls["triage"] == []
|
|
assert env.calls["repairs"] == []
|
|
|
|
|
|
@pytest.mark.parametrize("status", ["timeout", "lost", "error", "failed", "cancelled"])
|
|
def test_unfinished_hermes_run_requires_human(monkeypatch, status) -> None:
|
|
env = _prepare(monkeypatch, run=_run(status=status, error="boom"))
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
assert summary["jobs"][JOB]["reason"] == f"hermes_run_{status}"
|
|
assert _statuses(env.storage) == ["detected", "human_required"]
|
|
diagnosis = _events(env.storage, module.DIAGNOSIS_EVENT_TYPE)[0]
|
|
assert diagnosis["run"]["status"] == status
|
|
assert diagnosis["outcome"] is None
|
|
assert diagnosis["authorized"] is False
|
|
assert env.calls["repairs"] == []
|
|
|
|
|
|
def test_invalid_response_requires_human(monkeypatch) -> None:
|
|
rejected_before = _counter("unknown", "rejected")
|
|
env = _prepare(monkeypatch, run=_run(output="no json here"))
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
assert summary["jobs"][JOB]["reason"].startswith("response_invalid")
|
|
assert _statuses(env.storage) == ["detected", "human_required"]
|
|
assert _counter("unknown", "rejected") == rejected_before + 1.0
|
|
|
|
|
|
def test_model_human_required_is_rejected(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch, run=_run(output=_model_output(human_required=True)))
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
assert summary["jobs"][JOB]["reason"] == "human_required"
|
|
assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"]
|
|
assert env.calls["repairs"] == []
|
|
|
|
|
|
def test_missing_signature_is_rejected(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch, signature=False)
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
assert summary["jobs"][JOB]["reason"] == "evidence_signature_missing"
|
|
assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"]
|
|
assert env.calls["repairs"] == []
|
|
|
|
|
|
def test_non_allowlisted_action_is_rejected(monkeypatch) -> None:
|
|
rejected_before = _counter("unknown", "rejected")
|
|
output = _model_output(requested_action={"type": "run_ariadne_job", "id": "other_action"})
|
|
env = _prepare(monkeypatch, run=_run(output=output))
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
assert summary["jobs"][JOB]["reason"].startswith("action_not_allowlisted")
|
|
assert _counter("unknown", "rejected") == rejected_before + 1.0
|
|
|
|
|
|
def test_prior_action_blocks_second_action(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch)
|
|
_seed_incident(env.storage, "detected")
|
|
env.storage.record_event(
|
|
module.ACTION_EVENT_TYPE,
|
|
{"incident_id": INCIDENT_ID, "action": "repair_demo_fixture", "result": "requested"},
|
|
)
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
assert summary["jobs"][JOB]["reason"] == "max_actions_reached"
|
|
assert env.calls["repairs"] == []
|
|
|
|
|
|
def test_repair_failure_marks_failed_and_human(monkeypatch) -> None:
|
|
failed_before = _counter("repair_demo_fixture", "failed")
|
|
env = _prepare(
|
|
monkeypatch,
|
|
repair={"job_name": "hermes-demo-repair-12", "succeeded": False, "error": "repair job failed"},
|
|
)
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
assert summary["jobs"][JOB] == {
|
|
"status": "failed",
|
|
"incident_id": INCIDENT_ID,
|
|
"reason": "repair job failed",
|
|
}
|
|
assert _statuses(env.storage) == ["detected", "diagnosed", "repairing", "failed"]
|
|
actions = _events(env.storage, module.ACTION_EVENT_TYPE)
|
|
assert [action["result"] for action in actions] == ["requested", "accepted", "failed"]
|
|
assert env.calls["rebuilds"] == []
|
|
assert _counter("repair_demo_fixture", "failed") == failed_before + 1.0
|
|
assert _gauge("12", "failed") == 1.0
|
|
assert _gauge("12", "human_required") == 1.0
|
|
|
|
|
|
def test_rebuild_trigger_failure_marks_failed(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch, rebuild={"requested": False, "error": "rebuild http 500"})
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
assert summary["jobs"][JOB]["status"] == "failed"
|
|
assert summary["jobs"][JOB]["reason"] == "rebuild http 500"
|
|
actions = _events(env.storage, module.ACTION_EVENT_TYPE)
|
|
assert [action["result"] for action in actions] == ["requested", "accepted", "failed"]
|
|
|
|
|
|
def test_jenkins_fetch_failure_skips_job(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch, jenkins_exc=RuntimeError("boom"))
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
assert summary["jobs"][JOB] == {"status": "skipped"}
|
|
assert env.storage.events == []
|
|
|
|
|
|
def test_building_build_is_skipped(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch, last_build=_build(12, None, building=True))
|
|
assert module.run_hermes_autotriage(env.storage)["jobs"][JOB] == {"status": "skipped"}
|
|
assert env.storage.events == []
|
|
|
|
|
|
def test_empty_jenkins_base_url_skips(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch, cfg=_settings(jenkins_base_url=""))
|
|
assert module.run_hermes_autotriage(env.storage)["jobs"][JOB] == {"status": "skipped"}
|
|
assert env.calls["gets"] == []
|
|
|
|
|
|
def test_non_terminal_result_is_ignored(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch, last_build=_build(12, "ABORTED"))
|
|
assert module.run_hermes_autotriage(env.storage)["jobs"][JOB] == {
|
|
"status": "ignored",
|
|
"result": "ABORTED",
|
|
}
|
|
assert env.storage.events == []
|
|
|
|
|
|
def test_event_detail_tolerates_bad_payloads() -> None:
|
|
assert module._event_detail({"detail": "not-json"}) is None
|
|
assert module._event_detail({"detail": "[1,2]"}) is None
|
|
assert module._event_detail({"detail": 5}) is None
|
|
assert module._event_detail("not-a-row") is None
|
|
assert module._int_value("not-a-number") == 0
|
|
|
|
|
|
def test_jenkins_request_uses_basic_auth_and_tree(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch, last_build=_build(13, "SUCCESS"))
|
|
module.run_hermes_autotriage(env.storage)
|
|
assert env.calls["client_kwargs"]["auth"] == ("user", "token")
|
|
url, params = env.calls["gets"][0]
|
|
assert url == f"https://ci.example/job/{JOB}/api/json"
|
|
assert params == {"tree": "lastBuild[number,result,building,timestamp,duration,url]"}
|