diff --git a/scripts/ops/hermes_triage_monitor.py b/scripts/ops/hermes_triage_monitor.py index c4c9a3bca..3d2c095c1 100755 --- a/scripts/ops/hermes_triage_monitor.py +++ b/scripts/ops/hermes_triage_monitor.py @@ -38,7 +38,7 @@ JENKINS = os.environ.get("JENKINS_URL", "https://ci.bstein.dev") GITEA = os.environ.get("GITEA_URL", "https://scm.bstein.dev") GRAFANA = os.environ.get("GRAFANA_URL", "https://metrics.bstein.dev") HERMES_UI = os.environ.get("HERMES_URL", "https://agent.bstein.dev") -POLL_SECONDS = 4 +POLL_SECONDS = 6 BOLD, DIM, GREEN, CYAN, YELLOW, RED, RESET = ( "\033[1m", "\033[2m", "\033[32m", "\033[36m", "\033[33m", "\033[31m", "\033[0m" @@ -49,49 +49,49 @@ STAGES: dict[str, tuple[str, str, str, str]] = { "detect": ( "Detect and gather", "A failed build becomes one incident", - "intake: detector -> evidence_sources", + "Jenkins detector -> Evidence sources", f"{JENKINS}/job/{JOB}/ — the red build", ), "evidence": ( "Detect and gather", "Ariadne assembles the bounded evidence bundle", - "intake: collector -> bundle", + "Console reader -> Failure ranker -> Context filter -> Incident bundle", "", ), "hermes": ( "Hermes analysis", "Hermes returns a recommendation it cannot act on", - "hermes_plane: skills -> recommendation", + "Triage skills -> Structured recommendation", f"{HERMES_UI} — the agent run appears here", ), "gates": ( "Ariadne policy gates", "Ariadne decides on its own reading of the evidence", - "controls: response_check -> authorizer", + "Response check -> Scoped repair guard -> Action authorizer", "", ), "route": ( "Ariadne response", - "Which edge off the route diamond is taken", - "response: route", + "The policy result, and which branch it opens", + "Policy result -> Action registry", "", ), "response": ( "Ariadne response", "Ariadne executes, proposes, or escalates", - "response: operational_path / code_path / human_path", + "Action registry -> Scoped ConfigMap repair -> Action result", "", ), "verify": ( "Ariadne response", "One rebuild decides whether the repair held", - "response: operational_result -> validation build", + "Action result -> validation build", f"{JENKINS}/job/{JOB}/ — a new build starts on its own", ), "outputs": ( "Inspectable outputs", "Incident closed; the artifacts remain", - "outputs: audit_events, alert_output, issue_output", + "Ariadne records every outcome -> Audit events, Triage metrics", f"{GRAFANA}/d/atlas-testing — triage panels", ), } @@ -108,7 +108,9 @@ def run(cmd: list[str], *, show: bool = True, limit: int = 12) -> str: if show: print(f" {DIM}${RESET} {CYAN}{' '.join(cmd)}{RESET}") try: - done = subprocess.run(cmd, capture_output=True, text=True, timeout=25) + # Ariadne emits ~200 lines per tick, so a ten-minute window is a + # few thousand lines; the fetch needs room to finish. + done = subprocess.run(cmd, capture_output=True, text=True, timeout=60) out = done.stdout.strip() except Exception as exc: # a failed read must never stop the narration print(f" {RED}(command failed: {exc}){RESET}") @@ -134,7 +136,7 @@ def banner(key: str) -> None: print(f"{GREEN}{'─' * 70}{RESET}") -def ariadne_records(since: str = "15m") -> list[dict]: +def ariadne_records(since: str = "10m") -> list[dict]: """Return recent Ariadne log records that parse as JSON. Windowed by time, not line count. Ariadne emits roughly two hundred lines @@ -177,6 +179,28 @@ def job_states(records: list[dict]) -> list[dict]: return states +_DIAG_QUERY = ( + ". /vault/secrets/ariadne-env.sh >/dev/null 2>&1; python3 -c \"" + "import json,os,psycopg;" + "conn=psycopg.connect(os.environ['ARIADNE_DATABASE_URL']);" + "cur=conn.cursor();" + "cur.execute(\\\"select detail from ariadne_events where event_type='hermes_autotriage_diagnosis'" + " order by id desc limit 1\\\");" + "r=cur.fetchone()[0];" + "d=r if isinstance(r,dict) else json.loads(r);" + "print(json.dumps({'incident':d.get('incident_id'),'authorized':d.get('authorized')," + "'authorize_reason':d.get('authorize_reason'),'evidence_marker':d.get('evidence_marker')," + "'outcome':d.get('outcome')},indent=1))\"" +) + + +def diagnosis_event() -> None: + """Show the diagnosis Ariadne stored: what Hermes said, and the verdict.""" + + run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--", + "sh", "-c", _DIAG_QUERY], limit=16) + + def evidence_for(key: str) -> None: """Run the reads that show this stage actually happened.""" @@ -188,15 +212,28 @@ def evidence_for(key: str) -> None: run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--", "printenv", "ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST"], limit=2) elif key == "hermes": - run(["kubectl", "-n", "hermes", "get", "pods", "-l", "app=hermes", "--no-headers"], limit=3) - print(f" {DIM}Hermes holds no Git or Kubernetes write access; it returns JSON only{RESET}\n") + print(f" {DIM}what Hermes actually returned, as Ariadne stored it:{RESET}") + diagnosis_event() + print(f" {DIM}Hermes holds no Git or Kubernetes write access; this JSON is its" + f" entire output{RESET}\n") elif key == "gates": - for var in ("ARIADNE_HERMES_ALLOWED_ACTIONS", "ARIADNE_HERMES_MIN_CONFIDENCE"): - run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--", - "printenv", var], limit=2) + print(f" {DIM}authorized/authorize_reason above is the policy result: the verdict" + f" Ariadne reached on its own reading{RESET}") + run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--", + "printenv", "ARIADNE_HERMES_ALLOWED_ACTIONS"], limit=2) + run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--", + "printenv", "ARIADNE_HERMES_MIN_CONFIDENCE"], limit=2) + print(f" {DIM}only these two ids may ever be executed. Source fixes are not actions:" + f" they become pull requests and are never executed by Ariadne.{RESET}\n") + elif key == "route": + print(f" {DIM}the fixture repair is an operational action, so the Optional source" + f" proposal branch is not taken for this incident{RESET}\n") elif key in {"response", "outputs"}: run(["kubectl", "-n", NS_DEMO, "get", "cm", "hermes-triage-demo-fixture", "-o", "jsonpath={.data.state}"]) + if key == "response": + print(f" {DIM}Scoped ConfigMap repair on the chart: one merge patch to one" + f" ConfigMap, in process, no pod created{RESET}\n") if key == "outputs": print(f" {YELLOW}issues filed by triage: {GITEA}/bstein/ariadne/issues{RESET}\n") elif key == "verify": @@ -231,7 +268,7 @@ class Monitor: print(f"\n{BOLD} flow chart progress{RESET}") for key in ORDER: tick = f"{GREEN}✓{RESET}" if key in self.done else f"{DIM}·{RESET}" - print(f" {tick} {key:<9} {STAGES[key][0]}") + print(f" {tick} {key:<9} {STAGES[key][0]:<22} {DIM}{STAGES[key][2]}{RESET}") print() @@ -255,14 +292,23 @@ def main() -> None: monitor.mark("evidence", "bundle collected: console regions, tests, logs") monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema") monitor.mark("gates", "every gate passed; the predefined action was authorized") - monitor.mark("route", "route --|authorized action|--> registry -> operational_result") - monitor.mark("response", f"{state.get('repair', 'action')} on {state.get('target', '')}") + monitor.mark( + "route", + "policy result: authorized -> Action registry (Optional source proposal not taken)", + ) + monitor.mark( + "response", + f"Scoped ConfigMap repair: {state.get('repair', 'action')} on {state.get('target', '')}", + ) monitor.mark("verify", "one rebuild triggered with seeding disabled") if status == "human_required": monitor.mark("evidence", "bundle collected") monitor.mark("hermes", "diagnosis returned") monitor.mark("gates", f"Ariadne refused: {state.get('reason', '')} — nothing ran") - monitor.mark("route", "route --|human required|--> human_required -> gitea_issue") + monitor.mark( + "route", + "policy result: refused -> Diagnosis and next checks -> Ariadne opens an issue", + ) monitor.mark("response", "escalated; issue filed in the service repository") if status == "healthy" and state.get("resolved") and not monitor.summarised: monitor.mark("verify", "rebuild finished green")