diff --git a/scripts/ops/hermes_triage_monitor.py b/scripts/ops/hermes_triage_monitor.py index 3d2c095c1..c3ffe4538 100755 --- a/scripts/ops/hermes_triage_monitor.py +++ b/scripts/ops/hermes_triage_monitor.py @@ -194,6 +194,35 @@ _DIAG_QUERY = ( ) +_HISTORY_QUERY = ( + ". /vault/secrets/ariadne-env.sh >/dev/null 2>&1; python3 -c \"" + "import json,os,psycopg;" + "conn=psycopg.connect(os.environ['ARIADNE_DATABASE_URL']);" + "cur=conn.cursor();" + "cur.execute(\\\"select created_at,detail from ariadne_events where" + " event_type='hermes_autotriage_incident' order by id desc limit 40\\\");" + "rows=[(t,d if isinstance(d,dict) else json.loads(d)) for t,d in cur.fetchall()];" + "rows=[r for r in rows if r[1].get('incident_id')=='__INCIDENT__'];" + "[print(str(t)[11:19], v.get('status'), json.dumps(v.get('phase') or {})[:80])" + " for t,v in reversed(rows)]\"" +) + + +def incident_history(incident: str) -> None: + """Show this incident's recorded state changes, oldest first. + + Durable evidence on purpose. A stage describes a moment that has passed, + so reading live cluster state at that point misrepresents it: by the time + the detection stage is narrated the repair has already run, and the + fixture would read healthy as though it had never failed. + """ + + if not incident: + return + run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--", + "sh", "-c", _HISTORY_QUERY.replace("__INCIDENT__", incident)], limit=10) + + def diagnosis_event() -> None: """Show the diagnosis Ariadne stored: what Hermes said, and the verdict.""" @@ -201,12 +230,13 @@ def diagnosis_event() -> None: "sh", "-c", _DIAG_QUERY], limit=16) -def evidence_for(key: str) -> None: +def evidence_for(key: str, incident: str = "") -> None: """Run the reads that show this stage actually happened.""" if key == "detect": - run(["kubectl", "-n", NS_DEMO, "get", "cm", "hermes-triage-demo-fixture", - "-o", "jsonpath={.data.state}"]) + print(f" {DIM}the incident's recorded state changes; durable, so it still reads" + f" true after the repair has run:{RESET}") + incident_history(incident) elif key == "evidence": print(f" {DIM}the bundle is what Ariadne sends Hermes; the audit trail keeps it{RESET}\n") run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--", @@ -255,7 +285,7 @@ class Monitor: self.done.add(key) banner(key) print(f" {BOLD}what happened:{RESET} {evidence}\n") - evidence_for(key) + evidence_for(key, self.incident) def new_incident(self, incident: str) -> None: if incident and incident != self.incident: