From bbb4f6b1852141be663ae234308486746d6a46d3 Mon Sep 17 00:00:00 2001 From: jenkins Date: Thu, 6 Aug 2026 18:52:08 -0300 Subject: [PATCH] fix(demo): show durable evidence at detection, not live cluster state The detection stage read the fixture ConfigMap, which is a live value. Every stage describes a moment that has passed, and by the time detection is narrated the repair has already run - so it printed 'healthy' and implied the fixture had never failed, contradicting the premise of the whole run. Replay made it certain rather than occasional. It now prints the incident's recorded state changes, which stay true afterwards: 21:48:03 detected {result: FAILURE, ...} 21:48:18 diagnosed {classification: known_demo_fixture_failure, ...} 21:48:18 repairing {action: repair_demo_fixture} 21:48:18 awaiting_rebuild {repair: configmap_patch, ...} 21:49:03 resolved {resolved_by_build: 35} The fixture read stays on the response stage, where healthy is the point. Co-Authored-By: Claude Opus 5 --- scripts/ops/hermes_triage_monitor.py | 38 +++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/scripts/ops/hermes_triage_monitor.py b/scripts/ops/hermes_triage_monitor.py index 3d2c095c1..c3ffe4538 100755 --- a/scripts/ops/hermes_triage_monitor.py +++ b/scripts/ops/hermes_triage_monitor.py @@ -194,6 +194,35 @@ _DIAG_QUERY = ( ) +_HISTORY_QUERY = ( + ". /vault/secrets/ariadne-env.sh >/dev/null 2>&1; python3 -c \"" + "import json,os,psycopg;" + "conn=psycopg.connect(os.environ['ARIADNE_DATABASE_URL']);" + "cur=conn.cursor();" + "cur.execute(\\\"select created_at,detail from ariadne_events where" + " event_type='hermes_autotriage_incident' order by id desc limit 40\\\");" + "rows=[(t,d if isinstance(d,dict) else json.loads(d)) for t,d in cur.fetchall()];" + "rows=[r for r in rows if r[1].get('incident_id')=='__INCIDENT__'];" + "[print(str(t)[11:19], v.get('status'), json.dumps(v.get('phase') or {})[:80])" + " for t,v in reversed(rows)]\"" +) + + +def incident_history(incident: str) -> None: + """Show this incident's recorded state changes, oldest first. + + Durable evidence on purpose. A stage describes a moment that has passed, + so reading live cluster state at that point misrepresents it: by the time + the detection stage is narrated the repair has already run, and the + fixture would read healthy as though it had never failed. + """ + + if not incident: + return + run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--", + "sh", "-c", _HISTORY_QUERY.replace("__INCIDENT__", incident)], limit=10) + + def diagnosis_event() -> None: """Show the diagnosis Ariadne stored: what Hermes said, and the verdict.""" @@ -201,12 +230,13 @@ def diagnosis_event() -> None: "sh", "-c", _DIAG_QUERY], limit=16) -def evidence_for(key: str) -> None: +def evidence_for(key: str, incident: str = "") -> None: """Run the reads that show this stage actually happened.""" if key == "detect": - run(["kubectl", "-n", NS_DEMO, "get", "cm", "hermes-triage-demo-fixture", - "-o", "jsonpath={.data.state}"]) + print(f" {DIM}the incident's recorded state changes; durable, so it still reads" + f" true after the repair has run:{RESET}") + incident_history(incident) elif key == "evidence": print(f" {DIM}the bundle is what Ariadne sends Hermes; the audit trail keeps it{RESET}\n") run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--", @@ -255,7 +285,7 @@ class Monitor: self.done.add(key) banner(key) print(f" {BOLD}what happened:{RESET} {evidence}\n") - evidence_for(key) + evidence_for(key, self.incident) def new_incident(self, incident: str) -> None: if incident and incident != self.incident: