diff --git a/scripts/ops/hermes_triage_monitor.py b/scripts/ops/hermes_triage_monitor.py index bb0e225c1..f92422849 100755 --- a/scripts/ops/hermes_triage_monitor.py +++ b/scripts/ops/hermes_triage_monitor.py @@ -42,6 +42,25 @@ GRAFANA = os.environ.get("GRAFANA_URL", "https://metrics.bstein.dev") HERMES_UI = os.environ.get("HERMES_URL", "https://agent.bstein.dev") POLL_SECONDS = 6 +# `--filter ` restricts the monitor to incidents whose id contains that +# text; `--incident ` pins it to exactly one. Without either, it follows +# the newest incident it can see for this job, which is what someone who just +# triggered a build actually wants. +def _argv_option(name: str) -> str: + """Read `--name value` or `--name=value` from argv, or "" when absent.""" + + prefix = f"--{name}=" + for index, arg in enumerate(sys.argv[1:]): + if arg.startswith(prefix): + return arg[len(prefix) :].strip() + if arg == f"--{name}" and index + 2 <= len(sys.argv[1:]): + return sys.argv[index + 2].strip() + return "" + + +FILTER = _argv_option("filter") +PIN = _argv_option("incident") + # The fixture job repairs a ConfigMap; the code job proposes a patch. Evidence # that suits one is false for the other, so the stages branch on it. IS_CODE_JOB = JOB == "hermes-code-demo" @@ -183,7 +202,31 @@ def job_states(records: list[dict]) -> list[dict]: continue if state: states.append(state) - return states + return _one_incident(states) + + +def _one_incident(states: list[dict]) -> list[dict]: + """Narrow a window's ticks to the single incident worth narrating. + + A ten-minute window routinely holds two incidents - the build just pushed + and the one before it - and interleaving them produces a transcript that + reads as though the system is doing everything twice. Following one at a + time is both clearer and closer to the truth: the diagram describes the + life of one incident. + + `--incident` pins an exact id, `--filter` matches a substring, and + otherwise the newest incident in the window wins. + """ + + if PIN: + return [s for s in states if str(s.get("incident_id") or "") == PIN] + if FILTER: + states = [s for s in states if FILTER in str(s.get("incident_id") or "")] + ids = [str(s.get("incident_id") or "") for s in states if s.get("incident_id")] + if not ids: + return states + newest = ids[-1] + return [s for s in states if str(s.get("incident_id") or "") == newest] _DIAG_QUERY = ( @@ -191,15 +234,20 @@ _DIAG_QUERY = ( "import json,os,psycopg;" "conn=psycopg.connect(os.environ['ARIADNE_DATABASE_URL']);" "cur=conn.cursor();" - "cur.execute(\\\"select detail from ariadne_events where event_type='hermes_autotriage_diagnosis'" - " order by id desc limit 1\\\");" - "r=cur.fetchone()[0];" - "d=r if isinstance(r,dict) else json.loads(r);" - "print(json.dumps({'incident':d.get('incident_id'),'authorized':d.get('authorized')," - "'authorize_reason':d.get('authorize_reason'),'evidence_marker':d.get('evidence_marker')," - "'run_id':(d.get('run') or {}).get('run_id')," - "'run_seconds':(d.get('run') or {}).get('duration_seconds')," - "'outcome':d.get('outcome')},indent=1))\"" + "cur.execute(\\\"select event_type,detail from ariadne_events where event_type in" + " ('hermes_autotriage_diagnosis','hermes_autotriage_code_proposal')" + " order by id desc limit 200\\\");" + "rows=[(t, d if isinstance(d,dict) else json.loads(d)) for t,d in cur.fetchall()];" + "rows=[(t,d) for t,d in rows if d.get('incident_id')=='__INCIDENT__'];" + "print(json.dumps({'note':'no Hermes run recorded for this incident yet'},indent=1))" + " if not rows else None;" + "t,d=(rows[0] if rows else ('',{}));" + "print(json.dumps({'event':t,'incident':d.get('incident_id')," + "'authorized':d.get('authorized'),'authorize_reason':d.get('authorize_reason')," + "'validated':d.get('validated'),'reject_reason':d.get('reject_reason')," + "'chosen_path':d.get('chosen_path'),'url':d.get('url')," + "'run_id':d.get('run_id') or (d.get('run') or {}).get('run_id')," + "'outcome':d.get('outcome')},indent=1)) if rows else None\"" ) @@ -269,11 +317,17 @@ def incident_history(incident: str) -> None: "sh", "-c", _HISTORY_QUERY.replace("__INCIDENT__", incident)], limit=10) -def diagnosis_event() -> str: - """Show the diagnosis Ariadne stored, and return its Hermes run id.""" +def diagnosis_event(incident: str) -> str: + """Show the diagnosis Ariadne stored for one incident, and return its run id. + + Scoped to the incident on purpose. Reading the newest diagnosis in the + table meant an unrelated service's run could appear in the middle of this + incident's narration - which is worse than showing nothing, because it + looks like the answer to the question on screen. + """ out = run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--", - "sh", "-c", _DIAG_QUERY], limit=26) + "sh", "-c", _DIAG_QUERY.replace("__INCIDENT__", incident)], limit=26) match = re.search(r'"run_id":\s*"([^"]+)"', out or "") return match.group(1) if match else "" @@ -307,7 +361,7 @@ def evidence_for(key: str, incident: str = "") -> None: bundle_sample(incident) elif key == "hermes": print(f" {DIM}what Hermes actually returned, as Ariadne stored it:{RESET}") - run_id = diagnosis_event() + run_id = diagnosis_event(incident) hermes_run_link(run_id) print(f" {DIM}Hermes holds no Git or Kubernetes write access; this JSON is its" f" entire output. When outcome.suggested_remediation is populated, Hermes found" @@ -371,30 +425,36 @@ class Monitor: """Track which Test Automation Diagram stage the current incident has reached.""" def __init__(self) -> None: - self.done: set[str] = set() + # Keyed by incident. A single shared set was cleared whenever the + # incident changed, so two live incidents in the same window wiped each + # other's progress and reprinted every stage on every poll. + self.done: dict[str, set[str]] = {} self.incident = "" - self.summarised = False + self.summarised: set[str] = set() self.last_tick = "" def mark(self, key: str, evidence: str) -> None: - if key in self.done: + seen = self.done.setdefault(self.incident, set()) + if key in seen: return - self.done.add(key) + seen.add(key) banner(key) print(f" {BOLD}what happened:{RESET} {evidence}\n") evidence_for(key, self.incident) def new_incident(self, incident: str) -> None: - if incident and incident != self.incident: - self.incident = incident - self.done.clear() - self.summarised = False + if not incident or incident == self.incident: + return + self.incident = incident + if incident not in self.done: + self.done[incident] = set() print(f"\n{BOLD}{YELLOW}══ incident {incident} ══{RESET}") def checklist(self) -> None: print(f"\n{BOLD} Test Automation Diagram progress{RESET}") + seen = self.done.get(self.incident, set()) for key in ORDER: - tick = f"{GREEN}✓{RESET}" if key in self.done else f"{DIM}·{RESET}" + tick = f"{GREEN}✓{RESET}" if key in seen else f"{DIM}·{RESET}" print(f" {tick} {key:<9} {STAGES[key][0]:<22} {DIM}{STAGES[key][2]}{RESET}") print() @@ -405,7 +465,14 @@ def main() -> None: print(f"{DIM}Every command is echoed before it runs. Read-only. Ctrl-C to stop.{RESET}") print(f"{YELLOW}Jenkins: {JENKINS}/job/{JOB}/{RESET}") print(f"{YELLOW}Gitea: {GITEA}/bstein{RESET}") - print(f"{YELLOW}Grafana: {GRAFANA}/d/atlas-testing{RESET}\n") + print(f"{YELLOW}Grafana: {GRAFANA}/d/atlas-testing{RESET}") + if PIN: + print(f"{DIM}following incident {PIN} only{RESET}\n") + elif FILTER: + print(f"{DIM}following incidents matching {FILTER!r}{RESET}\n") + else: + print(f"{DIM}following the newest incident for {JOB};" + f" pass --filter TEXT or --incident ID to pin one{RESET}\n") monitor = Monitor() while True: states = job_states(ariadne_records()) @@ -464,11 +531,11 @@ def main() -> None: "the branch build validates the proposal; the incident stays" " human-required either way", ) - if status == "healthy" and state.get("resolved") and not monitor.summarised: + if status == "healthy" and state.get("resolved") and monitor.incident not in monitor.summarised: monitor.mark("verify", "rebuild finished green") monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}") monitor.checklist() - monitor.summarised = True + monitor.summarised.add(monitor.incident) status = str(states[-1].get("status") or "") if states else "" if status and status != monitor.last_tick: