diff --git a/scripts/ops/hermes_triage_monitor.py b/scripts/ops/hermes_triage_monitor.py index a38258d48..eda574833 100755 --- a/scripts/ops/hermes_triage_monitor.py +++ b/scripts/ops/hermes_triage_monitor.py @@ -1,22 +1,25 @@ #!/usr/bin/env python3 -"""Stream the triage flow as it happens, in the stages of the flow chart. +"""Narrate the triage flow live, showing every command it runs. -Run in a second terminal alongside the demo. Every stage name below is a -subgraph in `mermaid/TestAutomation.mmd`, and the route line quotes that -chart's own edge labels, so an audience can follow along on the diagram: +Run in a second terminal beside the demo. When a stage of +`mermaid/TestAutomation.mmd` is reached this prints, in order: - Detect and gather -> Hermes analysis -> Ariadne policy gates - -> Ariadne response -> Inspectable outputs + the stage banner, naming the chart subgraph + the service UI to look at, if one changes at that stage + each command, echoed before it runs, then its output -Each stage prints the evidence that moved it, because the claim the chart -makes is that every step is answerable from data rather than asserted. +Echoing the commands is the point. An audience watching a dashboard has to +take the result on trust; watching `kubectl` run against the cluster and +reading the raw answer is the difference between a demonstration and an +assertion. -Two honest limits. The chart draws finer nodes inside each subgraph - the -collector, the response check, the marker check, the authorizer - and this -reports at subgraph granularity, not per node. And it follows one incident on -one job at a time, whereas the chart describes the system as a whole. +Two honest limits. This reports at subgraph granularity, not per node: the +chart draws the collector, response check, marker check and authorizer +separately, and a refusal here names the gate rather than walking all twelve. +And it follows one incident on one job, whereas the chart describes the whole +system. -Read-only: it polls Kubernetes and changes nothing. +Read-only. Every command below is a read; nothing here changes the cluster. """ from __future__ import annotations @@ -31,48 +34,110 @@ from datetime import datetime, timezone JOB = os.environ.get("MONITOR_JOB", "hermes-triage-demo") NS_DEMO = "hermes-triage-demo" NS_ARIADNE = "maintenance" +JENKINS = os.environ.get("JENKINS_URL", "https://ci.bstein.dev") +GITEA = os.environ.get("GITEA_URL", "https://scm.bstein.dev") +GRAFANA = os.environ.get("GRAFANA_URL", "https://metrics.bstein.dev") +HERMES_UI = os.environ.get("HERMES_URL", "https://agent.bstein.dev") POLL_SECONDS = 4 -STAGES = [ - ("detect", "Detect and gather", "Jenkins failure detector opens an incident"), - ("evidence", "Detect and gather", "Ariadne collects the bounded evidence bundle"), - ("hermes", "Hermes analysis", "Hermes returns a classification it cannot act on"), - ("gates", "Ariadne policy gates", "Ariadne authorizes or refuses, on its own reading"), - ("route", "Ariadne response", "Which branch off the route diamond was taken"), - ("response", "Ariadne response", "Ariadne executes, proposes, or escalates"), - ("verify", "Ariadne response", "One rebuild decides whether the repair worked"), - ("outputs", "Inspectable outputs", "Incident closed; artifacts left behind"), -] - -BOLD, DIM, GREEN, YELLOW, RED, RESET = ( - "\033[1m", "\033[2m", "\033[32m", "\033[33m", "\033[31m", "\033[0m" +BOLD, DIM, GREEN, CYAN, YELLOW, RED, RESET = ( + "\033[1m", "\033[2m", "\033[32m", "\033[36m", "\033[33m", "\033[31m", "\033[0m" ) - -def sh(*args: str, timeout: int = 20) -> str: - """Run a command and return stdout, or "" on any failure.""" - - try: - out = subprocess.run(args, capture_output=True, text=True, timeout=timeout) - return out.stdout.strip() - except Exception: - return "" +# stage key -> (chart subgraph, what it means, chart path, UI worth showing) +STAGES: dict[str, tuple[str, str, str, str]] = { + "detect": ( + "Detect and gather", + "A failed build becomes one incident", + "intake: detector -> evidence_sources", + f"{JENKINS}/job/{JOB}/ — the red build", + ), + "evidence": ( + "Detect and gather", + "Ariadne assembles the bounded evidence bundle", + "intake: collector -> bundle", + "", + ), + "hermes": ( + "Hermes analysis", + "Hermes returns a recommendation it cannot act on", + "hermes_plane: skills -> recommendation", + f"{HERMES_UI} — the agent run appears here", + ), + "gates": ( + "Ariadne policy gates", + "Ariadne decides on its own reading of the evidence", + "controls: response_check -> authorizer", + "", + ), + "route": ( + "Ariadne response", + "Which edge off the route diamond is taken", + "response: route", + "", + ), + "response": ( + "Ariadne response", + "Ariadne executes, proposes, or escalates", + "response: operational_path / code_path / human_path", + "", + ), + "verify": ( + "Ariadne response", + "One rebuild decides whether the repair held", + "response: operational_result -> validation build", + f"{JENKINS}/job/{JOB}/ — a new build starts on its own", + ), + "outputs": ( + "Inspectable outputs", + "Incident closed; the artifacts remain", + "outputs: audit_events, alert_output, issue_output", + f"{GRAFANA}/d/atlas-testing — triage panels", + ), +} +ORDER = ["detect", "evidence", "hermes", "gates", "route", "response", "verify", "outputs"] def stamp() -> str: return datetime.now(timezone.utc).strftime("%H:%M:%S") -def fixture_state() -> str: - return sh("kubectl", "-n", NS_DEMO, "get", "cm", "hermes-triage-demo-fixture", - "-o", "jsonpath={.data.state}") +def run(cmd: list[str], *, show: bool = True, limit: int = 12) -> str: + """Echo a command, run it, print its output, and return stdout.""" + + if show: + print(f" {DIM}${RESET} {CYAN}{' '.join(cmd)}{RESET}") + try: + done = subprocess.run(cmd, capture_output=True, text=True, timeout=25) + out = done.stdout.strip() + except Exception as exc: # a failed read must never stop the narration + print(f" {RED}(command failed: {exc}){RESET}") + return "" + if show: + for line in (out.split("\n")[:limit] or ["(no output)"]): + print(f" {line[:150]}") + print() + return out -def ariadne_lines(tail: int = 400) -> list[dict]: - """Return recent Ariadne log records that parse as JSON.""" +def quiet(cmd: list[str]) -> str: + return run(cmd, show=False) - raw = sh("kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne", - f"--tail={tail}", timeout=25) + +def banner(key: str) -> None: + subgraph, meaning, path, ui = STAGES[key] + print(f"\n{GREEN}{'─' * 70}{RESET}") + print(f"{GREEN}{BOLD} {stamp()} {subgraph}{RESET}{GREEN} — {meaning}{RESET}") + print(f"{DIM} chart: {path}{RESET}") + if ui: + print(f"{YELLOW} look at: {ui}{RESET}") + print(f"{GREEN}{'─' * 70}{RESET}") + + +def ariadne_records(tail: int = 400) -> list[dict]: + raw = quiet( + ["kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne", f"--tail={tail}"] + ) records = [] for line in raw.split("\n"): line = line.strip() @@ -85,8 +150,6 @@ def ariadne_lines(tail: int = 400) -> list[dict]: def job_state(records: list[dict]) -> dict: - """Pull this job's latest auto-triage tick result out of the log.""" - for record in reversed(records): if record.get("event") == "hermes_autotriage" and record.get("jobs"): try: @@ -96,109 +159,102 @@ def job_state(records: list[dict]) -> dict: return {} +def evidence_for(key: str) -> None: + """Run the reads that show this stage actually happened.""" + + if key == "detect": + run(["kubectl", "-n", NS_DEMO, "get", "cm", "hermes-triage-demo-fixture", + "-o", "jsonpath={.data.state}"]) + elif key == "evidence": + print(f" {DIM}the bundle is what Ariadne sends Hermes; the audit trail keeps it{RESET}\n") + run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--", + "printenv", "ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST"], limit=2) + elif key == "hermes": + run(["kubectl", "-n", "hermes", "get", "pods", "-l", "app=hermes", "--no-headers"], limit=3) + print(f" {DIM}Hermes holds no Git or Kubernetes write access; it returns JSON only{RESET}\n") + elif key == "gates": + for var in ("ARIADNE_HERMES_ALLOWED_ACTIONS", "ARIADNE_HERMES_MIN_CONFIDENCE"): + run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--", + "printenv", var], limit=2) + elif key in {"response", "outputs"}: + run(["kubectl", "-n", NS_DEMO, "get", "cm", "hermes-triage-demo-fixture", + "-o", "jsonpath={.data.state}"]) + if key == "outputs": + print(f" {YELLOW}issues filed by triage: {GITEA}/bstein/ariadne/issues{RESET}\n") + elif key == "verify": + print(f" {DIM}exactly one rebuild is triggered; it never retries in a loop{RESET}\n") + + class Monitor: """Track which flow-chart stage the current incident has reached.""" def __init__(self) -> None: - self.done: dict[str, str] = {} + self.done: set[str] = set() self.incident = "" - self.last_line = "" self.summarised = False + self.last_tick = "" def mark(self, key: str, evidence: str) -> None: - """Record a stage as reached and print it once, with its evidence.""" - if key in self.done: return - self.done[key] = evidence - _, subgraph, meaning = next(s for s in STAGES if s[0] == key) - print(f"{GREEN}{stamp()} ✓ {BOLD}{subgraph}{RESET}{GREEN} — {meaning}{RESET}") - print(f" {DIM}evidence:{RESET} {evidence}") - sys.stdout.flush() + self.done.add(key) + banner(key) + print(f" {BOLD}what happened:{RESET} {evidence}\n") + evidence_for(key) - def checklist(self) -> None: - """Print the stage checklist as it currently stands.""" - - print(f"\n{BOLD} flow chart progress{RESET}") - for key, subgraph, _meaning in STAGES: - mark = f"{GREEN}✓{RESET}" if key in self.done else f"{DIM}·{RESET}" - label = subgraph if key in self.done else f"{DIM}{subgraph}{RESET}" - print(f" {mark} {key:<9} {label}") - print() - sys.stdout.flush() - - def reset_for(self, incident: str) -> None: + def new_incident(self, incident: str) -> None: if incident and incident != self.incident: self.incident = incident self.done.clear() self.summarised = False - print(f"\n{BOLD}{YELLOW}{stamp()} ── incident {incident} ──{RESET}") - sys.stdout.flush() + print(f"\n{BOLD}{YELLOW}══ incident {incident} ══{RESET}") + + def checklist(self) -> None: + print(f"\n{BOLD} flow chart progress{RESET}") + for key in ORDER: + tick = f"{GREEN}✓{RESET}" if key in self.done else f"{DIM}·{RESET}" + print(f" {tick} {key:<9} {STAGES[key][0]}") + print() def main() -> None: - print(f"{BOLD}Watching {JOB}. Stages follow mermaid/TestAutomation.mmd.{RESET}") - print(f"{DIM}Read-only. Ctrl-C to stop.{RESET}\n") + print(f"{BOLD}Hermes triage monitor — following mermaid/TestAutomation.mmd{RESET}") + print(f"{DIM}Every command is echoed before it runs. Read-only. Ctrl-C to stop.{RESET}") + print(f"{YELLOW}Jenkins: {JENKINS}/job/{JOB}/{RESET}") + print(f"{YELLOW}Gitea: {GITEA}/bstein{RESET}") + print(f"{YELLOW}Grafana: {GRAFANA}/d/atlas-testing{RESET}\n") monitor = Monitor() while True: - records = ariadne_lines() - state = job_state(records) + state = job_state(ariadne_records()) status = str(state.get("status") or "") - incident = str(state.get("incident_id") or "") - monitor.reset_for(incident) - - if incident and status in {"human_required", "awaiting_rebuild", "deduped", "failed"}: - monitor.mark("detect", f"incident {incident} opened from a terminal build failure") - - for record in records: - if record.get("logger", "").endswith("hermes_autotriage") and record.get("status") == "ok": - continue - msg = str(record.get("message") or "") - if "run_triage" in msg or "hermes run" in msg.lower(): - monitor.mark("hermes", msg[:110]) - if record.get("event") == "hermes_code_flow": - monitor.mark("route", "code_path (optional source proposal) also taken") - monitor.mark("response", f"code proposal: {record.get('status')} -> gitea_pr") + monitor.new_incident(str(state.get("incident_id") or "")) + if status in {"human_required", "awaiting_rebuild", "deduped", "failed"}: + monitor.mark("detect", f"incident {monitor.incident} opened from a terminal failure") if status == "awaiting_rebuild": - monitor.mark("evidence", "bundle collected; console regions, tests and logs attached") + monitor.mark("evidence", "bundle collected: console regions, tests, logs") monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema") - monitor.mark("gates", "every gate passed; Ariadne authorized the predefined action") - # The chart labels this edge "authorized action"; naming it lets a - # viewer point at the branch being taken rather than infer it. + monitor.mark("gates", "every gate passed; the predefined action was authorized") monitor.mark("route", "route --|authorized action|--> registry -> operational_result") - monitor.mark( - "response", - f"{state.get('repair', 'action')} on {state.get('target', 'target')}", - ) + monitor.mark("response", f"{state.get('repair', 'action')} on {state.get('target', '')}") monitor.mark("verify", "one rebuild triggered with seeding disabled") if status == "human_required": monitor.mark("evidence", "bundle collected") monitor.mark("hermes", "diagnosis returned") - monitor.mark( - "gates", - f"Ariadne refused: {state.get('reason', 'human_required')} — no action taken", - ) + monitor.mark("gates", f"Ariadne refused: {state.get('reason', '')} — nothing ran") monitor.mark("route", "route --|human required|--> human_required -> gitea_issue") - monitor.mark("response", "escalated to a human; issue filed in the service repository") + monitor.mark("response", "escalated; issue filed in the service repository") if status == "healthy" and state.get("resolved") and not monitor.summarised: monitor.mark("verify", "rebuild finished green") monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}") monitor.checklist() - # The resolving tick stays newest until another incident opens, so - # the completed checklist is printed once rather than every poll. + # The resolving tick stays newest until another incident opens. monitor.summarised = True - line = f"{status}|{incident}|{fixture_state()}" - if line != monitor.last_line: - monitor.last_line = line - fixture = fixture_state() - colour = GREEN if fixture == "healthy" else RED - print( - f"{DIM}{stamp()}{RESET} tick: {status or 'idle':<16} " - f"fixture={colour}{fixture or '?'}{RESET} {DIM}{incident}{RESET}" - ) - sys.stdout.flush() + if status and status != monitor.last_tick: + monitor.last_tick = status + print(f"{DIM}{stamp()} tick: {status}{RESET}") + sys.stdout.flush() time.sleep(POLL_SECONDS)