diff --git a/scripts/ops/hermes_triage_demo.sh b/scripts/ops/hermes_triage_demo.sh index f1d654a00..2bfcbda4f 100755 --- a/scripts/ops/hermes_triage_demo.sh +++ b/scripts/ops/hermes_triage_demo.sh @@ -5,6 +5,8 @@ # hermes_triage_demo.sh code # proposal loop: fail -> Hermes patch -> PR # hermes_triage_demo.sh status # current incident/alert state, no changes # hermes_triage_demo.sh preflight # confirm the lab is ready to demo +# hermes_triage_demo.sh reset # restore the demo to its pre-run state +# hermes_triage_demo.sh monitor # stream the flow chart stages live # # FIRST RUN: copy hermes_triage_demo.env.example to hermes_triage_demo.env in # this directory and fill it in. That file is git-ignored precisely so it can @@ -76,6 +78,65 @@ for line in sys.stdin: print(" ", d["timestamp"][11:19], d.get("jobs"))' | tail -"${2:-5}" } +cmd_monitor() { + exec python3 "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_triage_monitor.py" +} + +# Restores only what the demo itself creates. Real service repositories are +# never touched: those issues are genuine triage records and deleting them to +# tidy a demo would destroy the evidence the system exists to produce. +cmd_reset() { + require_jenkins + say "Reset — restoring the demo to its pre-run state" + + note "fixture -> healthy" + kubectl -n "$DEMO_NS" patch cm hermes-triage-demo-fixture \ + --type merge -p '{"data":{"state":"healthy"}}' >/dev/null 2>&1 && + note " fixture: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')" || + note " fixture patch failed (is the demo namespace present?)" + + local gitea="${GITEA_URL:-https://scm.bstein.dev}" + if [ -z "${GITEA_TOKEN:-}" ]; then + note "GITEA_TOKEN unset; skipping pull request and branch cleanup" + else + note "closing open repair pull requests on hermes-code-demo" + local prs + prs="$(curl -s -H "Authorization: token $GITEA_TOKEN" \ + "$gitea/api/v1/repos/bstein/hermes-code-demo/pulls?state=open" | + python3 -c 'import json,sys +for p in json.load(sys.stdin): + print(p["number"], p["head"]["ref"])' 2>/dev/null || true)" + if [ -z "$prs" ]; then + note " none open" + else + while read -r num ref; do + [ -z "$num" ] && continue + curl -s -o /dev/null -X PATCH -H "Authorization: token $GITEA_TOKEN" \ + -H "Content-Type: application/json" -d '{"state":"closed"}' \ + "$gitea/api/v1/repos/bstein/hermes-code-demo/pulls/$num" + curl -s -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \ + "$gitea/api/v1/repos/bstein/hermes-code-demo/branches/$ref" + note " closed #$num and deleted branch $ref" + done <<< "$prs" + fi + fi + + if [ -d "$CODE_REPO_DIR/.git" ]; then + note "restoring the demo repository working state" + ( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q --ff-only 2>/dev/null || true ) + if grep -q 'percent / 100' "$CODE_REPO_DIR/src/discount.py" 2>/dev/null; then + note " src/discount.py is correct; demo is armable" + else + note " src/discount.py still carries the seeded defect — merge or revert it before demoing" + fi + else + note "demo repository not cloned at $CODE_REPO_DIR; skipping" + fi + + say "Ready" + note "run 'preflight' next, then arm the fixture build" +} + cmd_preflight() { require_jenkins say "Preflight" @@ -200,5 +261,7 @@ case "${1:-}" in code) cmd_code ;; status) cmd_status ;; preflight) cmd_preflight ;; + reset) cmd_reset ;; + monitor) cmd_monitor ;; *) sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;; esac diff --git a/scripts/ops/hermes_triage_monitor.py b/scripts/ops/hermes_triage_monitor.py new file mode 100755 index 000000000..495e61b9a --- /dev/null +++ b/scripts/ops/hermes_triage_monitor.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Stream the triage flow as it happens, in the stages of the flow chart. + +Run in a second terminal alongside the demo. Each stage below is a subgraph in +`mermaid/TestAutomation.mmd`, so what prints here can be followed on the chart: + + Detect and gather -> Hermes analysis -> Ariadne policy gates + -> Ariadne response -> Inspectable outputs + +Every stage prints the evidence that moved it, because the point of the chart +is that each step is answerable from data rather than asserted. + +Read-only: it polls Kubernetes, Jenkins and Gitea and changes nothing. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from datetime import datetime, timezone + +JOB = os.environ.get("MONITOR_JOB", "hermes-triage-demo") +NS_DEMO = "hermes-triage-demo" +NS_ARIADNE = "maintenance" +POLL_SECONDS = 4 + +STAGES = [ + ("detect", "Detect and gather", "Jenkins failure detector opens an incident"), + ("evidence", "Detect and gather", "Ariadne collects the bounded evidence bundle"), + ("hermes", "Hermes analysis", "Hermes returns a classification it cannot act on"), + ("gates", "Ariadne policy gates", "Ariadne authorizes or refuses, on its own reading"), + ("response", "Ariadne response", "Ariadne executes, proposes, or escalates"), + ("verify", "Ariadne response", "One rebuild decides whether the repair worked"), + ("outputs", "Inspectable outputs", "Incident closed; artifacts left behind"), +] + +BOLD, DIM, GREEN, YELLOW, RED, RESET = ( + "\033[1m", "\033[2m", "\033[32m", "\033[33m", "\033[31m", "\033[0m" +) + + +def sh(*args: str, timeout: int = 20) -> str: + """Run a command and return stdout, or "" on any failure.""" + + try: + out = subprocess.run(args, capture_output=True, text=True, timeout=timeout) + return out.stdout.strip() + except Exception: + return "" + + +def stamp() -> str: + return datetime.now(timezone.utc).strftime("%H:%M:%S") + + +def fixture_state() -> str: + return sh("kubectl", "-n", NS_DEMO, "get", "cm", "hermes-triage-demo-fixture", + "-o", "jsonpath={.data.state}") + + +def ariadne_lines(tail: int = 400) -> list[dict]: + """Return recent Ariadne log records that parse as JSON.""" + + raw = sh("kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne", + f"--tail={tail}", timeout=25) + records = [] + for line in raw.split("\n"): + line = line.strip() + if line.startswith("{"): + try: + records.append(json.loads(line)) + except ValueError: + continue + return records + + +def job_state(records: list[dict]) -> dict: + """Pull this job's latest auto-triage tick result out of the log.""" + + for record in reversed(records): + if record.get("event") == "hermes_autotriage" and record.get("jobs"): + try: + return json.loads(record["jobs"]).get(JOB) or {} + except ValueError: + return {} + return {} + + +class Monitor: + """Track which flow-chart stage the current incident has reached.""" + + def __init__(self) -> None: + self.done: dict[str, str] = {} + self.incident = "" + self.last_line = "" + + def mark(self, key: str, evidence: str) -> None: + """Record a stage as reached and print it once, with its evidence.""" + + if key in self.done: + return + self.done[key] = evidence + _, subgraph, meaning = next(s for s in STAGES if s[0] == key) + print(f"{GREEN}{stamp()} ✓ {BOLD}{subgraph}{RESET}{GREEN} — {meaning}{RESET}") + print(f" {DIM}evidence:{RESET} {evidence}") + sys.stdout.flush() + + def checklist(self) -> None: + """Print the stage checklist as it currently stands.""" + + print(f"\n{BOLD} flow chart progress{RESET}") + for key, subgraph, _meaning in STAGES: + mark = f"{GREEN}✓{RESET}" if key in self.done else f"{DIM}·{RESET}" + label = subgraph if key in self.done else f"{DIM}{subgraph}{RESET}" + print(f" {mark} {key:<9} {label}") + print() + sys.stdout.flush() + + def reset_for(self, incident: str) -> None: + if incident and incident != self.incident: + self.incident = incident + self.done.clear() + print(f"\n{BOLD}{YELLOW}{stamp()} ── incident {incident} ──{RESET}") + sys.stdout.flush() + + +def main() -> None: + print(f"{BOLD}Watching {JOB}. Stages follow mermaid/TestAutomation.mmd.{RESET}") + print(f"{DIM}Read-only. Ctrl-C to stop.{RESET}\n") + monitor = Monitor() + while True: + records = ariadne_lines() + state = job_state(records) + status = str(state.get("status") or "") + incident = str(state.get("incident_id") or "") + monitor.reset_for(incident) + + if incident and status in {"human_required", "awaiting_rebuild", "deduped", "failed"}: + monitor.mark("detect", f"incident {incident} opened from a terminal build failure") + + for record in records: + if record.get("logger", "").endswith("hermes_autotriage") and record.get("status") == "ok": + continue + msg = str(record.get("message") or "") + if "run_triage" in msg or "hermes run" in msg.lower(): + monitor.mark("hermes", msg[:110]) + if record.get("event") == "hermes_code_flow": + monitor.mark("response", f"code proposal: {record.get('status')}") + + if status == "awaiting_rebuild": + monitor.mark("evidence", "bundle collected; console regions, tests and logs attached") + monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema") + monitor.mark("gates", "every gate passed; Ariadne authorized the predefined action") + monitor.mark( + "response", + f"{state.get('repair', 'action')} on {state.get('target', 'target')}", + ) + monitor.mark("verify", "one rebuild triggered with seeding disabled") + if status == "human_required": + monitor.mark("evidence", "bundle collected") + monitor.mark("hermes", "diagnosis returned") + monitor.mark( + "gates", + f"Ariadne refused: {state.get('reason', 'human_required')} — no action taken", + ) + monitor.mark("response", "escalated to a human; issue filed in the service repository") + if status == "healthy" and state.get("resolved"): + monitor.mark("verify", "rebuild finished green") + monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}") + monitor.checklist() + + line = f"{status}|{incident}|{fixture_state()}" + if line != monitor.last_line: + monitor.last_line = line + fixture = fixture_state() + colour = GREEN if fixture == "healthy" else RED + print( + f"{DIM}{stamp()}{RESET} tick: {status or 'idle':<16} " + f"fixture={colour}{fixture or '?'}{RESET} {DIM}{incident}{RESET}" + ) + sys.stdout.flush() + time.sleep(POLL_SECONDS) + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\nstopped")