feat(demo): narrate the monitor by echoing the commands it runs
An audience watching a dashboard has to take the result on trust. Watching kubectl run against the cluster and reading the raw answer is the difference between a demonstration and an assertion, so every read is now printed before it runs and its output shown beneath. Each stage also names the service UI worth looking at when something changes there: the Jenkins job as the build goes red and again when the rebuild starts on its own, the Hermes dashboard as the agent run appears, the Grafana triage panels and the filed issues at the end. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
jenkins
parent
cf9cc19d13
commit
16d771aa69
@ -1,22 +1,25 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Stream the triage flow as it happens, in the stages of the flow chart.
|
||||
"""Narrate the triage flow live, showing every command it runs.
|
||||
|
||||
Run in a second terminal alongside the demo. Every stage name below is a
|
||||
subgraph in `mermaid/TestAutomation.mmd`, and the route line quotes that
|
||||
chart's own edge labels, so an audience can follow along on the diagram:
|
||||
Run in a second terminal beside the demo. When a stage of
|
||||
`mermaid/TestAutomation.mmd` is reached this prints, in order:
|
||||
|
||||
Detect and gather -> Hermes analysis -> Ariadne policy gates
|
||||
-> Ariadne response -> Inspectable outputs
|
||||
the stage banner, naming the chart subgraph
|
||||
the service UI to look at, if one changes at that stage
|
||||
each command, echoed before it runs, then its output
|
||||
|
||||
Each stage prints the evidence that moved it, because the claim the chart
|
||||
makes is that every step is answerable from data rather than asserted.
|
||||
Echoing the commands is the point. An audience watching a dashboard has to
|
||||
take the result on trust; watching `kubectl` run against the cluster and
|
||||
reading the raw answer is the difference between a demonstration and an
|
||||
assertion.
|
||||
|
||||
Two honest limits. The chart draws finer nodes inside each subgraph - the
|
||||
collector, the response check, the marker check, the authorizer - and this
|
||||
reports at subgraph granularity, not per node. And it follows one incident on
|
||||
one job at a time, whereas the chart describes the system as a whole.
|
||||
Two honest limits. This reports at subgraph granularity, not per node: the
|
||||
chart draws the collector, response check, marker check and authorizer
|
||||
separately, and a refusal here names the gate rather than walking all twelve.
|
||||
And it follows one incident on one job, whereas the chart describes the whole
|
||||
system.
|
||||
|
||||
Read-only: it polls Kubernetes and changes nothing.
|
||||
Read-only. Every command below is a read; nothing here changes the cluster.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@ -31,48 +34,110 @@ from datetime import datetime, timezone
|
||||
JOB = os.environ.get("MONITOR_JOB", "hermes-triage-demo")
|
||||
NS_DEMO = "hermes-triage-demo"
|
||||
NS_ARIADNE = "maintenance"
|
||||
JENKINS = os.environ.get("JENKINS_URL", "https://ci.bstein.dev")
|
||||
GITEA = os.environ.get("GITEA_URL", "https://scm.bstein.dev")
|
||||
GRAFANA = os.environ.get("GRAFANA_URL", "https://metrics.bstein.dev")
|
||||
HERMES_UI = os.environ.get("HERMES_URL", "https://agent.bstein.dev")
|
||||
POLL_SECONDS = 4
|
||||
|
||||
STAGES = [
|
||||
("detect", "Detect and gather", "Jenkins failure detector opens an incident"),
|
||||
("evidence", "Detect and gather", "Ariadne collects the bounded evidence bundle"),
|
||||
("hermes", "Hermes analysis", "Hermes returns a classification it cannot act on"),
|
||||
("gates", "Ariadne policy gates", "Ariadne authorizes or refuses, on its own reading"),
|
||||
("route", "Ariadne response", "Which branch off the route diamond was taken"),
|
||||
("response", "Ariadne response", "Ariadne executes, proposes, or escalates"),
|
||||
("verify", "Ariadne response", "One rebuild decides whether the repair worked"),
|
||||
("outputs", "Inspectable outputs", "Incident closed; artifacts left behind"),
|
||||
]
|
||||
|
||||
BOLD, DIM, GREEN, YELLOW, RED, RESET = (
|
||||
"\033[1m", "\033[2m", "\033[32m", "\033[33m", "\033[31m", "\033[0m"
|
||||
BOLD, DIM, GREEN, CYAN, YELLOW, RED, RESET = (
|
||||
"\033[1m", "\033[2m", "\033[32m", "\033[36m", "\033[33m", "\033[31m", "\033[0m"
|
||||
)
|
||||
|
||||
|
||||
def sh(*args: str, timeout: int = 20) -> str:
|
||||
"""Run a command and return stdout, or "" on any failure."""
|
||||
|
||||
try:
|
||||
out = subprocess.run(args, capture_output=True, text=True, timeout=timeout)
|
||||
return out.stdout.strip()
|
||||
except Exception:
|
||||
return ""
|
||||
# stage key -> (chart subgraph, what it means, chart path, UI worth showing)
|
||||
STAGES: dict[str, tuple[str, str, str, str]] = {
|
||||
"detect": (
|
||||
"Detect and gather",
|
||||
"A failed build becomes one incident",
|
||||
"intake: detector -> evidence_sources",
|
||||
f"{JENKINS}/job/{JOB}/ — the red build",
|
||||
),
|
||||
"evidence": (
|
||||
"Detect and gather",
|
||||
"Ariadne assembles the bounded evidence bundle",
|
||||
"intake: collector -> bundle",
|
||||
"",
|
||||
),
|
||||
"hermes": (
|
||||
"Hermes analysis",
|
||||
"Hermes returns a recommendation it cannot act on",
|
||||
"hermes_plane: skills -> recommendation",
|
||||
f"{HERMES_UI} — the agent run appears here",
|
||||
),
|
||||
"gates": (
|
||||
"Ariadne policy gates",
|
||||
"Ariadne decides on its own reading of the evidence",
|
||||
"controls: response_check -> authorizer",
|
||||
"",
|
||||
),
|
||||
"route": (
|
||||
"Ariadne response",
|
||||
"Which edge off the route diamond is taken",
|
||||
"response: route",
|
||||
"",
|
||||
),
|
||||
"response": (
|
||||
"Ariadne response",
|
||||
"Ariadne executes, proposes, or escalates",
|
||||
"response: operational_path / code_path / human_path",
|
||||
"",
|
||||
),
|
||||
"verify": (
|
||||
"Ariadne response",
|
||||
"One rebuild decides whether the repair held",
|
||||
"response: operational_result -> validation build",
|
||||
f"{JENKINS}/job/{JOB}/ — a new build starts on its own",
|
||||
),
|
||||
"outputs": (
|
||||
"Inspectable outputs",
|
||||
"Incident closed; the artifacts remain",
|
||||
"outputs: audit_events, alert_output, issue_output",
|
||||
f"{GRAFANA}/d/atlas-testing — triage panels",
|
||||
),
|
||||
}
|
||||
ORDER = ["detect", "evidence", "hermes", "gates", "route", "response", "verify", "outputs"]
|
||||
|
||||
|
||||
def stamp() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%H:%M:%S")
|
||||
|
||||
|
||||
def fixture_state() -> str:
|
||||
return sh("kubectl", "-n", NS_DEMO, "get", "cm", "hermes-triage-demo-fixture",
|
||||
"-o", "jsonpath={.data.state}")
|
||||
def run(cmd: list[str], *, show: bool = True, limit: int = 12) -> str:
|
||||
"""Echo a command, run it, print its output, and return stdout."""
|
||||
|
||||
if show:
|
||||
print(f" {DIM}${RESET} {CYAN}{' '.join(cmd)}{RESET}")
|
||||
try:
|
||||
done = subprocess.run(cmd, capture_output=True, text=True, timeout=25)
|
||||
out = done.stdout.strip()
|
||||
except Exception as exc: # a failed read must never stop the narration
|
||||
print(f" {RED}(command failed: {exc}){RESET}")
|
||||
return ""
|
||||
if show:
|
||||
for line in (out.split("\n")[:limit] or ["(no output)"]):
|
||||
print(f" {line[:150]}")
|
||||
print()
|
||||
return out
|
||||
|
||||
|
||||
def ariadne_lines(tail: int = 400) -> list[dict]:
|
||||
"""Return recent Ariadne log records that parse as JSON."""
|
||||
def quiet(cmd: list[str]) -> str:
|
||||
return run(cmd, show=False)
|
||||
|
||||
raw = sh("kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne",
|
||||
f"--tail={tail}", timeout=25)
|
||||
|
||||
def banner(key: str) -> None:
|
||||
subgraph, meaning, path, ui = STAGES[key]
|
||||
print(f"\n{GREEN}{'─' * 70}{RESET}")
|
||||
print(f"{GREEN}{BOLD} {stamp()} {subgraph}{RESET}{GREEN} — {meaning}{RESET}")
|
||||
print(f"{DIM} chart: {path}{RESET}")
|
||||
if ui:
|
||||
print(f"{YELLOW} look at: {ui}{RESET}")
|
||||
print(f"{GREEN}{'─' * 70}{RESET}")
|
||||
|
||||
|
||||
def ariadne_records(tail: int = 400) -> list[dict]:
|
||||
raw = quiet(
|
||||
["kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne", f"--tail={tail}"]
|
||||
)
|
||||
records = []
|
||||
for line in raw.split("\n"):
|
||||
line = line.strip()
|
||||
@ -85,8 +150,6 @@ def ariadne_lines(tail: int = 400) -> list[dict]:
|
||||
|
||||
|
||||
def job_state(records: list[dict]) -> dict:
|
||||
"""Pull this job's latest auto-triage tick result out of the log."""
|
||||
|
||||
for record in reversed(records):
|
||||
if record.get("event") == "hermes_autotriage" and record.get("jobs"):
|
||||
try:
|
||||
@ -96,109 +159,102 @@ def job_state(records: list[dict]) -> dict:
|
||||
return {}
|
||||
|
||||
|
||||
def evidence_for(key: str) -> None:
|
||||
"""Run the reads that show this stage actually happened."""
|
||||
|
||||
if key == "detect":
|
||||
run(["kubectl", "-n", NS_DEMO, "get", "cm", "hermes-triage-demo-fixture",
|
||||
"-o", "jsonpath={.data.state}"])
|
||||
elif key == "evidence":
|
||||
print(f" {DIM}the bundle is what Ariadne sends Hermes; the audit trail keeps it{RESET}\n")
|
||||
run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
|
||||
"printenv", "ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST"], limit=2)
|
||||
elif key == "hermes":
|
||||
run(["kubectl", "-n", "hermes", "get", "pods", "-l", "app=hermes", "--no-headers"], limit=3)
|
||||
print(f" {DIM}Hermes holds no Git or Kubernetes write access; it returns JSON only{RESET}\n")
|
||||
elif key == "gates":
|
||||
for var in ("ARIADNE_HERMES_ALLOWED_ACTIONS", "ARIADNE_HERMES_MIN_CONFIDENCE"):
|
||||
run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
|
||||
"printenv", var], limit=2)
|
||||
elif key in {"response", "outputs"}:
|
||||
run(["kubectl", "-n", NS_DEMO, "get", "cm", "hermes-triage-demo-fixture",
|
||||
"-o", "jsonpath={.data.state}"])
|
||||
if key == "outputs":
|
||||
print(f" {YELLOW}issues filed by triage: {GITEA}/bstein/ariadne/issues{RESET}\n")
|
||||
elif key == "verify":
|
||||
print(f" {DIM}exactly one rebuild is triggered; it never retries in a loop{RESET}\n")
|
||||
|
||||
|
||||
class Monitor:
|
||||
"""Track which flow-chart stage the current incident has reached."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.done: dict[str, str] = {}
|
||||
self.done: set[str] = set()
|
||||
self.incident = ""
|
||||
self.last_line = ""
|
||||
self.summarised = False
|
||||
self.last_tick = ""
|
||||
|
||||
def mark(self, key: str, evidence: str) -> None:
|
||||
"""Record a stage as reached and print it once, with its evidence."""
|
||||
|
||||
if key in self.done:
|
||||
return
|
||||
self.done[key] = evidence
|
||||
_, subgraph, meaning = next(s for s in STAGES if s[0] == key)
|
||||
print(f"{GREEN}{stamp()} ✓ {BOLD}{subgraph}{RESET}{GREEN} — {meaning}{RESET}")
|
||||
print(f" {DIM}evidence:{RESET} {evidence}")
|
||||
sys.stdout.flush()
|
||||
self.done.add(key)
|
||||
banner(key)
|
||||
print(f" {BOLD}what happened:{RESET} {evidence}\n")
|
||||
evidence_for(key)
|
||||
|
||||
def checklist(self) -> None:
|
||||
"""Print the stage checklist as it currently stands."""
|
||||
|
||||
print(f"\n{BOLD} flow chart progress{RESET}")
|
||||
for key, subgraph, _meaning in STAGES:
|
||||
mark = f"{GREEN}✓{RESET}" if key in self.done else f"{DIM}·{RESET}"
|
||||
label = subgraph if key in self.done else f"{DIM}{subgraph}{RESET}"
|
||||
print(f" {mark} {key:<9} {label}")
|
||||
print()
|
||||
sys.stdout.flush()
|
||||
|
||||
def reset_for(self, incident: str) -> None:
|
||||
def new_incident(self, incident: str) -> None:
|
||||
if incident and incident != self.incident:
|
||||
self.incident = incident
|
||||
self.done.clear()
|
||||
self.summarised = False
|
||||
print(f"\n{BOLD}{YELLOW}{stamp()} ── incident {incident} ──{RESET}")
|
||||
sys.stdout.flush()
|
||||
print(f"\n{BOLD}{YELLOW}══ incident {incident} ══{RESET}")
|
||||
|
||||
def checklist(self) -> None:
|
||||
print(f"\n{BOLD} flow chart progress{RESET}")
|
||||
for key in ORDER:
|
||||
tick = f"{GREEN}✓{RESET}" if key in self.done else f"{DIM}·{RESET}"
|
||||
print(f" {tick} {key:<9} {STAGES[key][0]}")
|
||||
print()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
print(f"{BOLD}Watching {JOB}. Stages follow mermaid/TestAutomation.mmd.{RESET}")
|
||||
print(f"{DIM}Read-only. Ctrl-C to stop.{RESET}\n")
|
||||
print(f"{BOLD}Hermes triage monitor — following mermaid/TestAutomation.mmd{RESET}")
|
||||
print(f"{DIM}Every command is echoed before it runs. Read-only. Ctrl-C to stop.{RESET}")
|
||||
print(f"{YELLOW}Jenkins: {JENKINS}/job/{JOB}/{RESET}")
|
||||
print(f"{YELLOW}Gitea: {GITEA}/bstein{RESET}")
|
||||
print(f"{YELLOW}Grafana: {GRAFANA}/d/atlas-testing{RESET}\n")
|
||||
monitor = Monitor()
|
||||
while True:
|
||||
records = ariadne_lines()
|
||||
state = job_state(records)
|
||||
state = job_state(ariadne_records())
|
||||
status = str(state.get("status") or "")
|
||||
incident = str(state.get("incident_id") or "")
|
||||
monitor.reset_for(incident)
|
||||
|
||||
if incident and status in {"human_required", "awaiting_rebuild", "deduped", "failed"}:
|
||||
monitor.mark("detect", f"incident {incident} opened from a terminal build failure")
|
||||
|
||||
for record in records:
|
||||
if record.get("logger", "").endswith("hermes_autotriage") and record.get("status") == "ok":
|
||||
continue
|
||||
msg = str(record.get("message") or "")
|
||||
if "run_triage" in msg or "hermes run" in msg.lower():
|
||||
monitor.mark("hermes", msg[:110])
|
||||
if record.get("event") == "hermes_code_flow":
|
||||
monitor.mark("route", "code_path (optional source proposal) also taken")
|
||||
monitor.mark("response", f"code proposal: {record.get('status')} -> gitea_pr")
|
||||
monitor.new_incident(str(state.get("incident_id") or ""))
|
||||
|
||||
if status in {"human_required", "awaiting_rebuild", "deduped", "failed"}:
|
||||
monitor.mark("detect", f"incident {monitor.incident} opened from a terminal failure")
|
||||
if status == "awaiting_rebuild":
|
||||
monitor.mark("evidence", "bundle collected; console regions, tests and logs attached")
|
||||
monitor.mark("evidence", "bundle collected: console regions, tests, logs")
|
||||
monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema")
|
||||
monitor.mark("gates", "every gate passed; Ariadne authorized the predefined action")
|
||||
# The chart labels this edge "authorized action"; naming it lets a
|
||||
# viewer point at the branch being taken rather than infer it.
|
||||
monitor.mark("gates", "every gate passed; the predefined action was authorized")
|
||||
monitor.mark("route", "route --|authorized action|--> registry -> operational_result")
|
||||
monitor.mark(
|
||||
"response",
|
||||
f"{state.get('repair', 'action')} on {state.get('target', 'target')}",
|
||||
)
|
||||
monitor.mark("response", f"{state.get('repair', 'action')} on {state.get('target', '')}")
|
||||
monitor.mark("verify", "one rebuild triggered with seeding disabled")
|
||||
if status == "human_required":
|
||||
monitor.mark("evidence", "bundle collected")
|
||||
monitor.mark("hermes", "diagnosis returned")
|
||||
monitor.mark(
|
||||
"gates",
|
||||
f"Ariadne refused: {state.get('reason', 'human_required')} — no action taken",
|
||||
)
|
||||
monitor.mark("gates", f"Ariadne refused: {state.get('reason', '')} — nothing ran")
|
||||
monitor.mark("route", "route --|human required|--> human_required -> gitea_issue")
|
||||
monitor.mark("response", "escalated to a human; issue filed in the service repository")
|
||||
monitor.mark("response", "escalated; issue filed in the service repository")
|
||||
if status == "healthy" and state.get("resolved") and not monitor.summarised:
|
||||
monitor.mark("verify", "rebuild finished green")
|
||||
monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}")
|
||||
monitor.checklist()
|
||||
# The resolving tick stays newest until another incident opens, so
|
||||
# the completed checklist is printed once rather than every poll.
|
||||
# The resolving tick stays newest until another incident opens.
|
||||
monitor.summarised = True
|
||||
|
||||
line = f"{status}|{incident}|{fixture_state()}"
|
||||
if line != monitor.last_line:
|
||||
monitor.last_line = line
|
||||
fixture = fixture_state()
|
||||
colour = GREEN if fixture == "healthy" else RED
|
||||
print(
|
||||
f"{DIM}{stamp()}{RESET} tick: {status or 'idle':<16} "
|
||||
f"fixture={colour}{fixture or '?'}{RESET} {DIM}{incident}{RESET}"
|
||||
)
|
||||
sys.stdout.flush()
|
||||
if status and status != monitor.last_tick:
|
||||
monitor.last_tick = status
|
||||
print(f"{DIM}{stamp()} tick: {status}{RESET}")
|
||||
sys.stdout.flush()
|
||||
time.sleep(POLL_SECONDS)
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user