feat(demo): add reset and monitor commands
Some checks failed
Tests / Declarative: Post Actions testing.tests.test_repo_structure.test_knowledge_service_mirror_matches_source failed
Some checks failed
Tests / Declarative: Post Actions testing.tests.test_repo_structure.test_knowledge_service_mirror_matches_source failed
reset restores only what the demo itself creates: the fixture ConfigMap, the demo repository's open repair pull requests and their branches. Real service repositories are deliberately untouched - those issues are genuine triage records, and deleting them to tidy a demo would destroy the evidence the system exists to produce. monitor streams the flow in the stages of mermaid/TestAutomation.mmd, so a second terminal can be followed against the chart: Detect and gather, Hermes analysis, Ariadne policy gates, Ariadne response, Inspectable outputs. Each stage prints the evidence that moved it, since the claim the chart makes is that every step is answerable from data rather than asserted. Read-only. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
f96fb30522
commit
93ff096fa1
@ -5,6 +5,8 @@
|
||||
# hermes_triage_demo.sh code # proposal loop: fail -> Hermes patch -> PR
|
||||
# hermes_triage_demo.sh status # current incident/alert state, no changes
|
||||
# hermes_triage_demo.sh preflight # confirm the lab is ready to demo
|
||||
# hermes_triage_demo.sh reset # restore the demo to its pre-run state
|
||||
# hermes_triage_demo.sh monitor # stream the flow chart stages live
|
||||
#
|
||||
# FIRST RUN: copy hermes_triage_demo.env.example to hermes_triage_demo.env in
|
||||
# this directory and fill it in. That file is git-ignored precisely so it can
|
||||
@ -76,6 +78,65 @@ for line in sys.stdin:
|
||||
print(" ", d["timestamp"][11:19], d.get("jobs"))' | tail -"${2:-5}"
|
||||
}
|
||||
|
||||
cmd_monitor() {
|
||||
exec python3 "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_triage_monitor.py"
|
||||
}
|
||||
|
||||
# Restores only what the demo itself creates. Real service repositories are
|
||||
# never touched: those issues are genuine triage records and deleting them to
|
||||
# tidy a demo would destroy the evidence the system exists to produce.
|
||||
cmd_reset() {
|
||||
require_jenkins
|
||||
say "Reset — restoring the demo to its pre-run state"
|
||||
|
||||
note "fixture -> healthy"
|
||||
kubectl -n "$DEMO_NS" patch cm hermes-triage-demo-fixture \
|
||||
--type merge -p '{"data":{"state":"healthy"}}' >/dev/null 2>&1 &&
|
||||
note " fixture: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')" ||
|
||||
note " fixture patch failed (is the demo namespace present?)"
|
||||
|
||||
local gitea="${GITEA_URL:-https://scm.bstein.dev}"
|
||||
if [ -z "${GITEA_TOKEN:-}" ]; then
|
||||
note "GITEA_TOKEN unset; skipping pull request and branch cleanup"
|
||||
else
|
||||
note "closing open repair pull requests on hermes-code-demo"
|
||||
local prs
|
||||
prs="$(curl -s -H "Authorization: token $GITEA_TOKEN" \
|
||||
"$gitea/api/v1/repos/bstein/hermes-code-demo/pulls?state=open" |
|
||||
python3 -c 'import json,sys
|
||||
for p in json.load(sys.stdin):
|
||||
print(p["number"], p["head"]["ref"])' 2>/dev/null || true)"
|
||||
if [ -z "$prs" ]; then
|
||||
note " none open"
|
||||
else
|
||||
while read -r num ref; do
|
||||
[ -z "$num" ] && continue
|
||||
curl -s -o /dev/null -X PATCH -H "Authorization: token $GITEA_TOKEN" \
|
||||
-H "Content-Type: application/json" -d '{"state":"closed"}' \
|
||||
"$gitea/api/v1/repos/bstein/hermes-code-demo/pulls/$num"
|
||||
curl -s -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \
|
||||
"$gitea/api/v1/repos/bstein/hermes-code-demo/branches/$ref"
|
||||
note " closed #$num and deleted branch $ref"
|
||||
done <<< "$prs"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -d "$CODE_REPO_DIR/.git" ]; then
|
||||
note "restoring the demo repository working state"
|
||||
( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q --ff-only 2>/dev/null || true )
|
||||
if grep -q 'percent / 100' "$CODE_REPO_DIR/src/discount.py" 2>/dev/null; then
|
||||
note " src/discount.py is correct; demo is armable"
|
||||
else
|
||||
note " src/discount.py still carries the seeded defect — merge or revert it before demoing"
|
||||
fi
|
||||
else
|
||||
note "demo repository not cloned at $CODE_REPO_DIR; skipping"
|
||||
fi
|
||||
|
||||
say "Ready"
|
||||
note "run 'preflight' next, then arm the fixture build"
|
||||
}
|
||||
|
||||
cmd_preflight() {
|
||||
require_jenkins
|
||||
say "Preflight"
|
||||
@ -200,5 +261,7 @@ case "${1:-}" in
|
||||
code) cmd_code ;;
|
||||
status) cmd_status ;;
|
||||
preflight) cmd_preflight ;;
|
||||
reset) cmd_reset ;;
|
||||
monitor) cmd_monitor ;;
|
||||
*) sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;;
|
||||
esac
|
||||
|
||||
192
scripts/ops/hermes_triage_monitor.py
Executable file
192
scripts/ops/hermes_triage_monitor.py
Executable file
@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Stream the triage flow as it happens, in the stages of the flow chart.
|
||||
|
||||
Run in a second terminal alongside the demo. Each stage below is a subgraph in
|
||||
`mermaid/TestAutomation.mmd`, so what prints here can be followed on the chart:
|
||||
|
||||
Detect and gather -> Hermes analysis -> Ariadne policy gates
|
||||
-> Ariadne response -> Inspectable outputs
|
||||
|
||||
Every stage prints the evidence that moved it, because the point of the chart
|
||||
is that each step is answerable from data rather than asserted.
|
||||
|
||||
Read-only: it polls Kubernetes, Jenkins and Gitea and changes nothing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
JOB = os.environ.get("MONITOR_JOB", "hermes-triage-demo")
|
||||
NS_DEMO = "hermes-triage-demo"
|
||||
NS_ARIADNE = "maintenance"
|
||||
POLL_SECONDS = 4
|
||||
|
||||
STAGES = [
|
||||
("detect", "Detect and gather", "Jenkins failure detector opens an incident"),
|
||||
("evidence", "Detect and gather", "Ariadne collects the bounded evidence bundle"),
|
||||
("hermes", "Hermes analysis", "Hermes returns a classification it cannot act on"),
|
||||
("gates", "Ariadne policy gates", "Ariadne authorizes or refuses, on its own reading"),
|
||||
("response", "Ariadne response", "Ariadne executes, proposes, or escalates"),
|
||||
("verify", "Ariadne response", "One rebuild decides whether the repair worked"),
|
||||
("outputs", "Inspectable outputs", "Incident closed; artifacts left behind"),
|
||||
]
|
||||
|
||||
BOLD, DIM, GREEN, YELLOW, RED, RESET = (
|
||||
"\033[1m", "\033[2m", "\033[32m", "\033[33m", "\033[31m", "\033[0m"
|
||||
)
|
||||
|
||||
|
||||
def sh(*args: str, timeout: int = 20) -> str:
|
||||
"""Run a command and return stdout, or "" on any failure."""
|
||||
|
||||
try:
|
||||
out = subprocess.run(args, capture_output=True, text=True, timeout=timeout)
|
||||
return out.stdout.strip()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def stamp() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%H:%M:%S")
|
||||
|
||||
|
||||
def fixture_state() -> str:
|
||||
return sh("kubectl", "-n", NS_DEMO, "get", "cm", "hermes-triage-demo-fixture",
|
||||
"-o", "jsonpath={.data.state}")
|
||||
|
||||
|
||||
def ariadne_lines(tail: int = 400) -> list[dict]:
|
||||
"""Return recent Ariadne log records that parse as JSON."""
|
||||
|
||||
raw = sh("kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne",
|
||||
f"--tail={tail}", timeout=25)
|
||||
records = []
|
||||
for line in raw.split("\n"):
|
||||
line = line.strip()
|
||||
if line.startswith("{"):
|
||||
try:
|
||||
records.append(json.loads(line))
|
||||
except ValueError:
|
||||
continue
|
||||
return records
|
||||
|
||||
|
||||
def job_state(records: list[dict]) -> dict:
|
||||
"""Pull this job's latest auto-triage tick result out of the log."""
|
||||
|
||||
for record in reversed(records):
|
||||
if record.get("event") == "hermes_autotriage" and record.get("jobs"):
|
||||
try:
|
||||
return json.loads(record["jobs"]).get(JOB) or {}
|
||||
except ValueError:
|
||||
return {}
|
||||
return {}
|
||||
|
||||
|
||||
class Monitor:
|
||||
"""Track which flow-chart stage the current incident has reached."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.done: dict[str, str] = {}
|
||||
self.incident = ""
|
||||
self.last_line = ""
|
||||
|
||||
def mark(self, key: str, evidence: str) -> None:
|
||||
"""Record a stage as reached and print it once, with its evidence."""
|
||||
|
||||
if key in self.done:
|
||||
return
|
||||
self.done[key] = evidence
|
||||
_, subgraph, meaning = next(s for s in STAGES if s[0] == key)
|
||||
print(f"{GREEN}{stamp()} ✓ {BOLD}{subgraph}{RESET}{GREEN} — {meaning}{RESET}")
|
||||
print(f" {DIM}evidence:{RESET} {evidence}")
|
||||
sys.stdout.flush()
|
||||
|
||||
def checklist(self) -> None:
|
||||
"""Print the stage checklist as it currently stands."""
|
||||
|
||||
print(f"\n{BOLD} flow chart progress{RESET}")
|
||||
for key, subgraph, _meaning in STAGES:
|
||||
mark = f"{GREEN}✓{RESET}" if key in self.done else f"{DIM}·{RESET}"
|
||||
label = subgraph if key in self.done else f"{DIM}{subgraph}{RESET}"
|
||||
print(f" {mark} {key:<9} {label}")
|
||||
print()
|
||||
sys.stdout.flush()
|
||||
|
||||
def reset_for(self, incident: str) -> None:
|
||||
if incident and incident != self.incident:
|
||||
self.incident = incident
|
||||
self.done.clear()
|
||||
print(f"\n{BOLD}{YELLOW}{stamp()} ── incident {incident} ──{RESET}")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
print(f"{BOLD}Watching {JOB}. Stages follow mermaid/TestAutomation.mmd.{RESET}")
|
||||
print(f"{DIM}Read-only. Ctrl-C to stop.{RESET}\n")
|
||||
monitor = Monitor()
|
||||
while True:
|
||||
records = ariadne_lines()
|
||||
state = job_state(records)
|
||||
status = str(state.get("status") or "")
|
||||
incident = str(state.get("incident_id") or "")
|
||||
monitor.reset_for(incident)
|
||||
|
||||
if incident and status in {"human_required", "awaiting_rebuild", "deduped", "failed"}:
|
||||
monitor.mark("detect", f"incident {incident} opened from a terminal build failure")
|
||||
|
||||
for record in records:
|
||||
if record.get("logger", "").endswith("hermes_autotriage") and record.get("status") == "ok":
|
||||
continue
|
||||
msg = str(record.get("message") or "")
|
||||
if "run_triage" in msg or "hermes run" in msg.lower():
|
||||
monitor.mark("hermes", msg[:110])
|
||||
if record.get("event") == "hermes_code_flow":
|
||||
monitor.mark("response", f"code proposal: {record.get('status')}")
|
||||
|
||||
if status == "awaiting_rebuild":
|
||||
monitor.mark("evidence", "bundle collected; console regions, tests and logs attached")
|
||||
monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema")
|
||||
monitor.mark("gates", "every gate passed; Ariadne authorized the predefined action")
|
||||
monitor.mark(
|
||||
"response",
|
||||
f"{state.get('repair', 'action')} on {state.get('target', 'target')}",
|
||||
)
|
||||
monitor.mark("verify", "one rebuild triggered with seeding disabled")
|
||||
if status == "human_required":
|
||||
monitor.mark("evidence", "bundle collected")
|
||||
monitor.mark("hermes", "diagnosis returned")
|
||||
monitor.mark(
|
||||
"gates",
|
||||
f"Ariadne refused: {state.get('reason', 'human_required')} — no action taken",
|
||||
)
|
||||
monitor.mark("response", "escalated to a human; issue filed in the service repository")
|
||||
if status == "healthy" and state.get("resolved"):
|
||||
monitor.mark("verify", "rebuild finished green")
|
||||
monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}")
|
||||
monitor.checklist()
|
||||
|
||||
line = f"{status}|{incident}|{fixture_state()}"
|
||||
if line != monitor.last_line:
|
||||
monitor.last_line = line
|
||||
fixture = fixture_state()
|
||||
colour = GREEN if fixture == "healthy" else RED
|
||||
print(
|
||||
f"{DIM}{stamp()}{RESET} tick: {status or 'idle':<16} "
|
||||
f"fixture={colour}{fixture or '?'}{RESET} {DIM}{incident}{RESET}"
|
||||
)
|
||||
sys.stdout.flush()
|
||||
time.sleep(POLL_SECONDS)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
print("\nstopped")
|
||||
Loading…
x
Reference in New Issue
Block a user