bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(demo): show the diagnosis and the policy result, in the chart's own words
Some checks failed
Tests / Declarative: Post Actions testing.tests.test_repo_structure.test_knowledge_service_mirror_matches_source failed
Details

Browse Source 
The Hermes stage proved only that a pod was running, when the substantive
evidence is the JSON Hermes returned; it now reads that from the audit trail,
so classification, confidence and requested action are on screen. The policy
gates stage never showed the verdict it was describing; authorized and
authorize_reason now appear with it.

The chart paths were paraphrases. They are now the node labels from
mermaid/TestAutomation.mmd verbatim - Response check, Scoped repair guard,
Action authorizer, Action registry, Scoped ConfigMap repair, Action result -
so a line on screen can be found on the diagram. The checklist carries them too.

The Optional source proposal branch was silently absent rather than explained.
The route stage now states it was not taken and why: a fixture repair is an
operational action, and source fixes are not actions at all.

Also widened the log window and its timeout; the fetch occasionally exceeded
25s and dropped a poll.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
jenkins 2026-08-06 18:14:13 -03:00
parent 39cc744d19
commit 6da0cba96d
1 changed files with 67 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
scripts/ops/hermes_triage_monitor.py
View File

@ -38,7 +38,7 @@ JENKINS = os.environ.get("JENKINS_URL", "https://ci.bstein.dev")
GITEA = os.environ.get("GITEA_URL", "https://scm.bstein.dev")
GRAFANA = os.environ.get("GRAFANA_URL", "https://metrics.bstein.dev")
HERMES_UI = os.environ.get("HERMES_URL", "https://agent.bstein.dev")
POLL_SECONDS = 4
POLL_SECONDS = 6
BOLD, DIM, GREEN, CYAN, YELLOW, RED, RESET = (
"\033[1m", "\033[2m", "\033[32m", "\033[36m", "\033[33m", "\033[31m", "\033[0m"
@ -49,49 +49,49 @@ STAGES: dict[str, tuple[str, str, str, str]] = {
"detect": (
"Detect and gather",
"A failed build becomes one incident",
"intake: detector -> evidence_sources",
"Jenkins detector -> Evidence sources",
f"{JENKINS}/job/{JOB}/ — the red build",
),
"evidence": (
"Detect and gather",
"Ariadne assembles the bounded evidence bundle",
"intake: collector -> bundle",
"Console reader -> Failure ranker -> Context filter -> Incident bundle",
"",
),
"hermes": (
"Hermes analysis",
"Hermes returns a recommendation it cannot act on",
"hermes_plane: skills -> recommendation",
"Triage skills -> Structured recommendation",
f"{HERMES_UI} — the agent run appears here",
),
"gates": (
"Ariadne policy gates",
"Ariadne decides on its own reading of the evidence",
"controls: response_check -> authorizer",
"Response check -> Scoped repair guard -> Action authorizer",
"",
),
"route": (
"Ariadne response",
"Which edge off the route diamond is taken",
"response: route",
"The policy result, and which branch it opens",
"Policy result -> Action registry",
"",
),
"response": (
"Ariadne response",
"Ariadne executes, proposes, or escalates",
"response: operational_path / code_path / human_path",
"Action registry -> Scoped ConfigMap repair -> Action result",
"",
),
"verify": (
"Ariadne response",
"One rebuild decides whether the repair held",
"response: operational_result -> validation build",
"Action result -> validation build",
f"{JENKINS}/job/{JOB}/ — a new build starts on its own",
),
"outputs": (
"Inspectable outputs",
"Incident closed; the artifacts remain",
"outputs: audit_events, alert_output, issue_output",
"Ariadne records every outcome -> Audit events, Triage metrics",
f"{GRAFANA}/d/atlas-testing — triage panels",
),
}
@ -108,7 +108,9 @@ def run(cmd: list[str], *, show: bool = True, limit: int = 12) -> str:
if show:
print(f" {DIM}${RESET} {CYAN}{' '.join(cmd)}{RESET}")
try:
done = subprocess.run(cmd, capture_output=True, text=True, timeout=25)
# Ariadne emits ~200 lines per tick, so a ten-minute window is a
# few thousand lines; the fetch needs room to finish.
done = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
out = done.stdout.strip()
except Exception as exc: # a failed read must never stop the narration
print(f" {RED}(command failed: {exc}){RESET}")
@ -134,7 +136,7 @@ def banner(key: str) -> None:
print(f"{GREEN}{'─' * 70}{RESET}")
def ariadne_records(since: str = "15m") -> list[dict]:
def ariadne_records(since: str = "10m") -> list[dict]:
"""Return recent Ariadne log records that parse as JSON.
Windowed by time, not line count. Ariadne emits roughly two hundred lines
@ -177,6 +179,28 @@ def job_states(records: list[dict]) -> list[dict]:
return states
_DIAG_QUERY = (
". /vault/secrets/ariadne-env.sh >/dev/null 2>&1; python3 -c \""
"import json,os,psycopg;"
"conn=psycopg.connect(os.environ['ARIADNE_DATABASE_URL']);"
"cur=conn.cursor();"
"cur.execute(\\\"select detail from ariadne_events where event_type='hermes_autotriage_diagnosis'"
" order by id desc limit 1\\\");"
"r=cur.fetchone()[0];"
"d=r if isinstance(r,dict) else json.loads(r);"
"print(json.dumps({'incident':d.get('incident_id'),'authorized':d.get('authorized'),"
"'authorize_reason':d.get('authorize_reason'),'evidence_marker':d.get('evidence_marker'),"
"'outcome':d.get('outcome')},indent=1))\""
)
def diagnosis_event() -> None:
"""Show the diagnosis Ariadne stored: what Hermes said, and the verdict."""
run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
"sh", "-c", _DIAG_QUERY], limit=16)
def evidence_for(key: str) -> None:
"""Run the reads that show this stage actually happened."""
@ -188,15 +212,28 @@ def evidence_for(key: str) -> None:
run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
"printenv", "ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST"], limit=2)
elif key == "hermes":
run(["kubectl", "-n", "hermes", "get", "pods", "-l", "app=hermes", "--no-headers"], limit=3)
print(f" {DIM}Hermes holds no Git or Kubernetes write access; it returns JSON only{RESET}\n")
print(f" {DIM}what Hermes actually returned, as Ariadne stored it:{RESET}")
diagnosis_event()
print(f" {DIM}Hermes holds no Git or Kubernetes write access; this JSON is its"
f" entire output{RESET}\n")
elif key == "gates":
for var in ("ARIADNE_HERMES_ALLOWED_ACTIONS", "ARIADNE_HERMES_MIN_CONFIDENCE"):
run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
"printenv", var], limit=2)
print(f" {DIM}authorized/authorize_reason above is the policy result: the verdict"
f" Ariadne reached on its own reading{RESET}")
run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
"printenv", "ARIADNE_HERMES_ALLOWED_ACTIONS"], limit=2)
run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
"printenv", "ARIADNE_HERMES_MIN_CONFIDENCE"], limit=2)
print(f" {DIM}only these two ids may ever be executed. Source fixes are not actions:"
f" they become pull requests and are never executed by Ariadne.{RESET}\n")
elif key == "route":
print(f" {DIM}the fixture repair is an operational action, so the Optional source"
f" proposal branch is not taken for this incident{RESET}\n")
elif key in {"response", "outputs"}:
run(["kubectl", "-n", NS_DEMO, "get", "cm", "hermes-triage-demo-fixture",
"-o", "jsonpath={.data.state}"])
if key == "response":
print(f" {DIM}Scoped ConfigMap repair on the chart: one merge patch to one"
f" ConfigMap, in process, no pod created{RESET}\n")
if key == "outputs":
print(f" {YELLOW}issues filed by triage: {GITEA}/bstein/ariadne/issues{RESET}\n")
elif key == "verify":
@ -231,7 +268,7 @@ class Monitor:
print(f"\n{BOLD} flow chart progress{RESET}")
for key in ORDER:
tick = f"{GREEN}✓{RESET}" if key in self.done else f"{DIM}·{RESET}"
print(f" {tick} {key:<9} {STAGES[key][0]}")
print(f" {tick} {key:<9} {STAGES[key][0]:<22} {DIM}{STAGES[key][2]}{RESET}")
print()
@ -255,14 +292,23 @@ def main() -> None:
monitor.mark("evidence", "bundle collected: console regions, tests, logs")
monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema")
monitor.mark("gates", "every gate passed; the predefined action was authorized")
monitor.mark("route", "route --|authorized action|--> registry -> operational_result")
monitor.mark("response", f"{state.get('repair', 'action')} on {state.get('target', '')}")
monitor.mark(
"route",
"policy result: authorized -> Action registry (Optional source proposal not taken)",
)
monitor.mark(
"response",
f"Scoped ConfigMap repair: {state.get('repair', 'action')} on {state.get('target', '')}",
)
monitor.mark("verify", "one rebuild triggered with seeding disabled")
if status == "human_required":
monitor.mark("evidence", "bundle collected")
monitor.mark("hermes", "diagnosis returned")
monitor.mark("gates", f"Ariadne refused: {state.get('reason', '')} — nothing ran")
monitor.mark("route", "route --|human required|--> human_required -> gitea_issue")
monitor.mark(
"route",
"policy result: refused -> Diagnosis and next checks -> Ariadne opens an issue",
)
monitor.mark("response", "escalated; issue filed in the service repository")
if status == "healthy" and state.get("resolved") and not monitor.summarised:
monitor.mark("verify", "rebuild finished green")