bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(demo): follow one incident, and never show another service's run

Browse Source 
Two bugs, both visible in a single demo transcript.

The stage memory was one shared set cleared whenever the incident changed. A
ten-minute window routinely holds two incidents - the build just pushed and
the one before it - so they wiped each other's progress and every stage
reprinted on every poll, forever. Memory is now per incident, and the monitor
follows one incident at a time: --incident pins an exact id, --filter matches
a substring, and otherwise the newest wins, which is what someone who just
triggered a build wants.

Worse: the diagnosis panel read the newest diagnosis in the whole table, not
the one for the incident on screen. During a code-demo run it displayed
ananke/249 - a different service, a different failure - directly beneath the
heading naming this incident. Showing an unrelated answer is worse than
showing nothing, because nothing about it looks wrong.

It is now scoped to the incident and reads both event types, since the code
path records a code_proposal rather than a diagnosis - which is why nothing
matched and the fallback took over. There is no fallback any more: an incident
with no recorded run says so.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
jenkins 2026-08-07 04:42:46 -03:00
parent 5090cd85b5
commit 62697bbb9e
1 changed files with 93 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
scripts/ops/hermes_triage_monitor.py
View File

@ -42,6 +42,25 @@ GRAFANA = os.environ.get("GRAFANA_URL", "https://metrics.bstein.dev")
HERMES_UI = os.environ.get("HERMES_URL", "https://agent.bstein.dev")
POLL_SECONDS = 6
# `--filter <text>` restricts the monitor to incidents whose id contains that
# text; `--incident <id>` pins it to exactly one. Without either, it follows
# the newest incident it can see for this job, which is what someone who just
# triggered a build actually wants.
def _argv_option(name: str) -> str:
"""Read `--name value` or `--name=value` from argv, or "" when absent."""
prefix = f"--{name}="
for index, arg in enumerate(sys.argv[1:]):
if arg.startswith(prefix):
return arg[len(prefix) :].strip()
if arg == f"--{name}" and index + 2 <= len(sys.argv[1:]):
return sys.argv[index + 2].strip()
return ""
FILTER = _argv_option("filter")
PIN = _argv_option("incident")
# The fixture job repairs a ConfigMap; the code job proposes a patch. Evidence
# that suits one is false for the other, so the stages branch on it.
IS_CODE_JOB = JOB == "hermes-code-demo"
@ -183,7 +202,31 @@ def job_states(records: list[dict]) -> list[dict]:
continue
if state:
states.append(state)
return states
return _one_incident(states)
def _one_incident(states: list[dict]) -> list[dict]:
"""Narrow a window's ticks to the single incident worth narrating.
A ten-minute window routinely holds two incidents - the build just pushed
and the one before it - and interleaving them produces a transcript that
reads as though the system is doing everything twice. Following one at a
time is both clearer and closer to the truth: the diagram describes the
life of one incident.
`--incident` pins an exact id, `--filter` matches a substring, and
otherwise the newest incident in the window wins.
"""
if PIN:
return [s for s in states if str(s.get("incident_id") or "") == PIN]
if FILTER:
states = [s for s in states if FILTER in str(s.get("incident_id") or "")]
ids = [str(s.get("incident_id") or "") for s in states if s.get("incident_id")]
if not ids:
return states
newest = ids[-1]
return [s for s in states if str(s.get("incident_id") or "") == newest]
_DIAG_QUERY = (
@ -191,15 +234,20 @@ _DIAG_QUERY = (
"import json,os,psycopg;"
"conn=psycopg.connect(os.environ['ARIADNE_DATABASE_URL']);"
"cur=conn.cursor();"
"cur.execute(\\\"select detail from ariadne_events where event_type='hermes_autotriage_diagnosis'"
" order by id desc limit 1\\\");"
"r=cur.fetchone()[0];"
"d=r if isinstance(r,dict) else json.loads(r);"
"print(json.dumps({'incident':d.get('incident_id'),'authorized':d.get('authorized'),"
"'authorize_reason':d.get('authorize_reason'),'evidence_marker':d.get('evidence_marker'),"
"'run_id':(d.get('run') or {}).get('run_id'),"
"'run_seconds':(d.get('run') or {}).get('duration_seconds'),"
"'outcome':d.get('outcome')},indent=1))\""
"cur.execute(\\\"select event_type,detail from ariadne_events where event_type in"
" ('hermes_autotriage_diagnosis','hermes_autotriage_code_proposal')"
" order by id desc limit 200\\\");"
"rows=[(t, d if isinstance(d,dict) else json.loads(d)) for t,d in cur.fetchall()];"
"rows=[(t,d) for t,d in rows if d.get('incident_id')=='__INCIDENT__'];"
"print(json.dumps({'note':'no Hermes run recorded for this incident yet'},indent=1))"
" if not rows else None;"
"t,d=(rows[0] if rows else ('',{}));"
"print(json.dumps({'event':t,'incident':d.get('incident_id'),"
"'authorized':d.get('authorized'),'authorize_reason':d.get('authorize_reason'),"
"'validated':d.get('validated'),'reject_reason':d.get('reject_reason'),"
"'chosen_path':d.get('chosen_path'),'url':d.get('url'),"
"'run_id':d.get('run_id') or (d.get('run') or {}).get('run_id'),"
"'outcome':d.get('outcome')},indent=1)) if rows else None\""
)
@ -269,11 +317,17 @@ def incident_history(incident: str) -> None:
"sh", "-c", _HISTORY_QUERY.replace("__INCIDENT__", incident)], limit=10)
def diagnosis_event() -> str:
"""Show the diagnosis Ariadne stored, and return its Hermes run id."""
def diagnosis_event(incident: str) -> str:
"""Show the diagnosis Ariadne stored for one incident, and return its run id.
Scoped to the incident on purpose. Reading the newest diagnosis in the
table meant an unrelated service's run could appear in the middle of this
incident's narration - which is worse than showing nothing, because it
looks like the answer to the question on screen.
"""
out = run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
"sh", "-c", _DIAG_QUERY], limit=26)
"sh", "-c", _DIAG_QUERY.replace("__INCIDENT__", incident)], limit=26)
match = re.search(r'"run_id":\s*"([^"]+)"', out or "")
return match.group(1) if match else ""
@ -307,7 +361,7 @@ def evidence_for(key: str, incident: str = "") -> None:
bundle_sample(incident)
elif key == "hermes":
print(f" {DIM}what Hermes actually returned, as Ariadne stored it:{RESET}")
run_id = diagnosis_event()
run_id = diagnosis_event(incident)
hermes_run_link(run_id)
print(f" {DIM}Hermes holds no Git or Kubernetes write access; this JSON is its"
f" entire output. When outcome.suggested_remediation is populated, Hermes found"
@ -371,30 +425,36 @@ class Monitor:
"""Track which Test Automation Diagram stage the current incident has reached."""
def __init__(self) -> None:
self.done: set[str] = set()
# Keyed by incident. A single shared set was cleared whenever the
# incident changed, so two live incidents in the same window wiped each
# other's progress and reprinted every stage on every poll.
self.done: dict[str, set[str]] = {}
self.incident = ""
self.summarised = False
self.summarised: set[str] = set()
self.last_tick = ""
def mark(self, key: str, evidence: str) -> None:
if key in self.done:
seen = self.done.setdefault(self.incident, set())
if key in seen:
return
self.done.add(key)
seen.add(key)
banner(key)
print(f" {BOLD}what happened:{RESET} {evidence}\n")
evidence_for(key, self.incident)
def new_incident(self, incident: str) -> None:
if incident and incident != self.incident:
self.incident = incident
self.done.clear()
self.summarised = False
if not incident or incident == self.incident:
return
self.incident = incident
if incident not in self.done:
self.done[incident] = set()
print(f"\n{BOLD}{YELLOW}══ incident {incident} ══{RESET}")
def checklist(self) -> None:
print(f"\n{BOLD} Test Automation Diagram progress{RESET}")
seen = self.done.get(self.incident, set())
for key in ORDER:
tick = f"{GREEN}✓{RESET}" if key in self.done else f"{DIM}·{RESET}"
tick = f"{GREEN}✓{RESET}" if key in seen else f"{DIM}·{RESET}"
print(f" {tick} {key:<9} {STAGES[key][0]:<22} {DIM}{STAGES[key][2]}{RESET}")
print()
@ -405,7 +465,14 @@ def main() -> None:
print(f"{DIM}Every command is echoed before it runs. Read-only. Ctrl-C to stop.{RESET}")
print(f"{YELLOW}Jenkins: {JENKINS}/job/{JOB}/{RESET}")
print(f"{YELLOW}Gitea: {GITEA}/bstein{RESET}")
print(f"{YELLOW}Grafana: {GRAFANA}/d/atlas-testing{RESET}\n")
print(f"{YELLOW}Grafana: {GRAFANA}/d/atlas-testing{RESET}")
if PIN:
print(f"{DIM}following incident {PIN} only{RESET}\n")
elif FILTER:
print(f"{DIM}following incidents matching {FILTER!r}{RESET}\n")
else:
print(f"{DIM}following the newest incident for {JOB};"
f" pass --filter TEXT or --incident ID to pin one{RESET}\n")
monitor = Monitor()
while True:
states = job_states(ariadne_records())
@ -464,11 +531,11 @@ def main() -> None:
"the branch build validates the proposal; the incident stays"
" human-required either way",
)
if status == "healthy" and state.get("resolved") and not monitor.summarised:
if status == "healthy" and state.get("resolved") and monitor.incident not in monitor.summarised:
monitor.mark("verify", "rebuild finished green")
monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}")
monitor.checklist()
monitor.summarised = True
monitor.summarised.add(monitor.incident)
status = str(states[-1].get("status") or "") if states else ""
if status and status != monitor.last_tick: