diff --git a/scripts/ops/hermes_triage_monitor.py b/scripts/ops/hermes_triage_monitor.py index c3ffe4538..d4e133313 100755 --- a/scripts/ops/hermes_triage_monitor.py +++ b/scripts/ops/hermes_triage_monitor.py @@ -190,6 +190,8 @@ _DIAG_QUERY = ( "d=r if isinstance(r,dict) else json.loads(r);" "print(json.dumps({'incident':d.get('incident_id'),'authorized':d.get('authorized')," "'authorize_reason':d.get('authorize_reason'),'evidence_marker':d.get('evidence_marker')," + "'run_id':(d.get('run') or {}).get('run_id')," + "'run_seconds':(d.get('run') or {}).get('duration_seconds')," "'outcome':d.get('outcome')},indent=1))\"" ) @@ -208,6 +210,43 @@ _HISTORY_QUERY = ( ) +_BUNDLE_QUERY = ( + "python3 -c \"" + "from ariadne.services import hermes_autotriage_evidence as ev;" + "lb={'number':__BUILD__,'result':'FAILURE','building':False,'timestamp':0,'duration':0,'url':''};" + "b=ev.collect_evidence('__INCIDENT__','__JOB__',lb);" + "j=b['jenkins']; regions=j.get('console_failures') or []; tests=j.get('failed_tests') or [];" + "recs=(b.get('log_evidence') or {}).get('records') or [];" + "print('jenkins.console_failures : %d region(s), truncated=%s' % (len(regions), j.get('console_truncated')));" + "[print(' | ' + l[:110]) for l in ((regions[0].get('text') or '').strip().split(chr(10))[-3:] if regions else [])];" + "print('jenkins.failed_tests : %d' % len(tests));" + "[print(' | %s :: %s' % (t.get('className'), t.get('name'))) for t in tests[:2]];" + "print('log_evidence.records : %d from OpenSearch kube-*' % len(recs));" + "[print(' | [%s] %s' % (r.get('namespace'), (r.get('message') or '')[:90])) for r in recs[:2]]\"" +) + + +def bundle_sample(incident: str) -> None: + """Show a trimmed sample of the bundle that was sent to Hermes. + + Rebuilt from the same collector Ariadne used. A finished build's console + does not change, so this is durable rather than a live reading, and it is + trimmed hard on purpose: the point is to show what kind of evidence each + source contributes, not to reprint the bundle. + """ + + if "/" not in incident: + return + job, _, build = incident.rpartition("/") + query = ( + _BUNDLE_QUERY.replace("__BUILD__", build) + .replace("__INCIDENT__", incident) + .replace("__JOB__", job) + ) + run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--", + "sh", "-c", ". /vault/secrets/ariadne-env.sh >/dev/null 2>&1; " + query], limit=14) + + def incident_history(incident: str) -> None: """Show this incident's recorded state changes, oldest first. @@ -238,11 +277,13 @@ def evidence_for(key: str, incident: str = "") -> None: f" true after the repair has run:{RESET}") incident_history(incident) elif key == "evidence": - print(f" {DIM}the bundle is what Ariadne sends Hermes; the audit trail keeps it{RESET}\n") - run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--", - "printenv", "ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST"], limit=2) + print(f" {DIM}a sample of what each source contributed to the bundle Hermes" + f" received:{RESET}") + bundle_sample(incident) elif key == "hermes": - print(f" {DIM}what Hermes actually returned, as Ariadne stored it:{RESET}") + print(f" {DIM}what Hermes actually returned, as Ariadne stored it. The run_id below" + f" identifies this run in the Hermes dashboard, where the prompt it was given" + f" and every tool call it made are visible:{RESET}") diagnosis_event() print(f" {DIM}Hermes holds no Git or Kubernetes write access; this JSON is its" f" entire output{RESET}\n")