From 39cc744d193e695a1ab9e1139cc72ea9f77b69b1 Mon Sep 17 00:00:00 2001 From: jenkins Date: Thu, 6 Aug 2026 17:32:51 -0300 Subject: [PATCH] fix(demo): stop the monitor losing steps to poll timing It read only the newest tick from a --tail=400 window. Ariadne now emits roughly two hundred lines per tick, so that window held about two ticks, and a step that lasted a single tick - the repair, most importantly - vanished if a poll landed after it. Build 30 was triaged, repaired and resolved correctly while the monitor printed nothing at all. The window is now time-based, and every tick inside it is replayed rather than just the last. mark() was already idempotent, so replaying is safe and the monitor recovers stages it was not running for. Co-Authored-By: Claude Opus 5 --- scripts/ops/hermes_triage_monitor.py | 84 +++++++++++++++++----------- 1 file changed, 52 insertions(+), 32 deletions(-) diff --git a/scripts/ops/hermes_triage_monitor.py b/scripts/ops/hermes_triage_monitor.py index eda574833..c4c9a3bca 100755 --- a/scripts/ops/hermes_triage_monitor.py +++ b/scripts/ops/hermes_triage_monitor.py @@ -134,9 +134,16 @@ def banner(key: str) -> None: print(f"{GREEN}{'─' * 70}{RESET}") -def ariadne_records(tail: int = 400) -> list[dict]: +def ariadne_records(since: str = "15m") -> list[dict]: + """Return recent Ariadne log records that parse as JSON. + + Windowed by time, not line count. Ariadne emits roughly two hundred lines + per tick once it is polling every job, so a --tail window wide enough to + be useful is impossible to guess and a narrow one silently drops ticks. + """ + raw = quiet( - ["kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne", f"--tail={tail}"] + ["kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne", f"--since={since}"] ) records = [] for line in raw.split("\n"): @@ -149,14 +156,25 @@ def ariadne_records(tail: int = 400) -> list[dict]: return records -def job_state(records: list[dict]) -> dict: - for record in reversed(records): +def job_states(records: list[dict]) -> list[dict]: + """Return every tick result for this job in the window, oldest first. + + The stages are reconstructed from all of them rather than from the newest + alone. A tick reporting the repair is replaced by the next tick within + seconds, so reading only the latest state means a poll landing at the + wrong moment loses that step permanently. + """ + + states = [] + for record in records: if record.get("event") == "hermes_autotriage" and record.get("jobs"): try: - return json.loads(record["jobs"]).get(JOB) or {} + state = json.loads(record["jobs"]).get(JOB) except ValueError: - return {} - return {} + continue + if state: + states.append(state) + return states def evidence_for(key: str) -> None: @@ -225,31 +243,33 @@ def main() -> None: print(f"{YELLOW}Grafana: {GRAFANA}/d/atlas-testing{RESET}\n") monitor = Monitor() while True: - state = job_state(ariadne_records()) - status = str(state.get("status") or "") - monitor.new_incident(str(state.get("incident_id") or "")) - - if status in {"human_required", "awaiting_rebuild", "deduped", "failed"}: - monitor.mark("detect", f"incident {monitor.incident} opened from a terminal failure") - if status == "awaiting_rebuild": - monitor.mark("evidence", "bundle collected: console regions, tests, logs") - monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema") - monitor.mark("gates", "every gate passed; the predefined action was authorized") - monitor.mark("route", "route --|authorized action|--> registry -> operational_result") - monitor.mark("response", f"{state.get('repair', 'action')} on {state.get('target', '')}") - monitor.mark("verify", "one rebuild triggered with seeding disabled") - if status == "human_required": - monitor.mark("evidence", "bundle collected") - monitor.mark("hermes", "diagnosis returned") - monitor.mark("gates", f"Ariadne refused: {state.get('reason', '')} — nothing ran") - monitor.mark("route", "route --|human required|--> human_required -> gitea_issue") - monitor.mark("response", "escalated; issue filed in the service repository") - if status == "healthy" and state.get("resolved") and not monitor.summarised: - monitor.mark("verify", "rebuild finished green") - monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}") - monitor.checklist() - # The resolving tick stays newest until another incident opens. - monitor.summarised = True + states = job_states(ariadne_records()) + # Replay every tick in the window, so a step that lasted one tick is + # never lost to poll timing. mark() is idempotent. + for state in states: + status = str(state.get("status") or "") + monitor.new_incident(str(state.get("incident_id") or "")) + if status in {"human_required", "awaiting_rebuild", "deduped", "failed"}: + monitor.mark("detect", f"incident {monitor.incident} opened from a terminal failure") + if status == "awaiting_rebuild": + monitor.mark("evidence", "bundle collected: console regions, tests, logs") + monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema") + monitor.mark("gates", "every gate passed; the predefined action was authorized") + monitor.mark("route", "route --|authorized action|--> registry -> operational_result") + monitor.mark("response", f"{state.get('repair', 'action')} on {state.get('target', '')}") + monitor.mark("verify", "one rebuild triggered with seeding disabled") + if status == "human_required": + monitor.mark("evidence", "bundle collected") + monitor.mark("hermes", "diagnosis returned") + monitor.mark("gates", f"Ariadne refused: {state.get('reason', '')} — nothing ran") + monitor.mark("route", "route --|human required|--> human_required -> gitea_issue") + monitor.mark("response", "escalated; issue filed in the service repository") + if status == "healthy" and state.get("resolved") and not monitor.summarised: + monitor.mark("verify", "rebuild finished green") + monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}") + monitor.checklist() + monitor.summarised = True + status = str(states[-1].get("status") or "") if states else "" if status and status != monitor.last_tick: monitor.last_tick = status