diff --git a/scripts/ops/hermes_triage_monitor.py b/scripts/ops/hermes_triage_monitor.py index eda574833..c4c9a3bca 100755 --- a/scripts/ops/hermes_triage_monitor.py +++ b/scripts/ops/hermes_triage_monitor.py @@ -134,9 +134,16 @@ def banner(key: str) -> None: print(f"{GREEN}{'─' * 70}{RESET}") -def ariadne_records(tail: int = 400) -> list[dict]: +def ariadne_records(since: str = "15m") -> list[dict]: + """Return recent Ariadne log records that parse as JSON. + + Windowed by time, not line count. Ariadne emits roughly two hundred lines + per tick once it is polling every job, so a --tail window wide enough to + be useful is impossible to guess and a narrow one silently drops ticks. + """ + raw = quiet( - ["kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne", f"--tail={tail}"] + ["kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne", f"--since={since}"] ) records = [] for line in raw.split("\n"): @@ -149,14 +156,25 @@ def ariadne_records(tail: int = 400) -> list[dict]: return records -def job_state(records: list[dict]) -> dict: - for record in reversed(records): +def job_states(records: list[dict]) -> list[dict]: + """Return every tick result for this job in the window, oldest first. + + The stages are reconstructed from all of them rather than from the newest + alone. A tick reporting the repair is replaced by the next tick within + seconds, so reading only the latest state means a poll landing at the + wrong moment loses that step permanently. + """ + + states = [] + for record in records: if record.get("event") == "hermes_autotriage" and record.get("jobs"): try: - return json.loads(record["jobs"]).get(JOB) or {} + state = json.loads(record["jobs"]).get(JOB) except ValueError: - return {} - return {} + continue + if state: + states.append(state) + return states def evidence_for(key: str) -> None: @@ -225,31 +243,33 @@ def main() -> None: print(f"{YELLOW}Grafana: {GRAFANA}/d/atlas-testing{RESET}\n") monitor = Monitor() while True: - state = job_state(ariadne_records()) - status = str(state.get("status") or "") - monitor.new_incident(str(state.get("incident_id") or "")) - - if status in {"human_required", "awaiting_rebuild", "deduped", "failed"}: - monitor.mark("detect", f"incident {monitor.incident} opened from a terminal failure") - if status == "awaiting_rebuild": - monitor.mark("evidence", "bundle collected: console regions, tests, logs") - monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema") - monitor.mark("gates", "every gate passed; the predefined action was authorized") - monitor.mark("route", "route --|authorized action|--> registry -> operational_result") - monitor.mark("response", f"{state.get('repair', 'action')} on {state.get('target', '')}") - monitor.mark("verify", "one rebuild triggered with seeding disabled") - if status == "human_required": - monitor.mark("evidence", "bundle collected") - monitor.mark("hermes", "diagnosis returned") - monitor.mark("gates", f"Ariadne refused: {state.get('reason', '')} — nothing ran") - monitor.mark("route", "route --|human required|--> human_required -> gitea_issue") - monitor.mark("response", "escalated; issue filed in the service repository") - if status == "healthy" and state.get("resolved") and not monitor.summarised: - monitor.mark("verify", "rebuild finished green") - monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}") - monitor.checklist() - # The resolving tick stays newest until another incident opens. - monitor.summarised = True + states = job_states(ariadne_records()) + # Replay every tick in the window, so a step that lasted one tick is + # never lost to poll timing. mark() is idempotent. + for state in states: + status = str(state.get("status") or "") + monitor.new_incident(str(state.get("incident_id") or "")) + if status in {"human_required", "awaiting_rebuild", "deduped", "failed"}: + monitor.mark("detect", f"incident {monitor.incident} opened from a terminal failure") + if status == "awaiting_rebuild": + monitor.mark("evidence", "bundle collected: console regions, tests, logs") + monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema") + monitor.mark("gates", "every gate passed; the predefined action was authorized") + monitor.mark("route", "route --|authorized action|--> registry -> operational_result") + monitor.mark("response", f"{state.get('repair', 'action')} on {state.get('target', '')}") + monitor.mark("verify", "one rebuild triggered with seeding disabled") + if status == "human_required": + monitor.mark("evidence", "bundle collected") + monitor.mark("hermes", "diagnosis returned") + monitor.mark("gates", f"Ariadne refused: {state.get('reason', '')} — nothing ran") + monitor.mark("route", "route --|human required|--> human_required -> gitea_issue") + monitor.mark("response", "escalated; issue filed in the service repository") + if status == "healthy" and state.get("resolved") and not monitor.summarised: + monitor.mark("verify", "rebuild finished green") + monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}") + monitor.checklist() + monitor.summarised = True + status = str(states[-1].get("status") or "") if states else "" if status and status != monitor.last_tick: monitor.last_tick = status