fix(demo): stop the monitor losing steps to poll timing
Some checks failed
Tests / Declarative: Post Actions testing.tests.test_repo_structure.test_knowledge_service_mirror_matches_source failed

It read only the newest tick from a --tail=400 window. Ariadne now emits
roughly two hundred lines per tick, so that window held about two ticks, and a
step that lasted a single tick - the repair, most importantly - vanished if a
poll landed after it. Build 30 was triaged, repaired and resolved correctly
while the monitor printed nothing at all.

The window is now time-based, and every tick inside it is replayed rather than
just the last. mark() was already idempotent, so replaying is safe and the
monitor recovers stages it was not running for.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
jenkins 2026-08-06 17:32:51 -03:00
parent 51338921b6
commit 39cc744d19

View File

@ -134,9 +134,16 @@ def banner(key: str) -> None:
print(f"{GREEN}{'' * 70}{RESET}") print(f"{GREEN}{'' * 70}{RESET}")
def ariadne_records(tail: int = 400) -> list[dict]: def ariadne_records(since: str = "15m") -> list[dict]:
"""Return recent Ariadne log records that parse as JSON.
Windowed by time, not line count. Ariadne emits roughly two hundred lines
per tick once it is polling every job, so a --tail window wide enough to
be useful is impossible to guess and a narrow one silently drops ticks.
"""
raw = quiet( raw = quiet(
["kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne", f"--tail={tail}"] ["kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne", f"--since={since}"]
) )
records = [] records = []
for line in raw.split("\n"): for line in raw.split("\n"):
@ -149,14 +156,25 @@ def ariadne_records(tail: int = 400) -> list[dict]:
return records return records
def job_state(records: list[dict]) -> dict: def job_states(records: list[dict]) -> list[dict]:
for record in reversed(records): """Return every tick result for this job in the window, oldest first.
The stages are reconstructed from all of them rather than from the newest
alone. A tick reporting the repair is replaced by the next tick within
seconds, so reading only the latest state means a poll landing at the
wrong moment loses that step permanently.
"""
states = []
for record in records:
if record.get("event") == "hermes_autotriage" and record.get("jobs"): if record.get("event") == "hermes_autotriage" and record.get("jobs"):
try: try:
return json.loads(record["jobs"]).get(JOB) or {} state = json.loads(record["jobs"]).get(JOB)
except ValueError: except ValueError:
return {} continue
return {} if state:
states.append(state)
return states
def evidence_for(key: str) -> None: def evidence_for(key: str) -> None:
@ -225,31 +243,33 @@ def main() -> None:
print(f"{YELLOW}Grafana: {GRAFANA}/d/atlas-testing{RESET}\n") print(f"{YELLOW}Grafana: {GRAFANA}/d/atlas-testing{RESET}\n")
monitor = Monitor() monitor = Monitor()
while True: while True:
state = job_state(ariadne_records()) states = job_states(ariadne_records())
status = str(state.get("status") or "") # Replay every tick in the window, so a step that lasted one tick is
monitor.new_incident(str(state.get("incident_id") or "")) # never lost to poll timing. mark() is idempotent.
for state in states:
if status in {"human_required", "awaiting_rebuild", "deduped", "failed"}: status = str(state.get("status") or "")
monitor.mark("detect", f"incident {monitor.incident} opened from a terminal failure") monitor.new_incident(str(state.get("incident_id") or ""))
if status == "awaiting_rebuild": if status in {"human_required", "awaiting_rebuild", "deduped", "failed"}:
monitor.mark("evidence", "bundle collected: console regions, tests, logs") monitor.mark("detect", f"incident {monitor.incident} opened from a terminal failure")
monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema") if status == "awaiting_rebuild":
monitor.mark("gates", "every gate passed; the predefined action was authorized") monitor.mark("evidence", "bundle collected: console regions, tests, logs")
monitor.mark("route", "route --|authorized action|--> registry -> operational_result") monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema")
monitor.mark("response", f"{state.get('repair', 'action')} on {state.get('target', '')}") monitor.mark("gates", "every gate passed; the predefined action was authorized")
monitor.mark("verify", "one rebuild triggered with seeding disabled") monitor.mark("route", "route --|authorized action|--> registry -> operational_result")
if status == "human_required": monitor.mark("response", f"{state.get('repair', 'action')} on {state.get('target', '')}")
monitor.mark("evidence", "bundle collected") monitor.mark("verify", "one rebuild triggered with seeding disabled")
monitor.mark("hermes", "diagnosis returned") if status == "human_required":
monitor.mark("gates", f"Ariadne refused: {state.get('reason', '')} — nothing ran") monitor.mark("evidence", "bundle collected")
monitor.mark("route", "route --|human required|--> human_required -> gitea_issue") monitor.mark("hermes", "diagnosis returned")
monitor.mark("response", "escalated; issue filed in the service repository") monitor.mark("gates", f"Ariadne refused: {state.get('reason', '')} — nothing ran")
if status == "healthy" and state.get("resolved") and not monitor.summarised: monitor.mark("route", "route --|human required|--> human_required -> gitea_issue")
monitor.mark("verify", "rebuild finished green") monitor.mark("response", "escalated; issue filed in the service repository")
monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}") if status == "healthy" and state.get("resolved") and not monitor.summarised:
monitor.checklist() monitor.mark("verify", "rebuild finished green")
# The resolving tick stays newest until another incident opens. monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}")
monitor.summarised = True monitor.checklist()
monitor.summarised = True
status = str(states[-1].get("status") or "") if states else ""
if status and status != monitor.last_tick: if status and status != monitor.last_tick:
monitor.last_tick = status monitor.last_tick = status