bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(demo): stop the monitor losing steps to poll timing
Some checks failed
Tests / Declarative: Post Actions testing.tests.test_repo_structure.test_knowledge_service_mirror_matches_source failed
Details

Browse Source 
It read only the newest tick from a --tail=400 window. Ariadne now emits
roughly two hundred lines per tick, so that window held about two ticks, and a
step that lasted a single tick - the repair, most importantly - vanished if a
poll landed after it. Build 30 was triaged, repaired and resolved correctly
while the monitor printed nothing at all.

The window is now time-based, and every tick inside it is replayed rather than
just the last. mark() was already idempotent, so replaying is safe and the
monitor recovers stages it was not running for.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
jenkins 2026-08-06 17:32:51 -03:00
parent 51338921b6
commit 39cc744d19
1 changed files with 52 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
scripts/ops/hermes_triage_monitor.py
View File

@ -134,9 +134,16 @@ def banner(key: str) -> None:
print(f"{GREEN}{'─' * 70}{RESET}")
def ariadne_records(tail: int = 400) -> list[dict]:
def ariadne_records(since: str = "15m") -> list[dict]:
"""Return recent Ariadne log records that parse as JSON.
Windowed by time, not line count. Ariadne emits roughly two hundred lines
per tick once it is polling every job, so a --tail window wide enough to
be useful is impossible to guess and a narrow one silently drops ticks.
"""
raw = quiet(
["kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne", f"--tail={tail}"]
["kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne", f"--since={since}"]
)
records = []
for line in raw.split("\n"):
@ -149,14 +156,25 @@ def ariadne_records(tail: int = 400) -> list[dict]:
return records
def job_state(records: list[dict]) -> dict:
for record in reversed(records):
def job_states(records: list[dict]) -> list[dict]:
"""Return every tick result for this job in the window, oldest first.
The stages are reconstructed from all of them rather than from the newest
alone. A tick reporting the repair is replaced by the next tick within
seconds, so reading only the latest state means a poll landing at the
wrong moment loses that step permanently.
"""
states = []
for record in records:
if record.get("event") == "hermes_autotriage" and record.get("jobs"):
try:
return json.loads(record["jobs"]).get(JOB) or {}
state = json.loads(record["jobs"]).get(JOB)
except ValueError:
return {}
return {}
continue
if state:
states.append(state)
return states
def evidence_for(key: str) -> None:
@ -225,31 +243,33 @@ def main() -> None:
print(f"{YELLOW}Grafana: {GRAFANA}/d/atlas-testing{RESET}\n")
monitor = Monitor()
while True:
state = job_state(ariadne_records())
status = str(state.get("status") or "")
monitor.new_incident(str(state.get("incident_id") or ""))
if status in {"human_required", "awaiting_rebuild", "deduped", "failed"}:
monitor.mark("detect", f"incident {monitor.incident} opened from a terminal failure")
if status == "awaiting_rebuild":
monitor.mark("evidence", "bundle collected: console regions, tests, logs")
monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema")
monitor.mark("gates", "every gate passed; the predefined action was authorized")
monitor.mark("route", "route --|authorized action|--> registry -> operational_result")
monitor.mark("response", f"{state.get('repair', 'action')} on {state.get('target', '')}")
monitor.mark("verify", "one rebuild triggered with seeding disabled")
if status == "human_required":
monitor.mark("evidence", "bundle collected")
monitor.mark("hermes", "diagnosis returned")
monitor.mark("gates", f"Ariadne refused: {state.get('reason', '')} — nothing ran")
monitor.mark("route", "route --|human required|--> human_required -> gitea_issue")
monitor.mark("response", "escalated; issue filed in the service repository")
if status == "healthy" and state.get("resolved") and not monitor.summarised:
monitor.mark("verify", "rebuild finished green")
monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}")
monitor.checklist()
# The resolving tick stays newest until another incident opens.
monitor.summarised = True
states = job_states(ariadne_records())
# Replay every tick in the window, so a step that lasted one tick is
# never lost to poll timing. mark() is idempotent.
for state in states:
status = str(state.get("status") or "")
monitor.new_incident(str(state.get("incident_id") or ""))
if status in {"human_required", "awaiting_rebuild", "deduped", "failed"}:
monitor.mark("detect", f"incident {monitor.incident} opened from a terminal failure")
if status == "awaiting_rebuild":
monitor.mark("evidence", "bundle collected: console regions, tests, logs")
monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema")
monitor.mark("gates", "every gate passed; the predefined action was authorized")
monitor.mark("route", "route --|authorized action|--> registry -> operational_result")
monitor.mark("response", f"{state.get('repair', 'action')} on {state.get('target', '')}")
monitor.mark("verify", "one rebuild triggered with seeding disabled")
if status == "human_required":
monitor.mark("evidence", "bundle collected")
monitor.mark("hermes", "diagnosis returned")
monitor.mark("gates", f"Ariadne refused: {state.get('reason', '')} — nothing ran")
monitor.mark("route", "route --|human required|--> human_required -> gitea_issue")
monitor.mark("response", "escalated; issue filed in the service repository")
if status == "healthy" and state.get("resolved") and not monitor.summarised:
monitor.mark("verify", "rebuild finished green")
monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}")
monitor.checklist()
monitor.summarised = True
status = str(states[-1].get("status") or "") if states else ""
if status and status != monitor.last_tick:
monitor.last_tick = status