feat(hermes-autotriage): restart-safe incident gauge rehydration

Gauges are process-local, so a pod restart could silently drop an active
human_required signal before the vmalert hold window elapsed. Metrics move
to hermes_autotriage_metrics; every tick republishes gauges from stored
incident state, and an incident superseded by a newer successful build is
zeroed so the alert clears once the job is green again.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

ariadne/services/hermes_autotriage.py

@@ -5,13 +5,21 @@ import time
 from typing import Any
 
 import httpx
-from prometheus_client import Counter, Gauge
 
 from ..settings import settings
 from ..utils.logging import get_logger
 from . import hermes_agent_client, hermes_autotriage_repair
 from . import hermes_autotriage_decision as hermes_decision
 from . import hermes_autotriage_evidence as hermes_evidence
+from .hermes_autotriage_metrics import (
+    HERMES_TRIAGE_ACTION_TOTAL,
+    HERMES_TRIAGE_DURATION_SECONDS,
+    HERMES_TRIAGE_INCIDENT,
+    HERMES_TRIAGE_LAST_SUCCESS_TS,
+    INCIDENT_STATUSES,
+    refresh_incident_gauges,
+    set_incident_gauge,
+)
 
 
 logger = get_logger(__name__)
@@ -19,15 +27,6 @@ logger = get_logger(__name__)
 INCIDENT_EVENT_TYPE = "hermes_autotriage_incident"
 DIAGNOSIS_EVENT_TYPE = "hermes_autotriage_diagnosis"
 ACTION_EVENT_TYPE = "hermes_autotriage_action"
-INCIDENT_STATUSES = (
-    "detected",
-    "diagnosed",
-    "repairing",
-    "awaiting_rebuild",
-    "resolved",
-    "human_required",
-    "failed",
-)
 EXPECTED_CLASSIFICATION = "known_demo_fixture_failure"
 REBUILD_FAILED_REASON = "repair rebuild failed"
 
@@ -52,27 +51,6 @@ Do not perform mutations.
 Bundle:
 __BUNDLE__"""
 
-HERMES_TRIAGE_INCIDENT = Gauge(
-    "ariadne_hermes_triage_incident",
-    "Hermes auto-triage incident state (1=current status, 0=other statuses)",
-    ["jenkins_job", "build", "status"],
-)
-HERMES_TRIAGE_ACTION_TOTAL = Counter(
-    "ariadne_hermes_triage_action_total",
-    "Hermes auto-triage remediation actions by result",
-    ["action", "result"],
-)
-HERMES_TRIAGE_LAST_SUCCESS_TS = Gauge(
-    "ariadne_hermes_triage_last_success_timestamp_seconds",
-    "Last Hermes auto-triage incident resolution timestamp",
-)
-HERMES_TRIAGE_DURATION_SECONDS = Gauge(
-    "ariadne_hermes_triage_duration_seconds",
-    "Duration of the latest Hermes auto-triage phase in seconds",
-    ["phase"],
-)
-
-
 def run_hermes_autotriage(storage: Any) -> dict[str, Any]:
     """Run one Hermes auto-triage tick over the allowlisted Jenkins jobs.
 
@@ -130,6 +108,7 @@ def _process_job(storage: Any, job: str, incidents: dict[str, dict[str, Any]]) -
     last_build = _fetch_last_build(job)
     if last_build is None or last_build.get("building") or last_build.get("number") is None:
         return {"status": "skipped"}
+    refresh_incident_gauges(job, last_build, incidents)
     result = str(last_build.get("result") or "").upper()
     if result == "SUCCESS":
         return _resolve_on_success(storage, job, last_build, incidents)
@@ -343,16 +322,7 @@ def _record_incident(
     """Append an incident event and publish its one-hot status gauge."""
 
     storage.record_event(INCIDENT_EVENT_TYPE, {**base, "status": status, "phase": phase or {}})
-    _set_incident_gauge(str(base["job"]), str(base["build_number"]), {status, *extra_statuses})
-
-
-def _set_incident_gauge(job: str, build: str, active: set[str]) -> None:
-    """Set the incident gauge to 1 for active statuses and 0 for the rest."""
-
-    for status in INCIDENT_STATUSES:
-        HERMES_TRIAGE_INCIDENT.labels(jenkins_job=job, build=build, status=status).set(
-            1.0 if status in active else 0.0
-        )
+    set_incident_gauge(str(base["job"]), str(base["build_number"]), {status, *extra_statuses})
 
 
 def _record_action(

ariadne/services/hermes_autotriage_metrics.py

@@ -0,0 +1,95 @@
+"""Prometheus metrics for Hermes auto-triage, with restart-safe rehydration.
+
+The incident gauge is one-hot per (jenkins_job, build): the current status
+holds 1 and every other status holds 0. Because gauge state lives in
+process memory, each scheduler tick republishes the gauges for known
+incidents so pod restarts cannot silently drop an active human-required
+signal before the alert's hold window elapses.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from prometheus_client import Counter, Gauge
+
+
+INCIDENT_STATUSES = (
+    "detected",
+    "diagnosed",
+    "repairing",
+    "awaiting_rebuild",
+    "resolved",
+    "human_required",
+    "failed",
+)
+
+HERMES_TRIAGE_INCIDENT = Gauge(
+    "ariadne_hermes_triage_incident",
+    "Hermes auto-triage incident state (1=current status, 0=other statuses)",
+    ["jenkins_job", "build", "status"],
+)
+HERMES_TRIAGE_ACTION_TOTAL = Counter(
+    "ariadne_hermes_triage_action_total",
+    "Hermes auto-triage remediation actions by result",
+    ["action", "result"],
+)
+HERMES_TRIAGE_LAST_SUCCESS_TS = Gauge(
+    "ariadne_hermes_triage_last_success_timestamp_seconds",
+    "Last Hermes auto-triage incident resolution timestamp",
+)
+HERMES_TRIAGE_DURATION_SECONDS = Gauge(
+    "ariadne_hermes_triage_duration_seconds",
+    "Duration of the latest Hermes auto-triage phase in seconds",
+    ["phase"],
+)
+
+
+def set_incident_gauge(job: str, build: str, active: set[str]) -> None:
+    """Set the incident gauge to 1 for active statuses and 0 for the rest.
+
+    Inputs: the Jenkins job name, the build number as a string, and the
+    set of statuses that should read 1. Outputs: none.
+    """
+
+    for status in INCIDENT_STATUSES:
+        HERMES_TRIAGE_INCIDENT.labels(jenkins_job=job, build=build, status=status).set(
+            1 if status in active else 0
+        )
+
+
+def refresh_incident_gauges(job: str, last_build: dict[str, Any], incidents: dict[str, dict[str, Any]]) -> None:
+    """Republish incident gauges for one job from stored incident state.
+
+    Inputs: the job name, its Jenkins lastBuild summary, and the folded
+    latest-state-per-incident map. Outputs: none. Unresolved incidents keep
+    their one-hot status (a repair failure also keeps human_required); an
+    incident superseded by a newer successful build is zeroed so the
+    human-required alert clears once the job is green again. Resolved
+    incidents are left quiet.
+    """
+
+    number = _int_value(last_build.get("number"))
+    superseded = str(last_build.get("result") or "").upper() == "SUCCESS"
+    for incident in incidents.values():
+        if incident.get("job") != job:
+            continue
+        status = str(incident.get("status") or "")
+        if status not in INCIDENT_STATUSES or status == "resolved":
+            continue
+        build = str(incident.get("build_number"))
+        if superseded and _int_value(incident.get("build_number")) < number:
+            set_incident_gauge(job, build, set())
+        elif status == "failed":
+            set_incident_gauge(job, build, {"failed", "human_required"})
+        else:
+            set_incident_gauge(job, build, {status})
+
+
+def _int_value(value: Any) -> int:
+    """Coerce a value to int, defaulting to zero."""
+
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return 0

tests/test_hermes_autotriage_metrics.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from ariadne.services import hermes_autotriage_metrics as module
+
+
+JOB = "hermes-triage-demo"
+
+
+def _gauge(build: str, status: str) -> float:
+    return module.HERMES_TRIAGE_INCIDENT.labels(
+        jenkins_job=JOB, build=build, status=status
+    )._value.get()
+
+
+def _incident(build: int, status: str) -> dict:
+    return {
+        "incident_id": f"{JOB}/{build}",
+        "job": JOB,
+        "build_number": build,
+        "status": status,
+    }
+
+
+def test_set_incident_gauge_is_one_hot() -> None:
+    module.set_incident_gauge(JOB, "90", {"diagnosed"})
+    assert _gauge("90", "diagnosed") == 1
+    for status in module.INCIDENT_STATUSES:
+        if status != "diagnosed":
+            assert _gauge("90", status) == 0
+
+
+def test_refresh_republishes_active_incident_after_restart() -> None:
+    """A human_required incident must survive a gauge-wiping pod restart."""
+
+    incidents = {f"{JOB}/91": _incident(91, "human_required")}
+    last_build = {"number": 91, "result": "FAILURE"}
+    module.refresh_incident_gauges(JOB, last_build, incidents)
+    assert _gauge("91", "human_required") == 1
+
+
+def test_refresh_clears_incident_superseded_by_newer_green_build() -> None:
+    incidents = {f"{JOB}/92": _incident(92, "human_required")}
+    module.refresh_incident_gauges(JOB, {"number": 92, "result": "FAILURE"}, incidents)
+    assert _gauge("92", "human_required") == 1
+    module.refresh_incident_gauges(JOB, {"number": 93, "result": "SUCCESS"}, incidents)
+    assert _gauge("92", "human_required") == 0
+
+
+def test_refresh_keeps_failed_incident_alerting_with_human_required() -> None:
+    incidents = {f"{JOB}/94": _incident(94, "failed")}
+    module.refresh_incident_gauges(JOB, {"number": 94, "result": "FAILURE"}, incidents)
+    assert _gauge("94", "failed") == 1
+    assert _gauge("94", "human_required") == 1
+
+
+def test_refresh_skips_resolved_foreign_and_unknown_incidents() -> None:
+    incidents = {
+        f"{JOB}/95": _incident(95, "resolved"),
+        "other-job/9": {**_incident(9, "human_required"), "job": "other-job", "incident_id": "other-job/9"},
+        f"{JOB}/96": _incident(96, "not-a-status"),
+    }
+    module.refresh_incident_gauges(JOB, {"number": 97, "result": "FAILURE"}, incidents)
+    assert _gauge("95", "resolved") == 0
+    assert _gauge("96", "detected") == 0
+
+
+def test_refresh_same_build_green_does_not_clear() -> None:
+    """A success at the same build number is not a supersession."""
+
+    incidents = {f"{JOB}/98": _incident(98, "human_required")}
+    module.refresh_incident_gauges(JOB, {"number": 98, "result": "SUCCESS"}, incidents)
+    assert _gauge("98", "human_required") == 1