diff --git a/ariadne/services/hermes_autotriage.py b/ariadne/services/hermes_autotriage.py index c2b95c7..1c8f927 100644 --- a/ariadne/services/hermes_autotriage.py +++ b/ariadne/services/hermes_autotriage.py @@ -5,13 +5,21 @@ import time from typing import Any import httpx -from prometheus_client import Counter, Gauge from ..settings import settings from ..utils.logging import get_logger from . import hermes_agent_client, hermes_autotriage_repair from . import hermes_autotriage_decision as hermes_decision from . import hermes_autotriage_evidence as hermes_evidence +from .hermes_autotriage_metrics import ( + HERMES_TRIAGE_ACTION_TOTAL, + HERMES_TRIAGE_DURATION_SECONDS, + HERMES_TRIAGE_INCIDENT, + HERMES_TRIAGE_LAST_SUCCESS_TS, + INCIDENT_STATUSES, + refresh_incident_gauges, + set_incident_gauge, +) logger = get_logger(__name__) @@ -19,15 +27,6 @@ logger = get_logger(__name__) INCIDENT_EVENT_TYPE = "hermes_autotriage_incident" DIAGNOSIS_EVENT_TYPE = "hermes_autotriage_diagnosis" ACTION_EVENT_TYPE = "hermes_autotriage_action" -INCIDENT_STATUSES = ( - "detected", - "diagnosed", - "repairing", - "awaiting_rebuild", - "resolved", - "human_required", - "failed", -) EXPECTED_CLASSIFICATION = "known_demo_fixture_failure" REBUILD_FAILED_REASON = "repair rebuild failed" @@ -52,27 +51,6 @@ Do not perform mutations. Bundle: __BUNDLE__""" -HERMES_TRIAGE_INCIDENT = Gauge( - "ariadne_hermes_triage_incident", - "Hermes auto-triage incident state (1=current status, 0=other statuses)", - ["jenkins_job", "build", "status"], -) -HERMES_TRIAGE_ACTION_TOTAL = Counter( - "ariadne_hermes_triage_action_total", - "Hermes auto-triage remediation actions by result", - ["action", "result"], -) -HERMES_TRIAGE_LAST_SUCCESS_TS = Gauge( - "ariadne_hermes_triage_last_success_timestamp_seconds", - "Last Hermes auto-triage incident resolution timestamp", -) -HERMES_TRIAGE_DURATION_SECONDS = Gauge( - "ariadne_hermes_triage_duration_seconds", - "Duration of the latest Hermes auto-triage phase in seconds", - ["phase"], -) - - def run_hermes_autotriage(storage: Any) -> dict[str, Any]: """Run one Hermes auto-triage tick over the allowlisted Jenkins jobs. @@ -130,6 +108,7 @@ def _process_job(storage: Any, job: str, incidents: dict[str, dict[str, Any]]) - last_build = _fetch_last_build(job) if last_build is None or last_build.get("building") or last_build.get("number") is None: return {"status": "skipped"} + refresh_incident_gauges(job, last_build, incidents) result = str(last_build.get("result") or "").upper() if result == "SUCCESS": return _resolve_on_success(storage, job, last_build, incidents) @@ -343,16 +322,7 @@ def _record_incident( """Append an incident event and publish its one-hot status gauge.""" storage.record_event(INCIDENT_EVENT_TYPE, {**base, "status": status, "phase": phase or {}}) - _set_incident_gauge(str(base["job"]), str(base["build_number"]), {status, *extra_statuses}) - - -def _set_incident_gauge(job: str, build: str, active: set[str]) -> None: - """Set the incident gauge to 1 for active statuses and 0 for the rest.""" - - for status in INCIDENT_STATUSES: - HERMES_TRIAGE_INCIDENT.labels(jenkins_job=job, build=build, status=status).set( - 1.0 if status in active else 0.0 - ) + set_incident_gauge(str(base["job"]), str(base["build_number"]), {status, *extra_statuses}) def _record_action( diff --git a/ariadne/services/hermes_autotriage_metrics.py b/ariadne/services/hermes_autotriage_metrics.py new file mode 100644 index 0000000..7d561f0 --- /dev/null +++ b/ariadne/services/hermes_autotriage_metrics.py @@ -0,0 +1,95 @@ +"""Prometheus metrics for Hermes auto-triage, with restart-safe rehydration. + +The incident gauge is one-hot per (jenkins_job, build): the current status +holds 1 and every other status holds 0. Because gauge state lives in +process memory, each scheduler tick republishes the gauges for known +incidents so pod restarts cannot silently drop an active human-required +signal before the alert's hold window elapses. +""" + +from __future__ import annotations + +from typing import Any + +from prometheus_client import Counter, Gauge + + +INCIDENT_STATUSES = ( + "detected", + "diagnosed", + "repairing", + "awaiting_rebuild", + "resolved", + "human_required", + "failed", +) + +HERMES_TRIAGE_INCIDENT = Gauge( + "ariadne_hermes_triage_incident", + "Hermes auto-triage incident state (1=current status, 0=other statuses)", + ["jenkins_job", "build", "status"], +) +HERMES_TRIAGE_ACTION_TOTAL = Counter( + "ariadne_hermes_triage_action_total", + "Hermes auto-triage remediation actions by result", + ["action", "result"], +) +HERMES_TRIAGE_LAST_SUCCESS_TS = Gauge( + "ariadne_hermes_triage_last_success_timestamp_seconds", + "Last Hermes auto-triage incident resolution timestamp", +) +HERMES_TRIAGE_DURATION_SECONDS = Gauge( + "ariadne_hermes_triage_duration_seconds", + "Duration of the latest Hermes auto-triage phase in seconds", + ["phase"], +) + + +def set_incident_gauge(job: str, build: str, active: set[str]) -> None: + """Set the incident gauge to 1 for active statuses and 0 for the rest. + + Inputs: the Jenkins job name, the build number as a string, and the + set of statuses that should read 1. Outputs: none. + """ + + for status in INCIDENT_STATUSES: + HERMES_TRIAGE_INCIDENT.labels(jenkins_job=job, build=build, status=status).set( + 1 if status in active else 0 + ) + + +def refresh_incident_gauges(job: str, last_build: dict[str, Any], incidents: dict[str, dict[str, Any]]) -> None: + """Republish incident gauges for one job from stored incident state. + + Inputs: the job name, its Jenkins lastBuild summary, and the folded + latest-state-per-incident map. Outputs: none. Unresolved incidents keep + their one-hot status (a repair failure also keeps human_required); an + incident superseded by a newer successful build is zeroed so the + human-required alert clears once the job is green again. Resolved + incidents are left quiet. + """ + + number = _int_value(last_build.get("number")) + superseded = str(last_build.get("result") or "").upper() == "SUCCESS" + for incident in incidents.values(): + if incident.get("job") != job: + continue + status = str(incident.get("status") or "") + if status not in INCIDENT_STATUSES or status == "resolved": + continue + build = str(incident.get("build_number")) + if superseded and _int_value(incident.get("build_number")) < number: + set_incident_gauge(job, build, set()) + elif status == "failed": + set_incident_gauge(job, build, {"failed", "human_required"}) + else: + set_incident_gauge(job, build, {status}) + + +def _int_value(value: Any) -> int: + """Coerce a value to int, defaulting to zero.""" + + try: + return int(value) + except (TypeError, ValueError): + return 0 diff --git a/tests/test_hermes_autotriage_metrics.py b/tests/test_hermes_autotriage_metrics.py new file mode 100644 index 0000000..8bcdfda --- /dev/null +++ b/tests/test_hermes_autotriage_metrics.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from ariadne.services import hermes_autotriage_metrics as module + + +JOB = "hermes-triage-demo" + + +def _gauge(build: str, status: str) -> float: + return module.HERMES_TRIAGE_INCIDENT.labels( + jenkins_job=JOB, build=build, status=status + )._value.get() + + +def _incident(build: int, status: str) -> dict: + return { + "incident_id": f"{JOB}/{build}", + "job": JOB, + "build_number": build, + "status": status, + } + + +def test_set_incident_gauge_is_one_hot() -> None: + module.set_incident_gauge(JOB, "90", {"diagnosed"}) + assert _gauge("90", "diagnosed") == 1 + for status in module.INCIDENT_STATUSES: + if status != "diagnosed": + assert _gauge("90", status) == 0 + + +def test_refresh_republishes_active_incident_after_restart() -> None: + """A human_required incident must survive a gauge-wiping pod restart.""" + + incidents = {f"{JOB}/91": _incident(91, "human_required")} + last_build = {"number": 91, "result": "FAILURE"} + module.refresh_incident_gauges(JOB, last_build, incidents) + assert _gauge("91", "human_required") == 1 + + +def test_refresh_clears_incident_superseded_by_newer_green_build() -> None: + incidents = {f"{JOB}/92": _incident(92, "human_required")} + module.refresh_incident_gauges(JOB, {"number": 92, "result": "FAILURE"}, incidents) + assert _gauge("92", "human_required") == 1 + module.refresh_incident_gauges(JOB, {"number": 93, "result": "SUCCESS"}, incidents) + assert _gauge("92", "human_required") == 0 + + +def test_refresh_keeps_failed_incident_alerting_with_human_required() -> None: + incidents = {f"{JOB}/94": _incident(94, "failed")} + module.refresh_incident_gauges(JOB, {"number": 94, "result": "FAILURE"}, incidents) + assert _gauge("94", "failed") == 1 + assert _gauge("94", "human_required") == 1 + + +def test_refresh_skips_resolved_foreign_and_unknown_incidents() -> None: + incidents = { + f"{JOB}/95": _incident(95, "resolved"), + "other-job/9": {**_incident(9, "human_required"), "job": "other-job", "incident_id": "other-job/9"}, + f"{JOB}/96": _incident(96, "not-a-status"), + } + module.refresh_incident_gauges(JOB, {"number": 97, "result": "FAILURE"}, incidents) + assert _gauge("95", "resolved") == 0 + assert _gauge("96", "detected") == 0 + + +def test_refresh_same_build_green_does_not_clear() -> None: + """A success at the same build number is not a supersession.""" + + incidents = {f"{JOB}/98": _incident(98, "human_required")} + module.refresh_incident_gauges(JOB, {"number": 98, "result": "SUCCESS"}, incidents) + assert _gauge("98", "human_required") == 1