950d014707
The code path was gated on a single job id, so homegrown services could never produce a pull request. Escalated incidents on mapped repositories now additionally attempt a bounded patch proposal, and the filed issue links it. - hermes_code_flow.propose_for_incident: additive entry point invoked only from the escalation branch, so an auto-remediated failure never also gets a patch and a failed Hermes run never spends tokens on one - three gates before any HTTP call: code enabled, not the legacy demo job, and the job has a repo mapping; unmapped jobs make zero calls - never raises: a failed proposal cannot change the incident outcome or break the tick - issue body links the proposal when a pull request was opened - propose_code_fix added to the bounded action-label set The legacy demo-job short-circuit is untouched and all of its tests pass unchanged. 8 new tests; 480 pass in the hermes suite. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
105 lines
3.7 KiB
Python
105 lines
3.7 KiB
Python
"""Prometheus metrics for Hermes auto-triage, with restart-safe rehydration.
|
|
|
|
The incident gauge is one-hot per (jenkins_job, build): the current status
|
|
holds 1 and every other status holds 0. Because gauge state lives in
|
|
process memory, each scheduler tick republishes the gauges for known
|
|
incidents so pod restarts cannot silently drop an active human-required
|
|
signal before the alert's hold window elapses.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from prometheus_client import Counter, Gauge
|
|
|
|
|
|
INCIDENT_STATUSES = (
|
|
"detected",
|
|
"diagnosed",
|
|
"repairing",
|
|
"awaiting_rebuild",
|
|
"resolved",
|
|
"human_required",
|
|
"failed",
|
|
)
|
|
|
|
PROPOSE_CODE_FIX_ACTION = "propose_code_fix"
|
|
|
|
# The bounded `action` label set for HERMES_TRIAGE_ACTION_TOTAL. Anything the
|
|
# model asks for that is neither deployed in hermes_allowed_actions nor listed
|
|
# here collapses to "unknown", so a hallucinated action id can never mint a new
|
|
# metric series. `propose_code_fix` is the orchestrator's own additive step
|
|
# rather than a model-requested remediation, so it has to be named here.
|
|
KNOWN_ACTION_LABELS = ("repair_demo_fixture", "retry_transient_infra", PROPOSE_CODE_FIX_ACTION)
|
|
|
|
HERMES_TRIAGE_INCIDENT = Gauge(
|
|
"ariadne_hermes_triage_incident",
|
|
"Hermes auto-triage incident state (1=current status, 0=other statuses)",
|
|
["jenkins_job", "build", "status"],
|
|
)
|
|
HERMES_TRIAGE_ACTION_TOTAL = Counter(
|
|
"ariadne_hermes_triage_action_total",
|
|
"Hermes auto-triage remediation actions by result",
|
|
["action", "result"],
|
|
)
|
|
HERMES_TRIAGE_LAST_SUCCESS_TS = Gauge(
|
|
"ariadne_hermes_triage_last_success_timestamp_seconds",
|
|
"Last Hermes auto-triage incident resolution timestamp",
|
|
)
|
|
HERMES_TRIAGE_DURATION_SECONDS = Gauge(
|
|
"ariadne_hermes_triage_duration_seconds",
|
|
"Duration of the latest Hermes auto-triage phase in seconds",
|
|
["phase"],
|
|
)
|
|
|
|
|
|
def set_incident_gauge(job: str, build: str, active: set[str]) -> None:
|
|
"""Set the incident gauge to 1 for active statuses and 0 for the rest.
|
|
|
|
Inputs: the Jenkins job name, the build number as a string, and the
|
|
set of statuses that should read 1. Outputs: none.
|
|
"""
|
|
|
|
for status in INCIDENT_STATUSES:
|
|
HERMES_TRIAGE_INCIDENT.labels(jenkins_job=job, build=build, status=status).set(
|
|
1 if status in active else 0
|
|
)
|
|
|
|
|
|
def refresh_incident_gauges(job: str, last_build: dict[str, Any], incidents: dict[str, dict[str, Any]]) -> None:
|
|
"""Republish incident gauges for one job from stored incident state.
|
|
|
|
Inputs: the job name, its Jenkins lastBuild summary, and the folded
|
|
latest-state-per-incident map. Outputs: none. Unresolved incidents keep
|
|
their one-hot status (a repair failure also keeps human_required); an
|
|
incident superseded by a newer successful build is zeroed so the
|
|
human-required alert clears once the job is green again. Resolved
|
|
incidents are left quiet.
|
|
"""
|
|
|
|
number = _int_value(last_build.get("number"))
|
|
superseded = str(last_build.get("result") or "").upper() == "SUCCESS"
|
|
for incident in incidents.values():
|
|
if incident.get("job") != job:
|
|
continue
|
|
status = str(incident.get("status") or "")
|
|
if status not in INCIDENT_STATUSES or status == "resolved":
|
|
continue
|
|
build = str(incident.get("build_number"))
|
|
if superseded and _int_value(incident.get("build_number")) < number:
|
|
set_incident_gauge(job, build, set())
|
|
elif status == "failed":
|
|
set_incident_gauge(job, build, {"failed", "human_required"})
|
|
else:
|
|
set_incident_gauge(job, build, {status})
|
|
|
|
|
|
def _int_value(value: Any) -> int:
|
|
"""Coerce a value to int, defaulting to zero."""
|
|
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return 0
|