e89de924c0
Gauges are process-local, so a pod restart could silently drop an active human_required signal before the vmalert hold window elapsed. Metrics move to hermes_autotriage_metrics; every tick republishes gauges from stored incident state, and an incident superseded by a newer successful build is zeroed so the alert clears once the job is green again. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
73 lines
2.6 KiB
Python
73 lines
2.6 KiB
Python
from __future__ import annotations
|
|
|
|
from ariadne.services import hermes_autotriage_metrics as module
|
|
|
|
|
|
JOB = "hermes-triage-demo"
|
|
|
|
|
|
def _gauge(build: str, status: str) -> float:
|
|
return module.HERMES_TRIAGE_INCIDENT.labels(
|
|
jenkins_job=JOB, build=build, status=status
|
|
)._value.get()
|
|
|
|
|
|
def _incident(build: int, status: str) -> dict:
|
|
return {
|
|
"incident_id": f"{JOB}/{build}",
|
|
"job": JOB,
|
|
"build_number": build,
|
|
"status": status,
|
|
}
|
|
|
|
|
|
def test_set_incident_gauge_is_one_hot() -> None:
|
|
module.set_incident_gauge(JOB, "90", {"diagnosed"})
|
|
assert _gauge("90", "diagnosed") == 1
|
|
for status in module.INCIDENT_STATUSES:
|
|
if status != "diagnosed":
|
|
assert _gauge("90", status) == 0
|
|
|
|
|
|
def test_refresh_republishes_active_incident_after_restart() -> None:
|
|
"""A human_required incident must survive a gauge-wiping pod restart."""
|
|
|
|
incidents = {f"{JOB}/91": _incident(91, "human_required")}
|
|
last_build = {"number": 91, "result": "FAILURE"}
|
|
module.refresh_incident_gauges(JOB, last_build, incidents)
|
|
assert _gauge("91", "human_required") == 1
|
|
|
|
|
|
def test_refresh_clears_incident_superseded_by_newer_green_build() -> None:
|
|
incidents = {f"{JOB}/92": _incident(92, "human_required")}
|
|
module.refresh_incident_gauges(JOB, {"number": 92, "result": "FAILURE"}, incidents)
|
|
assert _gauge("92", "human_required") == 1
|
|
module.refresh_incident_gauges(JOB, {"number": 93, "result": "SUCCESS"}, incidents)
|
|
assert _gauge("92", "human_required") == 0
|
|
|
|
|
|
def test_refresh_keeps_failed_incident_alerting_with_human_required() -> None:
|
|
incidents = {f"{JOB}/94": _incident(94, "failed")}
|
|
module.refresh_incident_gauges(JOB, {"number": 94, "result": "FAILURE"}, incidents)
|
|
assert _gauge("94", "failed") == 1
|
|
assert _gauge("94", "human_required") == 1
|
|
|
|
|
|
def test_refresh_skips_resolved_foreign_and_unknown_incidents() -> None:
|
|
incidents = {
|
|
f"{JOB}/95": _incident(95, "resolved"),
|
|
"other-job/9": {**_incident(9, "human_required"), "job": "other-job", "incident_id": "other-job/9"},
|
|
f"{JOB}/96": _incident(96, "not-a-status"),
|
|
}
|
|
module.refresh_incident_gauges(JOB, {"number": 97, "result": "FAILURE"}, incidents)
|
|
assert _gauge("95", "resolved") == 0
|
|
assert _gauge("96", "detected") == 0
|
|
|
|
|
|
def test_refresh_same_build_green_does_not_clear() -> None:
|
|
"""A success at the same build number is not a supersession."""
|
|
|
|
incidents = {f"{JOB}/98": _incident(98, "human_required")}
|
|
module.refresh_incident_gauges(JOB, {"number": 98, "result": "SUCCESS"}, incidents)
|
|
assert _gauge("98", "human_required") == 1
|