83 lines
2.9 KiB
Python
83 lines
2.9 KiB
Python
|
"""Keep the public availability figure honest about what it measured."""
|
||
|
|
|
||
|
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
import yaml
|
||
|
|
|
||
|
|
|
||
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||
|
|
|
||
|
|
# On 2026-08-18 the metrics volume filled and the availability pipeline failed
|
||
|
|
# in both directions at once: the stale yearly sample silently fell back to a
|
||
|
|
# one-hour Traefik ratio wearing the 365d title, and the 34-hour telemetry gap
|
||
|
|
# vanished from the yearly ratio as if it had been measured. These guardrails
|
||
|
|
# pin the corrections.
|
||
|
|
|
||
|
|
|
||
|
|
def _overview_panels() -> list[dict]:
|
||
|
|
"""Return the generated Overview dashboard's top-level panels."""
|
||
|
|
dashboard = json.loads(
|
||
|
|
(REPO_ROOT / "services/monitoring/dashboards/atlas-overview.json").read_text()
|
||
|
|
)
|
||
|
|
return dashboard["panels"]
|
||
|
|
|
||
|
|
|
||
|
|
def _panel(title_prefix: str) -> dict:
|
||
|
|
return next(
|
||
|
|
panel
|
||
|
|
for panel in _overview_panels()
|
||
|
|
if str(panel.get("title", "")).startswith(title_prefix)
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _panel_exprs(panel: dict) -> str:
|
||
|
|
return " ".join(target.get("expr", "") for target in panel.get("targets", []))
|
||
|
|
|
||
|
|
|
||
|
|
def _alert_rules() -> dict[str, dict]:
|
||
|
|
"""Return every provisioned Grafana alert rule keyed by uid."""
|
||
|
|
manifest = next(
|
||
|
|
document
|
||
|
|
for document in yaml.safe_load_all(
|
||
|
|
(REPO_ROOT / "services/monitoring/grafana-alerting-config.yaml").read_text()
|
||
|
|
)
|
||
|
|
if document
|
||
|
|
)
|
||
|
|
groups = yaml.safe_load(manifest["data"]["rules.yaml"])["groups"]
|
||
|
|
return {rule["uid"]: rule for group in groups for rule in group["rules"]}
|
||
|
|
|
||
|
|
|
||
|
|
def test_availability_panel_never_swaps_measurements() -> None:
|
||
|
|
"""A stale yearly rollup must render empty, not a 1h ratio in disguise."""
|
||
|
|
panel = _panel("Atlas Availability")
|
||
|
|
exprs = _panel_exprs(panel)
|
||
|
|
assert "atlas:availability:ratio_365d" in exprs
|
||
|
|
assert "traefik_entrypoint_requests_total" not in exprs
|
||
|
|
|
||
|
|
|
||
|
|
def test_availability_coverage_is_disclosed() -> None:
|
||
|
|
"""The overview must state how many days the yearly figure rests on."""
|
||
|
|
panel = _panel("Availability Coverage")
|
||
|
|
assert "atlas:availability:coverage_days_365d" in _panel_exprs(panel)
|
||
|
|
|
||
|
|
|
||
|
|
def test_stale_availability_rollup_pages() -> None:
|
||
|
|
"""A silent rollup once hid 34 hours of lost publishes; staleness must page."""
|
||
|
|
rules = _alert_rules()
|
||
|
|
stale = rules["atlas-availability-rollup-stale"]
|
||
|
|
assert "atlas:availability:ratio_365d" in stale["data"][0]["model"]["expr"]
|
||
|
|
assert stale["noDataState"] == "Alerting"
|
||
|
|
assert stale["execErrState"] == "Alerting"
|
||
|
|
assert stale["labels"]["severity"] == "warning"
|
||
|
|
|
||
|
|
|
||
|
|
def test_rollup_proves_its_publish_survived() -> None:
|
||
|
|
"""A read-only store accepts imports and drops them; the job must read back."""
|
||
|
|
source = (
|
||
|
|
REPO_ROOT / "services/monitoring/scripts/availability_rollup.py"
|
||
|
|
).read_text()
|
||
|
|
assert "def verify_stored" in source
|
||
|
|
assert "verify_stored(OUTPUT_METRIC" in source
|
||
|
|
assert "atlas:availability:coverage_days_365d" in source
|