2862594c62
The 2026-08-18 metrics-storage outage exposed two defects in the availability pipeline that distorted the figure in opposite directions at once. The Overview panel fell back to a live one-hour Traefik ratio whenever the yearly rollup sample went stale for 48h, and rendered it under the same "365d" title. When the rollup stopped publishing on 2026-08-18 the panel quietly swapped a 365-day measurement for a 60-minute one and read 99.74% instead of the recorded 99.95%. The fallback is removed: a stale rollup now renders no value, and a new atlas-availability-rollup-stale alert pages at 26h, well before the panel goes blank at 48h. The yearly ratio also silently excluded the 34-hour telemetry gap, because missing days contribute zero requests and zero failures. Absent data was read as "nothing happened" — had Atlas genuinely been down in that window, the figure would still have said 99.95%. Availability keeps its measured-days-only definition, which is correct, but coverage is now published alongside it and shown in a new panel, so a telemetry gap lowers disclosed coverage instead of vanishing. The title reads "365d window" to stop implying 365 days of data exist; request-v4 begins 2026-05-01. The rollup job reported healthy runs across a day and a half of lost publishes: a read-only VictoriaMetrics accepts an import and discards it. It now reads each sample back and fails loudly when the write did not survive. Not addressed here: availability is still measured from inside the platform via Traefik counters, so it cannot distinguish "Atlas down" from "telemetry down", and misses failures that never reach Traefik (DNS, TLS, node dead). An external synthetic prober is the real fix and needs a hosting decision.
230 lines
13 KiB
Python
230 lines
13 KiB
Python
"""Atlas overview dashboard rendering contracts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from test_dashboard_render_support import (
|
|
flatten_panels,
|
|
load_module,
|
|
)
|
|
|
|
|
|
def test_table_panel_options_and_filterable():
|
|
mod = load_module()
|
|
panel = mod.table_panel(
|
|
1,
|
|
"test",
|
|
"metric",
|
|
{"h": 1, "w": 1, "x": 0, "y": 0},
|
|
unit="percent",
|
|
transformations=[{"id": "labelsToFields", "options": {}}],
|
|
instant=True,
|
|
options={"showColumnFilters": False},
|
|
filterable=False,
|
|
footer={"show": False, "fields": "", "calcs": []},
|
|
format="table",
|
|
)
|
|
assert panel["fieldConfig"]["defaults"]["unit"] == "percent"
|
|
assert panel["fieldConfig"]["defaults"]["custom"]["filterable"] is False
|
|
assert panel["options"]["showHeader"] is True
|
|
assert panel["targets"][0]["format"] == "table"
|
|
|
|
def test_node_filter_and_expr_helpers():
|
|
mod = load_module()
|
|
expr = mod.node_filter("titan-.*")
|
|
assert "label_replace" in expr
|
|
cpu_expr = mod.node_cpu_expr("titan-.*")
|
|
mem_expr = mod.node_mem_expr("titan-.*")
|
|
assert "node_cpu_seconds_total" in cpu_expr
|
|
assert "clamp_max(clamp_min(" in cpu_expr
|
|
assert "* 100, 0), 100)" in cpu_expr
|
|
assert "node_memory_MemAvailable_bytes" in mem_expr
|
|
|
|
def test_overview_availability_panel_uses_recorded_365d_rollup():
|
|
mod = load_module()
|
|
dashboard = mod.build_overview()
|
|
panels_by_id = {panel["id"]: panel for panel in flatten_panels(dashboard["panels"])}
|
|
panel = panels_by_id[27]
|
|
|
|
assert panel["title"] == "Atlas Availability (365d window)"
|
|
availability_expr = panel["targets"][0]["expr"]
|
|
assert (
|
|
availability_expr
|
|
== 'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])'
|
|
)
|
|
# A stale rollup must render nothing rather than silently substituting the
|
|
# last hour of Traefik traffic under a yearly title, as it did on 2026-08-18.
|
|
assert "traefik_entrypoint_requests_total" not in availability_expr
|
|
assert "atlas:availability:failures_1d" not in availability_expr
|
|
assert "atlas:availability:requests_1d" not in availability_expr
|
|
assert "sum_over_time" not in availability_expr
|
|
assert "kube_node_status_condition" not in availability_expr
|
|
assert "kube_deployment_status_replicas_available" not in availability_expr
|
|
assert panel["targets"][0]["instant"] is True
|
|
assert "Every server-side 5xx" in panel["description"]
|
|
assert "never converted into downtime or uptime" in panel["description"]
|
|
|
|
coverage = panels_by_id[36]
|
|
assert coverage["title"] == "Availability Coverage (days)"
|
|
assert (
|
|
coverage["targets"][0]["expr"]
|
|
== 'last_over_time(atlas:availability:coverage_days_365d{scope="atlas",definition="request-v4"}[48h])'
|
|
)
|
|
assert "lowers this number instead of moving availability" in coverage["description"]
|
|
|
|
def test_overview_uses_readable_quality_power_and_gitops_panels():
|
|
mod = load_module()
|
|
dashboard = mod.build_overview()
|
|
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
|
|
|
|
assert dashboard["links"] == [
|
|
{"title": "Atlas Testing", "url": "/d/atlas-testing", "targetBlank": True}
|
|
]
|
|
assert "atlas-jobs" not in repr(dashboard)
|
|
assert "Platform Test Success Rate" not in panels_by_title
|
|
assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
|
|
assert panels_by_title["Test Category Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 13}
|
|
assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
|
|
assert panels_by_title["Test Category Health"]["targets"][0]["format"] == "time_series"
|
|
assert panels_by_title["Test Category Health"]["targets"][0]["range"] is True
|
|
assert "${overview_suite:regex}" not in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
|
assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
|
assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
|
assert panels_by_title["Test Category Health"]["timeFrom"] == "24h"
|
|
assert f'category=~"{mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
|
assert "manual" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
|
|
assert "unit" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
|
|
assert panels_by_title["UPS History (Power Draw)"]["gridPos"] == {"h": 6, "w": 6, "x": 3, "y": 7}
|
|
assert panels_by_title["Ariadne Run Volume"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 7}
|
|
assert panels_by_title["Pyrphoros UPS Current"]["gridPos"]["w"] == 3
|
|
assert panels_by_title["Current Enclosure Climate"]["gridPos"]["w"] == 3
|
|
assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["placement"] == "bottom"
|
|
assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["displayMode"] == "list"
|
|
assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "line"
|
|
assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["fillOpacity"] == 22
|
|
assert all(target["expr"].startswith("max(") for target in panels_by_title["UPS History (Power Draw)"]["targets"])
|
|
ups_overrides = panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["overrides"]
|
|
ups_override_by_name = {override["matcher"]["options"]: override for override in ups_overrides}
|
|
assert ups_override_by_name["Pyrphoros"]["properties"] == [
|
|
{"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-blue"}},
|
|
]
|
|
assert ups_override_by_name["Statera"]["properties"] == [
|
|
{"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-yellow"}},
|
|
]
|
|
assert panels_by_title["Ariadne Run Volume"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
|
|
assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["placement"] == "bottom"
|
|
assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["displayMode"] == "list"
|
|
assert "Fan History (0-10)" not in panels_by_title
|
|
assert panels_by_title["Fan Intensity History"]["type"] == "state-timeline"
|
|
assert panels_by_title["Fan Intensity History"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 13}
|
|
assert panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["max"] == 10
|
|
assert panels_by_title["Fan Intensity History"]["targets"][0]["legendFormat"] == "{{fan}}"
|
|
fan_steps = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["thresholds"]["steps"]
|
|
assert [step["value"] for step in fan_steps] == [None, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
|
assert fan_steps[0]["color"] == "#8f1d1d"
|
|
assert fan_steps[5]["color"] == "#d4b106"
|
|
assert fan_steps[-1]["color"] == "#1f60c4"
|
|
fan_mappings = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["mappings"][0]["options"]
|
|
assert fan_mappings["0"]["text"] == "Off"
|
|
assert fan_mappings["10"]["text"] == "10"
|
|
assert "Tent Interior" in panels_by_title["Fan Intensity History"]["targets"][0]["expr"]
|
|
assert panels_by_title["Fan Intensity History"]["options"]["legend"]["displayMode"] == "list"
|
|
assert panels_by_title["Fan Intensity History"]["options"]["legend"]["showLegend"] is False
|
|
assert panels_by_title["Fan Intensity History"]["options"]["mergeValues"] is False
|
|
assert panels_by_title["Fan Intensity History"]["options"]["showValue"] == "auto"
|
|
|
|
assert panels_by_title["Flux Source"]["type"] == "stat"
|
|
assert panels_by_title["Flux Source"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 7}
|
|
assert panels_by_title["Flux Source"]["targets"][0]["legendFormat"] == "{{branch}}"
|
|
assert panels_by_title["Current Gate Health"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 9}
|
|
assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Current Gate Health"]["targets"][0]["expr"]
|
|
assert panels_by_title["CI Run Success (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 11}
|
|
assert panels_by_title["Suites With Runs (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 15}
|
|
suites_reporting_expr = panels_by_title["Suites With Runs (24h)"]["targets"][0]["expr"]
|
|
assert "> bool 0" in suites_reporting_expr
|
|
assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in suites_reporting_expr
|
|
assert "bstein-home" not in suites_reporting_expr
|
|
assert "published quality-gate run" in panels_by_title["Suites With Runs (24h)"]["description"]
|
|
assert panels_by_title["Avg Coverage"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 17}
|
|
assert "LOC Clean Suites" not in panels_by_title
|
|
assert panels_by_title["GitOps Health"]["type"] == "state-timeline"
|
|
assert panels_by_title["GitOps Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 7}
|
|
gitops_expr = panels_by_title["GitOps Health"]["targets"][0]["expr"]
|
|
assert "Kustomizations Not Suspended" in gitops_expr
|
|
assert "HelmReleases Not Suspended" in gitops_expr
|
|
assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
|
|
assert panels_by_title["Test Category Health"]["options"]["legend"]["showLegend"] is False
|
|
assert panels_by_title["Test Category Health"]["options"]["mergeValues"] is False
|
|
assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto"
|
|
assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9
|
|
assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
|
|
assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
|
assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"])
|
|
|
|
pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"]
|
|
assert "backup-telemetry-missing" in pvc_backup_expr
|
|
assert 'pvc_backup_(count|last_success_timestamp_seconds|health_reason)' in pvc_backup_expr
|
|
|
|
gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
|
|
assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
|
|
assert "nvidia_gpu_device_utilization_percent" in gpu_expr
|
|
assert "jetson_gr3d_active_seconds_total" in gpu_expr
|
|
assert "increase" in gpu_expr
|
|
assert "sum_over_time" not in gpu_expr
|
|
assert "count_over_time" not in gpu_expr
|
|
assert "avg_over_time" in gpu_expr
|
|
assert "$__range" in gpu_expr
|
|
assert "$__range_s" in gpu_expr
|
|
assert "sum by (namespace)" in gpu_expr
|
|
assert 'namespace", "shared"' not in gpu_expr
|
|
assert "kube_pod_container_resource_requests" in gpu_expr
|
|
assert mod.GPU_RESOURCE_REGEX in gpu_expr
|
|
assert '"gpu_source", "nvidia"' in gpu_expr
|
|
assert '"gpu_source", "jetson"' in gpu_expr
|
|
assert "100 *" in gpu_expr
|
|
assert "100 -" not in gpu_expr
|
|
assert 'namespace", "unattributed"' in gpu_expr
|
|
assert 'namespace", "idle"' in gpu_expr
|
|
assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
|
|
assert "selected time range" in panels_by_title["Namespace GPU Utilization"]["description"]
|
|
assert "continuously sampled" in panels_by_title["Namespace GPU Utilization"]["description"]
|
|
assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
|
|
|
|
def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
|
|
mod = load_module()
|
|
dashboard = mod.build_gpu_dashboard()
|
|
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
|
|
|
|
node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"]
|
|
assert "nvidia_gpu_device_utilization_percent" in node_expr
|
|
assert "DCGM_FI_DEV_GPU_UTIL" in node_expr
|
|
assert "jetson_gr3d_freq_percent" in node_expr
|
|
assert "unless on(node)" in node_expr
|
|
assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
|
|
|
|
process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"]
|
|
assert "nvidia_process_gpu_sm_util_percent" in process_expr
|
|
assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
|
|
|
|
def test_overview_and_testing_panels_all_have_concise_descriptions():
|
|
mod = load_module()
|
|
|
|
for dashboard in [mod.build_overview(), mod.build_jobs_dashboard(), mod.build_testing_dashboard()]:
|
|
panels = flatten_panels(dashboard["panels"])
|
|
assert all(panel.get("description") for panel in panels if panel["type"] != "row")
|
|
assert "85.7" not in repr(dashboard)
|
|
|
|
def test_render_configmap_writes(tmp_path):
|
|
mod = load_module()
|
|
mod.DASHBOARD_DIR = tmp_path / "dash"
|
|
mod.ROOT = tmp_path
|
|
uid = "atlas-test"
|
|
info = {"configmap": tmp_path / "cm.yaml"}
|
|
data = {"title": "Atlas Test"}
|
|
mod.write_json(uid, data)
|
|
mod.render_configmap(uid, info)
|
|
json_path = mod.DASHBOARD_DIR / f"{uid}.json"
|
|
assert json_path.exists()
|
|
content = (tmp_path / "cm.yaml").read_text()
|
|
assert "kind: ConfigMap" in content
|
|
assert f"{uid}.json" in content
|