atlas-iac/scripts/tests/test_dashboards_render_overview.py

230 lines
13 KiB
Python
Raw Normal View History

"""Atlas overview dashboard rendering contracts."""
from __future__ import annotations
from test_dashboard_render_support import (
flatten_panels,
load_module,
)
def test_table_panel_options_and_filterable():
mod = load_module()
panel = mod.table_panel(
1,
"test",
"metric",
{"h": 1, "w": 1, "x": 0, "y": 0},
unit="percent",
transformations=[{"id": "labelsToFields", "options": {}}],
instant=True,
options={"showColumnFilters": False},
filterable=False,
footer={"show": False, "fields": "", "calcs": []},
format="table",
)
assert panel["fieldConfig"]["defaults"]["unit"] == "percent"
assert panel["fieldConfig"]["defaults"]["custom"]["filterable"] is False
assert panel["options"]["showHeader"] is True
assert panel["targets"][0]["format"] == "table"
def test_node_filter_and_expr_helpers():
mod = load_module()
expr = mod.node_filter("titan-.*")
assert "label_replace" in expr
cpu_expr = mod.node_cpu_expr("titan-.*")
mem_expr = mod.node_mem_expr("titan-.*")
assert "node_cpu_seconds_total" in cpu_expr
assert "clamp_max(clamp_min(" in cpu_expr
assert "* 100, 0), 100)" in cpu_expr
assert "node_memory_MemAvailable_bytes" in mem_expr
def test_overview_availability_panel_uses_recorded_365d_rollup():
mod = load_module()
dashboard = mod.build_overview()
fix(monitoring): measure Atlas availability honestly across telemetry gaps The 2026-08-18 metrics-storage outage exposed two defects in the availability pipeline that distorted the figure in opposite directions at once. The Overview panel fell back to a live one-hour Traefik ratio whenever the yearly rollup sample went stale for 48h, and rendered it under the same "365d" title. When the rollup stopped publishing on 2026-08-18 the panel quietly swapped a 365-day measurement for a 60-minute one and read 99.74% instead of the recorded 99.95%. The fallback is removed: a stale rollup now renders no value, and a new atlas-availability-rollup-stale alert pages at 26h, well before the panel goes blank at 48h. The yearly ratio also silently excluded the 34-hour telemetry gap, because missing days contribute zero requests and zero failures. Absent data was read as "nothing happened" — had Atlas genuinely been down in that window, the figure would still have said 99.95%. Availability keeps its measured-days-only definition, which is correct, but coverage is now published alongside it and shown in a new panel, so a telemetry gap lowers disclosed coverage instead of vanishing. The title reads "365d window" to stop implying 365 days of data exist; request-v4 begins 2026-05-01. The rollup job reported healthy runs across a day and a half of lost publishes: a read-only VictoriaMetrics accepts an import and discards it. It now reads each sample back and fails loudly when the write did not survive. Not addressed here: availability is still measured from inside the platform via Traefik counters, so it cannot distinguish "Atlas down" from "telemetry down", and misses failures that never reach Traefik (DNS, TLS, node dead). An external synthetic prober is the real fix and needs a hosting decision.
2026-08-20 02:03:34 +00:00
panels_by_id = {panel["id"]: panel for panel in flatten_panels(dashboard["panels"])}
panel = panels_by_id[27]
fix(monitoring): measure Atlas availability honestly across telemetry gaps The 2026-08-18 metrics-storage outage exposed two defects in the availability pipeline that distorted the figure in opposite directions at once. The Overview panel fell back to a live one-hour Traefik ratio whenever the yearly rollup sample went stale for 48h, and rendered it under the same "365d" title. When the rollup stopped publishing on 2026-08-18 the panel quietly swapped a 365-day measurement for a 60-minute one and read 99.74% instead of the recorded 99.95%. The fallback is removed: a stale rollup now renders no value, and a new atlas-availability-rollup-stale alert pages at 26h, well before the panel goes blank at 48h. The yearly ratio also silently excluded the 34-hour telemetry gap, because missing days contribute zero requests and zero failures. Absent data was read as "nothing happened" — had Atlas genuinely been down in that window, the figure would still have said 99.95%. Availability keeps its measured-days-only definition, which is correct, but coverage is now published alongside it and shown in a new panel, so a telemetry gap lowers disclosed coverage instead of vanishing. The title reads "365d window" to stop implying 365 days of data exist; request-v4 begins 2026-05-01. The rollup job reported healthy runs across a day and a half of lost publishes: a read-only VictoriaMetrics accepts an import and discards it. It now reads each sample back and fails loudly when the write did not survive. Not addressed here: availability is still measured from inside the platform via Traefik counters, so it cannot distinguish "Atlas down" from "telemetry down", and misses failures that never reach Traefik (DNS, TLS, node dead). An external synthetic prober is the real fix and needs a hosting decision.
2026-08-20 02:03:34 +00:00
assert panel["title"] == "Atlas Availability (365d window)"
availability_expr = panel["targets"][0]["expr"]
assert (
fix(monitoring): measure Atlas availability honestly across telemetry gaps The 2026-08-18 metrics-storage outage exposed two defects in the availability pipeline that distorted the figure in opposite directions at once. The Overview panel fell back to a live one-hour Traefik ratio whenever the yearly rollup sample went stale for 48h, and rendered it under the same "365d" title. When the rollup stopped publishing on 2026-08-18 the panel quietly swapped a 365-day measurement for a 60-minute one and read 99.74% instead of the recorded 99.95%. The fallback is removed: a stale rollup now renders no value, and a new atlas-availability-rollup-stale alert pages at 26h, well before the panel goes blank at 48h. The yearly ratio also silently excluded the 34-hour telemetry gap, because missing days contribute zero requests and zero failures. Absent data was read as "nothing happened" — had Atlas genuinely been down in that window, the figure would still have said 99.95%. Availability keeps its measured-days-only definition, which is correct, but coverage is now published alongside it and shown in a new panel, so a telemetry gap lowers disclosed coverage instead of vanishing. The title reads "365d window" to stop implying 365 days of data exist; request-v4 begins 2026-05-01. The rollup job reported healthy runs across a day and a half of lost publishes: a read-only VictoriaMetrics accepts an import and discards it. It now reads each sample back and fails loudly when the write did not survive. Not addressed here: availability is still measured from inside the platform via Traefik counters, so it cannot distinguish "Atlas down" from "telemetry down", and misses failures that never reach Traefik (DNS, TLS, node dead). An external synthetic prober is the real fix and needs a hosting decision.
2026-08-20 02:03:34 +00:00
availability_expr
== 'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])'
)
fix(monitoring): measure Atlas availability honestly across telemetry gaps The 2026-08-18 metrics-storage outage exposed two defects in the availability pipeline that distorted the figure in opposite directions at once. The Overview panel fell back to a live one-hour Traefik ratio whenever the yearly rollup sample went stale for 48h, and rendered it under the same "365d" title. When the rollup stopped publishing on 2026-08-18 the panel quietly swapped a 365-day measurement for a 60-minute one and read 99.74% instead of the recorded 99.95%. The fallback is removed: a stale rollup now renders no value, and a new atlas-availability-rollup-stale alert pages at 26h, well before the panel goes blank at 48h. The yearly ratio also silently excluded the 34-hour telemetry gap, because missing days contribute zero requests and zero failures. Absent data was read as "nothing happened" — had Atlas genuinely been down in that window, the figure would still have said 99.95%. Availability keeps its measured-days-only definition, which is correct, but coverage is now published alongside it and shown in a new panel, so a telemetry gap lowers disclosed coverage instead of vanishing. The title reads "365d window" to stop implying 365 days of data exist; request-v4 begins 2026-05-01. The rollup job reported healthy runs across a day and a half of lost publishes: a read-only VictoriaMetrics accepts an import and discards it. It now reads each sample back and fails loudly when the write did not survive. Not addressed here: availability is still measured from inside the platform via Traefik counters, so it cannot distinguish "Atlas down" from "telemetry down", and misses failures that never reach Traefik (DNS, TLS, node dead). An external synthetic prober is the real fix and needs a hosting decision.
2026-08-20 02:03:34 +00:00
# A stale rollup must render nothing rather than silently substituting the
# last hour of Traefik traffic under a yearly title, as it did on 2026-08-18.
assert "traefik_entrypoint_requests_total" not in availability_expr
assert "atlas:availability:failures_1d" not in availability_expr
assert "atlas:availability:requests_1d" not in availability_expr
assert "sum_over_time" not in availability_expr
assert "kube_node_status_condition" not in availability_expr
assert "kube_deployment_status_replicas_available" not in availability_expr
assert panel["targets"][0]["instant"] is True
assert "Every server-side 5xx" in panel["description"]
fix(monitoring): measure Atlas availability honestly across telemetry gaps The 2026-08-18 metrics-storage outage exposed two defects in the availability pipeline that distorted the figure in opposite directions at once. The Overview panel fell back to a live one-hour Traefik ratio whenever the yearly rollup sample went stale for 48h, and rendered it under the same "365d" title. When the rollup stopped publishing on 2026-08-18 the panel quietly swapped a 365-day measurement for a 60-minute one and read 99.74% instead of the recorded 99.95%. The fallback is removed: a stale rollup now renders no value, and a new atlas-availability-rollup-stale alert pages at 26h, well before the panel goes blank at 48h. The yearly ratio also silently excluded the 34-hour telemetry gap, because missing days contribute zero requests and zero failures. Absent data was read as "nothing happened" — had Atlas genuinely been down in that window, the figure would still have said 99.95%. Availability keeps its measured-days-only definition, which is correct, but coverage is now published alongside it and shown in a new panel, so a telemetry gap lowers disclosed coverage instead of vanishing. The title reads "365d window" to stop implying 365 days of data exist; request-v4 begins 2026-05-01. The rollup job reported healthy runs across a day and a half of lost publishes: a read-only VictoriaMetrics accepts an import and discards it. It now reads each sample back and fails loudly when the write did not survive. Not addressed here: availability is still measured from inside the platform via Traefik counters, so it cannot distinguish "Atlas down" from "telemetry down", and misses failures that never reach Traefik (DNS, TLS, node dead). An external synthetic prober is the real fix and needs a hosting decision.
2026-08-20 02:03:34 +00:00
assert "never converted into downtime or uptime" in panel["description"]
coverage = panels_by_id[36]
assert coverage["title"] == "Availability Coverage (days)"
assert (
coverage["targets"][0]["expr"]
== 'last_over_time(atlas:availability:coverage_days_365d{scope="atlas",definition="request-v4"}[48h])'
)
assert "lowers this number instead of moving availability" in coverage["description"]
def test_overview_uses_readable_quality_power_and_gitops_panels():
mod = load_module()
dashboard = mod.build_overview()
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
assert dashboard["links"] == [
{"title": "Atlas Testing", "url": "/d/atlas-testing", "targetBlank": True}
]
assert "atlas-jobs" not in repr(dashboard)
assert "Platform Test Success Rate" not in panels_by_title
assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
assert panels_by_title["Test Category Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 13}
assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
assert panels_by_title["Test Category Health"]["targets"][0]["format"] == "time_series"
assert panels_by_title["Test Category Health"]["targets"][0]["range"] is True
assert "${overview_suite:regex}" not in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert panels_by_title["Test Category Health"]["timeFrom"] == "24h"
assert f'category=~"{mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert "manual" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
assert "unit" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
assert panels_by_title["UPS History (Power Draw)"]["gridPos"] == {"h": 6, "w": 6, "x": 3, "y": 7}
assert panels_by_title["Ariadne Run Volume"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 7}
assert panels_by_title["Pyrphoros UPS Current"]["gridPos"]["w"] == 3
assert panels_by_title["Current Enclosure Climate"]["gridPos"]["w"] == 3
assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["placement"] == "bottom"
assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["displayMode"] == "list"
assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "line"
assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["fillOpacity"] == 22
assert all(target["expr"].startswith("max(") for target in panels_by_title["UPS History (Power Draw)"]["targets"])
ups_overrides = panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["overrides"]
ups_override_by_name = {override["matcher"]["options"]: override for override in ups_overrides}
assert ups_override_by_name["Pyrphoros"]["properties"] == [
{"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-blue"}},
]
assert ups_override_by_name["Statera"]["properties"] == [
{"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-yellow"}},
]
assert panels_by_title["Ariadne Run Volume"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["placement"] == "bottom"
assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["displayMode"] == "list"
assert "Fan History (0-10)" not in panels_by_title
assert panels_by_title["Fan Intensity History"]["type"] == "state-timeline"
assert panels_by_title["Fan Intensity History"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 13}
assert panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["max"] == 10
assert panels_by_title["Fan Intensity History"]["targets"][0]["legendFormat"] == "{{fan}}"
fan_steps = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["thresholds"]["steps"]
assert [step["value"] for step in fan_steps] == [None, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
assert fan_steps[0]["color"] == "#8f1d1d"
assert fan_steps[5]["color"] == "#d4b106"
assert fan_steps[-1]["color"] == "#1f60c4"
fan_mappings = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["mappings"][0]["options"]
assert fan_mappings["0"]["text"] == "Off"
assert fan_mappings["10"]["text"] == "10"
assert "Tent Interior" in panels_by_title["Fan Intensity History"]["targets"][0]["expr"]
assert panels_by_title["Fan Intensity History"]["options"]["legend"]["displayMode"] == "list"
assert panels_by_title["Fan Intensity History"]["options"]["legend"]["showLegend"] is False
assert panels_by_title["Fan Intensity History"]["options"]["mergeValues"] is False
assert panels_by_title["Fan Intensity History"]["options"]["showValue"] == "auto"
assert panels_by_title["Flux Source"]["type"] == "stat"
assert panels_by_title["Flux Source"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 7}
assert panels_by_title["Flux Source"]["targets"][0]["legendFormat"] == "{{branch}}"
assert panels_by_title["Current Gate Health"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 9}
assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Current Gate Health"]["targets"][0]["expr"]
assert panels_by_title["CI Run Success (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 11}
assert panels_by_title["Suites With Runs (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 15}
suites_reporting_expr = panels_by_title["Suites With Runs (24h)"]["targets"][0]["expr"]
assert "> bool 0" in suites_reporting_expr
assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in suites_reporting_expr
assert "bstein-home" not in suites_reporting_expr
assert "published quality-gate run" in panels_by_title["Suites With Runs (24h)"]["description"]
assert panels_by_title["Avg Coverage"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 17}
assert "LOC Clean Suites" not in panels_by_title
assert panels_by_title["GitOps Health"]["type"] == "state-timeline"
assert panels_by_title["GitOps Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 7}
gitops_expr = panels_by_title["GitOps Health"]["targets"][0]["expr"]
assert "Kustomizations Not Suspended" in gitops_expr
assert "HelmReleases Not Suspended" in gitops_expr
assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
assert panels_by_title["Test Category Health"]["options"]["legend"]["showLegend"] is False
assert panels_by_title["Test Category Health"]["options"]["mergeValues"] is False
assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto"
assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9
assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"])
pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"]
assert "backup-telemetry-missing" in pvc_backup_expr
assert 'pvc_backup_(count|last_success_timestamp_seconds|health_reason)' in pvc_backup_expr
gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
assert "nvidia_gpu_device_utilization_percent" in gpu_expr
assert "jetson_gr3d_active_seconds_total" in gpu_expr
assert "increase" in gpu_expr
assert "sum_over_time" not in gpu_expr
assert "count_over_time" not in gpu_expr
assert "avg_over_time" in gpu_expr
assert "$__range" in gpu_expr
assert "$__range_s" in gpu_expr
assert "sum by (namespace)" in gpu_expr
assert 'namespace", "shared"' not in gpu_expr
assert "kube_pod_container_resource_requests" in gpu_expr
assert mod.GPU_RESOURCE_REGEX in gpu_expr
assert '"gpu_source", "nvidia"' in gpu_expr
assert '"gpu_source", "jetson"' in gpu_expr
assert "100 *" in gpu_expr
assert "100 -" not in gpu_expr
assert 'namespace", "unattributed"' in gpu_expr
assert 'namespace", "idle"' in gpu_expr
assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
assert "selected time range" in panels_by_title["Namespace GPU Utilization"]["description"]
assert "continuously sampled" in panels_by_title["Namespace GPU Utilization"]["description"]
assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
mod = load_module()
dashboard = mod.build_gpu_dashboard()
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"]
assert "nvidia_gpu_device_utilization_percent" in node_expr
assert "DCGM_FI_DEV_GPU_UTIL" in node_expr
assert "jetson_gr3d_freq_percent" in node_expr
assert "unless on(node)" in node_expr
assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"]
assert "nvidia_process_gpu_sm_util_percent" in process_expr
assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
def test_overview_and_testing_panels_all_have_concise_descriptions():
mod = load_module()
for dashboard in [mod.build_overview(), mod.build_jobs_dashboard(), mod.build_testing_dashboard()]:
panels = flatten_panels(dashboard["panels"])
assert all(panel.get("description") for panel in panels if panel["type"] != "row")
assert "85.7" not in repr(dashboard)
def test_render_configmap_writes(tmp_path):
mod = load_module()
mod.DASHBOARD_DIR = tmp_path / "dash"
mod.ROOT = tmp_path
uid = "atlas-test"
info = {"configmap": tmp_path / "cm.yaml"}
data = {"title": "Atlas Test"}
mod.write_json(uid, data)
mod.render_configmap(uid, info)
json_path = mod.DASHBOARD_DIR / f"{uid}.json"
assert json_path.exists()
content = (tmp_path / "cm.yaml").read_text()
assert "kind: ConfigMap" in content
assert f"{uid}.json" in content