2026-08-16 21:55:28 -03:00
|
|
|
"""Atlas overview dashboard rendering contracts."""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
from test_dashboard_render_support import (
|
|
|
|
|
flatten_panels,
|
|
|
|
|
load_module,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_table_panel_options_and_filterable():
|
|
|
|
|
mod = load_module()
|
|
|
|
|
panel = mod.table_panel(
|
|
|
|
|
1,
|
|
|
|
|
"test",
|
|
|
|
|
"metric",
|
|
|
|
|
{"h": 1, "w": 1, "x": 0, "y": 0},
|
|
|
|
|
unit="percent",
|
|
|
|
|
transformations=[{"id": "labelsToFields", "options": {}}],
|
|
|
|
|
instant=True,
|
|
|
|
|
options={"showColumnFilters": False},
|
|
|
|
|
filterable=False,
|
|
|
|
|
footer={"show": False, "fields": "", "calcs": []},
|
|
|
|
|
format="table",
|
|
|
|
|
)
|
|
|
|
|
assert panel["fieldConfig"]["defaults"]["unit"] == "percent"
|
|
|
|
|
assert panel["fieldConfig"]["defaults"]["custom"]["filterable"] is False
|
|
|
|
|
assert panel["options"]["showHeader"] is True
|
|
|
|
|
assert panel["targets"][0]["format"] == "table"
|
|
|
|
|
|
|
|
|
|
def test_node_filter_and_expr_helpers():
|
|
|
|
|
mod = load_module()
|
|
|
|
|
expr = mod.node_filter("titan-.*")
|
|
|
|
|
assert "label_replace" in expr
|
|
|
|
|
cpu_expr = mod.node_cpu_expr("titan-.*")
|
|
|
|
|
mem_expr = mod.node_mem_expr("titan-.*")
|
|
|
|
|
assert "node_cpu_seconds_total" in cpu_expr
|
|
|
|
|
assert "clamp_max(clamp_min(" in cpu_expr
|
|
|
|
|
assert "* 100, 0), 100)" in cpu_expr
|
|
|
|
|
assert "node_memory_MemAvailable_bytes" in mem_expr
|
|
|
|
|
|
|
|
|
|
def test_overview_availability_panel_uses_recorded_365d_rollup():
|
|
|
|
|
mod = load_module()
|
|
|
|
|
dashboard = mod.build_overview()
|
fix(monitoring): measure Atlas availability honestly across telemetry gaps
The 2026-08-18 metrics-storage outage exposed two defects in the
availability pipeline that distorted the figure in opposite directions at
once.
The Overview panel fell back to a live one-hour Traefik ratio whenever the
yearly rollup sample went stale for 48h, and rendered it under the same
"365d" title. When the rollup stopped publishing on 2026-08-18 the panel
quietly swapped a 365-day measurement for a 60-minute one and read 99.74%
instead of the recorded 99.95%. The fallback is removed: a stale rollup now
renders no value, and a new atlas-availability-rollup-stale alert pages at
26h, well before the panel goes blank at 48h.
The yearly ratio also silently excluded the 34-hour telemetry gap, because
missing days contribute zero requests and zero failures. Absent data was
read as "nothing happened" — had Atlas genuinely been down in that window,
the figure would still have said 99.95%. Availability keeps its
measured-days-only definition, which is correct, but coverage is now
published alongside it and shown in a new panel, so a telemetry gap lowers
disclosed coverage instead of vanishing. The title reads "365d window" to
stop implying 365 days of data exist; request-v4 begins 2026-05-01.
The rollup job reported healthy runs across a day and a half of lost
publishes: a read-only VictoriaMetrics accepts an import and discards it.
It now reads each sample back and fails loudly when the write did not
survive.
Not addressed here: availability is still measured from inside the platform
via Traefik counters, so it cannot distinguish "Atlas down" from "telemetry
down", and misses failures that never reach Traefik (DNS, TLS, node dead).
An external synthetic prober is the real fix and needs a hosting decision.
2026-08-20 02:03:34 +00:00
|
|
|
panels_by_id = {panel["id"]: panel for panel in flatten_panels(dashboard["panels"])}
|
|
|
|
|
panel = panels_by_id[27]
|
2026-08-16 21:55:28 -03:00
|
|
|
|
fix(monitoring): measure Atlas availability honestly across telemetry gaps
The 2026-08-18 metrics-storage outage exposed two defects in the
availability pipeline that distorted the figure in opposite directions at
once.
The Overview panel fell back to a live one-hour Traefik ratio whenever the
yearly rollup sample went stale for 48h, and rendered it under the same
"365d" title. When the rollup stopped publishing on 2026-08-18 the panel
quietly swapped a 365-day measurement for a 60-minute one and read 99.74%
instead of the recorded 99.95%. The fallback is removed: a stale rollup now
renders no value, and a new atlas-availability-rollup-stale alert pages at
26h, well before the panel goes blank at 48h.
The yearly ratio also silently excluded the 34-hour telemetry gap, because
missing days contribute zero requests and zero failures. Absent data was
read as "nothing happened" — had Atlas genuinely been down in that window,
the figure would still have said 99.95%. Availability keeps its
measured-days-only definition, which is correct, but coverage is now
published alongside it and shown in a new panel, so a telemetry gap lowers
disclosed coverage instead of vanishing. The title reads "365d window" to
stop implying 365 days of data exist; request-v4 begins 2026-05-01.
The rollup job reported healthy runs across a day and a half of lost
publishes: a read-only VictoriaMetrics accepts an import and discards it.
It now reads each sample back and fails loudly when the write did not
survive.
Not addressed here: availability is still measured from inside the platform
via Traefik counters, so it cannot distinguish "Atlas down" from "telemetry
down", and misses failures that never reach Traefik (DNS, TLS, node dead).
An external synthetic prober is the real fix and needs a hosting decision.
2026-08-20 02:03:34 +00:00
|
|
|
assert panel["title"] == "Atlas Availability (365d window)"
|
2026-08-16 21:55:28 -03:00
|
|
|
availability_expr = panel["targets"][0]["expr"]
|
|
|
|
|
assert (
|
fix(monitoring): measure Atlas availability honestly across telemetry gaps
The 2026-08-18 metrics-storage outage exposed two defects in the
availability pipeline that distorted the figure in opposite directions at
once.
The Overview panel fell back to a live one-hour Traefik ratio whenever the
yearly rollup sample went stale for 48h, and rendered it under the same
"365d" title. When the rollup stopped publishing on 2026-08-18 the panel
quietly swapped a 365-day measurement for a 60-minute one and read 99.74%
instead of the recorded 99.95%. The fallback is removed: a stale rollup now
renders no value, and a new atlas-availability-rollup-stale alert pages at
26h, well before the panel goes blank at 48h.
The yearly ratio also silently excluded the 34-hour telemetry gap, because
missing days contribute zero requests and zero failures. Absent data was
read as "nothing happened" — had Atlas genuinely been down in that window,
the figure would still have said 99.95%. Availability keeps its
measured-days-only definition, which is correct, but coverage is now
published alongside it and shown in a new panel, so a telemetry gap lowers
disclosed coverage instead of vanishing. The title reads "365d window" to
stop implying 365 days of data exist; request-v4 begins 2026-05-01.
The rollup job reported healthy runs across a day and a half of lost
publishes: a read-only VictoriaMetrics accepts an import and discards it.
It now reads each sample back and fails loudly when the write did not
survive.
Not addressed here: availability is still measured from inside the platform
via Traefik counters, so it cannot distinguish "Atlas down" from "telemetry
down", and misses failures that never reach Traefik (DNS, TLS, node dead).
An external synthetic prober is the real fix and needs a hosting decision.
2026-08-20 02:03:34 +00:00
|
|
|
availability_expr
|
|
|
|
|
== 'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])'
|
2026-08-16 21:55:28 -03:00
|
|
|
)
|
fix(monitoring): measure Atlas availability honestly across telemetry gaps
The 2026-08-18 metrics-storage outage exposed two defects in the
availability pipeline that distorted the figure in opposite directions at
once.
The Overview panel fell back to a live one-hour Traefik ratio whenever the
yearly rollup sample went stale for 48h, and rendered it under the same
"365d" title. When the rollup stopped publishing on 2026-08-18 the panel
quietly swapped a 365-day measurement for a 60-minute one and read 99.74%
instead of the recorded 99.95%. The fallback is removed: a stale rollup now
renders no value, and a new atlas-availability-rollup-stale alert pages at
26h, well before the panel goes blank at 48h.
The yearly ratio also silently excluded the 34-hour telemetry gap, because
missing days contribute zero requests and zero failures. Absent data was
read as "nothing happened" — had Atlas genuinely been down in that window,
the figure would still have said 99.95%. Availability keeps its
measured-days-only definition, which is correct, but coverage is now
published alongside it and shown in a new panel, so a telemetry gap lowers
disclosed coverage instead of vanishing. The title reads "365d window" to
stop implying 365 days of data exist; request-v4 begins 2026-05-01.
The rollup job reported healthy runs across a day and a half of lost
publishes: a read-only VictoriaMetrics accepts an import and discards it.
It now reads each sample back and fails loudly when the write did not
survive.
Not addressed here: availability is still measured from inside the platform
via Traefik counters, so it cannot distinguish "Atlas down" from "telemetry
down", and misses failures that never reach Traefik (DNS, TLS, node dead).
An external synthetic prober is the real fix and needs a hosting decision.
2026-08-20 02:03:34 +00:00
|
|
|
# A stale rollup must render nothing rather than silently substituting the
|
|
|
|
|
# last hour of Traefik traffic under a yearly title, as it did on 2026-08-18.
|
|
|
|
|
assert "traefik_entrypoint_requests_total" not in availability_expr
|
2026-08-16 21:55:28 -03:00
|
|
|
assert "atlas:availability:failures_1d" not in availability_expr
|
|
|
|
|
assert "atlas:availability:requests_1d" not in availability_expr
|
|
|
|
|
assert "sum_over_time" not in availability_expr
|
|
|
|
|
assert "kube_node_status_condition" not in availability_expr
|
|
|
|
|
assert "kube_deployment_status_replicas_available" not in availability_expr
|
|
|
|
|
assert panel["targets"][0]["instant"] is True
|
|
|
|
|
assert "Every server-side 5xx" in panel["description"]
|
fix(monitoring): measure Atlas availability honestly across telemetry gaps
The 2026-08-18 metrics-storage outage exposed two defects in the
availability pipeline that distorted the figure in opposite directions at
once.
The Overview panel fell back to a live one-hour Traefik ratio whenever the
yearly rollup sample went stale for 48h, and rendered it under the same
"365d" title. When the rollup stopped publishing on 2026-08-18 the panel
quietly swapped a 365-day measurement for a 60-minute one and read 99.74%
instead of the recorded 99.95%. The fallback is removed: a stale rollup now
renders no value, and a new atlas-availability-rollup-stale alert pages at
26h, well before the panel goes blank at 48h.
The yearly ratio also silently excluded the 34-hour telemetry gap, because
missing days contribute zero requests and zero failures. Absent data was
read as "nothing happened" — had Atlas genuinely been down in that window,
the figure would still have said 99.95%. Availability keeps its
measured-days-only definition, which is correct, but coverage is now
published alongside it and shown in a new panel, so a telemetry gap lowers
disclosed coverage instead of vanishing. The title reads "365d window" to
stop implying 365 days of data exist; request-v4 begins 2026-05-01.
The rollup job reported healthy runs across a day and a half of lost
publishes: a read-only VictoriaMetrics accepts an import and discards it.
It now reads each sample back and fails loudly when the write did not
survive.
Not addressed here: availability is still measured from inside the platform
via Traefik counters, so it cannot distinguish "Atlas down" from "telemetry
down", and misses failures that never reach Traefik (DNS, TLS, node dead).
An external synthetic prober is the real fix and needs a hosting decision.
2026-08-20 02:03:34 +00:00
|
|
|
assert "never converted into downtime or uptime" in panel["description"]
|
|
|
|
|
|
|
|
|
|
coverage = panels_by_id[36]
|
|
|
|
|
assert coverage["title"] == "Availability Coverage (days)"
|
|
|
|
|
assert (
|
|
|
|
|
coverage["targets"][0]["expr"]
|
|
|
|
|
== 'last_over_time(atlas:availability:coverage_days_365d{scope="atlas",definition="request-v4"}[48h])'
|
|
|
|
|
)
|
|
|
|
|
assert "lowers this number instead of moving availability" in coverage["description"]
|
2026-08-16 21:55:28 -03:00
|
|
|
|
|
|
|
|
def test_overview_uses_readable_quality_power_and_gitops_panels():
|
|
|
|
|
mod = load_module()
|
|
|
|
|
dashboard = mod.build_overview()
|
|
|
|
|
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
|
|
|
|
|
|
|
|
|
|
assert dashboard["links"] == [
|
|
|
|
|
{"title": "Atlas Testing", "url": "/d/atlas-testing", "targetBlank": True}
|
|
|
|
|
]
|
|
|
|
|
assert "atlas-jobs" not in repr(dashboard)
|
|
|
|
|
assert "Platform Test Success Rate" not in panels_by_title
|
|
|
|
|
assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
|
|
|
|
|
assert panels_by_title["Test Category Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 13}
|
|
|
|
|
assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
|
|
|
|
|
assert panels_by_title["Test Category Health"]["targets"][0]["format"] == "time_series"
|
|
|
|
|
assert panels_by_title["Test Category Health"]["targets"][0]["range"] is True
|
|
|
|
|
assert "${overview_suite:regex}" not in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
|
|
|
|
assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
|
|
|
|
assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
|
|
|
|
assert panels_by_title["Test Category Health"]["timeFrom"] == "24h"
|
|
|
|
|
assert f'category=~"{mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
|
|
|
|
assert "manual" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
|
|
|
|
|
assert "unit" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
|
|
|
|
|
assert panels_by_title["UPS History (Power Draw)"]["gridPos"] == {"h": 6, "w": 6, "x": 3, "y": 7}
|
|
|
|
|
assert panels_by_title["Ariadne Run Volume"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 7}
|
|
|
|
|
assert panels_by_title["Pyrphoros UPS Current"]["gridPos"]["w"] == 3
|
|
|
|
|
assert panels_by_title["Current Enclosure Climate"]["gridPos"]["w"] == 3
|
|
|
|
|
assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["placement"] == "bottom"
|
|
|
|
|
assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["displayMode"] == "list"
|
|
|
|
|
assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "line"
|
|
|
|
|
assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["fillOpacity"] == 22
|
|
|
|
|
assert all(target["expr"].startswith("max(") for target in panels_by_title["UPS History (Power Draw)"]["targets"])
|
|
|
|
|
ups_overrides = panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["overrides"]
|
|
|
|
|
ups_override_by_name = {override["matcher"]["options"]: override for override in ups_overrides}
|
|
|
|
|
assert ups_override_by_name["Pyrphoros"]["properties"] == [
|
|
|
|
|
{"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-blue"}},
|
|
|
|
|
]
|
|
|
|
|
assert ups_override_by_name["Statera"]["properties"] == [
|
|
|
|
|
{"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-yellow"}},
|
|
|
|
|
]
|
|
|
|
|
assert panels_by_title["Ariadne Run Volume"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
|
|
|
|
|
assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["placement"] == "bottom"
|
|
|
|
|
assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["displayMode"] == "list"
|
|
|
|
|
assert "Fan History (0-10)" not in panels_by_title
|
|
|
|
|
assert panels_by_title["Fan Intensity History"]["type"] == "state-timeline"
|
|
|
|
|
assert panels_by_title["Fan Intensity History"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 13}
|
|
|
|
|
assert panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["max"] == 10
|
|
|
|
|
assert panels_by_title["Fan Intensity History"]["targets"][0]["legendFormat"] == "{{fan}}"
|
|
|
|
|
fan_steps = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["thresholds"]["steps"]
|
|
|
|
|
assert [step["value"] for step in fan_steps] == [None, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
|
|
|
|
assert fan_steps[0]["color"] == "#8f1d1d"
|
|
|
|
|
assert fan_steps[5]["color"] == "#d4b106"
|
|
|
|
|
assert fan_steps[-1]["color"] == "#1f60c4"
|
|
|
|
|
fan_mappings = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["mappings"][0]["options"]
|
|
|
|
|
assert fan_mappings["0"]["text"] == "Off"
|
|
|
|
|
assert fan_mappings["10"]["text"] == "10"
|
|
|
|
|
assert "Tent Interior" in panels_by_title["Fan Intensity History"]["targets"][0]["expr"]
|
|
|
|
|
assert panels_by_title["Fan Intensity History"]["options"]["legend"]["displayMode"] == "list"
|
|
|
|
|
assert panels_by_title["Fan Intensity History"]["options"]["legend"]["showLegend"] is False
|
|
|
|
|
assert panels_by_title["Fan Intensity History"]["options"]["mergeValues"] is False
|
|
|
|
|
assert panels_by_title["Fan Intensity History"]["options"]["showValue"] == "auto"
|
|
|
|
|
|
|
|
|
|
assert panels_by_title["Flux Source"]["type"] == "stat"
|
|
|
|
|
assert panels_by_title["Flux Source"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 7}
|
|
|
|
|
assert panels_by_title["Flux Source"]["targets"][0]["legendFormat"] == "{{branch}}"
|
|
|
|
|
assert panels_by_title["Current Gate Health"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 9}
|
|
|
|
|
assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Current Gate Health"]["targets"][0]["expr"]
|
|
|
|
|
assert panels_by_title["CI Run Success (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 11}
|
|
|
|
|
assert panels_by_title["Suites With Runs (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 15}
|
|
|
|
|
suites_reporting_expr = panels_by_title["Suites With Runs (24h)"]["targets"][0]["expr"]
|
|
|
|
|
assert "> bool 0" in suites_reporting_expr
|
|
|
|
|
assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in suites_reporting_expr
|
|
|
|
|
assert "bstein-home" not in suites_reporting_expr
|
|
|
|
|
assert "published quality-gate run" in panels_by_title["Suites With Runs (24h)"]["description"]
|
|
|
|
|
assert panels_by_title["Avg Coverage"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 17}
|
|
|
|
|
assert "LOC Clean Suites" not in panels_by_title
|
|
|
|
|
assert panels_by_title["GitOps Health"]["type"] == "state-timeline"
|
|
|
|
|
assert panels_by_title["GitOps Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 7}
|
|
|
|
|
gitops_expr = panels_by_title["GitOps Health"]["targets"][0]["expr"]
|
|
|
|
|
assert "Kustomizations Not Suspended" in gitops_expr
|
|
|
|
|
assert "HelmReleases Not Suspended" in gitops_expr
|
|
|
|
|
assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
|
|
|
|
|
assert panels_by_title["Test Category Health"]["options"]["legend"]["showLegend"] is False
|
|
|
|
|
assert panels_by_title["Test Category Health"]["options"]["mergeValues"] is False
|
|
|
|
|
assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto"
|
|
|
|
|
assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9
|
|
|
|
|
assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
|
|
|
|
|
assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
|
|
|
|
|
assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"])
|
|
|
|
|
|
|
|
|
|
pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"]
|
|
|
|
|
assert "backup-telemetry-missing" in pvc_backup_expr
|
|
|
|
|
assert 'pvc_backup_(count|last_success_timestamp_seconds|health_reason)' in pvc_backup_expr
|
|
|
|
|
|
|
|
|
|
gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
|
|
|
|
|
assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
|
|
|
|
|
assert "nvidia_gpu_device_utilization_percent" in gpu_expr
|
|
|
|
|
assert "jetson_gr3d_active_seconds_total" in gpu_expr
|
|
|
|
|
assert "increase" in gpu_expr
|
|
|
|
|
assert "sum_over_time" not in gpu_expr
|
|
|
|
|
assert "count_over_time" not in gpu_expr
|
|
|
|
|
assert "avg_over_time" in gpu_expr
|
|
|
|
|
assert "$__range" in gpu_expr
|
|
|
|
|
assert "$__range_s" in gpu_expr
|
|
|
|
|
assert "sum by (namespace)" in gpu_expr
|
|
|
|
|
assert 'namespace", "shared"' not in gpu_expr
|
|
|
|
|
assert "kube_pod_container_resource_requests" in gpu_expr
|
|
|
|
|
assert mod.GPU_RESOURCE_REGEX in gpu_expr
|
|
|
|
|
assert '"gpu_source", "nvidia"' in gpu_expr
|
|
|
|
|
assert '"gpu_source", "jetson"' in gpu_expr
|
|
|
|
|
assert "100 *" in gpu_expr
|
|
|
|
|
assert "100 -" not in gpu_expr
|
|
|
|
|
assert 'namespace", "unattributed"' in gpu_expr
|
|
|
|
|
assert 'namespace", "idle"' in gpu_expr
|
|
|
|
|
assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
|
|
|
|
|
assert "selected time range" in panels_by_title["Namespace GPU Utilization"]["description"]
|
|
|
|
|
assert "continuously sampled" in panels_by_title["Namespace GPU Utilization"]["description"]
|
|
|
|
|
assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
|
|
|
|
|
|
|
|
|
|
def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
|
|
|
|
|
mod = load_module()
|
|
|
|
|
dashboard = mod.build_gpu_dashboard()
|
|
|
|
|
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
|
|
|
|
|
|
|
|
|
|
node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"]
|
|
|
|
|
assert "nvidia_gpu_device_utilization_percent" in node_expr
|
|
|
|
|
assert "DCGM_FI_DEV_GPU_UTIL" in node_expr
|
|
|
|
|
assert "jetson_gr3d_freq_percent" in node_expr
|
|
|
|
|
assert "unless on(node)" in node_expr
|
|
|
|
|
assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
|
|
|
|
|
|
|
|
|
|
process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"]
|
|
|
|
|
assert "nvidia_process_gpu_sm_util_percent" in process_expr
|
|
|
|
|
assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
|
|
|
|
|
|
|
|
|
|
def test_overview_and_testing_panels_all_have_concise_descriptions():
|
|
|
|
|
mod = load_module()
|
|
|
|
|
|
|
|
|
|
for dashboard in [mod.build_overview(), mod.build_jobs_dashboard(), mod.build_testing_dashboard()]:
|
|
|
|
|
panels = flatten_panels(dashboard["panels"])
|
|
|
|
|
assert all(panel.get("description") for panel in panels if panel["type"] != "row")
|
|
|
|
|
assert "85.7" not in repr(dashboard)
|
|
|
|
|
|
|
|
|
|
def test_render_configmap_writes(tmp_path):
|
|
|
|
|
mod = load_module()
|
|
|
|
|
mod.DASHBOARD_DIR = tmp_path / "dash"
|
|
|
|
|
mod.ROOT = tmp_path
|
|
|
|
|
uid = "atlas-test"
|
|
|
|
|
info = {"configmap": tmp_path / "cm.yaml"}
|
|
|
|
|
data = {"title": "Atlas Test"}
|
|
|
|
|
mod.write_json(uid, data)
|
|
|
|
|
mod.render_configmap(uid, info)
|
|
|
|
|
json_path = mod.DASHBOARD_DIR / f"{uid}.json"
|
|
|
|
|
assert json_path.exists()
|
|
|
|
|
content = (tmp_path / "cm.yaml").read_text()
|
|
|
|
|
assert "kind: ConfigMap" in content
|
|
|
|
|
assert f"{uid}.json" in content
|