diff --git a/scripts/dashboards_render_atlas.py b/scripts/dashboards_render_atlas.py index eb46a4df8..bddddf46c 100644 --- a/scripts/dashboards_render_atlas.py +++ b/scripts/dashboards_render_atlas.py @@ -459,21 +459,23 @@ UPTIME_WINDOW = "365d" UPTIME_RECORDING_METRIC = ( f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="serving-v2"}}' ) -UPTIME_RECORDING_EXPR = f"last_over_time({UPTIME_RECORDING_METRIC}[24h])" TRAEFIK_READY_EXPR = ( "(" 'sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})' - " / clamp_min(" - 'sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)' - ")" + " > bool 0)" ) CONTROL_READY_FRACTION_EXPR = ( f"(sum(kube_node_status_condition{{condition=\"Ready\",status=\"true\",node=~\"{CONTROL_REGEX}\"}})" - f" / {CONTROL_TOTAL})" + " >= bool 2)" ) UPTIME_AVAIL_EXPR = ( f"min(({CONTROL_READY_FRACTION_EXPR}), ({TRAEFIK_READY_EXPR}))" ) +UPTIME_LIVE_FALLBACK_EXPR = f"avg_over_time(({UPTIME_AVAIL_EXPR})[1h:5m])" +UPTIME_RECORDING_EXPR = ( + f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) " + f"or on() {UPTIME_LIVE_FALLBACK_EXPR})" +) # Tie-breaker to deterministically pick one node per namespace when shares tie. NODE_TIEBREAKER = " + ".join( @@ -2141,7 +2143,7 @@ def build_overview(): "decimals": 4, "text_mode": "value", "instant": True, - "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Partial replica capacity remains available; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h.", + "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Partial replica capacity remains available; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available.", }, { "id": 4, diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index a38c063c0..88e883242 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -58,13 +58,19 @@ def test_overview_availability_panel_uses_recorded_365d_rollup(): panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27) assert panel["title"] == "Atlas Serving Availability" - assert panel["targets"][0]["expr"] == ( + availability_expr = panel["targets"][0]["expr"] + assert ( 'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="serving-v2"}[24h])' + in availability_expr ) + assert ">= bool 2" in availability_expr + assert "> bool 0" in availability_expr + assert "[1h:5m]" in availability_expr assert panel["targets"][0]["instant"] is True assert "at least one Traefik replica serving" in panel["description"] assert "Partial replica capacity remains available" in panel["description"] assert "monitoring gaps are excluded" in panel["description"] + assert "falls back to the live binary serving state" in panel["description"] def test_overview_uses_readable_quality_power_and_gitops_panels(): diff --git a/services/monitoring/dashboards/atlas-overview.json b/services/monitoring/dashboards/atlas-overview.json index 03e580be0..5c2d406f0 100644 --- a/services/monitoring/dashboards/atlas-overview.json +++ b/services/monitoring/dashboards/atlas-overview.json @@ -229,7 +229,7 @@ }, "targets": [ { - "expr": "last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h])", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h]) or on() avg_over_time((min(((sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-0a|titan-0b|titan-0c\"}) >= bool 2)), ((sum(kube_deployment_status_replicas_available{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}) > bool 0))))[1h:5m]))", "refId": "A", "instant": true } @@ -286,7 +286,7 @@ }, "textMode": "value" }, - "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Partial replica capacity remains available; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h." + "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Partial replica capacity remains available; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available." }, { "id": 4, diff --git a/services/monitoring/grafana-dashboard-overview.yaml b/services/monitoring/grafana-dashboard-overview.yaml index b9f869a38..827d412b5 100644 --- a/services/monitoring/grafana-dashboard-overview.yaml +++ b/services/monitoring/grafana-dashboard-overview.yaml @@ -238,7 +238,7 @@ data: }, "targets": [ { - "expr": "last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h])", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h]) or on() avg_over_time((min(((sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-0a|titan-0b|titan-0c\"}) >= bool 2)), ((sum(kube_deployment_status_replicas_available{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}) > bool 0))))[1h:5m]))", "refId": "A", "instant": true } @@ -295,7 +295,7 @@ data: }, "textMode": "value" }, - "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Partial replica capacity remains available; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h." + "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Partial replica capacity remains available; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available." }, { "id": 4,