monitoring: fall back to live serving state

scripts/dashboards_render_atlas.py

@@ -459,21 +459,23 @@ UPTIME_WINDOW = "365d"
 UPTIME_RECORDING_METRIC = (
     f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="serving-v2"}}'
 )
-UPTIME_RECORDING_EXPR = f"last_over_time({UPTIME_RECORDING_METRIC}[24h])"
 TRAEFIK_READY_EXPR = (
     "("
     'sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})'
-    " / clamp_min("
+    " > bool 0)"
-    'sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)'
-    ")"
 )
 CONTROL_READY_FRACTION_EXPR = (
     f"(sum(kube_node_status_condition{{condition=\"Ready\",status=\"true\",node=~\"{CONTROL_REGEX}\"}})"
-    f" / {CONTROL_TOTAL})"
+    " >= bool 2)"
 )
 UPTIME_AVAIL_EXPR = (
     f"min(({CONTROL_READY_FRACTION_EXPR}), ({TRAEFIK_READY_EXPR}))"
 )
+UPTIME_LIVE_FALLBACK_EXPR = f"avg_over_time(({UPTIME_AVAIL_EXPR})[1h:5m])"
+UPTIME_RECORDING_EXPR = (
+    f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) "
+    f"or on() {UPTIME_LIVE_FALLBACK_EXPR})"
+)
 
 # Tie-breaker to deterministically pick one node per namespace when shares tie.
 NODE_TIEBREAKER = " + ".join(
@@ -2141,7 +2143,7 @@ def build_overview():
             "decimals": 4,
             "text_mode": "value",
             "instant": True,
-            "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Partial replica capacity remains available; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h.",
+            "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Partial replica capacity remains available; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available.",
         },
         {
             "id": 4,

scripts/tests/test_dashboards_render_atlas.py

@@ -58,13 +58,19 @@ def test_overview_availability_panel_uses_recorded_365d_rollup():
     panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27)
 
     assert panel["title"] == "Atlas Serving Availability"
-    assert panel["targets"][0]["expr"] == (
+    availability_expr = panel["targets"][0]["expr"]
+    assert (
         'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="serving-v2"}[24h])'
+        in availability_expr
     )
+    assert ">= bool 2" in availability_expr
+    assert "> bool 0" in availability_expr
+    assert "[1h:5m]" in availability_expr
     assert panel["targets"][0]["instant"] is True
     assert "at least one Traefik replica serving" in panel["description"]
     assert "Partial replica capacity remains available" in panel["description"]
     assert "monitoring gaps are excluded" in panel["description"]
+    assert "falls back to the live binary serving state" in panel["description"]
 
 
 def test_overview_uses_readable_quality_power_and_gitops_panels():

services/monitoring/dashboards/atlas-overview.json

@@ -229,7 +229,7 @@
       },
       "targets": [
         {
-          "expr": "last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h])",
+          "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h]) or on() avg_over_time((min(((sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-0a|titan-0b|titan-0c\"}) >= bool 2)), ((sum(kube_deployment_status_replicas_available{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}) > bool 0))))[1h:5m]))",
           "refId": "A",
           "instant": true
         }
@@ -286,7 +286,7 @@
         },
         "textMode": "value"
       },
-      "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Partial replica capacity remains available; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h."
+      "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Partial replica capacity remains available; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available."
     },
     {
       "id": 4,

services/monitoring/grafana-dashboard-overview.yaml

@@ -238,7 +238,7 @@ data:
           },
           "targets": [
             {
-              "expr": "last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h])",
+              "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h]) or on() avg_over_time((min(((sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-0a|titan-0b|titan-0c\"}) >= bool 2)), ((sum(kube_deployment_status_replicas_available{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}) > bool 0))))[1h:5m]))",
               "refId": "A",
               "instant": true
             }
@@ -295,7 +295,7 @@ data:
             },
             "textMode": "value"
           },
-          "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Partial replica capacity remains available; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h."
+          "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Partial replica capacity remains available; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available."
         },
         {
           "id": 4,