monitoring: retain daily availability through retries

scripts/dashboards_render_atlas.py

@@ -472,7 +472,7 @@ UPTIME_LIVE_FALLBACK_EXPR = (
     f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))"
 )
 UPTIME_RECORDING_EXPR = (
-    f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) "
+    f"(last_over_time({UPTIME_RECORDING_METRIC}[48h]) "
     f"or on() {UPTIME_LIVE_FALLBACK_EXPR})"
 )
 
@@ -2142,7 +2142,7 @@ def build_overview():
             "decimals": 4,
             "text_mode": "value",
             "instant": True,
-            "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 24 hours and only falls back to the same one-hour request SLI before history exists.",
+            "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 48 hours so one delayed retry cannot cause a fallback, and only uses the same one-hour request SLI before history exists.",
         },
         {
             "id": 4,

scripts/tests/test_dashboards_render_atlas.py

@@ -60,7 +60,7 @@ def test_overview_availability_panel_uses_recorded_365d_rollup():
     assert panel["title"] == "Atlas Availability (365d)"
     availability_expr = panel["targets"][0]["expr"]
     assert (
-        'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[24h])'
+        'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])'
         in availability_expr
     )
     assert 'code=~"5.."' in availability_expr
@@ -74,6 +74,7 @@ def test_overview_availability_panel_uses_recorded_365d_rollup():
     assert "Every server-side 5xx" in panel["description"]
     assert "Replica counts, Grafana health" in panel["description"]
     assert "daily rollup job publishes one annual sample" in panel["description"]
+    assert "keeps it for up to 48 hours" in panel["description"]
 
 
 def test_overview_uses_readable_quality_power_and_gitops_panels():

services/monitoring/dashboards/atlas-overview.json

@@ -229,7 +229,7 @@
       },
       "targets": [
         {
-          "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
+          "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[48h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
           "refId": "A",
           "instant": true
         }
@@ -286,7 +286,7 @@
         },
         "textMode": "value"
       },
-      "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 24 hours and only falls back to the same one-hour request SLI before history exists."
+      "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 48 hours so one delayed retry cannot cause a fallback, and only uses the same one-hour request SLI before history exists."
     },
     {
       "id": 4,

services/monitoring/grafana-dashboard-overview.yaml

@@ -238,7 +238,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
+              "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[48h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
               "refId": "A",
               "instant": true
             }
@@ -295,7 +295,7 @@ data:
             },
             "textMode": "value"
           },
-          "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 24 hours and only falls back to the same one-hour request SLI before history exists."
+          "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 48 hours so one delayed retry cannot cause a fallback, and only uses the same one-hour request SLI before history exists."
         },
         {
           "id": 4,