monitoring: add daily availability rollups

scripts/dashboards_render_atlas.py

@@ -472,9 +472,9 @@ UPTIME_LIVE_FALLBACK_EXPR = (
     f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))"
 )
 UPTIME_COMPACT_FALLBACK_EXPR = (
-    "(1 - (sum_over_time(atlas:availability:failures_1h{"
+    "(1 - (sum_over_time(atlas:availability:failures_1d{"
     'scope="atlas",definition="request-v4"}[365d]) / '
-    "clamp_min(sum_over_time(atlas:availability:requests_1h{"
+    "clamp_min(sum_over_time(atlas:availability:requests_1d{"
     'scope="atlas",definition="request-v4"}[365d]), 1)))'
 )
 UPTIME_RECORDING_EXPR = (

scripts/tests/test_dashboards_render_atlas.py

@@ -65,8 +65,8 @@ def test_overview_availability_panel_uses_recorded_365d_rollup():
     )
     assert 'code=~"5.."' in availability_expr
     assert 'code=~"[1-5].."' in availability_expr
-    assert "atlas:availability:failures_1h" in availability_expr
+    assert "atlas:availability:failures_1d" in availability_expr
-    assert "atlas:availability:requests_1h" in availability_expr
+    assert "atlas:availability:requests_1d" in availability_expr
     assert "sum_over_time" in availability_expr
     assert "kube_node_status_condition" not in availability_expr
     assert "kube_deployment_status_replicas_available" not in availability_expr

scripts/tests/test_monitoring_query_capacity.py

@@ -57,8 +57,8 @@ def test_yearly_availability_reuses_the_hourly_rollup() -> None:
         if rule["record"] == "atlas:availability:ratio_365d"
     )
 
-    assert "atlas:availability:requests_1h" in yearly["expr"]
+    assert "atlas:availability:requests_1d" in yearly["expr"]
-    assert "atlas:availability:failures_1h" in yearly["expr"]
+    assert "atlas:availability:failures_1d" in yearly["expr"]
     assert 'definition="request-v4"' in yearly["expr"]
     assert "sum_over_time" in yearly["expr"]
     assert "[365d]" in yearly["expr"]
@@ -66,6 +66,23 @@ def test_yearly_availability_reuses_the_hourly_rollup() -> None:
     assert yearly["labels"]["definition"] == "request-v4"
 
 
+def test_daily_availability_rollups_use_the_same_request_sli() -> None:
+    """Keep annual availability cheap without changing its request definition."""
+    manifest = _documents(
+        REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
+    )[0]
+    groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
+    daily_group = next(
+        group for group in groups if group["name"] == "atlas.availability.rollup"
+    )
+    daily = {rule["record"]: rule for rule in daily_group["rules"]}
+
+    assert daily_group["interval"] == "1d"
+    assert 'code=~"[1-5].."' in daily["atlas:availability:requests_1d"]["expr"]
+    assert 'code=~"5.."' in daily["atlas:availability:failures_1d"]["expr"]
+    assert all(rule["labels"]["definition"] == "request-v4" for rule in daily.values())
+
+
 def test_availability_uses_request_failures_instead_of_replica_capacity() -> None:
     """Measure HTTP outcomes without treating redundant replica loss as downtime."""
     manifest = _documents(
@@ -116,6 +133,24 @@ def test_availability_backfill_replays_the_same_request_sli() -> None:
     assert "-replay.maxDatapointsPerQuery=48" in args
 
 
+def test_daily_availability_backfill_stays_below_query_timeout() -> None:
+    """Replay daily buckets in small chunks so raw history never starves Grafana."""
+    manifests = _documents(
+        REPO_ROOT / "services/monitoring/availability-daily-backfill-v4-job.yaml"
+    )
+    config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap")
+    job = next(manifest for manifest in manifests if manifest["kind"] == "Job")
+    backfill = yaml.safe_load(config["data"]["atlas-request-daily-history.yaml"])
+    group = backfill["groups"][0]
+    expressions = {rule["record"]: rule["expr"] for rule in group["rules"]}
+
+    assert group["interval"] == "1d"
+    assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1d"]
+    assert 'code=~"5.."' in expressions["atlas:availability:failures_1d"]
+    args = job["spec"]["template"]["spec"]["containers"][0]["args"]
+    assert "-replay.maxDatapointsPerQuery=4" in args
+
+
 def test_quality_rollups_do_not_run_every_minute() -> None:
     """Keep high-cardinality quality rollups below the backend saturation cadence."""
     manifest = _documents(

services/monitoring/availability-daily-backfill-v4-job.yaml

@@ -0,0 +1,92 @@
+# services/monitoring/availability-daily-backfill-v4-job.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: atlas-availability-request-v4-daily-backfill-rules
+  namespace: monitoring
+data:
+  atlas-request-daily-history.yaml: |
+    groups:
+      - name: atlas.availability.request.daily.backfill
+        interval: 1d
+        rules:
+          - record: atlas:availability:requests_1d
+            expr: |
+              sum(increase(
+                traefik_entrypoint_requests_total{
+                  entrypoint="websecure",
+                  protocol="http",
+                  code=~"[1-5].."
+                }[1d]
+              ))
+            labels:
+              definition: request-v4
+              scope: atlas
+              rollup: daily
+          - record: atlas:availability:failures_1d
+            expr: |
+              sum(increase(
+                traefik_entrypoint_requests_total{
+                  entrypoint="websecure",
+                  protocol="http",
+                  code=~"5.."
+                }[1d]
+              ))
+            labels:
+              definition: request-v4
+              scope: atlas
+              rollup: daily
+
+---
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: atlas-availability-request-v4-daily-backfill
+  namespace: monitoring
+spec:
+  backoffLimit: 2
+  template:
+    metadata:
+      labels:
+        app: atlas-availability-request-v4-daily-backfill
+    spec:
+      restartPolicy: Never
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values:
+                      - titan-22
+                      - titan-24
+      containers:
+        - name: vmalert-replay
+          image: victoriametrics/vmalert:v1.113.0
+          args:
+            - -datasource.url=http://victoria-metrics-single-server:8428
+            - -remoteWrite.url=http://victoria-metrics-single-server:8428
+            - -remoteWrite.flushInterval=1s
+            - -rule=/etc/vmalert/backfill/*.yaml
+            - -replay.timeFrom=2026-05-01T00:00:00Z
+            - -replay.timeTo=2026-08-04T00:00:00Z
+            - -replay.maxDatapointsPerQuery=4
+            - -replay.rulesDelay=2s
+            - -replay.disableProgressBar
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: "1"
+              memory: 512Mi
+          volumeMounts:
+            - name: rules
+              mountPath: /etc/vmalert/backfill
+              readOnly: true
+      volumes:
+        - name: rules
+          configMap:
+            name: atlas-availability-request-v4-daily-backfill-rules

services/monitoring/dashboards/atlas-overview.json

@@ -229,7 +229,7 @@
       },
       "targets": [
         {
-          "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
+          "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
           "refId": "A",
           "instant": true
         }

services/monitoring/grafana-dashboard-overview.yaml

@@ -238,7 +238,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
+              "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
               "refId": "A",
               "instant": true
             }

services/monitoring/kustomization.yaml

@@ -20,6 +20,7 @@ resources:
   - grafana-dashboard-testing.yaml
   - vmalert-atlas-availability.yaml
   - availability-backfill-v4-job.yaml
+  - availability-daily-backfill-v4-job.yaml
   - dcgm-exporter.yaml
   - nvidia-process-exporter.yaml
   - jetson-tegrastats-exporter.yaml

services/monitoring/vmalert-atlas-availability.yaml

@@ -38,6 +38,36 @@ data:
               scope: atlas
               rollup: hourly
       - name: atlas.availability.rollup
+        interval: 1d
+        eval_offset: 23h59m
+        rules:
+          - record: atlas:availability:requests_1d
+            expr: |
+              sum(increase(
+                traefik_entrypoint_requests_total{
+                  entrypoint="websecure",
+                  protocol="http",
+                  code=~"[1-5].."
+                }[1d]
+              ))
+            labels:
+              definition: request-v4
+              scope: atlas
+              rollup: daily
+          - record: atlas:availability:failures_1d
+            expr: |
+              sum(increase(
+                traefik_entrypoint_requests_total{
+                  entrypoint="websecure",
+                  protocol="http",
+                  code=~"5.."
+                }[1d]
+              ))
+            labels:
+              definition: request-v4
+              scope: atlas
+              rollup: daily
+      - name: atlas.availability.annual
         interval: 15m
         eval_offset: 14m
         rules:
@@ -45,7 +75,7 @@ data:
             expr: |
               1 - (
                 sum_over_time(
-                  atlas:availability:failures_1h{
+                  atlas:availability:failures_1d{
                     scope="atlas",
                     definition="request-v4"
                   }[365d]
@@ -53,7 +83,7 @@ data:
                 /
                 clamp_min(
                   sum_over_time(
-                    atlas:availability:requests_1h{
+                    atlas:availability:requests_1d{
                       scope="atlas",
                       definition="request-v4"
                     }[365d]