monitoring: publish availability outside query pool

2026-08-04 21:40:24 -03:00 · 2026-08-04 21:40:24 -03:00 · 2691581cbc
commit 2691581cbc
parent a8dda9815d
10 changed files with 293 additions and 57 deletions
--- a/scripts/dashboards_render_atlas.py
+++ b/scripts/dashboards_render_atlas.py
@ -471,15 +471,8 @@ UPTIME_LIVE_FALLBACK_EXPR = (
    f"(1 - (({AVAILABILITY_FAILURES_1H_EXPR} or on() vector(0)) / "
    f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))"
 )
-UPTIME_COMPACT_FALLBACK_EXPR = (
-    "(1 - (sum_over_time(atlas:availability:failures_1d{"
-    'scope="atlas",definition="request-v4"}[365d]) / '
-    "clamp_min(sum_over_time(atlas:availability:requests_1d{"
-    'scope="atlas",definition="request-v4"}[365d]), 1)))'
-)
 UPTIME_RECORDING_EXPR = (
    f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) "
-    f"or on() {UPTIME_COMPACT_FALLBACK_EXPR} "
    f"or on() {UPTIME_LIVE_FALLBACK_EXPR})"
 )

@ -2149,7 +2142,7 @@ def build_overview():
            "decimals": 4,
            "text_mode": "value",
            "instant": True,
-            "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful annual sample for up to 24 hours, can rebuild it from compact hourly rollups, and only falls back to the same one-hour request SLI before history exists.",
+            "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 24 hours and only falls back to the same one-hour request SLI before history exists.",
        },
        {
            "id": 4,
--- a/scripts/tests/test_availability_rollup.py
+++ b/scripts/tests/test_availability_rollup.py
@ -0,0 +1,52 @@
+"""Unit tests for the Atlas availability publisher."""
+
+import importlib.util
+import json
+from pathlib import Path
+
+import pytest
+
+
+def load_module():
+    """Load the service-owned rollup module without packaging it."""
+    path = (
+        Path(__file__).resolve().parents[2]
+        / "services/monitoring/scripts/availability_rollup.py"
+    )
+    spec = importlib.util.spec_from_file_location("availability_rollup", path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_parse_export_deduplicates_replay_boundaries() -> None:
+    """Keep only the final value when replay chunks share a timestamp."""
+    mod = load_module()
+    lines = [
+        (json.dumps({"timestamps": [1000, 2000], "values": [2, 3]}) + "\n").encode(),
+        (json.dumps({"timestamps": [2000, 3000], "values": [3, 5]}) + "\n").encode(),
+    ]
+
+    assert mod.parse_export(lines) == {1000: 2.0, 2000: 3.0, 3000: 5.0}
+
+
+def test_calculate_availability_uses_all_server_failures() -> None:
+    """Calculate one bounded successful-request ratio."""
+    mod = load_module()
+
+    assert mod.calculate_availability(1000, 2) == pytest.approx(0.998)
+    with pytest.raises(ValueError):
+        mod.calculate_availability(0, 0)
+    with pytest.raises(ValueError):
+        mod.calculate_availability(100, -1)
+
+
+def test_render_metric_publishes_only_the_request_v4_series() -> None:
+    """Render the single series selected by the Grafana panel."""
+    mod = load_module()
+
+    assert mod.render_metric(0.9995, 1234) == (
+        'atlas:availability:ratio_365d{definition="request-v4",scope="atlas",'
+        'rollup="yearly"} 0.999500000000 1234\n'
+    )
--- a/scripts/tests/test_dashboards_render_atlas.py
+++ b/scripts/tests/test_dashboards_render_atlas.py
@ -65,15 +65,15 @@ def test_overview_availability_panel_uses_recorded_365d_rollup():
    )
    assert 'code=~"5.."' in availability_expr
    assert 'code=~"[1-5].."' in availability_expr
-    assert "atlas:availability:failures_1d" in availability_expr
-    assert "atlas:availability:requests_1d" in availability_expr
-    assert "sum_over_time" in availability_expr
+    assert "atlas:availability:failures_1d" not in availability_expr
+    assert "atlas:availability:requests_1d" not in availability_expr
+    assert "sum_over_time" not in availability_expr
    assert "kube_node_status_condition" not in availability_expr
    assert "kube_deployment_status_replicas_available" not in availability_expr
    assert panel["targets"][0]["instant"] is True
    assert "Every server-side 5xx" in panel["description"]
    assert "Replica counts, Grafana health" in panel["description"]
-    assert "can rebuild it from compact hourly rollups" in panel["description"]
+    assert "daily rollup job publishes one annual sample" in panel["description"]


 def test_overview_uses_readable_quality_power_and_gitops_panels():
--- a/scripts/tests/test_monitoring_query_capacity.py
+++ b/scripts/tests/test_monitoring_query_capacity.py
@ -44,26 +44,28 @@ def test_victoria_metrics_has_dashboard_burst_headroom() -> None:
    assert {"titan-14", "titan-18"} <= set(hostname_rule["values"])


-def test_yearly_availability_reuses_the_hourly_rollup() -> None:
-    """Prevent the yearly rule from rescanning raw cluster metrics for 365 days."""
+def test_yearly_availability_is_published_outside_the_query_pool() -> None:
+    """Keep all long-range availability work out of Grafana and MetricsQL."""
    manifest = _documents(
        REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
    )[0]
    groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
    rules = [rule for group in groups for rule in group["rules"]]
-    yearly = next(
-        rule
-        for rule in rules
-        if rule["record"] == "atlas:availability:ratio_365d"
+    manifests = _documents(
+        REPO_ROOT / "services/monitoring/availability-rollup-cronjob.yaml"
    )
+    cronjob = next(manifest for manifest in manifests if manifest["kind"] == "CronJob")
+    source = (
+        REPO_ROOT / "services/monitoring/scripts/availability_rollup.py"
+    ).read_text()

-    assert "atlas:availability:requests_1d" in yearly["expr"]
-    assert "atlas:availability:failures_1d" in yearly["expr"]
-    assert 'definition="request-v4"' in yearly["expr"]
-    assert "sum_over_time" in yearly["expr"]
-    assert "[365d]" in yearly["expr"]
-    assert "traefik_entrypoint_requests_total" not in yearly["expr"]
-    assert yearly["labels"]["definition"] == "request-v4"
+    assert all(rule["record"] != "atlas:availability:ratio_365d" for rule in rules)
+    assert "atlas:availability:requests_1d" in source
+    assert "atlas:availability:failures_1d" in source
+    assert "/api/v1/export" in source
+    assert "/api/v1/import/prometheus" in source
+    assert cronjob["spec"]["schedule"] == "10 0 * * *"
+    assert cronjob["spec"]["concurrencyPolicy"] == "Forbid"


 def test_daily_availability_rollups_use_the_same_request_sli() -> None:
--- a/services/monitoring/availability-rollup-cronjob.yaml
+++ b/services/monitoring/availability-rollup-cronjob.yaml
@ -0,0 +1,103 @@
+# services/monitoring/availability-rollup-cronjob.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: atlas-availability-request-v4-rollup-initial
+  namespace: monitoring
+spec:
+  activeDeadlineSeconds: 900
+  backoffLimit: 2
+  template:
+    metadata:
+      labels:
+        app: atlas-availability-request-v4-rollup
+    spec:
+      restartPolicy: Never
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values:
+                      - titan-22
+                      - titan-24
+      containers:
+        - name: rollup
+          image: python:3.12-alpine
+          command: ["python", "/scripts/availability_rollup.py"]
+          env:
+            - name: VM_URL
+              value: http://victoria-metrics-single-server:8428
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 500m
+              memory: 256Mi
+          volumeMounts:
+            - name: script
+              mountPath: /scripts
+              readOnly: true
+      volumes:
+        - name: script
+          configMap:
+            name: atlas-availability-rollup-script
+
+---
+
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: atlas-availability-request-v4-rollup
+  namespace: monitoring
+spec:
+  schedule: "10 0 * * *"
+  timeZone: Etc/UTC
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: 900
+      backoffLimit: 2
+      template:
+        metadata:
+          labels:
+            app: atlas-availability-request-v4-rollup
+        spec:
+          restartPolicy: Never
+          affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                  - matchExpressions:
+                      - key: kubernetes.io/hostname
+                        operator: NotIn
+                        values:
+                          - titan-22
+                          - titan-24
+          containers:
+            - name: rollup
+              image: python:3.12-alpine
+              command: ["python", "/scripts/availability_rollup.py"]
+              env:
+                - name: VM_URL
+                  value: http://victoria-metrics-single-server:8428
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 64Mi
+                limits:
+                  cpu: 500m
+                  memory: 256Mi
+              volumeMounts:
+                - name: script
+                  mountPath: /scripts
+                  readOnly: true
+          volumes:
+            - name: script
+              configMap:
+                name: atlas-availability-rollup-script
--- a/services/monitoring/dashboards/atlas-overview.json
+++ b/services/monitoring/dashboards/atlas-overview.json
@ -229,7 +229,7 @@
      },
      "targets": [
        {
-          "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
+          "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
          "refId": "A",
          "instant": true
        }
@ -286,7 +286,7 @@
        },
        "textMode": "value"
      },
-      "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful annual sample for up to 24 hours, can rebuild it from compact hourly rollups, and only falls back to the same one-hour request SLI before history exists."
+      "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 24 hours and only falls back to the same one-hour request SLI before history exists."
    },
    {
      "id": 4,
--- a/services/monitoring/grafana-dashboard-overview.yaml
+++ b/services/monitoring/grafana-dashboard-overview.yaml
@ -238,7 +238,7 @@ data:
          },
          "targets": [
            {
-              "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
+              "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
              "refId": "A",
              "instant": true
            }
@ -295,7 +295,7 @@ data:
            },
            "textMode": "value"
          },
-          "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful annual sample for up to 24 hours, can rebuild it from compact hourly rollups, and only falls back to the same one-hour request SLI before history exists."
+          "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 24 hours and only falls back to the same one-hour request SLI before history exists."
        },
        {
          "id": 4,
--- a/services/monitoring/kustomization.yaml
+++ b/services/monitoring/kustomization.yaml
@ -21,6 +21,7 @@ resources:
  - vmalert-atlas-availability.yaml
  - availability-backfill-v4-job.yaml
  - availability-daily-backfill-v4-job.yaml
+  - availability-rollup-cronjob.yaml
  - dcgm-exporter.yaml
  - nvidia-process-exporter.yaml
  - jetson-tegrastats-exporter.yaml
@ -68,3 +69,9 @@ configMapGenerator:
      - platform_quality_suite_probe.sh=scripts/platform_quality_suite_probe.sh
    options:
      disableNameSuffixHash: true
+  - name: atlas-availability-rollup-script
+    namespace: monitoring
+    files:
+      - availability_rollup.py=scripts/availability_rollup.py
+    options:
+      disableNameSuffixHash: true
--- a/services/monitoring/scripts/availability_rollup.py
+++ b/services/monitoring/scripts/availability_rollup.py
@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""Publish Atlas request availability from deduplicated daily rollups."""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from datetime import datetime, timedelta, timezone
+from typing import Iterable
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+
+VM_URL = os.environ.get(
+    "VM_URL", "http://victoria-metrics-single-server:8428"
+).rstrip("/")
+SCOPE = "atlas"
+DEFINITION = "request-v4"
+REQUESTS_METRIC = "atlas:availability:requests_1d"
+FAILURES_METRIC = "atlas:availability:failures_1d"
+OUTPUT_METRIC = "atlas:availability:ratio_365d"
+WINDOW_DAYS = 365
+
+
+def parse_export(lines: Iterable[bytes]) -> dict[int, float]:
+    """Return the last exported value for each timestamp."""
+    points: dict[int, float] = {}
+    for raw_line in lines:
+        if not raw_line.strip():
+            continue
+        series = json.loads(raw_line)
+        for timestamp, value in zip(series["timestamps"], series["values"], strict=True):
+            points[int(timestamp)] = float(value)
+    return points
+
+
+def fetch_rollup(metric: str, start: datetime, end: datetime) -> dict[int, float]:
+    """Stream one compact rollup series from VictoriaMetrics."""
+    matcher = (
+        f'{{__name__="{metric}",scope="{SCOPE}",definition="{DEFINITION}"}}'
+    )
+    query = urlencode(
+        {
+            "match[]": matcher,
+            "start": start.isoformat().replace("+00:00", "Z"),
+            "end": end.isoformat().replace("+00:00", "Z"),
+        }
+    )
+    with urlopen(f"{VM_URL}/api/v1/export?{query}", timeout=600) as response:
+        return parse_export(response)
+
+
+def calculate_availability(requests: float, failures: float) -> float:
+    """Calculate the bounded successful-request ratio."""
+    if requests <= 0:
+        raise ValueError("availability requires at least one observed request")
+    if failures < 0:
+        raise ValueError("failed request count cannot be negative")
+    return max(0.0, min(1.0, 1.0 - (failures / requests)))
+
+
+def render_metric(value: float, timestamp_ms: int) -> str:
+    """Render one VictoriaMetrics Prometheus-import sample."""
+    return (
+        f'{OUTPUT_METRIC}{{definition="{DEFINITION}",scope="{SCOPE}",'
+        f'rollup="yearly"}} {value:.12f} {timestamp_ms}\n'
+    )
+
+
+def publish(value: float, timestamp_ms: int) -> None:
+    """Write the calculated annual ratio to VictoriaMetrics."""
+    request = Request(
+        f"{VM_URL}/api/v1/import/prometheus",
+        data=render_metric(value, timestamp_ms).encode(),
+        headers={"Content-Type": "text/plain"},
+        method="POST",
+    )
+    with urlopen(request, timeout=30) as response:
+        if response.status not in {200, 204}:
+            raise RuntimeError(f"VictoriaMetrics import returned HTTP {response.status}")
+
+
+def main() -> None:
+    """Rebuild and publish the rolling request-availability sample."""
+    end = datetime.now(timezone.utc)
+    start = end - timedelta(days=WINDOW_DAYS)
+    requests = sum(fetch_rollup(REQUESTS_METRIC, start, end).values())
+    failures = sum(fetch_rollup(FAILURES_METRIC, start, end).values())
+    availability = calculate_availability(requests, failures)
+    timestamp_ms = time.time_ns() // 1_000_000
+    publish(availability, timestamp_ms)
+    print(
+        json.dumps(
+            {
+                "requests": requests,
+                "failures": failures,
+                "availability_percent": availability * 100,
+                "timestamp_ms": timestamp_ms,
+            },
+            sort_keys=True,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()
--- a/services/monitoring/vmalert-atlas-availability.yaml
+++ b/services/monitoring/vmalert-atlas-availability.yaml
@ -67,34 +67,6 @@ data:
              definition: request-v4
              scope: atlas
              rollup: daily
-      - name: atlas.availability.annual
-        interval: 15m
-        eval_offset: 14m
-        rules:
-          - record: atlas:availability:ratio_365d
-            expr: |
-              1 - (
-                sum_over_time(
-                  atlas:availability:failures_1d{
-                    scope="atlas",
-                    definition="request-v4"
-                  }[365d]
-                )
-                /
-                clamp_min(
-                  sum_over_time(
-                    atlas:availability:requests_1d{
-                      scope="atlas",
-                      definition="request-v4"
-                    }[365d]
-                  ),
-                  1
-                )
-              )
-            labels:
-              definition: request-v4
-              scope: atlas
-              rollup: yearly
  platform-quality.yaml: |
    groups:
      - name: platform.quality