diff --git a/scripts/dashboards_render_atlas.py b/scripts/dashboards_render_atlas.py index 5df8eb1bd..0ac01db9d 100644 --- a/scripts/dashboards_render_atlas.py +++ b/scripts/dashboards_render_atlas.py @@ -471,15 +471,8 @@ UPTIME_LIVE_FALLBACK_EXPR = ( f"(1 - (({AVAILABILITY_FAILURES_1H_EXPR} or on() vector(0)) / " f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))" ) -UPTIME_COMPACT_FALLBACK_EXPR = ( - "(1 - (sum_over_time(atlas:availability:failures_1d{" - 'scope="atlas",definition="request-v4"}[365d]) / ' - "clamp_min(sum_over_time(atlas:availability:requests_1d{" - 'scope="atlas",definition="request-v4"}[365d]), 1)))' -) UPTIME_RECORDING_EXPR = ( f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) " - f"or on() {UPTIME_COMPACT_FALLBACK_EXPR} " f"or on() {UPTIME_LIVE_FALLBACK_EXPR})" ) @@ -2149,7 +2142,7 @@ def build_overview(): "decimals": 4, "text_mode": "value", "instant": True, - "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful annual sample for up to 24 hours, can rebuild it from compact hourly rollups, and only falls back to the same one-hour request SLI before history exists.", + "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 24 hours and only falls back to the same one-hour request SLI before history exists.", }, { "id": 4, diff --git a/scripts/tests/test_availability_rollup.py b/scripts/tests/test_availability_rollup.py new file mode 100644 index 000000000..917196ce8 --- /dev/null +++ b/scripts/tests/test_availability_rollup.py @@ -0,0 +1,52 @@ +"""Unit tests for the Atlas availability publisher.""" + +import importlib.util +import json +from pathlib import Path + +import pytest + + +def load_module(): + """Load the service-owned rollup module without packaging it.""" + path = ( + Path(__file__).resolve().parents[2] + / "services/monitoring/scripts/availability_rollup.py" + ) + spec = importlib.util.spec_from_file_location("availability_rollup", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_parse_export_deduplicates_replay_boundaries() -> None: + """Keep only the final value when replay chunks share a timestamp.""" + mod = load_module() + lines = [ + (json.dumps({"timestamps": [1000, 2000], "values": [2, 3]}) + "\n").encode(), + (json.dumps({"timestamps": [2000, 3000], "values": [3, 5]}) + "\n").encode(), + ] + + assert mod.parse_export(lines) == {1000: 2.0, 2000: 3.0, 3000: 5.0} + + +def test_calculate_availability_uses_all_server_failures() -> None: + """Calculate one bounded successful-request ratio.""" + mod = load_module() + + assert mod.calculate_availability(1000, 2) == pytest.approx(0.998) + with pytest.raises(ValueError): + mod.calculate_availability(0, 0) + with pytest.raises(ValueError): + mod.calculate_availability(100, -1) + + +def test_render_metric_publishes_only_the_request_v4_series() -> None: + """Render the single series selected by the Grafana panel.""" + mod = load_module() + + assert mod.render_metric(0.9995, 1234) == ( + 'atlas:availability:ratio_365d{definition="request-v4",scope="atlas",' + 'rollup="yearly"} 0.999500000000 1234\n' + ) diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index 6ef5ab7f9..c45ed8196 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -65,15 +65,15 @@ def test_overview_availability_panel_uses_recorded_365d_rollup(): ) assert 'code=~"5.."' in availability_expr assert 'code=~"[1-5].."' in availability_expr - assert "atlas:availability:failures_1d" in availability_expr - assert "atlas:availability:requests_1d" in availability_expr - assert "sum_over_time" in availability_expr + assert "atlas:availability:failures_1d" not in availability_expr + assert "atlas:availability:requests_1d" not in availability_expr + assert "sum_over_time" not in availability_expr assert "kube_node_status_condition" not in availability_expr assert "kube_deployment_status_replicas_available" not in availability_expr assert panel["targets"][0]["instant"] is True assert "Every server-side 5xx" in panel["description"] assert "Replica counts, Grafana health" in panel["description"] - assert "can rebuild it from compact hourly rollups" in panel["description"] + assert "daily rollup job publishes one annual sample" in panel["description"] def test_overview_uses_readable_quality_power_and_gitops_panels(): diff --git a/scripts/tests/test_monitoring_query_capacity.py b/scripts/tests/test_monitoring_query_capacity.py index 0f7c7c656..3f8126847 100644 --- a/scripts/tests/test_monitoring_query_capacity.py +++ b/scripts/tests/test_monitoring_query_capacity.py @@ -44,26 +44,28 @@ def test_victoria_metrics_has_dashboard_burst_headroom() -> None: assert {"titan-14", "titan-18"} <= set(hostname_rule["values"]) -def test_yearly_availability_reuses_the_hourly_rollup() -> None: - """Prevent the yearly rule from rescanning raw cluster metrics for 365 days.""" +def test_yearly_availability_is_published_outside_the_query_pool() -> None: + """Keep all long-range availability work out of Grafana and MetricsQL.""" manifest = _documents( REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" )[0] groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"] rules = [rule for group in groups for rule in group["rules"]] - yearly = next( - rule - for rule in rules - if rule["record"] == "atlas:availability:ratio_365d" + manifests = _documents( + REPO_ROOT / "services/monitoring/availability-rollup-cronjob.yaml" ) + cronjob = next(manifest for manifest in manifests if manifest["kind"] == "CronJob") + source = ( + REPO_ROOT / "services/monitoring/scripts/availability_rollup.py" + ).read_text() - assert "atlas:availability:requests_1d" in yearly["expr"] - assert "atlas:availability:failures_1d" in yearly["expr"] - assert 'definition="request-v4"' in yearly["expr"] - assert "sum_over_time" in yearly["expr"] - assert "[365d]" in yearly["expr"] - assert "traefik_entrypoint_requests_total" not in yearly["expr"] - assert yearly["labels"]["definition"] == "request-v4" + assert all(rule["record"] != "atlas:availability:ratio_365d" for rule in rules) + assert "atlas:availability:requests_1d" in source + assert "atlas:availability:failures_1d" in source + assert "/api/v1/export" in source + assert "/api/v1/import/prometheus" in source + assert cronjob["spec"]["schedule"] == "10 0 * * *" + assert cronjob["spec"]["concurrencyPolicy"] == "Forbid" def test_daily_availability_rollups_use_the_same_request_sli() -> None: diff --git a/services/monitoring/availability-rollup-cronjob.yaml b/services/monitoring/availability-rollup-cronjob.yaml new file mode 100644 index 000000000..c42ed33ad --- /dev/null +++ b/services/monitoring/availability-rollup-cronjob.yaml @@ -0,0 +1,103 @@ +# services/monitoring/availability-rollup-cronjob.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: atlas-availability-request-v4-rollup-initial + namespace: monitoring +spec: + activeDeadlineSeconds: 900 + backoffLimit: 2 + template: + metadata: + labels: + app: atlas-availability-request-v4-rollup + spec: + restartPolicy: Never + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-22 + - titan-24 + containers: + - name: rollup + image: python:3.12-alpine + command: ["python", "/scripts/availability_rollup.py"] + env: + - name: VM_URL + value: http://victoria-metrics-single-server:8428 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - name: script + mountPath: /scripts + readOnly: true + volumes: + - name: script + configMap: + name: atlas-availability-rollup-script + +--- + +apiVersion: batch/v1 +kind: CronJob +metadata: + name: atlas-availability-request-v4-rollup + namespace: monitoring +spec: + schedule: "10 0 * * *" + timeZone: Etc/UTC + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + activeDeadlineSeconds: 900 + backoffLimit: 2 + template: + metadata: + labels: + app: atlas-availability-request-v4-rollup + spec: + restartPolicy: Never + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-22 + - titan-24 + containers: + - name: rollup + image: python:3.12-alpine + command: ["python", "/scripts/availability_rollup.py"] + env: + - name: VM_URL + value: http://victoria-metrics-single-server:8428 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - name: script + mountPath: /scripts + readOnly: true + volumes: + - name: script + configMap: + name: atlas-availability-rollup-script diff --git a/services/monitoring/dashboards/atlas-overview.json b/services/monitoring/dashboards/atlas-overview.json index 991d24414..65a1c9a86 100644 --- a/services/monitoring/dashboards/atlas-overview.json +++ b/services/monitoring/dashboards/atlas-overview.json @@ -229,7 +229,7 @@ }, "targets": [ { - "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", "refId": "A", "instant": true } @@ -286,7 +286,7 @@ }, "textMode": "value" }, - "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful annual sample for up to 24 hours, can rebuild it from compact hourly rollups, and only falls back to the same one-hour request SLI before history exists." + "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 24 hours and only falls back to the same one-hour request SLI before history exists." }, { "id": 4, diff --git a/services/monitoring/grafana-dashboard-overview.yaml b/services/monitoring/grafana-dashboard-overview.yaml index be1238c1b..6416b11f3 100644 --- a/services/monitoring/grafana-dashboard-overview.yaml +++ b/services/monitoring/grafana-dashboard-overview.yaml @@ -238,7 +238,7 @@ data: }, "targets": [ { - "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", "refId": "A", "instant": true } @@ -295,7 +295,7 @@ data: }, "textMode": "value" }, - "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful annual sample for up to 24 hours, can rebuild it from compact hourly rollups, and only falls back to the same one-hour request SLI before history exists." + "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 24 hours and only falls back to the same one-hour request SLI before history exists." }, { "id": 4, diff --git a/services/monitoring/kustomization.yaml b/services/monitoring/kustomization.yaml index b96b64baa..3c54503d6 100644 --- a/services/monitoring/kustomization.yaml +++ b/services/monitoring/kustomization.yaml @@ -21,6 +21,7 @@ resources: - vmalert-atlas-availability.yaml - availability-backfill-v4-job.yaml - availability-daily-backfill-v4-job.yaml + - availability-rollup-cronjob.yaml - dcgm-exporter.yaml - nvidia-process-exporter.yaml - jetson-tegrastats-exporter.yaml @@ -68,3 +69,9 @@ configMapGenerator: - platform_quality_suite_probe.sh=scripts/platform_quality_suite_probe.sh options: disableNameSuffixHash: true + - name: atlas-availability-rollup-script + namespace: monitoring + files: + - availability_rollup.py=scripts/availability_rollup.py + options: + disableNameSuffixHash: true diff --git a/services/monitoring/scripts/availability_rollup.py b/services/monitoring/scripts/availability_rollup.py new file mode 100644 index 000000000..c8a738920 --- /dev/null +++ b/services/monitoring/scripts/availability_rollup.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Publish Atlas request availability from deduplicated daily rollups.""" + +from __future__ import annotations + +import json +import os +import time +from datetime import datetime, timedelta, timezone +from typing import Iterable +from urllib.parse import urlencode +from urllib.request import Request, urlopen + + +VM_URL = os.environ.get( + "VM_URL", "http://victoria-metrics-single-server:8428" +).rstrip("/") +SCOPE = "atlas" +DEFINITION = "request-v4" +REQUESTS_METRIC = "atlas:availability:requests_1d" +FAILURES_METRIC = "atlas:availability:failures_1d" +OUTPUT_METRIC = "atlas:availability:ratio_365d" +WINDOW_DAYS = 365 + + +def parse_export(lines: Iterable[bytes]) -> dict[int, float]: + """Return the last exported value for each timestamp.""" + points: dict[int, float] = {} + for raw_line in lines: + if not raw_line.strip(): + continue + series = json.loads(raw_line) + for timestamp, value in zip(series["timestamps"], series["values"], strict=True): + points[int(timestamp)] = float(value) + return points + + +def fetch_rollup(metric: str, start: datetime, end: datetime) -> dict[int, float]: + """Stream one compact rollup series from VictoriaMetrics.""" + matcher = ( + f'{{__name__="{metric}",scope="{SCOPE}",definition="{DEFINITION}"}}' + ) + query = urlencode( + { + "match[]": matcher, + "start": start.isoformat().replace("+00:00", "Z"), + "end": end.isoformat().replace("+00:00", "Z"), + } + ) + with urlopen(f"{VM_URL}/api/v1/export?{query}", timeout=600) as response: + return parse_export(response) + + +def calculate_availability(requests: float, failures: float) -> float: + """Calculate the bounded successful-request ratio.""" + if requests <= 0: + raise ValueError("availability requires at least one observed request") + if failures < 0: + raise ValueError("failed request count cannot be negative") + return max(0.0, min(1.0, 1.0 - (failures / requests))) + + +def render_metric(value: float, timestamp_ms: int) -> str: + """Render one VictoriaMetrics Prometheus-import sample.""" + return ( + f'{OUTPUT_METRIC}{{definition="{DEFINITION}",scope="{SCOPE}",' + f'rollup="yearly"}} {value:.12f} {timestamp_ms}\n' + ) + + +def publish(value: float, timestamp_ms: int) -> None: + """Write the calculated annual ratio to VictoriaMetrics.""" + request = Request( + f"{VM_URL}/api/v1/import/prometheus", + data=render_metric(value, timestamp_ms).encode(), + headers={"Content-Type": "text/plain"}, + method="POST", + ) + with urlopen(request, timeout=30) as response: + if response.status not in {200, 204}: + raise RuntimeError(f"VictoriaMetrics import returned HTTP {response.status}") + + +def main() -> None: + """Rebuild and publish the rolling request-availability sample.""" + end = datetime.now(timezone.utc) + start = end - timedelta(days=WINDOW_DAYS) + requests = sum(fetch_rollup(REQUESTS_METRIC, start, end).values()) + failures = sum(fetch_rollup(FAILURES_METRIC, start, end).values()) + availability = calculate_availability(requests, failures) + timestamp_ms = time.time_ns() // 1_000_000 + publish(availability, timestamp_ms) + print( + json.dumps( + { + "requests": requests, + "failures": failures, + "availability_percent": availability * 100, + "timestamp_ms": timestamp_ms, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/services/monitoring/vmalert-atlas-availability.yaml b/services/monitoring/vmalert-atlas-availability.yaml index f30946db2..c2e1f78e5 100644 --- a/services/monitoring/vmalert-atlas-availability.yaml +++ b/services/monitoring/vmalert-atlas-availability.yaml @@ -67,34 +67,6 @@ data: definition: request-v4 scope: atlas rollup: daily - - name: atlas.availability.annual - interval: 15m - eval_offset: 14m - rules: - - record: atlas:availability:ratio_365d - expr: | - 1 - ( - sum_over_time( - atlas:availability:failures_1d{ - scope="atlas", - definition="request-v4" - }[365d] - ) - / - clamp_min( - sum_over_time( - atlas:availability:requests_1d{ - scope="atlas", - definition="request-v4" - }[365d] - ), - 1 - ) - ) - labels: - definition: request-v4 - scope: atlas - rollup: yearly platform-quality.yaml: | groups: - name: platform.quality