monitoring: remove legacy availability series

scripts/tests/test_availability_cleanup.py

@@ -0,0 +1,28 @@
+"""Safety tests for the legacy Atlas availability cleanup."""
+
+import importlib.util
+from pathlib import Path
+
+
+def load_module():
+    """Load the service-owned cleanup module without packaging it."""
+    path = (
+        Path(__file__).resolve().parents[2]
+        / "services/monitoring/scripts/availability_cleanup.py"
+    )
+    spec = importlib.util.spec_from_file_location("availability_cleanup", path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_cleanup_selector_cannot_match_request_v4() -> None:
+    """Constrain deletion to obsolete annual Atlas definitions."""
+    mod = load_module()
+
+    assert '__name__="atlas:availability:ratio_365d"' in mod.LEGACY_SELECTOR
+    assert 'scope="atlas"' in mod.LEGACY_SELECTOR
+    assert 'definition!="request-v4"' in mod.LEGACY_SELECTOR
+    assert 'definition="request-v4"' in mod.PROTECTED_SELECTOR
+    assert "ratio_1h" not in mod.LEGACY_SELECTOR

services/monitoring/availability-legacy-cleanup-job.yaml

@@ -0,0 +1,47 @@
+# services/monitoring/availability-legacy-cleanup-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: atlas-availability-legacy-series-cleanup-v1
+  namespace: monitoring
+spec:
+  activeDeadlineSeconds: 300
+  backoffLimit: 2
+  template:
+    metadata:
+      labels:
+        app: atlas-availability-legacy-series-cleanup
+    spec:
+      restartPolicy: Never
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values:
+                      - titan-22
+                      - titan-24
+      containers:
+        - name: cleanup
+          image: python:3.12-alpine
+          command: ["python", "/scripts/availability_cleanup.py"]
+          env:
+            - name: VM_URL
+              value: http://victoria-metrics-single-server:8428
+          resources:
+            requests:
+              cpu: 25m
+              memory: 32Mi
+            limits:
+              cpu: 250m
+              memory: 128Mi
+          volumeMounts:
+            - name: script
+              mountPath: /scripts
+              readOnly: true
+      volumes:
+        - name: script
+          configMap:
+            name: atlas-availability-cleanup-script

services/monitoring/kustomization.yaml

@@ -22,6 +22,7 @@ resources:
   - availability-backfill-v4-job.yaml
   - availability-daily-backfill-v4-job.yaml
   - availability-rollup-cronjob.yaml
+  - availability-legacy-cleanup-job.yaml
   - dcgm-exporter.yaml
   - nvidia-process-exporter.yaml
   - jetson-tegrastats-exporter.yaml
@@ -75,3 +76,9 @@ configMapGenerator:
       - availability_rollup.py=scripts/availability_rollup.py
     options:
       disableNameSuffixHash: true
+  - name: atlas-availability-cleanup-script
+    namespace: monitoring
+    files:
+      - availability_cleanup.py=scripts/availability_cleanup.py
+    options:
+      disableNameSuffixHash: true

services/monitoring/scripts/availability_cleanup.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Remove obsolete Atlas annual-availability series after the request-v4 migration."""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from datetime import datetime, timedelta, timezone
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+
+VM_URL = os.environ.get(
+    "VM_URL", "http://victoria-metrics-single-server:8428"
+).rstrip("/")
+LEGACY_SELECTOR = (
+    '{__name__="atlas:availability:ratio_365d",scope="atlas",'
+    'definition!="request-v4"}'
+)
+PROTECTED_SELECTOR = (
+    '{__name__="atlas:availability:ratio_365d",scope="atlas",'
+    'definition="request-v4"}'
+)
+
+
+def list_series(selector: str) -> list[dict[str, str]]:
+    """List recently visible series matching an exact safety selector."""
+    end = datetime.now(timezone.utc)
+    start = end - timedelta(hours=48)
+    query = urlencode(
+        {
+            "match[]": selector,
+            "start": start.isoformat().replace("+00:00", "Z"),
+            "end": end.isoformat().replace("+00:00", "Z"),
+        }
+    )
+    with urlopen(f"{VM_URL}/api/v1/series?{query}", timeout=60) as response:
+        payload = json.load(response)
+    if payload.get("status") != "success":
+        raise RuntimeError(f"VictoriaMetrics series lookup failed: {payload}")
+    return payload["data"]
+
+
+def delete_legacy_series() -> None:
+    """Delete only annual Atlas series outside the protected request-v4 label set."""
+    query = urlencode({"match[]": LEGACY_SELECTOR})
+    request = Request(
+        f"{VM_URL}/api/v1/admin/tsdb/delete_series?{query}",
+        data=b"",
+        method="POST",
+    )
+    with urlopen(request, timeout=60) as response:
+        if response.status not in {200, 204}:
+            raise RuntimeError(f"VictoriaMetrics deletion returned HTTP {response.status}")
+
+
+def main() -> None:
+    """Guard the new series, delete retired definitions, and verify convergence."""
+    protected_before = list_series(PROTECTED_SELECTOR)
+    if len(protected_before) != 1:
+        raise RuntimeError(
+            f"expected exactly one protected request-v4 series, found {len(protected_before)}"
+        )
+    legacy_before = list_series(LEGACY_SELECTOR)
+    if legacy_before:
+        delete_legacy_series()
+
+    legacy_after = legacy_before
+    for _ in range(10):
+        legacy_after = list_series(LEGACY_SELECTOR)
+        if not legacy_after:
+            break
+        time.sleep(1)
+    if legacy_after:
+        raise RuntimeError(f"legacy availability series remain: {legacy_after}")
+    if len(list_series(PROTECTED_SELECTOR)) != 1:
+        raise RuntimeError("protected request-v4 series was not preserved")
+
+    print(
+        json.dumps(
+            {
+                "deleted_legacy_series": len(legacy_before),
+                "protected_series": len(protected_before),
+                "selector": LEGACY_SELECTOR,
+            },
+            sort_keys=True,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()