From ea53bec75d845336af9fdb87f013d42b1049ee83 Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 4 Aug 2026 22:49:48 -0300 Subject: [PATCH] monitoring: remove legacy availability series --- scripts/tests/test_availability_cleanup.py | 28 ++++++ .../availability-legacy-cleanup-job.yaml | 47 ++++++++++ services/monitoring/kustomization.yaml | 7 ++ .../scripts/availability_cleanup.py | 93 +++++++++++++++++++ 4 files changed, 175 insertions(+) create mode 100644 scripts/tests/test_availability_cleanup.py create mode 100644 services/monitoring/availability-legacy-cleanup-job.yaml create mode 100644 services/monitoring/scripts/availability_cleanup.py diff --git a/scripts/tests/test_availability_cleanup.py b/scripts/tests/test_availability_cleanup.py new file mode 100644 index 000000000..c907b60f9 --- /dev/null +++ b/scripts/tests/test_availability_cleanup.py @@ -0,0 +1,28 @@ +"""Safety tests for the legacy Atlas availability cleanup.""" + +import importlib.util +from pathlib import Path + + +def load_module(): + """Load the service-owned cleanup module without packaging it.""" + path = ( + Path(__file__).resolve().parents[2] + / "services/monitoring/scripts/availability_cleanup.py" + ) + spec = importlib.util.spec_from_file_location("availability_cleanup", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_cleanup_selector_cannot_match_request_v4() -> None: + """Constrain deletion to obsolete annual Atlas definitions.""" + mod = load_module() + + assert '__name__="atlas:availability:ratio_365d"' in mod.LEGACY_SELECTOR + assert 'scope="atlas"' in mod.LEGACY_SELECTOR + assert 'definition!="request-v4"' in mod.LEGACY_SELECTOR + assert 'definition="request-v4"' in mod.PROTECTED_SELECTOR + assert "ratio_1h" not in mod.LEGACY_SELECTOR diff --git a/services/monitoring/availability-legacy-cleanup-job.yaml b/services/monitoring/availability-legacy-cleanup-job.yaml new file mode 100644 index 000000000..70f30e079 --- /dev/null +++ b/services/monitoring/availability-legacy-cleanup-job.yaml @@ -0,0 +1,47 @@ +# services/monitoring/availability-legacy-cleanup-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: atlas-availability-legacy-series-cleanup-v1 + namespace: monitoring +spec: + activeDeadlineSeconds: 300 + backoffLimit: 2 + template: + metadata: + labels: + app: atlas-availability-legacy-series-cleanup + spec: + restartPolicy: Never + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-22 + - titan-24 + containers: + - name: cleanup + image: python:3.12-alpine + command: ["python", "/scripts/availability_cleanup.py"] + env: + - name: VM_URL + value: http://victoria-metrics-single-server:8428 + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 250m + memory: 128Mi + volumeMounts: + - name: script + mountPath: /scripts + readOnly: true + volumes: + - name: script + configMap: + name: atlas-availability-cleanup-script diff --git a/services/monitoring/kustomization.yaml b/services/monitoring/kustomization.yaml index 3c54503d6..a9ce091ea 100644 --- a/services/monitoring/kustomization.yaml +++ b/services/monitoring/kustomization.yaml @@ -22,6 +22,7 @@ resources: - availability-backfill-v4-job.yaml - availability-daily-backfill-v4-job.yaml - availability-rollup-cronjob.yaml + - availability-legacy-cleanup-job.yaml - dcgm-exporter.yaml - nvidia-process-exporter.yaml - jetson-tegrastats-exporter.yaml @@ -75,3 +76,9 @@ configMapGenerator: - availability_rollup.py=scripts/availability_rollup.py options: disableNameSuffixHash: true + - name: atlas-availability-cleanup-script + namespace: monitoring + files: + - availability_cleanup.py=scripts/availability_cleanup.py + options: + disableNameSuffixHash: true diff --git a/services/monitoring/scripts/availability_cleanup.py b/services/monitoring/scripts/availability_cleanup.py new file mode 100644 index 000000000..16cd9149a --- /dev/null +++ b/services/monitoring/scripts/availability_cleanup.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Remove obsolete Atlas annual-availability series after the request-v4 migration.""" + +from __future__ import annotations + +import json +import os +import time +from datetime import datetime, timedelta, timezone +from urllib.parse import urlencode +from urllib.request import Request, urlopen + + +VM_URL = os.environ.get( + "VM_URL", "http://victoria-metrics-single-server:8428" +).rstrip("/") +LEGACY_SELECTOR = ( + '{__name__="atlas:availability:ratio_365d",scope="atlas",' + 'definition!="request-v4"}' +) +PROTECTED_SELECTOR = ( + '{__name__="atlas:availability:ratio_365d",scope="atlas",' + 'definition="request-v4"}' +) + + +def list_series(selector: str) -> list[dict[str, str]]: + """List recently visible series matching an exact safety selector.""" + end = datetime.now(timezone.utc) + start = end - timedelta(hours=48) + query = urlencode( + { + "match[]": selector, + "start": start.isoformat().replace("+00:00", "Z"), + "end": end.isoformat().replace("+00:00", "Z"), + } + ) + with urlopen(f"{VM_URL}/api/v1/series?{query}", timeout=60) as response: + payload = json.load(response) + if payload.get("status") != "success": + raise RuntimeError(f"VictoriaMetrics series lookup failed: {payload}") + return payload["data"] + + +def delete_legacy_series() -> None: + """Delete only annual Atlas series outside the protected request-v4 label set.""" + query = urlencode({"match[]": LEGACY_SELECTOR}) + request = Request( + f"{VM_URL}/api/v1/admin/tsdb/delete_series?{query}", + data=b"", + method="POST", + ) + with urlopen(request, timeout=60) as response: + if response.status not in {200, 204}: + raise RuntimeError(f"VictoriaMetrics deletion returned HTTP {response.status}") + + +def main() -> None: + """Guard the new series, delete retired definitions, and verify convergence.""" + protected_before = list_series(PROTECTED_SELECTOR) + if len(protected_before) != 1: + raise RuntimeError( + f"expected exactly one protected request-v4 series, found {len(protected_before)}" + ) + legacy_before = list_series(LEGACY_SELECTOR) + if legacy_before: + delete_legacy_series() + + legacy_after = legacy_before + for _ in range(10): + legacy_after = list_series(LEGACY_SELECTOR) + if not legacy_after: + break + time.sleep(1) + if legacy_after: + raise RuntimeError(f"legacy availability series remain: {legacy_after}") + if len(list_series(PROTECTED_SELECTOR)) != 1: + raise RuntimeError("protected request-v4 series was not preserved") + + print( + json.dumps( + { + "deleted_legacy_series": len(legacy_before), + "protected_series": len(protected_before), + "selector": LEGACY_SELECTOR, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main()