titan-iac/services/monitoring/scripts/availability_cleanup.py

94 lines
2.9 KiB
Python

#!/usr/bin/env python3
"""Remove obsolete Atlas annual-availability series after the request-v4 migration."""
from __future__ import annotations
import json
import os
import time
from datetime import datetime, timedelta, timezone
from urllib.parse import urlencode
from urllib.request import Request, urlopen
VM_URL = os.environ.get(
"VM_URL", "http://victoria-metrics-single-server:8428"
).rstrip("/")
LEGACY_SELECTOR = (
'{__name__="atlas:availability:ratio_365d",scope="atlas",'
'definition!="request-v4"}'
)
PROTECTED_SELECTOR = (
'{__name__="atlas:availability:ratio_365d",scope="atlas",'
'definition="request-v4"}'
)
def list_series(selector: str) -> list[dict[str, str]]:
"""List recently visible series matching an exact safety selector."""
end = datetime.now(timezone.utc)
start = end - timedelta(hours=48)
query = urlencode(
{
"match[]": selector,
"start": start.isoformat().replace("+00:00", "Z"),
"end": end.isoformat().replace("+00:00", "Z"),
}
)
with urlopen(f"{VM_URL}/api/v1/series?{query}", timeout=60) as response:
payload = json.load(response)
if payload.get("status") != "success":
raise RuntimeError(f"VictoriaMetrics series lookup failed: {payload}")
return payload["data"]
def delete_legacy_series() -> None:
"""Delete only annual Atlas series outside the protected request-v4 label set."""
query = urlencode({"match[]": LEGACY_SELECTOR})
request = Request(
f"{VM_URL}/api/v1/admin/tsdb/delete_series?{query}",
data=b"",
method="POST",
)
with urlopen(request, timeout=60) as response:
if response.status not in {200, 204}:
raise RuntimeError(f"VictoriaMetrics deletion returned HTTP {response.status}")
def main() -> None:
"""Guard the new series, delete retired definitions, and verify convergence."""
protected_before = list_series(PROTECTED_SELECTOR)
if len(protected_before) != 1:
raise RuntimeError(
f"expected exactly one protected request-v4 series, found {len(protected_before)}"
)
legacy_before = list_series(LEGACY_SELECTOR)
if legacy_before:
delete_legacy_series()
legacy_after = legacy_before
for _ in range(10):
legacy_after = list_series(LEGACY_SELECTOR)
if not legacy_after:
break
time.sleep(1)
if legacy_after:
raise RuntimeError(f"legacy availability series remain: {legacy_after}")
if len(list_series(PROTECTED_SELECTOR)) != 1:
raise RuntimeError("protected request-v4 series was not preserved")
print(
json.dumps(
{
"deleted_legacy_series": len(legacy_before),
"protected_series": len(protected_before),
"selector": LEGACY_SELECTOR,
},
sort_keys=True,
)
)
if __name__ == "__main__":
main()