#!/usr/bin/env python3 """Publish Atlas request availability from deduplicated daily rollups.""" from __future__ import annotations import json import os import time from datetime import datetime, timedelta, timezone from typing import Iterable from urllib.parse import urlencode from urllib.request import Request, urlopen VM_URL = os.environ.get( "VM_URL", "http://victoria-metrics-single-server:8428" ).rstrip("/") SCOPE = "atlas" DEFINITION = "request-v4" REQUESTS_METRIC = "atlas:availability:requests_1d" FAILURES_METRIC = "atlas:availability:failures_1d" OUTPUT_METRIC = "atlas:availability:ratio_365d" COVERAGE_METRIC = "atlas:availability:coverage_days_365d" WINDOW_DAYS = 365 # Freshly imported samples sit in the in-memory buffer briefly before they # become searchable, so the read-back check retries instead of failing fast. VERIFY_ATTEMPTS = 6 VERIFY_DELAY_SECONDS = 10 def parse_export(lines: Iterable[bytes]) -> dict[int, float]: """Return the last exported value for each timestamp.""" points: dict[int, float] = {} for raw_line in lines: if not raw_line.strip(): continue series = json.loads(raw_line) for timestamp, value in zip(series["timestamps"], series["values"], strict=True): points[int(timestamp)] = float(value) return points def fetch_series(metric: str, start: datetime, end: datetime) -> dict[int, float]: """Stream one compact series from VictoriaMetrics.""" matcher = ( f'{{__name__="{metric}",scope="{SCOPE}",definition="{DEFINITION}"}}' ) query = urlencode( { "match[]": matcher, "start": start.isoformat().replace("+00:00", "Z"), "end": end.isoformat().replace("+00:00", "Z"), } ) with urlopen(f"{VM_URL}/api/v1/export?{query}", timeout=600) as response: return parse_export(response) def calculate_availability(requests: float, failures: float) -> float: """Calculate the bounded successful-request ratio.""" if requests <= 0: raise ValueError("availability requires at least one observed request") if failures < 0: raise ValueError("failed request count cannot be negative") return max(0.0, min(1.0, 1.0 - (failures / requests))) def render_metric(metric: str, value: float, timestamp_ms: int) -> str: """Render one VictoriaMetrics Prometheus-import sample.""" return ( f'{metric}{{definition="{DEFINITION}",scope="{SCOPE}",' f'rollup="yearly"}} {value:.12f} {timestamp_ms}\n' ) def publish(metric: str, value: float, timestamp_ms: int) -> None: """Write one calculated yearly sample to VictoriaMetrics.""" request = Request( f"{VM_URL}/api/v1/import/prometheus", data=render_metric(metric, value, timestamp_ms).encode(), headers={"Content-Type": "text/plain"}, method="POST", ) with urlopen(request, timeout=30) as response: if response.status not in {200, 204}: raise RuntimeError(f"VictoriaMetrics import returned HTTP {response.status}") def verify_stored(metric: str, value: float, timestamp_ms: int) -> None: """Prove the sample landed; a read-only store accepts writes and drops them. During the 2026-08-18 storage outage every import returned success while VictoriaMetrics silently discarded the samples, so this job reported healthy runs across a day and a half of lost publishes. Reading the sample back is the only evidence the write survived. """ when = datetime.fromtimestamp(timestamp_ms / 1000, tz=timezone.utc) for attempt in range(VERIFY_ATTEMPTS): if attempt: time.sleep(VERIFY_DELAY_SECONDS) points = fetch_series( metric, when - timedelta(minutes=5), when + timedelta(minutes=5) ) stored = points.get(timestamp_ms) if stored is not None and abs(stored - value) < 1e-9: return raise RuntimeError( f"{metric} sample at {timestamp_ms} is not readable after publish; " "VictoriaMetrics accepted and then discarded the write " "(is /storage read-only?)" ) def main() -> None: """Rebuild and publish the rolling request-availability samples.""" end = datetime.now(timezone.utc) start = end - timedelta(days=WINDOW_DAYS) requests = fetch_series(REQUESTS_METRIC, start, end) failures = fetch_series(FAILURES_METRIC, start, end) availability = calculate_availability( sum(requests.values()), sum(failures.values()) ) coverage_days = float(len(requests)) timestamp_ms = time.time_ns() // 1_000_000 publish(OUTPUT_METRIC, availability, timestamp_ms) publish(COVERAGE_METRIC, coverage_days, timestamp_ms) verify_stored(OUTPUT_METRIC, availability, timestamp_ms) verify_stored(COVERAGE_METRIC, coverage_days, timestamp_ms) print( json.dumps( { "availability_percent": availability * 100, "coverage_days": coverage_days, "failures": sum(failures.values()), "requests": sum(requests.values()), "timestamp_ms": timestamp_ms, }, sort_keys=True, ) ) if __name__ == "__main__": main()