atlas-iac/services/monitoring/scripts/availability_rollup.py

#!/usr/bin/env python3
"""Publish Atlas request availability from deduplicated daily rollups."""

from __future__ import annotations

import json
import os
import time
from datetime import datetime, timedelta, timezone
from typing import Iterable
from urllib.parse import urlencode
from urllib.request import Request, urlopen


VM_URL = os.environ.get(
    "VM_URL", "http://victoria-metrics-single-server:8428"
).rstrip("/")
SCOPE = "atlas"
DEFINITION = "request-v4"
REQUESTS_METRIC = "atlas:availability:requests_1d"
FAILURES_METRIC = "atlas:availability:failures_1d"
OUTPUT_METRIC = "atlas:availability:ratio_365d"
COVERAGE_METRIC = "atlas:availability:coverage_days_365d"
WINDOW_DAYS = 365
# Freshly imported samples sit in the in-memory buffer briefly before they
# become searchable, so the read-back check retries instead of failing fast.
VERIFY_ATTEMPTS = 6
VERIFY_DELAY_SECONDS = 10


def parse_export(lines: Iterable[bytes]) -> dict[int, float]:
    """Return the last exported value for each timestamp."""
    points: dict[int, float] = {}
    for raw_line in lines:
        if not raw_line.strip():
            continue
        series = json.loads(raw_line)
        for timestamp, value in zip(series["timestamps"], series["values"], strict=True):
            points[int(timestamp)] = float(value)
    return points


def fetch_series(metric: str, start: datetime, end: datetime) -> dict[int, float]:
    """Stream one compact series from VictoriaMetrics."""
    matcher = (
        f'{{__name__="{metric}",scope="{SCOPE}",definition="{DEFINITION}"}}'
    )
    query = urlencode(
        {
            "match[]": matcher,
            "start": start.isoformat().replace("+00:00", "Z"),
            "end": end.isoformat().replace("+00:00", "Z"),
        }
    )
    with urlopen(f"{VM_URL}/api/v1/export?{query}", timeout=600) as response:
        return parse_export(response)


def calculate_availability(requests: float, failures: float) -> float:
    """Calculate the bounded successful-request ratio."""
    if requests <= 0:
        raise ValueError("availability requires at least one observed request")
    if failures < 0:
        raise ValueError("failed request count cannot be negative")
    return max(0.0, min(1.0, 1.0 - (failures / requests)))


def render_metric(metric: str, value: float, timestamp_ms: int) -> str:
    """Render one VictoriaMetrics Prometheus-import sample."""
    return (
        f'{metric}{{definition="{DEFINITION}",scope="{SCOPE}",'
        f'rollup="yearly"}} {value:.12f} {timestamp_ms}\n'
    )


def publish(metric: str, value: float, timestamp_ms: int) -> None:
    """Write one calculated yearly sample to VictoriaMetrics."""
    request = Request(
        f"{VM_URL}/api/v1/import/prometheus",
        data=render_metric(metric, value, timestamp_ms).encode(),
        headers={"Content-Type": "text/plain"},
        method="POST",
    )
    with urlopen(request, timeout=30) as response:
        if response.status not in {200, 204}:
            raise RuntimeError(f"VictoriaMetrics import returned HTTP {response.status}")


def verify_stored(metric: str, value: float, timestamp_ms: int) -> None:
    """Prove the sample landed; a read-only store accepts writes and drops them.

    During the 2026-08-18 storage outage every import returned success while
    VictoriaMetrics silently discarded the samples, so this job reported
    healthy runs across a day and a half of lost publishes. Reading the sample
    back is the only evidence the write survived.
    """
    when = datetime.fromtimestamp(timestamp_ms / 1000, tz=timezone.utc)
    for attempt in range(VERIFY_ATTEMPTS):
        if attempt:
            time.sleep(VERIFY_DELAY_SECONDS)
        points = fetch_series(
            metric, when - timedelta(minutes=5), when + timedelta(minutes=5)
        )
        stored = points.get(timestamp_ms)
        if stored is not None and abs(stored - value) < 1e-9:
            return
    raise RuntimeError(
        f"{metric} sample at {timestamp_ms} is not readable after publish; "
        "VictoriaMetrics accepted and then discarded the write "
        "(is /storage read-only?)"
    )


def main() -> None:
    """Rebuild and publish the rolling request-availability samples."""
    end = datetime.now(timezone.utc)
    start = end - timedelta(days=WINDOW_DAYS)
    requests = fetch_series(REQUESTS_METRIC, start, end)
    failures = fetch_series(FAILURES_METRIC, start, end)
    availability = calculate_availability(
        sum(requests.values()), sum(failures.values())
    )
    coverage_days = float(len(requests))
    timestamp_ms = time.time_ns() // 1_000_000
    publish(OUTPUT_METRIC, availability, timestamp_ms)
    publish(COVERAGE_METRIC, coverage_days, timestamp_ms)
    verify_stored(OUTPUT_METRIC, availability, timestamp_ms)
    verify_stored(COVERAGE_METRIC, coverage_days, timestamp_ms)
    print(
        json.dumps(
            {
                "availability_percent": availability * 100,
                "coverage_days": coverage_days,
                "failures": sum(failures.values()),
                "requests": sum(requests.values()),
                "timestamp_ms": timestamp_ms,
            },
            sort_keys=True,
        )
    )


if __name__ == "__main__":
    main()