atlas-iac/services/hermes/scripts/cli_lane_metrics.py

#!/usr/bin/env python3
"""In-process Prometheus metrics for the direct CLI lane's routing decisions."""

from __future__ import annotations

import os
import sys
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Callable


METRIC_HELP = {
    "hermes_cli_route_decisions_total": (
        "Terminal outcome of each executed CLI lane route decision."
    ),
    "hermes_cli_provider_fallbacks_total": (
        "Cross-provider failovers taken by the CLI lane, by failure reason."
    ),
    "hermes_cli_router_selection_failures_total": (
        "Switchyard route selections that failed and blocked a card as transient."
    ),
    "hermes_cli_quota_fetch_failures_total": (
        "Quota snapshot reads that failed; routing then fails open."
    ),
    "hermes_cli_quota_remaining_percent": (
        "Binding remaining-quota percent per provider as seen by the CLI lane."
    ),
    "hermes_cli_quota_reset_timestamp_seconds": (
        "Unix reset time of the binding quota window as seen by the CLI lane."
    ),
    "hermes_cli_provider_soft_excluded": (
        "Whether new automatic work is currently routed away from a provider."
    ),
}
COUNTER_NAMES = frozenset(name for name in METRIC_HELP if name.endswith("_total"))


def _escape_label(value: str) -> str:
    """Escape one Prometheus label value."""
    return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')


class LaneMetricsRegistry:
    """Thread-safe counter and gauge store rendered in Prometheus text format."""

    def __init__(self) -> None:
        self._lock = threading.Lock()
        self._values: dict[tuple[str, tuple[tuple[str, str], ...]], float] = {}

    def _key(
        self, name: str, labels: dict[str, str] | None
    ) -> tuple[str, tuple[tuple[str, str], ...]]:
        return name, tuple(sorted((labels or {}).items()))

    def increment(
        self, name: str, labels: dict[str, str] | None = None, amount: float = 1.0
    ) -> None:
        key = self._key(name, labels)
        with self._lock:
            self._values[key] = self._values.get(key, 0.0) + amount

    def set_value(
        self, name: str, labels: dict[str, str] | None, value: float
    ) -> None:
        with self._lock:
            self._values[self._key(name, labels)] = value

    def render(self) -> bytes:
        """Render every recorded sample with stable ordering."""
        with self._lock:
            values = dict(self._values)
        lines: list[str] = []
        for name in sorted({name for name, _ in values}):
            kind = "counter" if name in COUNTER_NAMES else "gauge"
            lines.extend((f"# HELP {name} {METRIC_HELP[name]}", f"# TYPE {name} {kind}"))
            for (sample_name, labels), value in sorted(values.items()):
                if sample_name != name:
                    continue
                rendered = ",".join(
                    f'{key}="{_escape_label(text)}"' for key, text in labels
                )
                body = f"{{{rendered}}}" if rendered else ""
                lines.append(f"{name}{body} {value:.12g}")
        return ("\n".join(lines) + "\n").encode("utf-8")


METRICS = LaneMetricsRegistry()


def record_route_decision(
    provider: str, effort: str, classifier: str, outcome: str
) -> None:
    """Count one executed route with its terminal outcome."""
    METRICS.increment(
        "hermes_cli_route_decisions_total",
        {
            "provider": provider,
            "effort": effort,
            "classifier": classifier,
            "outcome": outcome,
        },
    )


def record_provider_fallback(
    from_provider: str, to_provider: str, reason: str
) -> None:
    """Count one cross-provider failover with its classified reason."""
    METRICS.increment(
        "hermes_cli_provider_fallbacks_total",
        {
            "from_provider": from_provider,
            "to_provider": to_provider,
            "reason": reason,
        },
    )


def record_router_selection_failure() -> None:
    """Count one Switchyard outage observed at a selection boundary."""
    METRICS.increment("hermes_cli_router_selection_failures_total")


def record_quota_fetch_failure() -> None:
    """Count one unavailable quota snapshot; routing continues fail-open."""
    METRICS.increment("hermes_cli_quota_fetch_failures_total")


def record_provider_quota(
    provider: str, remaining_percent: float, reset_timestamp: float | None
) -> None:
    """Publish the binding quota window the lane based its routing on."""
    labels = {"provider": provider}
    METRICS.set_value(
        "hermes_cli_quota_remaining_percent", labels, remaining_percent
    )
    if reset_timestamp is not None:
        METRICS.set_value(
            "hermes_cli_quota_reset_timestamp_seconds", labels, reset_timestamp
        )


def record_soft_exclusion(provider: str, excluded: bool) -> None:
    """Publish whether new automatic work avoids this provider right now."""
    METRICS.set_value(
        "hermes_cli_provider_soft_excluded",
        {"provider": provider},
        1.0 if excluded else 0.0,
    )


def start_metrics_server(
    port: int | None = None,
    *,
    health_check: Callable[[], bool] | None = None,
) -> ThreadingHTTPServer | None:
    """Serve lane metrics on the pod-local scrape port without blocking dispatch."""
    selected_port = (
        int(os.environ.get("HERMES_CLI_LANE_METRICS_PORT", "9011"))
        if port is None
        else port
    )

    class Handler(BaseHTTPRequestHandler):
        def do_GET(self) -> None:  # noqa: N802
            if self.path == "/metrics":
                payload = METRICS.render()
                self.send_response(200)
                self.send_header("Content-Type", "text/plain; version=0.0.4")
                self.send_header("Content-Length", str(len(payload)))
                self.end_headers()
                self.wfile.write(payload)
            elif self.path == "/healthz":
                try:
                    healthy = health_check is None or health_check()
                except Exception:
                    healthy = False
                self.send_response(200 if healthy else 503)
                self.end_headers()
            else:
                self.send_error(404)

        def log_message(self, _format: str, *_args: object) -> None:
            return

    try:
        server = ThreadingHTTPServer(("0.0.0.0", selected_port), Handler)
    except OSError as error:
        print(
            f"cli lane metrics server unavailable: {error}",
            file=sys.stderr,
            flush=True,
        )
        return None
    threading.Thread(target=server.serve_forever, daemon=True).start()
    return server