#!/usr/bin/env python3 """In-process Prometheus metrics for the direct CLI lane's routing decisions.""" from __future__ import annotations import os import sys import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Callable METRIC_HELP = { "hermes_cli_route_decisions_total": ( "Terminal outcome of each executed CLI lane route decision." ), "hermes_cli_provider_fallbacks_total": ( "Cross-provider failovers taken by the CLI lane, by failure reason." ), "hermes_cli_router_selection_failures_total": ( "Switchyard route selections that failed and blocked a card as transient." ), "hermes_cli_quota_fetch_failures_total": ( "Quota snapshot reads that failed; routing then fails open." ), "hermes_cli_quota_remaining_percent": ( "Binding remaining-quota percent per provider as seen by the CLI lane." ), "hermes_cli_quota_reset_timestamp_seconds": ( "Unix reset time of the binding quota window as seen by the CLI lane." ), "hermes_cli_provider_soft_excluded": ( "Whether new automatic work is currently routed away from a provider." ), } COUNTER_NAMES = frozenset(name for name in METRIC_HELP if name.endswith("_total")) def _escape_label(value: str) -> str: """Escape one Prometheus label value.""" return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') class LaneMetricsRegistry: """Thread-safe counter and gauge store rendered in Prometheus text format.""" def __init__(self) -> None: self._lock = threading.Lock() self._values: dict[tuple[str, tuple[tuple[str, str], ...]], float] = {} def _key( self, name: str, labels: dict[str, str] | None ) -> tuple[str, tuple[tuple[str, str], ...]]: return name, tuple(sorted((labels or {}).items())) def increment( self, name: str, labels: dict[str, str] | None = None, amount: float = 1.0 ) -> None: key = self._key(name, labels) with self._lock: self._values[key] = self._values.get(key, 0.0) + amount def set_value( self, name: str, labels: dict[str, str] | None, value: float ) -> None: with self._lock: self._values[self._key(name, labels)] = value def render(self) -> bytes: """Render every recorded sample with stable ordering.""" with self._lock: values = dict(self._values) lines: list[str] = [] for name in sorted({name for name, _ in values}): kind = "counter" if name in COUNTER_NAMES else "gauge" lines.extend((f"# HELP {name} {METRIC_HELP[name]}", f"# TYPE {name} {kind}")) for (sample_name, labels), value in sorted(values.items()): if sample_name != name: continue rendered = ",".join( f'{key}="{_escape_label(text)}"' for key, text in labels ) body = f"{{{rendered}}}" if rendered else "" lines.append(f"{name}{body} {value:.12g}") return ("\n".join(lines) + "\n").encode("utf-8") METRICS = LaneMetricsRegistry() def record_route_decision( provider: str, effort: str, classifier: str, outcome: str ) -> None: """Count one executed route with its terminal outcome.""" METRICS.increment( "hermes_cli_route_decisions_total", { "provider": provider, "effort": effort, "classifier": classifier, "outcome": outcome, }, ) def record_provider_fallback( from_provider: str, to_provider: str, reason: str ) -> None: """Count one cross-provider failover with its classified reason.""" METRICS.increment( "hermes_cli_provider_fallbacks_total", { "from_provider": from_provider, "to_provider": to_provider, "reason": reason, }, ) def record_router_selection_failure() -> None: """Count one Switchyard outage observed at a selection boundary.""" METRICS.increment("hermes_cli_router_selection_failures_total") def record_quota_fetch_failure() -> None: """Count one unavailable quota snapshot; routing continues fail-open.""" METRICS.increment("hermes_cli_quota_fetch_failures_total") def record_provider_quota( provider: str, remaining_percent: float, reset_timestamp: float | None ) -> None: """Publish the binding quota window the lane based its routing on.""" labels = {"provider": provider} METRICS.set_value( "hermes_cli_quota_remaining_percent", labels, remaining_percent ) if reset_timestamp is not None: METRICS.set_value( "hermes_cli_quota_reset_timestamp_seconds", labels, reset_timestamp ) def record_soft_exclusion(provider: str, excluded: bool) -> None: """Publish whether new automatic work avoids this provider right now.""" METRICS.set_value( "hermes_cli_provider_soft_excluded", {"provider": provider}, 1.0 if excluded else 0.0, ) def start_metrics_server( port: int | None = None, *, health_check: Callable[[], bool] | None = None, ) -> ThreadingHTTPServer | None: """Serve lane metrics on the pod-local scrape port without blocking dispatch.""" selected_port = ( int(os.environ.get("HERMES_CLI_LANE_METRICS_PORT", "9011")) if port is None else port ) class Handler(BaseHTTPRequestHandler): def do_GET(self) -> None: # noqa: N802 if self.path == "/metrics": payload = METRICS.render() self.send_response(200) self.send_header("Content-Type", "text/plain; version=0.0.4") self.send_header("Content-Length", str(len(payload))) self.end_headers() self.wfile.write(payload) elif self.path == "/healthz": try: healthy = health_check is None or health_check() except Exception: healthy = False self.send_response(200 if healthy else 503) self.end_headers() else: self.send_error(404) def log_message(self, _format: str, *_args: object) -> None: return try: server = ThreadingHTTPServer(("0.0.0.0", selected_port), Handler) except OSError as error: print( f"cli lane metrics server unavailable: {error}", file=sys.stderr, flush=True, ) return None threading.Thread(target=server.serve_forever, daemon=True).start() return server