#!/usr/bin/env python3 """Lane-observed provider health with failure classification and cooldowns.""" from __future__ import annotations import json import os import re import time from typing import Any from cli_lane_config import ( AUTH_COOLDOWN_SECONDS_DEFAULT, CAPACITY_COOLDOWN_SECONDS_DEFAULT, PROVIDER_HEALTH_PATHS, kanban_setting, utc_now, ) from cli_lane_files import load_json # Ordered from most to least specific so one output maps to one actionable # reason instead of the single conflated capacity signal. FAILURE_CLASSIFICATIONS = ( ("auth", re.compile(r"authentication|unauthorized|forbidden|oauth|token.*expired|401|403", re.I)), ("rate-limit", re.compile(r"rate.?limit|429|529|overload", re.I)), ("quota", re.compile(r"usage.?limit|quota|credit|exhaust|capacity", re.I)), ) def classify_capacity_failure(output: str) -> str: """Split the broad capacity regex into quota/auth/rate-limit/transport.""" for reason, pattern in FAILURE_CLASSIFICATIONS: if pattern.search(output): return reason return "transport" def _write_provider_health( provider: str, updates: dict[str, Any], clear: tuple[str, ...] = () ) -> None: """Merge one lane observation into the shared provider health snapshot.""" path = PROVIDER_HEALTH_PATHS[provider] value = load_json(path) for key in clear: value.pop(key, None) value.update(updates) try: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") temporary.write_text( json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8" ) os.replace(temporary, path) except OSError: # Health snapshots are advisory; never fail the lane over one write. pass def record_provider_success(provider: str) -> None: """Publish fresh evidence of success so the provider re-enters routing.""" _write_provider_health( provider, { "source": "cli-lane-runner", "state": "available", "authenticated": True, "checked_at": utc_now(), }, clear=("failure_reason", "failed_at", "cooldown_until"), ) def record_provider_failure( provider: str, reason: str, now: float | None = None ) -> float: """Persist one lane-observed failure with its re-admission cooldown.""" current = time.time() if now is None else now if reason == "auth": cooldown = kanban_setting( "provider_auth_cooldown_seconds", AUTH_COOLDOWN_SECONDS_DEFAULT ) state, authenticated = "unavailable", False else: cooldown = kanban_setting( "provider_capacity_cooldown_seconds", CAPACITY_COOLDOWN_SECONDS_DEFAULT ) state, authenticated = "capacity-limited", True cooldown_until = current + max(0.0, cooldown) _write_provider_health( provider, { "source": "cli-lane-runner", "state": state, "authenticated": authenticated, "checked_at": utc_now(), "failure_reason": reason, "failed_at": current, "cooldown_until": cooldown_until, }, ) return cooldown_until