034c8372c7
Based on PR #15 (fix/hermes-result-decomposition-reliability); stacked on the decomposed cli_lane modules. - cli_lane_quota: soft-exclude a provider from NEW cli-auto work below the remaining-quota threshold (both-below prefers more remaining; fetch failure fails open with a metric). - cli_lane_health: lane now writes provider health (G7) with classified failure reasons splitting the capacity conflation (quota/auth/ rate-limit/transport) and cooldown hysteresis; re-admission only on full cooldown expiry, passed quota reset, or fresh success (G4). - cli_lane_routing: capacity-limited health now excludes a provider (G3); cooldown/reset-aware re-admission. - cli_lane_failover: explicit cli-codex-*/cli-claude-* assignees fail closed as transient instead of switching providers (G5); fallback depth stays bounded at two hosted providers (G1) with effort preserved; Switchyard outages block transient, not capability (G9). - cli_lane_metrics: route-decision/fallback counters, quota and soft-exclusion gauges, pod-local scrape server (G6). - cli_lane_provider: worker env drops ANTHROPIC_API_KEY, CLAUDE_API_KEY, OPENAI_API_KEY, API_SERVER_KEY so no metered path exists (G10). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
103 lines
3.2 KiB
Python
103 lines
3.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Lane-observed provider health with failure classification and cooldowns."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import time
|
|
from typing import Any
|
|
|
|
from cli_lane_config import (
|
|
AUTH_COOLDOWN_SECONDS_DEFAULT,
|
|
CAPACITY_COOLDOWN_SECONDS_DEFAULT,
|
|
PROVIDER_HEALTH_PATHS,
|
|
kanban_setting,
|
|
utc_now,
|
|
)
|
|
from cli_lane_files import load_json
|
|
|
|
|
|
# Ordered from most to least specific so one output maps to one actionable
|
|
# reason instead of the single conflated capacity signal.
|
|
FAILURE_CLASSIFICATIONS = (
|
|
("auth", re.compile(r"authentication|unauthorized|forbidden|oauth|token.*expired|401|403", re.I)),
|
|
("rate-limit", re.compile(r"rate.?limit|429|529|overload", re.I)),
|
|
("quota", re.compile(r"usage.?limit|quota|credit|exhaust|capacity", re.I)),
|
|
)
|
|
|
|
|
|
def classify_capacity_failure(output: str) -> str:
|
|
"""Split the broad capacity regex into quota/auth/rate-limit/transport."""
|
|
for reason, pattern in FAILURE_CLASSIFICATIONS:
|
|
if pattern.search(output):
|
|
return reason
|
|
return "transport"
|
|
|
|
|
|
def _write_provider_health(
|
|
provider: str, updates: dict[str, Any], clear: tuple[str, ...] = ()
|
|
) -> None:
|
|
"""Merge one lane observation into the shared provider health snapshot."""
|
|
path = PROVIDER_HEALTH_PATHS[provider]
|
|
value = load_json(path)
|
|
for key in clear:
|
|
value.pop(key, None)
|
|
value.update(updates)
|
|
try:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
|
temporary.write_text(
|
|
json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
|
)
|
|
os.replace(temporary, path)
|
|
except OSError:
|
|
# Health snapshots are advisory; never fail the lane over one write.
|
|
pass
|
|
|
|
|
|
def record_provider_success(provider: str) -> None:
|
|
"""Publish fresh evidence of success so the provider re-enters routing."""
|
|
_write_provider_health(
|
|
provider,
|
|
{
|
|
"source": "cli-lane-runner",
|
|
"state": "available",
|
|
"authenticated": True,
|
|
"checked_at": utc_now(),
|
|
},
|
|
clear=("failure_reason", "failed_at", "cooldown_until"),
|
|
)
|
|
|
|
|
|
def record_provider_failure(
|
|
provider: str, reason: str, now: float | None = None
|
|
) -> float:
|
|
"""Persist one lane-observed failure with its re-admission cooldown."""
|
|
current = time.time() if now is None else now
|
|
if reason == "auth":
|
|
cooldown = kanban_setting(
|
|
"provider_auth_cooldown_seconds", AUTH_COOLDOWN_SECONDS_DEFAULT
|
|
)
|
|
state, authenticated = "unavailable", False
|
|
else:
|
|
cooldown = kanban_setting(
|
|
"provider_capacity_cooldown_seconds", CAPACITY_COOLDOWN_SECONDS_DEFAULT
|
|
)
|
|
state, authenticated = "capacity-limited", True
|
|
cooldown_until = current + max(0.0, cooldown)
|
|
_write_provider_health(
|
|
provider,
|
|
{
|
|
"source": "cli-lane-runner",
|
|
"state": state,
|
|
"authenticated": authenticated,
|
|
"checked_at": utc_now(),
|
|
"failure_reason": reason,
|
|
"failed_at": current,
|
|
"cooldown_until": cooldown_until,
|
|
},
|
|
)
|
|
return cooldown_until
|