titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

hermes: enforce quota-aware fail-closed cli-auto provider routing

Browse Source 
Based on PR #15 (fix/hermes-result-decomposition-reliability); stacked
on the decomposed cli_lane modules.

- cli_lane_quota: soft-exclude a provider from NEW cli-auto work below
  the remaining-quota threshold (both-below prefers more remaining;
  fetch failure fails open with a metric).
- cli_lane_health: lane now writes provider health (G7) with classified
  failure reasons splitting the capacity conflation (quota/auth/
  rate-limit/transport) and cooldown hysteresis; re-admission only on
  full cooldown expiry, passed quota reset, or fresh success (G4).
- cli_lane_routing: capacity-limited health now excludes a provider
  (G3); cooldown/reset-aware re-admission.
- cli_lane_failover: explicit cli-codex-*/cli-claude-* assignees fail
  closed as transient instead of switching providers (G5); fallback
  depth stays bounded at two hosted providers (G1) with effort
  preserved; Switchyard outages block transient, not capability (G9).
- cli_lane_metrics: route-decision/fallback counters, quota and
  soft-exclusion gauges, pod-local scrape server (G6).
- cli_lane_provider: worker env drops ANTHROPIC_API_KEY, CLAUDE_API_KEY,
  OPENAI_API_KEY, API_SERVER_KEY so no metered path exists (G10).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jenkins 2026-08-17 20:31:05 -03:00
parent 62d8cd984b
commit 034c8372c7
11 changed files with 870 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
services/hermes/scripts/cli_lane_config.py
View File

@ -11,6 +11,8 @@ from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import yaml
import cli_lane_goal
@ -31,6 +33,12 @@ DEFAULT_MAX_RUNTIME = 12 * 60 * 60
HEARTBEAT_SECONDS = 20
PROVIDER_HEALTH_MAX_AGE_SECONDS = 5 * 60
PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS = 12 * 60 * 60
QUOTA_MIN_REMAINING_PERCENT_DEFAULT = 15.0
CAPACITY_COOLDOWN_SECONDS_DEFAULT = 300.0
AUTH_COOLDOWN_SECONDS_DEFAULT = 3600.0
QUOTA_METRICS_URL = os.environ.get(
"HERMES_CLI_QUOTA_METRICS_URL", "http://127.0.0.1:9010/metrics"
)
KANBAN_STORAGE_ATTEMPTS = 5
ARTIFACT_GC_INTERVAL_SECONDS = 5 * 60
ARTIFACT_RETENTION_AGE_SECONDS = 30 * 24 * 60 * 60
@ -194,3 +202,18 @@ class TerminalRecoverySnapshot:
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def kanban_setting(name: str, default: float) -> float:
"""Read one numeric routing setting from the deployed kanban config block."""
try:
document = yaml.safe_load(
(DATA_ROOT / "config.yaml").read_text(encoding="utf-8")
)
except (OSError, yaml.YAMLError):
return default
kanban = document.get("kanban") if isinstance(document, dict) else None
value = kanban.get(name) if isinstance(kanban, dict) else None
if isinstance(value, bool) or not isinstance(value, (int, float)):
return default
return float(value)

2
services/hermes/scripts/cli_lane_dispatch.py
View File

@ -24,6 +24,7 @@ from cli_lane_config import (
)
from cli_lane_execution import execute_claim
from cli_lane_files import atomic_json
from cli_lane_metrics import start_metrics_server
from cli_lane_recovery import _has_pending_finalization, recover_pending_finalizations
from cli_lane_retention import maybe_gc_lane_artifacts
@ -155,6 +156,7 @@ def main() -> int:
RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
atomic_json(RESULT_SCHEMA_PATH, RESULT_SCHEMA, 0o644)
start_metrics_server()
capabilities = initialize_kanban_capabilities(kanban_db)
recover_orphans()
workers = max(1, min(int(os.environ.get("HERMES_CLI_LANE_CONCURRENCY", "4")), 8))

132
services/hermes/scripts/cli_lane_execution.py
View File

@ -15,6 +15,7 @@ from cli_lane_config import (
DEFAULT_MAX_RUNTIME,
TerminalFinalizationPending,
)
from cli_lane_failover import _routed_or_blocked, capacity_failover
from cli_lane_files import (
_terminal_identity,
_terminal_path,
@ -23,8 +24,11 @@ from cli_lane_files import (
state_path,
)
from cli_lane_finalization import _finalize_terminal_record, _recover_exact_run
from cli_lane_health import record_provider_success
from cli_lane_metrics import record_route_decision
from cli_lane_prompt import build_prompt, git_handoff, workspace_artifacts
from cli_lane_provider import run_provider
from cli_lane_quota import selection_constraint
from cli_lane_records import _persist_candidate, _write_terminal_record
from cli_lane_recovery import _has_pending_finalization
from cli_lane_routing import fresh_unavailable_provider, select_route
@ -115,21 +119,30 @@ def execute_claim(board: str, task_id: str) -> None:
try:
previous_route = state.get("current_route")
excluded_provider = (
fresh_unavailable_provider() if assignee == "cli-auto" else None
constraint = selection_constraint(assignee)
unavailable_provider = (
constraint.exclude_provider if constraint.source == "health" else None
)
unavailable_provider = excluded_provider
route = select_route(
context,
assignee,
exclude_provider=excluded_provider,
exclude_reason="is unavailable according to fresh native health"
if excluded_provider
else None,
route = _routed_or_blocked(
kanban_db,
board,
task_id,
run_id,
lambda: select_route(
context,
assignee,
exclude_provider=constraint.exclude_provider,
exclude_reason=constraint.exclude_reason,
),
)
if excluded_provider:
if route is None:
return
for note in constraint.notes:
comment(note)
if constraint.exclude_provider:
comment(
f"Provider health guard excluded {excluded_provider} before automatic routing.",
f"Provider routing guard ({constraint.source}) excluded "
f"{constraint.exclude_provider} before automatic routing.",
)
comment(
f"CLI route: {route.provider}/{route.model} at {route.effort}; classifier={route.classifier}; {route.reason}",
@ -181,44 +194,34 @@ def execute_claim(board: str, task_id: str) -> None:
goal_turn=goal_turn,
)
if result.capacity_failure:
unavailable_provider = route.provider
retry_context = (
context
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. "
+ "Select the alternate hosted provider at an appropriate effort."
boundary = capacity_failover(
kanban_db,
board=board,
task_id=task_id,
run_id=run_id,
assignee=assignee,
comment=comment,
context=context,
workspace=workspace,
state=state,
state_file=state_file,
log_path=log_path,
heartbeat=heartbeat,
deadline=deadline,
route=route,
result=result,
candidate_file=candidate_file,
goal_turn=goal_turn,
)
alternate = "claude" if route.provider == "codex" else "codex"
fallback = select_route(
retry_context,
f"cli-{alternate}-{route.effort}",
)
comment(
f"Provider fallback: {route.provider} -> {fallback.provider}; Jetson reclassified the retry boundary.",
)
result = run_provider(
fallback,
build_prompt(
context,
workspace,
git_handoff(workspace, result.output),
),
workspace,
state,
state_file,
log_path,
heartbeat,
max(1, int(deadline - time.monotonic())),
)
route = fallback
if result.structured:
candidate_file = _persist_candidate(
state,
state_file,
dict(result.structured),
route=route,
returncode=result.returncode,
goal_turn=goal_turn,
)
route = boundary.route
result = boundary.result
candidate_file = boundary.candidate_file
if boundary.unavailable_provider is not None:
unavailable_provider = boundary.unavailable_provider
if boundary.blocked:
break
if result.returncode == 0:
record_provider_success(route.provider)
structured = result.structured
if structured:
structured = dict(structured)
@ -329,6 +332,9 @@ def execute_claim(board: str, task_id: str) -> None:
"accepted worker result is durably journaled but "
f"Kanban finalization is {outcome}"
)
record_route_decision(
route.provider, route.effort, route.classifier, "completed"
)
break
can_continue = (
@ -358,14 +364,25 @@ def execute_claim(board: str, task_id: str) -> None:
excluded = unavailable_provider
if excluded is None and assignee == "cli-auto":
excluded = fresh_unavailable_provider()
next_route = select_route(
escalation_context,
assignee,
exclude_provider=excluded,
exclude_reason="is unavailable according to fresh native health"
if excluded
else None,
record_route_decision(
route.provider, route.effort, route.classifier, "goal-continued"
)
next_route = _routed_or_blocked(
kanban_db,
board,
task_id,
run_id,
lambda boundary=escalation_context, blocked=excluded: select_route(
boundary,
assignee,
exclude_provider=blocked,
exclude_reason="is unavailable according to fresh native health"
if blocked
else None,
),
)
if next_route is None:
break
comment(
f"Goal route {goal_turn}/{goal_max_turns}: "
f"{next_route.provider}/{next_route.model} at {next_route.effort}; "
@ -400,6 +417,9 @@ def execute_claim(board: str, task_id: str) -> None:
failure_kind = (
"transient" if result.capacity_failure else "capability"
)
record_route_decision(
route.provider, route.effort, route.classifier, f"blocked-{failure_kind}"
)
_board_call(
kanban_db,
board,

181
services/hermes/scripts/cli_lane_failover.py Normal file
View File

@ -0,0 +1,181 @@
#!/usr/bin/env python3
"""Bounded, fail-closed cross-provider failover for one capacity boundary."""
from __future__ import annotations
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable
from cli_lane_board import _board_call
from cli_lane_config import ProcessResult, Route
from cli_lane_health import classify_capacity_failure, record_provider_failure
from cli_lane_metrics import (
record_provider_fallback,
record_route_decision,
record_router_selection_failure,
)
from cli_lane_prompt import build_prompt, git_handoff
from cli_lane_provider import run_provider
from cli_lane_records import _persist_candidate
from cli_lane_routing import select_route
@dataclass
class FailoverOutcome:
"""Where one capacity boundary left the claim's route and result."""
route: Route
result: ProcessResult
candidate_file: Path | None
unavailable_provider: str | None
blocked: bool
def _block_transient(
kanban_db: Any, board: str, task_id: str, run_id: Any, reason: str
) -> None:
"""Park one card as retryable so it re-enters when providers recover."""
_board_call(
kanban_db,
board,
lambda fresh: kanban_db.block_task(
fresh,
task_id,
reason=reason,
kind="transient",
expected_run_id=run_id,
),
)
def _routed_or_blocked(
kanban_db: Any,
board: str,
task_id: str,
run_id: Any,
select: Callable[[], Route],
) -> Route | None:
"""Turn a Switchyard outage into a transient block, never a capability one."""
try:
return select()
except RuntimeError as error:
record_router_selection_failure()
_block_transient(
kanban_db,
board,
task_id,
run_id,
"Switchyard route selection is unavailable; the card retries "
f"when the router recovers: {error}",
)
return None
def capacity_failover(
kanban_db: Any,
*,
board: str,
task_id: str,
run_id: Any,
assignee: str,
comment: Callable[[str], None],
context: str,
workspace: Path,
state: dict[str, Any],
state_file: Path,
log_path: Path,
heartbeat: Callable[[str], bool],
deadline: float,
route: Route,
result: ProcessResult,
candidate_file: Path | None,
goal_turn: int,
) -> FailoverOutcome:
"""Apply the capacity policy once: record, fail closed, or fail over."""
failure_class = classify_capacity_failure(result.output)
record_provider_failure(route.provider, failure_class)
record_route_decision(
route.provider, route.effort, route.classifier, "capacity-failure"
)
if assignee != "cli-auto":
# Fail closed: an operator pinned this provider, so a capacity or
# auth failure must wait for that provider, never switch away.
_block_transient(
kanban_db,
board,
task_id,
run_id,
f"Manually pinned provider {route.provider} hit a "
f"{failure_class} failure; the explicit {assignee} lane never "
"switches providers. Retry after recovery or reassign to cli-auto.",
)
return FailoverOutcome(route, result, candidate_file, None, True)
retry_context = (
context
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. "
+ "Select the alternate hosted provider at an appropriate effort."
)
alternate = "claude" if route.provider == "codex" else "codex"
fallback = _routed_or_blocked(
kanban_db,
board,
task_id,
run_id,
lambda: select_route(retry_context, f"cli-{alternate}-{route.effort}"),
)
if fallback is None:
return FailoverOutcome(route, result, candidate_file, route.provider, True)
record_provider_fallback(route.provider, fallback.provider, failure_class)
comment(
f"Provider fallback: {route.provider} -> {fallback.provider} "
f"after a {failure_class} failure; Jetson reclassified the retry boundary.",
)
fallback_result = run_provider(
fallback,
build_prompt(
context,
workspace,
git_handoff(workspace, result.output),
),
workspace,
state,
state_file,
log_path,
heartbeat,
max(1, int(deadline - time.monotonic())),
)
if fallback_result.capacity_failure:
# Bounded failover depth: two hosted providers, never a third
# metered or local path.
fallback_class = classify_capacity_failure(fallback_result.output)
record_provider_failure(fallback.provider, fallback_class)
record_route_decision(
fallback.provider, fallback.effort, fallback.classifier, "capacity-failure"
)
_block_transient(
kanban_db,
board,
task_id,
run_id,
"Both hosted providers failed at this boundary "
f"({route.provider}: {failure_class}; {fallback.provider}: "
f"{fallback_class}); the lane never falls to a metered or "
"local path. Waiting for provider recovery.",
)
return FailoverOutcome(
fallback, fallback_result, candidate_file, route.provider, True
)
if fallback_result.structured:
candidate_file = _persist_candidate(
state,
state_file,
dict(fallback_result.structured),
route=fallback,
returncode=fallback_result.returncode,
goal_turn=goal_turn,
)
return FailoverOutcome(
fallback, fallback_result, candidate_file, route.provider, False
)

102
services/hermes/scripts/cli_lane_health.py Normal file
View File

@ -0,0 +1,102 @@
#!/usr/bin/env python3
"""Lane-observed provider health with failure classification and cooldowns."""
from __future__ import annotations
import json
import os
import re
import time
from typing import Any
from cli_lane_config import (
AUTH_COOLDOWN_SECONDS_DEFAULT,
CAPACITY_COOLDOWN_SECONDS_DEFAULT,
PROVIDER_HEALTH_PATHS,
kanban_setting,
utc_now,
)
from cli_lane_files import load_json
# Ordered from most to least specific so one output maps to one actionable
# reason instead of the single conflated capacity signal.
FAILURE_CLASSIFICATIONS = (
("auth", re.compile(r"authentication|unauthorized|forbidden|oauth|token.*expired|401|403", re.I)),
("rate-limit", re.compile(r"rate.?limit|429|529|overload", re.I)),
("quota", re.compile(r"usage.?limit|quota|credit|exhaust|capacity", re.I)),
)
def classify_capacity_failure(output: str) -> str:
"""Split the broad capacity regex into quota/auth/rate-limit/transport."""
for reason, pattern in FAILURE_CLASSIFICATIONS:
if pattern.search(output):
return reason
return "transport"
def _write_provider_health(
provider: str, updates: dict[str, Any], clear: tuple[str, ...] = ()
) -> None:
"""Merge one lane observation into the shared provider health snapshot."""
path = PROVIDER_HEALTH_PATHS[provider]
value = load_json(path)
for key in clear:
value.pop(key, None)
value.update(updates)
try:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
temporary.write_text(
json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
os.replace(temporary, path)
except OSError:
# Health snapshots are advisory; never fail the lane over one write.
pass
def record_provider_success(provider: str) -> None:
"""Publish fresh evidence of success so the provider re-enters routing."""
_write_provider_health(
provider,
{
"source": "cli-lane-runner",
"state": "available",
"authenticated": True,
"checked_at": utc_now(),
},
clear=("failure_reason", "failed_at", "cooldown_until"),
)
def record_provider_failure(
provider: str, reason: str, now: float | None = None
) -> float:
"""Persist one lane-observed failure with its re-admission cooldown."""
current = time.time() if now is None else now
if reason == "auth":
cooldown = kanban_setting(
"provider_auth_cooldown_seconds", AUTH_COOLDOWN_SECONDS_DEFAULT
)
state, authenticated = "unavailable", False
else:
cooldown = kanban_setting(
"provider_capacity_cooldown_seconds", CAPACITY_COOLDOWN_SECONDS_DEFAULT
)
state, authenticated = "capacity-limited", True
cooldown_until = current + max(0.0, cooldown)
_write_provider_health(
provider,
{
"source": "cli-lane-runner",
"state": state,
"authenticated": authenticated,
"checked_at": utc_now(),
"failure_reason": reason,
"failed_at": current,
"cooldown_until": cooldown_until,
},
)
return cooldown_until

188
services/hermes/scripts/cli_lane_metrics.py Normal file
View File

@ -0,0 +1,188 @@
#!/usr/bin/env python3
"""In-process Prometheus metrics for the direct CLI lane's routing decisions."""
from __future__ import annotations
import os
import sys
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
METRIC_HELP = {
"hermes_cli_route_decisions_total": (
"Terminal outcome of each executed CLI lane route decision."
),
"hermes_cli_provider_fallbacks_total": (
"Cross-provider failovers taken by the CLI lane, by failure reason."
),
"hermes_cli_router_selection_failures_total": (
"Switchyard route selections that failed and blocked a card as transient."
),
"hermes_cli_quota_fetch_failures_total": (
"Quota snapshot reads that failed; routing then fails open."
),
"hermes_cli_quota_remaining_percent": (
"Binding remaining-quota percent per provider as seen by the CLI lane."
),
"hermes_cli_quota_reset_timestamp_seconds": (
"Unix reset time of the binding quota window as seen by the CLI lane."
),
"hermes_cli_provider_soft_excluded": (
"Whether new automatic work is currently routed away from a provider."
),
}
COUNTER_NAMES = frozenset(name for name in METRIC_HELP if name.endswith("_total"))
def _escape_label(value: str) -> str:
"""Escape one Prometheus label value."""
return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
class LaneMetricsRegistry:
"""Thread-safe counter and gauge store rendered in Prometheus text format."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._values: dict[tuple[str, tuple[tuple[str, str], ...]], float] = {}
def _key(
self, name: str, labels: dict[str, str] | None
) -> tuple[str, tuple[tuple[str, str], ...]]:
return name, tuple(sorted((labels or {}).items()))
def increment(
self, name: str, labels: dict[str, str] | None = None, amount: float = 1.0
) -> None:
key = self._key(name, labels)
with self._lock:
self._values[key] = self._values.get(key, 0.0) + amount
def set_value(
self, name: str, labels: dict[str, str] | None, value: float
) -> None:
with self._lock:
self._values[self._key(name, labels)] = value
def render(self) -> bytes:
"""Render every recorded sample with stable ordering."""
with self._lock:
values = dict(self._values)
lines: list[str] = []
for name in sorted({name for name, _ in values}):
kind = "counter" if name in COUNTER_NAMES else "gauge"
lines.extend((f"# HELP {name} {METRIC_HELP[name]}", f"# TYPE {name} {kind}"))
for (sample_name, labels), value in sorted(values.items()):
if sample_name != name:
continue
rendered = ",".join(
f'{key}="{_escape_label(text)}"' for key, text in labels
)
body = f"{{{rendered}}}" if rendered else ""
lines.append(f"{name}{body} {value:.12g}")
return ("\n".join(lines) + "\n").encode("utf-8")
METRICS = LaneMetricsRegistry()
def record_route_decision(
provider: str, effort: str, classifier: str, outcome: str
) -> None:
"""Count one executed route with its terminal outcome."""
METRICS.increment(
"hermes_cli_route_decisions_total",
{
"provider": provider,
"effort": effort,
"classifier": classifier,
"outcome": outcome,
},
)
def record_provider_fallback(
from_provider: str, to_provider: str, reason: str
) -> None:
"""Count one cross-provider failover with its classified reason."""
METRICS.increment(
"hermes_cli_provider_fallbacks_total",
{
"from_provider": from_provider,
"to_provider": to_provider,
"reason": reason,
},
)
def record_router_selection_failure() -> None:
"""Count one Switchyard outage observed at a selection boundary."""
METRICS.increment("hermes_cli_router_selection_failures_total")
def record_quota_fetch_failure() -> None:
"""Count one unavailable quota snapshot; routing continues fail-open."""
METRICS.increment("hermes_cli_quota_fetch_failures_total")
def record_provider_quota(
provider: str, remaining_percent: float, reset_timestamp: float | None
) -> None:
"""Publish the binding quota window the lane based its routing on."""
labels = {"provider": provider}
METRICS.set_value(
"hermes_cli_quota_remaining_percent", labels, remaining_percent
)
if reset_timestamp is not None:
METRICS.set_value(
"hermes_cli_quota_reset_timestamp_seconds", labels, reset_timestamp
)
def record_soft_exclusion(provider: str, excluded: bool) -> None:
"""Publish whether new automatic work avoids this provider right now."""
METRICS.set_value(
"hermes_cli_provider_soft_excluded",
{"provider": provider},
1.0 if excluded else 0.0,
)
def start_metrics_server(port: int | None = None) -> ThreadingHTTPServer | None:
"""Serve lane metrics on the pod-local scrape port without blocking dispatch."""
selected_port = (
int(os.environ.get("HERMES_CLI_LANE_METRICS_PORT", "9011"))
if port is None
else port
)
class Handler(BaseHTTPRequestHandler):
def do_GET(self) -> None: # noqa: N802
if self.path == "/metrics":
payload = METRICS.render()
self.send_response(200)
self.send_header("Content-Type", "text/plain; version=0.0.4")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
elif self.path == "/healthz":
self.send_response(200)
self.end_headers()
else:
self.send_error(404)
def log_message(self, _format: str, *_args: object) -> None:
return
try:
server = ThreadingHTTPServer(("0.0.0.0", selected_port), Handler)
except OSError as error:
print(
f"cli lane metrics server unavailable: {error}",
file=sys.stderr,
flush=True,
)
return None
threading.Thread(target=server.serve_forever, daemon=True).start()
return server

4
services/hermes/scripts/cli_lane_provider.py
View File

@ -197,6 +197,10 @@ def _terminate_worker_process(
def _base_env() -> dict[str, str]:
"""Build a worker environment while preserving Vault-backed CLI homes."""
env = os.environ.copy()
# Subscription CLIs only: a vendor API key in the environment could
# silently flip a worker onto a metered path, so drop every key here.
for secret in ("ANTHROPIC_API_KEY", "CLAUDE_API_KEY", "OPENAI_API_KEY", "API_SERVER_KEY"):
env.pop(secret, None)
env.update(
{
"HOME": str(DATA_ROOT / "home"),

205
services/hermes/scripts/cli_lane_quota.py Normal file
View File

@ -0,0 +1,205 @@
#!/usr/bin/env python3
"""Quota-aware soft routing for automatic CLI lane provider selection."""
from __future__ import annotations
import re
import sys
import urllib.request
from dataclasses import dataclass
from typing import Callable
from cli_lane_config import (
QUOTA_METRICS_URL,
QUOTA_MIN_REMAINING_PERCENT_DEFAULT,
kanban_setting,
)
from cli_lane_metrics import (
record_provider_quota,
record_quota_fetch_failure,
record_soft_exclusion,
)
from cli_lane_routing import fresh_unavailable_provider
# Exporter accounts are labeled by vendor; the lane routes by CLI provider.
QUOTA_PROVIDER_LABELS = {"openai": "codex", "anthropic": "claude"}
# Gate only on account-wide windows: a model-specific Claude window (for
# example seven_day_opus) must not veto the whole provider.
GATED_QUOTA_WINDOWS = {
"codex": None,
"claude": ("five_hour", "seven_day"),
}
_SAMPLE_PATTERN = re.compile(
r"^atlas_ai_quota_(remaining_percent|reset_timestamp_seconds|fetch_success)"
r"\{([^}]*)\} (-?[0-9.eE+]+)$"
)
_LABEL_PATTERN = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="([^"]*)"')
@dataclass(frozen=True)
class ProviderQuota:
"""One provider's binding remaining quota as seen by the lane."""
remaining_percent: float
reset_timestamp: float | None
@dataclass(frozen=True)
class SelectionConstraint:
"""One provider exclusion decided at a routing boundary, with evidence."""
exclude_provider: str | None
exclude_reason: str | None
source: str | None
notes: tuple[str, ...]
def _gated_window(provider: str, labels: dict[str, str]) -> str | None:
"""Return the window key when a sample belongs to a gated account limit."""
if labels.get("limit") != provider:
return None
window = labels.get("window", "")
allowed = GATED_QUOTA_WINDOWS[provider]
if allowed is not None and window not in allowed:
return None
return window
def parse_quota_metrics(text: str) -> dict[str, ProviderQuota]:
"""Reduce exporter samples to each provider's tightest account window."""
remaining: dict[tuple[str, str], float] = {}
resets: dict[tuple[str, str], float] = {}
fetch_success: dict[str, bool] = {}
for line in text.splitlines():
match = _SAMPLE_PATTERN.match(line.strip())
if not match:
continue
metric, raw_labels, raw_value = match.groups()
labels = dict(_LABEL_PATTERN.findall(raw_labels))
provider = QUOTA_PROVIDER_LABELS.get(labels.get("provider", ""))
if provider is None:
continue
value = float(raw_value)
if metric == "fetch_success":
fetch_success[provider] = value >= 1.0
continue
window = _gated_window(provider, labels)
if window is None:
continue
if metric == "remaining_percent":
remaining[(provider, window)] = value
else:
resets[(provider, window)] = value
snapshot: dict[str, ProviderQuota] = {}
for provider in QUOTA_PROVIDER_LABELS.values():
windows = {
window: value
for (candidate, window), value in remaining.items()
if candidate == provider
}
if not windows or not fetch_success.get(provider, True):
# Failed or stale exporter fetches leave this provider ungated.
continue
binding = min(windows, key=lambda window: (windows[window], window))
snapshot[provider] = ProviderQuota(
remaining_percent=windows[binding],
reset_timestamp=resets.get((provider, binding)),
)
return snapshot
def fetch_quota_snapshot(
url: str = QUOTA_METRICS_URL,
open_url: Callable = urllib.request.urlopen,
) -> dict[str, ProviderQuota]:
"""Read the in-pod usage exporter; empty means the signal is unavailable."""
try:
with open_url(url, timeout=10) as response:
text = response.read(1 << 20).decode("utf-8", errors="replace")
snapshot = parse_quota_metrics(text)
except (OSError, ValueError) as error:
record_quota_fetch_failure()
print(
f"quota snapshot unavailable; routing fails open: {type(error).__name__}",
file=sys.stderr,
flush=True,
)
return {}
if not snapshot:
record_quota_fetch_failure()
print(
"quota snapshot carried no gated windows; routing fails open",
file=sys.stderr,
flush=True,
)
return snapshot
def quota_soft_exclusion(
snapshot: dict[str, ProviderQuota], threshold: float
) -> tuple[str | None, str | None]:
"""Choose at most one provider to soft-exclude from new automatic work."""
below = {
provider: quota.remaining_percent
for provider, quota in snapshot.items()
if quota.remaining_percent < threshold
}
if not below:
return None, None
excluded = min(below, key=lambda provider: (below[provider], provider))
if len(below) == len(snapshot) and len(snapshot) > 1:
preferred = next(name for name in snapshot if name != excluded)
note = (
"Quota degraded: every provider is below the "
f"{threshold:.6g}% remaining threshold; preferring {preferred} "
f"({snapshot[preferred].remaining_percent:.6g}% left) over "
f"{excluded} ({below[excluded]:.6g}% left) for new auto work."
)
return excluded, note
note = (
f"Quota guard: {excluded} has {below[excluded]:.6g}% remaining "
f"(threshold {threshold:.6g}%); routing new auto work to the other "
"provider while active work finishes."
)
return excluded, note
def selection_constraint(
assignee: str,
*,
snapshot_from: Callable[[], dict[str, ProviderQuota]] | None = None,
now: float | None = None,
) -> SelectionConstraint:
"""Combine provider health and quota gating for one selection boundary."""
if assignee != "cli-auto":
# Explicit lanes are operator decisions; they are never re-routed.
return SelectionConstraint(None, None, None, ())
snapshot = (snapshot_from or fetch_quota_snapshot)()
resets: dict[str, float] = {}
for provider, quota in snapshot.items():
record_provider_quota(provider, quota.remaining_percent, quota.reset_timestamp)
if quota.reset_timestamp is not None:
resets[provider] = quota.reset_timestamp
health_excluded = fresh_unavailable_provider(now=now, quota_resets=resets)
if health_excluded is not None:
return SelectionConstraint(
health_excluded,
"is unavailable according to fresh native health",
"health",
(),
)
threshold = kanban_setting(
"provider_quota_min_remaining_percent", QUOTA_MIN_REMAINING_PERCENT_DEFAULT
)
excluded, note = quota_soft_exclusion(snapshot, threshold)
for provider in snapshot:
record_soft_exclusion(provider, provider == excluded)
if excluded is None or note is None:
return SelectionConstraint(None, None, None, ())
return SelectionConstraint(
excluded,
"is below its remaining-quota routing threshold",
"quota",
(note,),
)

44
services/hermes/scripts/cli_lane_routing.py
View File

@ -31,12 +31,49 @@ def parse_assignee(assignee: str) -> tuple[str | None, str | None]:
raise ValueError(f"unsupported external lane assignee: {assignee}")
return match.group(1), match.group(2)
def fresh_unavailable_provider(now: float | None = None) -> str | None:
def _health_number(value: Any) -> float | None:
"""Return a plain numeric health field, rejecting booleans and strings."""
if isinstance(value, bool) or not isinstance(value, (int, float)):
return None
return float(value)
def _quota_reset_readmits(
health: dict[str, Any], current: float, reset_timestamp: float | None
) -> bool:
"""A passed provider quota reset ends a capacity cooldown early."""
if health.get("authenticated") is False:
# An authentication failure is not repaired by a quota window reset.
return False
failed_at = _health_number(health.get("failed_at"))
return (
reset_timestamp is not None
and failed_at is not None
and failed_at < reset_timestamp <= current
)
def fresh_unavailable_provider(
now: float | None = None,
quota_resets: dict[str, float] | None = None,
) -> str | None:
"""Return one recently proven-down provider for automatic route exclusion."""
current = time.time() if now is None else now
resets = quota_resets or {}
unavailable: list[str] = []
for provider, path in PROVIDER_HEALTH_PATHS.items():
health = load_json(path)
if health.get("state") not in ("unavailable", "capacity-limited"):
continue
cooldown_until = _health_number(health.get("cooldown_until"))
if cooldown_until is not None:
# Lane-recorded failures re-enter only after the full cooldown has
# elapsed (hysteresis), a quota reset passed, or fresh success
# rewrote the state above.
if current >= cooldown_until or _quota_reset_readmits(
health, current, resets.get(provider)
):
continue
unavailable.append(provider)
continue
try:
age = current - path.stat().st_mtime
except OSError:
@ -46,10 +83,7 @@ def fresh_unavailable_provider(now: float | None = None) -> str | None:
if health.get("authenticated") is False
else PROVIDER_HEALTH_MAX_AGE_SECONDS
)
if (
0 <= age <= max_age
and health.get("state") == "unavailable"
):
if 0 <= age <= max_age:
unavailable.append(provider)
# If both providers are down, keep the normal attempt/fallback path so the
# card records authoritative current errors instead of trusting snapshots.

34
services/hermes/scripts/cli_lane_runner.py
View File

@ -32,6 +32,11 @@ from cli_lane_board import (
)
from cli_lane_config import (
ARTIFACT_GC_INTERVAL_SECONDS,
AUTH_COOLDOWN_SECONDS_DEFAULT,
CAPACITY_COOLDOWN_SECONDS_DEFAULT,
QUOTA_METRICS_URL,
QUOTA_MIN_REMAINING_PERCENT_DEFAULT,
kanban_setting,
ARTIFACT_RETENTION_AGE_SECONDS,
ARTIFACT_RETENTION_BYTES,
ARTIFACT_RETENTION_COUNT,
@ -67,6 +72,35 @@ from cli_lane_evidence import (
_write_json_noreplace,
)
from cli_lane_execution import execute_claim
from cli_lane_failover import (
FailoverOutcome,
_block_transient,
_routed_or_blocked,
capacity_failover,
)
from cli_lane_health import (
classify_capacity_failure,
record_provider_failure,
record_provider_success,
)
from cli_lane_metrics import (
METRICS,
record_provider_fallback,
record_provider_quota,
record_quota_fetch_failure,
record_route_decision,
record_router_selection_failure,
record_soft_exclusion,
start_metrics_server,
)
from cli_lane_quota import (
ProviderQuota,
SelectionConstraint,
fetch_quota_snapshot,
parse_quota_metrics,
quota_soft_exclusion,
selection_constraint,
)
from cli_lane_files import (
_candidate_path,
_fsync_directory,

20
testing/quality_contract.json
View File

@ -26,12 +26,16 @@
"services/hermes/scripts/cli_lane_dispatch.py",
"services/hermes/scripts/cli_lane_evidence.py",
"services/hermes/scripts/cli_lane_execution.py",
"services/hermes/scripts/cli_lane_failover.py",
"services/hermes/scripts/cli_lane_files.py",
"services/hermes/scripts/cli_lane_finalization.py",
"services/hermes/scripts/cli_lane_goal.py",
"services/hermes/scripts/cli_lane_health.py",
"services/hermes/scripts/cli_lane_metrics.py",
"services/hermes/scripts/cli_lane_prompt.py",
"services/hermes/scripts/cli_lane_provider.py",
"services/hermes/scripts/cli_lane_quarantine.py",
"services/hermes/scripts/cli_lane_quota.py",
"services/hermes/scripts/cli_lane_records.py",
"services/hermes/scripts/cli_lane_recovery.py",
"services/hermes/scripts/cli_lane_retention.py",
@ -45,10 +49,10 @@
"testing/quality_gate.py",
"ci/tests/glue/test_ariadne_schedules.py",
"ci/tests/glue/test_glue_metrics.py",
"testing/tests/test_publish_test_metrics.py",
"testing/tests/test_supply_chain_report.py",
"testing/tests/test_semgrep_report.py",
"testing/tests/test_quality_contract.py",
"testing/tests/test_publish_test_metrics.py",
"testing/tests/test_supply_chain_report.py",
"testing/tests/test_semgrep_report.py",
"testing/tests/test_quality_contract.py",
"testing/tests/test_quality_gate.py"
],
"lint_paths": [
@ -66,12 +70,16 @@
"services/hermes/scripts/cli_lane_dispatch.py",
"services/hermes/scripts/cli_lane_evidence.py",
"services/hermes/scripts/cli_lane_execution.py",
"services/hermes/scripts/cli_lane_failover.py",
"services/hermes/scripts/cli_lane_files.py",
"services/hermes/scripts/cli_lane_finalization.py",
"services/hermes/scripts/cli_lane_goal.py",
"services/hermes/scripts/cli_lane_health.py",
"services/hermes/scripts/cli_lane_metrics.py",
"services/hermes/scripts/cli_lane_prompt.py",
"services/hermes/scripts/cli_lane_provider.py",
"services/hermes/scripts/cli_lane_quarantine.py",
"services/hermes/scripts/cli_lane_quota.py",
"services/hermes/scripts/cli_lane_records.py",
"services/hermes/scripts/cli_lane_recovery.py",
"services/hermes/scripts/cli_lane_retention.py",
@ -216,12 +224,16 @@
"services/hermes/scripts/cli_lane_dispatch.py",
"services/hermes/scripts/cli_lane_evidence.py",
"services/hermes/scripts/cli_lane_execution.py",
"services/hermes/scripts/cli_lane_failover.py",
"services/hermes/scripts/cli_lane_files.py",
"services/hermes/scripts/cli_lane_finalization.py",
"services/hermes/scripts/cli_lane_goal.py",
"services/hermes/scripts/cli_lane_health.py",
"services/hermes/scripts/cli_lane_metrics.py",
"services/hermes/scripts/cli_lane_prompt.py",
"services/hermes/scripts/cli_lane_provider.py",
"services/hermes/scripts/cli_lane_quarantine.py",
"services/hermes/scripts/cli_lane_quota.py",
"services/hermes/scripts/cli_lane_records.py",
"services/hermes/scripts/cli_lane_recovery.py",
"services/hermes/scripts/cli_lane_retention.py",