hermes: enforce quota-aware fail-closed cli-auto provider routing

Based on PR #15 (fix/hermes-result-decomposition-reliability); stacked
on the decomposed cli_lane modules.

- cli_lane_quota: soft-exclude a provider from NEW cli-auto work below
  the remaining-quota threshold (both-below prefers more remaining;
  fetch failure fails open with a metric).
- cli_lane_health: lane now writes provider health (G7) with classified
  failure reasons splitting the capacity conflation (quota/auth/
  rate-limit/transport) and cooldown hysteresis; re-admission only on
  full cooldown expiry, passed quota reset, or fresh success (G4).
- cli_lane_routing: capacity-limited health now excludes a provider
  (G3); cooldown/reset-aware re-admission.
- cli_lane_failover: explicit cli-codex-*/cli-claude-* assignees fail
  closed as transient instead of switching providers (G5); fallback
  depth stays bounded at two hosted providers (G1) with effort
  preserved; Switchyard outages block transient, not capability (G9).
- cli_lane_metrics: route-decision/fallback counters, quota and
  soft-exclusion gauges, pod-local scrape server (G6).
- cli_lane_provider: worker env drops ANTHROPIC_API_KEY, CLAUDE_API_KEY,
  OPENAI_API_KEY, API_SERVER_KEY so no metered path exists (G10).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

services/hermes/scripts/cli_lane_config.py

@@ -11,6 +11,8 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
 
+import yaml
+
 import cli_lane_goal
 
 
@@ -31,6 +33,12 @@ DEFAULT_MAX_RUNTIME = 12 * 60 * 60
 HEARTBEAT_SECONDS = 20
 PROVIDER_HEALTH_MAX_AGE_SECONDS = 5 * 60
 PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS = 12 * 60 * 60
+QUOTA_MIN_REMAINING_PERCENT_DEFAULT = 15.0
+CAPACITY_COOLDOWN_SECONDS_DEFAULT = 300.0
+AUTH_COOLDOWN_SECONDS_DEFAULT = 3600.0
+QUOTA_METRICS_URL = os.environ.get(
+    "HERMES_CLI_QUOTA_METRICS_URL", "http://127.0.0.1:9010/metrics"
+)
 KANBAN_STORAGE_ATTEMPTS = 5
 ARTIFACT_GC_INTERVAL_SECONDS = 5 * 60
 ARTIFACT_RETENTION_AGE_SECONDS = 30 * 24 * 60 * 60
@@ -194,3 +202,18 @@ class TerminalRecoverySnapshot:
 
 def utc_now() -> str:
     return datetime.now(timezone.utc).isoformat()
+
+
+def kanban_setting(name: str, default: float) -> float:
+    """Read one numeric routing setting from the deployed kanban config block."""
+    try:
+        document = yaml.safe_load(
+            (DATA_ROOT / "config.yaml").read_text(encoding="utf-8")
+        )
+    except (OSError, yaml.YAMLError):
+        return default
+    kanban = document.get("kanban") if isinstance(document, dict) else None
+    value = kanban.get(name) if isinstance(kanban, dict) else None
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return default
+    return float(value)

services/hermes/scripts/cli_lane_dispatch.py

@@ -24,6 +24,7 @@ from cli_lane_config import (
 )
 from cli_lane_execution import execute_claim
 from cli_lane_files import atomic_json
+from cli_lane_metrics import start_metrics_server
 from cli_lane_recovery import _has_pending_finalization, recover_pending_finalizations
 from cli_lane_retention import maybe_gc_lane_artifacts
 
@@ -155,6 +156,7 @@ def main() -> int:
 
     RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
     atomic_json(RESULT_SCHEMA_PATH, RESULT_SCHEMA, 0o644)
+    start_metrics_server()
     capabilities = initialize_kanban_capabilities(kanban_db)
     recover_orphans()
     workers = max(1, min(int(os.environ.get("HERMES_CLI_LANE_CONCURRENCY", "4")), 8))

services/hermes/scripts/cli_lane_execution.py

@@ -15,6 +15,7 @@ from cli_lane_config import (
     DEFAULT_MAX_RUNTIME,
     TerminalFinalizationPending,
 )
+from cli_lane_failover import _routed_or_blocked, capacity_failover
 from cli_lane_files import (
     _terminal_identity,
     _terminal_path,
@@ -23,8 +24,11 @@ from cli_lane_files import (
     state_path,
 )
 from cli_lane_finalization import _finalize_terminal_record, _recover_exact_run
+from cli_lane_health import record_provider_success
+from cli_lane_metrics import record_route_decision
 from cli_lane_prompt import build_prompt, git_handoff, workspace_artifacts
 from cli_lane_provider import run_provider
+from cli_lane_quota import selection_constraint
 from cli_lane_records import _persist_candidate, _write_terminal_record
 from cli_lane_recovery import _has_pending_finalization
 from cli_lane_routing import fresh_unavailable_provider, select_route
@@ -115,21 +119,30 @@ def execute_claim(board: str, task_id: str) -> None:
 
         try:
             previous_route = state.get("current_route")
-            excluded_provider = (
+            constraint = selection_constraint(assignee)
-                fresh_unavailable_provider() if assignee == "cli-auto" else None
+            unavailable_provider = (
+                constraint.exclude_provider if constraint.source == "health" else None
             )
-            unavailable_provider = excluded_provider
+            route = _routed_or_blocked(
-            route = select_route(
+                kanban_db,
-                context,
+                board,
-                assignee,
+                task_id,
-                exclude_provider=excluded_provider,
+                run_id,
-                exclude_reason="is unavailable according to fresh native health"
+                lambda: select_route(
-                if excluded_provider
+                    context,
-                else None,
+                    assignee,
+                    exclude_provider=constraint.exclude_provider,
+                    exclude_reason=constraint.exclude_reason,
+                ),
             )
-            if excluded_provider:
+            if route is None:
+                return
+            for note in constraint.notes:
+                comment(note)
+            if constraint.exclude_provider:
                 comment(
-                    f"Provider health guard excluded {excluded_provider} before automatic routing.",
+                    f"Provider routing guard ({constraint.source}) excluded "
+                    f"{constraint.exclude_provider} before automatic routing.",
                 )
             comment(
                 f"CLI route: {route.provider}/{route.model} at {route.effort}; classifier={route.classifier}; {route.reason}",
@@ -181,44 +194,34 @@ def execute_claim(board: str, task_id: str) -> None:
                         goal_turn=goal_turn,
                     )
                 if result.capacity_failure:
-                    unavailable_provider = route.provider
+                    boundary = capacity_failover(
-                    retry_context = (
+                        kanban_db,
-                        context
+                        board=board,
-                        + "\n\nRouting boundary: the first provider failed from capacity/authentication. "
+                        task_id=task_id,
-                        + "Select the alternate hosted provider at an appropriate effort."
+                        run_id=run_id,
+                        assignee=assignee,
+                        comment=comment,
+                        context=context,
+                        workspace=workspace,
+                        state=state,
+                        state_file=state_file,
+                        log_path=log_path,
+                        heartbeat=heartbeat,
+                        deadline=deadline,
+                        route=route,
+                        result=result,
+                        candidate_file=candidate_file,
+                        goal_turn=goal_turn,
                     )
-                    alternate = "claude" if route.provider == "codex" else "codex"
+                    route = boundary.route
-                    fallback = select_route(
+                    result = boundary.result
-                        retry_context,
+                    candidate_file = boundary.candidate_file
-                        f"cli-{alternate}-{route.effort}",
+                    if boundary.unavailable_provider is not None:
-                    )
+                        unavailable_provider = boundary.unavailable_provider
-                    comment(
+                    if boundary.blocked:
-                        f"Provider fallback: {route.provider} -> {fallback.provider}; Jetson reclassified the retry boundary.",
+                        break
-                    )
+                if result.returncode == 0:
-                    result = run_provider(
+                    record_provider_success(route.provider)
-                        fallback,
-                        build_prompt(
-                            context,
-                            workspace,
-                            git_handoff(workspace, result.output),
-                        ),
-                        workspace,
-                        state,
-                        state_file,
-                        log_path,
-                        heartbeat,
-                        max(1, int(deadline - time.monotonic())),
-                    )
-                    route = fallback
-                    if result.structured:
-                        candidate_file = _persist_candidate(
-                            state,
-                            state_file,
-                            dict(result.structured),
-                            route=route,
-                            returncode=result.returncode,
-                            goal_turn=goal_turn,
-                        )
                 structured = result.structured
                 if structured:
                     structured = dict(structured)
@@ -329,6 +332,9 @@ def execute_claim(board: str, task_id: str) -> None:
                             "accepted worker result is durably journaled but "
                             f"Kanban finalization is {outcome}"
                         )
+                    record_route_decision(
+                        route.provider, route.effort, route.classifier, "completed"
+                    )
                     break
 
                 can_continue = (
@@ -358,14 +364,25 @@ def execute_claim(board: str, task_id: str) -> None:
                     excluded = unavailable_provider
                     if excluded is None and assignee == "cli-auto":
                         excluded = fresh_unavailable_provider()
-                    next_route = select_route(
+                    record_route_decision(
-                        escalation_context,
+                        route.provider, route.effort, route.classifier, "goal-continued"
-                        assignee,
-                        exclude_provider=excluded,
-                        exclude_reason="is unavailable according to fresh native health"
-                        if excluded
-                        else None,
                     )
+                    next_route = _routed_or_blocked(
+                        kanban_db,
+                        board,
+                        task_id,
+                        run_id,
+                        lambda boundary=escalation_context, blocked=excluded: select_route(
+                            boundary,
+                            assignee,
+                            exclude_provider=blocked,
+                            exclude_reason="is unavailable according to fresh native health"
+                            if blocked
+                            else None,
+                        ),
+                    )
+                    if next_route is None:
+                        break
                     comment(
                         f"Goal route {goal_turn}/{goal_max_turns}: "
                         f"{next_route.provider}/{next_route.model} at {next_route.effort}; "
@@ -400,6 +417,9 @@ def execute_claim(board: str, task_id: str) -> None:
                 failure_kind = (
                     "transient" if result.capacity_failure else "capability"
                 )
+                record_route_decision(
+                    route.provider, route.effort, route.classifier, f"blocked-{failure_kind}"
+                )
                 _board_call(
                     kanban_db,
                     board,

services/hermes/scripts/cli_lane_failover.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""Bounded, fail-closed cross-provider failover for one capacity boundary."""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable
+
+from cli_lane_board import _board_call
+from cli_lane_config import ProcessResult, Route
+from cli_lane_health import classify_capacity_failure, record_provider_failure
+from cli_lane_metrics import (
+    record_provider_fallback,
+    record_route_decision,
+    record_router_selection_failure,
+)
+from cli_lane_prompt import build_prompt, git_handoff
+from cli_lane_provider import run_provider
+from cli_lane_records import _persist_candidate
+from cli_lane_routing import select_route
+
+
+@dataclass
+class FailoverOutcome:
+    """Where one capacity boundary left the claim's route and result."""
+
+    route: Route
+    result: ProcessResult
+    candidate_file: Path | None
+    unavailable_provider: str | None
+    blocked: bool
+
+
+def _block_transient(
+    kanban_db: Any, board: str, task_id: str, run_id: Any, reason: str
+) -> None:
+    """Park one card as retryable so it re-enters when providers recover."""
+    _board_call(
+        kanban_db,
+        board,
+        lambda fresh: kanban_db.block_task(
+            fresh,
+            task_id,
+            reason=reason,
+            kind="transient",
+            expected_run_id=run_id,
+        ),
+    )
+
+
+def _routed_or_blocked(
+    kanban_db: Any,
+    board: str,
+    task_id: str,
+    run_id: Any,
+    select: Callable[[], Route],
+) -> Route | None:
+    """Turn a Switchyard outage into a transient block, never a capability one."""
+    try:
+        return select()
+    except RuntimeError as error:
+        record_router_selection_failure()
+        _block_transient(
+            kanban_db,
+            board,
+            task_id,
+            run_id,
+            "Switchyard route selection is unavailable; the card retries "
+            f"when the router recovers: {error}",
+        )
+        return None
+
+
+def capacity_failover(
+    kanban_db: Any,
+    *,
+    board: str,
+    task_id: str,
+    run_id: Any,
+    assignee: str,
+    comment: Callable[[str], None],
+    context: str,
+    workspace: Path,
+    state: dict[str, Any],
+    state_file: Path,
+    log_path: Path,
+    heartbeat: Callable[[str], bool],
+    deadline: float,
+    route: Route,
+    result: ProcessResult,
+    candidate_file: Path | None,
+    goal_turn: int,
+) -> FailoverOutcome:
+    """Apply the capacity policy once: record, fail closed, or fail over."""
+    failure_class = classify_capacity_failure(result.output)
+    record_provider_failure(route.provider, failure_class)
+    record_route_decision(
+        route.provider, route.effort, route.classifier, "capacity-failure"
+    )
+    if assignee != "cli-auto":
+        # Fail closed: an operator pinned this provider, so a capacity or
+        # auth failure must wait for that provider, never switch away.
+        _block_transient(
+            kanban_db,
+            board,
+            task_id,
+            run_id,
+            f"Manually pinned provider {route.provider} hit a "
+            f"{failure_class} failure; the explicit {assignee} lane never "
+            "switches providers. Retry after recovery or reassign to cli-auto.",
+        )
+        return FailoverOutcome(route, result, candidate_file, None, True)
+    retry_context = (
+        context
+        + "\n\nRouting boundary: the first provider failed from capacity/authentication. "
+        + "Select the alternate hosted provider at an appropriate effort."
+    )
+    alternate = "claude" if route.provider == "codex" else "codex"
+    fallback = _routed_or_blocked(
+        kanban_db,
+        board,
+        task_id,
+        run_id,
+        lambda: select_route(retry_context, f"cli-{alternate}-{route.effort}"),
+    )
+    if fallback is None:
+        return FailoverOutcome(route, result, candidate_file, route.provider, True)
+    record_provider_fallback(route.provider, fallback.provider, failure_class)
+    comment(
+        f"Provider fallback: {route.provider} -> {fallback.provider} "
+        f"after a {failure_class} failure; Jetson reclassified the retry boundary.",
+    )
+    fallback_result = run_provider(
+        fallback,
+        build_prompt(
+            context,
+            workspace,
+            git_handoff(workspace, result.output),
+        ),
+        workspace,
+        state,
+        state_file,
+        log_path,
+        heartbeat,
+        max(1, int(deadline - time.monotonic())),
+    )
+    if fallback_result.capacity_failure:
+        # Bounded failover depth: two hosted providers, never a third
+        # metered or local path.
+        fallback_class = classify_capacity_failure(fallback_result.output)
+        record_provider_failure(fallback.provider, fallback_class)
+        record_route_decision(
+            fallback.provider, fallback.effort, fallback.classifier, "capacity-failure"
+        )
+        _block_transient(
+            kanban_db,
+            board,
+            task_id,
+            run_id,
+            "Both hosted providers failed at this boundary "
+            f"({route.provider}: {failure_class}; {fallback.provider}: "
+            f"{fallback_class}); the lane never falls to a metered or "
+            "local path. Waiting for provider recovery.",
+        )
+        return FailoverOutcome(
+            fallback, fallback_result, candidate_file, route.provider, True
+        )
+    if fallback_result.structured:
+        candidate_file = _persist_candidate(
+            state,
+            state_file,
+            dict(fallback_result.structured),
+            route=fallback,
+            returncode=fallback_result.returncode,
+            goal_turn=goal_turn,
+        )
+    return FailoverOutcome(
+        fallback, fallback_result, candidate_file, route.provider, False
+    )

services/hermes/scripts/cli_lane_health.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""Lane-observed provider health with failure classification and cooldowns."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import time
+from typing import Any
+
+from cli_lane_config import (
+    AUTH_COOLDOWN_SECONDS_DEFAULT,
+    CAPACITY_COOLDOWN_SECONDS_DEFAULT,
+    PROVIDER_HEALTH_PATHS,
+    kanban_setting,
+    utc_now,
+)
+from cli_lane_files import load_json
+
+
+# Ordered from most to least specific so one output maps to one actionable
+# reason instead of the single conflated capacity signal.
+FAILURE_CLASSIFICATIONS = (
+    ("auth", re.compile(r"authentication|unauthorized|forbidden|oauth|token.*expired|401|403", re.I)),
+    ("rate-limit", re.compile(r"rate.?limit|429|529|overload", re.I)),
+    ("quota", re.compile(r"usage.?limit|quota|credit|exhaust|capacity", re.I)),
+)
+
+
+def classify_capacity_failure(output: str) -> str:
+    """Split the broad capacity regex into quota/auth/rate-limit/transport."""
+    for reason, pattern in FAILURE_CLASSIFICATIONS:
+        if pattern.search(output):
+            return reason
+    return "transport"
+
+
+def _write_provider_health(
+    provider: str, updates: dict[str, Any], clear: tuple[str, ...] = ()
+) -> None:
+    """Merge one lane observation into the shared provider health snapshot."""
+    path = PROVIDER_HEALTH_PATHS[provider]
+    value = load_json(path)
+    for key in clear:
+        value.pop(key, None)
+    value.update(updates)
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
+        temporary.write_text(
+            json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+        )
+        os.replace(temporary, path)
+    except OSError:
+        # Health snapshots are advisory; never fail the lane over one write.
+        pass
+
+
+def record_provider_success(provider: str) -> None:
+    """Publish fresh evidence of success so the provider re-enters routing."""
+    _write_provider_health(
+        provider,
+        {
+            "source": "cli-lane-runner",
+            "state": "available",
+            "authenticated": True,
+            "checked_at": utc_now(),
+        },
+        clear=("failure_reason", "failed_at", "cooldown_until"),
+    )
+
+
+def record_provider_failure(
+    provider: str, reason: str, now: float | None = None
+) -> float:
+    """Persist one lane-observed failure with its re-admission cooldown."""
+    current = time.time() if now is None else now
+    if reason == "auth":
+        cooldown = kanban_setting(
+            "provider_auth_cooldown_seconds", AUTH_COOLDOWN_SECONDS_DEFAULT
+        )
+        state, authenticated = "unavailable", False
+    else:
+        cooldown = kanban_setting(
+            "provider_capacity_cooldown_seconds", CAPACITY_COOLDOWN_SECONDS_DEFAULT
+        )
+        state, authenticated = "capacity-limited", True
+    cooldown_until = current + max(0.0, cooldown)
+    _write_provider_health(
+        provider,
+        {
+            "source": "cli-lane-runner",
+            "state": state,
+            "authenticated": authenticated,
+            "checked_at": utc_now(),
+            "failure_reason": reason,
+            "failed_at": current,
+            "cooldown_until": cooldown_until,
+        },
+    )
+    return cooldown_until

services/hermes/scripts/cli_lane_metrics.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+"""In-process Prometheus metrics for the direct CLI lane's routing decisions."""
+
+from __future__ import annotations
+
+import os
+import sys
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+
+METRIC_HELP = {
+    "hermes_cli_route_decisions_total": (
+        "Terminal outcome of each executed CLI lane route decision."
+    ),
+    "hermes_cli_provider_fallbacks_total": (
+        "Cross-provider failovers taken by the CLI lane, by failure reason."
+    ),
+    "hermes_cli_router_selection_failures_total": (
+        "Switchyard route selections that failed and blocked a card as transient."
+    ),
+    "hermes_cli_quota_fetch_failures_total": (
+        "Quota snapshot reads that failed; routing then fails open."
+    ),
+    "hermes_cli_quota_remaining_percent": (
+        "Binding remaining-quota percent per provider as seen by the CLI lane."
+    ),
+    "hermes_cli_quota_reset_timestamp_seconds": (
+        "Unix reset time of the binding quota window as seen by the CLI lane."
+    ),
+    "hermes_cli_provider_soft_excluded": (
+        "Whether new automatic work is currently routed away from a provider."
+    ),
+}
+COUNTER_NAMES = frozenset(name for name in METRIC_HELP if name.endswith("_total"))
+
+
+def _escape_label(value: str) -> str:
+    """Escape one Prometheus label value."""
+    return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
+
+
+class LaneMetricsRegistry:
+    """Thread-safe counter and gauge store rendered in Prometheus text format."""
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._values: dict[tuple[str, tuple[tuple[str, str], ...]], float] = {}
+
+    def _key(
+        self, name: str, labels: dict[str, str] | None
+    ) -> tuple[str, tuple[tuple[str, str], ...]]:
+        return name, tuple(sorted((labels or {}).items()))
+
+    def increment(
+        self, name: str, labels: dict[str, str] | None = None, amount: float = 1.0
+    ) -> None:
+        key = self._key(name, labels)
+        with self._lock:
+            self._values[key] = self._values.get(key, 0.0) + amount
+
+    def set_value(
+        self, name: str, labels: dict[str, str] | None, value: float
+    ) -> None:
+        with self._lock:
+            self._values[self._key(name, labels)] = value
+
+    def render(self) -> bytes:
+        """Render every recorded sample with stable ordering."""
+        with self._lock:
+            values = dict(self._values)
+        lines: list[str] = []
+        for name in sorted({name for name, _ in values}):
+            kind = "counter" if name in COUNTER_NAMES else "gauge"
+            lines.extend((f"# HELP {name} {METRIC_HELP[name]}", f"# TYPE {name} {kind}"))
+            for (sample_name, labels), value in sorted(values.items()):
+                if sample_name != name:
+                    continue
+                rendered = ",".join(
+                    f'{key}="{_escape_label(text)}"' for key, text in labels
+                )
+                body = f"{{{rendered}}}" if rendered else ""
+                lines.append(f"{name}{body} {value:.12g}")
+        return ("\n".join(lines) + "\n").encode("utf-8")
+
+
+METRICS = LaneMetricsRegistry()
+
+
+def record_route_decision(
+    provider: str, effort: str, classifier: str, outcome: str
+) -> None:
+    """Count one executed route with its terminal outcome."""
+    METRICS.increment(
+        "hermes_cli_route_decisions_total",
+        {
+            "provider": provider,
+            "effort": effort,
+            "classifier": classifier,
+            "outcome": outcome,
+        },
+    )
+
+
+def record_provider_fallback(
+    from_provider: str, to_provider: str, reason: str
+) -> None:
+    """Count one cross-provider failover with its classified reason."""
+    METRICS.increment(
+        "hermes_cli_provider_fallbacks_total",
+        {
+            "from_provider": from_provider,
+            "to_provider": to_provider,
+            "reason": reason,
+        },
+    )
+
+
+def record_router_selection_failure() -> None:
+    """Count one Switchyard outage observed at a selection boundary."""
+    METRICS.increment("hermes_cli_router_selection_failures_total")
+
+
+def record_quota_fetch_failure() -> None:
+    """Count one unavailable quota snapshot; routing continues fail-open."""
+    METRICS.increment("hermes_cli_quota_fetch_failures_total")
+
+
+def record_provider_quota(
+    provider: str, remaining_percent: float, reset_timestamp: float | None
+) -> None:
+    """Publish the binding quota window the lane based its routing on."""
+    labels = {"provider": provider}
+    METRICS.set_value(
+        "hermes_cli_quota_remaining_percent", labels, remaining_percent
+    )
+    if reset_timestamp is not None:
+        METRICS.set_value(
+            "hermes_cli_quota_reset_timestamp_seconds", labels, reset_timestamp
+        )
+
+
+def record_soft_exclusion(provider: str, excluded: bool) -> None:
+    """Publish whether new automatic work avoids this provider right now."""
+    METRICS.set_value(
+        "hermes_cli_provider_soft_excluded",
+        {"provider": provider},
+        1.0 if excluded else 0.0,
+    )
+
+
+def start_metrics_server(port: int | None = None) -> ThreadingHTTPServer | None:
+    """Serve lane metrics on the pod-local scrape port without blocking dispatch."""
+    selected_port = (
+        int(os.environ.get("HERMES_CLI_LANE_METRICS_PORT", "9011"))
+        if port is None
+        else port
+    )
+
+    class Handler(BaseHTTPRequestHandler):
+        def do_GET(self) -> None:  # noqa: N802
+            if self.path == "/metrics":
+                payload = METRICS.render()
+                self.send_response(200)
+                self.send_header("Content-Type", "text/plain; version=0.0.4")
+                self.send_header("Content-Length", str(len(payload)))
+                self.end_headers()
+                self.wfile.write(payload)
+            elif self.path == "/healthz":
+                self.send_response(200)
+                self.end_headers()
+            else:
+                self.send_error(404)
+
+        def log_message(self, _format: str, *_args: object) -> None:
+            return
+
+    try:
+        server = ThreadingHTTPServer(("0.0.0.0", selected_port), Handler)
+    except OSError as error:
+        print(
+            f"cli lane metrics server unavailable: {error}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return None
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    return server

services/hermes/scripts/cli_lane_provider.py

@@ -197,6 +197,10 @@ def _terminate_worker_process(
 def _base_env() -> dict[str, str]:
     """Build a worker environment while preserving Vault-backed CLI homes."""
     env = os.environ.copy()
+    # Subscription CLIs only: a vendor API key in the environment could
+    # silently flip a worker onto a metered path, so drop every key here.
+    for secret in ("ANTHROPIC_API_KEY", "CLAUDE_API_KEY", "OPENAI_API_KEY", "API_SERVER_KEY"):
+        env.pop(secret, None)
     env.update(
         {
             "HOME": str(DATA_ROOT / "home"),

services/hermes/scripts/cli_lane_quota.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+"""Quota-aware soft routing for automatic CLI lane provider selection."""
+
+from __future__ import annotations
+
+import re
+import sys
+import urllib.request
+from dataclasses import dataclass
+from typing import Callable
+
+from cli_lane_config import (
+    QUOTA_METRICS_URL,
+    QUOTA_MIN_REMAINING_PERCENT_DEFAULT,
+    kanban_setting,
+)
+from cli_lane_metrics import (
+    record_provider_quota,
+    record_quota_fetch_failure,
+    record_soft_exclusion,
+)
+from cli_lane_routing import fresh_unavailable_provider
+
+
+# Exporter accounts are labeled by vendor; the lane routes by CLI provider.
+QUOTA_PROVIDER_LABELS = {"openai": "codex", "anthropic": "claude"}
+# Gate only on account-wide windows: a model-specific Claude window (for
+# example seven_day_opus) must not veto the whole provider.
+GATED_QUOTA_WINDOWS = {
+    "codex": None,
+    "claude": ("five_hour", "seven_day"),
+}
+_SAMPLE_PATTERN = re.compile(
+    r"^atlas_ai_quota_(remaining_percent|reset_timestamp_seconds|fetch_success)"
+    r"\{([^}]*)\} (-?[0-9.eE+]+)$"
+)
+_LABEL_PATTERN = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="([^"]*)"')
+
+
+@dataclass(frozen=True)
+class ProviderQuota:
+    """One provider's binding remaining quota as seen by the lane."""
+
+    remaining_percent: float
+    reset_timestamp: float | None
+
+
+@dataclass(frozen=True)
+class SelectionConstraint:
+    """One provider exclusion decided at a routing boundary, with evidence."""
+
+    exclude_provider: str | None
+    exclude_reason: str | None
+    source: str | None
+    notes: tuple[str, ...]
+
+
+def _gated_window(provider: str, labels: dict[str, str]) -> str | None:
+    """Return the window key when a sample belongs to a gated account limit."""
+    if labels.get("limit") != provider:
+        return None
+    window = labels.get("window", "")
+    allowed = GATED_QUOTA_WINDOWS[provider]
+    if allowed is not None and window not in allowed:
+        return None
+    return window
+
+
+def parse_quota_metrics(text: str) -> dict[str, ProviderQuota]:
+    """Reduce exporter samples to each provider's tightest account window."""
+    remaining: dict[tuple[str, str], float] = {}
+    resets: dict[tuple[str, str], float] = {}
+    fetch_success: dict[str, bool] = {}
+    for line in text.splitlines():
+        match = _SAMPLE_PATTERN.match(line.strip())
+        if not match:
+            continue
+        metric, raw_labels, raw_value = match.groups()
+        labels = dict(_LABEL_PATTERN.findall(raw_labels))
+        provider = QUOTA_PROVIDER_LABELS.get(labels.get("provider", ""))
+        if provider is None:
+            continue
+        value = float(raw_value)
+        if metric == "fetch_success":
+            fetch_success[provider] = value >= 1.0
+            continue
+        window = _gated_window(provider, labels)
+        if window is None:
+            continue
+        if metric == "remaining_percent":
+            remaining[(provider, window)] = value
+        else:
+            resets[(provider, window)] = value
+    snapshot: dict[str, ProviderQuota] = {}
+    for provider in QUOTA_PROVIDER_LABELS.values():
+        windows = {
+            window: value
+            for (candidate, window), value in remaining.items()
+            if candidate == provider
+        }
+        if not windows or not fetch_success.get(provider, True):
+            # Failed or stale exporter fetches leave this provider ungated.
+            continue
+        binding = min(windows, key=lambda window: (windows[window], window))
+        snapshot[provider] = ProviderQuota(
+            remaining_percent=windows[binding],
+            reset_timestamp=resets.get((provider, binding)),
+        )
+    return snapshot
+
+
+def fetch_quota_snapshot(
+    url: str = QUOTA_METRICS_URL,
+    open_url: Callable = urllib.request.urlopen,
+) -> dict[str, ProviderQuota]:
+    """Read the in-pod usage exporter; empty means the signal is unavailable."""
+    try:
+        with open_url(url, timeout=10) as response:
+            text = response.read(1 << 20).decode("utf-8", errors="replace")
+        snapshot = parse_quota_metrics(text)
+    except (OSError, ValueError) as error:
+        record_quota_fetch_failure()
+        print(
+            f"quota snapshot unavailable; routing fails open: {type(error).__name__}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return {}
+    if not snapshot:
+        record_quota_fetch_failure()
+        print(
+            "quota snapshot carried no gated windows; routing fails open",
+            file=sys.stderr,
+            flush=True,
+        )
+    return snapshot
+
+
+def quota_soft_exclusion(
+    snapshot: dict[str, ProviderQuota], threshold: float
+) -> tuple[str | None, str | None]:
+    """Choose at most one provider to soft-exclude from new automatic work."""
+    below = {
+        provider: quota.remaining_percent
+        for provider, quota in snapshot.items()
+        if quota.remaining_percent < threshold
+    }
+    if not below:
+        return None, None
+    excluded = min(below, key=lambda provider: (below[provider], provider))
+    if len(below) == len(snapshot) and len(snapshot) > 1:
+        preferred = next(name for name in snapshot if name != excluded)
+        note = (
+            "Quota degraded: every provider is below the "
+            f"{threshold:.6g}% remaining threshold; preferring {preferred} "
+            f"({snapshot[preferred].remaining_percent:.6g}% left) over "
+            f"{excluded} ({below[excluded]:.6g}% left) for new auto work."
+        )
+        return excluded, note
+    note = (
+        f"Quota guard: {excluded} has {below[excluded]:.6g}% remaining "
+        f"(threshold {threshold:.6g}%); routing new auto work to the other "
+        "provider while active work finishes."
+    )
+    return excluded, note
+
+
+def selection_constraint(
+    assignee: str,
+    *,
+    snapshot_from: Callable[[], dict[str, ProviderQuota]] | None = None,
+    now: float | None = None,
+) -> SelectionConstraint:
+    """Combine provider health and quota gating for one selection boundary."""
+    if assignee != "cli-auto":
+        # Explicit lanes are operator decisions; they are never re-routed.
+        return SelectionConstraint(None, None, None, ())
+    snapshot = (snapshot_from or fetch_quota_snapshot)()
+    resets: dict[str, float] = {}
+    for provider, quota in snapshot.items():
+        record_provider_quota(provider, quota.remaining_percent, quota.reset_timestamp)
+        if quota.reset_timestamp is not None:
+            resets[provider] = quota.reset_timestamp
+    health_excluded = fresh_unavailable_provider(now=now, quota_resets=resets)
+    if health_excluded is not None:
+        return SelectionConstraint(
+            health_excluded,
+            "is unavailable according to fresh native health",
+            "health",
+            (),
+        )
+    threshold = kanban_setting(
+        "provider_quota_min_remaining_percent", QUOTA_MIN_REMAINING_PERCENT_DEFAULT
+    )
+    excluded, note = quota_soft_exclusion(snapshot, threshold)
+    for provider in snapshot:
+        record_soft_exclusion(provider, provider == excluded)
+    if excluded is None or note is None:
+        return SelectionConstraint(None, None, None, ())
+    return SelectionConstraint(
+        excluded,
+        "is below its remaining-quota routing threshold",
+        "quota",
+        (note,),
+    )

services/hermes/scripts/cli_lane_routing.py

@@ -31,12 +31,49 @@ def parse_assignee(assignee: str) -> tuple[str | None, str | None]:
         raise ValueError(f"unsupported external lane assignee: {assignee}")
     return match.group(1), match.group(2)
 
-def fresh_unavailable_provider(now: float | None = None) -> str | None:
+def _health_number(value: Any) -> float | None:
+    """Return a plain numeric health field, rejecting booleans and strings."""
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    return float(value)
+
+def _quota_reset_readmits(
+    health: dict[str, Any], current: float, reset_timestamp: float | None
+) -> bool:
+    """A passed provider quota reset ends a capacity cooldown early."""
+    if health.get("authenticated") is False:
+        # An authentication failure is not repaired by a quota window reset.
+        return False
+    failed_at = _health_number(health.get("failed_at"))
+    return (
+        reset_timestamp is not None
+        and failed_at is not None
+        and failed_at < reset_timestamp <= current
+    )
+
+def fresh_unavailable_provider(
+    now: float | None = None,
+    quota_resets: dict[str, float] | None = None,
+) -> str | None:
     """Return one recently proven-down provider for automatic route exclusion."""
     current = time.time() if now is None else now
+    resets = quota_resets or {}
     unavailable: list[str] = []
     for provider, path in PROVIDER_HEALTH_PATHS.items():
         health = load_json(path)
+        if health.get("state") not in ("unavailable", "capacity-limited"):
+            continue
+        cooldown_until = _health_number(health.get("cooldown_until"))
+        if cooldown_until is not None:
+            # Lane-recorded failures re-enter only after the full cooldown has
+            # elapsed (hysteresis), a quota reset passed, or fresh success
+            # rewrote the state above.
+            if current >= cooldown_until or _quota_reset_readmits(
+                health, current, resets.get(provider)
+            ):
+                continue
+            unavailable.append(provider)
+            continue
         try:
             age = current - path.stat().st_mtime
         except OSError:
@@ -46,10 +83,7 @@ def fresh_unavailable_provider(now: float | None = None) -> str | None:
             if health.get("authenticated") is False
             else PROVIDER_HEALTH_MAX_AGE_SECONDS
         )
-        if (
+        if 0 <= age <= max_age:
-            0 <= age <= max_age
-            and health.get("state") == "unavailable"
-        ):
             unavailable.append(provider)
     # If both providers are down, keep the normal attempt/fallback path so the
     # card records authoritative current errors instead of trusting snapshots.

services/hermes/scripts/cli_lane_runner.py

@@ -32,6 +32,11 @@ from cli_lane_board import (
 )
 from cli_lane_config import (
     ARTIFACT_GC_INTERVAL_SECONDS,
+    AUTH_COOLDOWN_SECONDS_DEFAULT,
+    CAPACITY_COOLDOWN_SECONDS_DEFAULT,
+    QUOTA_METRICS_URL,
+    QUOTA_MIN_REMAINING_PERCENT_DEFAULT,
+    kanban_setting,
     ARTIFACT_RETENTION_AGE_SECONDS,
     ARTIFACT_RETENTION_BYTES,
     ARTIFACT_RETENTION_COUNT,
@@ -67,6 +72,35 @@ from cli_lane_evidence import (
     _write_json_noreplace,
 )
 from cli_lane_execution import execute_claim
+from cli_lane_failover import (
+    FailoverOutcome,
+    _block_transient,
+    _routed_or_blocked,
+    capacity_failover,
+)
+from cli_lane_health import (
+    classify_capacity_failure,
+    record_provider_failure,
+    record_provider_success,
+)
+from cli_lane_metrics import (
+    METRICS,
+    record_provider_fallback,
+    record_provider_quota,
+    record_quota_fetch_failure,
+    record_route_decision,
+    record_router_selection_failure,
+    record_soft_exclusion,
+    start_metrics_server,
+)
+from cli_lane_quota import (
+    ProviderQuota,
+    SelectionConstraint,
+    fetch_quota_snapshot,
+    parse_quota_metrics,
+    quota_soft_exclusion,
+    selection_constraint,
+)
 from cli_lane_files import (
     _candidate_path,
     _fsync_directory,

testing/quality_contract.json

@@ -26,12 +26,16 @@
     "services/hermes/scripts/cli_lane_dispatch.py",
     "services/hermes/scripts/cli_lane_evidence.py",
     "services/hermes/scripts/cli_lane_execution.py",
+    "services/hermes/scripts/cli_lane_failover.py",
     "services/hermes/scripts/cli_lane_files.py",
     "services/hermes/scripts/cli_lane_finalization.py",
     "services/hermes/scripts/cli_lane_goal.py",
+    "services/hermes/scripts/cli_lane_health.py",
+    "services/hermes/scripts/cli_lane_metrics.py",
     "services/hermes/scripts/cli_lane_prompt.py",
     "services/hermes/scripts/cli_lane_provider.py",
     "services/hermes/scripts/cli_lane_quarantine.py",
+    "services/hermes/scripts/cli_lane_quota.py",
     "services/hermes/scripts/cli_lane_records.py",
     "services/hermes/scripts/cli_lane_recovery.py",
     "services/hermes/scripts/cli_lane_retention.py",
@@ -45,10 +49,10 @@
     "testing/quality_gate.py",
     "ci/tests/glue/test_ariadne_schedules.py",
     "ci/tests/glue/test_glue_metrics.py",
-      "testing/tests/test_publish_test_metrics.py",
+    "testing/tests/test_publish_test_metrics.py",
-      "testing/tests/test_supply_chain_report.py",
+    "testing/tests/test_supply_chain_report.py",
-      "testing/tests/test_semgrep_report.py",
+    "testing/tests/test_semgrep_report.py",
-      "testing/tests/test_quality_contract.py",
+    "testing/tests/test_quality_contract.py",
     "testing/tests/test_quality_gate.py"
   ],
   "lint_paths": [
@@ -66,12 +70,16 @@
     "services/hermes/scripts/cli_lane_dispatch.py",
     "services/hermes/scripts/cli_lane_evidence.py",
     "services/hermes/scripts/cli_lane_execution.py",
+    "services/hermes/scripts/cli_lane_failover.py",
     "services/hermes/scripts/cli_lane_files.py",
     "services/hermes/scripts/cli_lane_finalization.py",
     "services/hermes/scripts/cli_lane_goal.py",
+    "services/hermes/scripts/cli_lane_health.py",
+    "services/hermes/scripts/cli_lane_metrics.py",
     "services/hermes/scripts/cli_lane_prompt.py",
     "services/hermes/scripts/cli_lane_provider.py",
     "services/hermes/scripts/cli_lane_quarantine.py",
+    "services/hermes/scripts/cli_lane_quota.py",
     "services/hermes/scripts/cli_lane_records.py",
     "services/hermes/scripts/cli_lane_recovery.py",
     "services/hermes/scripts/cli_lane_retention.py",
@@ -216,12 +224,16 @@
       "services/hermes/scripts/cli_lane_dispatch.py",
       "services/hermes/scripts/cli_lane_evidence.py",
       "services/hermes/scripts/cli_lane_execution.py",
+      "services/hermes/scripts/cli_lane_failover.py",
       "services/hermes/scripts/cli_lane_files.py",
       "services/hermes/scripts/cli_lane_finalization.py",
       "services/hermes/scripts/cli_lane_goal.py",
+      "services/hermes/scripts/cli_lane_health.py",
+      "services/hermes/scripts/cli_lane_metrics.py",
       "services/hermes/scripts/cli_lane_prompt.py",
       "services/hermes/scripts/cli_lane_provider.py",
       "services/hermes/scripts/cli_lane_quarantine.py",
+      "services/hermes/scripts/cli_lane_quota.py",
       "services/hermes/scripts/cli_lane_records.py",
       "services/hermes/scripts/cli_lane_recovery.py",
       "services/hermes/scripts/cli_lane_retention.py",