hermes: fail over exhausted Claude routes

services/hermes/scripts/claude_oauth_broker.py

@@ -38,6 +38,14 @@ REQUIRED_BETAS: Final = (
     "oauth-2025-04-20",
 )
 ROUTED_MODEL_PREFIX: Final = "route/claude/"
+CAPACITY_ERROR_MARKERS: Final = (
+    "extra usage",
+    "plan limits",
+    "usage limit",
+    "rate limit",
+    "credit balance",
+    "claude.ai/settings/usage",
+)
 
 
 def _translate_model(body: bytes) -> bytes:
@@ -109,6 +117,16 @@ def _merge_betas(incoming: str | None) -> str:
     return ",".join(values)
 
 
+def _normalized_upstream_status(status: int, body: bytes) -> int:
+    """Expose provider capacity exhaustion using Switchyard's retryable status."""
+    if status != 400:
+        return status
+    text = body.decode("utf-8", errors="replace").lower()
+    if any(marker in text for marker in CAPACITY_ERROR_MARKERS):
+        return 429
+    return status
+
+
 class Handler(BaseHTTPRequestHandler):
     """Stream Anthropic responses while keeping the OAuth token server-side."""
 
@@ -190,6 +208,25 @@ class Handler(BaseHTTPRequestHandler):
                     headers=headers,
                     content=body or None,
                 ) as response:
+                    if response.status_code >= 400:
+                        error_body = response.read()
+                        self.send_response(
+                            _normalized_upstream_status(
+                                response.status_code, error_body
+                            )
+                        )
+                        self.send_header(
+                            "Content-Type",
+                            response.headers.get("Content-Type", "application/json"),
+                        )
+                        self.send_header("Content-Length", str(len(error_body)))
+                        self.send_header("Cache-Control", "no-store")
+                        self.send_header("Connection", "close")
+                        self.end_headers()
+                        response_started = True
+                        self.wfile.write(error_body)
+                        return
+
                     self.send_response(response.status_code)
                     for name, value in response.headers.items():
                         if name.lower() in {

services/hermes/switchyard-deployment.yaml

@@ -22,7 +22,7 @@ spec:
       labels:
         app: hermes-switchyard
       annotations:
-        ai.bstein.dev/config-rev: "20260811-switchyard-authority-v11"
+        ai.bstein.dev/config-rev: "20260811-switchyard-authority-v12"
         prometheus.io/scrape: "true"
         prometheus.io/port: "9005"
         prometheus.io/path: /metrics

testing/tests/test_hermes_chat_quality.py

@@ -24,6 +24,26 @@ def _documents(path: Path) -> list[dict]:
     return [doc for doc in yaml.safe_load_all(path.read_text()) if doc]
 
 
+def _load_broker_module(name: str, filename: str, monkeypatch):
+    """Load one broker with its mounted routing-catalog dependency."""
+    catalog_path = HERMES / "scripts" / "routing_catalog.py"
+    catalog_spec = importlib.util.spec_from_file_location(
+        "routing_catalog", catalog_path
+    )
+    assert catalog_spec and catalog_spec.loader
+    catalog = importlib.util.module_from_spec(catalog_spec)
+    catalog_spec.loader.exec_module(catalog)
+    monkeypatch.setitem(sys.modules, "routing_catalog", catalog)
+    monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace())
+
+    broker_path = HERMES / "scripts" / filename
+    spec = importlib.util.spec_from_file_location(name, broker_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
 def test_chat_config_enables_real_research_compute_and_delegation():
     configmap = _documents(HERMES / "chat-configmap.yaml")[0]
     config = yaml.safe_load(configmap["data"]["config.yaml"])
@@ -488,12 +508,9 @@ def test_chat_reasoning_uses_switchyard_without_owner_credentials():
 
 def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch):
     """The relay is bounded, stateless, and rejects unapproved models."""
-    broker_path = HERMES / "scripts" / "codex_broker.py"
-    monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace())
-    spec = importlib.util.spec_from_file_location("hermes_codex_broker", broker_path)
-    assert spec and spec.loader
-    module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)
+    module = _load_broker_module(
+        "hermes_codex_broker", "codex_broker.py", monkeypatch
+    )
     monkeypatch.setattr(module, "TOKEN", "relay-secret")
 
     assert module._authorized("Bearer relay-secret") is True
@@ -595,6 +612,30 @@ def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch):
     assert module._access_token().startswith("header.")
 
 
+def test_claude_broker_exposes_capacity_exhaustion_as_retryable(monkeypatch):
+    """Subscription exhaustion must cross providers instead of surfacing as 400."""
+    module = _load_broker_module(
+        "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch
+    )
+    exhausted = json.dumps(
+        {
+            "type": "error",
+            "error": {
+                "type": "invalid_request_error",
+                "message": (
+                    "Third-party apps now draw from your extra usage, not your "
+                    "plan limits. Add more at claude.ai/settings/usage."
+                ),
+            },
+        }
+    ).encode()
+    malformed = b'{"error":{"message":"invalid tool schema"}}'
+
+    assert module._normalized_upstream_status(400, exhausted) == 429
+    assert module._normalized_upstream_status(400, malformed) == 400
+    assert module._normalized_upstream_status(403, exhausted) == 403
+
+
 def test_image_broker_returns_bytes_and_removes_owner_cache(tmp_path: Path, monkeypatch):
     """The broker must not retain a family user's generated image."""
     broker_path = HERMES / "scripts" / "image_broker.py"