diff --git a/services/hermes/scripts/claude_oauth_broker.py b/services/hermes/scripts/claude_oauth_broker.py index 1892c5584..c8a382889 100644 --- a/services/hermes/scripts/claude_oauth_broker.py +++ b/services/hermes/scripts/claude_oauth_broker.py @@ -38,6 +38,14 @@ REQUIRED_BETAS: Final = ( "oauth-2025-04-20", ) ROUTED_MODEL_PREFIX: Final = "route/claude/" +CAPACITY_ERROR_MARKERS: Final = ( + "extra usage", + "plan limits", + "usage limit", + "rate limit", + "credit balance", + "claude.ai/settings/usage", +) def _translate_model(body: bytes) -> bytes: @@ -109,6 +117,16 @@ def _merge_betas(incoming: str | None) -> str: return ",".join(values) +def _normalized_upstream_status(status: int, body: bytes) -> int: + """Expose provider capacity exhaustion using Switchyard's retryable status.""" + if status != 400: + return status + text = body.decode("utf-8", errors="replace").lower() + if any(marker in text for marker in CAPACITY_ERROR_MARKERS): + return 429 + return status + + class Handler(BaseHTTPRequestHandler): """Stream Anthropic responses while keeping the OAuth token server-side.""" @@ -190,6 +208,25 @@ class Handler(BaseHTTPRequestHandler): headers=headers, content=body or None, ) as response: + if response.status_code >= 400: + error_body = response.read() + self.send_response( + _normalized_upstream_status( + response.status_code, error_body + ) + ) + self.send_header( + "Content-Type", + response.headers.get("Content-Type", "application/json"), + ) + self.send_header("Content-Length", str(len(error_body))) + self.send_header("Cache-Control", "no-store") + self.send_header("Connection", "close") + self.end_headers() + response_started = True + self.wfile.write(error_body) + return + self.send_response(response.status_code) for name, value in response.headers.items(): if name.lower() in { diff --git a/services/hermes/switchyard-deployment.yaml b/services/hermes/switchyard-deployment.yaml index 39eec47ac..a0fd975ba 100644 --- a/services/hermes/switchyard-deployment.yaml +++ b/services/hermes/switchyard-deployment.yaml @@ -22,7 +22,7 @@ spec: labels: app: hermes-switchyard annotations: - ai.bstein.dev/config-rev: "20260811-switchyard-authority-v11" + ai.bstein.dev/config-rev: "20260811-switchyard-authority-v12" prometheus.io/scrape: "true" prometheus.io/port: "9005" prometheus.io/path: /metrics diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 8e0cf8113..0a36ce3bf 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -24,6 +24,26 @@ def _documents(path: Path) -> list[dict]: return [doc for doc in yaml.safe_load_all(path.read_text()) if doc] +def _load_broker_module(name: str, filename: str, monkeypatch): + """Load one broker with its mounted routing-catalog dependency.""" + catalog_path = HERMES / "scripts" / "routing_catalog.py" + catalog_spec = importlib.util.spec_from_file_location( + "routing_catalog", catalog_path + ) + assert catalog_spec and catalog_spec.loader + catalog = importlib.util.module_from_spec(catalog_spec) + catalog_spec.loader.exec_module(catalog) + monkeypatch.setitem(sys.modules, "routing_catalog", catalog) + monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) + + broker_path = HERMES / "scripts" / filename + spec = importlib.util.spec_from_file_location(name, broker_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def test_chat_config_enables_real_research_compute_and_delegation(): configmap = _documents(HERMES / "chat-configmap.yaml")[0] config = yaml.safe_load(configmap["data"]["config.yaml"]) @@ -488,12 +508,9 @@ def test_chat_reasoning_uses_switchyard_without_owner_credentials(): def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): """The relay is bounded, stateless, and rejects unapproved models.""" - broker_path = HERMES / "scripts" / "codex_broker.py" - monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace()) - spec = importlib.util.spec_from_file_location("hermes_codex_broker", broker_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + module = _load_broker_module( + "hermes_codex_broker", "codex_broker.py", monkeypatch + ) monkeypatch.setattr(module, "TOKEN", "relay-secret") assert module._authorized("Bearer relay-secret") is True @@ -595,6 +612,30 @@ def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): assert module._access_token().startswith("header.") +def test_claude_broker_exposes_capacity_exhaustion_as_retryable(monkeypatch): + """Subscription exhaustion must cross providers instead of surfacing as 400.""" + module = _load_broker_module( + "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch + ) + exhausted = json.dumps( + { + "type": "error", + "error": { + "type": "invalid_request_error", + "message": ( + "Third-party apps now draw from your extra usage, not your " + "plan limits. Add more at claude.ai/settings/usage." + ), + }, + } + ).encode() + malformed = b'{"error":{"message":"invalid tool schema"}}' + + assert module._normalized_upstream_status(400, exhausted) == 429 + assert module._normalized_upstream_status(400, malformed) == 400 + assert module._normalized_upstream_status(403, exhausted) == 403 + + def test_image_broker_returns_bytes_and_removes_owner_cache(tmp_path: Path, monkeypatch): """The broker must not retain a family user's generated image.""" broker_path = HERMES / "scripts" / "image_broker.py"