diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index ac2a6b75..4b1d40ce 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -1045,14 +1045,15 @@ spec: - {name: HERMES_CLAUDE_BROKER_PORT, value: "9006"} - {name: HERMES_CLAUDE_BROKER_KEY_FILE, value: /runtime-access/chat-relay-key} - {name: HERMES_CLAUDE_BROKER_READ_TIMEOUT, value: "1800"} + - {name: HERMES_CLAUDE_BROKER_CONCURRENCY, value: "2"} - {name: HERMES_CLAUDE_HEALTH_PATH, value: /opt/data/provider-health/claude.json} - {name: HERMES_ROUTING_CATALOG_PATH, value: /routing-catalog/catalog.json} readinessProbe: - httpGet: {path: /health, port: claude-broker} + tcpSocket: {port: claude-broker} initialDelaySeconds: 5 periodSeconds: 10 livenessProbe: - httpGet: {path: /health, port: claude-broker} + tcpSocket: {port: claude-broker} initialDelaySeconds: 30 periodSeconds: 30 securityContext: diff --git a/services/hermes/scripts/claude_oauth_broker.py b/services/hermes/scripts/claude_oauth_broker.py index df44605f..4e0dd1a3 100644 --- a/services/hermes/scripts/claude_oauth_broker.py +++ b/services/hermes/scripts/claude_oauth_broker.py @@ -525,7 +525,13 @@ class Handler(BaseHTTPRequestHandler): self.send_header("Content-Length", str(len(body))) self.send_header("Cache-Control", "no-store") self.end_headers() - self.wfile.write(body) + try: + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + # Kubernetes and callers may close a health/request socket after + # their own deadline. The broker is still healthy and must not + # emit a noisy handler traceback for that normal disconnect. + return def _error(self, status: int, error_type: str, message: str) -> None: self._json( @@ -623,7 +629,10 @@ class Handler(BaseHTTPRequestHandler): self.send_header("Cache-Control", "no-store") self.send_header("Content-Length", str(len(body))) self.end_headers() - self.wfile.write(body) + try: + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + return else: self._json(200, message) diff --git a/services/hermes/service.yaml b/services/hermes/service.yaml index e8cddd11..0fc8d939 100644 --- a/services/hermes/service.yaml +++ b/services/hermes/service.yaml @@ -88,6 +88,7 @@ metadata: app: hermes-agent spec: type: ClusterIP + publishNotReadyAddresses: true selector: app: hermes-agent ports: @@ -105,6 +106,7 @@ metadata: app: hermes-agent spec: type: ClusterIP + publishNotReadyAddresses: true selector: app: hermes-agent ports: @@ -122,6 +124,7 @@ metadata: app: hermes-local-image spec: type: ClusterIP + publishNotReadyAddresses: true selector: app: hermes-local-image ports: @@ -139,6 +142,7 @@ metadata: app: hermes-agent spec: type: ClusterIP + publishNotReadyAddresses: true selector: app: hermes-agent ports: diff --git a/testing/tests/test_hermes_claude_broker.py b/testing/tests/test_hermes_claude_broker.py index 4d1eb398..8b019fff 100644 --- a/testing/tests/test_hermes_claude_broker.py +++ b/testing/tests/test_hermes_claude_broker.py @@ -4,6 +4,7 @@ from __future__ import annotations import importlib.util import sys +from io import BytesIO from pathlib import Path from types import ModuleType @@ -99,3 +100,22 @@ def test_unadvertised_tool_call_is_retryable(monkeypatch) -> None: }, _payload(), ) + + +@pytest.mark.parametrize("disconnect", [BrokenPipeError, ConnectionResetError]) +def test_json_response_tolerates_disconnected_client(monkeypatch, disconnect) -> None: + module = _module(monkeypatch) + handler = object.__new__(module.Handler) + handler.request_version = "HTTP/1.1" + handler.command = "GET" + handler.send_response = lambda _status: None + handler.send_header = lambda _name, _value: None + handler.end_headers = lambda: None + + class Disconnected(BytesIO): + def write(self, _body): + raise disconnect + + handler.wfile = Disconnected() + + handler._json(200, {"ok": True}) diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index dc6ca7ee..30c74d3e 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -47,6 +47,14 @@ def _agent_deployment() -> dict: return yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) +def _services() -> dict[str, dict]: + return { + item["metadata"]["name"]: item + for item in yaml.safe_load_all((HERMES / "service.yaml").read_text()) + if item + } + + def _oauth_deployment(name: str) -> dict: documents = [ item @@ -718,6 +726,12 @@ def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): terminal_env = {item["name"]: item["value"] for item in terminal["env"]} assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" + claude_broker = containers["claude-broker"] + claude_env = {item["name"]: item["value"] for item in claude_broker["env"]} + assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2" + assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"} + assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"} + args = containers["oauth2-proxy"]["args"] terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/" dashboard_upstream = "--upstream=http://127.0.0.1:9119/" @@ -774,6 +788,18 @@ def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): ] == "hermes-hermes-agent-terminal-slash@kubernetescrd" +def test_broker_services_survive_sibling_container_readiness_loss(): + services = _services() + + for name in ( + "hermes-image-broker", + "hermes-codex-broker", + "hermes-local-image", + "hermes-claude-broker", + ): + assert services[name]["spec"]["publishNotReadyAddresses"] is True + + def test_agent_dashboard_reconnects_all_transient_websockets(): dockerfile = ( HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent"