From 0bec6edba3ff8ad4d71093e0e89a40dd9acbb89a Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 10 Aug 2026 23:44:56 -0300 Subject: [PATCH] hermes(agent): reconnect stock dashboard feeds --- dockerfiles/Dockerfile.hermes-agent | 84 ++++++++++++++++++++++++++ services/hermes/agent-deployment.yaml | 30 +++++---- testing/tests/test_hermes_cli_lanes.py | 26 ++++++++ 3 files changed, 129 insertions(+), 11 deletions(-) diff --git a/dockerfiles/Dockerfile.hermes-agent b/dockerfiles/Dockerfile.hermes-agent index 33af290e6..b43d6a9c4 100644 --- a/dockerfiles/Dockerfile.hermes-agent +++ b/dockerfiles/Dockerfile.hermes-agent @@ -48,6 +48,88 @@ if (!source.includes(attachBefore)) { source = source.replace(socketBefore, socketAfter); source = source.replace(attachBefore, attachAfter); fs.writeFileSync(path, source); + +// The terminal socket already reconnects indefinitely after a transient +// outage. The sidebar's events/gateway sockets did not: one deployment or +// network flap left the page in a permanent manual-Reconnect state. Retry the +// whole sidebar connection set with capped backoff while keeping auth/origin +// rejections fail-closed. +const sidebarPath = "/opt/hermes/web/src/components/ChatSidebar.tsx"; +let sidebar = fs.readFileSync(sidebarPath, "utf8"); +const retryStateBefore = ' const [version, setVersion] = useState(0);'; +const retryStateAfter = [ + retryStateBefore, + ' const eventsRetryAttempt = useRef(0);', +].join("\n"); +const socketStateBefore = [ + ' let unmounting = false;', + ' let ws: WebSocket | null = null;', +].join("\n"); +const socketStateAfter = [ + ' let unmounting = false;', + ' let ws: WebSocket | null = null;', + ' let reconnectTimer: ReturnType | null = null;', +].join("\n"); +const handlersBefore = [ + ' ws.addEventListener("error", () => surface(DISCONNECTED));', + '', + ' ws.addEventListener("close", (ev) => {', + ' if (ev.code === 4401 || ev.code === 4403) {', + ' surface(`events feed rejected (${ev.code}) — reload the page`);', + ' } else if (ev.code !== 1000) {', + ' surface(DISCONNECTED);', + ' }', + ' });', +].join("\n"); +const handlersAfter = [ + ' ws.addEventListener("open", () => {', + ' eventsRetryAttempt.current = 0;', + ' setError(null);', + ' });', + '', + ' ws.addEventListener("error", () => surface(DISCONNECTED));', + '', + ' ws.addEventListener("close", (ev) => {', + ' if (unmounting || ev.code === 1000) return;', + ' if (ev.code === 4401 || ev.code === 4403) {', + ' surface(`events feed rejected (${ev.code}) — reload the page`);', + ' return;', + ' }', + ' const attempt = Math.min(eventsRetryAttempt.current + 1, 6);', + ' eventsRetryAttempt.current = attempt;', + ' const delay = Math.min(500 * 2 ** (attempt - 1), 5000);', + ' surface(`${DISCONNECTED}; reconnecting…`);', + ' reconnectTimer = setTimeout(() => {', + ' reconnectTimer = null;', + ' if (!unmounting) setVersion((v) => v + 1);', + ' }, delay);', + ' });', +].join("\n"); +const cleanupBefore = [ + ' return () => {', + ' unmounting = true;', + ' ws?.close();', + ' };', +].join("\n"); +const cleanupAfter = [ + ' return () => {', + ' unmounting = true;', + ' if (reconnectTimer) clearTimeout(reconnectTimer);', + ' ws?.close();', + ' };', +].join("\n"); +for (const [before, after, label] of [ + [retryStateBefore, retryStateAfter, "retry state"], + [socketStateBefore, socketStateAfter, "socket state"], + [handlersBefore, handlersAfter, "socket handlers"], + [cleanupBefore, cleanupAfter, "socket cleanup"], +]) { + if (!sidebar.includes(before)) { + throw new Error(`Hermes ChatSidebar ${label} patch context changed`); + } + sidebar = sidebar.replace(before, after); +} +fs.writeFileSync(sidebarPath, sidebar); NODE # The upstream OIDC gate authenticates users but deliberately treats the @@ -395,6 +477,8 @@ RUN cd /opt/hermes/web \ && npm run build \ && grep -Fq 'if (unmounting) return;' src/pages/ChatPage.tsx \ && grep -Fq 'resume:${resumeParam}' src/pages/ChatPage.tsx \ + && grep -Fq 'eventsRetryAttempt.current' src/components/ChatSidebar.tsx \ + && grep -Fq 'reconnecting…' src/components/ChatSidebar.tsx \ && grep -Fq 'HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS' \ /opt/hermes/hermes_cli/dashboard_auth/middleware.py \ && grep -Fq '_resolve_request_route' \ diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index f676f96ce..ba4dc335b 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -170,7 +170,7 @@ spec: requests: {cpu: 25m, memory: 32Mi} limits: {cpu: 100m, memory: 64Mi} - name: install-agent-tools - image: registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 + image: registry.bstein.dev/bstein/hermes-agent@sha256:0577eb22d28b1ffc649405d31919b2527d9e591ccaa4a1e168dd6d6392a5a43f imagePullPolicy: IfNotPresent command: - sh @@ -218,7 +218,7 @@ spec: requests: {cpu: 100m, memory: 256Mi} limits: {cpu: "1", memory: 1Gi} - name: patch-auth - image: registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 + image: registry.bstein.dev/bstein/hermes-agent@sha256:0577eb22d28b1ffc649405d31919b2527d9e591ccaa4a1e168dd6d6392a5a43f imagePullPolicy: IfNotPresent command: - /opt/hermes/.venv/bin/python @@ -241,7 +241,7 @@ spec: requests: {cpu: 25m, memory: 64Mi} limits: {cpu: 100m, memory: 128Mi} - name: patch-tui-gateway - image: registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 + image: registry.bstein.dev/bstein/hermes-agent@sha256:0577eb22d28b1ffc649405d31919b2527d9e591ccaa4a1e168dd6d6392a5a43f imagePullPolicy: IfNotPresent command: - /opt/hermes/.venv/bin/python @@ -264,7 +264,7 @@ spec: requests: {cpu: 25m, memory: 64Mi} limits: {cpu: 100m, memory: 128Mi} - name: patch-codex-runtime - image: registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 + image: registry.bstein.dev/bstein/hermes-agent@sha256:0577eb22d28b1ffc649405d31919b2527d9e591ccaa4a1e168dd6d6392a5a43f imagePullPolicy: IfNotPresent command: - /opt/hermes/.venv/bin/python @@ -295,7 +295,7 @@ spec: requests: {cpu: 25m, memory: 64Mi} limits: {cpu: 100m, memory: 128Mi} - name: bootstrap-coordinator - image: registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 + image: registry.bstein.dev/bstein/hermes-agent@sha256:0577eb22d28b1ffc649405d31919b2527d9e591ccaa4a1e168dd6d6392a5a43f imagePullPolicy: IfNotPresent command: - /opt/hermes/.venv/bin/python @@ -321,7 +321,7 @@ spec: requests: {cpu: 50m, memory: 128Mi} limits: {cpu: 500m, memory: 512Mi} - name: configure-agent-clients - image: registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 + image: registry.bstein.dev/bstein/hermes-agent@sha256:0577eb22d28b1ffc649405d31919b2527d9e591ccaa4a1e168dd6d6392a5a43f imagePullPolicy: IfNotPresent command: - sh @@ -332,11 +332,17 @@ spec: set +a /opt/hermes/.venv/bin/python /opt/coordinator/configure_agent_clients.py /opt/hermes/.venv/bin/python /opt/coordinator/migrate_herdr_state.py + # Client configuration restores the persisted Codex CLI login. + # Refresh routing afterwards so AUTO sees the app-server lane on + # the first request instead of waiting for the hourly steward. + /opt/hermes/.venv/bin/python /opt/coordinator/hermes_coordinator.py --once env: - {name: HERMES_HOME, value: /opt/data} + - {name: HERMES_AUTH_FILE, value: /shared-auth/auth.json} - {name: HOME, value: /opt/data/home} - {name: CODEX_HOME, value: /opt/data/home/.codex} - {name: CLAUDE_CONFIG_DIR, value: /opt/data/home/.claude} + - {name: PYTHONPATH, value: /opt/hermes} - {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin} securityContext: allowPrivilegeEscalation: false @@ -346,12 +352,14 @@ spec: type: RuntimeDefault volumeMounts: - {name: home, mountPath: /opt/data} + - {name: provider-auth, mountPath: /shared-auth} - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} resources: requests: {cpu: 25m, memory: 32Mi} limits: {cpu: 250m, memory: 128Mi} - name: prepare-ttyd-index - image: registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 + image: registry.bstein.dev/bstein/hermes-agent@sha256:0577eb22d28b1ffc649405d31919b2527d9e591ccaa4a1e168dd6d6392a5a43f imagePullPolicy: IfNotPresent command: - /opt/hermes/.venv/bin/python @@ -373,7 +381,7 @@ spec: limits: {cpu: 250m, memory: 128Mi} containers: - name: hermes - image: registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 + image: registry.bstein.dev/bstein/hermes-agent@sha256:0577eb22d28b1ffc649405d31919b2527d9e591ccaa4a1e168dd6d6392a5a43f imagePullPolicy: IfNotPresent command: [/init, /opt/hermes/docker/main-wrapper.sh] args: [gateway, run] @@ -504,7 +512,7 @@ spec: - {name: allowlist, mountPath: /etc/oauth2-proxy, readOnly: true} - {name: oauth-tmp, mountPath: /tmp} - name: terminal - image: registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 + image: registry.bstein.dev/bstein/hermes-agent@sha256:0577eb22d28b1ffc649405d31919b2527d9e591ccaa4a1e168dd6d6392a5a43f imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: @@ -596,7 +604,7 @@ spec: requests: {cpu: 25m, memory: 64Mi} limits: {cpu: 500m, memory: 512Mi} - name: cli-lane-runner - image: registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 + image: registry.bstein.dev/bstein/hermes-agent@sha256:0577eb22d28b1ffc649405d31919b2527d9e591ccaa4a1e168dd6d6392a5a43f imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: @@ -637,7 +645,7 @@ spec: requests: {cpu: 100m, memory: 256Mi} limits: {cpu: "3", memory: 6Gi} - name: model-steward - image: registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 + image: registry.bstein.dev/bstein/hermes-agent@sha256:0577eb22d28b1ffc649405d31919b2527d9e591ccaa4a1e168dd6d6392a5a43f imagePullPolicy: IfNotPresent command: [/opt/hermes/.venv/bin/python, /opt/coordinator/hermes_coordinator.py, --loop, --interval, "3600"] env: diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index b506ae0a4..5ee258d7e 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -716,6 +716,32 @@ def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): ] == "hermes-hermes-agent-terminal-slash@kubernetescrd" +def test_agent_dashboard_reconnects_all_transient_websockets(): + dockerfile = ( + HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent" + ).read_text(encoding="utf-8") + assert "eventsRetryAttempt.current" in dockerfile + assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile + assert "events feed rejected (${ev.code}) — reload the page" in dockerfile + + +def test_agent_refreshes_routes_after_restoring_cli_logins(): + deployment = _agent_deployment() + init_containers = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["initContainers"] + } + configure = init_containers["configure-agent-clients"] + command = configure["command"][-1] + assert "configure_agent_clients.py" in command + assert command.index("configure_agent_clients.py") < command.index( + "hermes_coordinator.py --once" + ) + env = {item["name"]: item["value"] for item in configure["env"]} + assert env["HERMES_AUTH_FILE"] == "/shared-auth/auth.json" + assert env["PYTHONPATH"] == "/opt/hermes" + + def test_flux_health_checks_follow_the_owner_oauth_sidecar(): flux = yaml.safe_load(FLUX_HERMES.read_text()) checks = {