From b31be8f7e2f4a8d169c151ed67c3e191a9509f83 Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 11 Aug 2026 03:50:13 -0300 Subject: [PATCH] hermes(agent): recover stale dashboard sockets --- dockerfiles/Dockerfile.hermes-agent | 22 ++++++++++++++++++++++ services/hermes/agent-deployment.yaml | 26 +++++++++++++------------- testing/tests/test_hermes_cli_lanes.py | 3 +++ 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/dockerfiles/Dockerfile.hermes-agent b/dockerfiles/Dockerfile.hermes-agent index 30f0345c0..b1decc92a 100644 --- a/dockerfiles/Dockerfile.hermes-agent +++ b/dockerfiles/Dockerfile.hermes-agent @@ -97,6 +97,10 @@ const socketBefore = [ const socketAfter = [ ' let url: string;', ' try {', + ' // WebSocket upgrades cannot expose an HTTP 401 reliably through', + ' // every proxy. Probe REST first so fetchJSON can detect a rotated', + ' // loopback session token and reload this exact SPA route once.', + ' await api.getStatus();', ' url = await api.buildWsUrl("/api/pty", params);', ' } catch {', ' if (!unmounting) scheduleReconnect(1006);', @@ -154,6 +158,10 @@ const ticketBefore = [ const ticketAfter = [ ' let url: string;', ' try {', + ' // Run the shared HTTP auth recovery before each socket attempt.', + ' // A stale injected token otherwise appears as opaque WS code 1006', + ' // and this reconnect loop can never obtain the replacement token.', + ' await api.getStatus();', ' url = await buildWsUrl("/api/events", { channel });', ' } catch {', ' if (unmounting) return;', @@ -218,12 +226,24 @@ const cleanupAfter = [ ' ws?.close();', ' };', ].join("\n"); +const gatewayBefore = [ + ' gw.connect()', + ' .then(() => {', +].join("\n"); +const gatewayAfter = [ + ' // Give the ordinary HTTP client first chance to recover an expired', + ' // OAuth session or a dashboard token rotated by a server restart.', + ' api.getStatus()', + ' .then(() => gw.connect())', + ' .then(() => {', +].join("\n"); for (const [before, after, label] of [ [retryStateBefore, retryStateAfter, "retry state"], [socketStateBefore, socketStateAfter, "socket state"], [ticketBefore, ticketAfter, "ticket retry"], [handlersBefore, handlersAfter, "socket handlers"], [cleanupBefore, cleanupAfter, "socket cleanup"], + [gatewayBefore, gatewayAfter, "gateway auth preflight"], ]) { if (!sidebar.includes(before)) { throw new Error(`Hermes ChatSidebar ${label} patch context changed`); @@ -576,6 +596,8 @@ COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrat RUN cd /opt/hermes/web \ && npm run build \ + && grep -Fq 'await api.getStatus();' src/pages/ChatPage.tsx \ + && grep -Fq 'api.getStatus()' src/components/ChatSidebar.tsx \ && grep -Fq 'if (unmounting) return;' src/pages/ChatPage.tsx \ && grep -Fq 'resume:${resumeParam}' src/pages/ChatPage.tsx \ && grep -Fq 'eventsRetryAttempt.current' src/components/ChatSidebar.tsx \ diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index e0b7595e8..3a682543d 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -179,7 +179,7 @@ spec: requests: {cpu: 25m, memory: 32Mi} limits: {cpu: 100m, memory: 64Mi} - name: install-agent-tools - image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0 + image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37 imagePullPolicy: IfNotPresent command: - sh @@ -227,7 +227,7 @@ spec: requests: {cpu: 100m, memory: 256Mi} limits: {cpu: "1", memory: 1Gi} - name: patch-auth - image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0 + image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37 imagePullPolicy: IfNotPresent command: - /opt/hermes/.venv/bin/python @@ -250,7 +250,7 @@ spec: requests: {cpu: 25m, memory: 64Mi} limits: {cpu: 100m, memory: 128Mi} - name: patch-tui-gateway - image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0 + image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37 imagePullPolicy: IfNotPresent command: - /opt/hermes/.venv/bin/python @@ -273,7 +273,7 @@ spec: requests: {cpu: 25m, memory: 64Mi} limits: {cpu: 100m, memory: 128Mi} - name: patch-codex-runtime - image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0 + image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37 imagePullPolicy: IfNotPresent command: - /opt/hermes/.venv/bin/python @@ -306,7 +306,7 @@ spec: requests: {cpu: 25m, memory: 64Mi} limits: {cpu: 100m, memory: 128Mi} - name: bootstrap-coordinator - image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0 + image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37 imagePullPolicy: IfNotPresent command: - /opt/hermes/.venv/bin/python @@ -335,7 +335,7 @@ spec: requests: {cpu: 50m, memory: 128Mi} limits: {cpu: 500m, memory: 512Mi} - name: configure-agent-clients - image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0 + image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37 imagePullPolicy: IfNotPresent command: - sh @@ -373,7 +373,7 @@ spec: requests: {cpu: 25m, memory: 32Mi} limits: {cpu: 250m, memory: 128Mi} - name: prepare-ttyd-index - image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0 + image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37 imagePullPolicy: IfNotPresent command: - /opt/hermes/.venv/bin/python @@ -395,7 +395,7 @@ spec: limits: {cpu: 250m, memory: 128Mi} containers: - name: hermes - image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0 + image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37 imagePullPolicy: IfNotPresent command: [/init, /opt/hermes/docker/main-wrapper.sh] args: [gateway, run] @@ -530,7 +530,7 @@ spec: - {name: allowlist, mountPath: /etc/oauth2-proxy, readOnly: true} - {name: oauth-tmp, mountPath: /tmp} - name: terminal - image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0 + image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37 imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: @@ -623,7 +623,7 @@ spec: requests: {cpu: 25m, memory: 64Mi} limits: {cpu: 500m, memory: 512Mi} - name: cli-lane-runner - image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0 + image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37 imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: @@ -664,7 +664,7 @@ spec: requests: {cpu: 100m, memory: 256Mi} limits: {cpu: "3", memory: 6Gi} - name: model-steward - image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0 + image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37 imagePullPolicy: IfNotPresent command: [/opt/hermes/.venv/bin/python, /opt/coordinator/hermes_coordinator.py, --loop, --interval, "3600"] env: @@ -690,7 +690,7 @@ spec: requests: {cpu: 25m, memory: 64Mi} limits: {cpu: 250m, memory: 512Mi} - name: image-broker - image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0 + image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37 imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: @@ -737,7 +737,7 @@ spec: requests: {cpu: 50m, memory: 128Mi} limits: {cpu: "1", memory: 1Gi} - name: codex-broker - image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0 + image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37 imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index 66635871a..26bf653b6 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -731,6 +731,9 @@ def test_agent_dashboard_reconnects_all_transient_websockets(): assert "events feed rejected (${ev.code}) — reload the page" in dockerfile assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile + assert dockerfile.count("' await api.getStatus();',") == 2 + assert ".then(() => gw.connect())" in dockerfile + assert "dashboard token rotated by a server restart" in dockerfile def test_agent_refreshes_routes_after_restoring_cli_logins():