hermes(agent): recover stale dashboard sockets

dockerfiles/Dockerfile.hermes-agent

@@ -97,6 +97,10 @@ const socketBefore = [
 const socketAfter = [
   '      let url: string;',
   '      try {',
+  '        // WebSocket upgrades cannot expose an HTTP 401 reliably through',
+  '        // every proxy. Probe REST first so fetchJSON can detect a rotated',
+  '        // loopback session token and reload this exact SPA route once.',
+  '        await api.getStatus();',
   '        url = await api.buildWsUrl("/api/pty", params);',
   '      } catch {',
   '        if (!unmounting) scheduleReconnect(1006);',
@@ -154,6 +158,10 @@ const ticketBefore = [
 const ticketAfter = [
   '      let url: string;',
   '      try {',
+  '        // Run the shared HTTP auth recovery before each socket attempt.',
+  '        // A stale injected token otherwise appears as opaque WS code 1006',
+  '        // and this reconnect loop can never obtain the replacement token.',
+  '        await api.getStatus();',
   '        url = await buildWsUrl("/api/events", { channel });',
   '      } catch {',
   '        if (unmounting) return;',
@@ -218,12 +226,24 @@ const cleanupAfter = [
   '      ws?.close();',
   '    };',
 ].join("\n");
+const gatewayBefore = [
+  '    gw.connect()',
+  '      .then(() => {',
+].join("\n");
+const gatewayAfter = [
+  '    // Give the ordinary HTTP client first chance to recover an expired',
+  '    // OAuth session or a dashboard token rotated by a server restart.',
+  '    api.getStatus()',
+  '      .then(() => gw.connect())',
+  '      .then(() => {',
+].join("\n");
 for (const [before, after, label] of [
   [retryStateBefore, retryStateAfter, "retry state"],
   [socketStateBefore, socketStateAfter, "socket state"],
   [ticketBefore, ticketAfter, "ticket retry"],
   [handlersBefore, handlersAfter, "socket handlers"],
   [cleanupBefore, cleanupAfter, "socket cleanup"],
+  [gatewayBefore, gatewayAfter, "gateway auth preflight"],
 ]) {
   if (!sidebar.includes(before)) {
     throw new Error(`Hermes ChatSidebar ${label} patch context changed`);
@@ -576,6 +596,8 @@ COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrat
 
 RUN cd /opt/hermes/web \
     && npm run build \
+    && grep -Fq 'await api.getStatus();' src/pages/ChatPage.tsx \
+    && grep -Fq 'api.getStatus()' src/components/ChatSidebar.tsx \
     && grep -Fq 'if (unmounting) return;' src/pages/ChatPage.tsx \
     && grep -Fq 'resume:${resumeParam}' src/pages/ChatPage.tsx \
     && grep -Fq 'eventsRetryAttempt.current' src/components/ChatSidebar.tsx \

services/hermes/agent-deployment.yaml

@@ -179,7 +179,7 @@ spec:
             requests: {cpu: 25m, memory: 32Mi}
             limits: {cpu: 100m, memory: 64Mi}
         - name: install-agent-tools
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37
           imagePullPolicy: IfNotPresent
           command:
             - sh
@@ -227,7 +227,7 @@ spec:
             requests: {cpu: 100m, memory: 256Mi}
             limits: {cpu: "1", memory: 1Gi}
         - name: patch-auth
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37
           imagePullPolicy: IfNotPresent
           command:
             - /opt/hermes/.venv/bin/python
@@ -250,7 +250,7 @@ spec:
             requests: {cpu: 25m, memory: 64Mi}
             limits: {cpu: 100m, memory: 128Mi}
         - name: patch-tui-gateway
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37
           imagePullPolicy: IfNotPresent
           command:
             - /opt/hermes/.venv/bin/python
@@ -273,7 +273,7 @@ spec:
             requests: {cpu: 25m, memory: 64Mi}
             limits: {cpu: 100m, memory: 128Mi}
         - name: patch-codex-runtime
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37
           imagePullPolicy: IfNotPresent
           command:
             - /opt/hermes/.venv/bin/python
@@ -306,7 +306,7 @@ spec:
             requests: {cpu: 25m, memory: 64Mi}
             limits: {cpu: 100m, memory: 128Mi}
         - name: bootstrap-coordinator
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37
           imagePullPolicy: IfNotPresent
           command:
             - /opt/hermes/.venv/bin/python
@@ -335,7 +335,7 @@ spec:
             requests: {cpu: 50m, memory: 128Mi}
             limits: {cpu: 500m, memory: 512Mi}
         - name: configure-agent-clients
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37
           imagePullPolicy: IfNotPresent
           command:
             - sh
@@ -373,7 +373,7 @@ spec:
             requests: {cpu: 25m, memory: 32Mi}
             limits: {cpu: 250m, memory: 128Mi}
         - name: prepare-ttyd-index
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37
           imagePullPolicy: IfNotPresent
           command:
             - /opt/hermes/.venv/bin/python
@@ -395,7 +395,7 @@ spec:
             limits: {cpu: 250m, memory: 128Mi}
       containers:
         - name: hermes
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37
           imagePullPolicy: IfNotPresent
           command: [/init, /opt/hermes/docker/main-wrapper.sh]
           args: [gateway, run]
@@ -530,7 +530,7 @@ spec:
             - {name: allowlist, mountPath: /etc/oauth2-proxy, readOnly: true}
             - {name: oauth-tmp, mountPath: /tmp}
         - name: terminal
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37
           imagePullPolicy: IfNotPresent
           command: [/bin/sh, -ec]
           args:
@@ -623,7 +623,7 @@ spec:
             requests: {cpu: 25m, memory: 64Mi}
             limits: {cpu: 500m, memory: 512Mi}
         - name: cli-lane-runner
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37
           imagePullPolicy: IfNotPresent
           command: [/bin/sh, -ec]
           args:
@@ -664,7 +664,7 @@ spec:
             requests: {cpu: 100m, memory: 256Mi}
             limits: {cpu: "3", memory: 6Gi}
         - name: model-steward
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37
           imagePullPolicy: IfNotPresent
           command: [/opt/hermes/.venv/bin/python, /opt/coordinator/hermes_coordinator.py, --loop, --interval, "3600"]
           env:
@@ -690,7 +690,7 @@ spec:
             requests: {cpu: 25m, memory: 64Mi}
             limits: {cpu: 250m, memory: 512Mi}
         - name: image-broker
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37
           imagePullPolicy: IfNotPresent
           command: [/bin/sh, -ec]
           args:
@@ -737,7 +737,7 @@ spec:
             requests: {cpu: 50m, memory: 128Mi}
             limits: {cpu: "1", memory: 1Gi}
         - name: codex-broker
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:b413be5fcc0c01b0dd3e9ae0d54c23314a14f5e58f55da0d7cb47b408a2447e0
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:34d88e01a018f725f38fb19fd73b15db5bbc4da8abe5070c8ec5a71a2bcded37
           imagePullPolicy: IfNotPresent
           command: [/bin/sh, -ec]
           args:

testing/tests/test_hermes_cli_lanes.py

@@ -731,6 +731,9 @@ def test_agent_dashboard_reconnects_all_transient_websockets():
     assert "events feed rejected (${ev.code}) — reload the page" in dockerfile
     assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile
     assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile
+    assert dockerfile.count("'        await api.getStatus();',") == 2
+    assert ".then(() => gw.connect())" in dockerfile
+    assert "dashboard token rotated by a server restart" in dockerfile
 
 
 def test_agent_refreshes_routes_after_restoring_cli_logins():