diff --git a/services/hermes/NOTES.md b/services/hermes/NOTES.md index f86b1cbc8..c70e528ae 100644 --- a/services/hermes/NOTES.md +++ b/services/hermes/NOTES.md @@ -5,6 +5,13 @@ This is the mental model and demonstration script for the operator instance at consumer instance at `chat.hermes.bstein.dev` is intentionally separate and is not the place to perform infrastructure triage. +`agent.hermes.bstein.dev` is the owner-only engineering control plane. Its root +serves the stock Hermes dashboard, including embedded chat/TUI and the native +sessions, files, models, logs, Kanban, skills, plugins, MCP, profiles, and +configuration surfaces. `/terminal/` remains the raw full-screen TUI. The +community WebUI is intentionally reserved for isolated consumer chat, where +its conversation-first layout is a better fit. + ## Consumer chat and Telegram `chat.hermes.bstein.dev` uses the pinned Hermes WebUI rather than the operator diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index 1f1ca6b29..128c1b13e 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -16,10 +16,6 @@ data: fallback_providers: - provider: anthropic model: claude-sonnet-5 - - provider: custom - model: qwen2.5:14b-instruct-q4_0 - base_url: http://ollama.ai.svc.cluster.local:11434/v1 - api_key: ollama - provider: custom model: gpt-oss:20b base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1 @@ -228,7 +224,7 @@ data: restart. Manual lanes are `cli-codex-{low,medium,high,xhigh}` and `cli-claude-{low,medium,high,xhigh}`; they still call the Jetson for the audit record, then apply the explicit override. Observe workers through Kanban and - the WebUI worker list, not terminal panes. If Codex reports its first-use + the dashboard session/task lists, not terminal panes. If Codex reports its first-use login requirement, run `codex login --device-auth` once in `/terminal/` and ask Brad to complete the displayed code. @@ -248,9 +244,10 @@ data: START-HERE.md: | # Agent Hermes - The authenticated root of agent.hermes.bstein.dev opens the persistent - Hermes WebUI and its Project/Kanban controls. `/terminal/` opens the stock - Hermes TUI. Give Hermes + The authenticated root of agent.hermes.bstein.dev opens Hermes' stock + dashboard with embedded chat/TUI, sessions, files, models, logs, Kanban, + skills, plugins, MCP, profiles, and configuration. `/terminal/` opens the + raw full-screen Hermes TUI. Give Hermes the outcome you want and it will decompose dependent work, classify every delegated leaf on the Jetson, choose Codex or Claude, preserve the task on the Cassandra board, and synthesize the evidence. Persistent real Codex and diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index c7b76ec67..224ae38df 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -24,7 +24,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260810-owner-union-toolbox" + ai.bstein.dev/config-rev: "20260810-stock-dashboard-init-hardening" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-agent vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens @@ -39,6 +39,13 @@ spec: {{ . }} {{- end -}} {{- end }} + vault.hashicorp.com/agent-inject-secret-oidc-config: kv/data/atlas/hermes/agent-oidc + vault.hashicorp.com/agent-inject-template-oidc-config: | + {{- with secret "kv/data/atlas/hermes/agent-oidc" -}} + client_id = "{{ .Data.data.client_id }}" + client_secret = "{{ .Data.data.client_secret }}" + cookie_secret = "{{ .Data.data.cookie_secret }}" + {{- end -}} vault.hashicorp.com/agent-pre-populate-only: "true" vault.hashicorp.com/agent-init-first: "true" vault.hashicorp.com/agent-requests-cpu: 25m @@ -211,6 +218,29 @@ spec: resources: requests: {cpu: 25m, memory: 64Mi} limits: {cpu: 100m, memory: 128Mi} + - name: patch-tui-gateway + image: registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 + imagePullPolicy: IfNotPresent + command: + - /opt/hermes/.venv/bin/python + - /opt/coordinator/patch_tui_gateway.py + - /opt/hermes/tui_gateway/server.py + - /patched/server.py + securityContext: + allowPrivilegeEscalation: false + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: coordinator + mountPath: /opt/coordinator + readOnly: true + - name: tui-gateway-patch + mountPath: /patched + resources: + requests: {cpu: 25m, memory: 64Mi} + limits: {cpu: 100m, memory: 128Mi} - name: bootstrap-coordinator image: registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 imagePullPolicy: IfNotPresent @@ -343,48 +373,57 @@ spec: resources: requests: {cpu: 250m, memory: 512Mi} limits: {cpu: "2", memory: 4Gi} - - name: webui - image: registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d + - name: dashboard + image: registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 imagePullPolicy: IfNotPresent command: [/bin/sh, -ec] args: - | - api_key="$(sed -n 's/^API_SERVER_KEY=//p' /opt/data/.env | tail -n 1)" - test -n "${api_key}" - export API_SERVER_KEY="${api_key}" - export HERMES_WEBUI_GATEWAY_API_KEY="${api_key}" - exec /opt/hermes/.venv/bin/python /opt/hermes-webui/server.py + set -a + . /opt/data/.env + set +a + exec /opt/hermes/.venv/bin/hermes dashboard \ + --host 127.0.0.1 \ + --port 9119 \ + --skip-build \ + --no-open ports: - - {name: dashboard, containerPort: 8787, protocol: TCP} + - {name: dashboard, containerPort: 9119, protocol: TCP} env: - {name: HERMES_HOME, value: /opt/data} - {name: HERMES_AUTH_FILE, value: /shared-auth/auth.json} - {name: HOME, value: /opt/data/home} - - {name: HERMES_WEBUI_AGENT_DIR, value: /opt/hermes} - - {name: HERMES_WEBUI_HOST, value: 0.0.0.0} - - {name: HERMES_WEBUI_PORT, value: "8787"} - - {name: HERMES_WEBUI_STATE_DIR, value: /opt/data/webui} - - {name: HERMES_WEBUI_DEFAULT_WORKSPACE, value: /opt/data/workspace} - - {name: HERMES_WEBUI_CHAT_BACKEND, value: gateway} - - {name: HERMES_WEBUI_GATEWAY_BASE_URL, value: http://127.0.0.1:8642} - - {name: HERMES_WEBUI_GATEWAY_USE_RUNS_API, value: "true"} - - {name: HERMES_WEBUI_SKIP_ONBOARDING, value: "1"} - - {name: HERMES_WEBUI_SECURE, value: "1"} - - {name: HERMES_WEBUI_ALLOWED_ORIGINS, value: https://agent.hermes.bstein.dev} - - {name: HERMES_WEBUI_TRUST_FORWARDED_HOST, value: "1"} - - {name: HERMES_WEBUI_TRUST_FORWARDED_PROTO, value: "1"} + - {name: CODEX_HOME, value: /opt/data/home/.codex} + - {name: CLAUDE_CONFIG_DIR, value: /opt/data/home/.claude} + - {name: KUBECONFIG, value: /opt/data/home/.kube/config} + - {name: PYTHONPATH, value: /opt/hermes} + - {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin} + - {name: PLAYWRIGHT_BROWSERS_PATH, value: /opt/hermes/.playwright} + - {name: AGENT_BROWSER_EXECUTABLE_PATH, value: /opt/hermes/.playwright/chromium_headless_shell-1228/chrome-linux/headless_shell} + - {name: AGENT_BROWSER_ARGS, value: "--no-sandbox,--disable-dev-shm-usage"} + - {name: HERMES_DASHBOARD_FILES_ROOT, value: /opt/data/workspace} + - {name: HERMES_DASHBOARD_PUBLIC_URL, value: https://agent.hermes.bstein.dev} + - {name: HERMES_TUI_AGENT_INIT_TIMEOUT_S, value: "180"} volumeMounts: - {name: home, mountPath: /opt/data} - - {name: provider-auth, mountPath: /shared-auth, readOnly: true} + - {name: provider-auth, mountPath: /shared-auth} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} + - {name: tui-gateway-patch, mountPath: /opt/hermes/tui_gateway/server.py, subPath: server.py} + - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} + - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} - {name: tmp, mountPath: /tmp} + workingDir: /opt/data/workspace readinessProbe: - httpGet: {path: /health, port: dashboard} - initialDelaySeconds: 10 + exec: + command: [curl, -fsS, http://127.0.0.1:9119/api/status] + initialDelaySeconds: 20 periodSeconds: 10 timeoutSeconds: 5 livenessProbe: - httpGet: {path: /health, port: dashboard} - initialDelaySeconds: 30 + exec: + command: [curl, -fsS, http://127.0.0.1:9119/api/status] + initialDelaySeconds: 60 periodSeconds: 30 timeoutSeconds: 10 securityContext: @@ -398,8 +437,67 @@ spec: seccompProfile: type: RuntimeDefault resources: - requests: {cpu: 50m, memory: 128Mi} - limits: {cpu: 750m, memory: 1Gi} + requests: {cpu: 100m, memory: 256Mi} + limits: {cpu: "2", memory: 4Gi} + - name: oauth2-proxy + image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561 + imagePullPolicy: IfNotPresent + args: + - --provider=oidc + - --config=/vault/secrets/oidc-config + - --redirect-url=https://agent.hermes.bstein.dev/oauth2/callback + - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas + - --user-id-claim=sub + - --oidc-groups-claim=groups + - --allowed-group=/hermes-owner + - --code-challenge-method=S256 + - --scope=openid profile email + - --email-domain=* + - --authenticated-emails-file=/etc/oauth2-proxy/allowed-emails + - --set-xauthrequest=true + - --pass-user-headers=true + - --pass-basic-auth=false + - --pass-host-header=false + - --proxy-websockets=true + - --cookie-name=__Host-hermes_agent + - --cookie-path=/ + - --cookie-secure=true + - --cookie-samesite=lax + - --cookie-csrf-expire=10m + - --cookie-csrf-per-request=true + - --cookie-csrf-per-request-limit=8 + - --cookie-refresh=1h + - --cookie-expire=8h + - --upstream=http://127.0.0.1:7681/terminal/ + - --upstream=http://127.0.0.1:9119/ + - --http-address=0.0.0.0:4180 + - --skip-provider-button=true + - --reverse-proxy=true + - --trusted-proxy-ip=10.42.0.0/16 + ports: + - {name: auth-http, containerPort: 4180, protocol: TCP} + readinessProbe: + httpGet: {path: /ping, port: auth-http} + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: {path: /ping, port: auth-http} + initialDelaySeconds: 20 + periodSeconds: 20 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + resources: + requests: {cpu: 25m, memory: 64Mi} + limits: {cpu: 250m, memory: 256Mi} + volumeMounts: + - {name: allowlist, mountPath: /etc/oauth2-proxy, readOnly: true} + - {name: oauth-tmp, mountPath: /tmp} - name: terminal image: registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 imagePullPolicy: IfNotPresent @@ -411,7 +509,6 @@ spec: set +a exec /opt/data/tools/bin/ttyd \ --writable \ - --check-origin \ --base-path /terminal \ --interface 0.0.0.0 \ --port 7681 \ @@ -450,11 +547,13 @@ spec: - {name: PLAYWRIGHT_BROWSERS_PATH, value: /opt/hermes/.playwright} - {name: AGENT_BROWSER_EXECUTABLE_PATH, value: /opt/hermes/.playwright/chromium_headless_shell-1228/chrome-linux/headless_shell} - {name: AGENT_BROWSER_ARGS, value: "--no-sandbox,--disable-dev-shm-usage"} + - {name: HERMES_TUI_AGENT_INIT_TIMEOUT_S, value: "180"} volumeMounts: - {name: home, mountPath: /opt/data} - {name: provider-auth, mountPath: /shared-auth} - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} + - {name: tui-gateway-patch, mountPath: /opt/hermes/tui_gateway/server.py, subPath: server.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} - {name: tmp, mountPath: /tmp} @@ -570,12 +669,20 @@ spec: defaultMode: 0444 - name: auth-patch emptyDir: {} + - name: tui-gateway-patch + emptyDir: {} - name: auto-router-plugin configMap: name: hermes-auto-router-plugin - name: tmp emptyDir: sizeLimit: 4Gi + - name: oauth-tmp + emptyDir: + sizeLimit: 64Mi + - name: allowlist + configMap: + name: hermes-owner-allowlist - name: ttyd-index emptyDir: sizeLimit: 2Mi diff --git a/services/hermes/agent-ingress.yaml b/services/hermes/agent-ingress.yaml index 7273768e8..19ae39b71 100644 --- a/services/hermes/agent-ingress.yaml +++ b/services/hermes/agent-ingress.yaml @@ -10,23 +10,30 @@ spec: replacement: https://agent.hermes.bstein.dev/terminal/ permanent: false --- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: hermes-agent-stock-dashboard-headers + namespace: hermes +spec: + headers: + customRequestHeaders: + Origin: http://127.0.0.1:9119 +--- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: hermes-sites + name: hermes-agent-dashboard namespace: hermes annotations: - cert-manager.io/cluster-issuer: letsencrypt traefik.ingress.kubernetes.io/router.entrypoints: websecure - traefik.ingress.kubernetes.io/router.middlewares: hermes-hermes-agent-terminal-slash@kubernetescrd + traefik.ingress.kubernetes.io/router.middlewares: hermes-hermes-agent-stock-dashboard-headers@kubernetescrd traefik.ingress.kubernetes.io/router.tls: "true" spec: ingressClassName: traefik tls: - hosts: - agent.hermes.bstein.dev - - chat.hermes.bstein.dev - - triage.hermes.bstein.dev secretName: hermes-sites-tls rules: - host: agent.hermes.bstein.dev @@ -39,6 +46,52 @@ spec: name: oauth2-proxy-hermes-agent port: name: http +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: hermes-agent-terminal + namespace: hermes + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.middlewares: hermes-hermes-agent-terminal-slash@kubernetescrd + traefik.ingress.kubernetes.io/router.tls: "true" +spec: + ingressClassName: traefik + tls: + - hosts: + - agent.hermes.bstein.dev + secretName: hermes-sites-tls + rules: + - host: agent.hermes.bstein.dev + http: + paths: + - path: /terminal + pathType: Prefix + backend: + service: + name: oauth2-proxy-hermes-agent + port: + name: http +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: hermes-sites + namespace: hermes + annotations: + cert-manager.io/cluster-issuer: letsencrypt + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" +spec: + ingressClassName: traefik + tls: + - hosts: + - agent.hermes.bstein.dev + - chat.hermes.bstein.dev + - triage.hermes.bstein.dev + secretName: hermes-sites-tls + rules: - host: chat.hermes.bstein.dev http: paths: diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 7d28b3b35..603280fb7 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -58,6 +58,7 @@ configMapGenerator: - install_agent_tools.sh=scripts/install_agent_tools.sh - migrate_herdr_state.py=scripts/migrate_herdr_state.py - patch_hermes_auth.py=scripts/patch_hermes_auth.py + - patch_tui_gateway.py=scripts/patch_tui_gateway.py - patch_ttyd_index.py=scripts/patch_ttyd_index.py options: disableNameSuffixHash: true diff --git a/services/hermes/model-gate-configmap.yaml b/services/hermes/model-gate-configmap.yaml index cf179d84d..4dfde98fb 100644 --- a/services/hermes/model-gate-configmap.yaml +++ b/services/hermes/model-gate-configmap.yaml @@ -62,6 +62,34 @@ data: return owner + def _normalize_reasoning(body: bytes | None) -> bytes | None: + """Clamp hosted-only effort names to the local server's supported tier.""" + + if not body: + return body + try: + payload = json.loads(body) + except (TypeError, ValueError, json.JSONDecodeError): + return body + if not isinstance(payload, dict): + return body + + changed = False + for key in ("reasoning_effort", "reasoning"): + value = payload.get(key) + if isinstance(value, str) and value.lower() in {"xhigh", "max"}: + payload[key] = "high" + changed = True + elif isinstance(value, dict): + effort = value.get("effort") + if isinstance(effort, str) and effort.lower() in {"xhigh", "max"}: + value["effort"] = "high" + changed = True + if not changed: + return body + return json.dumps(payload, separators=(",", ":")).encode("utf-8") + + class Handler(BaseHTTPRequestHandler): """Proxy local model traffic while exposing health and ownership status.""" @@ -98,6 +126,7 @@ data: length = int(self.headers.get("Content-Length", "0") or 0) body = self.rfile.read(length) if length else None + body = _normalize_reasoning(body) headers = {"Content-Type": self.headers.get("Content-Type", "application/json")} if self.headers.get("Accept"): headers["Accept"] = self.headers["Accept"] diff --git a/services/hermes/networkpolicy.yaml b/services/hermes/networkpolicy.yaml index 435df0c0e..059a6ee0d 100644 --- a/services/hermes/networkpolicy.yaml +++ b/services/hermes/networkpolicy.yaml @@ -68,12 +68,14 @@ spec: policyTypes: [Ingress, Egress] ingress: - from: - - podSelector: + - namespaceSelector: matchLabels: - app: oauth2-proxy-hermes-agent + kubernetes.io/metadata.name: traefik + podSelector: + matchLabels: + app.kubernetes.io/name: traefik ports: - - {protocol: TCP, port: 7681} - - {protocol: TCP, port: 8787} + - {protocol: TCP, port: 4180} # agent.hermes.bstein.dev is an owner-only engineering workstation. The # browser boundary remains OAuth-protected, while its workers need to reach # every cluster namespace, Atlas LAN service, and hosted provider endpoint. @@ -316,7 +318,6 @@ spec: - key: app operator: In values: - - oauth2-proxy-hermes-agent - oauth2-proxy-hermes-chat - oauth2-proxy-hermes-triage policyTypes: [Ingress, Egress] diff --git a/services/hermes/oauth2-proxy.yaml b/services/hermes/oauth2-proxy.yaml index 9ef3746ab..fba9b67ce 100644 --- a/services/hermes/oauth2-proxy.yaml +++ b/services/hermes/oauth2-proxy.yaml @@ -15,9 +15,9 @@ metadata: namespace: hermes spec: selector: - app: oauth2-proxy-hermes-agent + app: hermes-agent ports: - - {name: http, port: 80, targetPort: http} + - {name: http, port: 80, targetPort: auth-http} --- apiVersion: v1 kind: Service @@ -43,103 +43,6 @@ spec: --- apiVersion: apps/v1 kind: Deployment -metadata: - name: oauth2-proxy-hermes-agent - namespace: hermes - labels: - app: oauth2-proxy-hermes-agent -spec: - replicas: 1 - revisionHistoryLimit: 2 - selector: - matchLabels: - app: oauth2-proxy-hermes-agent - template: - metadata: - labels: - app: oauth2-proxy-hermes-agent - annotations: - vault.hashicorp.com/agent-inject: "true" - vault.hashicorp.com/agent-pre-populate-only: "true" - vault.hashicorp.com/role: hermes-agent - vault.hashicorp.com/agent-inject-secret-oidc-config: kv/data/atlas/hermes/agent-oidc - vault.hashicorp.com/agent-inject-template-oidc-config: | - {{- with secret "kv/data/atlas/hermes/agent-oidc" -}} - client_id = "{{ .Data.data.client_id }}" - client_secret = "{{ .Data.data.client_secret }}" - cookie_secret = "{{ .Data.data.cookie_secret }}" - {{- end -}} - spec: - serviceAccountName: hermes-agent - automountServiceAccountToken: true - containers: - - name: oauth2-proxy - image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561 - imagePullPolicy: IfNotPresent - args: - - --provider=oidc - - --config=/vault/secrets/oidc-config - - --redirect-url=https://agent.hermes.bstein.dev/oauth2/callback - - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas - - --user-id-claim=sub - - --oidc-groups-claim=groups - - --allowed-group=/hermes-owner - - --code-challenge-method=S256 - - --scope=openid profile email - - --email-domain=* - - --authenticated-emails-file=/etc/oauth2-proxy/allowed-emails - - --set-xauthrequest=true - - --pass-user-headers=true - - --pass-basic-auth=false - - --proxy-websockets=true - - --cookie-name=__Host-hermes_agent - - --cookie-path=/ - - --cookie-secure=true - - --cookie-samesite=lax - - --cookie-csrf-expire=10m - - --cookie-csrf-per-request=true - - --cookie-csrf-per-request-limit=8 - - --cookie-refresh=1h - - --cookie-expire=8h - - --upstream=http://hermes-agent.hermes.svc.cluster.local:7681/terminal/ - - --upstream=http://hermes-agent.hermes.svc.cluster.local:8787/ - - --http-address=0.0.0.0:4180 - - --skip-provider-button=true - - --reverse-proxy=true - - --trusted-proxy-ip=10.42.0.0/16 - ports: - - {name: http, containerPort: 4180} - readinessProbe: - httpGet: {path: /ping, port: http} - initialDelaySeconds: 5 - periodSeconds: 10 - livenessProbe: - httpGet: {path: /ping, port: http} - initialDelaySeconds: 20 - periodSeconds: 20 - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: [ALL] - readOnlyRootFilesystem: true - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - resources: - requests: {cpu: 25m, memory: 64Mi} - limits: {cpu: 250m, memory: 256Mi} - volumeMounts: - - {name: allowlist, mountPath: /etc/oauth2-proxy, readOnly: true} - - {name: tmp, mountPath: /tmp} - volumes: - - name: allowlist - configMap: - name: hermes-owner-allowlist - - name: tmp - emptyDir: {sizeLimit: 64Mi} ---- -apiVersion: apps/v1 -kind: Deployment metadata: name: oauth2-proxy-hermes-triage namespace: hermes diff --git a/services/hermes/plugins/auto-router/__init__.py b/services/hermes/plugins/auto-router/__init__.py index a4e915c1a..ca5479cbb 100644 --- a/services/hermes/plugins/auto-router/__init__.py +++ b/services/hermes/plugins/auto-router/__init__.py @@ -26,6 +26,12 @@ JETSON_MODEL = os.environ.get( EFFORTS = ("low", "medium", "high", "xhigh") PROVIDERS = ("codex", "claude") EFFORT_RANK = {effort: rank for rank, effort in enumerate(EFFORTS)} +try: + PROVIDER_COOLDOWN_S = float( + os.environ.get("HERMES_PROVIDER_COOLDOWN_S", "900") + ) +except (TypeError, ValueError): + PROVIDER_COOLDOWN_S = 900.0 RISK_TERMS = { "credential", @@ -409,6 +415,37 @@ def _write_policy(value: dict[str, Any]) -> None: os.replace(temporary, POLICY_PATH) +def _provider_is_cooled_down(policy: dict[str, Any], provider: str) -> bool: + """Return whether a recent runtime fallback temporarily suppresses a lane.""" + cooldowns = policy.get("provider_cooldowns") + if not isinstance(cooldowns, dict): + return False + state = cooldowns.get(provider) + if not isinstance(state, dict): + return False + try: + return float(state.get("until_epoch") or 0) > time.time() + except (TypeError, ValueError): + return False + + +def _cool_down_provider( + policy: dict[str, Any], provider: str, actual_provider: str +) -> None: + """Circuit-break a provider after Hermes had to cross-provider fallback.""" + duration = max(60.0, min(PROVIDER_COOLDOWN_S, 3600.0)) + cooldowns = policy.get("provider_cooldowns") + if not isinstance(cooldowns, dict): + cooldowns = {} + cooldowns[provider] = { + "until_epoch": time.time() + duration, + "reason": "cross-provider runtime fallback", + "actual_provider": actual_provider, + "recorded_at": datetime.now(timezone.utc).isoformat(), + } + policy["provider_cooldowns"] = cooldowns + + def _split_route(route: str) -> tuple[str, str]: provider, separator, model = route.partition("/") if not separator or not provider or not model: @@ -417,14 +454,27 @@ def _split_route(route: str) -> tuple[str, str]: def select_route( - status: dict[str, Any], decision: Decision, model_override: str = "" + status: dict[str, Any], + decision: Decision, + model_override: str = "", + policy: dict[str, Any] | None = None, ) -> dict[str, Any]: """Resolve a connected managed provider/model chain for a decision.""" providers = status.get("providers") or {} + policy = policy if isinstance(policy, dict) else _current_policy() selected = decision.provider provider_key = "openai-codex" if selected == "codex" else "anthropic" alternate = "claude" if selected == "codex" else "codex" - if not bool((providers.get(provider_key) or {}).get("connected", True)): + alternate_key = "anthropic" if alternate == "claude" else "openai-codex" + selected_unavailable = ( + not bool((providers.get(provider_key) or {}).get("connected", True)) + or _provider_is_cooled_down(policy, provider_key) + ) + alternate_available = ( + bool((providers.get(alternate_key) or {}).get("connected", True)) + and not _provider_is_cooled_down(policy, alternate_key) + ) + if selected_unavailable and alternate_available: selected = alternate profile = f"{selected}-{decision.effort}" chain = (status.get("routes") or {}).get(profile) @@ -625,6 +675,8 @@ def _post_turn_route(ctx: Any, **kwargs: Any) -> None: fallback_used = ( actual_provider != target_provider or actual_model != target_model ) + if target_provider and actual_provider != target_provider: + _cool_down_provider(policy, target_provider, actual_provider) last.update( { "actual_provider": actual_provider, diff --git a/services/hermes/scripts/hermes_model_routing.py b/services/hermes/scripts/hermes_model_routing.py index b685a60f3..a92bef45e 100644 --- a/services/hermes/scripts/hermes_model_routing.py +++ b/services/hermes/scripts/hermes_model_routing.py @@ -16,12 +16,6 @@ import yaml CODEX_BASELINE = "gpt-5.6-terra" CLAUDE_BASELINE = "claude-opus-5" EFFORTS = ("low", "medium", "high", "xhigh") -JETSON_FALLBACK = { - "provider": "custom", - "model": "qwen2.5:14b-instruct-q4_0", - "base_url": "http://ollama.ai.svc.cluster.local:11434/v1", - "api_key": "ollama", -} ATLAS_FALLBACK = { "provider": "custom", "model": "gpt-oss:20b", @@ -337,7 +331,6 @@ def _profile_config( } config["fallback_providers"] = [ fallback, - copy.deepcopy(JETSON_FALLBACK), copy.deepcopy(ATLAS_FALLBACK), ] agent = config.setdefault("agent", {}) @@ -408,7 +401,6 @@ def configure_routes(root: Path, codex: Catalog, claude: Catalog) -> dict[str, A } base["fallback_providers"] = [ {"provider": "anthropic", "model": claude_coordinator}, - copy.deepcopy(JETSON_FALLBACK), copy.deepcopy(ATLAS_FALLBACK), ] base["model_catalog"] = {"enabled": True, "ttl_hours": 1} diff --git a/services/hermes/scripts/patch_tui_gateway.py b/services/hermes/scripts/patch_tui_gateway.py new file mode 100644 index 000000000..034d2b6e8 --- /dev/null +++ b/services/hermes/scripts/patch_tui_gateway.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Make Hermes' deferred TUI agent startup deadline operator-configurable.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + + +BEFORE = '''def _wait_agent(session: dict, rid: str, timeout: float = 30.0) -> dict | None: + ready = session.get("agent_ready") + if ready is not None and not ready.wait(timeout=timeout): + return _err(rid, 5032, "agent initialization timed out") +''' +AFTER = '''def _agent_init_timeout() -> float: + """Return the bounded startup allowance for tool-heavy TUI sessions.""" + try: + configured = float( + os.environ.get("HERMES_TUI_AGENT_INIT_TIMEOUT_S", "180") + ) + except (TypeError, ValueError): + configured = 180.0 + return max(30.0, min(configured, 900.0)) + + +def _wait_agent( + session: dict, rid: str, timeout: float | None = None +) -> dict | None: + ready = session.get("agent_ready") + wait_timeout = _agent_init_timeout() if timeout is None else timeout + if ready is not None and not ready.wait(timeout=wait_timeout): + return _err(rid, 5032, "agent initialization timed out") +''' + + +def patch(source: Path, destination: Path) -> None: + """Apply the narrow timeout override and fail on upstream drift.""" + content = source.read_text(encoding="utf-8") + if BEFORE not in content: + raise RuntimeError("Hermes TUI gateway patch context changed") + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(content.replace(BEFORE, AFTER, 1), encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("destination", type=Path) + args = parser.parse_args() + patch(args.source, args.destination) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/service.yaml b/services/hermes/service.yaml index afbe2ecae..c5974182c 100644 --- a/services/hermes/service.yaml +++ b/services/hermes/service.yaml @@ -41,25 +41,6 @@ spec: --- apiVersion: v1 kind: Service -metadata: - name: hermes-agent - namespace: hermes - labels: - app: hermes-agent -spec: - type: ClusterIP - selector: - app: hermes-agent - ports: - - name: dashboard - port: 8787 - targetPort: dashboard - - name: terminal - port: 7681 - targetPort: terminal ---- -apiVersion: v1 -kind: Service metadata: name: hermes-chat-tenant namespace: hermes diff --git a/testing/tests/test_hermes_auto_router.py b/testing/tests/test_hermes_auto_router.py index 17abfd874..6954b8ecd 100644 --- a/testing/tests/test_hermes_auto_router.py +++ b/testing/tests/test_hermes_auto_router.py @@ -99,6 +99,28 @@ def test_route_uses_managed_models_and_connected_provider_fallback(): assert fallback["provider"] == "anthropic" +def test_route_circuit_breaker_skips_recently_failed_provider(): + status = _status() + status["routes"]["codex-medium"] = [ + "openai-codex/gpt-5.6-terra", + "anthropic/claude-sonnet-5", + ] + decision = router.Decision( + "question", "medium", "claude", "jetson", "local vote" + ) + policy = { + "mode": "auto", + "provider_cooldowns": { + "anthropic": {"until_epoch": router.time.time() + 300} + }, + } + + plan = router.select_route(status, decision, policy=policy) + + assert plan["profile"] == "codex-medium" + assert plan["provider"] == "openai-codex" + + def test_local_classifier_accepts_only_bounded_route_decisions(): assert router._validated_local_route("?", "?", 1) is None decision = router._validated_local_route("A", "H", 1) @@ -477,6 +499,7 @@ def test_post_turn_records_and_announces_capacity_fallback(monkeypatch): assert outcome["fallback_used"] is True assert outcome["actual_provider"] == "openai-codex" assert outcome["actual_model"] == "gpt-5.6-terra" + assert written[-1]["provider_cooldowns"]["anthropic"]["until_epoch"] > router.time.time() assert agent.message.startswith("FALLBACK USED") diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index b3f89833e..af939ba7f 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -32,6 +32,7 @@ lanes = _load("cli_lane_runner") policy = _load("claude_command_policy") migration = _load("migrate_herdr_state") auth_patch = _load("patch_hermes_auth") +tui_gateway_patch = _load("patch_tui_gateway") ttyd_patch = _load("patch_ttyd_index") client_config = _load("configure_agent_clients") @@ -625,36 +626,89 @@ def test_agent_uses_one_native_kanban_control_plane(): assert "herdr-dispatch" not in rendered -def test_agent_root_is_webui_and_terminal_is_a_separate_path(): +def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path(): deployment = _agent_deployment() - terminal = next( - item for item in deployment["spec"]["template"]["spec"]["containers"] - if item["name"] == "terminal" - ) + pod = deployment["spec"]["template"]["spec"] + containers = {item["name"]: item for item in pod["containers"]} + assert "webui" not in containers + + dashboard = containers["dashboard"] + dashboard_command = dashboard["args"][0] + assert "/opt/hermes/.venv/bin/hermes dashboard" in dashboard_command + assert "--host 127.0.0.1" in dashboard_command + assert "--port 9119" in dashboard_command + assert "--skip-build" in dashboard_command + assert "--no-open" in dashboard_command + dashboard_env = {item["name"]: item["value"] for item in dashboard["env"]} + assert dashboard_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" + + terminal = containers["terminal"] command = terminal["args"][0] assert "--base-path /terminal" in command - assert "--check-origin" in command + assert "--check-origin" not in command assert "/usr/bin/tmux new-session -A" in command assert "--continue" in command assert "--yolo" in command + terminal_env = {item["name"]: item["value"] for item in terminal["env"]} + assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180" + + args = containers["oauth2-proxy"]["args"] + terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/" + dashboard_upstream = "--upstream=http://127.0.0.1:9119/" + assert terminal_upstream in args + assert dashboard_upstream in args + assert args.index(terminal_upstream) < args.index(dashboard_upstream) + assert "--pass-host-header=false" in args + + patch_init = next( + item for item in pod["initContainers"] + if item["name"] == "patch-tui-gateway" + ) + assert patch_init["command"][-1] == "/patched/server.py" + for name in ("dashboard", "terminal"): + mounts = containers[name]["volumeMounts"] + assert { + "name": "tui-gateway-patch", + "mountPath": "/opt/hermes/tui_gateway/server.py", + "subPath": "server.py", + } in mounts - oauth = _oauth_deployment("oauth2-proxy-hermes-agent") - args = oauth["spec"]["template"]["spec"]["containers"][0]["args"] - assert "--upstream=http://hermes-agent.hermes.svc.cluster.local:7681/terminal/" in args - assert "--upstream=http://hermes-agent.hermes.svc.cluster.local:8787/" in args - assert args.index("--upstream=http://hermes-agent.hermes.svc.cluster.local:7681/terminal/") < args.index("--upstream=http://hermes-agent.hermes.svc.cluster.local:8787/") ingress_documents = [ item for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text()) if item ] - middleware = next(item for item in ingress_documents if item["kind"] == "Middleware") - assert middleware["spec"]["redirectRegex"]["replacement"].endswith("/terminal/") + middlewares = { + item["metadata"]["name"]: item + for item in ingress_documents + if item["kind"] == "Middleware" + } + assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][ + "replacement" + ].endswith("/terminal/") + assert middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][ + "customRequestHeaders" + ]["Origin"] == "http://127.0.0.1:9119" + ingresses = { + item["metadata"]["name"]: item + for item in ingress_documents + if item["kind"] == "Ingress" + } + assert ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][ + "traefik.ingress.kubernetes.io/router.middlewares" + ] == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd" + assert ingresses["hermes-agent-terminal"]["metadata"]["annotations"][ + "traefik.ingress.kubernetes.io/router.middlewares" + ] == "hermes-hermes-agent-terminal-slash@kubernetescrd" def test_agent_auth_is_bstein_group_and_email_bounded(): - oauth = _oauth_deployment("oauth2-proxy-hermes-agent") - args = oauth["spec"]["template"]["spec"]["containers"][0]["args"] + deployment = _agent_deployment() + oauth = next( + item for item in deployment["spec"]["template"]["spec"]["containers"] + if item["name"] == "oauth2-proxy" + ) + args = oauth["args"] assert "--user-id-claim=sub" in args assert "--oidc-groups-claim=groups" in args assert "--allowed-group=/hermes-owner" in args @@ -674,11 +728,19 @@ def test_agent_network_boundary_allows_only_authenticated_web_surfaces(): isolation = next(item for item in documents if item.get("metadata", {}).get("name") == "hermes-agent-isolation") assert isolation["spec"]["ingress"] == [ { - "from": [{"podSelector": {"matchLabels": {"app": "oauth2-proxy-hermes-agent"}}}], - "ports": [ - {"protocol": "TCP", "port": 7681}, - {"protocol": "TCP", "port": 8787}, + "from": [ + { + "namespaceSelector": { + "matchLabels": { + "kubernetes.io/metadata.name": "traefik" + } + }, + "podSelector": { + "matchLabels": {"app.kubernetes.io/name": "traefik"} + }, + } ], + "ports": [{"protocol": "TCP", "port": 4180}], } ] assert isolation["spec"]["egress"] == [{}] @@ -735,7 +797,6 @@ def test_owner_agent_uses_only_the_canonical_hostname(): HERMES / "agent-configmap.yaml", HERMES / "agent-deployment.yaml", HERMES / "agent-ingress.yaml", - HERMES / "oauth2-proxy.yaml", Path(__file__).parents[2] / "scripts/ops/hermes_triage_monitor.py", ] for path in paths: @@ -774,6 +835,32 @@ def test_auth_patch_fails_closed_on_upstream_drift(tmp_path: Path): auth_patch.patch(source, tmp_path / "patched.py") +def test_tui_gateway_patch_extends_and_bounds_agent_startup(tmp_path: Path): + source = tmp_path / "server.py" + destination = tmp_path / "patched/server.py" + source.write_text( + "import os\n\n" + tui_gateway_patch.BEFORE + "\ndef unchanged():\n pass\n", + encoding="utf-8", + ) + + tui_gateway_patch.patch(source, destination) + + content = destination.read_text(encoding="utf-8") + assert "HERMES_TUI_AGENT_INIT_TIMEOUT_S" in content + assert 'configured = 180.0' in content + assert "return max(30.0, min(configured, 900.0))" in content + assert "timeout: float | None = None" in content + assert "ready.wait(timeout=wait_timeout)" in content + assert "def unchanged():" in content + + +def test_tui_gateway_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "server.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + tui_gateway_patch.patch(source, tmp_path / "patched.py") + + def test_ttyd_clipboard_and_reconnect_patch_remain_enabled(): source = '
' content = ttyd_patch.patch_html(source) diff --git a/testing/tests/test_hermes_coordinator.py b/testing/tests/test_hermes_coordinator.py index bd7d6e4b9..1c2c304dc 100644 --- a/testing/tests/test_hermes_coordinator.py +++ b/testing/tests/test_hermes_coordinator.py @@ -101,7 +101,8 @@ def test_configure_routes_builds_cross_provider_fallback_profiles(tmp_path: Path assert claude_profile["fallback_providers"][0]["provider"] == "openai-codex" assert codex_profile["toolsets"] == [] assert codex_profile["agent"]["reasoning_effort"] == "high" - assert codex_profile["fallback_providers"][1] == routing.JETSON_FALLBACK + assert codex_profile["fallback_providers"][1] == routing.ATLAS_FALLBACK + assert len(codex_profile["fallback_providers"]) == 2 assert routes["coordinator"][0] == "openai-codex/gpt-5.6-terra" assert "max" not in json.dumps(routes) diff --git a/testing/tests/test_hermes_model_gate.py b/testing/tests/test_hermes_model_gate.py new file mode 100644 index 000000000..e6974b5f6 --- /dev/null +++ b/testing/tests/test_hermes_model_gate.py @@ -0,0 +1,39 @@ +"""Focused contracts for Hermes's local inference admission proxy.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import yaml + + +HERMES = Path(__file__).parents[2] / "services/hermes" + + +def _model_gate_namespace() -> dict: + document = yaml.safe_load((HERMES / "model-gate-configmap.yaml").read_text()) + namespace = {"__name__": "hermes_model_gate_test"} + exec(compile(document["data"]["model_gate.py"], "model_gate.py", "exec"), namespace) + return namespace + + +def test_model_gate_clamps_hosted_only_reasoning_efforts(): + normalize = _model_gate_namespace()["_normalize_reasoning"] + + top_level = json.loads(normalize(b'{"reasoning_effort":"xhigh"}')) + string_reasoning = json.loads(normalize(b'{"reasoning":"max"}')) + nested = json.loads(normalize(b'{"reasoning":{"effort":"xhigh"}}')) + + assert top_level["reasoning_effort"] == "high" + assert string_reasoning["reasoning"] == "high" + assert nested["reasoning"]["effort"] == "high" + + +def test_model_gate_preserves_supported_and_non_json_requests(): + normalize = _model_gate_namespace()["_normalize_reasoning"] + supported = b'{"reasoning_effort":"medium","messages":[]}' + non_json = b"streamed-body" + + assert normalize(supported) == supported + assert normalize(non_json) == non_json