atlas-iac/services/hermes/chat-statefulset.yaml

# services/hermes/chat-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: hermes-chat-tenant
  namespace: hermes
  labels:
    app: hermes-chat-tenant
  annotations:
    kustomize.toolkit.fluxcd.io/force: enabled
spec:
  serviceName: hermes-chat-tenant
  replicas: 4
  podManagementPolicy: Parallel
  persistentVolumeClaimRetentionPolicy:
    whenDeleted: Retain
    whenScaled: Retain
  updateStrategy:
    type: RollingUpdate
  selector:
    matchLabels:
      app: hermes-chat-tenant
  template:
    metadata:
      labels:
        app: hermes-chat-tenant
      annotations:
        ai.bstein.dev/role: isolated-user-chat
        ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive
        ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject
        ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides
        ai.bstein.dev/config-rev: "20260816-telegram-topics"
        vault.hashicorp.com/agent-inject: "true"
        vault.hashicorp.com/role: hermes-chat
        vault.hashicorp.com/agent-inject-secret-chat-relay-key: kv/data/atlas/hermes/chat-telegram
        vault.hashicorp.com/agent-inject-template-chat-relay-key: |
          {{- with secret "kv/data/atlas/hermes/chat-telegram" -}}
          {{ .Data.data.relay_key }}
          {{- end }}
        vault.hashicorp.com/agent-pre-populate-only: "true"
        vault.hashicorp.com/agent-init-first: "true"
        vault.hashicorp.com/agent-requests-cpu: 25m
        vault.hashicorp.com/agent-requests-mem: 32Mi
        vault.hashicorp.com/agent-limits-cpu: 100m
        vault.hashicorp.com/agent-limits-mem: 128Mi
    spec:
      serviceAccountName: hermes-chat
      automountServiceAccountToken: true
      securityContext:
        fsGroup: 10000
        fsGroupChangePolicy: OnRootMismatch
        seccompProfile:
          type: RuntimeDefault
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: kubernetes.io/arch
                    operator: In
                    values: [arm64]
                  - key: node-role.kubernetes.io/worker
                    operator: In
                    values: ["true"]
                  - key: node-role.kubernetes.io/storage-backbone
                    operator: DoesNotExist
                  - key: kubernetes.io/hostname
                    operator: NotIn
                    values: [titan-05, titan-13, titan-14, titan-17, titan-18, titan-19]
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              preference:
                matchExpressions:
                  - key: node-role.kubernetes.io/storage-backbone
                    operator: DoesNotExist
            - weight: 100
              preference:
                matchExpressions:
                  - key: hardware
                    operator: In
                    values: [rpi5]
            - weight: 40
              preference:
                matchExpressions:
                  - key: hardware
                    operator: In
                    values: [rpi4]
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                labelSelector:
                  matchLabels:
                    app: hermes-chat-tenant
                topologyKey: kubernetes.io/hostname
      initContainers:
        - name: init-config
          image: busybox:1.37
          imagePullPolicy: IfNotPresent
          command:
            - sh
            - -c
            - |
              set -eu
              mkdir -p /opt/data/home/.local/bin /opt/data/logs /opt/data/workspace
              if [ ! -e /opt/data/workspace/.hermes-workspace-v1 ]; then
                if [ -d /legacy-home/workspace ]; then
                  cp -a /legacy-home/workspace/. /opt/data/workspace/
                fi
                touch /opt/data/workspace/.hermes-workspace-v1
              fi
              cp /config/config.yaml /opt/data/config.yaml
              cp /config/SOUL.md /opt/data/SOUL.md
              cp /config/AGENTS.md /opt/data/workspace/AGENTS.md
              touch /opt/data/.env
              for key in ANTHROPIC_API_KEY API_SERVER_KEY CLAUDE_API_KEY CLAUDE_CODE_OAUTH_TOKEN GITEA_TOKEN GITEA_USERNAME HERMES_IMAGE_BROKER_KEY OPENAI_API_KEY; do
                grep -v "^${key}=" /opt/data/.env > /opt/data/.env.tmp || true
                mv /opt/data/.env.tmp /opt/data/.env
              done
              rm -f /opt/data/auth.json
              chmod 0600 /opt/data/.env
              chown 10000:10000 \
                /opt/data \
                /opt/data/home \
                /opt/data/home/.local \
                /opt/data/home/.local/bin \
                /opt/data/logs \
                /opt/data/workspace \
                /opt/data/config.yaml \
                /opt/data/SOUL.md \
                /opt/data/workspace/AGENTS.md \
                /opt/data/.env
          securityContext:
            allowPrivilegeEscalation: false
            runAsUser: 0
            runAsGroup: 0
            seccompProfile:
              type: RuntimeDefault
          volumeMounts:
            - {name: home, mountPath: /opt/data}
            - {name: home, mountPath: /legacy-home, readOnly: true}
            - {name: workspace, mountPath: /opt/data/workspace}
            - {name: config, mountPath: /config, readOnly: true}
          resources:
            requests: {cpu: 25m, memory: 32Mi}
            limits: {cpu: 100m, memory: 64Mi}
        - name: stage-runtime-access
          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
          imagePullPolicy: IfNotPresent
          command:
            - /opt/hermes/.venv/bin/python
            - /opt/coordinator/stage_runtime_access.py
            - chat
          securityContext:
            allowPrivilegeEscalation: false
            runAsUser: 0
            runAsGroup: 0
            seccompProfile:
              type: RuntimeDefault
          volumeMounts:
            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
            - {name: runtime-access, mountPath: /runtime-access}
          resources:
            requests: {cpu: 25m, memory: 32Mi}
            limits: {cpu: 100m, memory: 64Mi}
        - name: patch-auth
          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
          imagePullPolicy: IfNotPresent
          command:
            - /opt/hermes/.venv/bin/python
            - /opt/coordinator/patch_hermes_auth.py
            - /opt/hermes/hermes_cli/auth.py
            - /patched/auth.py
          securityContext:
            allowPrivilegeEscalation: false
            runAsUser: 10000
            runAsGroup: 10000
            seccompProfile:
              type: RuntimeDefault
          volumeMounts:
            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
            - {name: auth-patch, mountPath: /patched}
          resources:
            requests: {cpu: 25m, memory: 64Mi}
            limits: {cpu: 100m, memory: 128Mi}
        - name: patch-stream-recovery
          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
          imagePullPolicy: IfNotPresent
          command:
            - /opt/hermes/.venv/bin/python
            - /opt/coordinator/patch_stream_recovery.py
            - /opt/hermes/agent/conversation_loop.py
            - /patched/conversation_loop.py
          securityContext:
            allowPrivilegeEscalation: false
            runAsUser: 10000
            runAsGroup: 10000
            seccompProfile:
              type: RuntimeDefault
          volumeMounts:
            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
            - {name: stream-recovery-patch, mountPath: /patched}
          resources:
            requests: {cpu: 25m, memory: 64Mi}
            limits: {cpu: 100m, memory: 128Mi}
        - name: patch-api-server-sessions
          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
          imagePullPolicy: IfNotPresent
          command: [/bin/sh, -ec]
          args:
            - |
              /opt/hermes/.venv/bin/python /opt/coordinator/patch_api_server_sessions.py \
                /opt/hermes/gateway/platforms/api_server.py /patched/api_server.py
              grep -Fq 'conversation_history = compact_telegram_history(conversation_history)' /patched/api_server.py
              grep -Fq 'full_history = compact_telegram_history(full_history)' /patched/api_server.py
              grep -Fq 'conversation_history_snapshot = compact_telegram_history(' /patched/api_server.py
              /opt/hermes/.venv/bin/python /opt/coordinator/migrate_telegram_api_sessions.py \
                /opt/data/state.db /opt/data/response_store.db
          securityContext:
            allowPrivilegeEscalation: false
            runAsUser: 10000
            runAsGroup: 10000
            seccompProfile:
              type: RuntimeDefault
          volumeMounts:
            - {name: home, mountPath: /opt/data}
            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
            - {name: api-server-patch, mountPath: /patched}
          resources:
            requests: {cpu: 25m, memory: 64Mi}
            limits: {cpu: 100m, memory: 128Mi}
        - name: patch-subprocess-secret-boundary
          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
          imagePullPolicy: IfNotPresent
          command:
            - /bin/sh
            - -ec
            - |
              /opt/hermes/.venv/bin/python /opt/coordinator/patch_subprocess_secret_boundary.py \
                /opt/hermes/tools/environments/local.py /patched/local.py
              /opt/hermes/.venv/bin/python /opt/coordinator/patch_process_output_redaction.py \
                /opt/hermes/tools/process_registry.py /patched/process_registry.py
          securityContext:
            allowPrivilegeEscalation: false
            runAsUser: 10000
            runAsGroup: 10000
            seccompProfile:
              type: RuntimeDefault
          volumeMounts:
            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
            - {name: subprocess-secret-patch, mountPath: /patched}
          resources:
            requests: {cpu: 25m, memory: 64Mi}
            limits: {cpu: 100m, memory: 128Mi}
      containers:
        - name: hermes
          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
          imagePullPolicy: IfNotPresent
          command: [/bin/sh, -ec]
          args:
            - |
              ordinal="${HOSTNAME##*-}"
              export HERMES_CODE_SANDBOX_URL="http://hermes-chat-sandbox-${ordinal}.hermes-chat-sandbox.hermes.svc.cluster.local:9080/v1/execute"
              set -a
              . /opt/data/.env
              set +a
              API_SERVER_KEY="$(tr -d '\r\n' < /runtime-access/chat-relay-key)"
              test -n "${API_SERVER_KEY}"
              export API_SERVER_KEY
              exec /opt/hermes/.venv/bin/hermes gateway run
          ports:
            - {name: api, containerPort: 8642, protocol: TCP}
          env:
            - {name: HERMES_HOME, value: /opt/data}
            - {name: HERMES_AUTH_FILE, value: /runtime-access/hermes-auth.json}
            - {name: HOME, value: /opt/data/home}
            - {name: TERMINAL_CWD, value: /opt/data/workspace}
            - {name: HERMES_WRITE_SAFE_ROOT, value: /opt/data/workspace}
            - {name: PATH, value: '/opt/data/home/.local/bin:/opt/hermes/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'}
            - {name: PLAYWRIGHT_BROWSERS_PATH, value: /opt/hermes/.playwright}
            - {name: AGENT_BROWSER_EXECUTABLE_PATH, value: /opt/hermes/.playwright/chromium_headless_shell-1228/chrome-linux/headless_shell}
            - {name: AGENT_BROWSER_ARGS, value: "--no-sandbox,--disable-dev-shm-usage"}
            - {name: HERMES_DASHBOARD, value: "0"}
            - {name: API_SERVER_ENABLED, value: "true"}
            - {name: API_SERVER_HOST, value: 0.0.0.0}
            - {name: API_SERVER_PORT, value: "8642"}
            - {name: API_SERVER_CORS_ORIGINS, value: 'https://chat.bstein.dev'}
            - {name: HERMES_IMAGE_BROKER_URL, value: 'http://hermes-image-broker.hermes.svc.cluster.local:9002'}
            - {name: HERMES_IMAGE_BROKER_KEY_FILE, value: /runtime-access/chat-relay-key}
            - {name: HERMES_AUTO_ROUTER_PROFILE, value: chat}
          volumeMounts:
            - {name: home, mountPath: /opt/data}
            - {name: workspace, mountPath: /opt/data/workspace}
            - {name: runtime-access, mountPath: /runtime-access}
            - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
            - {name: api-server-patch, mountPath: /opt/hermes/gateway/platforms/api_server.py, subPath: api_server.py}
            - {name: coordinator, mountPath: /opt/hermes/gateway/platforms/telegram_continuity.py, subPath: migrate_telegram_api_sessions.py, readOnly: true}
            - {name: stream-recovery-patch, mountPath: /opt/hermes/agent/conversation_loop.py, subPath: conversation_loop.py}
            - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/environments/local.py, subPath: local.py}
            - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py}
            - {name: image-plugin, mountPath: /opt/hermes/plugins/image_gen/atlas-broker, readOnly: true}
            - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}
          readinessProbe:
            tcpSocket: {port: api}
            initialDelaySeconds: 30
            periodSeconds: 10
            timeoutSeconds: 5
          livenessProbe:
            tcpSocket: {port: api}
            initialDelaySeconds: 90
            periodSeconds: 30
            timeoutSeconds: 10
          securityContext:
            allowPrivilegeEscalation: false
            runAsUser: 10000
            runAsGroup: 10000
            seccompProfile:
              type: RuntimeDefault
          resources:
            requests: {cpu: 250m, memory: 256Mi}
            limits: {cpu: "1", memory: 2Gi}
        - name: webui
          image: registry.bstein.dev/bstein/hermes-webui:git-708d6111014604b4858adbdd7fa53f1506da90bd-build-10-release@sha256:72ad65a6dd121c7e4fdc00b21e223585f973633b998c3c11b38dc9757a6c5dc9 # {"$imagepolicy": "hermes:hermes-webui-release"}
          imagePullPolicy: IfNotPresent
          command: [/bin/sh, -ec]
          args:
            - |
              api_key="$(tr -d '\r\n' < /runtime-access/chat-relay-key)"
              test -n "${api_key}"
              export API_SERVER_KEY="${api_key}"
              export HERMES_WEBUI_GATEWAY_API_KEY="${api_key}"
              exec /opt/hermes/.venv/bin/python /opt/hermes-webui/server.py
          ports:
            - {name: webui, containerPort: 8787, protocol: TCP}
          env:
            - {name: HERMES_HOME, value: /opt/data}
            - {name: HERMES_AUTH_FILE, value: /runtime-access/hermes-auth.json}
            - {name: HOME, value: /opt/data/home}
            - {name: HERMES_WEBUI_AGENT_DIR, value: /opt/hermes}
            - {name: HERMES_WEBUI_HOST, value: 0.0.0.0}
            - {name: HERMES_WEBUI_PORT, value: "8787"}
            - {name: HERMES_WEBUI_STATE_DIR, value: /opt/data/webui}
            - {name: HERMES_WEBUI_DEFAULT_WORKSPACE, value: /opt/data/workspace}
            - {name: HERMES_WEBUI_CHAT_BACKEND, value: gateway}
            - {name: HERMES_WEBUI_GATEWAY_BASE_URL, value: 'http://127.0.0.1:8642'}
            - {name: HERMES_WEBUI_GATEWAY_USE_RUNS_API, value: "true"}
            - {name: HERMES_WEBUI_SKIP_ONBOARDING, value: "1"}
            - {name: HERMES_WEBUI_SECURE, value: "1"}
            - {name: HERMES_WEBUI_COOKIE_NAME, value: hermes_chat_session}
            - {name: HERMES_WEBUI_PROFILE_COOKIE_NAME, value: hermes_chat_profile}
            - {name: HERMES_WEBUI_TRUSTED_AUTH_HEADER, value: X-Hermes-Tenant-Identity}
            # NetworkPolicy admits this port only from hermes-chat-router; the
            # CIDR lets the WebUI validate that router's changing pod address.
            - {name: HERMES_WEBUI_TRUSTED_PROXY_CIDRS, value: 10.42.0.0/16}
            - {name: HERMES_WEBUI_ALLOWED_ORIGINS, value: 'https://chat.bstein.dev'}
            - {name: HERMES_WEBUI_TRUST_FORWARDED_HOST, value: "1"}
            - {name: HERMES_WEBUI_TRUST_FORWARDED_PROTO, value: "1"}
            - {name: HERMES_ROUTER_PROFILE, value: chat}
            - {name: HERMES_STT_URL, value: 'http://hermes-stt.hermes.svc.cluster.local:9000/v1/audio/transcriptions'}
            - {name: HERMES_WEBUI_ATLAS_STT_STREAM_URL, value: 'http://hermes-stt.hermes.svc.cluster.local:9000/v1/audio/transcriptions/stream'}
            - {name: HERMES_LOCAL_STT_COMMAND, value: "/opt/hermes/.venv/bin/python /opt/coordinator/hermes_stt_client.py {input_path} --output-dir {output_dir} --language {language} --model {model}"}
            - {name: HERMES_WEBUI_ATLAS_TTS_URL, value: 'http://hermes-tts.hermes.svc.cluster.local:9001/v1/audio/speech'}
            - {name: HERMES_WEBUI_ATLAS_TTS_STREAM_URL, value: 'http://hermes-tts.hermes.svc.cluster.local:9001/v1/audio/speech/stream'}
          volumeMounts:
            - {name: home, mountPath: /opt/data}
            - {name: workspace, mountPath: /opt/data/workspace}
            - {name: runtime-access, mountPath: /runtime-access, readOnly: true}
            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
            - {name: tmp, mountPath: /tmp}
          readinessProbe:
            httpGet: {path: /health, port: webui}
            initialDelaySeconds: 10
            periodSeconds: 10
            timeoutSeconds: 5
          livenessProbe:
            httpGet: {path: /health, port: webui}
            initialDelaySeconds: 30
            periodSeconds: 30
            timeoutSeconds: 10
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop: [ALL]
            readOnlyRootFilesystem: true
            runAsUser: 10000
            runAsGroup: 10000
            seccompProfile:
              type: RuntimeDefault
          resources:
            requests: {cpu: 100m, memory: 224Mi}
            limits: {cpu: 750m, memory: 1Gi}
        - name: telegram-media
          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
          imagePullPolicy: IfNotPresent
          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/telegram_media_server.py]
          ports:
            - {name: telegram-media, containerPort: 8788, protocol: TCP}
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - {name: HERMES_MEDIA_RELAY_KEY_FILE, value: /runtime-access/chat-relay-key}
          volumeMounts:
            - {name: home, mountPath: /opt/data, readOnly: true}
            - {name: workspace, mountPath: /opt/data/workspace, readOnly: true}
            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
            - {name: runtime-access, mountPath: /runtime-access, readOnly: true}
          readinessProbe:
            httpGet: {path: /healthz, port: telegram-media}
            initialDelaySeconds: 2
            periodSeconds: 10
          livenessProbe:
            httpGet: {path: /healthz, port: telegram-media}
            initialDelaySeconds: 10
            periodSeconds: 20
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop: [ALL]
            readOnlyRootFilesystem: true
            runAsUser: 10000
            runAsGroup: 10000
            seccompProfile:
              type: RuntimeDefault
          resources:
            requests: {cpu: 10m, memory: 24Mi}
            limits: {cpu: 100m, memory: 64Mi}
      volumes:
        - name: runtime-access
          emptyDir:
            medium: Memory
            sizeLimit: 2Mi
        - name: config
          configMap:
            name: hermes-chat-config
        - name: coordinator
          configMap:
            name: hermes-coordinator
            defaultMode: 0555
        - name: auth-patch
          emptyDir: {}
        - name: api-server-patch
          emptyDir: {}
        - name: stream-recovery-patch
          emptyDir: {}
        - name: subprocess-secret-patch
          emptyDir: {}
        - name: auto-router-plugin
          configMap:
            name: hermes-auto-router-plugin
        - name: image-plugin
          configMap:
            name: hermes-chat-image-plugin
        - name: tmp
          emptyDir:
            sizeLimit: 256Mi
  volumeClaimTemplates:
    - metadata:
        name: home
        labels:
          app: hermes-chat-tenant
      spec:
        accessModes: [ReadWriteOnce]
        storageClassName: astreae
        resources:
          requests:
            storage: 10Gi
    - metadata:
        name: workspace
        labels:
          app: hermes-chat-tenant
          ai.bstein.dev/data: user-workspace
      spec:
        accessModes: [ReadWriteMany]
        storageClassName: astreae
        resources:
          requests:
            storage: 10Gi