atlas-iac/services/hermes/chat-statefulset.yaml

471 lines
21 KiB
YAML

# services/hermes/chat-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: hermes-chat-tenant
namespace: hermes
labels:
app: hermes-chat-tenant
annotations:
kustomize.toolkit.fluxcd.io/force: enabled
spec:
serviceName: hermes-chat-tenant
replicas: 4
podManagementPolicy: Parallel
persistentVolumeClaimRetentionPolicy:
whenDeleted: Retain
whenScaled: Retain
updateStrategy:
type: RollingUpdate
selector:
matchLabels:
app: hermes-chat-tenant
template:
metadata:
labels:
app: hermes-chat-tenant
annotations:
ai.bstein.dev/role: isolated-user-chat
ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive
ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject
ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides
ai.bstein.dev/config-rev: "20260816-telegram-topics"
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-chat
vault.hashicorp.com/agent-inject-secret-chat-relay-key: kv/data/atlas/hermes/chat-telegram
vault.hashicorp.com/agent-inject-template-chat-relay-key: |
{{- with secret "kv/data/atlas/hermes/chat-telegram" -}}
{{ .Data.data.relay_key }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 25m
vault.hashicorp.com/agent-requests-mem: 32Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
spec:
serviceAccountName: hermes-chat
automountServiceAccountToken: true
securityContext:
fsGroup: 10000
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/arch
operator: In
values: [arm64]
- key: node-role.kubernetes.io/worker
operator: In
values: ["true"]
- key: kubernetes.io/hostname
operator: NotIn
values: [titan-05, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19]
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: hardware
operator: In
values: [rpi5]
- weight: 40
preference:
matchExpressions:
- key: hardware
operator: In
values: [rpi4]
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
app: hermes-chat-tenant
topologyKey: kubernetes.io/hostname
initContainers:
- name: init-config
image: busybox:1.37
imagePullPolicy: IfNotPresent
command:
- sh
- -c
- |
set -eu
mkdir -p /opt/data/home/.local/bin /opt/data/logs /opt/data/workspace
if [ ! -e /opt/data/workspace/.hermes-workspace-v1 ]; then
if [ -d /legacy-home/workspace ]; then
cp -a /legacy-home/workspace/. /opt/data/workspace/
fi
touch /opt/data/workspace/.hermes-workspace-v1
fi
cp /config/config.yaml /opt/data/config.yaml
cp /config/SOUL.md /opt/data/SOUL.md
cp /config/AGENTS.md /opt/data/workspace/AGENTS.md
touch /opt/data/.env
for key in ANTHROPIC_API_KEY API_SERVER_KEY CLAUDE_API_KEY CLAUDE_CODE_OAUTH_TOKEN GITEA_TOKEN GITEA_USERNAME HERMES_IMAGE_BROKER_KEY OPENAI_API_KEY; do
grep -v "^${key}=" /opt/data/.env > /opt/data/.env.tmp || true
mv /opt/data/.env.tmp /opt/data/.env
done
rm -f /opt/data/auth.json
chmod 0600 /opt/data/.env
chown 10000:10000 \
/opt/data \
/opt/data/home \
/opt/data/home/.local \
/opt/data/home/.local/bin \
/opt/data/logs \
/opt/data/workspace \
/opt/data/config.yaml \
/opt/data/SOUL.md \
/opt/data/workspace/AGENTS.md \
/opt/data/.env
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
runAsGroup: 0
seccompProfile:
type: RuntimeDefault
volumeMounts:
- {name: home, mountPath: /opt/data}
- {name: home, mountPath: /legacy-home, readOnly: true}
- {name: workspace, mountPath: /opt/data/workspace}
- {name: config, mountPath: /config, readOnly: true}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: 100m, memory: 64Mi}
- name: stage-runtime-access
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command:
- /opt/hermes/.venv/bin/python
- /opt/coordinator/stage_runtime_access.py
- chat
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
runAsGroup: 0
seccompProfile:
type: RuntimeDefault
volumeMounts:
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: runtime-access, mountPath: /runtime-access}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: 100m, memory: 64Mi}
- name: patch-auth
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command:
- /opt/hermes/.venv/bin/python
- /opt/coordinator/patch_hermes_auth.py
- /opt/hermes/hermes_cli/auth.py
- /patched/auth.py
securityContext:
allowPrivilegeEscalation: false
runAsUser: 10000
runAsGroup: 10000
seccompProfile:
type: RuntimeDefault
volumeMounts:
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: auth-patch, mountPath: /patched}
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: 100m, memory: 128Mi}
- name: patch-stream-recovery
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command:
- /opt/hermes/.venv/bin/python
- /opt/coordinator/patch_stream_recovery.py
- /opt/hermes/agent/conversation_loop.py
- /patched/conversation_loop.py
securityContext:
allowPrivilegeEscalation: false
runAsUser: 10000
runAsGroup: 10000
seccompProfile:
type: RuntimeDefault
volumeMounts:
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: stream-recovery-patch, mountPath: /patched}
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: 100m, memory: 128Mi}
- name: patch-api-server-sessions
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command: [/bin/sh, -ec]
args:
- |
/opt/hermes/.venv/bin/python /opt/coordinator/patch_api_server_sessions.py \
/opt/hermes/gateway/platforms/api_server.py /patched/api_server.py
grep -Fq 'conversation_history = compact_telegram_history(conversation_history)' /patched/api_server.py
grep -Fq 'full_history = compact_telegram_history(full_history)' /patched/api_server.py
grep -Fq 'conversation_history_snapshot = compact_telegram_history(' /patched/api_server.py
/opt/hermes/.venv/bin/python /opt/coordinator/migrate_telegram_api_sessions.py \
/opt/data/state.db /opt/data/response_store.db
securityContext:
allowPrivilegeEscalation: false
runAsUser: 10000
runAsGroup: 10000
seccompProfile:
type: RuntimeDefault
volumeMounts:
- {name: home, mountPath: /opt/data}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: api-server-patch, mountPath: /patched}
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: 100m, memory: 128Mi}
- name: patch-subprocess-secret-boundary
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command:
- /bin/sh
- -ec
- |
/opt/hermes/.venv/bin/python /opt/coordinator/patch_subprocess_secret_boundary.py \
/opt/hermes/tools/environments/local.py /patched/local.py
/opt/hermes/.venv/bin/python /opt/coordinator/patch_process_output_redaction.py \
/opt/hermes/tools/process_registry.py /patched/process_registry.py
securityContext:
allowPrivilegeEscalation: false
runAsUser: 10000
runAsGroup: 10000
seccompProfile:
type: RuntimeDefault
volumeMounts:
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: subprocess-secret-patch, mountPath: /patched}
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: 100m, memory: 128Mi}
containers:
- name: hermes
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command: [/bin/sh, -ec]
args:
- |
ordinal="${HOSTNAME##*-}"
export HERMES_CODE_SANDBOX_URL="http://hermes-chat-sandbox-${ordinal}.hermes-chat-sandbox.hermes.svc.cluster.local:9080/v1/execute"
set -a
. /opt/data/.env
set +a
API_SERVER_KEY="$(tr -d '\r\n' < /runtime-access/chat-relay-key)"
test -n "${API_SERVER_KEY}"
export API_SERVER_KEY
exec /opt/hermes/.venv/bin/hermes gateway run
ports:
- {name: api, containerPort: 8642, protocol: TCP}
env:
- {name: HERMES_HOME, value: /opt/data}
- {name: HERMES_AUTH_FILE, value: /runtime-access/hermes-auth.json}
- {name: HOME, value: /opt/data/home}
- {name: TERMINAL_CWD, value: /opt/data/workspace}
- {name: HERMES_WRITE_SAFE_ROOT, value: /opt/data/workspace}
- {name: PATH, value: /opt/data/home/.local/bin:/opt/hermes/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin}
- {name: PLAYWRIGHT_BROWSERS_PATH, value: /opt/hermes/.playwright}
- {name: AGENT_BROWSER_EXECUTABLE_PATH, value: /opt/hermes/.playwright/chromium_headless_shell-1228/chrome-linux/headless_shell}
- {name: AGENT_BROWSER_ARGS, value: "--no-sandbox,--disable-dev-shm-usage"}
- {name: HERMES_DASHBOARD, value: "0"}
- {name: API_SERVER_ENABLED, value: "true"}
- {name: API_SERVER_HOST, value: 0.0.0.0}
- {name: API_SERVER_PORT, value: "8642"}
- {name: API_SERVER_CORS_ORIGINS, value: https://chat.bstein.dev}
- {name: HERMES_IMAGE_BROKER_URL, value: http://hermes-image-broker.hermes.svc.cluster.local:9002}
- {name: HERMES_IMAGE_BROKER_KEY_FILE, value: /runtime-access/chat-relay-key}
- {name: HERMES_AUTO_ROUTER_PROFILE, value: chat}
volumeMounts:
- {name: home, mountPath: /opt/data}
- {name: workspace, mountPath: /opt/data/workspace}
- {name: runtime-access, mountPath: /runtime-access}
- {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
- {name: api-server-patch, mountPath: /opt/hermes/gateway/platforms/api_server.py, subPath: api_server.py}
- {name: coordinator, mountPath: /opt/hermes/gateway/platforms/telegram_continuity.py, subPath: migrate_telegram_api_sessions.py, readOnly: true}
- {name: stream-recovery-patch, mountPath: /opt/hermes/agent/conversation_loop.py, subPath: conversation_loop.py}
- {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/environments/local.py, subPath: local.py}
- {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py}
- {name: image-plugin, mountPath: /opt/hermes/plugins/image_gen/atlas-broker, readOnly: true}
- {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}
readinessProbe:
tcpSocket: {port: api}
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
livenessProbe:
tcpSocket: {port: api}
initialDelaySeconds: 90
periodSeconds: 30
timeoutSeconds: 10
securityContext:
allowPrivilegeEscalation: false
runAsUser: 10000
runAsGroup: 10000
seccompProfile:
type: RuntimeDefault
resources:
requests: {cpu: 250m, memory: 512Mi}
limits: {cpu: "1", memory: 2Gi}
- name: webui
image: registry.bstein.dev/bstein/hermes-webui@sha256:c276a9e17c9057237472f39640c9ebac9d4359d9f62a34df52a759eef43167bf
imagePullPolicy: IfNotPresent
command: [/bin/sh, -ec]
args:
- |
api_key="$(tr -d '\r\n' < /runtime-access/chat-relay-key)"
test -n "${api_key}"
export API_SERVER_KEY="${api_key}"
export HERMES_WEBUI_GATEWAY_API_KEY="${api_key}"
exec /opt/hermes/.venv/bin/python /opt/hermes-webui/server.py
ports:
- {name: webui, containerPort: 8787, protocol: TCP}
env:
- {name: HERMES_HOME, value: /opt/data}
- {name: HERMES_AUTH_FILE, value: /runtime-access/hermes-auth.json}
- {name: HOME, value: /opt/data/home}
- {name: HERMES_WEBUI_AGENT_DIR, value: /opt/hermes}
- {name: HERMES_WEBUI_HOST, value: 0.0.0.0}
- {name: HERMES_WEBUI_PORT, value: "8787"}
- {name: HERMES_WEBUI_STATE_DIR, value: /opt/data/webui}
- {name: HERMES_WEBUI_DEFAULT_WORKSPACE, value: /opt/data/workspace}
- {name: HERMES_WEBUI_CHAT_BACKEND, value: gateway}
- {name: HERMES_WEBUI_GATEWAY_BASE_URL, value: http://127.0.0.1:8642}
- {name: HERMES_WEBUI_GATEWAY_USE_RUNS_API, value: "true"}
- {name: HERMES_WEBUI_SKIP_ONBOARDING, value: "1"}
- {name: HERMES_WEBUI_SECURE, value: "1"}
- {name: HERMES_WEBUI_COOKIE_NAME, value: hermes_chat_session}
- {name: HERMES_WEBUI_PROFILE_COOKIE_NAME, value: hermes_chat_profile}
- {name: HERMES_WEBUI_TRUSTED_AUTH_HEADER, value: X-Hermes-Tenant-Identity}
# NetworkPolicy admits this port only from hermes-chat-router; the
# CIDR lets the WebUI validate that router's changing pod address.
- {name: HERMES_WEBUI_TRUSTED_PROXY_CIDRS, value: 10.42.0.0/16}
- {name: HERMES_WEBUI_ALLOWED_ORIGINS, value: https://chat.bstein.dev}
- {name: HERMES_WEBUI_TRUST_FORWARDED_HOST, value: "1"}
- {name: HERMES_WEBUI_TRUST_FORWARDED_PROTO, value: "1"}
- {name: HERMES_ROUTER_PROFILE, value: chat}
- {name: HERMES_STT_URL, value: http://hermes-stt.hermes.svc.cluster.local:9000/v1/audio/transcriptions}
- {name: HERMES_LOCAL_STT_COMMAND, value: "/opt/hermes/.venv/bin/python /opt/coordinator/hermes_stt_client.py {input_path} --output-dir {output_dir} --language {language} --model {model}"}
- {name: HERMES_WEBUI_ATLAS_TTS_URL, value: http://hermes-tts.hermes.svc.cluster.local:9001/v1/audio/speech}
volumeMounts:
- {name: home, mountPath: /opt/data}
- {name: workspace, mountPath: /opt/data/workspace}
- {name: runtime-access, mountPath: /runtime-access, readOnly: true}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: tmp, mountPath: /tmp}
readinessProbe:
httpGet: {path: /health, port: webui}
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
livenessProbe:
httpGet: {path: /health, port: webui}
initialDelaySeconds: 30
periodSeconds: 30
timeoutSeconds: 10
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
readOnlyRootFilesystem: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile:
type: RuntimeDefault
resources:
requests: {cpu: 100m, memory: 256Mi}
limits: {cpu: 750m, memory: 1Gi}
- name: telegram-media
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/telegram_media_server.py]
ports:
- {name: telegram-media, containerPort: 8788, protocol: TCP}
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- {name: HERMES_MEDIA_RELAY_KEY_FILE, value: /runtime-access/chat-relay-key}
volumeMounts:
- {name: home, mountPath: /opt/data, readOnly: true}
- {name: workspace, mountPath: /opt/data/workspace, readOnly: true}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: runtime-access, mountPath: /runtime-access, readOnly: true}
readinessProbe:
httpGet: {path: /healthz, port: telegram-media}
initialDelaySeconds: 2
periodSeconds: 10
livenessProbe:
httpGet: {path: /healthz, port: telegram-media}
initialDelaySeconds: 10
periodSeconds: 20
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
readOnlyRootFilesystem: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile:
type: RuntimeDefault
resources:
requests: {cpu: 10m, memory: 24Mi}
limits: {cpu: 100m, memory: 64Mi}
volumes:
- name: runtime-access
emptyDir:
medium: Memory
sizeLimit: 2Mi
- name: config
configMap:
name: hermes-chat-config
- name: coordinator
configMap:
name: hermes-coordinator
defaultMode: 0555
- name: auth-patch
emptyDir: {}
- name: api-server-patch
emptyDir: {}
- name: stream-recovery-patch
emptyDir: {}
- name: subprocess-secret-patch
emptyDir: {}
- name: auto-router-plugin
configMap:
name: hermes-auto-router-plugin
- name: image-plugin
configMap:
name: hermes-chat-image-plugin
- name: tmp
emptyDir:
sizeLimit: 256Mi
volumeClaimTemplates:
- metadata:
name: home
labels:
app: hermes-chat-tenant
spec:
accessModes: [ReadWriteOnce]
storageClassName: astreae
resources:
requests:
storage: 10Gi
- metadata:
name: workspace
labels:
app: hermes-chat-tenant
ai.bstein.dev/data: user-workspace
spec:
accessModes: [ReadWriteMany]
storageClassName: astreae
resources:
requests:
storage: 10Gi