353 lines
15 KiB
YAML
353 lines
15 KiB
YAML
# services/hermes/chat-statefulset.yaml
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: hermes-chat-tenant
|
|
namespace: hermes
|
|
labels:
|
|
app: hermes-chat-tenant
|
|
annotations:
|
|
kustomize.toolkit.fluxcd.io/force: enabled
|
|
spec:
|
|
serviceName: hermes-chat-tenant
|
|
replicas: 4
|
|
podManagementPolicy: Parallel
|
|
persistentVolumeClaimRetentionPolicy:
|
|
whenDeleted: Retain
|
|
whenScaled: Retain
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
selector:
|
|
matchLabels:
|
|
app: hermes-chat-tenant
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: hermes-chat-tenant
|
|
annotations:
|
|
ai.bstein.dev/role: isolated-user-chat
|
|
ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive
|
|
ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject
|
|
ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides
|
|
ai.bstein.dev/config-rev: "20260812-keycloak-image-continuation"
|
|
vault.hashicorp.com/agent-inject: "true"
|
|
vault.hashicorp.com/role: hermes-chat
|
|
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
|
vault.hashicorp.com/agent-inject-template-anthropic-token: |
|
|
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
|
{{ .Data.data.anthropic_oauth_token }}
|
|
{{- end }}
|
|
vault.hashicorp.com/agent-inject-secret-chat-relay-key: kv/data/atlas/hermes/chat-telegram
|
|
vault.hashicorp.com/agent-inject-template-chat-relay-key: |
|
|
{{- with secret "kv/data/atlas/hermes/chat-telegram" -}}
|
|
{{ .Data.data.relay_key }}
|
|
{{- end }}
|
|
vault.hashicorp.com/agent-pre-populate-only: "true"
|
|
vault.hashicorp.com/agent-init-first: "true"
|
|
vault.hashicorp.com/agent-requests-cpu: 25m
|
|
vault.hashicorp.com/agent-requests-mem: 32Mi
|
|
vault.hashicorp.com/agent-limits-cpu: 100m
|
|
vault.hashicorp.com/agent-limits-mem: 128Mi
|
|
spec:
|
|
serviceAccountName: hermes-chat
|
|
automountServiceAccountToken: true
|
|
securityContext:
|
|
fsGroup: 10000
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
affinity:
|
|
nodeAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
nodeSelectorTerms:
|
|
- matchExpressions:
|
|
- key: kubernetes.io/arch
|
|
operator: In
|
|
values: [arm64]
|
|
- key: node-role.kubernetes.io/worker
|
|
operator: In
|
|
values: ["true"]
|
|
- key: kubernetes.io/hostname
|
|
operator: NotIn
|
|
values: [titan-05, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19]
|
|
preferredDuringSchedulingIgnoredDuringExecution:
|
|
- weight: 100
|
|
preference:
|
|
matchExpressions:
|
|
- key: hardware
|
|
operator: In
|
|
values: [rpi5]
|
|
- weight: 40
|
|
preference:
|
|
matchExpressions:
|
|
- key: hardware
|
|
operator: In
|
|
values: [rpi4]
|
|
podAntiAffinity:
|
|
preferredDuringSchedulingIgnoredDuringExecution:
|
|
- weight: 100
|
|
podAffinityTerm:
|
|
labelSelector:
|
|
matchLabels:
|
|
app: hermes-chat-tenant
|
|
topologyKey: kubernetes.io/hostname
|
|
initContainers:
|
|
- name: init-config
|
|
image: busybox:1.37
|
|
imagePullPolicy: IfNotPresent
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
set -eu
|
|
mkdir -p /opt/data/home/.local/bin /opt/data/logs /opt/data/workspace
|
|
if [ ! -e /opt/data/workspace/.hermes-workspace-v1 ]; then
|
|
if [ -d /legacy-home/workspace ]; then
|
|
cp -a /legacy-home/workspace/. /opt/data/workspace/
|
|
fi
|
|
touch /opt/data/workspace/.hermes-workspace-v1
|
|
fi
|
|
cp /config/config.yaml /opt/data/config.yaml
|
|
cp /config/SOUL.md /opt/data/SOUL.md
|
|
cp /config/AGENTS.md /opt/data/workspace/AGENTS.md
|
|
touch /opt/data/.env
|
|
relay_key=""
|
|
if [ -s /vault/secrets/chat-relay-key ]; then
|
|
relay_key="$(tr -d '\r\n' < /vault/secrets/chat-relay-key)"
|
|
fi
|
|
if [ -z "${relay_key}" ]; then
|
|
api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')"
|
|
relay_key="${api_key}"
|
|
fi
|
|
{ grep -v '^API_SERVER_KEY=' /opt/data/.env || true; printf 'API_SERVER_KEY=%s\n' "${relay_key}"; } > /opt/data/.env.tmp
|
|
mv /opt/data/.env.tmp /opt/data/.env
|
|
{ grep -v '^HERMES_IMAGE_BROKER_KEY=' /opt/data/.env || true; printf 'HERMES_IMAGE_BROKER_KEY=%s\n' "${relay_key}"; } > /opt/data/.env.tmp
|
|
mv /opt/data/.env.tmp /opt/data/.env
|
|
if [ -s /vault/secrets/anthropic-token ]; then
|
|
token="$(tr -d '\r\n' < /vault/secrets/anthropic-token)"
|
|
if [ -n "${token}" ]; then
|
|
{ grep -v '^CLAUDE_CODE_OAUTH_TOKEN=' /opt/data/.env || true; printf 'CLAUDE_CODE_OAUTH_TOKEN=%s\n' "${token}"; } > /opt/data/.env.tmp
|
|
mv /opt/data/.env.tmp /opt/data/.env
|
|
fi
|
|
fi
|
|
rm -f /vault/secrets/anthropic-token /vault/secrets/chat-relay-key
|
|
chmod 0600 /opt/data/.env
|
|
chown 10000:10000 \
|
|
/opt/data \
|
|
/opt/data/home \
|
|
/opt/data/home/.local \
|
|
/opt/data/home/.local/bin \
|
|
/opt/data/logs \
|
|
/opt/data/workspace \
|
|
/opt/data/config.yaml \
|
|
/opt/data/SOUL.md \
|
|
/opt/data/workspace/AGENTS.md \
|
|
/opt/data/.env
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
runAsUser: 0
|
|
runAsGroup: 0
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
volumeMounts:
|
|
- {name: home, mountPath: /opt/data}
|
|
- {name: home, mountPath: /legacy-home, readOnly: true}
|
|
- {name: workspace, mountPath: /opt/data/workspace}
|
|
- {name: config, mountPath: /config, readOnly: true}
|
|
resources:
|
|
requests: {cpu: 25m, memory: 32Mi}
|
|
limits: {cpu: 100m, memory: 64Mi}
|
|
- name: patch-auth
|
|
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
imagePullPolicy: IfNotPresent
|
|
command:
|
|
- /opt/hermes/.venv/bin/python
|
|
- /opt/coordinator/patch_hermes_auth.py
|
|
- /opt/hermes/hermes_cli/auth.py
|
|
- /patched/auth.py
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
runAsUser: 10000
|
|
runAsGroup: 10000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
volumeMounts:
|
|
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
|
- {name: auth-patch, mountPath: /patched}
|
|
resources:
|
|
requests: {cpu: 25m, memory: 64Mi}
|
|
limits: {cpu: 100m, memory: 128Mi}
|
|
containers:
|
|
- name: hermes
|
|
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
imagePullPolicy: IfNotPresent
|
|
command: [/bin/sh, -ec]
|
|
args:
|
|
- |
|
|
ordinal="${HOSTNAME##*-}"
|
|
export HERMES_CODE_SANDBOX_URL="http://hermes-chat-sandbox-${ordinal}.hermes-chat-sandbox.hermes.svc.cluster.local:9080/v1/execute"
|
|
set -a
|
|
. /opt/data/.env
|
|
set +a
|
|
exec /opt/hermes/.venv/bin/hermes gateway run
|
|
ports:
|
|
- {name: api, containerPort: 8642, protocol: TCP}
|
|
env:
|
|
- {name: HERMES_HOME, value: /opt/data}
|
|
- {name: HERMES_AUTH_FILE, value: /shared-auth/auth.json}
|
|
- {name: HOME, value: /opt/data/home}
|
|
- {name: TERMINAL_CWD, value: /opt/data/workspace}
|
|
- {name: HERMES_WRITE_SAFE_ROOT, value: /opt/data/workspace}
|
|
- {name: PATH, value: /opt/data/home/.local/bin:/opt/hermes/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin}
|
|
- {name: PLAYWRIGHT_BROWSERS_PATH, value: /opt/hermes/.playwright}
|
|
- {name: AGENT_BROWSER_EXECUTABLE_PATH, value: /opt/hermes/.playwright/chromium_headless_shell-1228/chrome-linux/headless_shell}
|
|
- {name: AGENT_BROWSER_ARGS, value: "--no-sandbox,--disable-dev-shm-usage"}
|
|
- {name: HERMES_DASHBOARD, value: "0"}
|
|
- {name: API_SERVER_ENABLED, value: "true"}
|
|
- {name: API_SERVER_HOST, value: 0.0.0.0}
|
|
- {name: API_SERVER_PORT, value: "8642"}
|
|
- {name: API_SERVER_CORS_ORIGINS, value: https://chat.hermes.bstein.dev}
|
|
- {name: HERMES_IMAGE_BROKER_URL, value: http://hermes-image-broker.hermes.svc.cluster.local:9002}
|
|
- {name: HERMES_AUTO_ROUTER_PROFILE, value: chat}
|
|
volumeMounts:
|
|
- {name: home, mountPath: /opt/data}
|
|
- {name: workspace, mountPath: /opt/data/workspace}
|
|
# The shared provider pool uses auth.lock to serialize token refresh
|
|
# across tenant gateways. Tenant files and conversations remain on
|
|
# their own PVCs; only provider credentials are shared here.
|
|
- {name: provider-auth, mountPath: /shared-auth}
|
|
- {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
|
|
- {name: image-plugin, mountPath: /opt/hermes/plugins/image_gen/atlas-broker, readOnly: true}
|
|
- {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}
|
|
readinessProbe:
|
|
tcpSocket: {port: api}
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
livenessProbe:
|
|
tcpSocket: {port: api}
|
|
initialDelaySeconds: 90
|
|
periodSeconds: 30
|
|
timeoutSeconds: 10
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
runAsUser: 10000
|
|
runAsGroup: 10000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
resources:
|
|
requests: {cpu: 250m, memory: 512Mi}
|
|
limits: {cpu: "1", memory: 2Gi}
|
|
- name: webui
|
|
image: registry.bstein.dev/bstein/hermes-webui@sha256:fb06acc864509d9aa367d1d3635c82c383dc14458bc8db69a917e1ddf4f71f72
|
|
imagePullPolicy: IfNotPresent
|
|
command: [/bin/sh, -ec]
|
|
args:
|
|
- |
|
|
api_key="$(sed -n 's/^API_SERVER_KEY=//p' /opt/data/.env | tail -n 1)"
|
|
test -n "${api_key}"
|
|
export API_SERVER_KEY="${api_key}"
|
|
export HERMES_WEBUI_GATEWAY_API_KEY="${api_key}"
|
|
exec /opt/hermes/.venv/bin/python /opt/hermes-webui/server.py
|
|
ports:
|
|
- {name: webui, containerPort: 8787, protocol: TCP}
|
|
env:
|
|
- {name: HERMES_HOME, value: /opt/data}
|
|
- {name: HERMES_AUTH_FILE, value: /shared-auth/auth.json}
|
|
- {name: HOME, value: /opt/data/home}
|
|
- {name: HERMES_WEBUI_AGENT_DIR, value: /opt/hermes}
|
|
- {name: HERMES_WEBUI_HOST, value: 0.0.0.0}
|
|
- {name: HERMES_WEBUI_PORT, value: "8787"}
|
|
- {name: HERMES_WEBUI_STATE_DIR, value: /opt/data/webui}
|
|
- {name: HERMES_WEBUI_DEFAULT_WORKSPACE, value: /opt/data/workspace}
|
|
- {name: HERMES_WEBUI_CHAT_BACKEND, value: gateway}
|
|
- {name: HERMES_WEBUI_GATEWAY_BASE_URL, value: http://127.0.0.1:8642}
|
|
- {name: HERMES_WEBUI_GATEWAY_USE_RUNS_API, value: "true"}
|
|
- {name: HERMES_WEBUI_SKIP_ONBOARDING, value: "1"}
|
|
- {name: HERMES_WEBUI_SECURE, value: "1"}
|
|
- {name: HERMES_WEBUI_COOKIE_NAME, value: hermes_chat_session}
|
|
- {name: HERMES_WEBUI_PROFILE_COOKIE_NAME, value: hermes_chat_profile}
|
|
- {name: HERMES_WEBUI_TRUSTED_AUTH_HEADER, value: X-Hermes-Tenant-Identity}
|
|
# NetworkPolicy admits this port only from hermes-chat-router; the
|
|
# CIDR lets the WebUI validate that router's changing pod address.
|
|
- {name: HERMES_WEBUI_TRUSTED_PROXY_CIDRS, value: 10.42.0.0/16}
|
|
- {name: HERMES_WEBUI_ALLOWED_ORIGINS, value: https://chat.hermes.bstein.dev}
|
|
- {name: HERMES_WEBUI_TRUST_FORWARDED_HOST, value: "1"}
|
|
- {name: HERMES_WEBUI_TRUST_FORWARDED_PROTO, value: "1"}
|
|
- {name: HERMES_ROUTER_PROFILE, value: chat}
|
|
- {name: HERMES_STT_URL, value: http://hermes-stt.hermes.svc.cluster.local:9000/v1/audio/transcriptions}
|
|
- {name: HERMES_LOCAL_STT_COMMAND, value: "/opt/hermes/.venv/bin/python /opt/coordinator/hermes_stt_client.py {input_path} --output-dir {output_dir} --language {language} --model {model}"}
|
|
- {name: HERMES_WEBUI_ATLAS_TTS_URL, value: http://hermes-tts.hermes.svc.cluster.local:9001/v1/audio/speech}
|
|
volumeMounts:
|
|
- {name: home, mountPath: /opt/data}
|
|
- {name: workspace, mountPath: /opt/data/workspace}
|
|
- {name: provider-auth, mountPath: /shared-auth, readOnly: true}
|
|
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
|
- {name: tmp, mountPath: /tmp}
|
|
readinessProbe:
|
|
httpGet: {path: /health, port: webui}
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
livenessProbe:
|
|
httpGet: {path: /health, port: webui}
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 30
|
|
timeoutSeconds: 10
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: [ALL]
|
|
readOnlyRootFilesystem: true
|
|
runAsUser: 10000
|
|
runAsGroup: 10000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
resources:
|
|
requests: {cpu: 100m, memory: 256Mi}
|
|
limits: {cpu: 750m, memory: 1Gi}
|
|
volumes:
|
|
- name: provider-auth
|
|
persistentVolumeClaim:
|
|
claimName: hermes-provider-auth
|
|
- name: config
|
|
configMap:
|
|
name: hermes-chat-config
|
|
- name: coordinator
|
|
configMap:
|
|
name: hermes-coordinator
|
|
defaultMode: 0555
|
|
- name: auth-patch
|
|
emptyDir: {}
|
|
- name: auto-router-plugin
|
|
configMap:
|
|
name: hermes-auto-router-plugin
|
|
- name: image-plugin
|
|
configMap:
|
|
name: hermes-chat-image-plugin
|
|
- name: tmp
|
|
emptyDir:
|
|
sizeLimit: 256Mi
|
|
volumeClaimTemplates:
|
|
- metadata:
|
|
name: home
|
|
labels:
|
|
app: hermes-chat-tenant
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: astreae
|
|
resources:
|
|
requests:
|
|
storage: 10Gi
|
|
- metadata:
|
|
name: workspace
|
|
labels:
|
|
app: hermes-chat-tenant
|
|
ai.bstein.dev/data: user-workspace
|
|
spec:
|
|
accessModes: [ReadWriteMany]
|
|
storageClassName: astreae
|
|
resources:
|
|
requests:
|
|
storage: 10Gi
|