355 lines
16 KiB
YAML
355 lines
16 KiB
YAML
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: hermes-execution-worker
|
|
namespace: hermes
|
|
labels:
|
|
app: hermes-execution-worker
|
|
spec:
|
|
serviceName: hermes-execution-worker
|
|
replicas: 3
|
|
podManagementPolicy: Parallel
|
|
revisionHistoryLimit: 2
|
|
selector:
|
|
matchLabels:
|
|
app: hermes-execution-worker
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: hermes-execution-worker
|
|
app.kubernetes.io/name: hermes-execution-worker
|
|
app.kubernetes.io/part-of: hermes
|
|
annotations:
|
|
ai.bstein.dev/role: fenced-execution-only
|
|
ai.bstein.dev/scm-boundary: ordinal-sidecar-with-assignment-bound-branch
|
|
ai.bstein.dev/model-policy: Switchyard AUTO with cross-provider fallback
|
|
ai.bstein.dev/storage: one durable RWO workspace and provider session home per ordinal
|
|
vault.hashicorp.com/agent-inject: "true"
|
|
vault.hashicorp.com/role: hermes-execution-worker
|
|
vault.hashicorp.com/agent-inject-containers: stage-worker-access
|
|
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
|
|
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
|
|
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
|
|
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
|
|
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
|
{{ printf "hermes-execution-pool-v1:%s" .Data.data.agent_api_key | sha256sum }}
|
|
{{- end }}
|
|
vault.hashicorp.com/agent-inject-secret-claude-credentials: kv/data/atlas/hermes/agent-tokens
|
|
vault.hashicorp.com/agent-inject-perms-claude-credentials: "0600"
|
|
vault.hashicorp.com/agent-inject-template-claude-credentials: |
|
|
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
|
{{ .Data.data.claude_credentials_json }}
|
|
{{- end }}
|
|
vault.hashicorp.com/agent-inject-secret-codex-auth: kv/data/atlas/hermes/agent-tokens
|
|
vault.hashicorp.com/agent-inject-perms-codex-auth: "0600"
|
|
vault.hashicorp.com/agent-inject-template-codex-auth: |
|
|
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
|
{{ .Data.data.codex_auth_json }}
|
|
{{- end }}
|
|
vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea
|
|
vault.hashicorp.com/agent-inject-perms-gitea-token: "0600"
|
|
vault.hashicorp.com/agent-inject-template-gitea-token: |
|
|
{{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
|
|
{{ .Data.data.token }}
|
|
{{- end }}
|
|
vault.hashicorp.com/agent-inject-secret-gitea-username: kv/data/atlas/hermes/developer-gitea
|
|
vault.hashicorp.com/agent-inject-perms-gitea-username: "0600"
|
|
vault.hashicorp.com/agent-inject-template-gitea-username: |
|
|
{{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
|
|
{{ .Data.data.username }}
|
|
{{- end }}
|
|
vault.hashicorp.com/agent-pre-populate-only: "true"
|
|
vault.hashicorp.com/agent-init-first: "true"
|
|
vault.hashicorp.com/agent-requests-cpu: 25m
|
|
vault.hashicorp.com/agent-requests-mem: 32Mi
|
|
vault.hashicorp.com/agent-limits-cpu: 100m
|
|
vault.hashicorp.com/agent-limits-mem: 128Mi
|
|
spec:
|
|
serviceAccountName: hermes-execution-worker
|
|
automountServiceAccountToken: false
|
|
enableServiceLinks: false
|
|
terminationGracePeriodSeconds: 30
|
|
securityContext:
|
|
fsGroup: 10000
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
affinity:
|
|
nodeAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
nodeSelectorTerms:
|
|
- matchExpressions:
|
|
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
|
|
- {key: node-role.kubernetes.io/accelerator, operator: Exists}
|
|
- {key: kubernetes.io/hostname, operator: In, values: [titan-20, titan-21]}
|
|
- matchExpressions:
|
|
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
|
|
- {key: hardware, operator: In, values: [rpi5]}
|
|
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
|
|
preferredDuringSchedulingIgnoredDuringExecution:
|
|
- weight: 100
|
|
preference:
|
|
matchExpressions:
|
|
- {key: node-role.kubernetes.io/accelerator, operator: Exists}
|
|
- weight: 50
|
|
preference:
|
|
matchExpressions:
|
|
- {key: hardware, operator: In, values: [rpi5]}
|
|
podAntiAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
- labelSelector:
|
|
matchLabels:
|
|
app: hermes-execution-worker
|
|
topologyKey: kubernetes.io/hostname
|
|
topologySpreadConstraints:
|
|
- maxSkew: 1
|
|
topologyKey: kubernetes.io/hostname
|
|
whenUnsatisfiable: DoNotSchedule
|
|
labelSelector:
|
|
matchLabels:
|
|
app: hermes-execution-worker
|
|
initContainers:
|
|
- name: stage-worker-access
|
|
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
imagePullPolicy: IfNotPresent
|
|
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-worker]
|
|
env:
|
|
- {name: HERMES_WORKER_ROOT, value: /workspace}
|
|
- {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
|
|
- {name: HERMES_SCM_ACCESS_ROOT, value: /scm-access}
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
runAsUser: 0
|
|
runAsGroup: 0
|
|
seccompProfile: {type: RuntimeDefault}
|
|
volumeMounts:
|
|
- {name: workspace, mountPath: /workspace}
|
|
- {name: runtime-access, mountPath: /runtime-access}
|
|
- {name: pool-access, mountPath: /pool-access}
|
|
- {name: scm-access, mountPath: /scm-access}
|
|
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
|
resources:
|
|
requests: {cpu: 25m, memory: 32Mi}
|
|
limits: {cpu: 100m, memory: 64Mi}
|
|
- name: install-provider-clis
|
|
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
imagePullPolicy: IfNotPresent
|
|
command: [/bin/sh, -ec]
|
|
args:
|
|
- |
|
|
tools=/worker-data/tools
|
|
mkdir -p "${tools}/bin"
|
|
if [ ! -f "${tools}/.cli-versions-0.147.0-2.1.226" ]; then
|
|
npm install --global --omit=dev --no-audit --no-fund --prefix "${tools}" \
|
|
@openai/codex@0.147.0 @anthropic-ai/claude-code@2.1.226
|
|
touch "${tools}/.cli-versions-0.147.0-2.1.226"
|
|
fi
|
|
test -x "${tools}/bin/codex"
|
|
test -x "${tools}/bin/claude"
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities: {drop: [ALL]}
|
|
runAsNonRoot: true
|
|
runAsUser: 10000
|
|
runAsGroup: 10000
|
|
seccompProfile: {type: RuntimeDefault}
|
|
volumeMounts:
|
|
- {name: tools, mountPath: /worker-data/tools}
|
|
resources:
|
|
requests: {cpu: 100m, memory: 128Mi}
|
|
limits: {cpu: "1", memory: 1Gi}
|
|
containers:
|
|
- name: execution-worker
|
|
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
imagePullPolicy: IfNotPresent
|
|
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_worker.py]
|
|
env:
|
|
- {name: HERMES_HOME, value: /worker-data}
|
|
- {name: HERMES_WORKER_ROOT, value: /workspace}
|
|
- {name: HOME, value: /worker-data/home}
|
|
- {name: CODEX_HOME, value: /runtime-access/codex}
|
|
- {name: CLAUDE_CONFIG_DIR, value: /runtime-access/claude}
|
|
- {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
|
|
- {name: PYTHONPATH, value: /opt/hermes}
|
|
- {name: PATH, value: /worker-data/tools/bin:/opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
|
|
- name: HERMES_WORKER_ORDINAL
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
|
|
- name: HERMES_WORKER_NODE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
startupProbe:
|
|
exec:
|
|
command: [/bin/sh, -ec, "test -w /workspace && test -w /runtime-access/codex/auth.json && test -w /runtime-access/claude/.credentials.json"]
|
|
periodSeconds: 5
|
|
failureThreshold: 60
|
|
readinessProbe:
|
|
exec:
|
|
command: [/bin/sh, -ec, "test -w /workspace && test -w /runtime-access/codex/auth.json"]
|
|
periodSeconds: 10
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities: {drop: [ALL]}
|
|
readOnlyRootFilesystem: true
|
|
runAsNonRoot: true
|
|
runAsUser: 10000
|
|
runAsGroup: 10000
|
|
seccompProfile: {type: RuntimeDefault}
|
|
volumeMounts:
|
|
- {name: workspace, mountPath: /workspace}
|
|
- {name: worker-data, mountPath: /worker-data}
|
|
- {name: tools, mountPath: /worker-data/tools, readOnly: true}
|
|
- {name: runtime-access, mountPath: /runtime-access}
|
|
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
|
- {name: tmp, mountPath: /tmp}
|
|
resources:
|
|
requests: {cpu: "1", memory: 2Gi, ephemeral-storage: 2Gi}
|
|
limits: {cpu: "5", memory: 6Gi, ephemeral-storage: 8Gi}
|
|
- name: execution-client
|
|
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
imagePullPolicy: IfNotPresent
|
|
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
|
|
env:
|
|
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
|
|
- name: HERMES_WORKER_ORDINAL
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
|
|
ports:
|
|
- {name: pool-client, containerPort: 9009, protocol: TCP}
|
|
startupProbe:
|
|
httpGet: {path: /ready, port: pool-client}
|
|
periodSeconds: 5
|
|
failureThreshold: 60
|
|
readinessProbe:
|
|
httpGet: {path: /ready, port: pool-client}
|
|
periodSeconds: 10
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities: {drop: [ALL]}
|
|
readOnlyRootFilesystem: true
|
|
runAsNonRoot: true
|
|
runAsUser: 10000
|
|
runAsGroup: 10000
|
|
seccompProfile: {type: RuntimeDefault}
|
|
volumeMounts:
|
|
- {name: pool-access, mountPath: /pool-access, readOnly: true}
|
|
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
|
- {name: tmp, mountPath: /tmp}
|
|
resources:
|
|
requests: {cpu: 25m, memory: 64Mi}
|
|
limits: {cpu: 250m, memory: 256Mi}
|
|
- name: scm-boundary
|
|
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
imagePullPolicy: IfNotPresent
|
|
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_scm.py]
|
|
env:
|
|
- {name: HERMES_WORKER_ROOT, value: /workspace}
|
|
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
|
|
- {name: HERMES_GITEA_TOKEN_FILE, value: /scm-access/gitea-token}
|
|
- {name: HERMES_GITEA_USERNAME_FILE, value: /scm-access/gitea-username}
|
|
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
|
|
- name: HERMES_WORKER_ORDINAL
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
|
|
ports:
|
|
- {name: scm, containerPort: 9008, protocol: TCP}
|
|
startupProbe:
|
|
httpGet: {path: /ready, port: scm}
|
|
periodSeconds: 5
|
|
failureThreshold: 60
|
|
readinessProbe:
|
|
httpGet: {path: /ready, port: scm}
|
|
periodSeconds: 10
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities: {drop: [ALL]}
|
|
readOnlyRootFilesystem: true
|
|
runAsNonRoot: true
|
|
runAsUser: 10000
|
|
runAsGroup: 10000
|
|
seccompProfile: {type: RuntimeDefault}
|
|
volumeMounts:
|
|
- {name: workspace, mountPath: /workspace}
|
|
- {name: pool-access, mountPath: /pool-access, readOnly: true}
|
|
- {name: scm-access, mountPath: /scm-access, readOnly: true}
|
|
- {name: scm-state, mountPath: /scm-state}
|
|
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
|
- {name: tmp, mountPath: /tmp}
|
|
resources:
|
|
requests: {cpu: 100m, memory: 128Mi}
|
|
limits: {cpu: "1", memory: 512Mi}
|
|
- name: credential-sync
|
|
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
imagePullPolicy: IfNotPresent
|
|
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/sync_runtime_credentials.py]
|
|
env:
|
|
- {name: HERMES_CREDENTIAL_SYNC_VAULT_ROLE, value: hermes-execution-credential-sync}
|
|
- {name: HERMES_CREDENTIAL_SYNC_INTERVAL, value: "300"}
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities: {drop: [ALL]}
|
|
readOnlyRootFilesystem: true
|
|
runAsNonRoot: true
|
|
runAsUser: 10000
|
|
runAsGroup: 10000
|
|
seccompProfile: {type: RuntimeDefault}
|
|
volumeMounts:
|
|
- {name: runtime-access, mountPath: /runtime-access}
|
|
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
|
- {name: vault-auth-token, mountPath: /var/run/secrets/kubernetes.io/serviceaccount, readOnly: true}
|
|
- {name: tmp, mountPath: /tmp}
|
|
resources:
|
|
requests: {cpu: 25m, memory: 64Mi}
|
|
limits: {cpu: 250m, memory: 256Mi}
|
|
volumes:
|
|
- name: worker-data
|
|
emptyDir: {sizeLimit: 128Mi}
|
|
- name: tools
|
|
emptyDir: {sizeLimit: 1Gi}
|
|
- name: runtime-access
|
|
emptyDir: {medium: Memory, sizeLimit: 128Mi}
|
|
- name: pool-access
|
|
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
|
- name: scm-access
|
|
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
|
- name: scm-state
|
|
emptyDir: {sizeLimit: 256Mi}
|
|
- name: coordinator
|
|
configMap:
|
|
name: hermes-execution-pool
|
|
defaultMode: 0555
|
|
- name: tmp
|
|
emptyDir: {sizeLimit: 2Gi}
|
|
- name: vault-auth-token
|
|
projected:
|
|
defaultMode: 0600
|
|
sources:
|
|
- serviceAccountToken:
|
|
audience: vault
|
|
expirationSeconds: 3600
|
|
path: token
|
|
- configMap:
|
|
name: kube-root-ca.crt
|
|
items:
|
|
- {key: ca.crt, path: ca.crt}
|
|
- downwardAPI:
|
|
items:
|
|
- {path: namespace, fieldRef: {fieldPath: metadata.namespace}}
|
|
volumeClaimTemplates:
|
|
- metadata:
|
|
name: workspace
|
|
labels:
|
|
app: hermes-execution-worker
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: astreae
|
|
resources:
|
|
requests:
|
|
storage: 30Gi
|