atlas-iac/services/hermes/execution-worker-statefulset.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: hermes-execution-worker
  namespace: hermes
  labels:
    app: hermes-execution-worker
spec:
  serviceName: hermes-execution-worker
  replicas: 3
  podManagementPolicy: Parallel
  revisionHistoryLimit: 2
  selector:
    matchLabels:
      app: hermes-execution-worker
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      partition: 0
  template:
    metadata:
      labels:
        app: hermes-execution-worker
        app.kubernetes.io/name: hermes-execution-worker
        app.kubernetes.io/part-of: hermes
      annotations:
        ai.bstein.dev/role: fenced-execution-only
        ai.bstein.dev/scm-boundary: mediated-pr14-broker-with-completion-gates
        ai.bstein.dev/model-policy: Switchyard AUTO with cross-provider fallback
        ai.bstein.dev/storage: separate durable RWO workspace and OAuth refresh ownership per ordinal
        vault.hashicorp.com/agent-inject: "true"
        vault.hashicorp.com/role: hermes-execution-worker
        vault.hashicorp.com/agent-inject-containers: stage-worker-access
        vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
        vault.hashicorp.com/agent-inject-secret-claude-credentials-0: kv/data/atlas/hermes/agent-tokens
        vault.hashicorp.com/agent-inject-perms-claude-credentials-0: "0600"
        vault.hashicorp.com/agent-inject-template-claude-credentials-0: |
          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
          {{ .Data.data.execution_worker_0_claude_credentials_json }}
          {{- end }}
        vault.hashicorp.com/agent-inject-secret-claude-credentials-1: kv/data/atlas/hermes/agent-tokens
        vault.hashicorp.com/agent-inject-perms-claude-credentials-1: "0600"
        vault.hashicorp.com/agent-inject-template-claude-credentials-1: |
          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
          {{ .Data.data.execution_worker_1_claude_credentials_json }}
          {{- end }}
        vault.hashicorp.com/agent-inject-secret-claude-credentials-2: kv/data/atlas/hermes/agent-tokens
        vault.hashicorp.com/agent-inject-perms-claude-credentials-2: "0600"
        vault.hashicorp.com/agent-inject-template-claude-credentials-2: |
          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
          {{ .Data.data.execution_worker_2_claude_credentials_json }}
          {{- end }}
        vault.hashicorp.com/agent-inject-secret-codex-auth-0: kv/data/atlas/hermes/agent-tokens
        vault.hashicorp.com/agent-inject-perms-codex-auth-0: "0600"
        vault.hashicorp.com/agent-inject-template-codex-auth-0: |
          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
          {{ .Data.data.execution_worker_0_codex_auth_json }}
          {{- end }}
        vault.hashicorp.com/agent-inject-secret-codex-auth-1: kv/data/atlas/hermes/agent-tokens
        vault.hashicorp.com/agent-inject-perms-codex-auth-1: "0600"
        vault.hashicorp.com/agent-inject-template-codex-auth-1: |
          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
          {{ .Data.data.execution_worker_1_codex_auth_json }}
          {{- end }}
        vault.hashicorp.com/agent-inject-secret-codex-auth-2: kv/data/atlas/hermes/agent-tokens
        vault.hashicorp.com/agent-inject-perms-codex-auth-2: "0600"
        vault.hashicorp.com/agent-inject-template-codex-auth-2: |
          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
          {{ .Data.data.execution_worker_2_codex_auth_json }}
          {{- end }}
        vault.hashicorp.com/agent-pre-populate-only: "true"
        vault.hashicorp.com/agent-init-first: "true"
        vault.hashicorp.com/agent-requests-cpu: 2m
        vault.hashicorp.com/agent-requests-mem: 16Mi
        vault.hashicorp.com/agent-limits-cpu: 100m
        vault.hashicorp.com/agent-limits-mem: 128Mi
    spec:
      serviceAccountName: hermes-execution-worker
      priorityClassName: scavenger
      automountServiceAccountToken: false
      enableServiceLinks: false
      terminationGracePeriodSeconds: 30
      securityContext:
        fsGroup: 10000
        fsGroupChangePolicy: OnRootMismatch
        seccompProfile:
          type: RuntimeDefault
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - {key: kubernetes.io/arch, operator: In, values: [arm64]}
                  - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
                  - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              preference:
                matchExpressions:
                  - {key: node-role.kubernetes.io/accelerator, operator: Exists}
            - weight: 50
              preference:
                matchExpressions:
                  - {key: hardware, operator: In, values: [rpi5]}
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchLabels:
                  app: hermes-execution-worker
              topologyKey: kubernetes.io/hostname
      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: kubernetes.io/hostname
          whenUnsatisfiable: DoNotSchedule
          labelSelector:
            matchLabels:
              app: hermes-execution-worker
      initContainers:
        - name: stage-worker-access
          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
          imagePullPolicy: IfNotPresent
          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-worker]
          env:
            - {name: HERMES_WORKER_ROOT, value: /workspace}
            - {name: HERMES_PROVIDER_ACCESS_ROOT, value: /provider-access}
            - name: HERMES_WORKER_ORDINAL
              valueFrom:
                fieldRef:
                  fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
          securityContext:
            allowPrivilegeEscalation: false
            runAsUser: 0
            runAsGroup: 0
            seccompProfile: {type: RuntimeDefault}
          volumeMounts:
            - {name: workspace, mountPath: /workspace}
            - {name: provider-access, mountPath: /provider-access}
            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
          resources:
            requests: {cpu: 2m, memory: 32Mi}
            limits: {cpu: 100m, memory: 64Mi}
        - name: install-provider-clis
          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
          imagePullPolicy: IfNotPresent
          command: [/bin/sh, -ec]
          args:
            - |
              tools=/worker-data/tools
              mkdir -p "${tools}/bin"
              if [ ! -f "${tools}/.cli-versions-0.147.0-2.1.226" ]; then
                npm install --global --omit=dev --no-audit --no-fund --prefix "${tools}" \
                  @openai/codex@0.147.0 @anthropic-ai/claude-code@2.1.226
                touch "${tools}/.cli-versions-0.147.0-2.1.226"
              fi
              test -x "${tools}/bin/codex"
              test -x "${tools}/bin/claude"
          securityContext:
            allowPrivilegeEscalation: false
            capabilities: {drop: [ALL]}
            runAsNonRoot: true
            runAsUser: 10000
            runAsGroup: 10000
            seccompProfile: {type: RuntimeDefault}
          volumeMounts:
            - {name: tools, mountPath: /worker-data/tools}
          resources:
            requests: {cpu: 5m, memory: 64Mi}
            limits: {cpu: "1", memory: 1Gi}
      containers:
        - name: execution-worker
          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
          imagePullPolicy: IfNotPresent
          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_worker.py]
          env:
            - {name: HERMES_HOME, value: /worker-data}
            - {name: HERMES_WORKER_ROOT, value: /workspace}
            - {name: HOME, value: /worker-data/home}
            - {name: CODEX_HOME, value: /provider-access/codex}
            - {name: CLAUDE_CONFIG_DIR, value: /provider-access/claude}
            - {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
            - {name: PYTHONPATH, value: /opt/hermes}
            - {name: PATH, value: /worker-data/tools/bin:/opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
            - name: HERMES_WORKER_ORDINAL
              valueFrom:
                fieldRef:
                  fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
            - name: HERMES_WORKER_NODE
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          startupProbe:
            exec:
              command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json && test -w /provider-access/claude/.credentials.json"]
            periodSeconds: 5
            failureThreshold: 60
          readinessProbe:
            exec:
              command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json"]
            periodSeconds: 10
          securityContext:
            allowPrivilegeEscalation: false
            capabilities: {drop: [ALL]}
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            runAsUser: 10000
            runAsGroup: 10000
            seccompProfile: {type: RuntimeDefault}
          volumeMounts:
            - {name: workspace, mountPath: /workspace}
            - {name: worker-data, mountPath: /worker-data}
            - {name: tools, mountPath: /worker-data/tools, readOnly: true}
            - {name: provider-access, mountPath: /provider-access}
            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
            - {name: tmp, mountPath: /tmp}
          resources:
            requests: {cpu: 5m, memory: 128Mi, ephemeral-storage: 1Gi}
            limits: {cpu: "2", memory: 4Gi, ephemeral-storage: 8Gi}
      volumes:
        - name: worker-data
          emptyDir: {sizeLimit: 128Mi}
        - name: tools
          emptyDir: {sizeLimit: 1Gi}
        - name: coordinator
          configMap:
            name: hermes-execution-pool
            defaultMode: 0555
        - name: tmp
          emptyDir: {sizeLimit: 2Gi}
        - name: vault-auth-token
          projected:
            defaultMode: 0600
            sources:
              - serviceAccountToken:
                  audience: vault
                  expirationSeconds: 3600
                  path: token
              - configMap:
                  name: kube-root-ca.crt
                  items:
                    - {key: ca.crt, path: ca.crt}
              - downwardAPI:
                  items:
                    - {path: namespace, fieldRef: {fieldPath: metadata.namespace}}
  volumeClaimTemplates:
    - metadata:
        name: workspace
        labels:
          app: hermes-execution-worker
      spec:
        accessModes: [ReadWriteOnce]
        storageClassName: astreae
        resources:
          requests:
            storage: 30Gi
    - metadata:
        name: provider-access
        labels:
          app: hermes-execution-worker
      spec:
        accessModes: [ReadWriteOnce]
        storageClassName: astreae
        resources:
          requests:
            storage: 1Gi