296 lines
13 KiB
YAML
296 lines
13 KiB
YAML
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: hermes-execution-worker
|
|
namespace: hermes
|
|
labels:
|
|
app: hermes-execution-worker
|
|
spec:
|
|
serviceName: hermes-execution-worker
|
|
replicas: 3
|
|
podManagementPolicy: Parallel
|
|
revisionHistoryLimit: 2
|
|
selector:
|
|
matchLabels:
|
|
app: hermes-execution-worker
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
rollingUpdate:
|
|
partition: 0
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: hermes-execution-worker
|
|
app.kubernetes.io/name: hermes-execution-worker
|
|
app.kubernetes.io/part-of: hermes
|
|
annotations:
|
|
ai.bstein.dev/role: fenced-execution-only
|
|
ai.bstein.dev/scm-boundary: mediated-pr14-broker-with-completion-gates
|
|
ai.bstein.dev/model-policy: Switchyard effort selection on the Claude subscription lane
|
|
ai.bstein.dev/storage: durable workspace and ordinal-private provider session state
|
|
ai.bstein.dev/config-rev: "20260822-claude-setup-token-v2"
|
|
vault.hashicorp.com/agent-inject: "true"
|
|
vault.hashicorp.com/role: hermes-execution-worker
|
|
vault.hashicorp.com/agent-inject-containers: stage-worker-access
|
|
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
|
|
vault.hashicorp.com/agent-inject-secret-claude-oauth-token: kv/data/atlas/hermes/agent-tokens
|
|
vault.hashicorp.com/agent-inject-perms-claude-oauth-token: "0600"
|
|
vault.hashicorp.com/agent-inject-template-claude-oauth-token: |
|
|
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
|
{{ .Data.data.claude_oauth_token }}
|
|
{{- end }}
|
|
vault.hashicorp.com/agent-pre-populate-only: "true"
|
|
vault.hashicorp.com/agent-init-first: "true"
|
|
vault.hashicorp.com/agent-requests-cpu: 2m
|
|
vault.hashicorp.com/agent-requests-mem: 16Mi
|
|
vault.hashicorp.com/agent-limits-cpu: 100m
|
|
vault.hashicorp.com/agent-limits-mem: 128Mi
|
|
spec:
|
|
serviceAccountName: hermes-execution-worker
|
|
priorityClassName: scavenger
|
|
automountServiceAccountToken: false
|
|
enableServiceLinks: false
|
|
terminationGracePeriodSeconds: 30
|
|
securityContext:
|
|
fsGroup: 10000
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
affinity:
|
|
nodeAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
nodeSelectorTerms:
|
|
- matchExpressions:
|
|
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
|
|
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
|
|
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
|
|
preferredDuringSchedulingIgnoredDuringExecution:
|
|
- weight: 100
|
|
preference:
|
|
matchExpressions:
|
|
- {key: node-role.kubernetes.io/accelerator, operator: Exists}
|
|
- weight: 50
|
|
preference:
|
|
matchExpressions:
|
|
- {key: hardware, operator: In, values: [rpi5]}
|
|
podAntiAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
- labelSelector:
|
|
matchLabels:
|
|
app: hermes-execution-worker
|
|
topologyKey: kubernetes.io/hostname
|
|
topologySpreadConstraints:
|
|
- maxSkew: 1
|
|
topologyKey: kubernetes.io/hostname
|
|
whenUnsatisfiable: DoNotSchedule
|
|
labelSelector:
|
|
matchLabels:
|
|
app: hermes-execution-worker
|
|
initContainers:
|
|
- name: stage-worker-access
|
|
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
imagePullPolicy: IfNotPresent
|
|
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-worker]
|
|
env:
|
|
- {name: HERMES_WORKER_ROOT, value: /workspace}
|
|
- {name: HERMES_PROVIDER_ACCESS_ROOT, value: /provider-access}
|
|
- name: HERMES_WORKER_ORDINAL
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
runAsUser: 0
|
|
runAsGroup: 0
|
|
seccompProfile: {type: RuntimeDefault}
|
|
volumeMounts:
|
|
- {name: claude-oauth-access, mountPath: /claude-oauth-access}
|
|
- {name: workspace, mountPath: /workspace}
|
|
- {name: provider-access, mountPath: /provider-access}
|
|
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
|
resources:
|
|
requests: {cpu: 2m, memory: 32Mi}
|
|
limits: {cpu: 100m, memory: 64Mi}
|
|
- name: install-worker-go
|
|
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
imagePullPolicy: IfNotPresent
|
|
command: [/bin/sh, -ec, "timeout 300 /bin/sh /opt/coordinator/install_worker_go.sh"]
|
|
env:
|
|
- {name: HERMES_WORKER_TOOLS_DIR, value: /worker-data/tools}
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities: {drop: [ALL]}
|
|
runAsNonRoot: true
|
|
runAsUser: 10000
|
|
runAsGroup: 10000
|
|
seccompProfile: {type: RuntimeDefault}
|
|
volumeMounts:
|
|
- {name: tools, mountPath: /worker-data/tools}
|
|
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
|
resources:
|
|
requests: {cpu: 5m, memory: 64Mi}
|
|
limits: {cpu: "1", memory: 256Mi}
|
|
- name: install-provider-clis
|
|
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
imagePullPolicy: IfNotPresent
|
|
command: [/bin/sh, -ec]
|
|
args:
|
|
# The tools volume is durable, so this install runs once per pinned
|
|
# version rather than on every Pod start. The version marker is only
|
|
# trusted when both binaries are actually present and executable, so
|
|
# a partial or pruned cache reinstalls instead of failing closed
|
|
# forever. The install itself is time-bounded so a stalled registry
|
|
# surfaces as a fast init failure with backoff instead of an
|
|
# open-ended wait that a Flux health timeout would have to absorb.
|
|
- |
|
|
tools=/worker-data/tools
|
|
marker="${tools}/.cli-versions-0.147.0-2.1.226"
|
|
mkdir -p "${tools}/bin"
|
|
if [ ! -f "${marker}" ] || [ ! -x "${tools}/bin/codex" ] \
|
|
|| [ ! -x "${tools}/bin/claude" ]; then
|
|
rm -f "${marker}"
|
|
timeout 900 npm install --global --omit=dev --no-audit --no-fund \
|
|
--fetch-timeout=120000 --fetch-retries=2 --prefix "${tools}" \
|
|
@openai/codex@0.147.0 @anthropic-ai/claude-code@2.1.226
|
|
touch "${marker}"
|
|
fi
|
|
test -x "${tools}/bin/codex"
|
|
test -x "${tools}/bin/claude"
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities: {drop: [ALL]}
|
|
runAsNonRoot: true
|
|
runAsUser: 10000
|
|
runAsGroup: 10000
|
|
seccompProfile: {type: RuntimeDefault}
|
|
volumeMounts:
|
|
- {name: tools, mountPath: /worker-data/tools}
|
|
resources:
|
|
requests: {cpu: 5m, memory: 64Mi}
|
|
limits: {cpu: "1", memory: 1Gi}
|
|
containers:
|
|
- name: execution-worker
|
|
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
imagePullPolicy: IfNotPresent
|
|
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_worker.py]
|
|
env:
|
|
- {name: HERMES_HOME, value: /worker-data}
|
|
- {name: HERMES_WORKER_ROOT, value: /workspace}
|
|
- {name: HOME, value: /worker-data/home}
|
|
- {name: CODEX_HOME, value: /provider-access/codex}
|
|
- {name: CLAUDE_CONFIG_DIR, value: /provider-access/claude}
|
|
- {name: CLAUDE_CODE_OAUTH_TOKEN_FILE, value: /claude-oauth-access/token}
|
|
- {name: HERMES_CLAUDE_BIN, value: /opt/coordinator/claude_oauth_exec}
|
|
- {name: HERMES_CLAUDE_NATIVE_BIN, value: /worker-data/tools/bin/claude}
|
|
# Codex OAuth is deliberately not copied from Hermes' shared refresh lineage.
|
|
- {name: HERMES_EXECUTION_DISABLED_PROVIDER, value: codex}
|
|
- {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
|
|
- {name: PYTHONPATH, value: /opt/hermes}
|
|
- {name: PATH, value: /worker-data/tools/bin:/opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
|
|
- name: HERMES_WORKER_ORDINAL
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
|
|
- name: HERMES_WORKER_NODE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
startupProbe:
|
|
exec:
|
|
command: [/bin/sh, -ec, "test -w /workspace && test -r /claude-oauth-access/token"]
|
|
periodSeconds: 5
|
|
failureThreshold: 60
|
|
readinessProbe:
|
|
exec:
|
|
command: [/bin/sh, -ec, "test -w /workspace && test -r /claude-oauth-access/token"]
|
|
periodSeconds: 10
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities: {drop: [ALL]}
|
|
readOnlyRootFilesystem: true
|
|
runAsNonRoot: true
|
|
runAsUser: 10000
|
|
runAsGroup: 10000
|
|
seccompProfile: {type: RuntimeDefault}
|
|
volumeMounts:
|
|
- {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true}
|
|
- {name: workspace, mountPath: /workspace}
|
|
- {name: worker-data, mountPath: /worker-data}
|
|
- {name: tools, mountPath: /worker-data/tools, readOnly: true}
|
|
- {name: provider-access, mountPath: /provider-access}
|
|
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
|
- {name: routing-catalog, mountPath: /routing-catalog, readOnly: true}
|
|
- {name: tmp, mountPath: /tmp}
|
|
resources:
|
|
requests: {cpu: 5m, memory: 128Mi, ephemeral-storage: 1Gi}
|
|
limits: {cpu: "2", memory: 4Gi, ephemeral-storage: 8Gi}
|
|
volumes:
|
|
- name: claude-oauth-access
|
|
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
|
- name: worker-data
|
|
emptyDir: {sizeLimit: 128Mi}
|
|
- name: coordinator
|
|
configMap:
|
|
name: hermes-execution-pool
|
|
defaultMode: 0555
|
|
- name: routing-catalog
|
|
persistentVolumeClaim:
|
|
claimName: hermes-routing-catalog
|
|
readOnly: true
|
|
- name: tmp
|
|
emptyDir: {sizeLimit: 2Gi}
|
|
- name: vault-auth-token
|
|
projected:
|
|
defaultMode: 0600
|
|
sources:
|
|
- serviceAccountToken:
|
|
audience: vault
|
|
expirationSeconds: 3600
|
|
path: token
|
|
- configMap:
|
|
name: kube-root-ca.crt
|
|
items:
|
|
- {key: ca.crt, path: ca.crt}
|
|
- downwardAPI:
|
|
items:
|
|
- {path: namespace, fieldRef: {fieldPath: metadata.namespace}}
|
|
volumeClaimTemplates:
|
|
# The workspace is shared with this ordinal's mediator, which performs every
|
|
# SCM operation on it. ReadWriteMany removes the cross-node Multi-Attach
|
|
# deadlock that a drain or preemption used to create between the two Pods,
|
|
# so each can be evicted and rescheduled independently. Longhorn already
|
|
# serves the hermes-chat tenant workspaces this way on the same class.
|
|
- metadata:
|
|
name: workspace
|
|
labels:
|
|
app: hermes-execution-worker
|
|
spec:
|
|
accessModes: [ReadWriteMany]
|
|
storageClassName: astreae
|
|
resources:
|
|
requests:
|
|
storage: 30Gi
|
|
# Provider CLIs are installed once per pinned version onto a durable volume
|
|
# instead of being re-downloaded into an emptyDir on every Pod start.
|
|
- metadata:
|
|
name: tools
|
|
labels:
|
|
app: hermes-execution-worker
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: astreae
|
|
resources:
|
|
requests:
|
|
storage: 2Gi
|
|
# Claude session state remains ordinal-private and survives Pod replacement.
|
|
- metadata:
|
|
name: provider-access
|
|
labels:
|
|
app: hermes-execution-worker
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: astreae
|
|
resources:
|
|
requests:
|
|
storage: 1Gi
|