atlas-iac/services/hermes/execution-mediator.yaml

477 lines
21 KiB
YAML
Raw Normal View History

# Privileged HMAC and SCM mediation runs outside the model Pods. Each mediator is
# colocated with exactly one ordinal's RWO workspace but has a distinct network
# identity, private HMAC derivation, and durable integrity state.
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: hermes-execution-mediator-state-0, namespace: hermes}
spec:
accessModes: [ReadWriteOnce]
storageClassName: astreae
resources: {requests: {storage: 1Gi}}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: hermes-execution-mediator-state-1, namespace: hermes}
spec:
accessModes: [ReadWriteOnce]
storageClassName: astreae
resources: {requests: {storage: 1Gi}}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: hermes-execution-mediator-state-2, namespace: hermes}
spec:
accessModes: [ReadWriteOnce]
storageClassName: astreae
resources: {requests: {storage: 1Gi}}
---
apiVersion: v1
kind: Service
metadata: {name: hermes-execution-mediator-0, namespace: hermes}
spec:
selector: {app: hermes-execution-mediator, pool-ordinal: "0"}
ports: [{name: mediator, port: 9009, targetPort: mediator}]
---
apiVersion: v1
kind: Service
metadata: {name: hermes-execution-mediator-1, namespace: hermes}
spec:
selector: {app: hermes-execution-mediator, pool-ordinal: "1"}
ports: [{name: mediator, port: 9009, targetPort: mediator}]
---
apiVersion: v1
kind: Service
metadata: {name: hermes-execution-mediator-2, namespace: hermes}
spec:
selector: {app: hermes-execution-mediator, pool-ordinal: "2"}
ports: [{name: mediator, port: 9009, targetPort: mediator}]
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: hermes-execution-mediator-0
namespace: hermes
labels: {app: hermes-execution-mediator, pool-ordinal: "0"}
spec:
replicas: 1
strategy: {type: Recreate}
selector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
template:
metadata:
labels: {app: hermes-execution-mediator, pool-ordinal: "0"}
annotations:
ai.bstein.dev/config-rev: execution-pool-v2-mediated
ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-execution-worker
vault.hashicorp.com/agent-inject-containers: stage-mediator-access
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256Hex }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 2m
vault.hashicorp.com/agent-requests-mem: 16Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
spec:
serviceAccountName: hermes-execution-worker
automountServiceAccountToken: false
enableServiceLinks: false
hermes: make pool lease recovery and release isolation safe Independent review t_5975c06a blocked this branch on a P1: a Kanban write that failed while a lease expired left a `lease_failed` row that was invisible to every pass, immortal to garbage collection, and fatal to the coordinator. It poisoned `reconcile()` forever with a conflicting-duplicate primary key, produced a spurious capability `block_task` from `dispatch()`, and -- because startup maintenance ran unguarded before the port bound, against a store on a PVC -- crash-looped the coordinator with no automatic recovery. `lease_failed` is now a retryable state that every maintenance pass drains, and a row only reaches a terminal state on authoritative evidence about its exact Kanban run, so nothing is collected before its outcome is known and nothing is silently dropped. Each row, task, and board is processed in isolation, and a coordinator-side fault is never converted into a Kanban mutation. Startup runs through the same guarded cycle as the steady-state loop. The wire protocol and the durable store are now separate modules, and the maintenance passes moved out of the coordinator, so each file stays under the managed line ceiling with room for the recovery logic. Also closes three consequential handoff risks the same review raised: * mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce claim, so a drain or preemption that moved only the lower-priority worker deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces already are on the same class), colocation is a preference, and the mediator shares the worker's preemption priority, so each Pod reschedules on its own. * the broker permits only branch creation, so a retry that added commits could never submit and the run's work was discarded with the failure. Submission now targets a fresh attempt- or content-scoped ref in the same reviewed namespace -- never an update -- and is idempotent under replay. A refused submission downgrades the result and says why instead of unwinding the run. * the provider CLIs were reinstalled into an emptyDir on every Pod start inside the 10m Flux health window for the whole hermes app. They now install once per pinned version onto a durable volume, re-verified against the real binaries and time-bounded, and the best-effort pool no longer gates the health of the app its dependents wait on. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
# Priority parity with the worker: a pressured node evicts the pair
# symmetrically instead of moving only the lower-priority half, and the
# pool never preempts anything else in the cluster.
priorityClassName: scavenger
securityContext:
fsGroup: 10000
fsGroupChangePolicy: OnRootMismatch
seccompProfile: {type: RuntimeDefault}
affinity:
hermes: make pool lease recovery and release isolation safe Independent review t_5975c06a blocked this branch on a P1: a Kanban write that failed while a lease expired left a `lease_failed` row that was invisible to every pass, immortal to garbage collection, and fatal to the coordinator. It poisoned `reconcile()` forever with a conflicting-duplicate primary key, produced a spurious capability `block_task` from `dispatch()`, and -- because startup maintenance ran unguarded before the port bound, against a store on a PVC -- crash-looped the coordinator with no automatic recovery. `lease_failed` is now a retryable state that every maintenance pass drains, and a row only reaches a terminal state on authoritative evidence about its exact Kanban run, so nothing is collected before its outcome is known and nothing is silently dropped. Each row, task, and board is processed in isolation, and a coordinator-side fault is never converted into a Kanban mutation. Startup runs through the same guarded cycle as the steady-state loop. The wire protocol and the durable store are now separate modules, and the maintenance passes moved out of the coordinator, so each file stays under the managed line ceiling with room for the recovery logic. Also closes three consequential handoff risks the same review raised: * mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce claim, so a drain or preemption that moved only the lower-priority worker deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces already are on the same class), colocation is a preference, and the mediator shares the worker's preemption priority, so each Pod reschedules on its own. * the broker permits only branch creation, so a retry that added commits could never submit and the run's work was discarded with the failure. Submission now targets a fresh attempt- or content-scoped ref in the same reviewed namespace -- never an update -- and is idempotent under replay. A refused submission downgrades the result and says why instead of unwinding the run. * the provider CLIs were reinstalled into an emptyDir on every Pod start inside the 10m Flux health window for the whole hermes app. They now install once per pinned version onto a durable volume, re-verified against the real binaries and time-bounded, and the best-effort pool no longer gates the health of the app its dependents wait on. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
# Colocation with the worker is a throughput preference, not a
# requirement. The shared workspace claim is ReadWriteMany, so this
# mediator still schedules and serves when its worker is Pending or
# placed elsewhere -- which is what makes a drain self-healing.
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
hermes: make pool lease recovery and release isolation safe Independent review t_5975c06a blocked this branch on a P1: a Kanban write that failed while a lease expired left a `lease_failed` row that was invisible to every pass, immortal to garbage collection, and fatal to the coordinator. It poisoned `reconcile()` forever with a conflicting-duplicate primary key, produced a spurious capability `block_task` from `dispatch()`, and -- because startup maintenance ran unguarded before the port bound, against a store on a PVC -- crash-looped the coordinator with no automatic recovery. `lease_failed` is now a retryable state that every maintenance pass drains, and a row only reaches a terminal state on authoritative evidence about its exact Kanban run, so nothing is collected before its outcome is known and nothing is silently dropped. Each row, task, and board is processed in isolation, and a coordinator-side fault is never converted into a Kanban mutation. Startup runs through the same guarded cycle as the steady-state loop. The wire protocol and the durable store are now separate modules, and the maintenance passes moved out of the coordinator, so each file stays under the managed line ceiling with room for the recovery logic. Also closes three consequential handoff risks the same review raised: * mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce claim, so a drain or preemption that moved only the lower-priority worker deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces already are on the same class), colocation is a preference, and the mediator shares the worker's preemption priority, so each Pod reschedules on its own. * the broker permits only branch creation, so a retry that added commits could never submit and the run's work was discarded with the failure. Submission now targets a fresh attempt- or content-scoped ref in the same reviewed namespace -- never an update -- and is idempotent under replay. A refused submission downgrades the result and says why instead of unwinding the run. * the provider CLIs were reinstalled into an emptyDir on every Pod start inside the 10m Flux health window for the whole hermes app. They now install once per pinned version onto a durable volume, re-verified against the real binaries and time-bounded, and the best-effort pool no longer gates the health of the app its dependents wait on. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
nodeSelectorTerms:
- matchExpressions:
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
podAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
statefulset.kubernetes.io/pod-name: hermes-execution-worker-0
topologyKey: kubernetes.io/hostname
initContainers:
- name: stage-mediator-access
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
env:
- {name: HERMES_WORKER_ORDINAL, value: "0"}
- {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
runAsGroup: 0
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: pool-access, mountPath: /pool-access}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
resources:
requests: {cpu: 2m, memory: 16Mi}
limits: {cpu: 100m, memory: 64Mi}
containers:
- name: execution-mediator
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
env:
- {name: HERMES_WORKER_ORDINAL, value: "0"}
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
startupProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 5
failureThreshold: 60
readinessProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 10
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: workspace, mountPath: /workspace}
- {name: scm-state, mountPath: /scm-state}
- {name: pool-access, mountPath: /pool-access, readOnly: true}
- {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 2m, memory: 64Mi}
limits: {cpu: 250m, memory: 256Mi}
volumes:
- name: workspace
persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-0}
- name: scm-state
persistentVolumeClaim: {claimName: hermes-execution-mediator-state-0}
- name: pool-access
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: scm-broker-client
configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
- name: coordinator
configMap: {name: hermes-execution-pool, defaultMode: 0555}
- name: tmp
emptyDir: {sizeLimit: 64Mi}
- name: vault-auth-token
projected:
defaultMode: 0600
sources:
- serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
- configMap:
name: kube-root-ca.crt
items: [{key: ca.crt, path: ca.crt}]
- downwardAPI:
items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: hermes-execution-mediator-1
namespace: hermes
labels: {app: hermes-execution-mediator, pool-ordinal: "1"}
spec:
replicas: 1
strategy: {type: Recreate}
selector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
template:
metadata:
labels: {app: hermes-execution-mediator, pool-ordinal: "1"}
annotations:
ai.bstein.dev/config-rev: execution-pool-v2-mediated
ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-execution-worker
vault.hashicorp.com/agent-inject-containers: stage-mediator-access
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256Hex }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 2m
vault.hashicorp.com/agent-requests-mem: 16Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
spec:
serviceAccountName: hermes-execution-worker
automountServiceAccountToken: false
enableServiceLinks: false
hermes: make pool lease recovery and release isolation safe Independent review t_5975c06a blocked this branch on a P1: a Kanban write that failed while a lease expired left a `lease_failed` row that was invisible to every pass, immortal to garbage collection, and fatal to the coordinator. It poisoned `reconcile()` forever with a conflicting-duplicate primary key, produced a spurious capability `block_task` from `dispatch()`, and -- because startup maintenance ran unguarded before the port bound, against a store on a PVC -- crash-looped the coordinator with no automatic recovery. `lease_failed` is now a retryable state that every maintenance pass drains, and a row only reaches a terminal state on authoritative evidence about its exact Kanban run, so nothing is collected before its outcome is known and nothing is silently dropped. Each row, task, and board is processed in isolation, and a coordinator-side fault is never converted into a Kanban mutation. Startup runs through the same guarded cycle as the steady-state loop. The wire protocol and the durable store are now separate modules, and the maintenance passes moved out of the coordinator, so each file stays under the managed line ceiling with room for the recovery logic. Also closes three consequential handoff risks the same review raised: * mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce claim, so a drain or preemption that moved only the lower-priority worker deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces already are on the same class), colocation is a preference, and the mediator shares the worker's preemption priority, so each Pod reschedules on its own. * the broker permits only branch creation, so a retry that added commits could never submit and the run's work was discarded with the failure. Submission now targets a fresh attempt- or content-scoped ref in the same reviewed namespace -- never an update -- and is idempotent under replay. A refused submission downgrades the result and says why instead of unwinding the run. * the provider CLIs were reinstalled into an emptyDir on every Pod start inside the 10m Flux health window for the whole hermes app. They now install once per pinned version onto a durable volume, re-verified against the real binaries and time-bounded, and the best-effort pool no longer gates the health of the app its dependents wait on. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
# Priority parity with the worker: a pressured node evicts the pair
# symmetrically instead of moving only the lower-priority half, and the
# pool never preempts anything else in the cluster.
priorityClassName: scavenger
securityContext:
fsGroup: 10000
fsGroupChangePolicy: OnRootMismatch
seccompProfile: {type: RuntimeDefault}
affinity:
hermes: make pool lease recovery and release isolation safe Independent review t_5975c06a blocked this branch on a P1: a Kanban write that failed while a lease expired left a `lease_failed` row that was invisible to every pass, immortal to garbage collection, and fatal to the coordinator. It poisoned `reconcile()` forever with a conflicting-duplicate primary key, produced a spurious capability `block_task` from `dispatch()`, and -- because startup maintenance ran unguarded before the port bound, against a store on a PVC -- crash-looped the coordinator with no automatic recovery. `lease_failed` is now a retryable state that every maintenance pass drains, and a row only reaches a terminal state on authoritative evidence about its exact Kanban run, so nothing is collected before its outcome is known and nothing is silently dropped. Each row, task, and board is processed in isolation, and a coordinator-side fault is never converted into a Kanban mutation. Startup runs through the same guarded cycle as the steady-state loop. The wire protocol and the durable store are now separate modules, and the maintenance passes moved out of the coordinator, so each file stays under the managed line ceiling with room for the recovery logic. Also closes three consequential handoff risks the same review raised: * mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce claim, so a drain or preemption that moved only the lower-priority worker deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces already are on the same class), colocation is a preference, and the mediator shares the worker's preemption priority, so each Pod reschedules on its own. * the broker permits only branch creation, so a retry that added commits could never submit and the run's work was discarded with the failure. Submission now targets a fresh attempt- or content-scoped ref in the same reviewed namespace -- never an update -- and is idempotent under replay. A refused submission downgrades the result and says why instead of unwinding the run. * the provider CLIs were reinstalled into an emptyDir on every Pod start inside the 10m Flux health window for the whole hermes app. They now install once per pinned version onto a durable volume, re-verified against the real binaries and time-bounded, and the best-effort pool no longer gates the health of the app its dependents wait on. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
# Colocation with the worker is a throughput preference, not a
# requirement. The shared workspace claim is ReadWriteMany, so this
# mediator still schedules and serves when its worker is Pending or
# placed elsewhere -- which is what makes a drain self-healing.
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
hermes: make pool lease recovery and release isolation safe Independent review t_5975c06a blocked this branch on a P1: a Kanban write that failed while a lease expired left a `lease_failed` row that was invisible to every pass, immortal to garbage collection, and fatal to the coordinator. It poisoned `reconcile()` forever with a conflicting-duplicate primary key, produced a spurious capability `block_task` from `dispatch()`, and -- because startup maintenance ran unguarded before the port bound, against a store on a PVC -- crash-looped the coordinator with no automatic recovery. `lease_failed` is now a retryable state that every maintenance pass drains, and a row only reaches a terminal state on authoritative evidence about its exact Kanban run, so nothing is collected before its outcome is known and nothing is silently dropped. Each row, task, and board is processed in isolation, and a coordinator-side fault is never converted into a Kanban mutation. Startup runs through the same guarded cycle as the steady-state loop. The wire protocol and the durable store are now separate modules, and the maintenance passes moved out of the coordinator, so each file stays under the managed line ceiling with room for the recovery logic. Also closes three consequential handoff risks the same review raised: * mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce claim, so a drain or preemption that moved only the lower-priority worker deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces already are on the same class), colocation is a preference, and the mediator shares the worker's preemption priority, so each Pod reschedules on its own. * the broker permits only branch creation, so a retry that added commits could never submit and the run's work was discarded with the failure. Submission now targets a fresh attempt- or content-scoped ref in the same reviewed namespace -- never an update -- and is idempotent under replay. A refused submission downgrades the result and says why instead of unwinding the run. * the provider CLIs were reinstalled into an emptyDir on every Pod start inside the 10m Flux health window for the whole hermes app. They now install once per pinned version onto a durable volume, re-verified against the real binaries and time-bounded, and the best-effort pool no longer gates the health of the app its dependents wait on. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
nodeSelectorTerms:
- matchExpressions:
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
podAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
statefulset.kubernetes.io/pod-name: hermes-execution-worker-1
topologyKey: kubernetes.io/hostname
initContainers:
- name: stage-mediator-access
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
env:
- {name: HERMES_WORKER_ORDINAL, value: "1"}
- {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
runAsGroup: 0
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: pool-access, mountPath: /pool-access}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
resources:
requests: {cpu: 2m, memory: 16Mi}
limits: {cpu: 100m, memory: 64Mi}
containers:
- name: execution-mediator
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
env:
- {name: HERMES_WORKER_ORDINAL, value: "1"}
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
startupProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 5
failureThreshold: 60
readinessProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 10
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: workspace, mountPath: /workspace}
- {name: scm-state, mountPath: /scm-state}
- {name: pool-access, mountPath: /pool-access, readOnly: true}
- {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 2m, memory: 64Mi}
limits: {cpu: 250m, memory: 256Mi}
volumes:
- name: workspace
persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-1}
- name: scm-state
persistentVolumeClaim: {claimName: hermes-execution-mediator-state-1}
- name: pool-access
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: scm-broker-client
configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
- name: coordinator
configMap: {name: hermes-execution-pool, defaultMode: 0555}
- name: tmp
emptyDir: {sizeLimit: 64Mi}
- name: vault-auth-token
projected:
defaultMode: 0600
sources:
- serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
- configMap:
name: kube-root-ca.crt
items: [{key: ca.crt, path: ca.crt}]
- downwardAPI:
items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: hermes-execution-mediator-2
namespace: hermes
labels: {app: hermes-execution-mediator, pool-ordinal: "2"}
spec:
replicas: 1
strategy: {type: Recreate}
selector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
template:
metadata:
labels: {app: hermes-execution-mediator, pool-ordinal: "2"}
annotations:
ai.bstein.dev/config-rev: execution-pool-v2-mediated
ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-execution-worker
vault.hashicorp.com/agent-inject-containers: stage-mediator-access
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256Hex }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 2m
vault.hashicorp.com/agent-requests-mem: 16Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
spec:
serviceAccountName: hermes-execution-worker
automountServiceAccountToken: false
enableServiceLinks: false
hermes: make pool lease recovery and release isolation safe Independent review t_5975c06a blocked this branch on a P1: a Kanban write that failed while a lease expired left a `lease_failed` row that was invisible to every pass, immortal to garbage collection, and fatal to the coordinator. It poisoned `reconcile()` forever with a conflicting-duplicate primary key, produced a spurious capability `block_task` from `dispatch()`, and -- because startup maintenance ran unguarded before the port bound, against a store on a PVC -- crash-looped the coordinator with no automatic recovery. `lease_failed` is now a retryable state that every maintenance pass drains, and a row only reaches a terminal state on authoritative evidence about its exact Kanban run, so nothing is collected before its outcome is known and nothing is silently dropped. Each row, task, and board is processed in isolation, and a coordinator-side fault is never converted into a Kanban mutation. Startup runs through the same guarded cycle as the steady-state loop. The wire protocol and the durable store are now separate modules, and the maintenance passes moved out of the coordinator, so each file stays under the managed line ceiling with room for the recovery logic. Also closes three consequential handoff risks the same review raised: * mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce claim, so a drain or preemption that moved only the lower-priority worker deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces already are on the same class), colocation is a preference, and the mediator shares the worker's preemption priority, so each Pod reschedules on its own. * the broker permits only branch creation, so a retry that added commits could never submit and the run's work was discarded with the failure. Submission now targets a fresh attempt- or content-scoped ref in the same reviewed namespace -- never an update -- and is idempotent under replay. A refused submission downgrades the result and says why instead of unwinding the run. * the provider CLIs were reinstalled into an emptyDir on every Pod start inside the 10m Flux health window for the whole hermes app. They now install once per pinned version onto a durable volume, re-verified against the real binaries and time-bounded, and the best-effort pool no longer gates the health of the app its dependents wait on. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
# Priority parity with the worker: a pressured node evicts the pair
# symmetrically instead of moving only the lower-priority half, and the
# pool never preempts anything else in the cluster.
priorityClassName: scavenger
securityContext:
fsGroup: 10000
fsGroupChangePolicy: OnRootMismatch
seccompProfile: {type: RuntimeDefault}
affinity:
hermes: make pool lease recovery and release isolation safe Independent review t_5975c06a blocked this branch on a P1: a Kanban write that failed while a lease expired left a `lease_failed` row that was invisible to every pass, immortal to garbage collection, and fatal to the coordinator. It poisoned `reconcile()` forever with a conflicting-duplicate primary key, produced a spurious capability `block_task` from `dispatch()`, and -- because startup maintenance ran unguarded before the port bound, against a store on a PVC -- crash-looped the coordinator with no automatic recovery. `lease_failed` is now a retryable state that every maintenance pass drains, and a row only reaches a terminal state on authoritative evidence about its exact Kanban run, so nothing is collected before its outcome is known and nothing is silently dropped. Each row, task, and board is processed in isolation, and a coordinator-side fault is never converted into a Kanban mutation. Startup runs through the same guarded cycle as the steady-state loop. The wire protocol and the durable store are now separate modules, and the maintenance passes moved out of the coordinator, so each file stays under the managed line ceiling with room for the recovery logic. Also closes three consequential handoff risks the same review raised: * mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce claim, so a drain or preemption that moved only the lower-priority worker deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces already are on the same class), colocation is a preference, and the mediator shares the worker's preemption priority, so each Pod reschedules on its own. * the broker permits only branch creation, so a retry that added commits could never submit and the run's work was discarded with the failure. Submission now targets a fresh attempt- or content-scoped ref in the same reviewed namespace -- never an update -- and is idempotent under replay. A refused submission downgrades the result and says why instead of unwinding the run. * the provider CLIs were reinstalled into an emptyDir on every Pod start inside the 10m Flux health window for the whole hermes app. They now install once per pinned version onto a durable volume, re-verified against the real binaries and time-bounded, and the best-effort pool no longer gates the health of the app its dependents wait on. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
# Colocation with the worker is a throughput preference, not a
# requirement. The shared workspace claim is ReadWriteMany, so this
# mediator still schedules and serves when its worker is Pending or
# placed elsewhere -- which is what makes a drain self-healing.
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
hermes: make pool lease recovery and release isolation safe Independent review t_5975c06a blocked this branch on a P1: a Kanban write that failed while a lease expired left a `lease_failed` row that was invisible to every pass, immortal to garbage collection, and fatal to the coordinator. It poisoned `reconcile()` forever with a conflicting-duplicate primary key, produced a spurious capability `block_task` from `dispatch()`, and -- because startup maintenance ran unguarded before the port bound, against a store on a PVC -- crash-looped the coordinator with no automatic recovery. `lease_failed` is now a retryable state that every maintenance pass drains, and a row only reaches a terminal state on authoritative evidence about its exact Kanban run, so nothing is collected before its outcome is known and nothing is silently dropped. Each row, task, and board is processed in isolation, and a coordinator-side fault is never converted into a Kanban mutation. Startup runs through the same guarded cycle as the steady-state loop. The wire protocol and the durable store are now separate modules, and the maintenance passes moved out of the coordinator, so each file stays under the managed line ceiling with room for the recovery logic. Also closes three consequential handoff risks the same review raised: * mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce claim, so a drain or preemption that moved only the lower-priority worker deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces already are on the same class), colocation is a preference, and the mediator shares the worker's preemption priority, so each Pod reschedules on its own. * the broker permits only branch creation, so a retry that added commits could never submit and the run's work was discarded with the failure. Submission now targets a fresh attempt- or content-scoped ref in the same reviewed namespace -- never an update -- and is idempotent under replay. A refused submission downgrades the result and says why instead of unwinding the run. * the provider CLIs were reinstalled into an emptyDir on every Pod start inside the 10m Flux health window for the whole hermes app. They now install once per pinned version onto a durable volume, re-verified against the real binaries and time-bounded, and the best-effort pool no longer gates the health of the app its dependents wait on. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
nodeSelectorTerms:
- matchExpressions:
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
podAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
statefulset.kubernetes.io/pod-name: hermes-execution-worker-2
topologyKey: kubernetes.io/hostname
initContainers:
- name: stage-mediator-access
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
env:
- {name: HERMES_WORKER_ORDINAL, value: "2"}
- {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
runAsGroup: 0
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: pool-access, mountPath: /pool-access}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
resources:
requests: {cpu: 2m, memory: 16Mi}
limits: {cpu: 100m, memory: 64Mi}
containers:
- name: execution-mediator
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
env:
- {name: HERMES_WORKER_ORDINAL, value: "2"}
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
startupProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 5
failureThreshold: 60
readinessProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 10
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: workspace, mountPath: /workspace}
- {name: scm-state, mountPath: /scm-state}
- {name: pool-access, mountPath: /pool-access, readOnly: true}
- {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 2m, memory: 64Mi}
limits: {cpu: 250m, memory: 256Mi}
volumes:
- name: workspace
persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-2}
- name: scm-state
persistentVolumeClaim: {claimName: hermes-execution-mediator-state-2}
- name: pool-access
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: scm-broker-client
configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
- name: coordinator
configMap: {name: hermes-execution-pool, defaultMode: 0555}
- name: tmp
emptyDir: {sizeLimit: 64Mi}
- name: vault-auth-token
projected:
defaultMode: 0600
sources:
- serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
- configMap:
name: kube-root-ca.crt
items: [{key: ca.crt, path: ca.crt}]
- downwardAPI:
items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]