2026-08-17 16:31:15 +00:00
|
|
|
apiVersion: apps/v1
|
|
|
|
|
kind: StatefulSet
|
|
|
|
|
metadata:
|
|
|
|
|
name: hermes-execution-worker
|
|
|
|
|
namespace: hermes
|
|
|
|
|
labels:
|
|
|
|
|
app: hermes-execution-worker
|
|
|
|
|
spec:
|
|
|
|
|
serviceName: hermes-execution-worker
|
|
|
|
|
replicas: 3
|
|
|
|
|
podManagementPolicy: Parallel
|
|
|
|
|
revisionHistoryLimit: 2
|
|
|
|
|
selector:
|
|
|
|
|
matchLabels:
|
|
|
|
|
app: hermes-execution-worker
|
|
|
|
|
updateStrategy:
|
|
|
|
|
type: RollingUpdate
|
|
|
|
|
rollingUpdate:
|
|
|
|
|
partition: 0
|
|
|
|
|
template:
|
|
|
|
|
metadata:
|
|
|
|
|
labels:
|
|
|
|
|
app: hermes-execution-worker
|
|
|
|
|
app.kubernetes.io/name: hermes-execution-worker
|
|
|
|
|
app.kubernetes.io/part-of: hermes
|
|
|
|
|
annotations:
|
|
|
|
|
ai.bstein.dev/role: fenced-execution-only
|
|
|
|
|
ai.bstein.dev/scm-boundary: mediated-pr14-broker-with-completion-gates
|
|
|
|
|
ai.bstein.dev/model-policy: Switchyard AUTO with cross-provider fallback
|
|
|
|
|
ai.bstein.dev/storage: separate durable RWO workspace and OAuth refresh ownership per ordinal
|
|
|
|
|
vault.hashicorp.com/agent-inject: "true"
|
|
|
|
|
vault.hashicorp.com/role: hermes-execution-worker
|
|
|
|
|
vault.hashicorp.com/agent-inject-containers: stage-worker-access
|
|
|
|
|
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
|
|
|
|
|
vault.hashicorp.com/agent-inject-secret-claude-credentials-0: kv/data/atlas/hermes/agent-tokens
|
|
|
|
|
vault.hashicorp.com/agent-inject-perms-claude-credentials-0: "0600"
|
|
|
|
|
vault.hashicorp.com/agent-inject-template-claude-credentials-0: |
|
|
|
|
|
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
|
|
|
|
{{ .Data.data.execution_worker_0_claude_credentials_json }}
|
|
|
|
|
{{- end }}
|
|
|
|
|
vault.hashicorp.com/agent-inject-secret-claude-credentials-1: kv/data/atlas/hermes/agent-tokens
|
|
|
|
|
vault.hashicorp.com/agent-inject-perms-claude-credentials-1: "0600"
|
|
|
|
|
vault.hashicorp.com/agent-inject-template-claude-credentials-1: |
|
|
|
|
|
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
|
|
|
|
{{ .Data.data.execution_worker_1_claude_credentials_json }}
|
|
|
|
|
{{- end }}
|
|
|
|
|
vault.hashicorp.com/agent-inject-secret-claude-credentials-2: kv/data/atlas/hermes/agent-tokens
|
|
|
|
|
vault.hashicorp.com/agent-inject-perms-claude-credentials-2: "0600"
|
|
|
|
|
vault.hashicorp.com/agent-inject-template-claude-credentials-2: |
|
|
|
|
|
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
|
|
|
|
{{ .Data.data.execution_worker_2_claude_credentials_json }}
|
|
|
|
|
{{- end }}
|
|
|
|
|
vault.hashicorp.com/agent-inject-secret-codex-auth-0: kv/data/atlas/hermes/agent-tokens
|
|
|
|
|
vault.hashicorp.com/agent-inject-perms-codex-auth-0: "0600"
|
|
|
|
|
vault.hashicorp.com/agent-inject-template-codex-auth-0: |
|
|
|
|
|
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
|
|
|
|
{{ .Data.data.execution_worker_0_codex_auth_json }}
|
|
|
|
|
{{- end }}
|
|
|
|
|
vault.hashicorp.com/agent-inject-secret-codex-auth-1: kv/data/atlas/hermes/agent-tokens
|
|
|
|
|
vault.hashicorp.com/agent-inject-perms-codex-auth-1: "0600"
|
|
|
|
|
vault.hashicorp.com/agent-inject-template-codex-auth-1: |
|
|
|
|
|
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
|
|
|
|
{{ .Data.data.execution_worker_1_codex_auth_json }}
|
|
|
|
|
{{- end }}
|
|
|
|
|
vault.hashicorp.com/agent-inject-secret-codex-auth-2: kv/data/atlas/hermes/agent-tokens
|
|
|
|
|
vault.hashicorp.com/agent-inject-perms-codex-auth-2: "0600"
|
|
|
|
|
vault.hashicorp.com/agent-inject-template-codex-auth-2: |
|
|
|
|
|
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
|
|
|
|
{{ .Data.data.execution_worker_2_codex_auth_json }}
|
|
|
|
|
{{- end }}
|
|
|
|
|
vault.hashicorp.com/agent-pre-populate-only: "true"
|
|
|
|
|
vault.hashicorp.com/agent-init-first: "true"
|
|
|
|
|
vault.hashicorp.com/agent-requests-cpu: 2m
|
|
|
|
|
vault.hashicorp.com/agent-requests-mem: 16Mi
|
|
|
|
|
vault.hashicorp.com/agent-limits-cpu: 100m
|
|
|
|
|
vault.hashicorp.com/agent-limits-mem: 128Mi
|
|
|
|
|
spec:
|
|
|
|
|
serviceAccountName: hermes-execution-worker
|
|
|
|
|
priorityClassName: scavenger
|
|
|
|
|
automountServiceAccountToken: false
|
|
|
|
|
enableServiceLinks: false
|
|
|
|
|
terminationGracePeriodSeconds: 30
|
|
|
|
|
securityContext:
|
|
|
|
|
fsGroup: 10000
|
|
|
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
|
|
|
seccompProfile:
|
|
|
|
|
type: RuntimeDefault
|
|
|
|
|
affinity:
|
|
|
|
|
nodeAffinity:
|
|
|
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
|
|
|
nodeSelectorTerms:
|
|
|
|
|
- matchExpressions:
|
|
|
|
|
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
|
|
|
|
|
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
|
|
|
|
|
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
|
|
|
|
|
preferredDuringSchedulingIgnoredDuringExecution:
|
|
|
|
|
- weight: 100
|
|
|
|
|
preference:
|
|
|
|
|
matchExpressions:
|
|
|
|
|
- {key: node-role.kubernetes.io/accelerator, operator: Exists}
|
|
|
|
|
- weight: 50
|
|
|
|
|
preference:
|
|
|
|
|
matchExpressions:
|
|
|
|
|
- {key: hardware, operator: In, values: [rpi5]}
|
|
|
|
|
podAntiAffinity:
|
|
|
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
|
|
|
- labelSelector:
|
|
|
|
|
matchLabels:
|
|
|
|
|
app: hermes-execution-worker
|
|
|
|
|
topologyKey: kubernetes.io/hostname
|
|
|
|
|
topologySpreadConstraints:
|
|
|
|
|
- maxSkew: 1
|
|
|
|
|
topologyKey: kubernetes.io/hostname
|
|
|
|
|
whenUnsatisfiable: DoNotSchedule
|
|
|
|
|
labelSelector:
|
|
|
|
|
matchLabels:
|
|
|
|
|
app: hermes-execution-worker
|
|
|
|
|
initContainers:
|
|
|
|
|
- name: stage-worker-access
|
|
|
|
|
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
|
|
|
imagePullPolicy: IfNotPresent
|
|
|
|
|
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-worker]
|
|
|
|
|
env:
|
|
|
|
|
- {name: HERMES_WORKER_ROOT, value: /workspace}
|
|
|
|
|
- {name: HERMES_PROVIDER_ACCESS_ROOT, value: /provider-access}
|
|
|
|
|
- name: HERMES_WORKER_ORDINAL
|
|
|
|
|
valueFrom:
|
|
|
|
|
fieldRef:
|
|
|
|
|
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
|
|
|
|
|
securityContext:
|
|
|
|
|
allowPrivilegeEscalation: false
|
|
|
|
|
runAsUser: 0
|
|
|
|
|
runAsGroup: 0
|
|
|
|
|
seccompProfile: {type: RuntimeDefault}
|
|
|
|
|
volumeMounts:
|
|
|
|
|
- {name: workspace, mountPath: /workspace}
|
|
|
|
|
- {name: provider-access, mountPath: /provider-access}
|
|
|
|
|
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
|
|
|
|
resources:
|
|
|
|
|
requests: {cpu: 2m, memory: 32Mi}
|
|
|
|
|
limits: {cpu: 100m, memory: 64Mi}
|
|
|
|
|
- name: install-provider-clis
|
|
|
|
|
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
|
|
|
imagePullPolicy: IfNotPresent
|
|
|
|
|
command: [/bin/sh, -ec]
|
|
|
|
|
args:
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
# The tools volume is durable, so this install runs once per pinned
|
|
|
|
|
# version rather than on every Pod start. The version marker is only
|
|
|
|
|
# trusted when both binaries are actually present and executable, so
|
|
|
|
|
# a partial or pruned cache reinstalls instead of failing closed
|
|
|
|
|
# forever. The install itself is time-bounded so a stalled registry
|
|
|
|
|
# surfaces as a fast init failure with backoff instead of an
|
|
|
|
|
# open-ended wait that a Flux health timeout would have to absorb.
|
2026-08-17 16:31:15 +00:00
|
|
|
- |
|
|
|
|
|
tools=/worker-data/tools
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
marker="${tools}/.cli-versions-0.147.0-2.1.226"
|
2026-08-17 16:31:15 +00:00
|
|
|
mkdir -p "${tools}/bin"
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
if [ ! -f "${marker}" ] || [ ! -x "${tools}/bin/codex" ] \
|
|
|
|
|
|| [ ! -x "${tools}/bin/claude" ]; then
|
|
|
|
|
rm -f "${marker}"
|
|
|
|
|
timeout 900 npm install --global --omit=dev --no-audit --no-fund \
|
|
|
|
|
--fetch-timeout=120000 --fetch-retries=2 --prefix "${tools}" \
|
2026-08-17 16:31:15 +00:00
|
|
|
@openai/codex@0.147.0 @anthropic-ai/claude-code@2.1.226
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
touch "${marker}"
|
2026-08-17 16:31:15 +00:00
|
|
|
fi
|
|
|
|
|
test -x "${tools}/bin/codex"
|
|
|
|
|
test -x "${tools}/bin/claude"
|
|
|
|
|
securityContext:
|
|
|
|
|
allowPrivilegeEscalation: false
|
|
|
|
|
capabilities: {drop: [ALL]}
|
|
|
|
|
runAsNonRoot: true
|
|
|
|
|
runAsUser: 10000
|
|
|
|
|
runAsGroup: 10000
|
|
|
|
|
seccompProfile: {type: RuntimeDefault}
|
|
|
|
|
volumeMounts:
|
|
|
|
|
- {name: tools, mountPath: /worker-data/tools}
|
|
|
|
|
resources:
|
|
|
|
|
requests: {cpu: 5m, memory: 64Mi}
|
|
|
|
|
limits: {cpu: "1", memory: 1Gi}
|
|
|
|
|
containers:
|
|
|
|
|
- name: execution-worker
|
|
|
|
|
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
|
|
|
|
imagePullPolicy: IfNotPresent
|
|
|
|
|
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_worker.py]
|
|
|
|
|
env:
|
|
|
|
|
- {name: HERMES_HOME, value: /worker-data}
|
|
|
|
|
- {name: HERMES_WORKER_ROOT, value: /workspace}
|
|
|
|
|
- {name: HOME, value: /worker-data/home}
|
|
|
|
|
- {name: CODEX_HOME, value: /provider-access/codex}
|
|
|
|
|
- {name: CLAUDE_CONFIG_DIR, value: /provider-access/claude}
|
|
|
|
|
- {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
|
|
|
|
|
- {name: PYTHONPATH, value: /opt/hermes}
|
|
|
|
|
- {name: PATH, value: /worker-data/tools/bin:/opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
|
|
|
|
|
- name: HERMES_WORKER_ORDINAL
|
|
|
|
|
valueFrom:
|
|
|
|
|
fieldRef:
|
|
|
|
|
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
|
|
|
|
|
- name: HERMES_WORKER_NODE
|
|
|
|
|
valueFrom:
|
|
|
|
|
fieldRef:
|
|
|
|
|
fieldPath: spec.nodeName
|
|
|
|
|
startupProbe:
|
|
|
|
|
exec:
|
|
|
|
|
command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json && test -w /provider-access/claude/.credentials.json"]
|
|
|
|
|
periodSeconds: 5
|
|
|
|
|
failureThreshold: 60
|
|
|
|
|
readinessProbe:
|
|
|
|
|
exec:
|
|
|
|
|
command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json"]
|
|
|
|
|
periodSeconds: 10
|
|
|
|
|
securityContext:
|
|
|
|
|
allowPrivilegeEscalation: false
|
|
|
|
|
capabilities: {drop: [ALL]}
|
|
|
|
|
readOnlyRootFilesystem: true
|
|
|
|
|
runAsNonRoot: true
|
|
|
|
|
runAsUser: 10000
|
|
|
|
|
runAsGroup: 10000
|
|
|
|
|
seccompProfile: {type: RuntimeDefault}
|
|
|
|
|
volumeMounts:
|
|
|
|
|
- {name: workspace, mountPath: /workspace}
|
|
|
|
|
- {name: worker-data, mountPath: /worker-data}
|
|
|
|
|
- {name: tools, mountPath: /worker-data/tools, readOnly: true}
|
|
|
|
|
- {name: provider-access, mountPath: /provider-access}
|
|
|
|
|
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
|
|
|
|
- {name: tmp, mountPath: /tmp}
|
|
|
|
|
resources:
|
|
|
|
|
requests: {cpu: 5m, memory: 128Mi, ephemeral-storage: 1Gi}
|
|
|
|
|
limits: {cpu: "2", memory: 4Gi, ephemeral-storage: 8Gi}
|
|
|
|
|
volumes:
|
|
|
|
|
- name: worker-data
|
|
|
|
|
emptyDir: {sizeLimit: 128Mi}
|
|
|
|
|
- name: coordinator
|
|
|
|
|
configMap:
|
|
|
|
|
name: hermes-execution-pool
|
|
|
|
|
defaultMode: 0555
|
|
|
|
|
- name: tmp
|
|
|
|
|
emptyDir: {sizeLimit: 2Gi}
|
|
|
|
|
- name: vault-auth-token
|
|
|
|
|
projected:
|
|
|
|
|
defaultMode: 0600
|
|
|
|
|
sources:
|
|
|
|
|
- serviceAccountToken:
|
|
|
|
|
audience: vault
|
|
|
|
|
expirationSeconds: 3600
|
|
|
|
|
path: token
|
|
|
|
|
- configMap:
|
|
|
|
|
name: kube-root-ca.crt
|
|
|
|
|
items:
|
|
|
|
|
- {key: ca.crt, path: ca.crt}
|
|
|
|
|
- downwardAPI:
|
|
|
|
|
items:
|
|
|
|
|
- {path: namespace, fieldRef: {fieldPath: metadata.namespace}}
|
|
|
|
|
volumeClaimTemplates:
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
# The workspace is shared with this ordinal's mediator, which performs every
|
|
|
|
|
# SCM operation on it. ReadWriteMany removes the cross-node Multi-Attach
|
|
|
|
|
# deadlock that a drain or preemption used to create between the two Pods,
|
|
|
|
|
# so each can be evicted and rescheduled independently. Longhorn already
|
|
|
|
|
# serves the hermes-chat tenant workspaces this way on the same class.
|
2026-08-17 16:31:15 +00:00
|
|
|
- metadata:
|
|
|
|
|
name: workspace
|
|
|
|
|
labels:
|
|
|
|
|
app: hermes-execution-worker
|
|
|
|
|
spec:
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
accessModes: [ReadWriteMany]
|
2026-08-17 16:31:15 +00:00
|
|
|
storageClassName: astreae
|
|
|
|
|
resources:
|
|
|
|
|
requests:
|
|
|
|
|
storage: 30Gi
|
hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.
`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.
The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.
Also closes three consequential handoff risks the same review raised:
* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
claim, so a drain or preemption that moved only the lower-priority worker
deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
already are on the same class), colocation is a preference, and the mediator
shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
never submit and the run's work was discarded with the failure. Submission
now targets a fresh attempt- or content-scoped ref in the same reviewed
namespace -- never an update -- and is idempotent under replay. A refused
submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
the 10m Flux health window for the whole hermes app. They now install once
per pinned version onto a durable volume, re-verified against the real
binaries and time-bounded, and the best-effort pool no longer gates the
health of the app its dependents wait on.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
|
|
|
# Provider CLIs are installed once per pinned version onto a durable volume
|
|
|
|
|
# instead of being re-downloaded into an emptyDir on every Pod start.
|
|
|
|
|
- metadata:
|
|
|
|
|
name: tools
|
|
|
|
|
labels:
|
|
|
|
|
app: hermes-execution-worker
|
|
|
|
|
spec:
|
|
|
|
|
accessModes: [ReadWriteOnce]
|
|
|
|
|
storageClassName: astreae
|
|
|
|
|
resources:
|
|
|
|
|
requests:
|
|
|
|
|
storage: 2Gi
|
|
|
|
|
# OAuth refresh ownership is single-mounter by design: keep it ReadWriteOnce.
|
2026-08-17 16:31:15 +00:00
|
|
|
- metadata:
|
|
|
|
|
name: provider-access
|
|
|
|
|
labels:
|
|
|
|
|
app: hermes-execution-worker
|
|
|
|
|
spec:
|
|
|
|
|
accessModes: [ReadWriteOnce]
|
|
|
|
|
storageClassName: astreae
|
|
|
|
|
resources:
|
|
|
|
|
requests:
|
|
|
|
|
storage: 1Gi
|