titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

hermes: repair distributed pool integration boundaries

Browse Source
This commit is contained in:
Hermes Agent 2026-08-17 12:52:23 +00:00
parent 5dedb67ec6
commit 0fc97529c2
48 changed files with 4550 additions and 1607 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ci/scripts/semgrep_report.py
View File

@ -52,7 +52,7 @@ def build_report(
results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else []
severity_counts = {severity: 0 for severity in SEVERITIES}
severity_counts = dict.fromkeys(SEVERITIES, 0)
blocking_findings = 0
for finding in results:
severity = _finding_severity(finding)

2
scripts/tests/test_dashboards_render_atlas_drilldowns.py
View File

@ -1,6 +1,6 @@
"""Detailed Atlas Jobs dashboard collapse and drilldown contracts."""
from scripts.tests.test_dashboards_render_atlas import load_module
from scripts.tests.test_dashboard_render_support import load_module
def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():

3
services/hermes-scm-broker/deployment.yaml
View File

@ -17,6 +17,7 @@ spec:
labels:
app: hermes-scm-broker
annotations:
ai.bstein.dev/config-rev: scm-boundary-v2
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-scm-broker
vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea
@ -91,7 +92,7 @@ spec:
volumes:
- name: broker-code
configMap:
name: hermes-scm-boundary
name: hermes-scm-boundary-v2
defaultMode: 0555
- name: tmp
emptyDir:

6
services/hermes-scm-broker/networkpolicy.yaml
View File

@ -17,6 +17,12 @@ spec:
podSelector:
matchLabels:
app: hermes-agent
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: hermes
podSelector:
matchLabels:
app: hermes-execution-mediator
ports:
- {protocol: TCP, port: 9081}
egress:

26
services/hermes/NOTES.md
View File

@ -257,6 +257,32 @@ Use this short explanation:
leading different real incidents, improving the skill after failures, and
teaching the architecture without prompts.
## Distributed execution-pool rollout boundary
- Do not reconcile the pool until the approved credential owner has provisioned
six distinct Vault fields: `execution_worker_{0,1,2}_claude_credentials_json`
and `execution_worker_{0,1,2}_codex_auth_json`. Each ordinal needs an
independent account or refresh-token lineage; copying one rotating refresh
token into multiple fields recreates the lost-update failure this design
prevents.
- Worker credential files live on separate ordinal-owned RWO `provider-access`
claims. Provider refresh updates these durable private copies across Pod
restarts; they are never synchronized back to Vault. Rotate one bootstrap
credential at a time through the reviewed Vault workflow and reinitialize
only that ordinal after human approval.
- Existing task worktrees remain on the single local owner lane. Only tasks
without `workspace_path` enter the distributed pool, where each ordinal owns
one RWO checkout. Repository and base-branch identity comes from the canonical
board registry rather than task-supplied metadata.
- Model Pods have no pool key, broker mount, broker egress, or Kubernetes token.
Ordinal mediator Deployments are separate network identities colocated with
the matching workspace PVC; they alone authenticate exact-run results and
reach the PR14 SCM broker.
- A hashed execution-pool ConfigMap, protocol-version readiness checks, and
versioned SCM boundary name make code/config changes controlled rollouts. A
rollout is still a human-reviewed operation; this repository change does not
reconcile or deploy it.
## Your shortest path to fluency
1. Explain the request diagram without looking.

2
services/hermes/agent-deployment.yaml
View File

@ -1191,7 +1191,7 @@ spec:
- {key: openai.yaml, path: agents/openai.yaml}
- name: scm-boundary
configMap:
name: hermes-scm-boundary
name: hermes-scm-boundary-v2
defaultMode: 0555
- name: image-policy
configMap:

8
services/hermes/execution-coordinator-patch.yaml
View File

@ -11,12 +11,14 @@ spec:
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-execution-pool-v1:%s" .Data.data.agent_api_key | sha256sum }}
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
{{- end }}
spec:
containers:
- name: cli-lane-runner
$patch: delete
env:
- {name: HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY, value: "true"}
- {name: HERMES_CLI_LANE_CONCURRENCY, value: "1"}
- name: execution-pool-coordinator
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
@ -26,8 +28,6 @@ spec:
- {name: HOME, value: /opt/data/home}
- {name: PYTHONPATH, value: /opt/hermes}
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- {name: HERMES_EXECUTION_DEFAULT_REPO_URL, value: https://scm.bstein.dev/atlas/titan-iac.git}
- {name: HERMES_EXECUTION_DEFAULT_BASE_BRANCH, value: main}
- {name: PATH, value: /opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
securityContext:
allowPrivilegeEscalation: false

425
services/hermes/execution-mediator.yaml Normal file
View File

@ -0,0 +1,425 @@
# Privileged HMAC and SCM mediation runs outside the model Pods. Each mediator is
# colocated with exactly one ordinal's RWO workspace but has a distinct network
# identity, private HMAC derivation, and durable integrity state.
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: hermes-execution-mediator-state-0, namespace: hermes}
spec:
accessModes: [ReadWriteOnce]
storageClassName: astreae
resources: {requests: {storage: 1Gi}}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: hermes-execution-mediator-state-1, namespace: hermes}
spec:
accessModes: [ReadWriteOnce]
storageClassName: astreae
resources: {requests: {storage: 1Gi}}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: hermes-execution-mediator-state-2, namespace: hermes}
spec:
accessModes: [ReadWriteOnce]
storageClassName: astreae
resources: {requests: {storage: 1Gi}}
---
apiVersion: v1
kind: Service
metadata: {name: hermes-execution-mediator-0, namespace: hermes}
spec:
selector: {app: hermes-execution-mediator, pool-ordinal: "0"}
ports: [{name: mediator, port: 9009, targetPort: mediator}]
---
apiVersion: v1
kind: Service
metadata: {name: hermes-execution-mediator-1, namespace: hermes}
spec:
selector: {app: hermes-execution-mediator, pool-ordinal: "1"}
ports: [{name: mediator, port: 9009, targetPort: mediator}]
---
apiVersion: v1
kind: Service
metadata: {name: hermes-execution-mediator-2, namespace: hermes}
spec:
selector: {app: hermes-execution-mediator, pool-ordinal: "2"}
ports: [{name: mediator, port: 9009, targetPort: mediator}]
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: hermes-execution-mediator-0
namespace: hermes
labels: {app: hermes-execution-mediator, pool-ordinal: "0"}
spec:
replicas: 1
strategy: {type: Recreate}
selector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
template:
metadata:
labels: {app: hermes-execution-mediator, pool-ordinal: "0"}
annotations:
ai.bstein.dev/config-rev: execution-pool-v2-mediated
ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-execution-worker
vault.hashicorp.com/agent-inject-containers: stage-mediator-access
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 2m
vault.hashicorp.com/agent-requests-mem: 16Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
spec:
serviceAccountName: hermes-execution-worker
automountServiceAccountToken: false
enableServiceLinks: false
securityContext:
fsGroup: 10000
fsGroupChangePolicy: OnRootMismatch
seccompProfile: {type: RuntimeDefault}
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
statefulset.kubernetes.io/pod-name: hermes-execution-worker-0
topologyKey: kubernetes.io/hostname
initContainers:
- name: stage-mediator-access
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
env:
- {name: HERMES_WORKER_ORDINAL, value: "0"}
- {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
runAsGroup: 0
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: pool-access, mountPath: /pool-access}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
resources:
requests: {cpu: 2m, memory: 16Mi}
limits: {cpu: 100m, memory: 64Mi}
containers:
- name: execution-mediator
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
env:
- {name: HERMES_WORKER_ORDINAL, value: "0"}
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
startupProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 5
failureThreshold: 60
readinessProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 10
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: workspace, mountPath: /workspace}
- {name: scm-state, mountPath: /scm-state}
- {name: pool-access, mountPath: /pool-access, readOnly: true}
- {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 2m, memory: 64Mi}
limits: {cpu: 250m, memory: 256Mi}
volumes:
- name: workspace
persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-0}
- name: scm-state
persistentVolumeClaim: {claimName: hermes-execution-mediator-state-0}
- name: pool-access
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: scm-broker-client
configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
- name: coordinator
configMap: {name: hermes-execution-pool, defaultMode: 0555}
- name: tmp
emptyDir: {sizeLimit: 64Mi}
- name: vault-auth-token
projected:
defaultMode: 0600
sources:
- serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
- configMap:
name: kube-root-ca.crt
items: [{key: ca.crt, path: ca.crt}]
- downwardAPI:
items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: hermes-execution-mediator-1
namespace: hermes
labels: {app: hermes-execution-mediator, pool-ordinal: "1"}
spec:
replicas: 1
strategy: {type: Recreate}
selector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
template:
metadata:
labels: {app: hermes-execution-mediator, pool-ordinal: "1"}
annotations:
ai.bstein.dev/config-rev: execution-pool-v2-mediated
ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-execution-worker
vault.hashicorp.com/agent-inject-containers: stage-mediator-access
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 2m
vault.hashicorp.com/agent-requests-mem: 16Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
spec:
serviceAccountName: hermes-execution-worker
automountServiceAccountToken: false
enableServiceLinks: false
securityContext:
fsGroup: 10000
fsGroupChangePolicy: OnRootMismatch
seccompProfile: {type: RuntimeDefault}
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
statefulset.kubernetes.io/pod-name: hermes-execution-worker-1
topologyKey: kubernetes.io/hostname
initContainers:
- name: stage-mediator-access
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
env:
- {name: HERMES_WORKER_ORDINAL, value: "1"}
- {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
runAsGroup: 0
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: pool-access, mountPath: /pool-access}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
resources:
requests: {cpu: 2m, memory: 16Mi}
limits: {cpu: 100m, memory: 64Mi}
containers:
- name: execution-mediator
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
env:
- {name: HERMES_WORKER_ORDINAL, value: "1"}
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
startupProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 5
failureThreshold: 60
readinessProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 10
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: workspace, mountPath: /workspace}
- {name: scm-state, mountPath: /scm-state}
- {name: pool-access, mountPath: /pool-access, readOnly: true}
- {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 2m, memory: 64Mi}
limits: {cpu: 250m, memory: 256Mi}
volumes:
- name: workspace
persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-1}
- name: scm-state
persistentVolumeClaim: {claimName: hermes-execution-mediator-state-1}
- name: pool-access
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: scm-broker-client
configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
- name: coordinator
configMap: {name: hermes-execution-pool, defaultMode: 0555}
- name: tmp
emptyDir: {sizeLimit: 64Mi}
- name: vault-auth-token
projected:
defaultMode: 0600
sources:
- serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
- configMap:
name: kube-root-ca.crt
items: [{key: ca.crt, path: ca.crt}]
- downwardAPI:
items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: hermes-execution-mediator-2
namespace: hermes
labels: {app: hermes-execution-mediator, pool-ordinal: "2"}
spec:
replicas: 1
strategy: {type: Recreate}
selector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
template:
metadata:
labels: {app: hermes-execution-mediator, pool-ordinal: "2"}
annotations:
ai.bstein.dev/config-rev: execution-pool-v2-mediated
ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-execution-worker
vault.hashicorp.com/agent-inject-containers: stage-mediator-access
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 2m
vault.hashicorp.com/agent-requests-mem: 16Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
spec:
serviceAccountName: hermes-execution-worker
automountServiceAccountToken: false
enableServiceLinks: false
securityContext:
fsGroup: 10000
fsGroupChangePolicy: OnRootMismatch
seccompProfile: {type: RuntimeDefault}
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
statefulset.kubernetes.io/pod-name: hermes-execution-worker-2
topologyKey: kubernetes.io/hostname
initContainers:
- name: stage-mediator-access
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
env:
- {name: HERMES_WORKER_ORDINAL, value: "2"}
- {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
runAsGroup: 0
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: pool-access, mountPath: /pool-access}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
resources:
requests: {cpu: 2m, memory: 16Mi}
limits: {cpu: 100m, memory: 64Mi}
containers:
- name: execution-mediator
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
env:
- {name: HERMES_WORKER_ORDINAL, value: "2"}
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
startupProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 5
failureThreshold: 60
readinessProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 10
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: workspace, mountPath: /workspace}
- {name: scm-state, mountPath: /scm-state}
- {name: pool-access, mountPath: /pool-access, readOnly: true}
- {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 2m, memory: 64Mi}
limits: {cpu: 250m, memory: 256Mi}
volumes:
- name: workspace
persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-2}
- name: scm-state
persistentVolumeClaim: {claimName: hermes-execution-mediator-state-2}
- name: pool-access
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: scm-broker-client
configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
- name: coordinator
configMap: {name: hermes-execution-pool, defaultMode: 0555}
- name: tmp
emptyDir: {sizeLimit: 64Mi}
- name: vault-auth-token
projected:
defaultMode: 0600
sources:
- serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
- configMap:
name: kube-root-ca.crt
items: [{key: ca.crt, path: ca.crt}]
- downwardAPI:
items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]

152
services/hermes/execution-worker-networkpolicy.yaml
View File

@ -22,24 +22,10 @@ spec:
podSelector:
matchLabels: {app: vault}
ports: [{protocol: TCP, port: 8200}]
- to:
- podSelector:
matchLabels: {app: hermes-agent}
ports: [{protocol: TCP, port: 9007}]
- to:
- podSelector:
matchLabels: {app: hermes-switchyard}
ports: [{protocol: TCP, port: 9005}]
- to:
- podSelector:
matchLabels: {app: hermes-model-gate}
ports: [{protocol: TCP, port: 8080}]
- to:
- namespaceSelector:
matchLabels: {kubernetes.io/metadata.name: gitea}
podSelector:
matchLabels: {app: gitea}
ports: [{protocol: TCP, port: 3000}]
- to:
- ipBlock:
cidr: 0.0.0.0/0
@ -55,6 +41,57 @@ spec:
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-worker-mediator-0
namespace: hermes
spec:
podSelector:
matchLabels:
app: hermes-execution-worker
apps.kubernetes.io/pod-index: "0"
policyTypes: [Egress]
egress:
- to:
- podSelector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
ports: [{protocol: TCP, port: 9009}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-worker-mediator-1
namespace: hermes
spec:
podSelector:
matchLabels:
app: hermes-execution-worker
apps.kubernetes.io/pod-index: "1"
policyTypes: [Egress]
egress:
- to:
- podSelector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
ports: [{protocol: TCP, port: 9009}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-worker-mediator-2
namespace: hermes
spec:
podSelector:
matchLabels:
app: hermes-execution-worker
apps.kubernetes.io/pod-index: "2"
policyTypes: [Egress]
egress:
- to:
- podSelector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
ports: [{protocol: TCP, port: 9009}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-pool-ingress
namespace: hermes
@ -65,11 +102,96 @@ spec:
ingress:
- from:
- podSelector:
matchLabels: {app: hermes-execution-worker}
matchLabels: {app: hermes-execution-mediator}
ports: [{protocol: TCP, port: 9007}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-mediator-isolation
namespace: hermes
spec:
podSelector:
matchLabels: {app: hermes-execution-mediator}
policyTypes: [Ingress, Egress]
ingress: []
egress:
- to:
- namespaceSelector:
matchLabels: {kubernetes.io/metadata.name: kube-system}
podSelector:
matchLabels: {k8s-app: kube-dns}
ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}]
- to:
- namespaceSelector:
matchLabels: {kubernetes.io/metadata.name: vault}
podSelector:
matchLabels: {app: vault}
ports: [{protocol: TCP, port: 8200}]
- to:
- podSelector:
matchLabels: {app: hermes-agent}
ports: [{protocol: TCP, port: 9007}]
- to:
- namespaceSelector:
matchLabels: {kubernetes.io/metadata.name: hermes-scm}
podSelector:
matchLabels: {app: hermes-scm-broker}
ports: [{protocol: TCP, port: 9081}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-mediator-worker-0
namespace: hermes
spec:
podSelector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
policyTypes: [Ingress]
ingress:
- from:
- podSelector:
matchLabels:
app: hermes-execution-worker
apps.kubernetes.io/pod-index: "0"
ports: [{protocol: TCP, port: 9009}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-mediator-worker-1
namespace: hermes
spec:
podSelector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
policyTypes: [Ingress]
ingress:
- from:
- podSelector:
matchLabels:
app: hermes-execution-worker
apps.kubernetes.io/pod-index: "1"
ports: [{protocol: TCP, port: 9009}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-mediator-worker-2
namespace: hermes
spec:
podSelector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
policyTypes: [Ingress]
ingress:
- from:
- podSelector:
matchLabels:
app: hermes-execution-worker
apps.kubernetes.io/pod-index: "2"
ports: [{protocol: TCP, port: 9009}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-switchyard-ingress
namespace: hermes

214
services/hermes/execution-worker-statefulset.yaml
View File

@ -15,6 +15,8 @@ spec:
app: hermes-execution-worker
updateStrategy:
type: RollingUpdate
rollingUpdate:
partition: 0
template:
metadata:
labels:
@ -23,51 +25,58 @@ spec:
app.kubernetes.io/part-of: hermes
annotations:
ai.bstein.dev/role: fenced-execution-only
ai.bstein.dev/scm-boundary: ordinal-sidecar-with-assignment-bound-branch
ai.bstein.dev/scm-boundary: mediated-pr14-broker-with-completion-gates
ai.bstein.dev/model-policy: Switchyard AUTO with cross-provider fallback
ai.bstein.dev/storage: one durable RWO workspace and provider session home per ordinal
ai.bstein.dev/storage: separate durable RWO workspace and OAuth refresh ownership per ordinal
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-execution-worker
vault.hashicorp.com/agent-inject-containers: stage-worker-access
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
vault.hashicorp.com/agent-inject-secret-claude-credentials-0: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-claude-credentials-0: "0600"
vault.hashicorp.com/agent-inject-template-claude-credentials-0: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-execution-pool-v1:%s" .Data.data.agent_api_key | sha256sum }}
{{ .Data.data.execution_worker_0_claude_credentials_json }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-claude-credentials: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-claude-credentials: "0600"
vault.hashicorp.com/agent-inject-template-claude-credentials: |
vault.hashicorp.com/agent-inject-secret-claude-credentials-1: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-claude-credentials-1: "0600"
vault.hashicorp.com/agent-inject-template-claude-credentials-1: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ .Data.data.claude_credentials_json }}
{{ .Data.data.execution_worker_1_claude_credentials_json }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-codex-auth: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-codex-auth: "0600"
vault.hashicorp.com/agent-inject-template-codex-auth: |
vault.hashicorp.com/agent-inject-secret-claude-credentials-2: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-claude-credentials-2: "0600"
vault.hashicorp.com/agent-inject-template-claude-credentials-2: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ .Data.data.codex_auth_json }}
{{ .Data.data.execution_worker_2_claude_credentials_json }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea
vault.hashicorp.com/agent-inject-perms-gitea-token: "0600"
vault.hashicorp.com/agent-inject-template-gitea-token: |
{{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
{{ .Data.data.token }}
vault.hashicorp.com/agent-inject-secret-codex-auth-0: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-codex-auth-0: "0600"
vault.hashicorp.com/agent-inject-template-codex-auth-0: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ .Data.data.execution_worker_0_codex_auth_json }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-gitea-username: kv/data/atlas/hermes/developer-gitea
vault.hashicorp.com/agent-inject-perms-gitea-username: "0600"
vault.hashicorp.com/agent-inject-template-gitea-username: |
{{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
{{ .Data.data.username }}
vault.hashicorp.com/agent-inject-secret-codex-auth-1: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-codex-auth-1: "0600"
vault.hashicorp.com/agent-inject-template-codex-auth-1: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ .Data.data.execution_worker_1_codex_auth_json }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-codex-auth-2: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-codex-auth-2: "0600"
vault.hashicorp.com/agent-inject-template-codex-auth-2: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ .Data.data.execution_worker_2_codex_auth_json }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 25m
vault.hashicorp.com/agent-requests-mem: 32Mi
vault.hashicorp.com/agent-requests-cpu: 2m
vault.hashicorp.com/agent-requests-mem: 16Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
spec:
serviceAccountName: hermes-execution-worker
priorityClassName: scavenger
automountServiceAccountToken: false
enableServiceLinks: false
terminationGracePeriodSeconds: 30
@ -82,12 +91,8 @@ spec:
nodeSelectorTerms:
- matchExpressions:
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
- {key: node-role.kubernetes.io/accelerator, operator: Exists}
- {key: kubernetes.io/hostname, operator: In, values: [titan-20, titan-21]}
- matchExpressions:
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
- {key: hardware, operator: In, values: [rpi5]}
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
@ -117,8 +122,11 @@ spec:
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-worker]
env:
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
- {name: HERMES_SCM_ACCESS_ROOT, value: /scm-access}
- {name: HERMES_PROVIDER_ACCESS_ROOT, value: /provider-access}
- name: HERMES_WORKER_ORDINAL
valueFrom:
fieldRef:
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
@ -126,12 +134,10 @@ spec:
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: workspace, mountPath: /workspace}
- {name: runtime-access, mountPath: /runtime-access}
- {name: pool-access, mountPath: /pool-access}
- {name: scm-access, mountPath: /scm-access}
- {name: provider-access, mountPath: /provider-access}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
resources:
requests: {cpu: 25m, memory: 32Mi}
requests: {cpu: 2m, memory: 32Mi}
limits: {cpu: 100m, memory: 64Mi}
- name: install-provider-clis
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
@ -158,7 +164,7 @@ spec:
volumeMounts:
- {name: tools, mountPath: /worker-data/tools}
resources:
requests: {cpu: 100m, memory: 128Mi}
requests: {cpu: 5m, memory: 64Mi}
limits: {cpu: "1", memory: 1Gi}
containers:
- name: execution-worker
@ -169,8 +175,8 @@ spec:
- {name: HERMES_HOME, value: /worker-data}
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HOME, value: /worker-data/home}
- {name: CODEX_HOME, value: /runtime-access/codex}
- {name: CLAUDE_CONFIG_DIR, value: /runtime-access/claude}
- {name: CODEX_HOME, value: /provider-access/codex}
- {name: CLAUDE_CONFIG_DIR, value: /provider-access/claude}
- {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
- {name: PYTHONPATH, value: /opt/hermes}
- {name: PATH, value: /worker-data/tools/bin:/opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
@ -184,12 +190,12 @@ spec:
fieldPath: spec.nodeName
startupProbe:
exec:
command: [/bin/sh, -ec, "test -w /workspace && test -w /runtime-access/codex/auth.json && test -w /runtime-access/claude/.credentials.json"]
command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json && test -w /provider-access/claude/.credentials.json"]
periodSeconds: 5
failureThreshold: 60
readinessProbe:
exec:
command: [/bin/sh, -ec, "test -w /workspace && test -w /runtime-access/codex/auth.json"]
command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json"]
periodSeconds: 10
securityContext:
allowPrivilegeEscalation: false
@ -203,123 +209,17 @@ spec:
- {name: workspace, mountPath: /workspace}
- {name: worker-data, mountPath: /worker-data}
- {name: tools, mountPath: /worker-data/tools, readOnly: true}
- {name: runtime-access, mountPath: /runtime-access}
- {name: provider-access, mountPath: /provider-access}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: "1", memory: 2Gi, ephemeral-storage: 2Gi}
limits: {cpu: "5", memory: 6Gi, ephemeral-storage: 8Gi}
- name: execution-client
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
env:
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- name: HERMES_WORKER_ORDINAL
valueFrom:
fieldRef:
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
ports:
- {name: pool-client, containerPort: 9009, protocol: TCP}
startupProbe:
httpGet: {path: /ready, port: pool-client}
periodSeconds: 5
failureThreshold: 60
readinessProbe:
httpGet: {path: /ready, port: pool-client}
periodSeconds: 10
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: pool-access, mountPath: /pool-access, readOnly: true}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: 250m, memory: 256Mi}
- name: scm-boundary
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_scm.py]
env:
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- {name: HERMES_GITEA_TOKEN_FILE, value: /scm-access/gitea-token}
- {name: HERMES_GITEA_USERNAME_FILE, value: /scm-access/gitea-username}
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
- name: HERMES_WORKER_ORDINAL
valueFrom:
fieldRef:
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
ports:
- {name: scm, containerPort: 9008, protocol: TCP}
startupProbe:
httpGet: {path: /ready, port: scm}
periodSeconds: 5
failureThreshold: 60
readinessProbe:
httpGet: {path: /ready, port: scm}
periodSeconds: 10
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: workspace, mountPath: /workspace}
- {name: pool-access, mountPath: /pool-access, readOnly: true}
- {name: scm-access, mountPath: /scm-access, readOnly: true}
- {name: scm-state, mountPath: /scm-state}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 100m, memory: 128Mi}
limits: {cpu: "1", memory: 512Mi}
- name: credential-sync
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/sync_runtime_credentials.py]
env:
- {name: HERMES_CREDENTIAL_SYNC_VAULT_ROLE, value: hermes-execution-credential-sync}
- {name: HERMES_CREDENTIAL_SYNC_INTERVAL, value: "300"}
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: runtime-access, mountPath: /runtime-access}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: vault-auth-token, mountPath: /var/run/secrets/kubernetes.io/serviceaccount, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: 250m, memory: 256Mi}
requests: {cpu: 5m, memory: 128Mi, ephemeral-storage: 1Gi}
limits: {cpu: "2", memory: 4Gi, ephemeral-storage: 8Gi}
volumes:
- name: worker-data
emptyDir: {sizeLimit: 128Mi}
- name: tools
emptyDir: {sizeLimit: 1Gi}
- name: runtime-access
emptyDir: {medium: Memory, sizeLimit: 128Mi}
- name: pool-access
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: scm-access
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: scm-state
emptyDir: {sizeLimit: 256Mi}
- name: coordinator
configMap:
name: hermes-execution-pool
@ -352,3 +252,13 @@ spec:
resources:
requests:
storage: 30Gi
- metadata:
name: provider-access
labels:
app: hermes-execution-worker
spec:
accessModes: [ReadWriteOnce]
storageClassName: astreae
resources:
requests:
storage: 1Gi

23
services/hermes/kustomization.yaml
View File

@ -43,6 +43,7 @@ resources:
- agent-ingress.yaml
- execution-worker-rbac.yaml
- execution-worker-statefulset.yaml
- execution-mediator.yaml
- execution-worker-networkpolicy.yaml
patches:
@ -64,18 +65,32 @@ configMapGenerator:
- name: hermes-execution-pool
namespace: hermes
files:
- cli_lane_board.py=scripts/cli_lane_board.py
- cli_lane_config.py=scripts/cli_lane_config.py
- cli_lane_dispatch.py=scripts/cli_lane_dispatch.py
- cli_lane_evidence.py=scripts/cli_lane_evidence.py
- cli_lane_execution.py=scripts/cli_lane_execution.py
- cli_lane_files.py=scripts/cli_lane_files.py
- cli_lane_finalization.py=scripts/cli_lane_finalization.py
- cli_lane_goal.py=scripts/cli_lane_goal.py
- cli_lane_prompt.py=scripts/cli_lane_prompt.py
- cli_lane_provider.py=scripts/cli_lane_provider.py
- cli_lane_quarantine.py=scripts/cli_lane_quarantine.py
- cli_lane_records.py=scripts/cli_lane_records.py
- cli_lane_recovery.py=scripts/cli_lane_recovery.py
- cli_lane_retention.py=scripts/cli_lane_retention.py
- cli_lane_routing.py=scripts/cli_lane_routing.py
- cli_lane_runner.py=scripts/cli_lane_runner.py
- execution_pool_askpass.sh=scripts/execution_pool_askpass.sh
- execution_pool_protocol.py=scripts/execution_pool_protocol.py
- execution_pool_project.py=scripts/execution_pool_project.py
- execution_pool_coordinator.py=scripts/execution_pool_coordinator.py
- execution_pool_server.py=scripts/execution_pool_server.py
- execution_pool_client.py=scripts/execution_pool_client.py
- execution_pool_worker.py=scripts/execution_pool_worker.py
- execution_pool_scm.py=scripts/execution_pool_scm.py
- gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py
- scm_broker_client.py=scm-common/scripts/scm_broker_client.py
- stage_runtime_access.py=scripts/stage_runtime_access.py
- sync_runtime_credentials.py=scripts/sync_runtime_credentials.py
options:
disableNameSuffixHash: true
- name: hermes-coordinator
namespace: hermes
files:

2
services/hermes/scm-common/kustomization.yaml
View File

@ -2,7 +2,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
configMapGenerator:
- name: hermes-scm-boundary
- name: hermes-scm-boundary-v2
files:
- gitea_api.py=scripts/gitea_api.py
- gitea_api_policy.py=scripts/gitea_api_policy.py

25
services/hermes/scm-common/scripts/scm_broker.py
View File

@ -7,7 +7,6 @@ import argparse
import base64
import json
import re
import socket
import tempfile
import time
import urllib.error
@ -45,7 +44,8 @@ GIT_PATH_RE = re.compile(
r"(?P<operation>info/refs|git-upload-pack|git-receive-pack)\Z"
)
FEATURE_REF_RE = re.compile(
r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z"
r"refs/heads/(?:(?:feature|fix|chore|docs|test|refactor|wt|review|hermes|handoff)/"
r"[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z"
)
@ -109,7 +109,9 @@ def _spool_bounded(
"""Copy a fixed-length exchange through a bounded-memory disk spool."""
if not 0 <= length <= maximum:
raise PolicyError("SCM request exceeds the safe size limit")
spool = tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_LIMIT, dir="/tmp")
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns lifetime
max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
)
remaining = length
forms = _credential_forms(token)
carry = b""
@ -218,7 +220,7 @@ def _content_length(headers: object, maximum: int) -> int:
def _credential_forms(token: str) -> tuple[bytes, ...]:
basic = base64.b64encode(f"{GIT_USER}:{token}".encode("utf-8"))
basic = base64.b64encode(f"{GIT_USER}:{token}".encode())
return token.encode("utf-8"), basic, b"Basic " + basic
@ -239,7 +241,7 @@ def _receive_prefix(body: bytes | BinaryIO) -> bytes:
def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None:
"""Permit only creation of new, namespaced feature branches."""
"""Permit only creation of new branches in reviewed namespaces."""
body = _receive_prefix(body)
_reject_credential_bytes(body, token, "Git request")
position = 0
@ -262,7 +264,7 @@ def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None:
raise PolicyError("Git receive-pack ref command is invalid")
old_sha, new_sha, raw_ref = fields
if old_sha != ZERO_SHA or new_sha == ZERO_SHA:
raise PolicyError("Git broker permits only new feature-branch creation")
raise PolicyError("Git broker permits only new namespaced branch creation")
try:
ref = raw_ref.decode("ascii")
except UnicodeDecodeError as exc:
@ -398,7 +400,7 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
self._stream(200, expected, body, length)
finally:
body.close()
except (OSError, PolicyError, socket.timeout, urllib.error.URLError, ValueError):
except (TimeoutError, OSError, PolicyError, urllib.error.URLError, ValueError):
self._reject()
def do_POST(self) -> None:
@ -409,7 +411,14 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
self._control()
else:
self._git_rpc()
except (OSError, PolicyError, socket.timeout, urllib.error.URLError, ValueError, json.JSONDecodeError):
except (
TimeoutError,
OSError,
PolicyError,
urllib.error.URLError,
ValueError,
json.JSONDecodeError,
):
self._reject()
def _control(self) -> None:

37
services/hermes/scripts/cli_lane_dispatch.py
View File

@ -7,6 +7,7 @@ import concurrent.futures
import os
import sys
import time
from collections.abc import Callable
from typing import Any
from cli_lane_board import _external, _record_board_access_error, _task_value
@ -23,6 +24,16 @@ from cli_lane_recovery import _has_pending_finalization, recover_pending_finaliz
from cli_lane_retention import maybe_gc_lane_artifacts
OWNED_WORKSPACES_ONLY = os.environ.get(
"HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY", ""
).strip().lower() in {"1", "true", "yes", "on"}
def _owns_local_workspace(task: Any) -> bool:
"""Keep legacy/dirty worktree tasks on their existing single-host owner."""
return bool(str(_task_value(task, "workspace_path", "") or "").strip())
def _board_slug(board: Any) -> str:
if isinstance(board, dict):
return str(board.get("slug") or board.get("id") or "")
@ -56,7 +67,11 @@ def recover_orphans() -> None:
continue
try:
for task in kanban_db.list_tasks(conn):
if _external(task) and str(_task_value(task, "status", "")) == "running":
if (
_external(task)
and str(_task_value(task, "status", "")) == "running"
and (not OWNED_WORKSPACES_ONLY or _owns_local_workspace(task))
):
task_id = str(_task_value(task, "id"))
run_id = _task_value(task, "current_run_id", None)
if not isinstance(run_id, int):
@ -75,7 +90,11 @@ def recover_orphans() -> None:
finally:
conn.close()
def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str]]:
def claim_ready(
active: set[tuple[str, str]],
limit: int,
eligible: Callable[[str, Any], bool] | None = None,
) -> list[tuple[str, str]]:
"""Atomically claim external ready tasks across all non-archived boards."""
from hermes_cli import kanban_db
@ -97,15 +116,21 @@ def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str
for task in tasks:
task_id = str(_task_value(task, "id", ""))
assignee = str(_task_value(task, "assignee", "") or "")
if task_id and not assignee and str(_task_value(task, "status", "")) == "ready":
if kanban_db.assign_task(conn, task_id, "cli-auto"):
task = kanban_db.get_task(conn, task_id)
assignee = "cli-auto"
if (
task_id
and not assignee
and str(_task_value(task, "status", "")) == "ready"
and kanban_db.assign_task(conn, task_id, "cli-auto")
):
task = kanban_db.get_task(conn, task_id)
assignee = "cli-auto"
if (
not task_id
or (board, task_id) in active
or not assignee.startswith(EXTERNAL_PREFIX)
or str(_task_value(task, "status", "")) != "ready"
or (OWNED_WORKSPACES_ONLY and not _owns_local_workspace(task))
or (eligible is not None and not eligible(board, task))
):
continue
try:

19
services/hermes/scripts/execution_pool_askpass.sh
View File

@ -1,19 +0,0 @@
#!/usr/bin/env sh
set -eu
case "${1:-}" in
*Username*)
if [ -s "${HERMES_SCM_USERNAME_FILE}" ]; then
tr -d '\r\n' <"${HERMES_SCM_USERNAME_FILE}"
else
printf '%s' hermes-automation
fi
printf '\n'
;;
*Password*)
test -s "${HERMES_SCM_PASSWORD_FILE}"
tr -d '\r\n' <"${HERMES_SCM_PASSWORD_FILE}"
printf '\n'
;;
*) exit 1 ;;
esac

163
services/hermes/scripts/execution_pool_client.py
View File

@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Ordinal-local signing boundary for a model-facing Hermes worker."""
"""Ordinal-local signing and SCM mediation boundary for a model worker."""
from __future__ import annotations
@ -12,8 +12,10 @@ from http.server import BaseHTTPRequestHandler
from pathlib import Path
from typing import Any
import cli_lane_goal
from execution_pool_protocol import (
MAX_WIRE_BYTES,
PROTOCOL_VERSION,
BoundedHTTPServer,
ProtocolError,
canonical_json,
@ -22,49 +24,80 @@ from execution_pool_protocol import (
sign_envelope,
verify_envelope,
)
from execution_pool_scm import Boundary as SCMBoundary
KEY_PATH = Path(os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/pool-access/execution-pool-key"))
KEY_PATH = Path(
os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/pool-access/execution-pool-key")
)
COORDINATOR = os.environ.get(
"HERMES_EXECUTION_POOL_URL", "http://hermes-execution-pool.hermes.svc.cluster.local:9007"
"HERMES_EXECUTION_POOL_URL",
"http://hermes-execution-pool.hermes.svc.cluster.local:9007",
).rstrip("/")
ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
PORT = int(os.environ.get("HERMES_EXECUTION_CLIENT_PORT", "9009"))
RESULT_FIELDS = frozenset(
{"status", "summary", "changed_files", "tests_run", "artifacts", "findings", "blockers"}
)
def _binding(value: dict[str, Any]) -> dict[str, Any]:
return {name: value[name] for name in (
"board", "task_id", "run_id", "worker_ordinal", "attempt"
)}
return {
name: value[name]
for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt")
}
def _validate_result(payload: Any) -> dict[str, Any]:
if not isinstance(payload, dict):
raise ProtocolError("terminal result payload must be an object")
structured = payload.get("structured")
if not isinstance(structured, dict) or set(structured) != RESULT_FIELDS:
raise ProtocolError("terminal result fields do not match the reviewed schema")
if structured.get("status") not in cli_lane_goal.RESULT_STATUSES:
raise ProtocolError("terminal result status is invalid")
if not isinstance(structured.get("summary"), str) or not structured["summary"].strip():
raise ProtocolError("terminal result summary is required")
for name in RESULT_FIELDS - {"status", "summary"}:
value = structured.get(name)
if not isinstance(value, list) or any(not isinstance(item, str) for item in value):
raise ProtocolError(f"terminal result {name} must be a text list")
return payload
class ClientBoundary:
"""Hold the master key and fence local requests to the current ordinal run."""
"""Keep HMAC and SCM authority outside the model-facing container."""
def __init__(self, key: bytes):
def __init__(self, key: bytes, scm: SCMBoundary | None = None):
self.key = key
self.scm = scm or SCMBoundary(key)
self.current: dict[str, Any] | None = None
self.lock = threading.RLock()
def _post(self, path: str, envelope: dict[str, Any]) -> dict[str, Any]:
request = urllib.request.Request(
COORDINATOR + path, data=canonical_json(envelope), method="POST",
COORDINATOR + path,
data=canonical_json(envelope),
method="POST",
headers={"Content-Type": "application/json", "Cache-Control": "no-store"},
)
with urllib.request.urlopen(request, timeout=60) as response:
body = response.read(MAX_WIRE_BYTES + 1)
if len(body) > MAX_WIRE_BYTES:
raise ProtocolError("coordinator response exceeds the wire limit")
value = json.loads(body)
return verify_envelope(self.key, value)
return verify_envelope(self.key, json.loads(body))
def poll(self) -> dict[str, Any]:
poll_binding = {
"board": "", "task_id": "", "run_id": "",
"worker_ordinal": ORDINAL, "attempt": 0,
"board": "",
"task_id": "",
"run_id": "",
"worker_ordinal": ORDINAL,
"attempt": 0,
}
response = self._post(
"/v1/poll", sign_envelope(self.key, "poll", poll_binding, {"ready": True})
"/v1/poll",
sign_envelope(self.key, "poll", poll_binding, {"ready": True}),
)
with self.lock:
if response["kind"] == "ack":
@ -72,32 +105,61 @@ class ClientBoundary:
return {"assignment": None}
if response["kind"] != "assignment" or response["worker_ordinal"] != ORDINAL:
raise ProtocolError("coordinator returned a foreign assignment")
self.current = _binding(response)
return {"assignment": response}
checkout = self.scm.checkout(response)
self.current = response
assignment = {
**_binding(response),
"payload": response["payload"],
"workspace": checkout["workspace"],
"baseline_sha": checkout["baseline_sha"],
"protocol_version": PROTOCOL_VERSION,
}
return {"assignment": assignment}
def forward(self, kind: str, request: dict[str, Any]) -> dict[str, Any]:
supplied = request.get("binding")
def _current_for(self, supplied: Any) -> tuple[dict[str, Any], dict[str, Any]]:
if not isinstance(supplied, dict):
raise ProtocolError("local request binding must be an object")
if self.current is None or supplied != _binding(self.current):
raise ProtocolError("local request does not own the current assignment")
return self.current, dict(supplied)
def heartbeat(self, request: dict[str, Any]) -> dict[str, Any]:
payload = request.get("payload")
if not isinstance(supplied, dict) or not isinstance(payload, dict):
raise ProtocolError("local request binding and payload must be objects")
if not isinstance(payload, dict):
raise ProtocolError("heartbeat payload must be an object")
with self.lock:
if self.current is None or supplied != self.current:
raise ProtocolError("local request does not own the current assignment")
binding = dict(self.current)
response = self._post(
f"/v1/{kind}", sign_envelope(self.key, kind, binding, payload)
)
if response["kind"] != "ack" or _binding(response) != binding:
raise ProtocolError("coordinator acknowledgement binding changed")
if kind == "result" and response["payload"].get("accepted"):
with self.lock:
self.current = None
_assignment, binding = self._current_for(request.get("binding"))
response = self._post(
"/v1/heartbeat",
sign_envelope(self.key, "heartbeat", binding, payload),
)
if response["kind"] != "ack" or _binding(response) != binding:
raise ProtocolError("coordinator acknowledgement binding changed")
return {"ack": response["payload"]}
def finish(self, request: dict[str, Any]) -> dict[str, Any]:
payload = _validate_result(request.get("payload"))
with self.lock:
assignment, binding = self._current_for(request.get("binding"))
structured = payload["structured"]
if structured["status"] == "completed" and int(payload.get("returncode", 1)) == 0:
submission = self.scm.submit(assignment, request)
pull = str(submission.get("pull_request") or "")
if pull and pull not in structured["artifacts"]:
structured["artifacts"].append(pull)
response = self._post(
"/v1/result", sign_envelope(self.key, "result", binding, payload)
)
if response["kind"] != "ack" or _binding(response) != binding:
raise ProtocolError("coordinator acknowledgement binding changed")
if response["payload"].get("accepted"):
self.current = None
return {"ack": response["payload"], "structured": structured}
def handler_factory(boundary: ClientBoundary) -> type[BaseHTTPRequestHandler]:
class Handler(BaseHTTPRequestHandler):
server_version = "hermes-execution-client/1"
server_version = f"hermes-execution-mediator/{PROTOCOL_VERSION}"
def _reply(self, status: int, value: dict[str, Any]) -> None:
body = canonical_json(value)
@ -109,22 +171,31 @@ def handler_factory(boundary: ClientBoundary) -> type[BaseHTTPRequestHandler]:
self.wfile.write(body)
def do_GET(self) -> None: # noqa: N802
self._reply(200, {"ready": True}) if self.path == "/ready" else self._reply(404, {"error": "not found"})
value = {"ready": True, "protocol_version": PROTOCOL_VERSION}
self._reply(200, value) if self.path == "/ready" else self._reply(
404, {"error": "not found"}
)
def do_POST(self) -> None: # noqa: N802
try:
length = int(self.headers.get("Content-Length", "0"))
request = parse_wire(self.rfile.read(length)) if 0 < length <= MAX_WIRE_BYTES else None
if not request or set(request) - {"operation", "binding", "payload"}:
raise ProtocolError("invalid local client request")
request = (
parse_wire(self.rfile.read(length))
if 0 < length <= MAX_WIRE_BYTES
else None
)
allowed = {"operation", "binding", "payload", "title", "body"}
if not request or set(request) - allowed:
raise ProtocolError("invalid local mediator request")
operation = str(request.get("operation") or "")
if operation == "poll":
result = boundary.poll()
elif operation in {"heartbeat", "result"}:
result = boundary.forward(operation, request)
else:
raise ProtocolError("unsupported local client operation")
self._reply(200, result)
routes = {
"poll": boundary.poll,
"heartbeat": lambda: boundary.heartbeat(request),
"finish": lambda: boundary.finish(request),
}
if operation not in routes:
raise ProtocolError("unsupported local mediator operation")
self._reply(200, routes[operation]())
except (ProtocolError, OSError, ValueError, urllib.error.URLError) as error:
self._reply(409, {"error": str(error)[:2000]})
@ -139,10 +210,12 @@ def main() -> int:
raise SystemExit("HERMES_WORKER_ORDINAL must be 0, 1, or 2")
key = read_key(KEY_PATH)
BoundedHTTPServer(
("0.0.0.0", PORT), handler_factory(ClientBoundary(key)), max_workers=4
("0.0.0.0", PORT),
handler_factory(ClientBoundary(key)),
max_workers=4,
).serve_forever()
return 0
if __name__ == "__main__":
if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint
raise SystemExit(main())

284
services/hermes/scripts/execution_pool_coordinator.py
View File

@ -3,7 +3,6 @@
from __future__ import annotations
import argparse
import json
import os
import re
@ -16,33 +15,23 @@ from pathlib import Path
from typing import Any
import cli_lane_goal
import cli_lane_runner
import cli_lane_dispatch
from cli_lane_config import canonical_run_id
from execution_pool_project import (
ProjectPolicyError,
distributed_workspace_eligible,
resolve_assignment,
)
from execution_pool_protocol import (
MAX_ACTIVITY_BYTES,
MAX_WIRE_BYTES,
BoundedHTTPServer,
PoolStore,
ProtocolError,
canonical_json,
parse_wire,
read_key,
derive_ordinal_key,
envelope_ordinal,
sign_envelope,
verify_envelope,
)
DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data"))
STATE_ROOT = DATA_ROOT / "execution-pool"
KEY_PATH = Path(os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/runtime-access/execution-pool-key"))
PORT = int(os.environ.get("HERMES_EXECUTION_POOL_PORT", "9007"))
RETENTION_SECONDS = int(os.environ.get("HERMES_EXECUTION_POOL_RETENTION_SECONDS", "1209600"))
ATLAS_REPO = re.compile(r"^https://scm\.bstein\.dev/atlas/[A-Za-z0-9_.-]+\.git$")
BRANCH = re.compile(r"^(?:feature|fix|chore|docs|test|refactor)/[A-Za-z0-9][A-Za-z0-9._/-]{0,119}$")
DEFAULT_REPO = os.environ.get(
"HERMES_EXECUTION_DEFAULT_REPO_URL",
"https://scm.bstein.dev/atlas/titan-iac.git",
)
DEFAULT_BASE_BRANCH = os.environ.get("HERMES_EXECUTION_DEFAULT_BASE_BRANCH", "main")
REDACTIONS = (
re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/-]{12,}"),
re.compile(
@ -59,43 +48,21 @@ def _task_value(task: Any, name: str, default: Any = None) -> Any:
return getattr(task, name, default)
def resolve_scm(task: Any) -> tuple[str, str, str]:
"""Resolve reviewed SCM metadata without executing in a legacy worktree."""
raw = str(_task_value(task, "workspace_path", "") or "")
if raw:
raise RuntimeError(
"task owns a legacy local workspace; preserve or commit its tracked and "
"untracked files before distributed assignment"
)
remote = str(_task_value(task, "repo_url", "") or DEFAULT_REPO)
if not ATLAS_REPO.fullmatch(remote) or "@" in remote:
raise RuntimeError("task origin is outside the reviewed Atlas SCM boundary")
task_id = str(_task_value(task, "id", "task"))
branch = str(_task_value(task, "branch_name", "") or "")
if not branch:
branch = f"feature/hermes-{re.sub(r'[^A-Za-z0-9._-]', '-', task_id)[:80]}"
if not BRANCH.fullmatch(branch) or ".." in branch or "//" in branch:
raise RuntimeError("task feature branch is outside the SCM branch policy")
base_branch = str(
_task_value(task, "base_branch", "") or DEFAULT_BASE_BRANCH
)
if (
not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,119}", base_branch)
or ".." in base_branch
or "//" in base_branch
):
raise RuntimeError("repository default branch is outside the SCM policy")
return remote, branch, base_branch
def resolve_scm(task: Any, board: str = "titan-iac") -> tuple[str, str, str]:
"""Resolve SCM only through the canonical board/project registry."""
return resolve_assignment(board, task)
def assignment_payload(kanban_db: Any, connection: Any, task: Any) -> dict[str, Any]:
def assignment_payload(
kanban_db: Any, connection: Any, task: Any, board: str = "titan-iac"
) -> dict[str, Any]:
context = kanban_db.build_worker_context(connection, str(_task_value(task, "id")))
if not isinstance(context, str):
context = json.dumps(context, default=str, sort_keys=True)
encoded = context.encode("utf-8")
if len(encoded) > 32 * 1024:
raise RuntimeError("Kanban worker context exceeds the 32KiB assignment limit")
repo_url, branch, base_branch = resolve_scm(task)
repo_url, branch, base_branch = resolve_scm(task, board)
runtime = int(_task_value(task, "max_runtime_seconds", 0) or 12 * 60 * 60)
runtime = max(60, min(runtime, 12 * 60 * 60))
return {
@ -171,10 +138,22 @@ class Coordinator:
"""Own all transitions between durable assignments and Hermes Kanban."""
def __init__(self, key: bytes, store: PoolStore):
self.key = key
self.master_key = key
self.store = store
self._kanban_lock = threading.RLock()
def _key(self, envelope: dict[str, Any]) -> bytes:
return derive_ordinal_key(self.master_key, envelope_ordinal(envelope))
def _verify(self, envelope: dict[str, Any], kind: str) -> dict[str, Any]:
return verify_envelope(self._key(envelope), envelope, expected_kind=kind)
def _sign(
self, kind: str, binding: dict[str, Any], payload: dict[str, Any]
) -> dict[str, Any]:
key = derive_ordinal_key(self.master_key, int(binding["worker_ordinal"]))
return sign_envelope(key, kind, binding, payload)
@staticmethod
def _binding(record: dict[str, Any]) -> dict[str, Any]:
return {name: record[name] for name in (
@ -182,20 +161,18 @@ class Coordinator:
)}
def poll(self, envelope: dict[str, Any]) -> dict[str, Any]:
verified = verify_envelope(self.key, envelope, expected_kind="poll")
verified = self._verify(envelope, "poll")
record = self.store.offer(int(verified["worker_ordinal"]))
if record is None:
binding = {**self._binding({
"board": "", "task_id": "", "run_id": "",
"worker_ordinal": verified["worker_ordinal"], "attempt": 0,
})}
return sign_envelope(self.key, "ack", binding, {"assignment": None})
return sign_envelope(
self.key, "assignment", self._binding(record), record["payload"]
)
return self._sign("ack", binding, {"assignment": None})
return self._sign("assignment", self._binding(record), record["payload"])
def heartbeat(self, envelope: dict[str, Any]) -> dict[str, Any]:
verified = verify_envelope(self.key, envelope, expected_kind="heartbeat")
verified = self._verify(envelope, "heartbeat")
accepted, duplicate = self.store.heartbeat(verified)
from hermes_cli import kanban_db
@ -204,10 +181,13 @@ class Coordinator:
with self._kanban_lock, kanban_db.scoped_current_board(verified["board"]):
connection = kanban_db.connect(board=verified["board"])
try:
run_id = canonical_run_id(verified["run_id"])
if run_id is None:
raise ProtocolError("worker run ID is not a canonical SQLite run")
alive = kanban_db.heartbeat_worker(
connection, verified["task_id"],
note=note or "distributed worker active",
expected_run_id=verified["run_id"],
expected_run_id=run_id,
)
if not alive:
raise ProtocolError("Kanban run no longer owns this worker")
@ -215,18 +195,18 @@ class Coordinator:
_append_activity(kanban_db, verified)
finally:
connection.close()
return sign_envelope(
self.key, "ack", self._binding(verified),
return self._sign(
"ack", self._binding(verified),
{"accepted": accepted, "duplicate": duplicate},
)
def result(self, envelope: dict[str, Any]) -> dict[str, Any]:
verified = verify_envelope(self.key, envelope, expected_kind="result")
verified = self._verify(envelope, "result")
record, duplicate = self.store.accept_result(verified)
if record.get("state") == "result":
self.finalize(record)
return sign_envelope(
self.key, "ack", self._binding(verified),
return self._sign(
"ack", self._binding(verified),
{"accepted": True, "duplicate": duplicate},
)
@ -235,6 +215,10 @@ class Coordinator:
from hermes_cli import kanban_db
binding = self._binding(record)
run_id = canonical_run_id(binding["run_id"])
if run_id is None:
self.store.finalize(binding, "stale")
return
payload = record.get("result")
if not isinstance(payload, dict):
raise ProtocolError("result payload must be an object")
@ -245,7 +229,10 @@ class Coordinator:
connection = kanban_db.connect(board=binding["board"])
try:
task = kanban_db.get_task(connection, binding["task_id"])
if task is None or _task_value(task, "current_run_id") != binding["run_id"]:
current_run_id = canonical_run_id(
_task_value(task, "current_run_id", None)
) if task is not None else None
if current_run_id is None or current_run_id != run_id:
self.store.finalize(binding, "stale")
return
_append_terminal_activity(kanban_db, record)
@ -273,7 +260,7 @@ class Coordinator:
connection, binding["task_id"],
result=json.dumps(structured, sort_keys=True),
summary=str(structured.get("summary") or "Completed"),
metadata=metadata, expected_run_id=binding["run_id"],
metadata=metadata, expected_run_id=run_id,
)
else:
reason = problem or "; ".join(map(str, structured.get("blockers", [])))
@ -281,7 +268,7 @@ class Coordinator:
changed = kanban_db.block_task(
connection, binding["task_id"], reason=reason,
kind="transient" if payload.get("capacity_failure") else "capability",
expected_run_id=binding["run_id"],
expected_run_id=run_id,
)
self.store.finalize(binding, "finalized" if changed else "stale")
finally:
@ -294,6 +281,37 @@ class Coordinator:
except (OSError, sqlite3.Error) as error:
print(f"result recovery deferred: {error}", file=sys.stderr, flush=True)
def expire_leases(self) -> None:
"""Fence dead attempts; after bounded retries, surface and release the run."""
from hermes_cli import kanban_db
for record in self.store.expire_leases():
if record.get("state") != "lease_failed":
continue
binding = self._binding(record)
run_id = canonical_run_id(binding["run_id"])
if run_id is None:
self.store.finalize(binding, "stale")
continue
with self._kanban_lock, kanban_db.scoped_current_board(binding["board"]):
connection = kanban_db.connect(board=binding["board"])
try:
changed = kanban_db.block_task(
connection,
binding["task_id"],
reason=(
"Distributed worker lease expired after "
f"{binding['attempt']} fenced attempts"
),
kind="transient",
expected_run_id=run_id,
)
self.store.finalize(
binding, "finalized" if changed else "stale"
)
finally:
connection.close()
def reconcile(self) -> None:
"""Recover the narrow claim/assignment crash gaps without double execution."""
from hermes_cli import kanban_db
@ -309,7 +327,12 @@ class Coordinator:
connection = kanban_db.connect(board=record["board"])
try:
task = kanban_db.get_task(connection, record["task_id"])
current = str(_task_value(task, "current_run_id", "") or "") if task else ""
current_run_id = (
canonical_run_id(_task_value(task, "current_run_id", None))
if task
else None
)
current = str(current_run_id or "")
status = str(_task_value(task, "status", "") or "") if task else ""
finally:
connection.close()
@ -321,7 +344,7 @@ class Coordinator:
if not ordinals:
return
for raw_board in kanban_db.list_boards(include_archived=False):
board = cli_lane_runner._board_slug(raw_board)
board = cli_lane_dispatch._board_slug(raw_board)
if not board:
continue
with kanban_db.scoped_current_board(board):
@ -330,7 +353,10 @@ class Coordinator:
tasks = kanban_db.list_tasks(connection)
for task in tasks:
task_id = str(_task_value(task, "id", "") or "")
run_id = str(_task_value(task, "current_run_id", "") or "")
raw_run_id = canonical_run_id(
_task_value(task, "current_run_id", None)
)
run_id = str(raw_run_id or "")
assignee = str(_task_value(task, "assignee", "") or "")
if (
not ordinals
@ -338,12 +364,12 @@ class Coordinator:
or not run_id
or str(_task_value(task, "status", "")) != "running"
or not assignee.startswith("cli-")
or not distributed_workspace_eligible(task)
or (board, task_id, run_id) in active_runs
):
continue
try:
payload = assignment_payload(kanban_db, connection, task)
kanban_db.set_branch_name(connection, task_id, payload["branch"])
payload = assignment_payload(kanban_db, connection, task, board)
except Exception as error:
kanban_db.block_task(
connection, task_id,
@ -351,7 +377,7 @@ class Coordinator:
"Distributed assignment recovery failed: "
f"{type(error).__name__}: {error}"
),
kind="capability", expected_run_id=run_id,
kind="capability", expected_run_id=raw_run_id,
)
continue
ordinal = ordinals.pop(0)
@ -373,9 +399,13 @@ class Coordinator:
ordinals = self.store.available_ordinals()
if not ordinals:
return
claimed = cli_lane_runner.claim_ready(set(), len(ordinals))
claimed = cli_lane_dispatch.claim_ready(
set(), len(ordinals),
lambda _board, task: distributed_workspace_eligible(task),
)
for ordinal, (board, task_id) in zip(ordinals, claimed, strict=False):
run_id = ""
database_run_id: int | None = None
try:
with kanban_db.scoped_current_board(board):
connection = kanban_db.connect(board=board)
@ -383,9 +413,29 @@ class Coordinator:
task = kanban_db.get_task(connection, task_id)
if task is None:
continue
run_id = str(_task_value(task, "current_run_id", "") or "")
payload = assignment_payload(kanban_db, connection, task)
kanban_db.set_branch_name(connection, task_id, payload["branch"])
if not distributed_workspace_eligible(task):
raw_run = canonical_run_id(
_task_value(task, "current_run_id", None)
)
if raw_run is not None:
kanban_db.block_task(
connection, task_id,
reason=(
"Distributed claim fenced because an existing "
"workspace is owned by the local lane"
),
kind="capability",
expected_run_id=raw_run,
)
continue
raw_run = canonical_run_id(
_task_value(task, "current_run_id", None)
)
if raw_run is None:
raise ProjectPolicyError("claimed task has no canonical run ID")
database_run_id = raw_run
run_id = str(raw_run)
payload = assignment_payload(kanban_db, connection, task, board)
finally:
connection.close()
binding = {
@ -400,92 +450,24 @@ class Coordinator:
try:
kanban_db.block_task(
connection, task_id, reason=reason, kind="capability",
expected_run_id=run_id or None,
expected_run_id=database_run_id,
)
finally:
connection.close()
def handler_factory(coordinator: Coordinator) -> type[BaseHTTPRequestHandler]:
class Handler(BaseHTTPRequestHandler):
server_version = "hermes-execution-pool/1"
"""Compatibility export for tests and the mounted coordinator entry point."""
from execution_pool_server import handler_factory as factory
def _reply(self, status: int, value: dict[str, Any]) -> None:
body = canonical_json(value)
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(body)
def do_GET(self) -> None: # noqa: N802
if self.path != "/ready":
self._reply(404, {"error": "not found"})
return
try:
coordinator.store.available_ordinals()
self._reply(200, {"ready": True})
except (OSError, sqlite3.Error):
self._reply(503, {"ready": False})
def do_POST(self) -> None: # noqa: N802
try:
length = int(self.headers.get("Content-Length", "0"))
if length <= 0 or length > MAX_WIRE_BYTES:
raise ProtocolError("invalid content length")
envelope = parse_wire(self.rfile.read(length))
routes = {
"/v1/poll": coordinator.poll,
"/v1/heartbeat": coordinator.heartbeat,
"/v1/result": coordinator.result,
}
if self.path not in routes:
self._reply(404, {"error": "not found"})
return
self._reply(200, routes[self.path](envelope))
except ProtocolError as error:
self._reply(409, {"error": str(error)})
except Exception as error:
print(f"pool request failed: {type(error).__name__}: {error}", file=sys.stderr, flush=True)
self._reply(503, {"error": "coordinator unavailable"})
def log_message(self, _format: str, *_arguments: Any) -> None:
return
return Handler
return factory(coordinator)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--once", action="store_true")
args = parser.parse_args()
key = read_key(KEY_PATH)
store = PoolStore(STATE_ROOT / "assignments.db")
coordinator = Coordinator(key, store)
coordinator.recover_results()
coordinator.reconcile()
coordinator.dispatch()
if args.once:
return 0
server = BoundedHTTPServer(
("0.0.0.0", PORT), handler_factory(coordinator), max_workers=8
)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
while True:
for operation in (
coordinator.recover_results,
coordinator.reconcile,
coordinator.dispatch,
lambda: store.garbage_collect(RETENTION_SECONDS),
):
try:
operation()
except Exception as error:
print(f"pool maintenance deferred: {type(error).__name__}: {error}", file=sys.stderr, flush=True)
time.sleep(5)
from execution_pool_server import run
return run(Coordinator)
if __name__ == "__main__":
if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint
raise SystemExit(main())

152
services/hermes/scripts/execution_pool_project.py Normal file
View File

@ -0,0 +1,152 @@
#!/usr/bin/env python3
"""Canonical Atlas project and Git-ref policy for distributed execution."""
from __future__ import annotations
import json
import os
import re
import stat
import subprocess
from pathlib import Path
from typing import Any
DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data"))
PROJECT_ROOT = DATA_ROOT / "workspace/projects"
BOARD_ROOT = DATA_ROOT / "kanban/boards"
ATLAS_REPO = re.compile(
r"https://scm\.bstein\.dev/atlas/(?P<repo>[A-Za-z0-9][A-Za-z0-9_.-]{0,99})\.git\Z"
)
SAFE_PREFIXES = frozenset(
{"feature", "fix", "chore", "docs", "test", "refactor", "wt", "review", "hermes", "handoff"}
)
IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z")
GIT = "/usr/bin/git"
MAX_BOARD_BYTES = 64 * 1024
class ProjectPolicyError(ValueError):
"""Canonical project metadata or a requested ref failed closed."""
def _task_value(task: Any, name: str, default: Any = None) -> Any:
return getattr(task, name, default)
def _run_git(workdir: Path, *arguments: str) -> str:
completed = subprocess.run(
[GIT, "-C", str(workdir), *arguments],
check=False,
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=30,
env={
"HOME": "/nonexistent",
"PATH": "/usr/bin:/bin",
"GIT_CONFIG_NOSYSTEM": "1",
"GIT_TERMINAL_PROMPT": "0",
},
)
if completed.returncode:
raise ProjectPolicyError("canonical project Git metadata is unavailable")
return completed.stdout.strip()
def validate_branch(value: object, *, feature: bool) -> str:
"""Validate a complete local branch name with Git and a reviewed namespace."""
if not isinstance(value, str) or not value or len(value) > 200:
raise ProjectPolicyError("branch name exceeds the safe limit")
if not value.isascii() or value.startswith("-"):
raise ProjectPolicyError("branch name must be canonical ASCII")
prefix = value.split("/", 1)[0]
if feature and ("/" not in value or prefix not in SAFE_PREFIXES):
raise ProjectPolicyError("task branch is outside the reviewed namespace")
completed = subprocess.run(
[GIT, "check-ref-format", f"refs/heads/{value}"],
check=False,
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=10,
env={"PATH": "/usr/bin:/bin", "GIT_CONFIG_NOSYSTEM": "1"},
)
if completed.returncode:
raise ProjectPolicyError("branch name is not a valid Git ref")
return value
def _read_board(board: str) -> dict[str, Any]:
if not IDENTIFIER.fullmatch(board):
raise ProjectPolicyError("board slug is invalid")
path = BOARD_ROOT / board / "board.json"
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
try:
info = os.fstat(descriptor)
if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= MAX_BOARD_BYTES:
raise ProjectPolicyError("board registry entry is not a bounded regular file")
raw = os.read(descriptor, MAX_BOARD_BYTES + 1)
finally:
os.close(descriptor)
try:
value = json.loads(raw)
except (UnicodeError, json.JSONDecodeError) as error:
raise ProjectPolicyError("board registry entry is malformed") from error
if not isinstance(value, dict) or value.get("slug") != board or value.get("archived") is True:
raise ProjectPolicyError("board registry identity is unavailable or archived")
return value
def resolve_project(board: str) -> tuple[str, str, Path]:
"""Resolve repo and base exclusively from the canonical board registry.
A board may be created before its primary checkout. In that case the
Atlas repository naming contract and ``main`` are the registry defaults;
once a checkout exists, its credential-free origin and remote HEAD must
agree with that identity. This keeps a missing checkout from silently
routing work to some other project's repository.
"""
entry = _read_board(board)
raw_workdir = entry.get("default_workdir")
if not isinstance(raw_workdir, str) or not Path(raw_workdir).is_absolute():
raise ProjectPolicyError("board default_workdir is missing")
workdir = Path(raw_workdir).resolve(strict=False)
project_root = PROJECT_ROOT.resolve(strict=True)
try:
workdir.relative_to(project_root)
except ValueError as error:
raise ProjectPolicyError("board workdir is outside the Atlas project registry") from error
remote = f"https://scm.bstein.dev/atlas/{board}.git"
if not ATLAS_REPO.fullmatch(remote):
raise ProjectPolicyError("board repository identity is invalid")
if not workdir.exists():
return remote, "main", workdir
if not workdir.is_dir():
raise ProjectPolicyError("board checkout is not a directory")
checkout_remote = _run_git(workdir, "remote", "get-url", "origin")
if "@" in checkout_remote or checkout_remote != remote:
raise ProjectPolicyError("board origin disagrees with the Atlas registry")
try:
base = _run_git(workdir, "symbolic-ref", "--short", "refs/remotes/origin/HEAD")
if not base.startswith("origin/"):
raise ProjectPolicyError("origin HEAD is not canonical")
base = base.removeprefix("origin/")
except ProjectPolicyError:
base = "main"
return remote, validate_branch(base, feature=False), workdir
def resolve_assignment(board: str, task: Any) -> tuple[str, str, str]:
"""Resolve the exact repo/base and safe task branch for one board task."""
repo, base, _workdir = resolve_project(board)
task_id = str(_task_value(task, "id", "") or "")
if not IDENTIFIER.fullmatch(task_id):
raise ProjectPolicyError("task identity is invalid")
branch = str(_task_value(task, "branch_name", "") or f"wt/{task_id}")
return repo, validate_branch(branch, feature=True), base
def distributed_workspace_eligible(task: Any) -> bool:
"""Only pathless tasks migrate; an existing worktree stays with its local owner."""
return not str(_task_value(task, "workspace_path", "") or "").strip()

105
services/hermes/scripts/execution_pool_protocol.py
View File

@ -22,6 +22,7 @@ MAX_WIRE_BYTES = 64 * 1024
MAX_ACTIVITY_BYTES = 12 * 1024
MAX_CLOCK_SKEW = 30
MAX_ENVELOPE_LIFETIME = 300
PROTOCOL_VERSION = 2
IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
KINDS = frozenset({"poll", "assignment", "heartbeat", "result", "ack"})
@ -64,9 +65,7 @@ class BoundedHTTPServer(ThreadingHTTPServer):
def canonical_json(value: Any) -> bytes:
"""Encode one value deterministically for digests and signatures."""
return json.dumps(
value, ensure_ascii=False, separators=(",", ":"), sort_keys=True
).encode("utf-8")
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
def atomic_json(path: Path, value: dict[str, Any], mode: int = 0o600) -> None:
@ -95,6 +94,24 @@ def payload_digest(payload: Any) -> str:
return hashlib.sha256(canonical_json(payload)).hexdigest()
def derive_ordinal_key(master: bytes, ordinal: int) -> bytes:
"""Derive one cryptographically isolated worker authority from the pool root."""
if ordinal not in range(3) or not 32 <= len(master) <= 4096:
raise ProtocolError("pool key derivation input is invalid")
context = f"hermes-execution-pool-v2:worker:{ordinal}".encode()
return hmac.new(master, context, hashlib.sha256).hexdigest().encode()
def envelope_ordinal(envelope: Any) -> int:
"""Read only the bounded ordinal needed to select a verification key."""
if not isinstance(envelope, dict):
raise ProtocolError("invalid pool message")
value = envelope.get("worker_ordinal")
if type(value) is not int or value not in range(3):
raise ProtocolError("worker binding is outside the pool")
return value
def read_key(path: Path) -> bytes:
"""Read a private regular file without following a final symlink."""
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
@ -115,7 +132,9 @@ def read_key(path: Path) -> bytes:
def _identifier(name: str, value: Any, *, allow_empty: bool = False) -> str:
text = str(value or "")
if not isinstance(value, str):
raise ProtocolError(f"invalid {name}")
text = value
if allow_empty and not text:
return text
if not IDENTIFIER.fullmatch(text):
@ -137,7 +156,7 @@ def sign_envelope(
current = int(time.time()) if now is None else int(now)
lifetime = max(1, min(int(lifetime), MAX_ENVELOPE_LIFETIME))
envelope = {
"version": 1,
"version": PROTOCOL_VERSION,
"kind": kind,
"board": str(binding.get("board") or ""),
"task_id": str(binding.get("task_id") or ""),
@ -169,14 +188,14 @@ def verify_envelope(
"""Verify structure, lifetime, digest, and HMAC before using a message."""
if not isinstance(envelope, dict) or len(canonical_json(envelope)) > MAX_WIRE_BYTES:
raise ProtocolError("invalid or oversized pool message")
required = {
"version", "kind", "board", "task_id", "run_id", "worker_ordinal",
"attempt", "delivery_id", "issued_at", "expires_at", "payload_digest",
"payload", "signature",
}
if set(envelope) != required or envelope.get("version") != 1:
raise ProtocolError("pool message fields do not match version 1")
kind = str(envelope["kind"])
required = {"version", "kind", "board", "task_id", "run_id", "worker_ordinal", "attempt", "delivery_id", "issued_at", "expires_at", "payload_digest", "payload", "signature"}
if set(envelope) != required or envelope.get("version") != PROTOCOL_VERSION:
raise ProtocolError(
f"pool message fields do not match version {PROTOCOL_VERSION}"
)
kind = envelope["kind"]
if not isinstance(kind, str):
raise ProtocolError("unexpected message kind")
if kind not in KINDS or (expected_kind and kind != expected_kind):
raise ProtocolError("unexpected message kind")
empty_binding = kind in {"poll", "ack"}
@ -184,13 +203,10 @@ def verify_envelope(
_identifier("task_id", envelope["task_id"], allow_empty=empty_binding)
_identifier("run_id", envelope["run_id"], allow_empty=empty_binding)
_identifier("delivery_id", envelope["delivery_id"])
try:
ordinal = int(envelope["worker_ordinal"])
attempt = int(envelope["attempt"])
issued = int(envelope["issued_at"])
expires = int(envelope["expires_at"])
except (TypeError, ValueError) as error:
raise ProtocolError("invalid numeric binding") from error
numeric = tuple(envelope[name] for name in ("worker_ordinal", "attempt", "issued_at", "expires_at"))
if any(type(value) is not int for value in numeric):
raise ProtocolError("invalid numeric binding")
ordinal, attempt, issued, expires = numeric
if ordinal not in range(3) or attempt < 0:
raise ProtocolError("worker binding is outside the pool")
current = int(time.time()) if now is None else int(now)
@ -374,6 +390,8 @@ class PoolStore:
row = self._matching(connection, envelope)
if row["state"] not in {"assigned", "running"}:
raise ProtocolError("assignment is no longer running")
if row["state"] == "running" and float(row["lease_until"]) < now:
raise ProtocolError("assignment lease expired")
if duplicate and duplicate[0] != delivery_digest:
raise ProtocolError("delivery identifier was reused")
if not duplicate:
@ -401,6 +419,8 @@ class PoolStore:
return self._record(row) or {}, True
if row["state"] not in {"assigned", "running"}:
raise ProtocolError("assignment cannot accept a result")
if row["state"] == "running" and float(row["lease_until"]) < now:
raise ProtocolError("assignment lease expired")
connection.execute(
"""UPDATE assignments SET state='result',result_digest=?,result_json=?,updated_at=?
WHERE board=? AND task_id=? AND run_id=?""",
@ -416,18 +436,53 @@ class PoolStore:
def pending_results(self) -> list[dict[str, Any]]:
with self._connect() as connection:
rows = connection.execute(
"SELECT * FROM assignments WHERE state='result' ORDER BY updated_at"
).fetchall()
rows = connection.execute("SELECT * FROM assignments WHERE state='result' ORDER BY updated_at").fetchall()
return [self._record(row) or {} for row in rows]
def expire_leases(
self, *, now: float | None = None, max_attempts: int = 3
) -> list[dict[str, Any]]:
"""Fence expired attempts and re-offer or terminally release their ordinals."""
current = time.time() if now is None else float(now)
maximum = max(1, min(int(max_attempts), 10))
changed: list[dict[str, Any]] = []
with self._lock, self._connect() as connection:
connection.execute("BEGIN IMMEDIATE")
rows = connection.execute(
"""SELECT * FROM assignments WHERE state='running'
AND lease_until > 0 AND lease_until < ? ORDER BY updated_at""",
(current,),
).fetchall()
for row in rows:
if int(row["attempt"]) >= maximum:
state, attempt = "lease_failed", int(row["attempt"])
else:
state, attempt = "assigned", int(row["attempt"]) + 1
connection.execute(
"""UPDATE assignments SET state=?,attempt=?,lease_until=0,
last_heartbeat=0,updated_at=? WHERE board=? AND task_id=? AND run_id=?
AND attempt=? AND state='running'""",
(
state, attempt, current, row["board"], row["task_id"],
row["run_id"], row["attempt"],
),
)
updated = connection.execute(
"SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?",
(row["board"], row["task_id"], row["run_id"]),
).fetchone()
if updated is not None:
changed.append(self._record(updated) or {})
connection.commit()
return changed
def finalize(self, binding: dict[str, Any], state: str) -> None:
if state not in {"finalized", "stale"}:
raise ProtocolError("invalid terminal assignment state")
with self._lock, self._connect() as connection:
connection.execute(
"UPDATE assignments SET state=?,updated_at=? WHERE board=? AND task_id=? AND run_id=?",
(state, time.time(), binding["board"], binding["task_id"], binding["run_id"]),
"UPDATE assignments SET state=?,updated_at=? WHERE board=? AND task_id=? AND run_id=? AND worker_ordinal=? AND attempt=?",
(state, time.time(), *(binding[name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt"))),
)
def garbage_collect(self, retention_seconds: int) -> int:

523
services/hermes/scripts/execution_pool_scm.py
View File

@ -1,153 +1,97 @@
#!/usr/bin/env python3
"""Ordinal-scoped Atlas SCM boundary; the model container never gets its token."""
"""Assignment-bound Git gate routed exclusively through the PR14 SCM broker."""
from __future__ import annotations
import configparser
import json
import os
import re
import shutil
import stat
import subprocess
import threading
import urllib.error
import urllib.parse
import urllib.request
import uuid
from http.server import BaseHTTPRequestHandler
from pathlib import Path
from typing import Any
from execution_pool_protocol import (
MAX_WIRE_BYTES,
BoundedHTTPServer,
ProtocolError,
parse_wire,
read_key,
verify_envelope,
)
import scm_broker_client
from execution_pool_project import ATLAS_REPO, validate_branch
from execution_pool_protocol import ProtocolError, atomic_json, verify_envelope
WORKSPACE_ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
SCM_ROOT = Path(os.environ.get("HERMES_SCM_STATE_ROOT", "/scm-state"))
KEY_PATH = Path(os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/pool-access/execution-pool-key"))
TOKEN_PATH = Path(os.environ.get("HERMES_GITEA_TOKEN_FILE", "/vault/secrets/gitea-token"))
USERNAME_PATH = Path(os.environ.get("HERMES_GITEA_USERNAME_FILE", "/vault/secrets/gitea-username"))
ASKPASS = os.environ.get(
"HERMES_GITEA_ASKPASS", "/opt/coordinator/execution_pool_askpass.sh"
)
ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
PORT = int(os.environ.get("HERMES_SCM_BOUNDARY_PORT", "9008"))
REPO = re.compile(r"^https://scm\.bstein\.dev/atlas/([A-Za-z0-9_.-]+)\.git$")
BRANCH = re.compile(r"^(?:feature|fix|chore|docs|test|refactor)/[A-Za-z0-9][A-Za-z0-9._/-]{0,119}$")
MAX_BUNDLE_BYTES = 128 * 1024 * 1024
BROKER_ORIGIN = scm_broker_client.BROKER_ORIGIN.rstrip("/")
MAX_STATUS_BYTES = 4 * 1024 * 1024
def _private_text(path: Path) -> str:
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
try:
value = os.read(descriptor, 65537).decode("utf-8").strip()
finally:
os.close(descriptor)
if not value or len(value) > 65536:
raise ProtocolError(f"SCM credential is missing or invalid: {path.name}")
return value
def _git_env(authenticated: bool) -> dict[str, str]:
"""Expose credential paths only to explicit boundary-owned network calls."""
sensitive = {
"GIT_ASKPASS", "SSH_ASKPASS", "HERMES_SCM_PASSWORD_FILE",
"HERMES_SCM_USERNAME_FILE",
def _git_environment() -> dict[str, str]:
"""Run Git without credentials, prompts, ambient config, or hook execution."""
return {
"HOME": "/nonexistent",
"PATH": "/usr/bin:/bin",
"GIT_CONFIG_NOSYSTEM": "1",
"GIT_TERMINAL_PROMPT": "0",
}
environment = {
name: value for name, value in os.environ.items()
if name not in sensitive and not name.startswith("GIT_CONFIG_")
}
environment["GIT_TERMINAL_PROMPT"] = "0"
if authenticated:
environment.update(
{
"GIT_ASKPASS": ASKPASS,
"HERMES_SCM_PASSWORD_FILE": str(TOKEN_PATH),
"HERMES_SCM_USERNAME_FILE": str(USERNAME_PATH),
"GIT_CONFIG_COUNT": "1",
"GIT_CONFIG_KEY_0": (
"url.http://gitea.gitea.svc.cluster.local:3000/.insteadOf"
),
"GIT_CONFIG_VALUE_0": "https://scm.bstein.dev/",
}
)
return environment
def _run(
*arguments: str,
cwd: Path | None = None,
timeout: int = 300,
authenticated: bool = False,
) -> str:
def _run(*arguments: str, cwd: Path | None = None, timeout: int = 300) -> str:
command = [
"/usr/bin/git",
"-c",
"core.hooksPath=/dev/null",
"-c",
"core.fsmonitor=false",
*arguments,
]
completed = subprocess.run(
list(arguments), cwd=cwd, env=_git_env(authenticated), stdin=subprocess.DEVNULL,
text=True, capture_output=True, timeout=timeout, check=False,
command,
cwd=cwd,
env=_git_environment(),
stdin=subprocess.DEVNULL,
text=True,
capture_output=True,
timeout=timeout,
check=False,
)
if completed.returncode:
message = (completed.stderr or completed.stdout or "SCM operation failed")[-2000:]
raise RuntimeError(message.strip())
if len(completed.stdout.encode()) > MAX_STATUS_BYTES:
raise ProtocolError("SCM command output exceeds the safe limit")
return completed.stdout.strip()
def _regular_text(path: Path, limit: int, encoding: str = "utf-8") -> str:
"""Read bounded model-controlled metadata without following or blocking."""
flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0)
try:
descriptor = os.open(path, flags)
except OSError as error:
raise ProtocolError(f"workspace metadata is unavailable: {path.name}") from error
try:
info = os.fstat(descriptor)
if not stat.S_ISREG(info.st_mode) or info.st_size > limit:
raise ProtocolError(f"workspace metadata is invalid: {path.name}")
raw = os.read(descriptor, limit + 1)
if len(raw) > limit:
raise ProtocolError(f"workspace metadata is oversized: {path.name}")
return raw.decode(encoding)
except UnicodeError as error:
raise ProtocolError(f"workspace metadata is malformed: {path.name}") from error
finally:
os.close(descriptor)
def _binding(envelope: dict[str, Any]) -> tuple[dict[str, Any], str, str, str]:
if envelope["kind"] != "assignment" or int(envelope["worker_ordinal"]) != ORDINAL:
raise ProtocolError("assignment does not belong to this worker ordinal")
payload = envelope["payload"]
payload = envelope.get("payload")
if not isinstance(payload, dict):
raise ProtocolError("assignment payload must be an object")
repo = str(payload.get("repo_url") or "")
branch = str(payload.get("branch") or "")
base_branch = str(payload.get("base_branch") or "main")
match = REPO.fullmatch(repo)
if (
not match
or not BRANCH.fullmatch(branch)
or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,119}", base_branch)
or ".." in branch
or "//" in branch
or ".." in base_branch
or "//" in base_branch
):
raise ProtocolError("assignment is outside the Atlas SCM policy")
run_name = f"{envelope['task_id']}-{envelope['run_id']}"
return payload, repo, branch, match.group(1) + ":" + run_name
match = ATLAS_REPO.fullmatch(repo)
if not match:
raise ProtocolError("assignment repository is outside Atlas")
try:
branch = validate_branch(payload.get("branch"), feature=True)
base = validate_branch(payload.get("base_branch"), feature=False)
except ValueError as error:
raise ProtocolError(str(error)) from error
return payload, match.group("repo"), branch, base
def _broker_repo(repo: str) -> str:
return f"{BROKER_ORIGIN}/git/atlas/{repo}.git"
def workspace_path(envelope: dict[str, Any]) -> Path:
"""Derive a contained path; no caller-provided filesystem path is accepted."""
parts = (str(envelope["board"]), str(envelope["task_id"]), str(envelope["run_id"]))
if any(not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", part) for part in parts):
"""Derive a private ordinal path; no caller-provided path is accepted."""
parts = tuple(str(envelope[name]) for name in ("board", "task_id", "run_id"))
identifier = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z")
if any(not identifier.fullmatch(part) for part in parts):
raise ProtocolError("assignment path binding is invalid")
if WORKSPACE_ROOT.is_symlink():
raise ProtocolError("workspace root must not be a symlink")
workspace_root = WORKSPACE_ROOT.resolve()
root = WORKSPACE_ROOT / "runs"
if root.is_symlink():
@ -156,112 +100,77 @@ def workspace_path(envelope: dict[str, Any]) -> Path:
root = root.resolve()
root.relative_to(workspace_root)
candidate = root.joinpath(*parts)
if candidate.is_symlink():
raise ProtocolError("workspace must not be a symlink")
current = root
for part in parts[:-1]:
current = current / part
current /= part
if current.is_symlink():
raise ProtocolError("workspace parent must not be a symlink")
current.mkdir(mode=0o700, exist_ok=True)
if current.is_symlink():
raise ProtocolError("workspace parent must not be a symlink")
try:
candidate.resolve(strict=False).relative_to(root)
except ValueError as error:
raise ProtocolError("workspace escaped its ordinal root") from error
if candidate.is_symlink():
raise ProtocolError("workspace must not be a symlink")
candidate.resolve(strict=False).relative_to(root)
return candidate
def _workspace_identity(destination: Path, repo: str, branch: str) -> str:
"""Read identity as data; never execute Git in a model-controlled checkout."""
git_dir = destination / ".git"
if git_dir.is_symlink() or not git_dir.is_dir():
raise ProtocolError("workspace Git metadata must be a private directory")
raw_config = _regular_text(git_dir / "config", 64 * 1024)
parser = configparser.ConfigParser(interpolation=None, strict=True)
def _regular_text(path: Path, maximum: int) -> str:
descriptor = os.open(
path, os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0)
)
try:
parser.read_string(raw_config)
origin = parser.get('remote "origin"', "url")
except (configparser.Error, KeyError, UnicodeError) as error:
raise ProtocolError("workspace Git config cannot prove its origin") from error
if origin != repo:
raise ProtocolError("durable workspace origin does not match assignment")
head = _regular_text(git_dir / "HEAD", 4096).strip()
if head != f"ref: refs/heads/{branch}":
raise ProtocolError("durable workspace branch does not match assignment")
ref_path = git_dir / "refs/heads" / Path(branch)
if ref_path.is_symlink():
raise ProtocolError("workspace branch ref must not be a symlink")
if ref_path.exists():
commit = _regular_text(ref_path, 128, "ascii").strip()
else:
packed = _regular_text(git_dir / "packed-refs", 1024 * 1024, "ascii")
matches = [
line.split(" ", 1)[0]
for line in packed.splitlines()
if line.endswith(f" refs/heads/{branch}")
]
if len(matches) != 1:
raise ProtocolError("workspace branch ref is unavailable")
commit = matches[0]
if not re.fullmatch(r"[0-9a-f]{40,64}", commit):
raise ProtocolError("workspace branch ref is invalid")
return commit
info = os.fstat(descriptor)
if not stat.S_ISREG(info.st_mode) or info.st_size > maximum:
raise ProtocolError("private SCM state is invalid")
raw = os.read(descriptor, maximum + 1)
finally:
os.close(descriptor)
try:
return raw.decode().strip()
except UnicodeError as error:
raise ProtocolError("private SCM state is malformed") from error
def _private_repo(envelope: dict[str, Any]) -> Path:
def _state_path(envelope: dict[str, Any]) -> Path:
if SCM_ROOT.is_symlink():
raise ProtocolError("private SCM root must not be a symlink")
root = SCM_ROOT.resolve()
root.mkdir(mode=0o700, parents=True, exist_ok=True)
name = f"{envelope['board']}-{envelope['task_id']}-{envelope['run_id']}.git"
path = root / name
path.resolve(strict=False).relative_to(root)
name = "-".join(
str(envelope[name]) for name in ("board", "task_id", "run_id")
)
path = root / f"{name}.json"
if path.is_symlink():
raise ProtocolError("private SCM state must not be a symlink")
path.resolve(strict=False).relative_to(root)
return path
def _copy_bundle(source: Path, destination: Path) -> None:
"""Copy a bounded regular bundle into boundary-private storage."""
source_fd = os.open(
source, os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0)
)
temporary = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.tmp")
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
target_fd = -1
try:
info = os.fstat(source_fd)
if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= MAX_BUNDLE_BYTES:
raise ProtocolError("submission bundle is empty, oversized, or not regular")
target_fd = os.open(
temporary,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
0o600,
)
remaining = info.st_size
while remaining:
chunk = os.read(source_fd, min(1024 * 1024, remaining))
if not chunk:
raise ProtocolError("submission bundle ended early")
view = memoryview(chunk)
while view:
view = view[os.write(target_fd, view) :]
remaining -= len(chunk)
os.fsync(target_fd)
except Exception:
temporary.unlink(missing_ok=True)
raise
finally:
os.close(source_fd)
if target_fd >= 0:
os.close(target_fd)
try:
os.replace(temporary, destination)
finally:
temporary.unlink(missing_ok=True)
def _workspace_identity(destination: Path, repo: str, branch: str) -> str:
"""Validate the checkout using bounded Git plumbing with all hooks disabled."""
if destination.is_symlink() or not (destination / ".git").is_dir():
raise ProtocolError("workspace Git metadata is unavailable")
ref_path = destination / ".git/refs/heads"
for part in branch.split("/"):
ref_path /= part
if ref_path.is_symlink():
raise ProtocolError("workspace branch ref must not be a symlink")
origin = _run("remote", "get-url", "origin", cwd=destination)
if origin != f"https://scm.bstein.dev/atlas/{repo}.git":
raise ProtocolError("workspace origin does not match assignment")
broker = _run("remote", "get-url", "hermes-broker", cwd=destination)
if broker != _broker_repo(repo):
raise ProtocolError("workspace broker remote does not match assignment")
current = _run("symbolic-ref", "--short", "HEAD", cwd=destination)
if current != branch:
raise ProtocolError("workspace branch does not match assignment")
head = _run("rev-parse", "--verify", "HEAD", cwd=destination)
if not re.fullmatch(r"[0-9a-f]{40,64}", head):
raise ProtocolError("workspace HEAD is invalid")
return head
class Boundary:
"""The only process allowed to turn model output into an SCM/result handoff."""
def __init__(self, key: bytes):
self.key = key
self.lock = threading.RLock()
@ -270,181 +179,103 @@ class Boundary:
return verify_envelope(self.key, raw, expected_kind="assignment")
def checkout(self, envelope: dict[str, Any]) -> dict[str, Any]:
"""Clone/fetch exactly one assigned branch into its private durable path."""
payload, repo, branch, _ = _binding(envelope)
base_branch = str(payload.get("base_branch") or "main")
_payload, repo, branch, base = _binding(envelope)
destination = workspace_path(envelope)
state_path = _state_path(envelope)
with self.lock:
if (destination / ".git").exists():
return {
"workspace": str(destination), "preserved_dirty_state": True,
"baseline_sha": _workspace_identity(destination, repo, branch),
}
_workspace_identity(destination, repo, branch)
state = json.loads(_regular_text(state_path, 16 * 1024))
baseline = state.get("baseline_sha") if isinstance(state, dict) else None
if not isinstance(baseline, str) or not re.fullmatch(
r"[0-9a-f]{40,64}", baseline
):
raise ProtocolError("private SCM baseline is unavailable")
return {"workspace": str(destination), "baseline_sha": baseline}
if destination.exists() and any(destination.iterdir()):
raise ProtocolError("workspace is non-empty and unmanaged")
destination.parent.mkdir(parents=True, exist_ok=True)
broker = _broker_repo(repo)
try:
_run(
"git", "clone", "--single-branch", "--branch", branch,
"--no-tags", repo, str(destination), timeout=900,
authenticated=True,
"clone", "--single-branch", "--branch", branch, "--no-tags",
broker, str(destination), timeout=900,
)
except RuntimeError as error:
if destination.exists() and any(destination.iterdir()):
raise RuntimeError(
"assigned branch clone failed and left state for review"
raise ProtocolError(
"failed branch checkout left unmanaged workspace state"
) from error
if destination.exists():
destination.rmdir()
_run(
"git", "clone", "--single-branch", "--branch", base_branch,
"--no-tags", repo, str(destination), timeout=900,
authenticated=True,
"clone", "--single-branch", "--branch", base, "--no-tags",
broker, str(destination), timeout=900,
)
_run("git", "checkout", "-b", branch, cwd=destination)
_run("git", "config", "user.name", "Hermes Execution Worker", cwd=destination)
_run("git", "config", "user.email", "hermes@bstein.dev", cwd=destination)
return {
"workspace": str(destination), "preserved_dirty_state": False,
"baseline_sha": _run("git", "rev-parse", "HEAD", cwd=destination),
}
def submit(self, envelope: dict[str, Any], request: dict[str, Any]) -> dict[str, Any]:
"""Push only the assignment branch and create/reuse its draft pull request."""
_payload, repo, branch, repo_binding = _binding(envelope)
repo_name, _ = repo_binding.split(":", 1)
destination = workspace_path(envelope)
title = str(request.get("title") or f"Hermes task {envelope['task_id']}").strip()[:240]
body = str(request.get("body") or "Automated draft from Hermes execution pool.")[:12000]
if not title or not (destination / ".git").exists():
raise ProtocolError("submission workspace or title is invalid")
with self.lock:
_workspace_identity(destination, repo, branch)
private = _private_repo(envelope)
bundle = private.with_suffix(".bundle")
_copy_bundle(destination / ".git/hermes-submit.bundle", bundle)
if not private.exists():
_run("git", "init", "--bare", str(private))
_run("git", "--git-dir", str(private), "remote", "add", "origin", repo)
else:
_run("git", "--git-dir", str(private), "remote", "set-url", "origin", repo)
try:
_run(
"git", "--git-dir", str(private), "fetch", "origin",
f"+refs/heads/{branch}:refs/remotes/origin/{branch}", timeout=900,
authenticated=True,
)
except RuntimeError as error:
if "couldn't find remote ref" not in str(error).lower():
raise
_run("checkout", "-b", branch, cwd=destination)
_run(
"git", "--git-dir", str(private), "fetch", str(bundle),
"HEAD:refs/pool/candidate", timeout=900,
"remote", "set-url", "origin",
f"https://scm.bstein.dev/atlas/{repo}.git", cwd=destination,
)
_run(
"git", "--git-dir", str(private), "push", "origin",
f"refs/pool/candidate:refs/heads/{branch}", timeout=900,
authenticated=True,
_run("remote", "add", "hermes-broker", broker, cwd=destination)
_run("config", "user.name", "Hermes Execution Worker", cwd=destination)
_run("config", "user.email", "hermes@bstein.dev", cwd=destination)
baseline = _workspace_identity(destination, repo, branch)
atomic_json(
state_path,
{"baseline_sha": baseline, "repo": repo, "branch": branch},
)
pull = self._draft_pull(
repo_name, branch, str(_payload.get("base_branch") or "main"), title, body
)
bundle.unlink(missing_ok=True)
shutil.rmtree(private)
return {"workspace": str(destination), "branch": branch, "pull_request": pull}
return {"workspace": str(destination), "baseline_sha": baseline}
@staticmethod
def _api(path: str, data: dict[str, Any] | None = None) -> Any:
token = _private_text(TOKEN_PATH)
request = urllib.request.Request(
"http://gitea.gitea.svc.cluster.local:3000" + path,
data=json.dumps(data, separators=(",", ":")).encode() if data else None,
method="POST" if data else "GET",
headers={
"Authorization": f"token {token}", "Accept": "application/json",
"Content-Type": "application/json", "User-Agent": "hermes-scm-boundary/1",
},
def _draft(repo: str, branch: str, base: str, head: str, title: str, body: str) -> str:
query = urllib.parse.urlencode(
{"state": "open", "head": f"atlas:{branch}", "limit": 10}
)
existing = json.loads(
scm_broker_client.read(f"/api/v1/repos/atlas/{repo}/pulls?{query}")
)
with urllib.request.urlopen(request, timeout=30) as response:
body = response.read(1024 * 1024 + 1)
if len(body) > 1024 * 1024:
raise ProtocolError("Gitea response exceeds the SCM boundary limit")
value = json.loads(body)
if not isinstance(value, (dict, list)):
raise ProtocolError("Gitea response has an invalid shape")
return value
def _draft_pull(
self, repo: str, branch: str, base_branch: str, title: str, body: str
) -> str:
query = urllib.parse.urlencode({"state": "open", "head": f"atlas:{branch}", "limit": 10})
existing = self._api(f"/api/v1/repos/atlas/{repo}/pulls?{query}")
if isinstance(existing, list) and existing:
return str(existing[0].get("html_url") or "")
created = self._api(
f"/api/v1/repos/atlas/{repo}/pulls",
{
"base": base_branch,
"head": branch, "title": title, "body": body, "draft": True,
},
created = json.loads(
scm_broker_client.create_draft(
repo,
base=base,
head=branch,
head_sha=head,
title=title,
body=body,
)
)
return str(created.get("html_url") or "")
def garbage_collect(self, envelope: dict[str, Any]) -> dict[str, Any]:
"""Online requests never delete model-controlled or private SCM state."""
return {"removed": False, "eligible": False, "workspace": str(workspace_path(envelope))}
def handler_factory(boundary: Boundary) -> type[BaseHTTPRequestHandler]:
class Handler(BaseHTTPRequestHandler):
server_version = "hermes-scm-boundary/1"
def _reply(self, status: int, value: dict[str, Any]) -> None:
body = json.dumps(value, separators=(",", ":"), sort_keys=True).encode()
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(body)
def do_GET(self) -> None: # noqa: N802
self._reply(200, {"ready": True}) if self.path == "/ready" else self._reply(404, {"error": "not found"})
def do_POST(self) -> None: # noqa: N802
try:
length = int(self.headers.get("Content-Length", "0"))
request = parse_wire(self.rfile.read(length)) if 0 < length <= MAX_WIRE_BYTES else None
if not request or set(request) - {"operation", "assignment", "title", "body"}:
raise ProtocolError("invalid SCM request")
envelope = boundary.verify(request.get("assignment"))
operations = {
"checkout": lambda: boundary.checkout(envelope),
"submit": lambda: boundary.submit(envelope, request),
"gc-check": lambda: boundary.garbage_collect(envelope),
}
operation = str(request.get("operation") or "")
if operation not in operations:
raise ProtocolError("unsupported SCM operation")
self._reply(200, operations[operation]())
except (ProtocolError, RuntimeError, OSError, urllib.error.URLError) as error:
self._reply(409, {"error": str(error)[:2000]})
def log_message(self, _format: str, *_arguments: Any) -> None:
return
return Handler
def main() -> int:
if ORDINAL not in range(3):
raise SystemExit("HERMES_WORKER_ORDINAL must be 0, 1, or 2")
key = read_key(KEY_PATH)
_private_text(TOKEN_PATH)
server = BoundedHTTPServer(
("0.0.0.0", PORT), handler_factory(Boundary(key)), max_workers=4
)
server.serve_forever()
return 0
if __name__ == "__main__":
raise SystemExit(main())
def submit(self, envelope: dict[str, Any], request: dict[str, Any]) -> dict[str, Any]:
"""Enforce clean/committed state, broker push, and reviewed draft creation."""
_payload, repo, branch, base = _binding(envelope)
destination = workspace_path(envelope)
title = str(request.get("title") or f"Hermes task {envelope['task_id']}").strip()
body = str(request.get("body") or "Automated Hermes draft.")
if not title or len(title.encode()) > 512 or len(body.encode()) > 32 * 1024:
raise ProtocolError("pull-request metadata exceeds the safe limit")
with self.lock:
head = _workspace_identity(destination, repo, branch)
state = json.loads(_regular_text(_state_path(envelope), 16 * 1024))
baseline = state.get("baseline_sha") if isinstance(state, dict) else ""
if not isinstance(baseline, str) or not re.fullmatch(
r"[0-9a-f]{40,64}", baseline
):
raise ProtocolError("private SCM baseline is unavailable")
status = _run(
"status", "--porcelain=v1", "--untracked-files=all", cwd=destination
)
if status:
raise ProtocolError("workspace has uncommitted or untracked files")
ahead = int(_run("rev-list", "--count", f"{baseline}..{head}", cwd=destination))
if ahead <= 0:
return {"workspace": str(destination), "branch": branch, "pull_request": ""}
_run(
"push", "hermes-broker", f"HEAD:refs/heads/{branch}",
cwd=destination, timeout=900,
)
pull = self._draft(repo, branch, base, head, title, body)
return {"workspace": str(destination), "branch": branch, "pull_request": pull}

130
services/hermes/scripts/execution_pool_server.py Normal file
View File

@ -0,0 +1,130 @@
#!/usr/bin/env python3
"""Versioned HTTP and maintenance loop for the execution-pool coordinator."""
from __future__ import annotations
import argparse
import os
import sqlite3
import sys
import threading
import time
from http.server import BaseHTTPRequestHandler
from pathlib import Path
from typing import Any
from execution_pool_protocol import (
MAX_WIRE_BYTES,
PROTOCOL_VERSION,
BoundedHTTPServer,
PoolStore,
ProtocolError,
canonical_json,
parse_wire,
read_key,
)
DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data"))
STATE_ROOT = DATA_ROOT / "execution-pool"
KEY_PATH = Path(
os.environ.get(
"HERMES_EXECUTION_POOL_KEY_FILE", "/runtime-access/execution-pool-key"
)
)
PORT = int(os.environ.get("HERMES_EXECUTION_POOL_PORT", "9007"))
RETENTION_SECONDS = int(
os.environ.get("HERMES_EXECUTION_POOL_RETENTION_SECONDS", "1209600")
)
def handler_factory(coordinator: Any) -> type[BaseHTTPRequestHandler]:
class Handler(BaseHTTPRequestHandler):
server_version = f"hermes-execution-pool/{PROTOCOL_VERSION}"
def _reply(self, status: int, value: dict[str, Any]) -> None:
body = canonical_json(value)
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(body)
def do_GET(self) -> None: # noqa: N802
if self.path != "/ready":
self._reply(404, {"error": "not found"})
return
try:
coordinator.store.available_ordinals()
self._reply(200, {"ready": True, "version": PROTOCOL_VERSION})
except (OSError, sqlite3.Error):
self._reply(503, {"ready": False, "version": PROTOCOL_VERSION})
def do_POST(self) -> None: # noqa: N802
try:
length = int(self.headers.get("Content-Length", "0"))
if length <= 0 or length > MAX_WIRE_BYTES:
raise ProtocolError("invalid content length")
envelope = parse_wire(self.rfile.read(length))
routes = {
"/v1/poll": coordinator.poll,
"/v1/heartbeat": coordinator.heartbeat,
"/v1/result": coordinator.result,
}
if self.path not in routes:
self._reply(404, {"error": "not found"})
return
self._reply(200, routes[self.path](envelope))
except ProtocolError as error:
self._reply(409, {"error": str(error)})
except Exception as error:
print(
f"pool request failed: {type(error).__name__}: {error}",
file=sys.stderr,
flush=True,
)
self._reply(503, {"error": "coordinator unavailable"})
def log_message(self, _format: str, *_arguments: Any) -> None:
return
return Handler
def run(coordinator_type: type[Any]) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--once", action="store_true")
args = parser.parse_args()
store = PoolStore(STATE_ROOT / "assignments.db")
coordinator = coordinator_type(read_key(KEY_PATH), store)
for operation in (
coordinator.expire_leases,
coordinator.recover_results,
coordinator.reconcile,
coordinator.dispatch,
):
operation()
if args.once:
return 0
server = BoundedHTTPServer(
("0.0.0.0", PORT), handler_factory(coordinator), max_workers=8
)
threading.Thread(target=server.serve_forever, daemon=True).start()
while True:
for operation in (
coordinator.expire_leases,
coordinator.recover_results,
coordinator.reconcile,
coordinator.dispatch,
lambda: store.garbage_collect(RETENTION_SECONDS),
):
try:
operation()
except Exception as error:
print(
f"pool maintenance deferred: {type(error).__name__}: {error}",
file=sys.stderr,
flush=True,
)
time.sleep(5)

98
services/hermes/scripts/execution_pool_worker.py
View File

@ -25,9 +25,11 @@ from execution_pool_protocol import (
ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
CLIENT = os.environ.get("HERMES_EXECUTION_CLIENT_URL", "http://127.0.0.1:9009").rstrip("/")
SCM = os.environ.get("HERMES_SCM_BOUNDARY_URL", "http://127.0.0.1:9008").rstrip("/")
ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
CLIENT = os.environ.get(
"HERMES_EXECUTION_CLIENT_URL",
f"http://hermes-execution-mediator-{ORDINAL}.hermes.svc.cluster.local:9009",
).rstrip("/")
NODE = os.environ.get("HERMES_WORKER_NODE", "unknown")[:128]
RUN_PART = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
RETENTION_SECONDS = int(os.environ.get("HERMES_WORKER_RETENTION_SECONDS", "1209600"))
@ -59,19 +61,15 @@ def _poll() -> dict[str, Any] | None:
assignment = _client("poll").get("assignment")
if assignment is None:
return None
if not isinstance(assignment, dict) or int(assignment["worker_ordinal"]) != ORDINAL:
if (
not isinstance(assignment, dict)
or int(assignment["worker_ordinal"]) != ORDINAL
or int(assignment.get("protocol_version", 0)) != 2
):
raise ProtocolError("local boundary returned a foreign assignment")
return assignment
def _scm(operation: str, assignment: dict[str, Any], **values: Any) -> dict[str, Any]:
request = {"operation": operation, "assignment": assignment, **values}
response = _post(f"{SCM}/v1/scm", request, timeout=900)
if response.get("error"):
raise RuntimeError(str(response["error"]))
return response
def _binding(assignment: dict[str, Any]) -> dict[str, Any]:
return {name: assignment[name] for name in (
"board", "task_id", "run_id", "worker_ordinal", "attempt"
@ -82,7 +80,10 @@ def _state_path(assignment: dict[str, Any]) -> Path:
parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id"))
if any(not RUN_PART.fullmatch(part) for part in parts):
raise ProtocolError("assignment state path is invalid")
root = (ROOT / "session-state").resolve()
configured_root = ROOT / "session-state"
if configured_root.is_symlink():
raise ProtocolError("assignment state root must not be a symlink")
root = configured_root.resolve()
path = root.joinpath(*parts).with_suffix(".json")
path.parent.mkdir(parents=True, exist_ok=True)
if path.is_symlink() or any(parent.is_symlink() for parent in path.parents if parent != root.parent):
@ -261,13 +262,12 @@ def execute(assignment: dict[str, Any]) -> None:
if not isinstance(payload, dict):
raise ProtocolError("assignment payload is invalid")
_bind_provider_sessions(assignment)
checkout = _scm("checkout", assignment)
workspace = Path(str(checkout.get("workspace") or "")).resolve(strict=True)
workspace = Path(str(assignment.get("workspace") or "")).resolve(strict=True)
workspace.relative_to((ROOT / "runs").resolve())
state_file = _state_path(assignment)
state = cli_lane_runner.load_json(state_file)
state.update({**binding, "node": NODE, "workspace": str(workspace)})
state.setdefault("baseline_sha", str(checkout.get("baseline_sha") or ""))
state.setdefault("baseline_sha", str(assignment.get("baseline_sha") or ""))
atomic_json(state_file, state)
log_path = state_file.with_suffix(".log")
offset = 0
@ -339,21 +339,7 @@ def execute(assignment: dict[str, Any]) -> None:
baseline = str(state.get("baseline_sha") or "")
if not re.fullmatch(r"[0-9a-f]{40,64}", baseline):
raise RuntimeError("durable SCM baseline is missing or invalid")
ahead = int(_git(workspace, "rev-list", "--count", f"{baseline}..HEAD") or "0")
if ahead:
fresh = _refresh_assignment(binding)
bundle = workspace / ".git/hermes-submit.bundle"
bundle.unlink(missing_ok=True)
_git(workspace, "bundle", "create", str(bundle), "HEAD")
submission = _scm(
"submit", fresh,
title=str(structured.get("summary") or f"Hermes task {binding['task_id']}")[:240],
body=json.dumps(structured, indent=2, sort_keys=True)[:12000],
)
pull = str(submission.get("pull_request") or "")
if pull:
structured["artifacts"].append(pull)
bundle.unlink(missing_ok=True)
_refresh_assignment(binding)
structured = _bounded_result(structured)
activity, _ = _read_activity(log_path, offset)
result_payload = {
@ -368,13 +354,53 @@ def execute(assignment: dict[str, Any]) -> None:
},
"final_activity": activity,
}
response = _client("result", binding=binding, payload=result_payload)
response = _client(
"finish",
binding=binding,
payload=result_payload,
title=str(structured.get("summary") or f"Hermes task {binding['task_id']}")[:240],
body=json.dumps(structured, indent=2, sort_keys=True)[:12000],
)
if not response.get("ack", {}).get("accepted"):
raise ProtocolError("coordinator did not accept the terminal result")
state["terminal_at"] = time.time()
atomic_json(state_file, state)
def report_exception(assignment: dict[str, Any], error: Exception) -> bool:
"""Surface one worker exception as an exact-run transient result."""
binding = _binding(assignment)
detail = f"{type(error).__name__}: {error}"[:1500]
structured = {
"status": "blocked",
"summary": "Distributed worker failed before a safe handoff.",
"changed_files": [],
"tests_run": [],
"artifacts": [],
"findings": [],
"blockers": [detail],
}
payload = {
"structured": structured,
"returncode": 1,
# This is an execution-infrastructure failure, not an objective capability
# verdict. The exact-run result releases the ordinal for another task.
"capacity_failure": True,
"node": NODE,
"route": {},
"provider_sessions": {},
"final_activity": detail,
}
try:
return bool(
_client("finish", binding=binding, payload=payload).get("ack", {}).get(
"accepted"
)
)
except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError):
return False
def readiness() -> None:
if ORDINAL not in range(3):
raise ProtocolError("worker ordinal must be 0, 1, or 2")
@ -397,6 +423,7 @@ def readiness() -> None:
def main() -> int:
readiness()
while True:
assignment = None
try:
garbage_collect()
assignment = _poll()
@ -405,9 +432,14 @@ def main() -> int:
continue
execute(assignment)
except (OSError, RuntimeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as error:
print(f"worker {ORDINAL} deferred: {type(error).__name__}: {error}", flush=True)
surfaced = assignment is not None and report_exception(assignment, error)
print(
f"worker {ORDINAL} {'surfaced' if surfaced else 'deferred'}: "
f"{type(error).__name__}: {error}",
flush=True,
)
time.sleep(10)
if __name__ == "__main__":
if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint
raise SystemExit(main())

143
services/hermes/scripts/stage_runtime_access.py
View File

@ -4,8 +4,11 @@
from __future__ import annotations
import argparse
import hashlib
import hmac
import json
import os
import stat
import uuid
from pathlib import Path
@ -14,24 +17,42 @@ VAULT_ROOT = Path("/vault/secrets")
RUNTIME_ROOT = Path("/runtime-access")
PERSISTENT_HOME = Path("/opt/data/home")
WORKER_ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
PROVIDER_ACCESS_ROOT = Path(
os.environ.get("HERMES_PROVIDER_ACCESS_ROOT", "/provider-access")
)
POOL_ACCESS_ROOT = Path(os.environ.get("HERMES_POOL_ACCESS_ROOT", "/pool-access"))
SCM_ACCESS_ROOT = Path(os.environ.get("HERMES_SCM_ACCESS_ROOT", "/scm-access"))
OWNER_UID = 10000
OWNER_GID = 10000
def _owned_directory(path: Path) -> None:
"""Create one private directory owned by the unprivileged Hermes user."""
if path.is_symlink():
raise RuntimeError(f"private directory {path.name} must not be a symlink")
path.mkdir(mode=0o700, parents=True, exist_ok=True)
if path.is_symlink() or not path.is_dir():
raise RuntimeError(f"private directory {path.name} is invalid")
path.chmod(0o700)
os.chown(path, OWNER_UID, OWNER_GID)
def _copy_secret(source_name: str, destination: Path) -> str:
"""Copy a non-empty Vault projection without logging its value."""
value = (VAULT_ROOT / source_name).read_text(encoding="utf-8").strip()
def _read_secret(source_name: str) -> str:
path = VAULT_ROOT / source_name
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
try:
info = os.fstat(descriptor)
if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= 1024 * 1024:
raise RuntimeError(f"Vault projection {source_name} is invalid")
raw = os.read(descriptor, info.st_size + 1)
finally:
os.close(descriptor)
value = raw.decode("utf-8").strip()
if not value:
raise RuntimeError(f"Vault projection {source_name} is empty")
return value
def _write_secret(destination: Path, value: str) -> None:
temporary = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.tmp")
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(temporary, flags, 0o600)
@ -48,26 +69,65 @@ def _copy_secret(source_name: str, destination: Path) -> str:
os.replace(temporary, destination)
finally:
temporary.unlink(missing_ok=True)
def _copy_secret(source_name: str, destination: Path) -> str:
"""Copy a non-empty Vault projection without logging its value."""
value = _read_secret(source_name)
_write_secret(destination, value)
return value
def _validate_json_value(value: str, source_name: str, required: tuple[str, ...]) -> None:
"""Validate a provider credential shape without exposing its value."""
try:
document = json.loads(value)
except json.JSONDecodeError as error:
raise RuntimeError(f"credential {source_name} is not valid JSON") from error
current = document
for key in required:
if not isinstance(current, dict) or key not in current:
raise RuntimeError(f"credential {source_name} has an invalid shape")
current = current[key]
if not isinstance(current, str) or not current:
raise RuntimeError(f"credential {source_name} has an empty credential")
def _validated_json(source_name: str, destination: Path, required: tuple[str, ...]) -> None:
"""Stage one credential document after checking its expected shape."""
value = _copy_secret(source_name, destination)
try:
document = json.loads(value)
except json.JSONDecodeError:
_validate_json_value(value, source_name, required)
except RuntimeError:
destination.unlink(missing_ok=True)
raise
current = document
for key in required:
if not isinstance(current, dict) or key not in current:
destination.unlink(missing_ok=True)
raise RuntimeError(f"Vault projection {source_name} has an invalid shape")
current = current[key]
if not isinstance(current, str) or not current:
destination.unlink(missing_ok=True)
raise RuntimeError(f"Vault projection {source_name} has an empty credential")
def _bootstrap_json(
source_name: str, destination: Path, required: tuple[str, ...]
) -> None:
"""Initialize one ordinal credential once, then preserve provider refreshes."""
if destination.is_symlink():
raise RuntimeError(f"durable credential {destination.name} must not be a symlink")
try:
descriptor = os.open(
destination, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
)
except FileNotFoundError:
_validated_json(source_name, destination, required)
return
try:
info = os.fstat(descriptor)
if (
not stat.S_ISREG(info.st_mode)
or info.st_mode & 0o077
or not 0 < info.st_size <= 1024 * 1024
):
raise RuntimeError(f"durable credential {destination.name} is unsafe")
value = os.read(descriptor, info.st_size + 1).decode("utf-8").strip()
finally:
os.close(descriptor)
_validate_json_value(value, destination.name, required)
def _link_noncredential_state(
@ -190,36 +250,51 @@ def _durable_link(runtime: Path, durable: Path, name: str) -> None:
def stage_execution_worker() -> None:
"""Stage only subscription/channel auth and ordinal-private session state."""
"""Bootstrap and preserve ordinal-private subscription refresh ownership."""
provider_state = WORKER_ROOT / "provider-state"
for path in (
RUNTIME_ROOT,
RUNTIME_ROOT / "claude",
RUNTIME_ROOT / "codex",
POOL_ACCESS_ROOT,
SCM_ACCESS_ROOT,
provider_state,
PROVIDER_ACCESS_ROOT,
PROVIDER_ACCESS_ROOT / "claude",
PROVIDER_ACCESS_ROOT / "codex",
):
_owned_directory(path)
_copy_secret("execution-pool-key", POOL_ACCESS_ROOT / "execution-pool-key")
_copy_secret("gitea-token", SCM_ACCESS_ROOT / "gitea-token")
_copy_secret("gitea-username", SCM_ACCESS_ROOT / "gitea-username")
_validated_json(
"claude-credentials",
RUNTIME_ROOT / "claude" / ".credentials.json",
ordinal = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
if ordinal not in range(3):
raise RuntimeError("execution worker ordinal is invalid")
_bootstrap_json(
f"claude-credentials-{ordinal}",
PROVIDER_ACCESS_ROOT / "claude" / ".credentials.json",
("claudeAiOauth", "refreshToken"),
)
_validated_json(
"codex-auth",
RUNTIME_ROOT / "codex" / "auth.json",
_bootstrap_json(
f"codex-auth-{ordinal}",
PROVIDER_ACCESS_ROOT / "codex" / "auth.json",
("tokens", "refresh_token"),
)
_write_empty_auth_store()
def stage_execution_mediator() -> None:
"""Derive one ordinal HMAC authority in the isolated mediator Pod only."""
_owned_directory(POOL_ACCESS_ROOT)
ordinal = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
if ordinal not in range(3):
raise RuntimeError("execution mediator ordinal is invalid")
master = _read_secret("execution-pool-key").encode()
derived = hmac.new(
master,
f"hermes-execution-pool-v2:worker:{ordinal}".encode(),
hashlib.sha256,
).hexdigest()
_write_secret(POOL_ACCESS_ROOT / "execution-pool-key", derived)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("mode", choices=("agent", "chat", "triage", "execution-worker"))
parser.add_argument(
"mode",
choices=("agent", "chat", "triage", "execution-worker", "execution-mediator"),
)
args = parser.parse_args()
if args.mode == "agent":
stage_agent()
@ -227,8 +302,10 @@ def main() -> int:
stage_chat()
elif args.mode == "triage":
stage_triage()
else:
elif args.mode == "execution-worker":
stage_execution_worker()
else:
stage_execution_mediator()
print("Runtime access staged from Vault.", flush=True)
return 0

4
services/vault/scripts/vault_k8s_auth_configure.sh
View File

@ -224,9 +224,7 @@ write_policy_and_role "outline" "outline" "outline-vault" \
write_policy_and_role "planka" "planka" "planka-vault" \
"planka/* shared/postmark-relay" ""
write_policy_and_role "hermes-execution-worker" "hermes" "hermes-execution-worker" \
"hermes/agent-tokens hermes/developer-gitea" ""
write_policy_and_role "hermes-execution-credential-sync" "hermes" "hermes-execution-worker" \
"" "hermes/agent-tokens"
"hermes/agent-tokens" ""
write_policy_and_role "bstein-dev-home" "bstein-dev-home" "bstein-dev-home,bstein-dev-home-vault-sync" \
"portal/* shared/chat-ai-keys-runtime shared/portal-e2e-client shared/postmark-relay mailu/mailu-initial-account-secret shared/harbor-pull" ""
write_policy_and_role "gitea" "gitea" "gitea-vault" \

23
testing/quality_contract.json
View File

@ -20,6 +20,13 @@
"ci/scripts/supply_chain_report.py",
"services/mailu/scripts/mailu_sync.py",
"services/mailu/scripts/mailu_sync_listener.py",
"services/hermes/scripts/execution_pool_client.py",
"services/hermes/scripts/execution_pool_coordinator.py",
"services/hermes/scripts/execution_pool_project.py",
"services/hermes/scripts/execution_pool_protocol.py",
"services/hermes/scripts/execution_pool_scm.py",
"services/hermes/scripts/execution_pool_server.py",
"services/hermes/scripts/execution_pool_worker.py",
"testing/__init__.py",
"testing/quality_contract.py",
"testing/quality_docs.py",
@ -43,6 +50,13 @@
"services/comms/scripts/tests",
"services/mailu/scripts/mailu_sync.py",
"services/mailu/scripts/mailu_sync_listener.py",
"services/hermes/scripts/execution_pool_client.py",
"services/hermes/scripts/execution_pool_coordinator.py",
"services/hermes/scripts/execution_pool_project.py",
"services/hermes/scripts/execution_pool_protocol.py",
"services/hermes/scripts/execution_pool_scm.py",
"services/hermes/scripts/execution_pool_server.py",
"services/hermes/scripts/execution_pool_worker.py",
"testing/tests",
"testing"
],
@ -58,6 +72,7 @@
"coverage_sources": [
"ci/scripts",
"scripts.render.dashboards_render_atlas",
"services/hermes/scripts",
"services/mailu/scripts",
"testing"
],
@ -124,6 +139,7 @@
"ci/tests/**/*.py",
"scripts/tests/**/*.py",
"services/*/scripts/tests/**/*.py",
"services/hermes/scripts/execution_pool_*.py",
"services/mailu/scripts/mailu_sync.py",
"services/mailu/scripts/mailu_sync_listener.py"
],
@ -174,6 +190,13 @@
"ci/scripts/supply_chain_report.py",
"services/mailu/scripts/mailu_sync.py",
"services/mailu/scripts/mailu_sync_listener.py",
"services/hermes/scripts/execution_pool_client.py",
"services/hermes/scripts/execution_pool_coordinator.py",
"services/hermes/scripts/execution_pool_project.py",
"services/hermes/scripts/execution_pool_protocol.py",
"services/hermes/scripts/execution_pool_scm.py",
"services/hermes/scripts/execution_pool_server.py",
"services/hermes/scripts/execution_pool_worker.py",
"testing/quality_contract.py",
"testing/quality_docs.py",
"testing/quality_hygiene.py",

32
testing/tests/test_hermes_agent_access.py
View File

@ -78,15 +78,16 @@ def test_owner_agent_has_pinned_dedicated_node_ssh_access():
"secretProviderClass": "hermes-node-ssh-access"
}
reconciler = pod["containers"][0]["args"][0]
assert "cat /vault/secrets/node-ssh-public-key" in reconciler
assert "grep -qxF" in reconciler
assert "for user in atlas oceanus" in reconciler
assert "/host-etc/passwd" in reconciler
assert "chown \"${uid}:${gid}\"" in reconciler
host_passwd = next(
item for item in pod["volumes"] if item["name"] == "host-passwd"
)
assert host_passwd["hostPath"] == {"path": "/etc/passwd", "type": "File"}
assert "/opt/node-hardener/node_account_hardening.py" in reconciler
assert "--public-key-file /vault/secrets/node-ssh-public-key" in reconciler
assert "grep -qxF" not in reconciler
host_etc = next(item for item in pod["volumes"] if item["name"] == "host-etc")
assert host_etc["hostPath"] == {"path": "/etc", "type": "Directory"}
hardener = next(item for item in pod["volumes"] if item["name"] == "coordinator")
assert hardener["configMap"] == {
"name": "hermes-node-account-hardener",
"defaultMode": 0o555,
}
def test_owner_agent_tracks_no_ssh_identity_or_host_key_material():
@ -155,13 +156,16 @@ def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog():
rbac = [
item
for item in yaml.safe_load_all((HERMES / "agent-rbac.yaml").read_text())
for item in yaml.safe_load_all((HERMES / "rbac.yaml").read_text())
if item
]
binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding")
assert binding["subjects"] == [
{"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"}
]
bindings = [item for item in rbac if item["kind"] == "ClusterRoleBinding"]
assert bindings
assert all(
subject["name"] not in {"hermes-agent", "hermes-switchyard"}
for binding in bindings
for subject in binding["subjects"]
)
def test_switchyard_active_state_uses_a_relocatable_rwx_claim():

67
testing/tests/test_hermes_agent_security.py
View File

@ -182,21 +182,54 @@ def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces()
"ports": [{"protocol": "TCP", "port": 9010}],
},
]
assert isolation["spec"]["egress"] == [{}]
def test_owner_agent_has_cluster_admin_kubernetes_context():
config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text())
assert config["current-context"] == "atlas-owner"
assert config["contexts"][0]["context"]["namespace"] == "default"
rbac_path = HERMES / "agent-rbac.yaml"
documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item]
binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding")
assert binding["roleRef"] == {
"apiGroup": "rbac.authorization.k8s.io",
"kind": "ClusterRole",
"name": "cluster-admin",
}
assert binding["subjects"] == [
{"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"}
egress = isolation["spec"]["egress"]
assert {} not in egress
broker = next(
rule
for rule in egress
if rule.get("ports") == [{"protocol": "TCP", "port": 9081}]
)
assert broker["to"] == [
{
"namespaceSelector": {
"matchLabels": {"kubernetes.io/metadata.name": "hermes-scm"}
},
"podSelector": {"matchLabels": {"app": "hermes-scm-broker"}},
}
]
namespace_rule = next(
rule
for rule in egress
if rule.get("to", [{}])[0].get("namespaceSelector", {}).get(
"matchExpressions"
)
)
assert namespace_rule["to"][0]["namespaceSelector"]["matchExpressions"] == [
{
"key": "kubernetes.io/metadata.name",
"operator": "NotIn",
"values": ["gitea", "hermes-scm"],
}
]
public = next(
rule
for rule in egress
if rule.get("to", [{}])[0].get("ipBlock", {}).get("cidr") == "0.0.0.0/0"
)
assert "192.168.0.0/16" in public["to"][0]["ipBlock"]["except"]
def test_owner_agent_has_observer_kubernetes_context_without_cluster_admin():
config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text())
assert config["current-context"] == "atlas-observer"
assert config["contexts"][0]["context"]["namespace"] == "default"
rbac_path = HERMES / "rbac.yaml"
documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item]
bindings = [item for item in documents if item["kind"] == "ClusterRoleBinding"]
assert bindings
assert all(binding["roleRef"]["name"] != "cluster-admin" for binding in bindings)
assert all(
subject["name"] != "hermes-agent"
for binding in bindings
for subject in binding["subjects"]
)

11
testing/tests/test_hermes_chat_config.py
View File

@ -56,16 +56,17 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle():
assert "Atlas organization has private visibility" in instructions
assert "may be public or private" in instructions
assert "do not infer a\nrepository's visibility" in instructions
assert "already supplied through `GIT_ASKPASS`" in instructions
assert "Atlas Git access is supplied by the isolated SCM\nbroker" in soul
assert "Never call `kanban_show` without a known, non-empty task ID" in instructions
assert "bounded ad-hoc inspection and acceptance checks may" in instructions
assert "load implementation or TDD skills" in instructions
assert "Never call `kanban_show` without\na known, non-empty task ID" in soul
assert "must load a skill only when its workflow\nmaterially applies" in soul
assert "runtime-only `GIT_ASKPASS`" in soul
assert "`scm.bstein.dev` is Gitea, not GitHub" in instructions
assert "Never load or follow a GitHub/`gh`" in instructions
assert "/opt/coordinator/gitea_api.py METHOD /api/v1/..." in instructions
assert "no repository token is present in this pod" in soul
assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions
assert "Never load or follow a\nGitHub/`gh`" in instructions
assert "use `/opt/scm/gitea_api.py`" in instructions
assert "never bypass the client with raw HTTP" in instructions
assert "JENKINS_BASE_URL" in instructions
assert "Do not\nuse `git reset --hard`" in instructions
rendered = (HERMES / "agent-deployment.yaml").read_text()

182
testing/tests/test_hermes_chat_provider_auth.py
View File

@ -3,7 +3,6 @@
from __future__ import annotations
import base64
import importlib.util
import json
import sys
import time
@ -13,9 +12,6 @@ from types import ModuleType
import pytest
from testing.tests.test_hermes_chat_support import (
HERMES,
ROOT,
_documents,
_load_broker_module,
)
@ -455,181 +451,3 @@ def test_codex_broker_refreshes_and_persists_first_party_oauth(
# refresh token or create a second billing/authentication path.
assert module._access_token() == live
assert len(calls) == 1
def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch):
"""Claude traffic must use the native first-party CLI subscription lane."""
module = _load_broker_module(
"hermes_claude_broker", "claude_oauth_broker.py", monkeypatch
)
monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak")
monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak")
monkeypatch.setattr(
module,
"resolve_route",
lambda route: "claude-fable-5" if "/fable/" in route else route,
)
model, effort = module._route(
"route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}}
)
assert (model, effort) == ("claude-fable-5", "xhigh")
assert "ANTHROPIC_API_KEY" not in module._claude_environment()
assert "CLAUDE_API_KEY" not in module._claude_environment()
assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted")
def test_codex_native_health_overrides_historical_router_errors(
tmp_path: Path, monkeypatch
):
"""Fresh first-party health is authoritative over old Switchyard probes."""
plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py"
spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
health_path = tmp_path / "codex.json"
health_path.write_text(
json.dumps(
{
"state": "available",
"authenticated": True,
"transport": "codex-chatgpt-subscription",
}
)
)
monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path)
monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json")
monkeypatch.setattr(
module,
"_get_json",
lambda url: {"status": "ok"}
if url.endswith("/health")
else {
"models": {
"route/codex/terra/medium": {
"calls": 1,
"errors": 99,
"total_tokens": 12,
}
}
},
)
monkeypatch.setattr(module, "_codex_account", lambda: {})
monkeypatch.setattr(module, "_claude_account", lambda: {})
codex = module.provider_status_payload()["providers"]["codex"]
assert codex["errors"] == 99
assert codex["state"] == "available"
assert codex["native_health"]["transport"] == "codex-chatgpt-subscription"
def test_api_session_patch_accepts_parent_lineage(tmp_path: Path):
"""API-created workers must persist the originating Hermes session."""
module_path = HERMES / "scripts" / "patch_api_server_sessions.py"
spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
source = tmp_path / "api_server.py"
destination = tmp_path / "patched.py"
source.write_text(
"prefix\n"
+ module.BEFORE
+ "middle\n"
+ module.RUNS_BEFORE
+ "run body\n"
+ module.RUN_CLOSE_BEFORE
+ module.RESPONSES_SESSION_BEFORE
+ module.EVENT_CALLBACK_SIGNATURE_BEFORE
+ "callback docstring and push helper\n"
+ module.EVENT_CALLBACK_BODY_BEFORE
+ "tool start body\n"
+ module.EVENT_CALLBACK_END_BEFORE
+ module.EVENT_CALLBACK_CALL_BEFORE
+ module.RUN_SWEEP_BEFORE
+ "suffix\n",
encoding="utf-8",
)
module.patch(source, destination)
patched = destination.read_text(encoding="utf-8")
assert "X-Hermes-Parent-Session-Id" in patched
assert "parent_session_id=parent_session_id" in patched
assert "Parent session not found" in patched
assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched
assert "user_message.startswith(default_prefixes)" in patched
assert "session_parent_conflict" in patched
assert "X-Hermes-Conversation-Platform" in patched
assert "X-Hermes-Conversation-Title" in patched
assert 'conversation_platform != "telegram"' in patched
assert "db.record_gateway_session_peer(" in patched
assert 'display_name="Telegram"' in patched
assert "db.reopen_session(session_id)" in patched
assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched
assert "def _record_run_activity(" in patched
assert '"_thinking": "Hermes is reasoning"' in patched
assert '"run.started": "Worker started"' in patched
assert '"run.completed": "Worker completed"' in patched
assert '"reasoning.available": "Hermes finished a reasoning step"' in patched
assert '"subagent.progress": "Nested worker progress"' in patched
assert "redact_sensitive_text" in patched
assert 'getattr(os, "O_NOFOLLOW", 0)' in patched
assert "os.fchmod(fd, 0o600)" in patched
assert "session_id=session_id" in patched
assert 'self._record_run_activity(session_id, "run.started")' in patched
assert 'detail = tool_name if event_type in {' in patched
assert 'if event_type == "subagent.tool"' in patched
assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched
assert 'heartbeats.get(session_id, 0.0)' in patched
assert '"subagent.thinking",' in patched
assert "Stream retention and run lifetime are separate" in patched
assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched
assert patched.index("terminal_status = self._run_statuses") < patched.index(
"self._active_run_tasks.pop(run_id, None)"
)
def test_switchyard_brokers_and_native_claude_lane_use_the_right_images():
"""Thin brokers stay small while native Claude runs beside owner auth."""
dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers").read_text()
assert "httpx==0.28.1" in dockerfile
assert "worker_route_broker.py" in dockerfile
assert "routing_catalog.py" in dockerfile
deployment = _documents(HERMES / "switchyard-deployment.yaml")[0]
containers = {
container["name"]: container
for container in deployment["spec"]["template"]["spec"]["containers"]
}
expected = (
"registry.bstein.dev/bstein/hermes-switchyard-brokers@"
"sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083"
)
assert containers["worker-route-broker"]["image"] == expected
assert containers["classifier-broker"]["image"] == expected
assert "claude-oauth-broker" not in containers
agent = _documents(HERMES / "agent-deployment.yaml")[0]
agent_containers = {
container["name"]: container
for container in agent["spec"]["template"]["spec"]["containers"]
}
for container_name in ("hermes", "terminal"):
container = agent_containers[container_name]
environment = {item["name"]: item["value"] for item in container["env"]}
mounts = {item["name"]: item for item in container["volumeMounts"]}
assert environment["HERMES_ROUTING_CATALOG_PATH"] == "/routing-catalog/catalog.json"
assert environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json"
assert environment["HERMES_CLAUDE_HEALTH_PATH"] == "/opt/data/provider-health/claude.json"
assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog"
assert mounts["routing-catalog"]["readOnly"] is True
codex = agent_containers["codex-broker"]
codex_environment = {item["name"]: item["value"] for item in codex["env"]}
assert codex_environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json"
claude = agent_containers["claude-broker"]
assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@")
assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0]
assert any(
mount["name"] == "home" and mount["mountPath"] == "/opt/data"
for mount in claude["volumeMounts"]
)

196
testing/tests/test_hermes_chat_provider_runtime.py Normal file
View File

@ -0,0 +1,196 @@
"""Native provider health, API lineage, and runtime-image contracts."""
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
from testing.tests.test_hermes_chat_support import (
HERMES,
ROOT,
_documents,
_load_broker_module,
)
def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch):
"""Claude traffic must use the native first-party CLI subscription lane."""
module = _load_broker_module(
"hermes_claude_broker", "claude_oauth_broker.py", monkeypatch
)
monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak")
monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak")
monkeypatch.setattr(
module,
"resolve_route",
lambda route: "claude-fable-5" if "/fable/" in route else route,
)
model, effort = module._route(
"route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}}
)
assert (model, effort) == ("claude-fable-5", "xhigh")
assert "ANTHROPIC_API_KEY" not in module._claude_environment()
assert "CLAUDE_API_KEY" not in module._claude_environment()
assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted")
def test_codex_native_health_overrides_historical_router_errors(
tmp_path: Path, monkeypatch
):
"""Fresh first-party health is authoritative over old Switchyard probes."""
plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py"
spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
health_path = tmp_path / "codex.json"
health_path.write_text(
json.dumps(
{
"state": "available",
"authenticated": True,
"transport": "codex-chatgpt-subscription",
}
)
)
monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path)
monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json")
monkeypatch.setattr(
module,
"_get_json",
lambda url: {"status": "ok"}
if url.endswith("/health")
else {
"models": {
"route/codex/terra/medium": {
"calls": 1,
"errors": 99,
"total_tokens": 12,
}
}
},
)
monkeypatch.setattr(module, "_codex_account", lambda: {})
monkeypatch.setattr(module, "_claude_account", lambda: {})
codex = module.provider_status_payload()["providers"]["codex"]
assert codex["errors"] == 99
assert codex["state"] == "available"
assert codex["native_health"]["transport"] == "codex-chatgpt-subscription"
def test_api_session_patch_accepts_parent_lineage(tmp_path: Path):
"""API-created workers must persist the originating Hermes session."""
module_path = HERMES / "scripts" / "patch_api_server_sessions.py"
spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
source = tmp_path / "api_server.py"
destination = tmp_path / "patched.py"
source.write_text(
"prefix\n"
+ module.BEFORE
+ "middle\n"
+ module.RUNS_BEFORE
+ "run body\n"
+ module.RUN_CLOSE_BEFORE
+ module.RESPONSES_SESSION_BEFORE
+ module.EVENT_CALLBACK_SIGNATURE_BEFORE
+ "callback docstring and push helper\n"
+ module.EVENT_CALLBACK_BODY_BEFORE
+ "tool start body\n"
+ module.EVENT_CALLBACK_END_BEFORE
+ module.EVENT_CALLBACK_CALL_BEFORE
+ module.RUN_SWEEP_BEFORE
+ "suffix\n",
encoding="utf-8",
)
module.patch(source, destination)
patched = destination.read_text(encoding="utf-8")
assert "X-Hermes-Parent-Session-Id" in patched
assert "parent_session_id=parent_session_id" in patched
assert "Parent session not found" in patched
assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched
assert "user_message.startswith(default_prefixes)" in patched
assert "session_parent_conflict" in patched
assert "X-Hermes-Conversation-Platform" in patched
assert "X-Hermes-Conversation-Title" in patched
assert 'conversation_platform != "telegram"' in patched
assert "db.record_gateway_session_peer(" in patched
assert 'display_name="Telegram"' in patched
assert "db.reopen_session(session_id)" in patched
assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched
assert "def _record_run_activity(" in patched
assert '"_thinking": "Hermes is reasoning"' in patched
assert '"run.started": "Worker started"' in patched
assert '"run.completed": "Worker completed"' in patched
assert '"reasoning.available": "Hermes finished a reasoning step"' in patched
assert '"subagent.progress": "Nested worker progress"' in patched
assert "redact_sensitive_text" in patched
assert 'getattr(os, "O_NOFOLLOW", 0)' in patched
assert "os.fchmod(fd, 0o600)" in patched
assert "session_id=session_id" in patched
assert 'self._record_run_activity(session_id, "run.started")' in patched
assert 'detail = tool_name if event_type in {' in patched
assert 'if event_type == "subagent.tool"' in patched
assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched
assert 'heartbeats.get(session_id, 0.0)' in patched
assert '"subagent.thinking",' in patched
assert "Stream retention and run lifetime are separate" in patched
assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched
assert patched.index("terminal_status = self._run_statuses") < patched.index(
"self._active_run_tasks.pop(run_id, None)"
)
def test_switchyard_brokers_and_native_claude_lane_use_the_right_images():
"""Thin brokers stay small while native Claude runs beside owner auth."""
dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers").read_text()
assert "httpx==0.28.1" in dockerfile
assert "worker_route_broker.py" in dockerfile
assert "routing_catalog.py" in dockerfile
deployment = _documents(HERMES / "switchyard-deployment.yaml")[0]
containers = {
container["name"]: container
for container in deployment["spec"]["template"]["spec"]["containers"]
}
expected = (
"registry.bstein.dev/bstein/hermes-switchyard-brokers@"
"sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083"
)
assert containers["worker-route-broker"]["image"] == expected
assert containers["classifier-broker"]["image"] == expected
assert "claude-oauth-broker" not in containers
agent = _documents(HERMES / "agent-deployment.yaml")[0]
agent_containers = {
container["name"]: container
for container in agent["spec"]["template"]["spec"]["containers"]
}
for container_name in ("hermes", "terminal"):
container = agent_containers[container_name]
environment = {item["name"]: item["value"] for item in container["env"]}
mounts = {item["name"]: item for item in container["volumeMounts"]}
assert environment["HERMES_ROUTING_CATALOG_PATH"] == "/routing-catalog/catalog.json"
assert environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json"
assert environment["HERMES_CLAUDE_HEALTH_PATH"] == "/opt/data/provider-health/claude.json"
assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog"
assert mounts["routing-catalog"]["readOnly"] is True
codex = agent_containers["codex-broker"]
codex_environment = {item["name"]: item["value"] for item in codex["env"]}
assert codex_environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json"
claude = agent_containers["claude-broker"]
assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@")
assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0]
assert any(
mount["name"] == "home" and mount["mountPath"] == "/opt/data"
for mount in claude["volumeMounts"]
)

7
testing/tests/test_hermes_chat_voice.py
View File

@ -254,11 +254,14 @@ def test_chat_image_generation_uses_private_owner_broker():
vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text()
assert (
'"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram '
'hermes/developer-keycloak hermes/developer-gitea '
'hermes/developer-harbor hermes/developer-jenkins '
'hermes/developer-keycloak hermes/developer-harbor hermes/developer-jenkins '
'hermes/developer-ssh"'
in vault_policy
)
assert (
'write_policy_and_role "hermes-scm-broker" "hermes-scm" '
'"hermes-scm-broker"' in vault_policy
)
assert (
'write_policy_and_role "hermes-node-ssh" "hermes" '
'"hermes-node-ssh-access"' in vault_policy

12
testing/tests/test_hermes_cli_lanes_configuration.py
View File

@ -20,6 +20,8 @@ from testing.tests.test_hermes_cli_lanes_support import (
policy,
)
import cli_lane_execution as lane_execution
def test_workspace_preparation_failure_durably_blocks_the_claim(
tmp_path: Path, monkeypatch
@ -43,7 +45,7 @@ def test_workspace_preparation_failure_durably_blocks_the_claim(
)
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
monkeypatch.setattr(
lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json"
lane_execution, "state_path", lambda _board, _task_id: tmp_path / "state.json"
)
lanes.execute_claim("cassandra", "t_bad_worktree")
@ -102,21 +104,21 @@ def test_restart_provider_change_includes_explicit_workspace_handoff(
block_task=lambda *_args, **_kwargs: None,
)
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: state_file)
monkeypatch.setattr(lane_execution, "state_path", lambda _board, _task_id: state_file)
monkeypatch.setattr(
lanes,
lane_execution,
"select_route",
lambda *_args, **_kwargs: lanes.Route(
"codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
),
)
monkeypatch.setattr(
lanes,
lane_execution,
"git_handoff",
lambda _workspace, output: f"HANDOFF:{output}",
)
monkeypatch.setattr(
lanes,
lane_execution,
"run_provider",
lambda _route, prompt, *_args, **_kwargs: (
prompts.append(prompt)

392
testing/tests/test_hermes_cli_lanes_kanban.py
View File

@ -1,392 +0,0 @@
"""Kanban claim and lifecycle contracts for Hermes CLI lanes."""
from __future__ import annotations
import sys
from contextlib import nullcontext
from pathlib import Path
from types import SimpleNamespace
import pytest
from testing.tests.test_hermes_cli_lanes_support import (
lanes,
)
def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch):
task = SimpleNamespace(id="t_auto", assignee=None, status="ready")
assigned = []
class Connection:
def close(self):
return None
def assign_task(_conn, task_id, profile):
assigned.append((task_id, profile))
task.assignee = profile
return True
fake_db = SimpleNamespace(
list_boards=lambda include_archived=False: [{"slug": "cassandra"}],
scoped_current_board=lambda _board: nullcontext(),
connect=lambda board: Connection(),
recompute_ready=lambda _conn: None,
list_tasks=lambda _conn: [task],
assign_task=assign_task,
get_task=lambda _conn, _task_id: task,
claim_task=lambda _conn, _task_id, **_kwargs: task,
)
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")]
assert assigned == [("t_auto", "cli-auto")]
def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes(
monkeypatch, capsys
):
class CorruptBoardError(Exception):
pass
task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready")
class Connection:
def close(self):
return None
def connect(*, board):
if board == "cassandra":
raise CorruptBoardError("integrity_check failed")
return Connection()
fake_db = SimpleNamespace(
KanbanDbCorruptError=CorruptBoardError,
list_boards=lambda include_archived=False: [
{"slug": "cassandra"},
{"slug": "healthy"},
],
scoped_current_board=lambda _board: nullcontext(),
connect=connect,
recompute_ready=lambda _conn: None,
list_tasks=lambda _conn: [task],
claim_task=lambda _conn, _task_id, **_kwargs: task,
)
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
lanes.BOARD_CORRUPTION_ERRORS.clear()
assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")]
assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err
def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys):
task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready")
class Connection:
def __init__(self, board):
self.board = board
def close(self):
return None
def recompute_ready(connection):
if connection.board == "cassandra":
raise lanes.sqlite3.OperationalError("disk I/O error")
fake_db = SimpleNamespace(
list_boards=lambda include_archived=False: [
{"slug": "cassandra"},
{"slug": "healthy"},
],
scoped_current_board=lambda _board: nullcontext(),
connect=lambda board: Connection(board),
recompute_ready=recompute_ready,
list_tasks=lambda _conn: [task],
claim_task=lambda _conn, _task_id, **_kwargs: task,
)
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
lanes.BOARD_CORRUPTION_ERRORS.clear()
assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")]
error = capsys.readouterr().err
assert "temporarily skipping Kanban board 'cassandra'" in error
assert "storage OperationalError: disk I/O error" in error
def test_board_call_retries_storage_faults_on_fresh_connections():
connections = []
class Connection:
def __init__(self):
self.closed = False
def close(self):
self.closed = True
def connect(*, board):
assert board == "cassandra"
connection = Connection()
connections.append(connection)
return connection
attempts = []
def operation(_connection):
attempts.append(1)
if len(attempts) < 3:
raise lanes.sqlite3.OperationalError("disk I/O error")
return "healthy"
fake_db = SimpleNamespace(
scoped_current_board=lambda _board: nullcontext(),
connect=connect,
)
lanes.BOARD_CORRUPTION_ERRORS.clear()
assert lanes._board_call(fake_db, "cassandra", operation) == "healthy"
assert len(connections) == 3
assert all(connection.closed for connection in connections)
@pytest.mark.parametrize(
("result", "expected_action"),
[
(lanes.ProcessResult(0, "plain text only", None, False), "block"),
(
lanes.ProcessResult(
0,
"",
{
"status": "completed",
"summary": "done",
"changed_files": ["src/a.py"],
"tests_run": ["pytest -q"],
"artifacts": ["reports/result.json"],
"blockers": [],
},
False,
),
"complete",
),
(
lanes.ProcessResult(
0,
"",
{
"status": "completed",
"summary": "The full test suite is still running.",
"changed_files": ["src/a.py"],
"tests_run": ["pytest -q — in progress"],
"artifacts": [],
"blockers": [],
},
False,
),
"block",
),
],
)
def test_claim_requires_structured_evidence_and_surfaces_artifacts(
tmp_path: Path,
monkeypatch,
result,
expected_action,
):
task = SimpleNamespace(
id="t_worker",
current_run_id=4,
assignee="cli-auto",
max_runtime_seconds=60,
)
calls = []
heartbeats = []
connections = []
artifact = tmp_path / "reports/result.json"
artifact.parent.mkdir()
artifact.write_text("{}\n", encoding="utf-8")
class Connection:
def __init__(self):
self.closed = False
def close(self):
self.closed = True
def connect(*, board):
assert board == "cassandra"
connection = Connection()
connections.append(connection)
return connection
fake_db = SimpleNamespace(
scoped_current_board=lambda _board: nullcontext(),
connect=connect,
get_task=lambda _conn, _task_id: task,
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"),
set_branch_name=lambda *_args: None,
set_workspace_path=lambda *_args: None,
build_worker_context=lambda *_args: "bounded objective",
heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: (
heartbeats.append((note, expected_run_id)) or True
),
add_comment=lambda *_args: None,
complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)),
block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)),
)
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
monkeypatch.setattr(
lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json"
)
monkeypatch.setattr(
lanes,
"select_route",
lambda *_args, **_kwargs: lanes.Route(
"codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
),
)
def run_provider(*args, **_kwargs):
assert connections[0].closed
before_heartbeat = len(connections)
assert args[6]("working") is True
assert len(connections) == before_heartbeat + 1
assert connections[-1].closed
return result
monkeypatch.setattr(lanes, "run_provider", run_provider)
lanes.execute_claim("cassandra", "t_worker")
assert calls[0][0] == expected_action
assert heartbeats == [("working", 4)]
if expected_action == "complete":
assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)]
assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"]
else:
assert calls[0][1]["kind"] == "capability"
assert all(connection.closed for connection in connections)
def test_goal_card_continues_after_local_judge_rejects_progress(
tmp_path: Path,
monkeypatch,
):
task = SimpleNamespace(
id="t_goal",
current_run_id=12,
assignee="cli-auto",
max_runtime_seconds=300,
goal_mode=True,
goal_max_turns=3,
)
calls = []
comments = []
class Connection:
def close(self):
return None
fake_db = SimpleNamespace(
scoped_current_board=lambda _board: nullcontext(),
connect=lambda board: Connection(),
get_task=lambda _conn, _task_id: task,
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"),
set_branch_name=lambda *_args: None,
set_workspace_path=lambda *_args: None,
build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.",
heartbeat_worker=lambda *_args, **_kwargs: True,
add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)),
block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)),
)
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
monkeypatch.setattr(
lanes,
"state_path",
lambda _board, _task_id: tmp_path / "state.json",
)
claude_low = lanes.Route(
"claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, ()
)
codex_low = lanes.Route(
"codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, ()
)
codex_xhigh = lanes.Route(
"codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, ()
)
route_calls = []
def select_route(_prompt, assignee, **kwargs):
route_calls.append((assignee, kwargs))
if assignee == "cli-codex-low":
return codex_low
if len(route_calls) == 1:
return claude_low
return codex_xhigh
monkeypatch.setattr(lanes, "select_route", select_route)
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None)
reports = [
lanes.ProcessResult(1, "authentication expired", None, True),
lanes.ProcessResult(
0,
"first turn",
{
"status": "completed",
"summary": "Focused tests passed.",
"changed_files": ["src/a.py"],
"tests_run": ["pytest focused: passed"],
"artifacts": [],
"blockers": [],
},
False,
),
lanes.ProcessResult(
0,
"second turn",
{
"status": "completed",
"summary": "Full tests passed; commit pushed and remote HEAD verified.",
"changed_files": ["src/a.py"],
"tests_run": ["pytest full: passed"],
"artifacts": [],
"blockers": [],
},
False,
),
]
monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
verdicts = iter(
[
(False, "commit, push, and remote verification are missing"),
(True, "all explicit acceptance criteria have evidence"),
]
)
judge_contexts = []
def judge_goal_completion(objective, *_args, **_kwargs):
judge_contexts.append(objective)
return next(verdicts)
monkeypatch.setattr(
lanes.cli_lane_goal,
"judge_goal_completion",
judge_goal_completion,
)
lanes.execute_claim("cassandra", "t_goal")
assert calls[0][0] == "complete"
assert calls[0][1]["metadata"]["goal_turn"] == 2
assert any(
"Goal completion rejected; continuing turn 2/3" in item for item in comments
)
assert any(
"Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments
)
assert route_calls[2][1]["exclude_provider"] == "claude"
assert "prior rejected reports" in judge_contexts[1]
assert "commit, push, and remote verification are missing" in judge_contexts[1]
assert reports == []

9
testing/tests/test_hermes_coordinator_boards.py
View File

@ -141,7 +141,7 @@ def test_cassandra_sync_repairs_existing_worktree(
assert all("GITEA_TOKEN" not in environment for environment in environments)
def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch):
"""Missing credentials skip only the fetch, not the local origin repair."""
"""The credential-isolated broker needs no model-facing token to fetch."""
workspace = tmp_path / "cassandra"
(workspace / ".git").mkdir(parents=True)
monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace)
@ -156,8 +156,11 @@ def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch
monkeypatch.setattr(coordinator.subprocess, "run", run)
state = coordinator.sync_cassandra_repo({})
assert state == "ready; fetch skipped until Gitea token is configured"
assert [command[-3] for command in commands] == ["remote", "set-url"]
assert state == "ready"
assert len(commands) == 3
assert commands[0][-3:] == ["remote", "get-url", "origin"]
assert commands[1][-4:-1] == ["remote", "set-url", "origin"]
assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"]
def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks():
"""A project switch moves queued work without relocating active evidence."""

158
testing/tests/test_hermes_execution_pool.py
View File

@ -3,7 +3,6 @@
from __future__ import annotations
import json
import os
import subprocess
import sys
import threading
@ -17,7 +16,7 @@ import yaml
ROOT = Path(__file__).parents[2]
HERMES = ROOT / "services/hermes"
SCRIPTS = HERMES / "scripts"
sys.path.insert(0, str(SCRIPTS))
sys.path[:0] = [str(SCRIPTS), str(HERMES / "scm-common/scripts")]
import execution_pool_coordinator as coordinator # noqa: E402
import execution_pool_client as pool_client # noqa: E402
@ -132,13 +131,13 @@ def test_key_requires_private_regular_file_and_rejects_symlink(tmp_path):
def test_local_signing_boundary_rejects_foreign_or_unassigned_result(monkeypatch):
monkeypatch.setattr(pool_client, "ORDINAL", 0)
boundary = pool_client.ClientBoundary(KEY)
request = {"binding": binding(), "payload": {"returncode": 0}}
request = {"binding": binding(), "payload": {"note": "active"}}
with pytest.raises(protocol.ProtocolError, match="does not own"):
boundary.forward("result", request)
boundary.heartbeat(request)
boundary.current = binding()
request["binding"] = binding(worker_ordinal=1)
with pytest.raises(protocol.ProtocolError, match="does not own"):
boundary.forward("result", request)
boundary.heartbeat(request)
def test_simultaneous_claim_materialization_has_one_winner(tmp_path):
@ -282,8 +281,8 @@ def test_scm_workspace_is_ordinal_contained_and_rejects_symlink(tmp_path, monkey
def test_scm_boundary_rejects_other_repo_branch_and_ordinal(monkeypatch):
monkeypatch.setattr(scm, "ORDINAL", 0)
for changes, message in (
({"repo_url": "https://evil.example/atlas/titan-iac.git"}, "SCM policy"),
({"branch": "main"}, "SCM policy"),
({"repo_url": "https://evil.example/atlas/titan-iac.git"}, "outside Atlas"),
({"branch": "main"}, "reviewed namespace"),
):
signed = protocol.sign_envelope(
KEY, "assignment", binding(), assignment_payload(**changes)
@ -297,68 +296,17 @@ def test_scm_boundary_rejects_other_repo_branch_and_ordinal(monkeypatch):
scm._binding(signed)
def test_scm_identity_is_data_only_and_supports_packed_refs(tmp_path):
checkout = tmp_path / "checkout"
subprocess.run(["git", "init", "-q", str(checkout)], check=True)
subprocess.run(
["git", "-C", str(checkout), "remote", "add", "origin",
"https://scm.bstein.dev/atlas/titan-iac.git"], check=True,
)
subprocess.run(
["git", "-C", str(checkout), "checkout", "-qb", "feature/hermes-safe-pool"],
check=True,
)
subprocess.run(["git", "-C", str(checkout), "config", "user.email", "a@b.c"], check=True)
subprocess.run(["git", "-C", str(checkout), "config", "user.name", "Test"], check=True)
(checkout / "tracked").write_text("safe\n")
subprocess.run(["git", "-C", str(checkout), "add", "tracked"], check=True)
subprocess.run(["git", "-C", str(checkout), "commit", "-qm", "initial"], check=True)
expected = subprocess.check_output(
["git", "-C", str(checkout), "rev-parse", "HEAD"], text=True
).strip()
subprocess.run(["git", "-C", str(checkout), "pack-refs", "--all"], check=True)
assert scm._workspace_identity(
checkout, "https://scm.bstein.dev/atlas/titan-iac.git",
"feature/hermes-safe-pool",
) == expected
(checkout / ".git/refs/heads/feature").mkdir(parents=True)
(checkout / ".git/refs/heads/feature/hermes-safe-pool").symlink_to("/etc/passwd")
with pytest.raises(protocol.ProtocolError, match="symlink"):
scm._workspace_identity(
checkout, "https://scm.bstein.dev/atlas/titan-iac.git",
"feature/hermes-safe-pool",
)
def test_scm_bundle_copy_is_nofollow_bounded_and_fetchable(tmp_path, monkeypatch):
repo = tmp_path / "repo"
private = tmp_path / "private.git"
subprocess.run(["git", "init", "-q", str(repo)], check=True)
subprocess.run(["git", "-C", str(repo), "config", "user.email", "a@b.c"], check=True)
subprocess.run(["git", "-C", str(repo), "config", "user.name", "Test"], check=True)
(repo / "tracked").write_text("safe\n")
subprocess.run(["git", "-C", str(repo), "add", "tracked"], check=True)
subprocess.run(["git", "-C", str(repo), "commit", "-qm", "initial"], check=True)
bundle = repo / ".git/hermes-submit.bundle"
subprocess.run(["git", "-C", str(repo), "bundle", "create", str(bundle), "HEAD"], check=True)
copied = tmp_path / "copied.bundle"
scm._copy_bundle(bundle, copied)
subprocess.run(["git", "init", "--bare", "-q", str(private)], check=True)
subprocess.run(
["git", "--git-dir", str(private), "fetch", str(copied),
"HEAD:refs/pool/candidate"], check=True,
)
link = tmp_path / "bundle-link"
link.symlink_to(bundle)
with pytest.raises(OSError):
scm._copy_bundle(link, tmp_path / "rejected")
fifo = tmp_path / "bundle-fifo"
os.mkfifo(fifo)
with pytest.raises(protocol.ProtocolError, match="not regular"):
scm._copy_bundle(fifo, tmp_path / "rejected-fifo")
monkeypatch.setattr(scm, "MAX_BUNDLE_BYTES", 1)
with pytest.raises(protocol.ProtocolError, match="oversized"):
scm._copy_bundle(bundle, tmp_path / "rejected-large")
def test_scm_git_process_has_no_credential_or_ambient_configuration():
environment = scm._git_environment()
assert environment == {
"HOME": "/nonexistent",
"PATH": "/usr/bin:/bin",
"GIT_CONFIG_NOSYSTEM": "1",
"GIT_TERMINAL_PROMPT": "0",
}
assert scm._broker_repo("titan-iac").startswith("http://hermes-scm-broker.")
source = (SCRIPTS / "execution_pool_scm.py").read_text()
assert "GITEA_TOKEN" not in source and "GIT_ASKPASS" not in source
def test_activity_is_bounded_sanitized_and_nofollow(tmp_path):
@ -396,10 +344,12 @@ def test_three_node_statefulset_contract_and_cross_worker_isolation():
assert stateful["spec"]["replicas"] == 3
assert stateful["spec"]["podManagementPolicy"] == "Parallel"
assert pod["automountServiceAccountToken"] is False
assert len(stateful["spec"]["volumeClaimTemplates"]) == 1
claim = stateful["spec"]["volumeClaimTemplates"][0]
assert claim["metadata"]["name"] == "workspace"
assert claim["spec"]["accessModes"] == ["ReadWriteOnce"]
claims = {
item["metadata"]["name"]: item
for item in stateful["spec"]["volumeClaimTemplates"]
}
assert set(claims) == {"workspace", "provider-access"}
assert all(item["spec"]["accessModes"] == ["ReadWriteOnce"] for item in claims.values())
anti = pod["affinity"]["podAntiAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"]
assert anti[0]["topologyKey"] == "kubernetes.io/hostname"
spread = pod["topologySpreadConstraints"][0]
@ -409,17 +359,33 @@ def test_three_node_statefulset_contract_and_cross_worker_isolation():
def test_worker_placement_prefers_accelerators_and_preserves_exclusions():
stateful = _documents(HERMES / "execution-worker-statefulset.yaml")[0]
pod = stateful["spec"]["template"]["spec"]
assert pod["priorityClassName"] == "scavenger"
affinity = stateful["spec"]["template"]["spec"]["affinity"]["nodeAffinity"]
terms = affinity["requiredDuringSchedulingIgnoredDuringExecution"]["nodeSelectorTerms"]
accelerator = {item["key"]: item for item in terms[0]["matchExpressions"]}
assert accelerator["kubernetes.io/hostname"]["values"] == ["titan-20", "titan-21"]
assert accelerator["node-role.kubernetes.io/accelerator"]["operator"] == "Exists"
fallback = {item["key"]: item for item in terms[1]["matchExpressions"]}
assert fallback["hardware"]["values"] == ["rpi5"]
assert "node-role.kubernetes.io/worker" not in fallback
excluded = set(fallback["kubernetes.io/hostname"]["values"])
assert {"titan-04", "titan-08", "titan-13", "titan-14", "titan-17", "titan-18", "titan-19", "titan-22", "titan-24"} <= excluded
required = {item["key"]: item for item in terms[0]["matchExpressions"]}
assert required["node-role.kubernetes.io/worker"]["values"] == ["true"]
excluded = set(required["kubernetes.io/hostname"]["values"])
assert {"titan-04", "titan-13", "titan-14", "titan-17", "titan-18", "titan-19", "titan-22", "titan-24"} <= excluded
assert affinity["preferredDuringSchedulingIgnoredDuringExecution"][0]["weight"] == 100
assert affinity["preferredDuringSchedulingIgnoredDuringExecution"][1]["weight"] == 50
worker_container = pod["containers"][0]
assert worker_container["resources"] == {
"requests": {"cpu": "5m", "memory": "128Mi", "ephemeral-storage": "1Gi"},
"limits": {"cpu": "2", "memory": "4Gi", "ephemeral-storage": "8Gi"},
}
mediators = [
item
for item in _documents(HERMES / "execution-mediator.yaml")
if item["kind"] == "Deployment"
]
assert all(
item["spec"]["template"]["spec"]["containers"][0]["resources"][
"requests"
]
== {"cpu": "2m", "memory": "64Mi"}
for item in mediators
)
def test_model_worker_has_no_scm_or_cluster_credential_mount():
@ -432,17 +398,26 @@ def test_model_worker_has_no_scm_or_cluster_credential_mount():
mounts = {item["mountPath"] for item in worker["volumeMounts"]}
assert "/vault/secrets" not in mounts
assert "/pool-access" not in mounts
assert "/provider-access" in mounts
assert not any(item["name"] == "vault-auth-token" for item in worker["volumeMounts"])
assert pod["metadata"]["annotations"]["vault.hashicorp.com/agent-inject-containers"] == "stage-worker-access"
assert any(item["name"] == "vault-auth-token" for item in containers["credential-sync"]["volumeMounts"])
for boundary_name in ("execution-client", "scm-boundary"):
assert any(
item["mountPath"] == "/pool-access"
for item in containers[boundary_name]["volumeMounts"]
)
scm_mounts = {item["mountPath"] for item in containers["scm-boundary"]["volumeMounts"]}
assert "/scm-access" in scm_mounts
assert "/vault/secrets" not in scm_mounts
assert set(containers) == {"execution-worker"}
assert "execution-pool-key" not in str(pod["metadata"]["annotations"])
mediators = [
item for item in _documents(HERMES / "execution-mediator.yaml")
if item["kind"] == "Deployment"
]
assert len(mediators) == 3
for mediator in mediators:
spec = mediator["spec"]["template"]["spec"]
assert spec["automountServiceAccountToken"] is False
privileged = spec["containers"][0]
assert {mount["mountPath"] for mount in privileged["volumeMounts"]} >= {
"/pool-access", "/scm-state", "/opt/scm", "/workspace"
}
assert "/provider-access" not in {
mount["mountPath"] for mount in privileged["volumeMounts"]
}
def test_worker_service_account_has_no_kubernetes_permissions():
@ -471,12 +446,13 @@ def test_coordinator_remains_single_state_owner_and_workers_do_not_mount_home():
def test_worker_protocol_preserves_switchyard_fallback_and_visible_evidence():
source = (SCRIPTS / "execution_pool_worker.py").read_text()
coordinator_source = (SCRIPTS / "execution_pool_coordinator.py").read_text()
server_source = (SCRIPTS / "execution_pool_server.py").read_text()
assert "cli_lane_runner.select_route" in source
assert 'alternate = "claude" if route.provider == "codex" else "codex"' in source
assert "codex_thread_id" in source and "claude_session_id" in source
assert "final_activity" in source
assert "worker_ordinal" in coordinator_source and "provider_sessions" in coordinator_source
assert "coordinator.reconcile()" in coordinator_source
assert "coordinator.reconcile" in server_source
def test_retention_gc_removes_only_clean_terminal_workspace(tmp_path, monkeypatch):

98
testing/tests/test_hermes_execution_pool_assignment.py
View File

@ -13,9 +13,10 @@ import yaml
SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts"
HERMES = SCRIPTS.parent
sys.path.insert(0, str(SCRIPTS))
sys.path[:0] = [str(SCRIPTS), str(HERMES / "scm-common/scripts")]
import execution_pool_coordinator as coordinator # noqa: E402
import execution_pool_project as project # noqa: E402
import execution_pool_protocol as protocol # noqa: E402
import execution_pool_scm as scm # noqa: E402
import execution_pool_worker as worker # noqa: E402
@ -26,34 +27,34 @@ KEY = b"k" * 32
def test_new_task_uses_explicit_reviewed_atlas_default(monkeypatch):
monkeypatch.setattr(
coordinator,
"DEFAULT_REPO",
"https://scm.bstein.dev/atlas/titan-iac.git",
project,
"resolve_project",
lambda board: (
f"https://scm.bstein.dev/atlas/{board}.git", "main", Path("/unused")
),
)
task = SimpleNamespace(id="t_deadbeef", workspace_path="", branch_name="")
assert coordinator.resolve_scm(task) == (
assert project.resolve_assignment("titan-iac", task) == (
"https://scm.bstein.dev/atlas/titan-iac.git",
"feature/hermes-t_deadbeef",
"wt/t_deadbeef",
"main",
)
def test_legacy_local_workspace_fails_closed_without_running_git(tmp_path):
def test_legacy_local_workspace_remains_owned_by_local_lane(tmp_path):
task = SimpleNamespace(
id="t_deadbeef", workspace_path=str(tmp_path), branch_name="feature/safe"
)
with pytest.raises(RuntimeError, match="preserve or commit"):
coordinator.resolve_scm(task)
assert project.distributed_workspace_eligible(task) is False
def test_local_git_environment_excludes_credential_boundary_paths():
local = scm._git_env(False)
authenticated = scm._git_env(True)
local = scm._git_environment()
assert "HERMES_SCM_PASSWORD_FILE" not in local
assert "GIT_ASKPASS" not in local
assert authenticated["HERMES_SCM_PASSWORD_FILE"] == str(scm.TOKEN_PATH)
assert "GITEA_TOKEN" not in local
def test_finalized_duplicate_result_is_acknowledged_without_refinalizing(tmp_path):
@ -63,7 +64,8 @@ def test_finalized_duplicate_result_is_acknowledged_without_refinalizing(tmp_pat
}
store = protocol.PoolStore(tmp_path / "pool.db")
store.add(binding, {"context": "safe"})
result = protocol.sign_envelope(KEY, "result", binding, {"structured": {}})
ordinal_key = protocol.derive_ordinal_key(KEY, 0)
result = protocol.sign_envelope(ordinal_key, "result", binding, {"structured": {}})
store.accept_result(result)
store.finalize(binding, "finalized")
pool = coordinator.Coordinator(KEY, store)
@ -117,9 +119,13 @@ def test_internal_http_server_rejects_work_above_its_bound(monkeypatch):
def test_additive_patch_replaces_local_lane_without_touching_base_deployment():
patch = yaml.safe_load((HERMES / "execution-coordinator-patch.yaml").read_text())
containers = patch["spec"]["template"]["spec"]["containers"]
deleted = next(item for item in containers if item["name"] == "cli-lane-runner")
local = next(item for item in containers if item["name"] == "cli-lane-runner")
pool = next(item for item in containers if item["name"] == "execution-pool-coordinator")
assert deleted["$patch"] == "delete"
environment = {item["name"]: item["value"] for item in local["env"]}
assert environment == {
"HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY": "true",
"HERMES_CLI_LANE_CONCURRENCY": "1",
}
assert pool["resources"]["requests"] == {"cpu": "50m", "memory": "128Mi"}
access = next(item for item in pool["volumeMounts"] if item["name"] == "runtime-access")
assert access["subPath"] == "execution-pool-key" and access["readOnly"] is True
@ -132,27 +138,71 @@ def test_additive_network_policies_expose_only_worker_pool_and_switchyard_ports(
policies = {item["metadata"]["name"]: item for item in documents}
assert set(policies) == {
"hermes-execution-worker-isolation",
"hermes-execution-mediator-isolation",
"hermes-execution-pool-ingress",
"hermes-execution-switchyard-ingress",
*(f"hermes-execution-worker-mediator-{ordinal}" for ordinal in range(3)),
*(f"hermes-execution-mediator-worker-{ordinal}" for ordinal in range(3)),
}
assert policies["hermes-execution-pool-ingress"]["spec"]["ingress"][0]["ports"] == [
{"protocol": "TCP", "port": 9007}
]
pool_source = policies["hermes-execution-pool-ingress"]["spec"]["ingress"]
assert "hermes-execution-mediator" in str(pool_source)
worker_egress = policies["hermes-execution-worker-isolation"]["spec"]["egress"]
assert "hermes-scm-broker" not in str(worker_egress)
assert "hermes-execution-mediator" not in str(worker_egress)
for ordinal in range(3):
worker_policy = policies[f"hermes-execution-worker-mediator-{ordinal}"]
mediator_policy = policies[f"hermes-execution-mediator-worker-{ordinal}"]
assert worker_policy["spec"]["podSelector"]["matchLabels"][
"apps.kubernetes.io/pod-index"
] == str(ordinal)
assert worker_policy["spec"]["egress"][0]["to"][0]["podSelector"][
"matchLabels"
]["pool-ordinal"] == str(ordinal)
assert mediator_policy["spec"]["ingress"][0]["from"][0]["podSelector"][
"matchLabels"
]["apps.kubernetes.io/pod-index"] == str(ordinal)
@pytest.mark.parametrize(
"repo,base",
"branch",
[
("https://evil.example/atlas/titan-iac.git", "main"),
("https://token@scm.bstein.dev/atlas/titan-iac.git", "main"),
("https://scm.bstein.dev/atlas/titan-iac.git", "../main"),
"../main",
"main",
"feature/../../main",
],
)
def test_assignment_rejects_unreviewed_repo_or_base(repo, base):
def test_assignment_rejects_unreviewed_task_branch(monkeypatch, branch):
monkeypatch.setattr(
project,
"resolve_project",
lambda _board: (
"https://scm.bstein.dev/atlas/titan-iac.git", "main", Path("/unused")
),
)
task = SimpleNamespace(
id="t_deadbeef", workspace_path="", branch_name="feature/safe",
repo_url=repo, base_branch=base,
id="t_deadbeef", workspace_path="", branch_name=branch,
repo_url="https://evil.example/atlas/other.git", base_branch="../main",
)
with pytest.raises(RuntimeError, match="outside"):
coordinator.resolve_scm(task)
with pytest.raises(project.ProjectPolicyError):
project.resolve_assignment("titan-iac", task)
def test_registry_authority_ignores_unreviewed_task_repo_metadata(monkeypatch):
monkeypatch.setattr(
project,
"resolve_project",
lambda _board: (
"https://scm.bstein.dev/atlas/metis.git", "main", Path("/unused")
),
)
task = SimpleNamespace(
id="t_deadbeef", branch_name="review/t_deadbeef",
repo_url="https://evil.example/atlas/other.git", base_branch="../main",
)
assert project.resolve_assignment("metis", task)[:2] == (
"https://scm.bstein.dev/atlas/metis.git", "review/t_deadbeef"
)

293
testing/tests/test_hermes_execution_pool_coordinator_v2.py Normal file
View File

@ -0,0 +1,293 @@
"""Exact-run coordinator finalization, lease, and ownership contracts."""
from __future__ import annotations
import json
import sys
import types
from pathlib import Path
from types import SimpleNamespace
import pytest
ROOT = Path(__file__).parents[2]
SCRIPTS = ROOT / "services/hermes/scripts"
sys.path.insert(0, str(SCRIPTS))
import execution_pool_coordinator as coordinator # noqa: E402
import execution_pool_protocol as protocol # noqa: E402
MASTER = b"k" * 32
STRUCTURED = {
"status": "completed",
"summary": "Completed safely.",
"changed_files": ["safe.py"],
"tests_run": ["pytest"],
"artifacts": [],
"findings": [],
"blockers": [],
}
def binding(**changes):
value = {
"board": "metis", "task_id": "t_deadbeef", "run_id": "23",
"worker_ordinal": 0, "attempt": 1,
}
value.update(changes)
return value
def assignment_payload(**changes):
value = {
"context": "objective",
"assignee": "cli-auto",
"repo_url": "https://scm.bstein.dev/atlas/metis.git",
"branch": "wt/t_deadbeef",
"base_branch": "main",
"max_runtime_seconds": 3600,
"deadline_unix": 10_000_000_000,
}
value.update(changes)
return value
def signed(kind, exact_binding, payload):
key = protocol.derive_ordinal_key(MASTER, exact_binding["worker_ordinal"])
return protocol.sign_envelope(key, kind, exact_binding, payload)
class Scope:
def __enter__(self):
return self
def __exit__(self, *_args):
return False
class Connection:
def close(self):
return None
def install_kanban(monkeypatch, tasks, tmp_path, boards=None):
module = types.ModuleType("kanban_db")
module.tasks = {task.id: task for task in tasks}
module.completed = []
module.blocked = []
module.heartbeats = []
module.reclaimed = []
module.branches = []
module.scoped_current_board = lambda _board: Scope()
module.connect = lambda board=None: Connection()
module.get_task = lambda _connection, task_id: module.tasks.get(task_id)
module.list_tasks = lambda _connection: list(module.tasks.values())
module.list_boards = lambda include_archived=False: boards or ["metis"]
module.build_worker_context = lambda _connection, task_id: f"objective {task_id}"
module.worker_log_path = lambda task_id, board=None: str(
tmp_path / board / f"{task_id}.log"
)
def complete(_connection, task_id, **values):
module.completed.append((task_id, values))
module.tasks[task_id].status = "completed"
return True
def block(_connection, task_id, **values):
module.blocked.append((task_id, values))
module.tasks[task_id].status = "blocked"
return True
def heartbeat(_connection, task_id, **values):
module.heartbeats.append((task_id, values))
return True
module.complete_task = complete
module.block_task = block
module.heartbeat_worker = heartbeat
module.reclaim_task = lambda _c, task_id, **values: module.reclaimed.append(
(task_id, values)
)
package = types.ModuleType("hermes_cli")
package.kanban_db = module
monkeypatch.setitem(sys.modules, "hermes_cli", package)
monkeypatch.setitem(sys.modules, "hermes_cli.kanban_db", module)
return module
def task(**changes):
value = {
"id": "t_deadbeef",
"status": "running",
"current_run_id": 23,
"assignee": "cli-auto",
"workspace_path": "",
"branch_name": "wt/t_deadbeef",
"max_runtime_seconds": 3600,
"goal_mode": False,
"goal_max_turns": 1,
}
value.update(changes)
return SimpleNamespace(**value)
def store_with_assignment(tmp_path, exact_binding=None):
store = protocol.PoolStore(tmp_path / "pool.db")
store.add(exact_binding or binding(), assignment_payload())
return store
def test_assignment_payload_is_bounded_registry_derived_and_typed(monkeypatch):
monkeypatch.setattr(
coordinator,
"resolve_assignment",
lambda board, _task: (
f"https://scm.bstein.dev/atlas/{board}.git", "wt/t_deadbeef", "main"
),
)
kanban = SimpleNamespace(
build_worker_context=lambda _connection, _task_id: {"safe": True}
)
item = task(max_runtime_seconds=1, goal_max_turns=99)
value = coordinator.assignment_payload(kanban, object(), item, "metis")
assert json.loads(value["context"]) == {"safe": True}
assert value["max_runtime_seconds"] == 60
assert value["goal_max_turns"] == 12
assert value["repo_url"].endswith("/metis.git")
kanban.build_worker_context = lambda *_a: "x" * (32 * 1024 + 1)
with pytest.raises(RuntimeError, match="32KiB"):
coordinator.assignment_payload(kanban, object(), item, "metis")
def test_poll_uses_per_ordinal_key_for_empty_and_active_assignment(tmp_path):
store = protocol.PoolStore(tmp_path / "pool.db")
pool = coordinator.Coordinator(MASTER, store)
poll_binding = binding(board="", task_id="", run_id="", attempt=0)
request = signed("poll", poll_binding, {"ready": True})
empty = pool.poll(request)
assert protocol.verify_envelope(
protocol.derive_ordinal_key(MASTER, 0), empty, expected_kind="ack"
)["payload"] == {"assignment": None}
store.add(binding(), assignment_payload())
offered = protocol.verify_envelope(
protocol.derive_ordinal_key(MASTER, 0), pool.poll(request),
expected_kind="assignment",
)
assert offered["run_id"] == "23"
wrong_key = protocol.derive_ordinal_key(MASTER, 1)
with pytest.raises(protocol.ProtocolError, match="authentication"):
pool.poll(protocol.sign_envelope(wrong_key, "poll", poll_binding, {}))
def test_finalize_canonicalizes_integer_run_and_reconcile_does_not_reexecute(
tmp_path, monkeypatch
):
live = task()
kanban = install_kanban(monkeypatch, [live], tmp_path)
store = store_with_assignment(tmp_path)
pool = coordinator.Coordinator(MASTER, store)
envelope = signed(
"result",
binding(),
{
"structured": dict(STRUCTURED),
"returncode": 0,
"capacity_failure": False,
"node": "titan-05",
"route": {},
"provider_sessions": {},
"final_activity": "done\n",
},
)
ack = pool.result(envelope)
assert protocol.verify_envelope(
protocol.derive_ordinal_key(MASTER, 0), ack, expected_kind="ack"
)["payload"] == {"accepted": True, "duplicate": False}
assert kanban.completed[0][1]["expected_run_id"] == 23
row = store._connect().execute("SELECT state FROM assignments").fetchone()
assert row[0] == "finalized" and store.available_ordinals() == [0, 1, 2]
pool.reconcile()
assert store.active_assignments() == []
def test_finalize_fences_stale_run_and_blocks_failed_result_exactly(tmp_path, monkeypatch):
stale_task = task(current_run_id=24)
kanban = install_kanban(monkeypatch, [stale_task], tmp_path)
store = store_with_assignment(tmp_path)
pool = coordinator.Coordinator(MASTER, store)
pool.result(signed("result", binding(), {"structured": dict(STRUCTURED), "returncode": 0}))
assert not kanban.completed
assert store._connect().execute("SELECT state FROM assignments").fetchone()[0] == "stale"
second = binding(run_id="25", worker_ordinal=1)
failed_task = task(current_run_id=25)
kanban.tasks[failed_task.id] = failed_task
store.add(second, assignment_payload())
blocked = {**STRUCTURED, "status": "blocked", "blockers": ["capacity"]}
pool.result(
signed(
"result", second,
{"structured": blocked, "returncode": 1, "capacity_failure": True},
)
)
assert kanban.blocked[-1][1]["expected_run_id"] == 25
assert kanban.blocked[-1][1]["kind"] == "transient"
def test_finalize_rejects_invalid_payload_and_noncanonical_run(tmp_path, monkeypatch):
install_kanban(monkeypatch, [task()], tmp_path)
store = store_with_assignment(tmp_path)
pool = coordinator.Coordinator(MASTER, store)
with pytest.raises(protocol.ProtocolError, match="payload"):
pool.finalize({**binding(), "result": []})
bad = binding(run_id="run-bad", worker_ordinal=1)
store.add(bad, assignment_payload())
pool.finalize({**bad, "result": {"structured": {}}})
states = dict(store._connect().execute("SELECT run_id,state FROM assignments"))
assert states["run-bad"] == "stale"
def test_heartbeat_uses_integer_exact_api_and_surfaces_activity(tmp_path, monkeypatch):
kanban = install_kanban(monkeypatch, [task()], tmp_path)
store = store_with_assignment(tmp_path)
store.offer(0)
pool = coordinator.Coordinator(MASTER, store)
request = signed(
"heartbeat", binding(),
{"note": "active", "activity": "safe output\n"},
)
ack = pool.heartbeat(request)
assert kanban.heartbeats[0][1]["expected_run_id"] == 23
assert (tmp_path / "metis/t_deadbeef.log").read_text() == "safe output\n"
assert ack["kind"] == "ack"
kanban.heartbeat_worker = lambda *_a, **_k: False
with pytest.raises(protocol.ProtocolError, match="no longer owns"):
pool.heartbeat(signed("heartbeat", binding(), {}))
def test_recovery_defers_io_errors_and_lease_failure_releases_ordinal(
tmp_path, monkeypatch, capsys
):
kanban = install_kanban(monkeypatch, [task()], tmp_path)
store = store_with_assignment(tmp_path)
pool = coordinator.Coordinator(MASTER, store)
monkeypatch.setattr(store, "pending_results", lambda: [binding()])
monkeypatch.setattr(pool, "finalize", lambda _record: (_ for _ in ()).throw(OSError("busy")))
pool.recover_results()
assert "recovery deferred" in capsys.readouterr().err
monkeypatch.undo()
kanban = install_kanban(monkeypatch, [task()], tmp_path)
store = protocol.PoolStore(tmp_path / "leases.db")
exact = binding(attempt=3)
store.add(exact, assignment_payload())
store.offer(0)
with store._connect() as connection:
connection.execute("UPDATE assignments SET lease_until=1")
pool = coordinator.Coordinator(MASTER, store)
pool.expire_leases()
assert kanban.blocked[0][1]["expected_run_id"] == 23
assert kanban.blocked[0][1]["kind"] == "transient"
assert store.available_ordinals() == [0, 1, 2]

410
testing/tests/test_hermes_execution_pool_dispatch_v2.py Normal file
View File

@ -0,0 +1,410 @@
"""Coordinator dispatch migration and versioned server contracts."""
from __future__ import annotations
import sqlite3
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
ROOT = Path(__file__).parents[2]
SCRIPTS = ROOT / "services/hermes/scripts"
SCM_SCRIPTS = ROOT / "services/hermes/scm-common/scripts"
sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)]
import execution_pool_coordinator as coordinator # noqa: E402
import execution_pool_protocol as protocol # noqa: E402
import execution_pool_server as server # noqa: E402
from testing.tests.test_hermes_execution_pool_coordinator_v2 import ( # noqa: E402
MASTER,
assignment_payload,
binding,
install_kanban,
store_with_assignment,
task,
)
from testing.tests.test_hermes_execution_pool_mediator import http_request # noqa: E402
def test_reconcile_fences_old_run_then_recovers_current_pathless_run(
tmp_path, monkeypatch
):
item = task(current_run_id=24)
install_kanban(monkeypatch, [item], tmp_path)
store = store_with_assignment(tmp_path)
pool = coordinator.Coordinator(MASTER, store)
monkeypatch.setattr(
coordinator, "assignment_payload",
lambda _db, _connection, _task, _board: assignment_payload(),
)
pool.reconcile()
rows = store._connect().execute(
"SELECT run_id,state,worker_ordinal FROM assignments ORDER BY run_id"
).fetchall()
assert [tuple(row) for row in rows] == [
("23", "stale", 0), ("24", "assigned", 0)
]
def test_reconcile_preserves_owned_workspace_and_exactly_blocks_registry_failure(
tmp_path, monkeypatch
):
owned = task(id="t_owned", current_run_id=30, workspace_path="/owned")
broken = task(id="t_broken", current_run_id=31)
kanban = install_kanban(monkeypatch, [owned, broken], tmp_path)
store = protocol.PoolStore(tmp_path / "pool.db")
pool = coordinator.Coordinator(MASTER, store)
def prepare(_db, _connection, item, _board):
if item.id == "t_broken":
raise RuntimeError("registry unavailable")
return assignment_payload()
monkeypatch.setattr(coordinator, "assignment_payload", prepare)
pool.reconcile()
assert store.active_assignments() == []
assert kanban.blocked[0][0] == "t_broken"
assert kanban.blocked[0][1]["expected_run_id"] == 31
assert all(call[0] != "t_owned" for call in kanban.blocked)
def test_dispatch_claims_only_pathless_task_with_safe_assignment_branch(
tmp_path, monkeypatch
):
item = task(status="running")
install_kanban(monkeypatch, [item], tmp_path)
store = protocol.PoolStore(tmp_path / "pool.db")
pool = coordinator.Coordinator(MASTER, store)
observed = []
def claim(active, limit, eligible):
observed.append((active, limit, eligible("metis", item)))
return [("metis", item.id)]
monkeypatch.setattr(coordinator.cli_lane_dispatch, "claim_ready", claim)
monkeypatch.setattr(
coordinator, "assignment_payload",
lambda *_a: assignment_payload(branch="wt/t_deadbeef"),
)
pool.dispatch()
assert observed == [(set(), 3, True)]
record = store.active_assignments()[0]
assert record["run_id"] == "23"
assert record["payload"]["branch"] == "wt/t_deadbeef"
def test_dispatch_incompatible_unversioned_claim_api_fails_closed(
tmp_path, monkeypatch
):
owned = task(workspace_path="/opt/data/workspace/live")
kanban = install_kanban(monkeypatch, [owned], tmp_path)
store = protocol.PoolStore(tmp_path / "pool.db")
pool = coordinator.Coordinator(MASTER, store)
calls = []
def old_claim(active, limit):
calls.append((active, limit))
return [("metis", owned.id)]
monkeypatch.setattr(coordinator.cli_lane_dispatch, "claim_ready", old_claim)
with pytest.raises(TypeError):
pool.dispatch()
assert calls == []
assert store.active_assignments() == []
assert kanban.reclaimed == []
def test_dispatch_workspace_ownership_race_is_exactly_fenced(
tmp_path, monkeypatch
):
owned = task(workspace_path="/opt/data/workspace/live")
kanban = install_kanban(monkeypatch, [owned], tmp_path)
store = protocol.PoolStore(tmp_path / "pool.db")
pool = coordinator.Coordinator(MASTER, store)
monkeypatch.setattr(
coordinator.cli_lane_dispatch,
"claim_ready",
lambda *_a: [("metis", owned.id)],
)
pool.dispatch()
assert store.active_assignments() == []
assert kanban.blocked == [
(
owned.id,
{
"reason": (
"Distributed claim fenced because an existing workspace "
"is owned by the local lane"
),
"kind": "capability",
"expected_run_id": 23,
},
)
]
def test_dispatch_preparation_failure_is_surfaced_and_full_pool_does_not_claim(
tmp_path, monkeypatch
):
broken = task(current_run_id=None)
kanban = install_kanban(monkeypatch, [broken], tmp_path)
store = protocol.PoolStore(tmp_path / "pool.db")
pool = coordinator.Coordinator(MASTER, store)
monkeypatch.setattr(
coordinator.cli_lane_dispatch,
"claim_ready",
lambda *_a: [("metis", broken.id)],
)
pool.dispatch()
assert kanban.blocked[0][1]["expected_run_id"] is None
assert "canonical run ID" in kanban.blocked[0][1]["reason"]
full = protocol.PoolStore(tmp_path / "full.db")
for ordinal in range(3):
full.add(
binding(task_id=f"t_{ordinal}", run_id=str(ordinal + 1), worker_ordinal=ordinal),
assignment_payload(),
)
pool = coordinator.Coordinator(MASTER, full)
monkeypatch.setattr(
coordinator.cli_lane_dispatch,
"claim_ready",
lambda *_a: (_ for _ in ()).throw(AssertionError("must not claim")),
)
pool.dispatch()
def test_lease_expiry_with_noncanonical_run_is_released_as_stale(tmp_path, monkeypatch):
install_kanban(monkeypatch, [task()], tmp_path)
store = protocol.PoolStore(tmp_path / "pool.db")
exact = binding(run_id="bad-run", attempt=3)
store.add(exact, assignment_payload())
store.offer(0)
with store._connect() as connection:
connection.execute("UPDATE assignments SET lease_until=1")
coordinator.Coordinator(MASTER, store).expire_leases()
assert store._connect().execute("SELECT state FROM assignments").fetchone()[0] == "stale"
def test_terminal_activity_marker_prevents_duplicate_append(tmp_path):
class Kanban:
worker_log_path = staticmethod(
lambda task_id, board=None: str(tmp_path / f"{board}-{task_id}.log")
)
record = {
**binding(),
"result_digest": "a" * 64,
"result": {"final_activity": "terminal evidence\n"},
}
coordinator._append_terminal_activity(Kanban, record)
coordinator._append_terminal_activity(Kanban, record)
text = (tmp_path / "metis-t_deadbeef.log").read_text()
assert text.count("execution-pool-result") == 1
empty = {**record, "result": {"final_activity": ""}}
coordinator._append_terminal_activity(Kanban, empty)
def test_activity_and_terminal_log_reject_invalid_or_symlink_targets(tmp_path):
class Kanban:
worker_log_path = staticmethod(lambda *_a, **_k: str(tmp_path / "worker.log"))
with pytest.raises(protocol.ProtocolError, match="payload"):
coordinator._append_activity(Kanban, {**binding(), "payload": []})
coordinator._append_activity(Kanban, {**binding(), "payload": {"activity": ""}})
(tmp_path / "worker.log").symlink_to(tmp_path / "target")
with pytest.raises(protocol.ProtocolError, match="symlink"):
coordinator._append_terminal_activity(
Kanban,
{
**binding(), "result_digest": "a" * 64,
"result": {"final_activity": "terminal"},
},
)
def test_heartbeat_bad_run_duplicate_and_unstructured_result_paths(
tmp_path, monkeypatch
):
item = task()
kanban = install_kanban(monkeypatch, [item], tmp_path)
bad_binding = binding(run_id="bad-run")
bad_store = protocol.PoolStore(tmp_path / "bad.db")
bad_store.add(bad_binding, assignment_payload())
bad_store.offer(0)
pool = coordinator.Coordinator(MASTER, bad_store)
key = protocol.derive_ordinal_key(MASTER, 0)
with pytest.raises(protocol.ProtocolError, match="canonical"):
pool.heartbeat(protocol.sign_envelope(key, "heartbeat", bad_binding, {}))
store = store_with_assignment(tmp_path / "duplicate")
store.offer(0)
pool = coordinator.Coordinator(MASTER, store)
request = protocol.sign_envelope(
key, "heartbeat", binding(), {"activity": "once"},
delivery_id="same-heartbeat",
)
pool.heartbeat(request)
pool.heartbeat(request)
assert (tmp_path / "metis/t_deadbeef.log").read_text() == "once"
second = binding(run_id="24", worker_ordinal=1)
item.current_run_id = 24
store.add(second, assignment_payload())
pool.result(
protocol.sign_envelope(
protocol.derive_ordinal_key(MASTER, 1), "result", second,
{"structured": [], "returncode": 1},
)
)
assert kanban.blocked[-1][1]["expected_run_id"] == 24
def test_reconcile_and_dispatch_cover_empty_error_and_missing_task_paths(
tmp_path, monkeypatch
):
item = task()
kanban = install_kanban(monkeypatch, [item], tmp_path, boards=["", "metis"])
store = store_with_assignment(tmp_path)
pool = coordinator.Coordinator(MASTER, store)
original_connect = kanban.connect
kanban.list_boards = lambda include_archived=False: []
kanban.connect = lambda board=None: (_ for _ in ()).throw(OSError("busy"))
pool.reconcile()
kanban.connect = original_connect
kanban.list_boards = lambda include_archived=False: [""]
pool.reconcile()
full = SimpleNamespace(
active_assignments=lambda: [], available_ordinals=lambda: []
)
coordinator.Coordinator(MASTER, full).reconcile()
empty = protocol.PoolStore(tmp_path / "missing.db")
monkeypatch.setattr(
coordinator.cli_lane_dispatch, "claim_ready", lambda *_a: [("metis", "missing")]
)
coordinator.Coordinator(MASTER, empty).dispatch()
def test_nonterminal_lease_expiry_is_reoffered_without_kanban_block(tmp_path, monkeypatch):
kanban = install_kanban(monkeypatch, [task()], tmp_path)
store = store_with_assignment(tmp_path)
store.offer(0)
with store._connect() as connection:
connection.execute("UPDATE assignments SET lease_until=1")
coordinator.Coordinator(MASTER, store).expire_leases()
assert kanban.blocked == []
assert store.active_assignments()[0]["attempt"] == 2
class HTTPStore:
def __init__(self, fail=False):
self.fail = fail
def available_ordinals(self):
if self.fail:
raise sqlite3.Error("busy")
return [0, 1, 2]
class HTTPCoordinator:
def __init__(self, fail_ready=False):
self.store = HTTPStore(fail_ready)
@staticmethod
def poll(value):
return {"route": "poll", "value": value}
@staticmethod
def heartbeat(_value):
raise protocol.ProtocolError("stale")
@staticmethod
def result(_value):
raise RuntimeError("storage")
def test_versioned_server_routes_readiness_and_errors():
handler = server.handler_factory(HTTPCoordinator())
assert http_request(handler, "/ready") == (200, {"ready": True, "version": 2})
assert http_request(handler, "/missing")[0] == 404
body = protocol.canonical_json({"safe": True})
assert http_request(handler, "/v1/poll", body=body) == (
200, {"route": "poll", "value": {"safe": True}}
)
assert http_request(handler, "/v1/heartbeat", body=body)[0] == 409
assert http_request(handler, "/v1/result", body=body)[0] == 503
assert http_request(handler, "/unknown", body=body)[0] == 404
assert http_request(handler, "/v1/poll", body=b"")[0] == 409
assert http_request(server.handler_factory(HTTPCoordinator(True)), "/ready")[0] == 503
class RunCoordinator:
instances = []
def __init__(self, key, store):
self.key = key
self.store = store
self.calls = []
self.dispatch_count = 0
self.__class__.instances.append(self)
def expire_leases(self):
self.calls.append("expire")
def recover_results(self):
self.calls.append("recover")
def reconcile(self):
self.calls.append("reconcile")
def dispatch(self):
self.calls.append("dispatch")
self.dispatch_count += 1
if self.dispatch_count > 1:
raise RuntimeError("deferred")
def test_server_once_and_maintenance_loop_are_versioned_and_resilient(
tmp_path, monkeypatch, capsys
):
monkeypatch.setattr(server, "STATE_ROOT", tmp_path)
monkeypatch.setattr(server, "read_key", lambda _path: MASTER)
monkeypatch.setattr(sys, "argv", ["pool", "--once"])
assert server.run(RunCoordinator) == 0
assert RunCoordinator.instances[-1].calls == [
"expire", "recover", "reconcile", "dispatch"
]
started = []
class FakeServer:
def __init__(self, address, _handler, max_workers):
started.append((address, max_workers))
def serve_forever(self):
return
monkeypatch.setattr(sys, "argv", ["pool"])
monkeypatch.setattr(server, "BoundedHTTPServer", FakeServer)
monkeypatch.setattr(
server.time,
"sleep",
lambda _seconds: (_ for _ in ()).throw(StopIteration()),
)
with pytest.raises(StopIteration):
server.run(RunCoordinator)
assert started == [(("0.0.0.0", server.PORT), 8)]
assert "maintenance deferred" in capsys.readouterr().err
def test_coordinator_compatibility_exports_delegate_to_server(monkeypatch):
marker = object()
monkeypatch.setattr(server, "handler_factory", lambda value: (marker, value))
assert coordinator.handler_factory(marker) == (marker, marker)
monkeypatch.setattr(server, "run", lambda value: 17 if value is coordinator.Coordinator else 0)
assert coordinator.main() == 17

448
testing/tests/test_hermes_execution_pool_mediator.py Normal file
View File

@ -0,0 +1,448 @@
"""Isolated mediator, SCM gate, and model-facing API contracts."""
from __future__ import annotations
import json
import subprocess
import sys
import threading
import urllib.error
import urllib.request
from pathlib import Path
import pytest
ROOT = Path(__file__).parents[2]
SCRIPTS = ROOT / "services/hermes/scripts"
SCM_SCRIPTS = ROOT / "services/hermes/scm-common/scripts"
sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)]
import execution_pool_client as client # noqa: E402
import execution_pool_protocol as protocol # noqa: E402
import execution_pool_scm as scm # noqa: E402
KEY = b"d" * 64
RESULT = {
"status": "completed",
"summary": "Completed safely.",
"changed_files": ["safe.py"],
"tests_run": ["pytest"],
"artifacts": [],
"findings": [],
"blockers": [],
}
def binding(**changes):
value = {
"board": "metis", "task_id": "t_deadbeef", "run_id": "42",
"worker_ordinal": 0, "attempt": 1,
}
value.update(changes)
return value
def payload(**changes):
value = {
"context": "safe objective",
"repo_url": "https://scm.bstein.dev/atlas/metis.git",
"branch": "wt/t_deadbeef",
"base_branch": "main",
}
value.update(changes)
return value
def assignment(**payload_changes):
return protocol.sign_envelope(
KEY, "assignment", binding(), payload(**payload_changes)
)
class Response:
def __init__(self, body):
self.body = body
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self, _size=-1):
return self.body
class FakeSCM:
def __init__(self):
self.submitted = []
def checkout(self, _envelope):
return {"workspace": "/workspace/run", "baseline_sha": "a" * 40}
def submit(self, envelope, request):
self.submitted.append((envelope, request))
return {"pull_request": "https://scm.bstein.dev/atlas/metis/pulls/7"}
def test_result_schema_validation_rejects_every_unsafe_shape():
invalid = [
None,
{},
{"structured": []},
{"structured": {**RESULT, "extra": True}},
{"structured": {**RESULT, "status": "unknown"}},
{"structured": {**RESULT, "summary": ""}},
{"structured": {**RESULT, "tests_run": "pytest"}},
{"structured": {**RESULT, "tests_run": [7]}},
]
for value in invalid:
with pytest.raises(protocol.ProtocolError):
client._validate_result(value)
assert client._validate_result({"structured": dict(RESULT)})["structured"] == RESULT
def test_client_post_verifies_response_and_bounds_body(monkeypatch):
boundary = client.ClientBoundary(KEY, FakeSCM())
ack = protocol.sign_envelope(KEY, "ack", binding(), {"accepted": True})
monkeypatch.setattr(client.urllib.request, "urlopen", lambda *_a, **_k: Response(protocol.canonical_json(ack)))
assert boundary._post("/v1/result", ack)["kind"] == "ack"
monkeypatch.setattr(
client.urllib.request,
"urlopen",
lambda *_a, **_k: Response(b"x" * (protocol.MAX_WIRE_BYTES + 1)),
)
with pytest.raises(protocol.ProtocolError, match="exceeds"):
boundary._post("/v1/result", ack)
def test_poll_materializes_only_ordinal_owned_assignment(monkeypatch):
monkeypatch.setattr(client, "ORDINAL", 0)
boundary = client.ClientBoundary(KEY, FakeSCM())
boundary._post = lambda *_a: protocol.sign_envelope(
KEY,
"ack",
binding(board="", task_id="", run_id="", attempt=0),
{"assignment": None},
)
assert boundary.poll() == {"assignment": None}
boundary._post = lambda *_a: assignment()
result = boundary.poll()["assignment"]
assert result["workspace"] == "/workspace/run"
assert result["protocol_version"] == 2
assert "signature" not in result
boundary._post = lambda *_a: protocol.sign_envelope(
KEY, "assignment", binding(worker_ordinal=1), payload()
)
with pytest.raises(protocol.ProtocolError, match="foreign"):
boundary.poll()
def test_heartbeat_and_finish_require_exact_current_binding(monkeypatch):
fake_scm = FakeSCM()
boundary = client.ClientBoundary(KEY, fake_scm)
current = assignment()
boundary.current = current
def post(path, envelope):
kind = "heartbeat" if path.endswith("heartbeat") else "result"
protocol.verify_envelope(KEY, envelope, expected_kind=kind)
return protocol.sign_envelope(
KEY, "ack", binding(), {"accepted": True, "duplicate": False}
)
boundary._post = post
assert boundary.heartbeat(
{"binding": binding(), "payload": {"note": "active"}}
)["ack"]["accepted"]
request = {
"binding": binding(),
"payload": {"structured": dict(RESULT), "returncode": 0},
"title": "Safe change",
"body": "Evidence",
}
finished = boundary.finish(request)
assert finished["ack"]["accepted"] and boundary.current is None
assert fake_scm.submitted
assert finished["structured"]["artifacts"] == [
"https://scm.bstein.dev/atlas/metis/pulls/7"
]
boundary.current = current
with pytest.raises(protocol.ProtocolError, match="heartbeat payload"):
boundary.heartbeat({"binding": binding(), "payload": "bad"})
with pytest.raises(protocol.ProtocolError, match="object"):
boundary._current_for(None)
with pytest.raises(protocol.ProtocolError, match="does not own"):
boundary._current_for(binding(attempt=2))
def test_noncompleted_finish_never_invokes_scm_and_bad_ack_is_rejected():
fake_scm = FakeSCM()
boundary = client.ClientBoundary(KEY, fake_scm)
boundary.current = assignment()
bad = protocol.sign_envelope(KEY, "ack", binding(attempt=2), {"accepted": True})
boundary._post = lambda *_a: bad
blocked = {**RESULT, "status": "blocked", "blockers": ["capacity"]}
with pytest.raises(protocol.ProtocolError, match="binding changed"):
boundary.finish(
{"binding": binding(), "payload": {"structured": blocked, "returncode": 1}}
)
assert fake_scm.submitted == []
boundary._post = lambda *_a: protocol.sign_envelope(
KEY, "result", binding(), {"accepted": True}
)
with pytest.raises(protocol.ProtocolError, match="binding changed"):
boundary.heartbeat({"binding": binding(), "payload": {}})
def http_request(handler, path, *, body=None):
server = protocol.BoundedHTTPServer(("127.0.0.1", 0), handler, max_workers=2)
thread = threading.Thread(target=server.handle_request)
thread.start()
request = urllib.request.Request(
f"http://127.0.0.1:{server.server_port}{path}",
data=body,
method="POST" if body is not None else "GET",
headers={"Content-Type": "application/json"},
)
try:
with urllib.request.urlopen(request, timeout=3) as response:
return response.status, json.loads(response.read())
except urllib.error.HTTPError as error:
return error.code, json.loads(error.read())
finally:
thread.join(timeout=3)
server.server_close()
def test_model_api_exposes_only_gated_state_machine_operations():
class Boundary:
poll = staticmethod(lambda: {"assignment": None})
heartbeat = staticmethod(lambda request: {"heartbeat": request["payload"]})
finish = staticmethod(lambda request: {"finish": request["payload"]})
handler = client.handler_factory(Boundary())
assert http_request(handler, "/ready") == (
200, {"protocol_version": 2, "ready": True}
)
assert http_request(handler, "/missing")[0] == 404
for operation in ("poll", "heartbeat", "finish"):
body = protocol.canonical_json(
{"operation": operation, "binding": binding(), "payload": {}}
)
assert http_request(handler, "/v1/client", body=body)[0] == 200
for value in (
{"operation": "bypass"},
{"operation": "poll", "authority": "steal"},
):
status, response = http_request(
handler, "/v1/client", body=protocol.canonical_json(value)
)
assert status == 409 and response.get("error")
def test_client_main_validates_ordinal_and_starts_bounded_server(monkeypatch):
monkeypatch.setattr(client, "ORDINAL", -1)
with pytest.raises(SystemExit, match="ORDINAL"):
client.main()
started = []
class Server:
def __init__(self, address, _handler, max_workers):
started.append((address, max_workers))
def serve_forever(self):
return
monkeypatch.setattr(client, "ORDINAL", 2)
monkeypatch.setattr(client, "read_key", lambda _path: KEY)
monkeypatch.setattr(client, "BoundedHTTPServer", Server)
monkeypatch.setattr(client, "SCMBoundary", lambda _key: FakeSCM())
assert client.main() == 0
assert started == [(("0.0.0.0", client.PORT), 4)]
def init_checkout(path, branch="wt/t_deadbeef"):
subprocess.run(["git", "init", "-q", str(path)], check=True)
commands = [
("remote", "add", "origin", "https://scm.bstein.dev/atlas/metis.git"),
(
"remote",
"add",
"hermes-broker",
"http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/"
"git/atlas/metis.git",
),
("checkout", "-qb", branch),
("config", "user.email", "test@example.com"),
("config", "user.name", "Test"),
]
for command in commands:
subprocess.run(["git", "-C", str(path), *command], check=True)
(path / "tracked").write_text("safe\n")
subprocess.run(["git", "-C", str(path), "add", "tracked"], check=True)
subprocess.run(["git", "-C", str(path), "commit", "-qm", "initial"], check=True)
return subprocess.check_output(
["git", "-C", str(path), "rev-parse", "HEAD"], text=True
).strip()
def test_scm_run_and_binding_validation(tmp_path, monkeypatch):
checkout = tmp_path / "checkout"
head = init_checkout(checkout)
assert scm._run("rev-parse", "HEAD", cwd=checkout) == head
with pytest.raises(RuntimeError):
scm._run("rev-parse", "missing", cwd=checkout)
monkeypatch.setattr(scm, "MAX_STATUS_BYTES", 1)
with pytest.raises(protocol.ProtocolError, match="output"):
scm._run("rev-parse", "HEAD", cwd=checkout)
monkeypatch.setattr(scm, "ORDINAL", 0)
assert scm._binding(assignment())[1:] == ("metis", "wt/t_deadbeef", "main")
for envelope in (
{**assignment(), "kind": "result"},
{**assignment(), "payload": []},
assignment(repo_url="https://evil.example/metis.git"),
assignment(branch="main"),
assignment(base_branch="../main"),
protocol.sign_envelope(KEY, "assignment", binding(worker_ordinal=1), payload()),
):
with pytest.raises(protocol.ProtocolError):
scm._binding(envelope)
def test_scm_private_paths_and_text_are_bounded(tmp_path, monkeypatch):
monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace")
monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state")
target = scm.workspace_path(assignment())
assert target.name == "42"
assert scm._state_path(assignment()).name == "metis-t_deadbeef-42.json"
bad = {**assignment(), "task_id": "../bad"}
with pytest.raises(protocol.ProtocolError, match="binding"):
scm.workspace_path(bad)
state = tmp_path / "regular"
state.write_text("safe")
assert scm._regular_text(state, 10) == "safe"
with pytest.raises(protocol.ProtocolError, match="invalid"):
scm._regular_text(state, 1)
binary = tmp_path / "binary"
binary.write_bytes(b"\xff")
with pytest.raises(protocol.ProtocolError, match="malformed"):
scm._regular_text(binary, 10)
linked_state = tmp_path / "linked-state"
linked_state.mkdir()
(tmp_path / "state-link").symlink_to(linked_state, target_is_directory=True)
monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state-link")
with pytest.raises(protocol.ProtocolError):
scm._state_path(assignment())
def test_boundary_verification_existing_checkout_and_missing_baseline(tmp_path, monkeypatch):
workspace = tmp_path / "workspace"
state_root = tmp_path / "state"
monkeypatch.setattr(scm, "WORKSPACE_ROOT", workspace)
monkeypatch.setattr(scm, "SCM_ROOT", state_root)
monkeypatch.setattr(scm, "ORDINAL", 0)
boundary = scm.Boundary(KEY)
assert boundary.verify(assignment())["kind"] == "assignment"
destination = scm.workspace_path(assignment())
baseline = init_checkout(destination)
protocol.atomic_json(
scm._state_path(assignment()),
{"baseline_sha": baseline, "repo": "metis", "branch": "wt/t_deadbeef"},
)
assert boundary.checkout(assignment())["baseline_sha"] == baseline
scm._state_path(assignment()).write_text("{}")
with pytest.raises(protocol.ProtocolError, match="baseline"):
boundary.checkout(assignment())
@pytest.mark.parametrize("feature_exists", [True, False])
def test_new_checkout_uses_broker_and_safe_base_fallback(
tmp_path, monkeypatch, feature_exists
):
monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace")
monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state")
monkeypatch.setattr(scm, "ORDINAL", 0)
calls = []
def run(*arguments, cwd=None, timeout=300):
calls.append((arguments, cwd, timeout))
if arguments[0] == "clone":
branch = arguments[arguments.index("--branch") + 1]
destination = Path(arguments[-1])
if branch == "wt/t_deadbeef" and not feature_exists:
raise RuntimeError("missing branch")
(destination / ".git").mkdir(parents=True)
return ""
monkeypatch.setattr(scm, "_run", run)
monkeypatch.setattr(scm, "_workspace_identity", lambda *_a: "a" * 40)
result = scm.Boundary(KEY).checkout(assignment())
assert result["baseline_sha"] == "a" * 40
assert any("hermes-scm-broker" in str(call) for call in calls)
if not feature_exists:
assert any(call[0][0] == "checkout" for call in calls)
def test_failed_clone_never_deletes_unmanaged_state(tmp_path, monkeypatch):
monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace")
monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state")
monkeypatch.setattr(scm, "ORDINAL", 0)
def fail(*arguments, **_kwargs):
destination = Path(arguments[-1])
destination.mkdir(parents=True, exist_ok=True)
(destination / "preserved").write_text("owner data")
raise RuntimeError("clone failed")
monkeypatch.setattr(scm, "_run", fail)
with pytest.raises(protocol.ProtocolError, match="unmanaged"):
scm.Boundary(KEY).checkout(assignment())
assert (scm.workspace_path(assignment()) / "preserved").read_text() == "owner data"
def test_draft_reuse_create_and_submit_gates(tmp_path, monkeypatch):
existing = json.dumps([{"html_url": "https://scm/pulls/1"}]).encode()
monkeypatch.setattr(scm.scm_broker_client, "read", lambda _path: existing)
assert scm.Boundary._draft("metis", "wt/task", "main", "a" * 40, "t", "b") == "https://scm/pulls/1"
monkeypatch.setattr(scm.scm_broker_client, "read", lambda _path: b"[]")
monkeypatch.setattr(
scm.scm_broker_client, "create_draft",
lambda *_a, **_k: json.dumps({"html_url": "https://scm/pulls/2"}).encode(),
)
assert scm.Boundary._draft("metis", "wt/task", "main", "a" * 40, "t", "b") == "https://scm/pulls/2"
monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace")
monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state")
monkeypatch.setattr(scm, "ORDINAL", 0)
destination = scm.workspace_path(assignment())
destination.mkdir(parents=True)
state = scm._state_path(assignment())
protocol.atomic_json(state, {"baseline_sha": "a" * 40})
boundary = scm.Boundary(KEY)
monkeypatch.setattr(scm, "_workspace_identity", lambda *_a: "b" * 40)
outputs = {"status": "", "rev-list": "1", "push": ""}
def run(*arguments, **_kwargs):
return outputs.get(arguments[0], "")
monkeypatch.setattr(scm, "_run", run)
monkeypatch.setattr(boundary, "_draft", lambda *_a: "https://scm/pulls/3")
result = boundary.submit(assignment(), {"title": "safe", "body": "evidence"})
assert result["pull_request"] == "https://scm/pulls/3"
outputs["status"] = "?? untracked"
with pytest.raises(protocol.ProtocolError, match="uncommitted"):
boundary.submit(assignment(), {})
outputs["status"] = ""
outputs["rev-list"] = "0"
assert boundary.submit(assignment(), {})["pull_request"] == ""
with pytest.raises(protocol.ProtocolError, match="metadata"):
boundary.submit(assignment(), {"title": "x" * 513, "body": "x"})

211
testing/tests/test_hermes_execution_pool_project.py Normal file
View File

@ -0,0 +1,211 @@
"""Canonical board-registry and Git-ref assignment contracts."""
from __future__ import annotations
import json
import shutil
import subprocess
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts"
sys.path.insert(0, str(SCRIPTS))
import execution_pool_project as project # noqa: E402
def git(*arguments: str, cwd: Path | None = None) -> str:
return subprocess.check_output(
["git", *arguments], cwd=cwd, text=True, stderr=subprocess.DEVNULL
).strip()
def registry(tmp_path, monkeypatch, board="metis", remote=None, base=None):
projects = tmp_path / "projects"
boards = tmp_path / "boards"
checkout = projects / board
checkout.mkdir(parents=True)
git("init", "-q", cwd=checkout)
git(
"remote", "add", "origin",
remote or f"https://scm.bstein.dev/atlas/{board}.git",
cwd=checkout,
)
if base:
git(
"symbolic-ref", "refs/remotes/origin/HEAD", f"refs/remotes/origin/{base}",
cwd=checkout,
)
entry = boards / board
entry.mkdir(parents=True)
(entry / "board.json").write_text(
json.dumps({"slug": board, "default_workdir": str(checkout)})
)
monkeypatch.setattr(project, "PROJECT_ROOT", projects)
monkeypatch.setattr(project, "BOARD_ROOT", boards)
return checkout, entry / "board.json"
@pytest.mark.parametrize(
"branch",
[
"feature/safe", "fix/safe", "chore/safe", "docs/safe", "test/safe",
"refactor/safe", "wt/t_deadbeef", "review/t_deadbeef", "hermes/safe",
"handoff/safe",
],
)
def test_every_established_feature_namespace_is_accepted(branch):
assert project.validate_branch(branch, feature=True) == branch
@pytest.mark.parametrize(
"branch",
["main", "unknown/safe", "../main", "feature/../../main", "feature/.hidden", "-bad", "é/safe", "x" * 201],
)
def test_ref_validation_denies_traversal_and_unreviewed_namespaces(branch):
with pytest.raises(project.ProjectPolicyError):
project.validate_branch(branch, feature=True)
def test_canonical_registry_resolves_repo_base_and_default_branch(tmp_path, monkeypatch):
checkout, _ = registry(tmp_path, monkeypatch, base="trunk")
assert project.resolve_project("metis") == (
"https://scm.bstein.dev/atlas/metis.git", "trunk", checkout.resolve()
)
task = SimpleNamespace(id="t_deadbeef", branch_name="")
assert project.resolve_assignment("metis", task) == (
"https://scm.bstein.dev/atlas/metis.git", "wt/t_deadbeef", "trunk"
)
@pytest.mark.parametrize("board", ["cassandra", "metis", "soteria", "titan-iac"])
def test_every_atlas_project_uses_its_own_registry_checkout(tmp_path, monkeypatch, board):
registry(tmp_path, monkeypatch, board=board)
repo, base, _ = project.resolve_project(board)
assert repo == f"https://scm.bstein.dev/atlas/{board}.git"
assert base == "main"
def test_registered_project_without_checkout_keeps_its_own_repo_identity(
tmp_path, monkeypatch
):
checkout, _ = registry(tmp_path, monkeypatch, board="metis")
shutil.rmtree(checkout)
assert project.resolve_project("metis") == (
"https://scm.bstein.dev/atlas/metis.git",
"main",
checkout.resolve(),
)
def test_registry_rejects_project_slug_too_long_for_an_atlas_repo(
tmp_path, monkeypatch
):
board = "m" * 101
checkout, _ = registry(tmp_path, monkeypatch, board=board)
shutil.rmtree(checkout)
with pytest.raises(project.ProjectPolicyError, match="repository identity"):
project.resolve_project(board)
def test_noncanonical_remote_head_falls_back_to_main(tmp_path, monkeypatch):
checkout, _ = registry(tmp_path, monkeypatch)
real_run = project._run_git
def run_git(workdir, *arguments):
if arguments[0] == "symbolic-ref":
return "heads/not-origin"
return real_run(workdir, *arguments)
monkeypatch.setattr(project, "_run_git", run_git)
assert project.resolve_project("metis") == (
"https://scm.bstein.dev/atlas/metis.git",
"main",
checkout.resolve(),
)
def test_registry_rejects_slug_traversal_symlink_and_malformed_documents(tmp_path, monkeypatch):
_, board_file = registry(tmp_path, monkeypatch)
with pytest.raises(project.ProjectPolicyError, match="slug"):
project._read_board("../metis")
board_file.write_text("not-json")
with pytest.raises(project.ProjectPolicyError, match="malformed"):
project._read_board("metis")
board_file.write_text(json.dumps({"slug": "other", "default_workdir": "/tmp"}))
with pytest.raises(project.ProjectPolicyError, match="identity"):
project._read_board("metis")
board_file.unlink()
board_file.symlink_to("/etc/passwd")
with pytest.raises(OSError):
project._read_board("metis")
def test_registry_rejects_oversized_archived_and_non_regular_entry(tmp_path, monkeypatch):
_, board_file = registry(tmp_path, monkeypatch)
board_file.write_text("x" * (project.MAX_BOARD_BYTES + 1))
with pytest.raises(project.ProjectPolicyError, match="bounded"):
project._read_board("metis")
board_file.write_text(json.dumps({"slug": "metis", "archived": True}))
with pytest.raises(project.ProjectPolicyError, match="archived"):
project._read_board("metis")
board_file.unlink()
board_file.mkdir()
with pytest.raises(project.ProjectPolicyError, match="regular"):
project._read_board("metis")
@pytest.mark.parametrize(
"remote",
[
"https://token@scm.bstein.dev/atlas/metis.git",
"https://evil.example/atlas/metis.git",
"ssh://git@scm.bstein.dev/atlas/metis.git",
],
)
def test_registry_rejects_credentialed_or_non_atlas_origin(tmp_path, monkeypatch, remote):
registry(tmp_path, monkeypatch, remote=remote)
with pytest.raises(project.ProjectPolicyError, match="origin"):
project.resolve_project("metis")
def test_registry_rejects_missing_or_outside_workdir(tmp_path, monkeypatch):
checkout, board_file = registry(tmp_path, monkeypatch)
board_file.write_text(json.dumps({"slug": "metis"}))
with pytest.raises(project.ProjectPolicyError, match="default_workdir"):
project.resolve_project("metis")
outside = tmp_path / "outside"
checkout.rename(outside)
board_file.write_text(
json.dumps({"slug": "metis", "default_workdir": str(outside)})
)
with pytest.raises(project.ProjectPolicyError, match="outside"):
project.resolve_project("metis")
def test_registry_rejects_non_directory_checkout(tmp_path, monkeypatch):
checkout, _ = registry(tmp_path, monkeypatch)
shutil.rmtree(checkout)
checkout.write_text("not a checkout")
with pytest.raises(project.ProjectPolicyError, match="not a directory"):
project.resolve_project("metis")
def test_git_metadata_failure_and_invalid_task_identity_fail_closed(tmp_path, monkeypatch):
checkout, _ = registry(tmp_path, monkeypatch)
with pytest.raises(project.ProjectPolicyError, match="metadata"):
project._run_git(checkout, "remote", "get-url", "missing")
with pytest.raises(project.ProjectPolicyError, match="identity"):
project.resolve_assignment("metis", SimpleNamespace(id="../../bad"))
def test_workspace_migration_boundary_is_explicit():
assert project.distributed_workspace_eligible(SimpleNamespace(workspace_path=""))
assert project.distributed_workspace_eligible(SimpleNamespace())
assert not project.distributed_workspace_eligible(
SimpleNamespace(workspace_path="/opt/data/workspace/live")
)

241
testing/tests/test_hermes_execution_pool_protocol_v2.py Normal file
View File

@ -0,0 +1,241 @@
"""Version-2 protocol reliability and exact-attempt fencing contracts."""
from __future__ import annotations
import hashlib
import hmac
import json
import sys
import threading
import urllib.request
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import pytest
SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts"
sys.path.insert(0, str(SCRIPTS))
import execution_pool_protocol as protocol # noqa: E402
MASTER = b"m" * 32
def binding(**changes):
value = {
"board": "metis",
"task_id": "t_deadbeef",
"run_id": "42",
"worker_ordinal": 0,
"attempt": 1,
}
value.update(changes)
return value
def resign(envelope, key=MASTER):
unsigned = dict(envelope)
unsigned.pop("signature", None)
envelope["signature"] = hmac.new(
key, protocol.canonical_json(unsigned), hashlib.sha256
).hexdigest()
return envelope
def test_atomic_json_is_private_durable_and_rejects_symlink_parent(tmp_path):
target = tmp_path / "state/value.json"
protocol.atomic_json(target, {"safe": True})
assert json.loads(target.read_text()) == {"safe": True}
assert target.stat().st_mode & 0o777 == 0o600
outside = tmp_path / "outside"
outside.mkdir()
(tmp_path / "linked").symlink_to(outside, target_is_directory=True)
with pytest.raises(protocol.ProtocolError, match="directory"):
protocol.atomic_json(tmp_path / "linked/value.json", {"safe": False})
def test_ordinal_derivation_and_selector_fail_closed():
keys = [protocol.derive_ordinal_key(MASTER, ordinal) for ordinal in range(3)]
assert len(set(keys)) == 3
for master, ordinal in ((b"short", 0), (MASTER, -1), (MASTER, 3)):
with pytest.raises(protocol.ProtocolError, match="derivation"):
protocol.derive_ordinal_key(master, ordinal)
for value in (None, [], {}, {"worker_ordinal": True}, {"worker_ordinal": 3}):
with pytest.raises(protocol.ProtocolError):
protocol.envelope_ordinal(value)
def test_key_reader_bounds_content_and_type(tmp_path):
for content in (b"x" * 31, b"x" * 4097):
path = tmp_path / f"key-{len(content)}"
path.write_bytes(content)
path.chmod(0o600)
with pytest.raises(protocol.ProtocolError, match="length"):
protocol.read_key(path)
directory = tmp_path / "directory"
directory.mkdir(mode=0o700)
with pytest.raises(protocol.ProtocolError, match="regular"):
protocol.read_key(directory)
def test_envelope_rejects_kind_fields_numeric_lifetime_and_digest():
with pytest.raises(protocol.ProtocolError, match="unsupported"):
protocol.sign_envelope(MASTER, "admin", binding(), {})
valid = protocol.sign_envelope(MASTER, "heartbeat", binding(), {})
extra = {**valid, "extra": True}
with pytest.raises(protocol.ProtocolError, match="fields"):
protocol.verify_envelope(MASTER, extra)
wrong_version = resign({**valid, "version": 1})
with pytest.raises(protocol.ProtocolError, match="version"):
protocol.verify_envelope(MASTER, wrong_version)
with pytest.raises(protocol.ProtocolError, match="unexpected"):
protocol.verify_envelope(MASTER, valid, expected_kind="result")
numeric = resign({**valid, "attempt": "not-a-number"})
with pytest.raises(protocol.ProtocolError, match="numeric"):
protocol.verify_envelope(MASTER, numeric)
numeric_text = resign({**valid, "attempt": "1"})
with pytest.raises(protocol.ProtocolError, match="numeric"):
protocol.verify_envelope(MASTER, numeric_text)
typed_identifier = resign({**valid, "run_id": 42})
with pytest.raises(protocol.ProtocolError, match="run_id"):
protocol.verify_envelope(MASTER, typed_identifier)
lifetime = resign({**valid, "expires_at": valid["issued_at"]})
with pytest.raises(protocol.ProtocolError, match="lifetime"):
protocol.verify_envelope(MASTER, lifetime)
digest = resign({**valid, "payload_digest": "0" * 64})
with pytest.raises(protocol.ProtocolError, match="digest"):
protocol.verify_envelope(MASTER, digest)
with pytest.raises(protocol.ProtocolError, match="oversized"):
protocol.verify_envelope(MASTER, {"value": "x" * protocol.MAX_WIRE_BYTES})
def test_wire_parser_rejects_non_object_and_empty():
for body, message in ((b"", "empty"), (b"[]", "object")):
with pytest.raises(protocol.ProtocolError, match=message):
protocol.parse_wire(body)
def test_bounded_http_server_sets_timeout_and_releases_slot():
handled = threading.Event()
class Handler(BaseHTTPRequestHandler):
def do_GET(self): # noqa: N802
handled.set()
self.send_response(204)
self.end_headers()
def log_message(self, *_args):
return
server = protocol.BoundedHTTPServer(("127.0.0.1", 0), Handler, max_workers=1)
thread = threading.Thread(target=server.handle_request)
thread.start()
with urllib.request.urlopen(
f"http://127.0.0.1:{server.server_port}/", timeout=3
) as response:
assert response.status == 204
thread.join(timeout=3)
assert handled.is_set()
acquired = False
for _ in range(100):
acquired = server._slots.acquire(blocking=False)
if acquired:
break
threading.Event().wait(0.01)
assert acquired
server._slots.release()
server.server_close()
def test_bounded_server_releases_slot_when_thread_dispatch_raises(monkeypatch):
server = protocol.BoundedHTTPServer(
("127.0.0.1", 0), BaseHTTPRequestHandler, max_workers=1
)
def explode(*_args):
raise RuntimeError("dispatch failed")
monkeypatch.setattr(ThreadingHTTPServer, "process_request", explode)
with pytest.raises(RuntimeError, match="dispatch"):
server.process_request(object(), ("127.0.0.1", 1))
assert server._slots.acquire(blocking=False)
server._slots.release()
server.server_close()
def test_store_fences_ordinal_attempt_state_and_expired_lease(tmp_path):
store = protocol.PoolStore(tmp_path / "pool.db", lease_seconds=60)
assert protocol.PoolStore._record(None) is None
store.add(binding(), {"context": "safe"})
store.offer(0)
foreign = protocol.sign_envelope(
MASTER, "heartbeat", binding(worker_ordinal=1), {"note": "foreign"}
)
with pytest.raises(protocol.ProtocolError, match="ordinal"):
store.heartbeat(foreign)
with store._connect() as connection:
connection.execute("UPDATE assignments SET lease_until=1")
expired = protocol.sign_envelope(MASTER, "heartbeat", binding(), {"note": "late"})
with pytest.raises(protocol.ProtocolError, match="lease expired"):
store.heartbeat(expired)
terminal = protocol.sign_envelope(MASTER, "result", binding(), {"structured": {}})
with pytest.raises(protocol.ProtocolError, match="lease expired"):
store.accept_result(terminal)
def test_store_pending_result_invalid_state_and_exact_finalize(tmp_path):
store = protocol.PoolStore(tmp_path / "pool.db")
store.add(binding(), {"context": "safe"})
result = protocol.sign_envelope(MASTER, "result", binding(), {"structured": {}})
record, duplicate = store.accept_result(result)
assert duplicate is False and record["state"] == "result"
assert store.pending_results()[0]["result"] == {"structured": {}}
with pytest.raises(protocol.ProtocolError, match="terminal"):
store.finalize(binding(), "invalid")
store.finalize(binding(attempt=2), "stale")
assert store.pending_results()[0]["state"] == "result"
store.finalize(binding(), "finalized")
heartbeat = protocol.sign_envelope(MASTER, "heartbeat", binding(), {})
with pytest.raises(protocol.ProtocolError, match="running"):
store.heartbeat(heartbeat)
def test_expired_attempt_is_reoffered_then_terminally_released(tmp_path):
store = protocol.PoolStore(tmp_path / "pool.db")
store.add(binding(), {"context": "safe"})
store.offer(0)
changed = store.expire_leases(now=10_000_000_000, max_attempts=2)
assert changed[0]["state"] == "assigned" and changed[0]["attempt"] == 2
stale = protocol.sign_envelope(MASTER, "heartbeat", binding(), {})
with pytest.raises(protocol.ProtocolError, match="attempt is stale"):
store.heartbeat(stale)
offered = store.offer(0)
assert offered and offered["attempt"] == 2
changed = store.expire_leases(now=10_000_000_001, max_attempts=2)
assert changed[0]["state"] == "lease_failed"
assert store.available_ordinals() == [0, 1, 2]
def test_store_garbage_collection_removes_old_terminal_and_deliveries(tmp_path):
store = protocol.PoolStore(tmp_path / "pool.db")
store.add(binding(), {"context": "safe"})
heartbeat = protocol.sign_envelope(
MASTER, "heartbeat", binding(), {}, delivery_id="old-delivery"
)
store.heartbeat(heartbeat)
store.finalize(binding(), "stale")
with store._connect() as connection:
connection.execute("UPDATE assignments SET updated_at=1")
connection.execute("UPDATE deliveries SET received_at=1")
assert store.garbage_collect(3600) == 1
assert store.active_assignments() == []
with store._connect() as connection:
assert connection.execute("SELECT count(*) FROM deliveries").fetchone()[0] == 0

200
testing/tests/test_hermes_execution_pool_scm_tampering.py Normal file
View File

@ -0,0 +1,200 @@
"""SCM checkout ownership and private-baseline corruption probes."""
from __future__ import annotations
import sys
import subprocess
from pathlib import Path
import pytest
ROOT = Path(__file__).parents[2]
sys.path[:0] = [
str(ROOT / "services/hermes/scripts"),
str(ROOT / "services/hermes/scm-common/scripts"),
]
import execution_pool_protocol as protocol # noqa: E402
import execution_pool_scm as scm # noqa: E402
from testing.tests.test_hermes_execution_pool_mediator import ( # noqa: E402
KEY,
assignment,
init_checkout,
)
def test_checkout_and_submit_reject_unmanaged_or_corrupt_private_state(
tmp_path, monkeypatch
):
monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace")
monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state")
monkeypatch.setattr(scm, "ORDINAL", 0)
destination = scm.workspace_path(assignment())
destination.mkdir(parents=True)
(destination / "owner-file").write_text("preserve")
boundary = scm.Boundary(KEY)
with pytest.raises(protocol.ProtocolError, match="non-empty"):
boundary.checkout(assignment())
(destination / "owner-file").unlink()
def clone_with_empty_failure(*arguments, **_kwargs):
if arguments[0] == "clone" and arguments[4] == "wt/t_deadbeef":
destination.mkdir(parents=True, exist_ok=True)
raise RuntimeError("missing")
if arguments[0] == "clone":
(destination / ".git").mkdir(parents=True)
return ""
monkeypatch.setattr(scm, "_run", clone_with_empty_failure)
monkeypatch.setattr(scm, "_workspace_identity", lambda *_a: "a" * 40)
assert boundary.checkout(assignment())["baseline_sha"] == "a" * 40
protocol.atomic_json(scm._state_path(assignment()), {"baseline_sha": "bad"})
with pytest.raises(protocol.ProtocolError, match="baseline"):
boundary.submit(assignment(), {})
def test_scm_identity_is_data_only_and_supports_packed_refs(tmp_path):
checkout = tmp_path / "checkout"
subprocess.run(["git", "init", "-q", str(checkout)], check=True)
subprocess.run(
[
"git", "-C", str(checkout), "remote", "add", "origin",
"https://scm.bstein.dev/atlas/titan-iac.git",
],
check=True,
)
subprocess.run(
[
"git", "-C", str(checkout), "remote", "add", "hermes-broker",
scm._broker_repo("titan-iac"),
],
check=True,
)
subprocess.run(
[
"git", "-C", str(checkout), "checkout", "-qb",
"feature/hermes-safe-pool",
],
check=True,
)
subprocess.run(
["git", "-C", str(checkout), "config", "user.email", "a@b.c"],
check=True,
)
subprocess.run(
["git", "-C", str(checkout), "config", "user.name", "Test"],
check=True,
)
(checkout / "tracked").write_text("safe\n")
subprocess.run(["git", "-C", str(checkout), "add", "tracked"], check=True)
subprocess.run(
["git", "-C", str(checkout), "commit", "-qm", "initial"], check=True
)
expected = subprocess.check_output(
["git", "-C", str(checkout), "rev-parse", "HEAD"], text=True
).strip()
subprocess.run(
["git", "-C", str(checkout), "pack-refs", "--all"], check=True
)
assert scm._workspace_identity(
checkout,
"titan-iac",
"feature/hermes-safe-pool",
) == expected
(checkout / ".git/refs/heads/feature").mkdir(parents=True)
(checkout / ".git/refs/heads/feature/hermes-safe-pool").symlink_to(
"/etc/passwd"
)
with pytest.raises(protocol.ProtocolError, match="symlink"):
scm._workspace_identity(
checkout,
"titan-iac",
"feature/hermes-safe-pool",
)
def test_scm_paths_and_identity_reject_symlink_and_git_tampering(
tmp_path, monkeypatch
):
workspace = tmp_path / "workspace"
workspace.mkdir()
workspace_link = tmp_path / "workspace-link"
workspace_link.symlink_to(workspace, target_is_directory=True)
monkeypatch.setattr(scm, "WORKSPACE_ROOT", workspace_link)
with pytest.raises(protocol.ProtocolError, match="root"):
scm.workspace_path(assignment())
monkeypatch.setattr(scm, "WORKSPACE_ROOT", workspace)
(workspace / "runs").mkdir()
(workspace / "runs/metis").symlink_to(tmp_path, target_is_directory=True)
with pytest.raises(protocol.ProtocolError, match="parent"):
scm.workspace_path(assignment())
state = tmp_path / "state"
state.mkdir()
monkeypatch.setattr(scm, "SCM_ROOT", state)
expected_state = state / "metis-t_deadbeef-42.json"
expected_state.symlink_to("/etc/passwd")
with pytest.raises(protocol.ProtocolError, match="state"):
scm._state_path(assignment())
checkout = tmp_path / "checkout"
checkout.mkdir()
with pytest.raises(protocol.ProtocolError, match="metadata"):
scm._workspace_identity(checkout, "metis", "wt/t_deadbeef")
init_checkout(checkout)
subprocess.run(
[
"git", "-C", str(checkout), "remote", "set-url", "origin",
"https://evil.example/repo.git",
],
check=True,
)
with pytest.raises(protocol.ProtocolError, match="origin"):
scm._workspace_identity(checkout, "metis", "wt/t_deadbeef")
subprocess.run(
[
"git", "-C", str(checkout), "remote", "set-url", "origin",
"https://scm.bstein.dev/atlas/metis.git",
],
check=True,
)
subprocess.run(
[
"git", "-C", str(checkout), "remote", "set-url", "hermes-broker",
"http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/"
"git/atlas/soteria.git",
],
check=True,
)
with pytest.raises(protocol.ProtocolError, match="broker remote"):
scm._workspace_identity(checkout, "metis", "wt/t_deadbeef")
subprocess.run(
[
"git", "-C", str(checkout), "remote", "set-url", "hermes-broker",
"http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/"
"git/atlas/metis.git",
],
check=True,
)
with pytest.raises(protocol.ProtocolError, match="branch"):
scm._workspace_identity(checkout, "metis", "review/other")
monkeypatch.setattr(
scm,
"_run",
lambda *arguments, **_kwargs: {
"remote": (
"http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/"
"git/atlas/metis.git"
if arguments[-1] == "hermes-broker"
else "https://scm.bstein.dev/atlas/metis.git"
),
"symbolic-ref": "wt/t_deadbeef",
"rev-parse": "invalid",
}[arguments[0]],
)
with pytest.raises(protocol.ProtocolError, match="HEAD"):
scm._workspace_identity(checkout, "metis", "wt/t_deadbeef")

246
testing/tests/test_hermes_execution_pool_worker_execute.py Normal file
View File

@ -0,0 +1,246 @@
"""Worker execution, fallback, terminal handoff, and exception contracts."""
from __future__ import annotations
import json
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).parents[2]
SCRIPTS = ROOT / "services/hermes/scripts"
sys.path.insert(0, str(SCRIPTS))
from cli_lane_config import ProcessResult, Route # noqa: E402
import execution_pool_protocol as protocol # noqa: E402
import execution_pool_worker as worker # noqa: E402
from testing.tests.test_hermes_execution_pool_worker_v2 import assignment # noqa: E402
def route(provider="codex", effort="high"):
return Route(
provider=provider,
model=f"{provider}-model",
effort=effort,
profile="agent",
classifier="test",
reason="unit",
latency_ms=1,
fallback_chain=(provider,),
)
def completed_result(**changes):
structured = {
"status": "completed",
"summary": "Completed safely.",
"changed_files": ["safe.py"],
"tests_run": ["pytest"],
"artifacts": [],
"findings": [],
"blockers": [],
}
value = ProcessResult(
returncode=0,
output="done",
structured=structured,
capacity_failure=False,
)
for name, replacement in changes.items():
setattr(value, name, replacement)
return value
def prepare_execute(tmp_path, monkeypatch, *, item=None, results=None, git_status=""):
worker_root = tmp_path / "worker"
workspace = worker_root / "runs/metis/t_deadbeef/23"
workspace.mkdir(parents=True)
exact = item or assignment(
workspace=str(workspace),
payload={
"context": "safe objective",
"assignee": "cli-auto",
"deadline_unix": 10_000_000_000,
"max_runtime_seconds": 3600,
},
)
monkeypatch.setattr(worker, "ROOT", worker_root)
monkeypatch.setattr(worker, "ORDINAL", 0)
monkeypatch.setattr(worker, "NODE", "titan-05")
monkeypatch.setattr(worker, "_bind_provider_sessions", lambda _a: None)
monkeypatch.setattr(worker.cli_lane_runner, "load_json", lambda _path: {})
routes = [route("codex"), route("claude")]
route_calls = []
def select_route(*arguments, **keywords):
route_calls.append((arguments, keywords))
return routes[min(len(route_calls) - 1, len(routes) - 1)]
monkeypatch.setattr(worker.cli_lane_runner, "select_route", select_route)
monkeypatch.setattr(worker.cli_lane_runner, "fresh_unavailable_provider", lambda: "claude")
monkeypatch.setattr(worker.cli_lane_runner, "git_handoff", lambda *_a: "\nhandoff")
outcomes = list(results or [completed_result()])
provider_calls = []
def run_provider(*arguments):
provider_calls.append(arguments)
return outcomes.pop(0)
monkeypatch.setattr(worker.cli_lane_runner, "run_provider", run_provider)
client_calls = []
def client(operation, **values):
client_calls.append((operation, values))
return {"ack": {"accepted": True}}
monkeypatch.setattr(worker, "_client", client)
monkeypatch.setattr(worker, "_git", lambda *_a: git_status)
refreshed = []
monkeypatch.setattr(worker, "_refresh_assignment", lambda exact: refreshed.append(exact))
return exact, workspace, client_calls, provider_calls, route_calls, refreshed
def test_execute_completed_clean_result_refreshes_and_finishes_exact_run(
tmp_path, monkeypatch
):
exact, _workspace, calls, providers, routes, refreshed = prepare_execute(
tmp_path, monkeypatch
)
worker.execute(exact)
assert len(providers) == 1 and len(routes) == 1
assert refreshed == [worker._binding(exact)]
operations = [name for name, _values in calls]
assert operations == ["heartbeat", "finish"]
finish = calls[-1][1]["payload"]
assert finish["structured"]["status"] == "completed"
assert finish["node"] == "titan-05"
state = json.loads(worker._state_path(exact).read_text())
assert state["terminal_at"] > 0 and state["baseline_sha"] == "a" * 40
def test_execute_capacity_fallback_changes_provider_and_preserves_handoff(
tmp_path, monkeypatch
):
first = completed_result(capacity_failure=True, output="capacity")
second = completed_result()
exact, _workspace, calls, providers, routes, _refreshed = prepare_execute(
tmp_path, monkeypatch, results=[first, second]
)
worker.execute(exact)
assert len(providers) == 2 and len(routes) == 2
assert providers[1][0].provider == "claude"
assert providers[1][1].endswith("handoff")
heartbeats = [value for operation, value in calls if operation == "heartbeat"]
assert any("fallback=codex->claude" in value["payload"]["note"] for value in heartbeats)
def test_execute_dirty_workspace_downgrades_completed_result(tmp_path, monkeypatch):
exact, _workspace, calls, _providers, _routes, refreshed = prepare_execute(
tmp_path, monkeypatch, git_status="?? untracked"
)
worker.execute(exact)
structured = calls[-1][1]["payload"]["structured"]
assert structured["status"] == "incomplete"
assert "uncommitted or untracked" in structured["blockers"][0]
assert refreshed == []
def test_execute_fills_missing_lists_for_failed_provider_result(tmp_path, monkeypatch):
failed = ProcessResult(
returncode=1,
output="failed",
structured={"status": "blocked", "summary": "Provider failed."},
capacity_failure=False,
)
exact, _workspace, calls, _providers, _routes, _refreshed = prepare_execute(
tmp_path, monkeypatch, results=[failed]
)
worker.execute(exact)
structured = calls[-1][1]["payload"]["structured"]
for name in ("changed_files", "tests_run", "artifacts", "findings", "blockers"):
assert structured[name] == []
def test_execute_rejects_payload_lease_baseline_and_terminal_ack_failures(
tmp_path, monkeypatch
):
exact, *_ = prepare_execute(tmp_path, monkeypatch)
with pytest.raises(protocol.ProtocolError, match="payload"):
worker.execute({**exact, "payload": []})
monkeypatch.setattr(worker, "_client", lambda *_a, **_k: {"ack": {"accepted": False}})
with pytest.raises(protocol.ProtocolError, match="lease"):
worker.execute(exact)
exact, *_ = prepare_execute(tmp_path / "baseline", monkeypatch)
exact["baseline_sha"] = "invalid"
with pytest.raises(RuntimeError, match="baseline"):
worker.execute(exact)
exact, *_ = prepare_execute(tmp_path / "ack", monkeypatch)
calls = []
def client(operation, **_values):
calls.append(operation)
return {"ack": {"accepted": operation != "finish"}}
monkeypatch.setattr(worker, "_client", client)
with pytest.raises(protocol.ProtocolError, match="terminal"):
worker.execute(exact)
def test_execute_heartbeat_transport_failure_loses_lease(tmp_path, monkeypatch):
exact, *_ = prepare_execute(tmp_path, monkeypatch)
monkeypatch.setattr(
worker, "_client", lambda *_a, **_k: (_ for _ in ()).throw(ValueError("offline"))
)
with pytest.raises(protocol.ProtocolError, match="lease"):
worker.execute(exact)
def test_report_exception_is_transient_exact_and_transport_safe(monkeypatch):
calls = []
def client(operation, **values):
calls.append((operation, values))
return {"ack": {"accepted": True}}
monkeypatch.setattr(worker, "_client", client)
assert worker.report_exception(assignment(), RuntimeError("boom"))
payload = calls[0][1]["payload"]
assert payload["capacity_failure"] is True
assert payload["structured"]["status"] == "blocked"
assert payload["structured"]["blockers"] == ["RuntimeError: boom"]
monkeypatch.setattr(
worker, "_client", lambda *_a, **_k: (_ for _ in ()).throw(OSError("offline"))
)
assert worker.report_exception(assignment(), OSError("boom")) is False
def test_main_idle_and_exception_paths_do_not_spin_silently(monkeypatch, capsys):
monkeypatch.setattr(worker, "readiness", lambda: None)
monkeypatch.setattr(worker, "garbage_collect", lambda: 0)
monkeypatch.setattr(worker, "_poll", lambda: None)
monkeypatch.setattr(
worker.time, "sleep", lambda _seconds: (_ for _ in ()).throw(SystemExit("stop"))
)
with pytest.raises(SystemExit, match="stop"):
worker.main()
exact = assignment()
monkeypatch.setattr(worker, "_poll", lambda: exact)
monkeypatch.setattr(worker, "execute", lambda _a: (_ for _ in ()).throw(RuntimeError("boom")))
monkeypatch.setattr(worker, "report_exception", lambda _a, _e: True)
with pytest.raises(SystemExit, match="stop"):
worker.main()
assert "surfaced" in capsys.readouterr().out
monkeypatch.setattr(
worker, "garbage_collect", lambda: (_ for _ in ()).throw(RuntimeError("gc"))
)
monkeypatch.setattr(worker, "report_exception", lambda *_a: False)
with pytest.raises(SystemExit, match="stop"):
worker.main()
assert "deferred" in capsys.readouterr().out

292
testing/tests/test_hermes_execution_pool_worker_v2.py Normal file
View File

@ -0,0 +1,292 @@
"""Model worker path, evidence, retention, and readiness contracts."""
from __future__ import annotations
import json
import os
import subprocess
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).parents[2]
SCRIPTS = ROOT / "services/hermes/scripts"
sys.path.insert(0, str(SCRIPTS))
import execution_pool_protocol as protocol # noqa: E402
import execution_pool_worker as worker # noqa: E402
def assignment(**changes):
value = {
"board": "metis", "task_id": "t_deadbeef", "run_id": "23",
"worker_ordinal": 0, "attempt": 1, "protocol_version": 2,
"workspace": "/workspace/runs/metis/t_deadbeef/23",
"baseline_sha": "a" * 40,
"payload": {"context": "safe objective", "assignee": "cli-auto"},
}
value.update(changes)
return value
class Response:
def __init__(self, value):
self.value = value
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self, _size=-1):
return self.value
def test_post_client_and_poll_validate_every_response_boundary(monkeypatch):
monkeypatch.setattr(
worker.urllib.request, "urlopen",
lambda *_a, **_k: Response(json.dumps({"safe": True}).encode()),
)
assert worker._post("http://mediator", {}) == {"safe": True}
monkeypatch.setattr(
worker.urllib.request, "urlopen",
lambda *_a, **_k: Response(b"x" * (64 * 1024 + 1)),
)
with pytest.raises(protocol.ProtocolError, match="wire"):
worker._post("http://mediator", {})
monkeypatch.setattr(
worker.urllib.request, "urlopen", lambda *_a, **_k: Response(b"[]")
)
with pytest.raises(protocol.ProtocolError, match="object"):
worker._post("http://mediator", {})
monkeypatch.setattr(worker, "_post", lambda *_a, **_k: {"error": "denied"})
with pytest.raises(protocol.ProtocolError, match="denied"):
worker._client("poll")
monkeypatch.setattr(worker, "_post", lambda *_a, **_k: {"safe": True})
assert worker._client("poll") == {"safe": True}
monkeypatch.setattr(worker, "ORDINAL", 0)
monkeypatch.setattr(worker, "_client", lambda *_a, **_k: {"assignment": None})
assert worker._poll() is None
for value in ([], assignment(worker_ordinal=1), assignment(protocol_version=1)):
monkeypatch.setattr(worker, "_client", lambda *_a, value=value, **_k: {"assignment": value})
with pytest.raises(protocol.ProtocolError, match="foreign"):
worker._poll()
monkeypatch.setattr(worker, "_client", lambda *_a, **_k: {"assignment": assignment()})
assert worker._poll()["task_id"] == "t_deadbeef"
assert worker._binding(assignment()) == {
"board": "metis", "task_id": "t_deadbeef", "run_id": "23",
"worker_ordinal": 0, "attempt": 1,
}
def test_state_path_rejects_traversal_root_and_leaf_symlinks(tmp_path, monkeypatch):
monkeypatch.setattr(worker, "ROOT", tmp_path)
path = worker._state_path(assignment())
assert path == tmp_path / "session-state/metis/t_deadbeef/23.json"
with pytest.raises(protocol.ProtocolError, match="invalid"):
worker._state_path(assignment(task_id="../bad"))
state_root = tmp_path / "session-state"
outside = tmp_path / "outside"
path.parent.rmdir()
path.parent.parent.rmdir()
state_root.rmdir()
outside.mkdir()
state_root.symlink_to(outside, target_is_directory=True)
with pytest.raises(protocol.ProtocolError, match="root"):
worker._state_path(assignment())
state_root.unlink()
path = worker._state_path(assignment())
path.symlink_to("/etc/passwd")
with pytest.raises(protocol.ProtocolError, match="symlink"):
worker._state_path(assignment())
def prepare_provider_roots(tmp_path, monkeypatch):
worker_root = tmp_path / "worker"
data_root = tmp_path / "data"
codex = tmp_path / "runtime/codex"
claude = tmp_path / "runtime/claude"
(worker_root / "provider-state").mkdir(parents=True)
data_root.mkdir()
codex.mkdir(parents=True)
claude.mkdir(parents=True)
monkeypatch.setattr(worker, "ROOT", worker_root)
monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root)
monkeypatch.setenv("CODEX_HOME", str(codex))
monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(claude))
return worker_root, data_root, codex, claude
def test_provider_session_binding_replaces_only_owned_symlinks(tmp_path, monkeypatch):
worker_root, data_root, codex, claude = prepare_provider_roots(tmp_path, monkeypatch)
old = tmp_path / "old"
old.mkdir()
(data_root / "home").symlink_to(old, target_is_directory=True)
(codex / "sessions").symlink_to(old, target_is_directory=True)
worker._bind_provider_sessions(assignment())
assert "metis/t_deadbeef/23" in str((data_root / "home").resolve())
assert "metis/t_deadbeef/23" in str((codex / "sessions").resolve())
settings = worker_root / "provider-state/metis/t_deadbeef/23/home/.claude/settings.json"
assert json.loads(settings.read_text()) == {}
worker._bind_provider_sessions(assignment())
with pytest.raises(protocol.ProtocolError, match="binding"):
worker._bind_provider_sessions(assignment(run_id="../bad"))
monkeypatch.setattr(worker, "ROOT", tmp_path / "missing")
with pytest.raises(protocol.ProtocolError, match="unavailable"):
worker._bind_provider_sessions(assignment())
def test_provider_session_binding_rejects_durable_and_runtime_tampering(
tmp_path, monkeypatch
):
worker_root, data_root, codex, _claude = prepare_provider_roots(tmp_path, monkeypatch)
run_parent = worker_root / "provider-state/metis"
run_parent.symlink_to(tmp_path, target_is_directory=True)
with pytest.raises(protocol.ProtocolError, match="session path"):
worker._bind_provider_sessions(assignment())
run_parent.unlink()
home = worker_root / "provider-state/metis/t_deadbeef/23/home"
home.parent.mkdir(parents=True)
home.symlink_to(tmp_path, target_is_directory=True)
with pytest.raises(protocol.ProtocolError, match="HOME"):
worker._bind_provider_sessions(assignment())
home.unlink()
(data_root / "home").write_text("not-owned")
with pytest.raises(protocol.ProtocolError, match="task-bound"):
worker._bind_provider_sessions(assignment())
(data_root / "home").unlink()
(codex / "sessions").write_text("not-owned")
with pytest.raises(protocol.ProtocolError, match="not a symlink"):
worker._bind_provider_sessions(assignment())
(codex / "sessions").unlink()
durable_sessions = (
worker_root / "provider-state/metis/t_deadbeef/23/codex/sessions"
)
durable_sessions.rmdir()
durable_sessions.symlink_to(tmp_path, target_is_directory=True)
with pytest.raises(protocol.ProtocolError, match="session path"):
worker._bind_provider_sessions(assignment())
def test_prompt_activity_git_and_result_bounding(tmp_path, monkeypatch):
text = worker._prompt("objective", tmp_path, worker._binding(assignment()))
assert "no Kubernetes identity" in text and "objective" in text
missing = tmp_path / "missing.log"
assert worker._read_activity(missing, 7) == ("", 7)
log = tmp_path / "worker.log"
log.write_text("abc")
assert worker._read_activity(log, 0) == ("abc", 3)
assert worker._read_activity(log, 99) == ("abc", 3)
fifo = tmp_path / "fifo"
os.mkfifo(fifo)
with pytest.raises(protocol.ProtocolError, match="regular"):
worker._read_activity(fifo, 0)
repo = tmp_path / "repo"
subprocess.run(["git", "init", "-q", str(repo)], check=True)
assert worker._git(repo, "status", "--porcelain") == ""
with pytest.raises(RuntimeError, match="ambiguous"):
worker._git(repo, "rev-parse", "missing")
value = {
"status": "completed", "summary": "s" * 20_000,
"changed_files": ["x" * 3000] * 100,
"tests_run": "not-list", "artifacts": [], "findings": [], "blockers": [],
}
bounded = worker._bounded_result(value)
assert len(protocol.canonical_json(bounded)) <= 32 * 1024
assert bounded["tests_run"] == [] and len(bounded["summary"]) <= 8000
def test_refresh_assignment_requires_exact_binding(monkeypatch):
exact = worker._binding(assignment())
monkeypatch.setattr(worker, "_poll", lambda: assignment())
assert worker._refresh_assignment(exact)["run_id"] == "23"
for value in (None, assignment(attempt=2)):
monkeypatch.setattr(worker, "_poll", lambda value=value: value)
with pytest.raises(protocol.ProtocolError, match="changed"):
worker._refresh_assignment(exact)
def write_gc_state(root, task_id, workspace, terminal_at):
path = root / f"session-state/metis/{task_id}/23.json"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps({"terminal_at": terminal_at, "workspace": str(workspace)}))
return path
def init_clean_repo(path):
subprocess.run(["git", "init", "-q", str(path)], check=True)
subprocess.run(["git", "-C", str(path), "config", "user.email", "a@b.c"], check=True)
subprocess.run(["git", "-C", str(path), "config", "user.name", "Test"], check=True)
(path / "tracked").write_text("safe")
subprocess.run(["git", "-C", str(path), "add", "tracked"], check=True)
subprocess.run(["git", "-C", str(path), "commit", "-qm", "initial"], check=True)
def test_retention_skips_young_outside_dirty_and_symlink_workspaces(tmp_path, monkeypatch):
monkeypatch.setattr(worker, "ROOT", tmp_path)
monkeypatch.setattr(worker, "RETENTION_SECONDS", 3600)
run_root = tmp_path / "runs/metis"
clean = run_root / "clean/23"
dirty = run_root / "dirty/23"
clean.mkdir(parents=True)
dirty.mkdir(parents=True)
init_clean_repo(clean)
init_clean_repo(dirty)
(dirty / "untracked").write_text("dirty")
now = 10_000
write_gc_state(tmp_path, "clean", clean, 1)
dirty_state = write_gc_state(tmp_path, "dirty", dirty, 1)
young = write_gc_state(tmp_path, "young", clean, now)
outside = write_gc_state(tmp_path, "outside", tmp_path / "missing", 1)
assert worker.garbage_collect(now=now) == 1
assert not clean.exists() and dirty.exists()
assert dirty_state.exists() and young.exists() and outside.exists()
link = run_root / "link/23"
link.parent.mkdir()
link.symlink_to(dirty, target_is_directory=True)
link_state = write_gc_state(tmp_path, "link", link, 1)
assert worker.garbage_collect(now=now) == 0
assert link_state.exists()
def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monkeypatch):
worker_root = tmp_path / "worker"
data_root = tmp_path / "data"
codex = tmp_path / "codex"
claude = tmp_path / "claude"
for path in (worker_root, worker_root / "provider-state", data_root, codex, claude):
path.mkdir(parents=True, exist_ok=True)
(codex / "auth.json").write_text("{}")
(claude / ".credentials.json").write_text("{}")
schema = tmp_path / "schema/result.json"
monkeypatch.setattr(worker, "ORDINAL", 0)
monkeypatch.setattr(worker, "ROOT", worker_root)
monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root)
monkeypatch.setattr(worker.cli_lane_runner, "RESULT_SCHEMA_PATH", schema)
monkeypatch.setenv("CODEX_HOME", str(codex))
monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(claude))
polled = []
monkeypatch.setattr(worker, "_poll", lambda: polled.append(True))
worker.readiness()
assert polled and json.loads(schema.read_text()) == worker.cli_lane_runner.RESULT_SCHEMA
monkeypatch.setattr(worker, "ORDINAL", 3)
with pytest.raises(protocol.ProtocolError, match="ordinal"):
worker.readiness()
monkeypatch.setattr(worker, "ORDINAL", 0)
monkeypatch.setattr(worker, "ROOT", tmp_path / "missing")
with pytest.raises(protocol.ProtocolError, match="path"):
worker.readiness()
monkeypatch.setattr(worker, "ROOT", worker_root)
(codex / "auth.json").unlink()
with pytest.raises(protocol.ProtocolError, match="credential"):
worker.readiness()

2
testing/tests/test_hermes_gitea_pr_integration.py
View File

@ -142,7 +142,7 @@ def test_flux_manifest_isolates_vault_token_in_separate_broker_only():
)
)
boundary = common["configMapGenerator"][0]
assert boundary["name"] == "hermes-scm-boundary"
assert boundary["name"] == "hermes-scm-boundary-v2"
assert "gitea_api.py=scripts/gitea_api.py" in boundary["files"]
assert "gitea_api_policy.py=scripts/gitea_api_policy.py" in boundary["files"]
assert "scm_broker_client.py=scripts/scm_broker_client.py" in boundary["files"]

6
testing/tests/test_hermes_node_account_privilege_audit.py
View File

@ -2,6 +2,7 @@
from __future__ import annotations
import shutil
import subprocess
from pathlib import Path
@ -115,8 +116,11 @@ def test_visudo_valid_numeric_and_alias_grants_fail_closed(
monkeypatch.setattr(module, "ACCOUNT_UID", 1200)
policy = module.HOST_ETC / "sudoers"
policy.write_text(value, encoding="utf-8")
visudo = shutil.which("visudo")
if visudo is None:
pytest.skip("visudo is not installed in this test environment")
validation = subprocess.run(
["/usr/bin/visudo", "-c", "-f", str(policy)],
[visudo, "-c", "-f", str(policy)],
check=False,
capture_output=True,
text=True,

62
testing/tests/test_hermes_runtime_access.py
View File

@ -3,6 +3,8 @@
from __future__ import annotations
import importlib.util
import hashlib
import hmac
import json
import sys
import urllib.request
@ -205,41 +207,51 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp
assert auth == {"version": 1, "providers": {}, "credential_pool": {}}
def test_execution_worker_separates_channel_key_and_durable_sessions(
def test_execution_worker_and_mediator_separate_credentials_and_hmac(
tmp_path: Path, monkeypatch
):
stage = _load("stage_runtime_access")
vault = tmp_path / "vault"
runtime = tmp_path / "runtime"
worker = tmp_path / "worker"
provider_access = tmp_path / "provider-access"
pool_access = tmp_path / "pool-access"
scm_access = tmp_path / "scm-access"
vault.mkdir()
(vault / "execution-pool-key").write_text("e" * 64)
(vault / "claude-credentials").write_text(
(vault / "claude-credentials-1").write_text(
json.dumps({"claudeAiOauth": {"refreshToken": "claude-refresh"}})
)
(vault / "codex-auth").write_text(
(vault / "codex-auth-1").write_text(
json.dumps({"tokens": {"refresh_token": "codex-refresh"}})
)
(vault / "gitea-token").write_text("test-gitea-token")
(vault / "gitea-username").write_text("hermes-automation")
monkeypatch.setattr(stage, "VAULT_ROOT", vault)
monkeypatch.setattr(stage, "RUNTIME_ROOT", runtime)
monkeypatch.setattr(stage, "WORKER_ROOT", worker)
monkeypatch.setattr(stage, "PROVIDER_ACCESS_ROOT", provider_access)
monkeypatch.setattr(stage, "POOL_ACCESS_ROOT", pool_access)
monkeypatch.setattr(stage, "SCM_ACCESS_ROOT", scm_access)
monkeypatch.setenv("HERMES_WORKER_ORDINAL", "1")
monkeypatch.setattr(stage.os, "chown", lambda *_args: None)
monkeypatch.setattr(stage.os, "fchown", lambda *_args: None)
stage.stage_execution_worker()
assert (pool_access / "execution-pool-key").read_text().strip() == "e" * 64
assert not pool_access.exists()
assert not (runtime / "execution-pool-key").exists()
assert (scm_access / "gitea-token").read_text().strip() == "test-gitea-token"
assert not (runtime / "codex/sessions").exists()
assert not (runtime / "claude/projects").exists()
assert (runtime / "codex/auth.json").stat().st_mode & 0o777 == 0o600
assert (provider_access / "codex/auth.json").stat().st_mode & 0o777 == 0o600
refreshed = {"tokens": {"refresh_token": "provider-rotated"}}
(provider_access / "codex/auth.json").write_text(json.dumps(refreshed))
(provider_access / "codex/auth.json").chmod(0o600)
stage.stage_execution_worker()
assert json.loads((provider_access / "codex/auth.json").read_text()) == refreshed
master = "e" * 64
(vault / "execution-pool-key").write_text(master)
stage.stage_execution_mediator()
expected = hmac.new(
master.encode(), b"hermes-execution-pool-v2:worker:1", hashlib.sha256
).hexdigest()
assert (pool_access / "execution-pool-key").read_text().strip() == expected
def test_execution_worker_fails_closed_without_channel_credential(
@ -248,19 +260,14 @@ def test_execution_worker_fails_closed_without_channel_credential(
stage = _load("stage_runtime_access")
vault = tmp_path / "vault"
vault.mkdir()
(vault / "claude-credentials").write_text(
(vault / "claude-credentials-0").write_text(
json.dumps({"claudeAiOauth": {"refreshToken": "claude-refresh"}})
)
(vault / "codex-auth").write_text(
json.dumps({"tokens": {"refresh_token": "codex-refresh"}})
)
(vault / "gitea-token").write_text("test-gitea-token")
(vault / "gitea-username").write_text("hermes-automation")
monkeypatch.setattr(stage, "VAULT_ROOT", vault)
monkeypatch.setattr(stage, "RUNTIME_ROOT", tmp_path / "runtime")
monkeypatch.setattr(stage, "WORKER_ROOT", tmp_path / "worker")
monkeypatch.setattr(stage, "POOL_ACCESS_ROOT", tmp_path / "pool-access")
monkeypatch.setattr(stage, "SCM_ACCESS_ROOT", tmp_path / "scm-access")
monkeypatch.setattr(stage, "PROVIDER_ACCESS_ROOT", tmp_path / "provider-access")
monkeypatch.setenv("HERMES_WORKER_ORDINAL", "0")
monkeypatch.setattr(stage.os, "chown", lambda *_args: None)
monkeypatch.setattr(stage.os, "fchown", lambda *_args: None)
@ -268,6 +275,21 @@ def test_execution_worker_fails_closed_without_channel_credential(
stage.stage_execution_worker()
def test_execution_worker_rejects_durable_credential_symlink(
tmp_path: Path, monkeypatch
):
stage = _load("stage_runtime_access")
provider = tmp_path / "provider"
provider.mkdir()
(provider / "claude").symlink_to(tmp_path, target_is_directory=True)
monkeypatch.setattr(stage, "WORKER_ROOT", tmp_path / "worker")
monkeypatch.setattr(stage, "PROVIDER_ACCESS_ROOT", provider)
monkeypatch.setenv("HERMES_WORKER_ORDINAL", "0")
monkeypatch.setattr(stage.os, "chown", lambda *_args: None)
with pytest.raises(RuntimeError, match="symlink"):
stage.stage_execution_worker()
def test_invalid_runtime_json_is_removed(tmp_path: Path, monkeypatch):
stage = _load("stage_runtime_access")
vault = tmp_path / "vault"
@ -281,7 +303,7 @@ def test_invalid_runtime_json_is_removed(tmp_path: Path, monkeypatch):
try:
stage._validated_json("credential", destination, ("token",))
except json.JSONDecodeError:
except RuntimeError:
pass
else:
raise AssertionError("invalid credential JSON should fail staging")

11
testing/tests/test_hermes_scm_broker.py
View File

@ -48,7 +48,7 @@ def test_broker_metadata_boundary_reuses_explicit_read_allowlist():
api.authorize_request("GET", path, None)
def test_receive_pack_allows_only_new_namespaced_feature_branch():
def test_receive_pack_allows_only_new_branches_in_reviewed_namespaces():
broker = _load("scm_broker")
zero = b"0" * 40
commit = b"1" * 40
@ -57,9 +57,18 @@ def test_receive_pack_allows_only_new_namespaced_feature_branch():
_receive_command(zero, commit, b"refs/heads/hermes/focused-fix"),
"runtime-sentinel",
)
broker._validate_receive_pack(
_receive_command(zero, commit, b"refs/heads/wt/t_deadbeef"),
"runtime-sentinel",
)
broker._validate_receive_pack(
_receive_command(zero, commit, b"refs/heads/review/t_deadbeef"),
"runtime-sentinel",
)
for old, new, ref in (
(commit, b"2" * 40, b"refs/heads/hermes/focused-fix"),
(commit, zero, b"refs/heads/hermes/focused-fix"),
(commit, commit, b"refs/heads/hermes/focused-fix"),
(zero, commit, b"refs/heads/main"),
(zero, commit, b"refs/heads/master"),
(zero, commit, b"refs/tags/release"),