From 0fc97529c222627678b55431461486cce5f78f35 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 12:52:23 +0000 Subject: [PATCH] hermes: repair distributed pool integration boundaries --- ci/scripts/semgrep_report.py | 2 +- ...test_dashboards_render_atlas_drilldowns.py | 2 +- services/hermes-scm-broker/deployment.yaml | 3 +- services/hermes-scm-broker/networkpolicy.yaml | 6 + services/hermes/NOTES.md | 26 + services/hermes/agent-deployment.yaml | 2 +- .../hermes/execution-coordinator-patch.yaml | 8 +- services/hermes/execution-mediator.yaml | 425 ++++++++++++++ .../execution-worker-networkpolicy.yaml | 152 ++++- .../hermes/execution-worker-statefulset.yaml | 214 +++---- services/hermes/kustomization.yaml | 23 +- services/hermes/scm-common/kustomization.yaml | 2 +- .../hermes/scm-common/scripts/scm_broker.py | 25 +- services/hermes/scripts/cli_lane_dispatch.py | 37 +- .../hermes/scripts/execution_pool_askpass.sh | 19 - .../hermes/scripts/execution_pool_client.py | 163 ++++-- .../scripts/execution_pool_coordinator.py | 284 +++++----- .../hermes/scripts/execution_pool_project.py | 152 +++++ .../hermes/scripts/execution_pool_protocol.py | 105 +++- services/hermes/scripts/execution_pool_scm.py | 523 ++++++------------ .../hermes/scripts/execution_pool_server.py | 130 +++++ .../hermes/scripts/execution_pool_worker.py | 98 ++-- .../hermes/scripts/stage_runtime_access.py | 143 +++-- .../vault/scripts/vault_k8s_auth_configure.sh | 4 +- testing/quality_contract.json | 23 + testing/tests/test_hermes_agent_access.py | 32 +- testing/tests/test_hermes_agent_security.py | 67 ++- testing/tests/test_hermes_chat_config.py | 11 +- .../tests/test_hermes_chat_provider_auth.py | 182 ------ .../test_hermes_chat_provider_runtime.py | 196 +++++++ testing/tests/test_hermes_chat_voice.py | 7 +- .../test_hermes_cli_lanes_configuration.py | 12 +- testing/tests/test_hermes_cli_lanes_kanban.py | 392 ------------- .../tests/test_hermes_coordinator_boards.py | 9 +- testing/tests/test_hermes_execution_pool.py | 158 +++--- .../test_hermes_execution_pool_assignment.py | 98 +++- ...st_hermes_execution_pool_coordinator_v2.py | 293 ++++++++++ .../test_hermes_execution_pool_dispatch_v2.py | 410 ++++++++++++++ .../test_hermes_execution_pool_mediator.py | 448 +++++++++++++++ .../test_hermes_execution_pool_project.py | 211 +++++++ .../test_hermes_execution_pool_protocol_v2.py | 241 ++++++++ ...est_hermes_execution_pool_scm_tampering.py | 200 +++++++ ...st_hermes_execution_pool_worker_execute.py | 246 ++++++++ .../test_hermes_execution_pool_worker_v2.py | 292 ++++++++++ .../tests/test_hermes_gitea_pr_integration.py | 2 +- ...est_hermes_node_account_privilege_audit.py | 6 +- testing/tests/test_hermes_runtime_access.py | 62 ++- testing/tests/test_hermes_scm_broker.py | 11 +- 48 files changed, 4550 insertions(+), 1607 deletions(-) create mode 100644 services/hermes/execution-mediator.yaml delete mode 100755 services/hermes/scripts/execution_pool_askpass.sh create mode 100644 services/hermes/scripts/execution_pool_project.py create mode 100644 services/hermes/scripts/execution_pool_server.py create mode 100644 testing/tests/test_hermes_chat_provider_runtime.py delete mode 100644 testing/tests/test_hermes_cli_lanes_kanban.py create mode 100644 testing/tests/test_hermes_execution_pool_coordinator_v2.py create mode 100644 testing/tests/test_hermes_execution_pool_dispatch_v2.py create mode 100644 testing/tests/test_hermes_execution_pool_mediator.py create mode 100644 testing/tests/test_hermes_execution_pool_project.py create mode 100644 testing/tests/test_hermes_execution_pool_protocol_v2.py create mode 100644 testing/tests/test_hermes_execution_pool_scm_tampering.py create mode 100644 testing/tests/test_hermes_execution_pool_worker_execute.py create mode 100644 testing/tests/test_hermes_execution_pool_worker_v2.py diff --git a/ci/scripts/semgrep_report.py b/ci/scripts/semgrep_report.py index b4ed5a2d..2408f3e9 100644 --- a/ci/scripts/semgrep_report.py +++ b/ci/scripts/semgrep_report.py @@ -52,7 +52,7 @@ def build_report( results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else [] errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else [] - severity_counts = {severity: 0 for severity in SEVERITIES} + severity_counts = dict.fromkeys(SEVERITIES, 0) blocking_findings = 0 for finding in results: severity = _finding_severity(finding) diff --git a/scripts/tests/test_dashboards_render_atlas_drilldowns.py b/scripts/tests/test_dashboards_render_atlas_drilldowns.py index 08871b71..904379c9 100644 --- a/scripts/tests/test_dashboards_render_atlas_drilldowns.py +++ b/scripts/tests/test_dashboards_render_atlas_drilldowns.py @@ -1,6 +1,6 @@ """Detailed Atlas Jobs dashboard collapse and drilldown contracts.""" -from scripts.tests.test_dashboards_render_atlas import load_module +from scripts.tests.test_dashboard_render_support import load_module def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): diff --git a/services/hermes-scm-broker/deployment.yaml b/services/hermes-scm-broker/deployment.yaml index 649ada1d..ba62f451 100644 --- a/services/hermes-scm-broker/deployment.yaml +++ b/services/hermes-scm-broker/deployment.yaml @@ -17,6 +17,7 @@ spec: labels: app: hermes-scm-broker annotations: + ai.bstein.dev/config-rev: scm-boundary-v2 vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-scm-broker vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea @@ -91,7 +92,7 @@ spec: volumes: - name: broker-code configMap: - name: hermes-scm-boundary + name: hermes-scm-boundary-v2 defaultMode: 0555 - name: tmp emptyDir: diff --git a/services/hermes-scm-broker/networkpolicy.yaml b/services/hermes-scm-broker/networkpolicy.yaml index 8187e333..9c1a0542 100644 --- a/services/hermes-scm-broker/networkpolicy.yaml +++ b/services/hermes-scm-broker/networkpolicy.yaml @@ -17,6 +17,12 @@ spec: podSelector: matchLabels: app: hermes-agent + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hermes + podSelector: + matchLabels: + app: hermes-execution-mediator ports: - {protocol: TCP, port: 9081} egress: diff --git a/services/hermes/NOTES.md b/services/hermes/NOTES.md index c70e528a..400bf44f 100644 --- a/services/hermes/NOTES.md +++ b/services/hermes/NOTES.md @@ -257,6 +257,32 @@ Use this short explanation: leading different real incidents, improving the skill after failures, and teaching the architecture without prompts. +## Distributed execution-pool rollout boundary + +- Do not reconcile the pool until the approved credential owner has provisioned + six distinct Vault fields: `execution_worker_{0,1,2}_claude_credentials_json` + and `execution_worker_{0,1,2}_codex_auth_json`. Each ordinal needs an + independent account or refresh-token lineage; copying one rotating refresh + token into multiple fields recreates the lost-update failure this design + prevents. +- Worker credential files live on separate ordinal-owned RWO `provider-access` + claims. Provider refresh updates these durable private copies across Pod + restarts; they are never synchronized back to Vault. Rotate one bootstrap + credential at a time through the reviewed Vault workflow and reinitialize + only that ordinal after human approval. +- Existing task worktrees remain on the single local owner lane. Only tasks + without `workspace_path` enter the distributed pool, where each ordinal owns + one RWO checkout. Repository and base-branch identity comes from the canonical + board registry rather than task-supplied metadata. +- Model Pods have no pool key, broker mount, broker egress, or Kubernetes token. + Ordinal mediator Deployments are separate network identities colocated with + the matching workspace PVC; they alone authenticate exact-run results and + reach the PR14 SCM broker. +- A hashed execution-pool ConfigMap, protocol-version readiness checks, and + versioned SCM boundary name make code/config changes controlled rollouts. A + rollout is still a human-reviewed operation; this repository change does not + reconcile or deploy it. + ## Your shortest path to fluency 1. Explain the request diagram without looking. diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index ccbc36ea..ed0533e7 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -1191,7 +1191,7 @@ spec: - {key: openai.yaml, path: agents/openai.yaml} - name: scm-boundary configMap: - name: hermes-scm-boundary + name: hermes-scm-boundary-v2 defaultMode: 0555 - name: image-policy configMap: diff --git a/services/hermes/execution-coordinator-patch.yaml b/services/hermes/execution-coordinator-patch.yaml index 93192937..ec9ab80b 100644 --- a/services/hermes/execution-coordinator-patch.yaml +++ b/services/hermes/execution-coordinator-patch.yaml @@ -11,12 +11,14 @@ spec: vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600" vault.hashicorp.com/agent-inject-template-execution-pool-key: | {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} - {{ printf "hermes-execution-pool-v1:%s" .Data.data.agent_api_key | sha256sum }} + {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }} {{- end }} spec: containers: - name: cli-lane-runner - $patch: delete + env: + - {name: HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY, value: "true"} + - {name: HERMES_CLI_LANE_CONCURRENCY, value: "1"} - name: execution-pool-coordinator image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 imagePullPolicy: IfNotPresent @@ -26,8 +28,6 @@ spec: - {name: HOME, value: /opt/data/home} - {name: PYTHONPATH, value: /opt/hermes} - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key} - - {name: HERMES_EXECUTION_DEFAULT_REPO_URL, value: https://scm.bstein.dev/atlas/titan-iac.git} - - {name: HERMES_EXECUTION_DEFAULT_BASE_BRANCH, value: main} - {name: PATH, value: /opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin} securityContext: allowPrivilegeEscalation: false diff --git a/services/hermes/execution-mediator.yaml b/services/hermes/execution-mediator.yaml new file mode 100644 index 00000000..d41ee9f9 --- /dev/null +++ b/services/hermes/execution-mediator.yaml @@ -0,0 +1,425 @@ +# Privileged HMAC and SCM mediation runs outside the model Pods. Each mediator is +# colocated with exactly one ordinal's RWO workspace but has a distinct network +# identity, private HMAC derivation, and durable integrity state. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: hermes-execution-mediator-state-0, namespace: hermes} +spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: {requests: {storage: 1Gi}} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: hermes-execution-mediator-state-1, namespace: hermes} +spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: {requests: {storage: 1Gi}} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: hermes-execution-mediator-state-2, namespace: hermes} +spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: {requests: {storage: 1Gi}} +--- +apiVersion: v1 +kind: Service +metadata: {name: hermes-execution-mediator-0, namespace: hermes} +spec: + selector: {app: hermes-execution-mediator, pool-ordinal: "0"} + ports: [{name: mediator, port: 9009, targetPort: mediator}] +--- +apiVersion: v1 +kind: Service +metadata: {name: hermes-execution-mediator-1, namespace: hermes} +spec: + selector: {app: hermes-execution-mediator, pool-ordinal: "1"} + ports: [{name: mediator, port: 9009, targetPort: mediator}] +--- +apiVersion: v1 +kind: Service +metadata: {name: hermes-execution-mediator-2, namespace: hermes} +spec: + selector: {app: hermes-execution-mediator, pool-ordinal: "2"} + ports: [{name: mediator, port: 9009, targetPort: mediator}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-execution-mediator-0 + namespace: hermes + labels: {app: hermes-execution-mediator, pool-ordinal: "0"} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"} + template: + metadata: + labels: {app: hermes-execution-mediator, pool-ordinal: "0"} + annotations: + ai.bstein.dev/config-rev: execution-pool-v2-mediated + ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-execution-worker + vault.hashicorp.com/agent-inject-containers: stage-mediator-access + vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token + vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600" + vault.hashicorp.com/agent-inject-template-execution-pool-key: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 2m + vault.hashicorp.com/agent-requests-mem: 16Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi + spec: + serviceAccountName: hermes-execution-worker + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: {type: RuntimeDefault} + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + statefulset.kubernetes.io/pod-name: hermes-execution-worker-0 + topologyKey: kubernetes.io/hostname + initContainers: + - name: stage-mediator-access + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator] + env: + - {name: HERMES_WORKER_ORDINAL, value: "0"} + - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: pool-access, mountPath: /pool-access} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + resources: + requests: {cpu: 2m, memory: 16Mi} + limits: {cpu: 100m, memory: 64Mi} + containers: + - name: execution-mediator + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py] + env: + - {name: HERMES_WORKER_ORDINAL, value: "0"} + - {name: HERMES_WORKER_ROOT, value: /workspace} + - {name: HERMES_SCM_STATE_ROOT, value: /scm-state} + - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key} + - {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes} + ports: [{name: mediator, containerPort: 9009, protocol: TCP}] + startupProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: workspace, mountPath: /workspace} + - {name: scm-state, mountPath: /scm-state} + - {name: pool-access, mountPath: /pool-access, readOnly: true} + - {name: scm-broker-client, mountPath: /opt/scm, readOnly: true} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 2m, memory: 64Mi} + limits: {cpu: 250m, memory: 256Mi} + volumes: + - name: workspace + persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-0} + - name: scm-state + persistentVolumeClaim: {claimName: hermes-execution-mediator-state-0} + - name: pool-access + emptyDir: {medium: Memory, sizeLimit: 1Mi} + - name: scm-broker-client + configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555} + - name: coordinator + configMap: {name: hermes-execution-pool, defaultMode: 0555} + - name: tmp + emptyDir: {sizeLimit: 64Mi} + - name: vault-auth-token + projected: + defaultMode: 0600 + sources: + - serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token} + - configMap: + name: kube-root-ca.crt + items: [{key: ca.crt, path: ca.crt}] + - downwardAPI: + items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-execution-mediator-1 + namespace: hermes + labels: {app: hermes-execution-mediator, pool-ordinal: "1"} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"} + template: + metadata: + labels: {app: hermes-execution-mediator, pool-ordinal: "1"} + annotations: + ai.bstein.dev/config-rev: execution-pool-v2-mediated + ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-execution-worker + vault.hashicorp.com/agent-inject-containers: stage-mediator-access + vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token + vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600" + vault.hashicorp.com/agent-inject-template-execution-pool-key: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 2m + vault.hashicorp.com/agent-requests-mem: 16Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi + spec: + serviceAccountName: hermes-execution-worker + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: {type: RuntimeDefault} + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + statefulset.kubernetes.io/pod-name: hermes-execution-worker-1 + topologyKey: kubernetes.io/hostname + initContainers: + - name: stage-mediator-access + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator] + env: + - {name: HERMES_WORKER_ORDINAL, value: "1"} + - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: pool-access, mountPath: /pool-access} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + resources: + requests: {cpu: 2m, memory: 16Mi} + limits: {cpu: 100m, memory: 64Mi} + containers: + - name: execution-mediator + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py] + env: + - {name: HERMES_WORKER_ORDINAL, value: "1"} + - {name: HERMES_WORKER_ROOT, value: /workspace} + - {name: HERMES_SCM_STATE_ROOT, value: /scm-state} + - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key} + - {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes} + ports: [{name: mediator, containerPort: 9009, protocol: TCP}] + startupProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: workspace, mountPath: /workspace} + - {name: scm-state, mountPath: /scm-state} + - {name: pool-access, mountPath: /pool-access, readOnly: true} + - {name: scm-broker-client, mountPath: /opt/scm, readOnly: true} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 2m, memory: 64Mi} + limits: {cpu: 250m, memory: 256Mi} + volumes: + - name: workspace + persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-1} + - name: scm-state + persistentVolumeClaim: {claimName: hermes-execution-mediator-state-1} + - name: pool-access + emptyDir: {medium: Memory, sizeLimit: 1Mi} + - name: scm-broker-client + configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555} + - name: coordinator + configMap: {name: hermes-execution-pool, defaultMode: 0555} + - name: tmp + emptyDir: {sizeLimit: 64Mi} + - name: vault-auth-token + projected: + defaultMode: 0600 + sources: + - serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token} + - configMap: + name: kube-root-ca.crt + items: [{key: ca.crt, path: ca.crt}] + - downwardAPI: + items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-execution-mediator-2 + namespace: hermes + labels: {app: hermes-execution-mediator, pool-ordinal: "2"} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"} + template: + metadata: + labels: {app: hermes-execution-mediator, pool-ordinal: "2"} + annotations: + ai.bstein.dev/config-rev: execution-pool-v2-mediated + ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-execution-worker + vault.hashicorp.com/agent-inject-containers: stage-mediator-access + vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token + vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600" + vault.hashicorp.com/agent-inject-template-execution-pool-key: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 2m + vault.hashicorp.com/agent-requests-mem: 16Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi + spec: + serviceAccountName: hermes-execution-worker + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: {type: RuntimeDefault} + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + statefulset.kubernetes.io/pod-name: hermes-execution-worker-2 + topologyKey: kubernetes.io/hostname + initContainers: + - name: stage-mediator-access + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator] + env: + - {name: HERMES_WORKER_ORDINAL, value: "2"} + - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: pool-access, mountPath: /pool-access} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + resources: + requests: {cpu: 2m, memory: 16Mi} + limits: {cpu: 100m, memory: 64Mi} + containers: + - name: execution-mediator + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py] + env: + - {name: HERMES_WORKER_ORDINAL, value: "2"} + - {name: HERMES_WORKER_ROOT, value: /workspace} + - {name: HERMES_SCM_STATE_ROOT, value: /scm-state} + - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key} + - {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes} + ports: [{name: mediator, containerPort: 9009, protocol: TCP}] + startupProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: workspace, mountPath: /workspace} + - {name: scm-state, mountPath: /scm-state} + - {name: pool-access, mountPath: /pool-access, readOnly: true} + - {name: scm-broker-client, mountPath: /opt/scm, readOnly: true} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 2m, memory: 64Mi} + limits: {cpu: 250m, memory: 256Mi} + volumes: + - name: workspace + persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-2} + - name: scm-state + persistentVolumeClaim: {claimName: hermes-execution-mediator-state-2} + - name: pool-access + emptyDir: {medium: Memory, sizeLimit: 1Mi} + - name: scm-broker-client + configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555} + - name: coordinator + configMap: {name: hermes-execution-pool, defaultMode: 0555} + - name: tmp + emptyDir: {sizeLimit: 64Mi} + - name: vault-auth-token + projected: + defaultMode: 0600 + sources: + - serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token} + - configMap: + name: kube-root-ca.crt + items: [{key: ca.crt, path: ca.crt}] + - downwardAPI: + items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}] diff --git a/services/hermes/execution-worker-networkpolicy.yaml b/services/hermes/execution-worker-networkpolicy.yaml index 06470b2a..93e46be8 100644 --- a/services/hermes/execution-worker-networkpolicy.yaml +++ b/services/hermes/execution-worker-networkpolicy.yaml @@ -22,24 +22,10 @@ spec: podSelector: matchLabels: {app: vault} ports: [{protocol: TCP, port: 8200}] - - to: - - podSelector: - matchLabels: {app: hermes-agent} - ports: [{protocol: TCP, port: 9007}] - to: - podSelector: matchLabels: {app: hermes-switchyard} ports: [{protocol: TCP, port: 9005}] - - to: - - podSelector: - matchLabels: {app: hermes-model-gate} - ports: [{protocol: TCP, port: 8080}] - - to: - - namespaceSelector: - matchLabels: {kubernetes.io/metadata.name: gitea} - podSelector: - matchLabels: {app: gitea} - ports: [{protocol: TCP, port: 3000}] - to: - ipBlock: cidr: 0.0.0.0/0 @@ -55,6 +41,57 @@ spec: --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy +metadata: + name: hermes-execution-worker-mediator-0 + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "0" + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"} + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-worker-mediator-1 + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "1" + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"} + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-worker-mediator-2 + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "2" + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"} + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy metadata: name: hermes-execution-pool-ingress namespace: hermes @@ -65,11 +102,96 @@ spec: ingress: - from: - podSelector: - matchLabels: {app: hermes-execution-worker} + matchLabels: {app: hermes-execution-mediator} ports: [{protocol: TCP, port: 9007}] --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy +metadata: + name: hermes-execution-mediator-isolation + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-execution-mediator} + policyTypes: [Ingress, Egress] + ingress: [] + egress: + - to: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: kube-system} + podSelector: + matchLabels: {k8s-app: kube-dns} + ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}] + - to: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: vault} + podSelector: + matchLabels: {app: vault} + ports: [{protocol: TCP, port: 8200}] + - to: + - podSelector: + matchLabels: {app: hermes-agent} + ports: [{protocol: TCP, port: 9007}] + - to: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: hermes-scm} + podSelector: + matchLabels: {app: hermes-scm-broker} + ports: [{protocol: TCP, port: 9081}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-mediator-worker-0 + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "0" + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-mediator-worker-1 + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "1" + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-mediator-worker-2 + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "2" + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy metadata: name: hermes-execution-switchyard-ingress namespace: hermes diff --git a/services/hermes/execution-worker-statefulset.yaml b/services/hermes/execution-worker-statefulset.yaml index ac3d9f00..6794f4da 100644 --- a/services/hermes/execution-worker-statefulset.yaml +++ b/services/hermes/execution-worker-statefulset.yaml @@ -15,6 +15,8 @@ spec: app: hermes-execution-worker updateStrategy: type: RollingUpdate + rollingUpdate: + partition: 0 template: metadata: labels: @@ -23,51 +25,58 @@ spec: app.kubernetes.io/part-of: hermes annotations: ai.bstein.dev/role: fenced-execution-only - ai.bstein.dev/scm-boundary: ordinal-sidecar-with-assignment-bound-branch + ai.bstein.dev/scm-boundary: mediated-pr14-broker-with-completion-gates ai.bstein.dev/model-policy: Switchyard AUTO with cross-provider fallback - ai.bstein.dev/storage: one durable RWO workspace and provider session home per ordinal + ai.bstein.dev/storage: separate durable RWO workspace and OAuth refresh ownership per ordinal vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-execution-worker vault.hashicorp.com/agent-inject-containers: stage-worker-access vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token - vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens - vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600" - vault.hashicorp.com/agent-inject-template-execution-pool-key: | + vault.hashicorp.com/agent-inject-secret-claude-credentials-0: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-claude-credentials-0: "0600" + vault.hashicorp.com/agent-inject-template-claude-credentials-0: | {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} - {{ printf "hermes-execution-pool-v1:%s" .Data.data.agent_api_key | sha256sum }} + {{ .Data.data.execution_worker_0_claude_credentials_json }} {{- end }} - vault.hashicorp.com/agent-inject-secret-claude-credentials: kv/data/atlas/hermes/agent-tokens - vault.hashicorp.com/agent-inject-perms-claude-credentials: "0600" - vault.hashicorp.com/agent-inject-template-claude-credentials: | + vault.hashicorp.com/agent-inject-secret-claude-credentials-1: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-claude-credentials-1: "0600" + vault.hashicorp.com/agent-inject-template-claude-credentials-1: | {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} - {{ .Data.data.claude_credentials_json }} + {{ .Data.data.execution_worker_1_claude_credentials_json }} {{- end }} - vault.hashicorp.com/agent-inject-secret-codex-auth: kv/data/atlas/hermes/agent-tokens - vault.hashicorp.com/agent-inject-perms-codex-auth: "0600" - vault.hashicorp.com/agent-inject-template-codex-auth: | + vault.hashicorp.com/agent-inject-secret-claude-credentials-2: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-claude-credentials-2: "0600" + vault.hashicorp.com/agent-inject-template-claude-credentials-2: | {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} - {{ .Data.data.codex_auth_json }} + {{ .Data.data.execution_worker_2_claude_credentials_json }} {{- end }} - vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea - vault.hashicorp.com/agent-inject-perms-gitea-token: "0600" - vault.hashicorp.com/agent-inject-template-gitea-token: | - {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} - {{ .Data.data.token }} + vault.hashicorp.com/agent-inject-secret-codex-auth-0: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-codex-auth-0: "0600" + vault.hashicorp.com/agent-inject-template-codex-auth-0: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.execution_worker_0_codex_auth_json }} {{- end }} - vault.hashicorp.com/agent-inject-secret-gitea-username: kv/data/atlas/hermes/developer-gitea - vault.hashicorp.com/agent-inject-perms-gitea-username: "0600" - vault.hashicorp.com/agent-inject-template-gitea-username: | - {{- with secret "kv/data/atlas/hermes/developer-gitea" -}} - {{ .Data.data.username }} + vault.hashicorp.com/agent-inject-secret-codex-auth-1: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-codex-auth-1: "0600" + vault.hashicorp.com/agent-inject-template-codex-auth-1: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.execution_worker_1_codex_auth_json }} + {{- end }} + vault.hashicorp.com/agent-inject-secret-codex-auth-2: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-codex-auth-2: "0600" + vault.hashicorp.com/agent-inject-template-codex-auth-2: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.execution_worker_2_codex_auth_json }} {{- end }} vault.hashicorp.com/agent-pre-populate-only: "true" vault.hashicorp.com/agent-init-first: "true" - vault.hashicorp.com/agent-requests-cpu: 25m - vault.hashicorp.com/agent-requests-mem: 32Mi + vault.hashicorp.com/agent-requests-cpu: 2m + vault.hashicorp.com/agent-requests-mem: 16Mi vault.hashicorp.com/agent-limits-cpu: 100m vault.hashicorp.com/agent-limits-mem: 128Mi spec: serviceAccountName: hermes-execution-worker + priorityClassName: scavenger automountServiceAccountToken: false enableServiceLinks: false terminationGracePeriodSeconds: 30 @@ -82,12 +91,8 @@ spec: nodeSelectorTerms: - matchExpressions: - {key: kubernetes.io/arch, operator: In, values: [arm64]} - - {key: node-role.kubernetes.io/accelerator, operator: Exists} - - {key: kubernetes.io/hostname, operator: In, values: [titan-20, titan-21]} - - matchExpressions: - - {key: kubernetes.io/arch, operator: In, values: [arm64]} - - {key: hardware, operator: In, values: [rpi5]} - - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]} + - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]} + - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]} preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 preference: @@ -117,8 +122,11 @@ spec: command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-worker] env: - {name: HERMES_WORKER_ROOT, value: /workspace} - - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access} - - {name: HERMES_SCM_ACCESS_ROOT, value: /scm-access} + - {name: HERMES_PROVIDER_ACCESS_ROOT, value: /provider-access} + - name: HERMES_WORKER_ORDINAL + valueFrom: + fieldRef: + fieldPath: metadata.labels['apps.kubernetes.io/pod-index'] securityContext: allowPrivilegeEscalation: false runAsUser: 0 @@ -126,12 +134,10 @@ spec: seccompProfile: {type: RuntimeDefault} volumeMounts: - {name: workspace, mountPath: /workspace} - - {name: runtime-access, mountPath: /runtime-access} - - {name: pool-access, mountPath: /pool-access} - - {name: scm-access, mountPath: /scm-access} + - {name: provider-access, mountPath: /provider-access} - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} resources: - requests: {cpu: 25m, memory: 32Mi} + requests: {cpu: 2m, memory: 32Mi} limits: {cpu: 100m, memory: 64Mi} - name: install-provider-clis image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 @@ -158,7 +164,7 @@ spec: volumeMounts: - {name: tools, mountPath: /worker-data/tools} resources: - requests: {cpu: 100m, memory: 128Mi} + requests: {cpu: 5m, memory: 64Mi} limits: {cpu: "1", memory: 1Gi} containers: - name: execution-worker @@ -169,8 +175,8 @@ spec: - {name: HERMES_HOME, value: /worker-data} - {name: HERMES_WORKER_ROOT, value: /workspace} - {name: HOME, value: /worker-data/home} - - {name: CODEX_HOME, value: /runtime-access/codex} - - {name: CLAUDE_CONFIG_DIR, value: /runtime-access/claude} + - {name: CODEX_HOME, value: /provider-access/codex} + - {name: CLAUDE_CONFIG_DIR, value: /provider-access/claude} - {name: HERMES_AUTO_ROUTER_PROFILE, value: agent} - {name: PYTHONPATH, value: /opt/hermes} - {name: PATH, value: /worker-data/tools/bin:/opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin} @@ -184,12 +190,12 @@ spec: fieldPath: spec.nodeName startupProbe: exec: - command: [/bin/sh, -ec, "test -w /workspace && test -w /runtime-access/codex/auth.json && test -w /runtime-access/claude/.credentials.json"] + command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json && test -w /provider-access/claude/.credentials.json"] periodSeconds: 5 failureThreshold: 60 readinessProbe: exec: - command: [/bin/sh, -ec, "test -w /workspace && test -w /runtime-access/codex/auth.json"] + command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json"] periodSeconds: 10 securityContext: allowPrivilegeEscalation: false @@ -203,123 +209,17 @@ spec: - {name: workspace, mountPath: /workspace} - {name: worker-data, mountPath: /worker-data} - {name: tools, mountPath: /worker-data/tools, readOnly: true} - - {name: runtime-access, mountPath: /runtime-access} + - {name: provider-access, mountPath: /provider-access} - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} - {name: tmp, mountPath: /tmp} resources: - requests: {cpu: "1", memory: 2Gi, ephemeral-storage: 2Gi} - limits: {cpu: "5", memory: 6Gi, ephemeral-storage: 8Gi} - - name: execution-client - image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 - imagePullPolicy: IfNotPresent - command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py] - env: - - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key} - - name: HERMES_WORKER_ORDINAL - valueFrom: - fieldRef: - fieldPath: metadata.labels['apps.kubernetes.io/pod-index'] - ports: - - {name: pool-client, containerPort: 9009, protocol: TCP} - startupProbe: - httpGet: {path: /ready, port: pool-client} - periodSeconds: 5 - failureThreshold: 60 - readinessProbe: - httpGet: {path: /ready, port: pool-client} - periodSeconds: 10 - securityContext: - allowPrivilegeEscalation: false - capabilities: {drop: [ALL]} - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 10000 - runAsGroup: 10000 - seccompProfile: {type: RuntimeDefault} - volumeMounts: - - {name: pool-access, mountPath: /pool-access, readOnly: true} - - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} - - {name: tmp, mountPath: /tmp} - resources: - requests: {cpu: 25m, memory: 64Mi} - limits: {cpu: 250m, memory: 256Mi} - - name: scm-boundary - image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 - imagePullPolicy: IfNotPresent - command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_scm.py] - env: - - {name: HERMES_WORKER_ROOT, value: /workspace} - - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key} - - {name: HERMES_GITEA_TOKEN_FILE, value: /scm-access/gitea-token} - - {name: HERMES_GITEA_USERNAME_FILE, value: /scm-access/gitea-username} - - {name: HERMES_SCM_STATE_ROOT, value: /scm-state} - - name: HERMES_WORKER_ORDINAL - valueFrom: - fieldRef: - fieldPath: metadata.labels['apps.kubernetes.io/pod-index'] - ports: - - {name: scm, containerPort: 9008, protocol: TCP} - startupProbe: - httpGet: {path: /ready, port: scm} - periodSeconds: 5 - failureThreshold: 60 - readinessProbe: - httpGet: {path: /ready, port: scm} - periodSeconds: 10 - securityContext: - allowPrivilegeEscalation: false - capabilities: {drop: [ALL]} - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 10000 - runAsGroup: 10000 - seccompProfile: {type: RuntimeDefault} - volumeMounts: - - {name: workspace, mountPath: /workspace} - - {name: pool-access, mountPath: /pool-access, readOnly: true} - - {name: scm-access, mountPath: /scm-access, readOnly: true} - - {name: scm-state, mountPath: /scm-state} - - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} - - {name: tmp, mountPath: /tmp} - resources: - requests: {cpu: 100m, memory: 128Mi} - limits: {cpu: "1", memory: 512Mi} - - name: credential-sync - image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 - imagePullPolicy: IfNotPresent - command: [/opt/hermes/.venv/bin/python, /opt/coordinator/sync_runtime_credentials.py] - env: - - {name: HERMES_CREDENTIAL_SYNC_VAULT_ROLE, value: hermes-execution-credential-sync} - - {name: HERMES_CREDENTIAL_SYNC_INTERVAL, value: "300"} - securityContext: - allowPrivilegeEscalation: false - capabilities: {drop: [ALL]} - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 10000 - runAsGroup: 10000 - seccompProfile: {type: RuntimeDefault} - volumeMounts: - - {name: runtime-access, mountPath: /runtime-access} - - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} - - {name: vault-auth-token, mountPath: /var/run/secrets/kubernetes.io/serviceaccount, readOnly: true} - - {name: tmp, mountPath: /tmp} - resources: - requests: {cpu: 25m, memory: 64Mi} - limits: {cpu: 250m, memory: 256Mi} + requests: {cpu: 5m, memory: 128Mi, ephemeral-storage: 1Gi} + limits: {cpu: "2", memory: 4Gi, ephemeral-storage: 8Gi} volumes: - name: worker-data emptyDir: {sizeLimit: 128Mi} - name: tools emptyDir: {sizeLimit: 1Gi} - - name: runtime-access - emptyDir: {medium: Memory, sizeLimit: 128Mi} - - name: pool-access - emptyDir: {medium: Memory, sizeLimit: 1Mi} - - name: scm-access - emptyDir: {medium: Memory, sizeLimit: 1Mi} - - name: scm-state - emptyDir: {sizeLimit: 256Mi} - name: coordinator configMap: name: hermes-execution-pool @@ -352,3 +252,13 @@ spec: resources: requests: storage: 30Gi + - metadata: + name: provider-access + labels: + app: hermes-execution-worker + spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: + requests: + storage: 1Gi diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 61bff865..1de6b74b 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -43,6 +43,7 @@ resources: - agent-ingress.yaml - execution-worker-rbac.yaml - execution-worker-statefulset.yaml + - execution-mediator.yaml - execution-worker-networkpolicy.yaml patches: @@ -64,18 +65,32 @@ configMapGenerator: - name: hermes-execution-pool namespace: hermes files: + - cli_lane_board.py=scripts/cli_lane_board.py + - cli_lane_config.py=scripts/cli_lane_config.py + - cli_lane_dispatch.py=scripts/cli_lane_dispatch.py + - cli_lane_evidence.py=scripts/cli_lane_evidence.py + - cli_lane_execution.py=scripts/cli_lane_execution.py + - cli_lane_files.py=scripts/cli_lane_files.py + - cli_lane_finalization.py=scripts/cli_lane_finalization.py - cli_lane_goal.py=scripts/cli_lane_goal.py + - cli_lane_prompt.py=scripts/cli_lane_prompt.py + - cli_lane_provider.py=scripts/cli_lane_provider.py + - cli_lane_quarantine.py=scripts/cli_lane_quarantine.py + - cli_lane_records.py=scripts/cli_lane_records.py + - cli_lane_recovery.py=scripts/cli_lane_recovery.py + - cli_lane_retention.py=scripts/cli_lane_retention.py + - cli_lane_routing.py=scripts/cli_lane_routing.py - cli_lane_runner.py=scripts/cli_lane_runner.py - - execution_pool_askpass.sh=scripts/execution_pool_askpass.sh - execution_pool_protocol.py=scripts/execution_pool_protocol.py + - execution_pool_project.py=scripts/execution_pool_project.py - execution_pool_coordinator.py=scripts/execution_pool_coordinator.py + - execution_pool_server.py=scripts/execution_pool_server.py - execution_pool_client.py=scripts/execution_pool_client.py - execution_pool_worker.py=scripts/execution_pool_worker.py - execution_pool_scm.py=scripts/execution_pool_scm.py + - gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py + - scm_broker_client.py=scm-common/scripts/scm_broker_client.py - stage_runtime_access.py=scripts/stage_runtime_access.py - - sync_runtime_credentials.py=scripts/sync_runtime_credentials.py - options: - disableNameSuffixHash: true - name: hermes-coordinator namespace: hermes files: diff --git a/services/hermes/scm-common/kustomization.yaml b/services/hermes/scm-common/kustomization.yaml index 17dc1da6..f2d222b5 100644 --- a/services/hermes/scm-common/kustomization.yaml +++ b/services/hermes/scm-common/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization configMapGenerator: - - name: hermes-scm-boundary + - name: hermes-scm-boundary-v2 files: - gitea_api.py=scripts/gitea_api.py - gitea_api_policy.py=scripts/gitea_api_policy.py diff --git a/services/hermes/scm-common/scripts/scm_broker.py b/services/hermes/scm-common/scripts/scm_broker.py index 22c2a637..02bd7e23 100644 --- a/services/hermes/scm-common/scripts/scm_broker.py +++ b/services/hermes/scm-common/scripts/scm_broker.py @@ -7,7 +7,6 @@ import argparse import base64 import json import re -import socket import tempfile import time import urllib.error @@ -45,7 +44,8 @@ GIT_PATH_RE = re.compile( r"(?Pinfo/refs|git-upload-pack|git-receive-pack)\Z" ) FEATURE_REF_RE = re.compile( - r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z" + r"refs/heads/(?:(?:feature|fix|chore|docs|test|refactor|wt|review|hermes|handoff)/" + r"[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z" ) @@ -109,7 +109,9 @@ def _spool_bounded( """Copy a fixed-length exchange through a bounded-memory disk spool.""" if not 0 <= length <= maximum: raise PolicyError("SCM request exceeds the safe size limit") - spool = tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_LIMIT, dir="/tmp") + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns lifetime + max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" + ) remaining = length forms = _credential_forms(token) carry = b"" @@ -218,7 +220,7 @@ def _content_length(headers: object, maximum: int) -> int: def _credential_forms(token: str) -> tuple[bytes, ...]: - basic = base64.b64encode(f"{GIT_USER}:{token}".encode("utf-8")) + basic = base64.b64encode(f"{GIT_USER}:{token}".encode()) return token.encode("utf-8"), basic, b"Basic " + basic @@ -239,7 +241,7 @@ def _receive_prefix(body: bytes | BinaryIO) -> bytes: def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None: - """Permit only creation of new, namespaced feature branches.""" + """Permit only creation of new branches in reviewed namespaces.""" body = _receive_prefix(body) _reject_credential_bytes(body, token, "Git request") position = 0 @@ -262,7 +264,7 @@ def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None: raise PolicyError("Git receive-pack ref command is invalid") old_sha, new_sha, raw_ref = fields if old_sha != ZERO_SHA or new_sha == ZERO_SHA: - raise PolicyError("Git broker permits only new feature-branch creation") + raise PolicyError("Git broker permits only new namespaced branch creation") try: ref = raw_ref.decode("ascii") except UnicodeDecodeError as exc: @@ -398,7 +400,7 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): self._stream(200, expected, body, length) finally: body.close() - except (OSError, PolicyError, socket.timeout, urllib.error.URLError, ValueError): + except (TimeoutError, OSError, PolicyError, urllib.error.URLError, ValueError): self._reject() def do_POST(self) -> None: @@ -409,7 +411,14 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): self._control() else: self._git_rpc() - except (OSError, PolicyError, socket.timeout, urllib.error.URLError, ValueError, json.JSONDecodeError): + except ( + TimeoutError, + OSError, + PolicyError, + urllib.error.URLError, + ValueError, + json.JSONDecodeError, + ): self._reject() def _control(self) -> None: diff --git a/services/hermes/scripts/cli_lane_dispatch.py b/services/hermes/scripts/cli_lane_dispatch.py index 7d386a8c..44f2b573 100644 --- a/services/hermes/scripts/cli_lane_dispatch.py +++ b/services/hermes/scripts/cli_lane_dispatch.py @@ -7,6 +7,7 @@ import concurrent.futures import os import sys import time +from collections.abc import Callable from typing import Any from cli_lane_board import _external, _record_board_access_error, _task_value @@ -23,6 +24,16 @@ from cli_lane_recovery import _has_pending_finalization, recover_pending_finaliz from cli_lane_retention import maybe_gc_lane_artifacts +OWNED_WORKSPACES_ONLY = os.environ.get( + "HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY", "" +).strip().lower() in {"1", "true", "yes", "on"} + + +def _owns_local_workspace(task: Any) -> bool: + """Keep legacy/dirty worktree tasks on their existing single-host owner.""" + return bool(str(_task_value(task, "workspace_path", "") or "").strip()) + + def _board_slug(board: Any) -> str: if isinstance(board, dict): return str(board.get("slug") or board.get("id") or "") @@ -56,7 +67,11 @@ def recover_orphans() -> None: continue try: for task in kanban_db.list_tasks(conn): - if _external(task) and str(_task_value(task, "status", "")) == "running": + if ( + _external(task) + and str(_task_value(task, "status", "")) == "running" + and (not OWNED_WORKSPACES_ONLY or _owns_local_workspace(task)) + ): task_id = str(_task_value(task, "id")) run_id = _task_value(task, "current_run_id", None) if not isinstance(run_id, int): @@ -75,7 +90,11 @@ def recover_orphans() -> None: finally: conn.close() -def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str]]: +def claim_ready( + active: set[tuple[str, str]], + limit: int, + eligible: Callable[[str, Any], bool] | None = None, +) -> list[tuple[str, str]]: """Atomically claim external ready tasks across all non-archived boards.""" from hermes_cli import kanban_db @@ -97,15 +116,21 @@ def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str for task in tasks: task_id = str(_task_value(task, "id", "")) assignee = str(_task_value(task, "assignee", "") or "") - if task_id and not assignee and str(_task_value(task, "status", "")) == "ready": - if kanban_db.assign_task(conn, task_id, "cli-auto"): - task = kanban_db.get_task(conn, task_id) - assignee = "cli-auto" + if ( + task_id + and not assignee + and str(_task_value(task, "status", "")) == "ready" + and kanban_db.assign_task(conn, task_id, "cli-auto") + ): + task = kanban_db.get_task(conn, task_id) + assignee = "cli-auto" if ( not task_id or (board, task_id) in active or not assignee.startswith(EXTERNAL_PREFIX) or str(_task_value(task, "status", "")) != "ready" + or (OWNED_WORKSPACES_ONLY and not _owns_local_workspace(task)) + or (eligible is not None and not eligible(board, task)) ): continue try: diff --git a/services/hermes/scripts/execution_pool_askpass.sh b/services/hermes/scripts/execution_pool_askpass.sh deleted file mode 100755 index 8e3758ae..00000000 --- a/services/hermes/scripts/execution_pool_askpass.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh -set -eu - -case "${1:-}" in - *Username*) - if [ -s "${HERMES_SCM_USERNAME_FILE}" ]; then - tr -d '\r\n' <"${HERMES_SCM_USERNAME_FILE}" - else - printf '%s' hermes-automation - fi - printf '\n' - ;; - *Password*) - test -s "${HERMES_SCM_PASSWORD_FILE}" - tr -d '\r\n' <"${HERMES_SCM_PASSWORD_FILE}" - printf '\n' - ;; - *) exit 1 ;; -esac diff --git a/services/hermes/scripts/execution_pool_client.py b/services/hermes/scripts/execution_pool_client.py index 76c22953..a1dc6622 100644 --- a/services/hermes/scripts/execution_pool_client.py +++ b/services/hermes/scripts/execution_pool_client.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Ordinal-local signing boundary for a model-facing Hermes worker.""" +"""Ordinal-local signing and SCM mediation boundary for a model worker.""" from __future__ import annotations @@ -12,8 +12,10 @@ from http.server import BaseHTTPRequestHandler from pathlib import Path from typing import Any +import cli_lane_goal from execution_pool_protocol import ( MAX_WIRE_BYTES, + PROTOCOL_VERSION, BoundedHTTPServer, ProtocolError, canonical_json, @@ -22,49 +24,80 @@ from execution_pool_protocol import ( sign_envelope, verify_envelope, ) +from execution_pool_scm import Boundary as SCMBoundary -KEY_PATH = Path(os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/pool-access/execution-pool-key")) +KEY_PATH = Path( + os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/pool-access/execution-pool-key") +) COORDINATOR = os.environ.get( - "HERMES_EXECUTION_POOL_URL", "http://hermes-execution-pool.hermes.svc.cluster.local:9007" + "HERMES_EXECUTION_POOL_URL", + "http://hermes-execution-pool.hermes.svc.cluster.local:9007", ).rstrip("/") ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1")) PORT = int(os.environ.get("HERMES_EXECUTION_CLIENT_PORT", "9009")) +RESULT_FIELDS = frozenset( + {"status", "summary", "changed_files", "tests_run", "artifacts", "findings", "blockers"} +) def _binding(value: dict[str, Any]) -> dict[str, Any]: - return {name: value[name] for name in ( - "board", "task_id", "run_id", "worker_ordinal", "attempt" - )} + return { + name: value[name] + for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt") + } + + +def _validate_result(payload: Any) -> dict[str, Any]: + if not isinstance(payload, dict): + raise ProtocolError("terminal result payload must be an object") + structured = payload.get("structured") + if not isinstance(structured, dict) or set(structured) != RESULT_FIELDS: + raise ProtocolError("terminal result fields do not match the reviewed schema") + if structured.get("status") not in cli_lane_goal.RESULT_STATUSES: + raise ProtocolError("terminal result status is invalid") + if not isinstance(structured.get("summary"), str) or not structured["summary"].strip(): + raise ProtocolError("terminal result summary is required") + for name in RESULT_FIELDS - {"status", "summary"}: + value = structured.get(name) + if not isinstance(value, list) or any(not isinstance(item, str) for item in value): + raise ProtocolError(f"terminal result {name} must be a text list") + return payload class ClientBoundary: - """Hold the master key and fence local requests to the current ordinal run.""" + """Keep HMAC and SCM authority outside the model-facing container.""" - def __init__(self, key: bytes): + def __init__(self, key: bytes, scm: SCMBoundary | None = None): self.key = key + self.scm = scm or SCMBoundary(key) self.current: dict[str, Any] | None = None self.lock = threading.RLock() def _post(self, path: str, envelope: dict[str, Any]) -> dict[str, Any]: request = urllib.request.Request( - COORDINATOR + path, data=canonical_json(envelope), method="POST", + COORDINATOR + path, + data=canonical_json(envelope), + method="POST", headers={"Content-Type": "application/json", "Cache-Control": "no-store"}, ) with urllib.request.urlopen(request, timeout=60) as response: body = response.read(MAX_WIRE_BYTES + 1) if len(body) > MAX_WIRE_BYTES: raise ProtocolError("coordinator response exceeds the wire limit") - value = json.loads(body) - return verify_envelope(self.key, value) + return verify_envelope(self.key, json.loads(body)) def poll(self) -> dict[str, Any]: poll_binding = { - "board": "", "task_id": "", "run_id": "", - "worker_ordinal": ORDINAL, "attempt": 0, + "board": "", + "task_id": "", + "run_id": "", + "worker_ordinal": ORDINAL, + "attempt": 0, } response = self._post( - "/v1/poll", sign_envelope(self.key, "poll", poll_binding, {"ready": True}) + "/v1/poll", + sign_envelope(self.key, "poll", poll_binding, {"ready": True}), ) with self.lock: if response["kind"] == "ack": @@ -72,32 +105,61 @@ class ClientBoundary: return {"assignment": None} if response["kind"] != "assignment" or response["worker_ordinal"] != ORDINAL: raise ProtocolError("coordinator returned a foreign assignment") - self.current = _binding(response) - return {"assignment": response} + checkout = self.scm.checkout(response) + self.current = response + assignment = { + **_binding(response), + "payload": response["payload"], + "workspace": checkout["workspace"], + "baseline_sha": checkout["baseline_sha"], + "protocol_version": PROTOCOL_VERSION, + } + return {"assignment": assignment} - def forward(self, kind: str, request: dict[str, Any]) -> dict[str, Any]: - supplied = request.get("binding") + def _current_for(self, supplied: Any) -> tuple[dict[str, Any], dict[str, Any]]: + if not isinstance(supplied, dict): + raise ProtocolError("local request binding must be an object") + if self.current is None or supplied != _binding(self.current): + raise ProtocolError("local request does not own the current assignment") + return self.current, dict(supplied) + + def heartbeat(self, request: dict[str, Any]) -> dict[str, Any]: payload = request.get("payload") - if not isinstance(supplied, dict) or not isinstance(payload, dict): - raise ProtocolError("local request binding and payload must be objects") + if not isinstance(payload, dict): + raise ProtocolError("heartbeat payload must be an object") with self.lock: - if self.current is None or supplied != self.current: - raise ProtocolError("local request does not own the current assignment") - binding = dict(self.current) - response = self._post( - f"/v1/{kind}", sign_envelope(self.key, kind, binding, payload) - ) - if response["kind"] != "ack" or _binding(response) != binding: - raise ProtocolError("coordinator acknowledgement binding changed") - if kind == "result" and response["payload"].get("accepted"): - with self.lock: - self.current = None + _assignment, binding = self._current_for(request.get("binding")) + response = self._post( + "/v1/heartbeat", + sign_envelope(self.key, "heartbeat", binding, payload), + ) + if response["kind"] != "ack" or _binding(response) != binding: + raise ProtocolError("coordinator acknowledgement binding changed") return {"ack": response["payload"]} + def finish(self, request: dict[str, Any]) -> dict[str, Any]: + payload = _validate_result(request.get("payload")) + with self.lock: + assignment, binding = self._current_for(request.get("binding")) + structured = payload["structured"] + if structured["status"] == "completed" and int(payload.get("returncode", 1)) == 0: + submission = self.scm.submit(assignment, request) + pull = str(submission.get("pull_request") or "") + if pull and pull not in structured["artifacts"]: + structured["artifacts"].append(pull) + response = self._post( + "/v1/result", sign_envelope(self.key, "result", binding, payload) + ) + if response["kind"] != "ack" or _binding(response) != binding: + raise ProtocolError("coordinator acknowledgement binding changed") + if response["payload"].get("accepted"): + self.current = None + return {"ack": response["payload"], "structured": structured} + def handler_factory(boundary: ClientBoundary) -> type[BaseHTTPRequestHandler]: class Handler(BaseHTTPRequestHandler): - server_version = "hermes-execution-client/1" + server_version = f"hermes-execution-mediator/{PROTOCOL_VERSION}" def _reply(self, status: int, value: dict[str, Any]) -> None: body = canonical_json(value) @@ -109,22 +171,31 @@ def handler_factory(boundary: ClientBoundary) -> type[BaseHTTPRequestHandler]: self.wfile.write(body) def do_GET(self) -> None: # noqa: N802 - self._reply(200, {"ready": True}) if self.path == "/ready" else self._reply(404, {"error": "not found"}) + value = {"ready": True, "protocol_version": PROTOCOL_VERSION} + self._reply(200, value) if self.path == "/ready" else self._reply( + 404, {"error": "not found"} + ) def do_POST(self) -> None: # noqa: N802 try: length = int(self.headers.get("Content-Length", "0")) - request = parse_wire(self.rfile.read(length)) if 0 < length <= MAX_WIRE_BYTES else None - if not request or set(request) - {"operation", "binding", "payload"}: - raise ProtocolError("invalid local client request") + request = ( + parse_wire(self.rfile.read(length)) + if 0 < length <= MAX_WIRE_BYTES + else None + ) + allowed = {"operation", "binding", "payload", "title", "body"} + if not request or set(request) - allowed: + raise ProtocolError("invalid local mediator request") operation = str(request.get("operation") or "") - if operation == "poll": - result = boundary.poll() - elif operation in {"heartbeat", "result"}: - result = boundary.forward(operation, request) - else: - raise ProtocolError("unsupported local client operation") - self._reply(200, result) + routes = { + "poll": boundary.poll, + "heartbeat": lambda: boundary.heartbeat(request), + "finish": lambda: boundary.finish(request), + } + if operation not in routes: + raise ProtocolError("unsupported local mediator operation") + self._reply(200, routes[operation]()) except (ProtocolError, OSError, ValueError, urllib.error.URLError) as error: self._reply(409, {"error": str(error)[:2000]}) @@ -139,10 +210,12 @@ def main() -> int: raise SystemExit("HERMES_WORKER_ORDINAL must be 0, 1, or 2") key = read_key(KEY_PATH) BoundedHTTPServer( - ("0.0.0.0", PORT), handler_factory(ClientBoundary(key)), max_workers=4 + ("0.0.0.0", PORT), + handler_factory(ClientBoundary(key)), + max_workers=4, ).serve_forever() return 0 -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint raise SystemExit(main()) diff --git a/services/hermes/scripts/execution_pool_coordinator.py b/services/hermes/scripts/execution_pool_coordinator.py index 8a128778..6fc0f5a4 100644 --- a/services/hermes/scripts/execution_pool_coordinator.py +++ b/services/hermes/scripts/execution_pool_coordinator.py @@ -3,7 +3,6 @@ from __future__ import annotations -import argparse import json import os import re @@ -16,33 +15,23 @@ from pathlib import Path from typing import Any import cli_lane_goal -import cli_lane_runner +import cli_lane_dispatch +from cli_lane_config import canonical_run_id +from execution_pool_project import ( + ProjectPolicyError, + distributed_workspace_eligible, + resolve_assignment, +) from execution_pool_protocol import ( MAX_ACTIVITY_BYTES, - MAX_WIRE_BYTES, - BoundedHTTPServer, PoolStore, ProtocolError, - canonical_json, - parse_wire, - read_key, + derive_ordinal_key, + envelope_ordinal, sign_envelope, verify_envelope, ) - -DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data")) -STATE_ROOT = DATA_ROOT / "execution-pool" -KEY_PATH = Path(os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/runtime-access/execution-pool-key")) -PORT = int(os.environ.get("HERMES_EXECUTION_POOL_PORT", "9007")) -RETENTION_SECONDS = int(os.environ.get("HERMES_EXECUTION_POOL_RETENTION_SECONDS", "1209600")) -ATLAS_REPO = re.compile(r"^https://scm\.bstein\.dev/atlas/[A-Za-z0-9_.-]+\.git$") -BRANCH = re.compile(r"^(?:feature|fix|chore|docs|test|refactor)/[A-Za-z0-9][A-Za-z0-9._/-]{0,119}$") -DEFAULT_REPO = os.environ.get( - "HERMES_EXECUTION_DEFAULT_REPO_URL", - "https://scm.bstein.dev/atlas/titan-iac.git", -) -DEFAULT_BASE_BRANCH = os.environ.get("HERMES_EXECUTION_DEFAULT_BASE_BRANCH", "main") REDACTIONS = ( re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/-]{12,}"), re.compile( @@ -59,43 +48,21 @@ def _task_value(task: Any, name: str, default: Any = None) -> Any: return getattr(task, name, default) -def resolve_scm(task: Any) -> tuple[str, str, str]: - """Resolve reviewed SCM metadata without executing in a legacy worktree.""" - raw = str(_task_value(task, "workspace_path", "") or "") - if raw: - raise RuntimeError( - "task owns a legacy local workspace; preserve or commit its tracked and " - "untracked files before distributed assignment" - ) - remote = str(_task_value(task, "repo_url", "") or DEFAULT_REPO) - if not ATLAS_REPO.fullmatch(remote) or "@" in remote: - raise RuntimeError("task origin is outside the reviewed Atlas SCM boundary") - task_id = str(_task_value(task, "id", "task")) - branch = str(_task_value(task, "branch_name", "") or "") - if not branch: - branch = f"feature/hermes-{re.sub(r'[^A-Za-z0-9._-]', '-', task_id)[:80]}" - if not BRANCH.fullmatch(branch) or ".." in branch or "//" in branch: - raise RuntimeError("task feature branch is outside the SCM branch policy") - base_branch = str( - _task_value(task, "base_branch", "") or DEFAULT_BASE_BRANCH - ) - if ( - not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,119}", base_branch) - or ".." in base_branch - or "//" in base_branch - ): - raise RuntimeError("repository default branch is outside the SCM policy") - return remote, branch, base_branch +def resolve_scm(task: Any, board: str = "titan-iac") -> tuple[str, str, str]: + """Resolve SCM only through the canonical board/project registry.""" + return resolve_assignment(board, task) -def assignment_payload(kanban_db: Any, connection: Any, task: Any) -> dict[str, Any]: +def assignment_payload( + kanban_db: Any, connection: Any, task: Any, board: str = "titan-iac" +) -> dict[str, Any]: context = kanban_db.build_worker_context(connection, str(_task_value(task, "id"))) if not isinstance(context, str): context = json.dumps(context, default=str, sort_keys=True) encoded = context.encode("utf-8") if len(encoded) > 32 * 1024: raise RuntimeError("Kanban worker context exceeds the 32KiB assignment limit") - repo_url, branch, base_branch = resolve_scm(task) + repo_url, branch, base_branch = resolve_scm(task, board) runtime = int(_task_value(task, "max_runtime_seconds", 0) or 12 * 60 * 60) runtime = max(60, min(runtime, 12 * 60 * 60)) return { @@ -171,10 +138,22 @@ class Coordinator: """Own all transitions between durable assignments and Hermes Kanban.""" def __init__(self, key: bytes, store: PoolStore): - self.key = key + self.master_key = key self.store = store self._kanban_lock = threading.RLock() + def _key(self, envelope: dict[str, Any]) -> bytes: + return derive_ordinal_key(self.master_key, envelope_ordinal(envelope)) + + def _verify(self, envelope: dict[str, Any], kind: str) -> dict[str, Any]: + return verify_envelope(self._key(envelope), envelope, expected_kind=kind) + + def _sign( + self, kind: str, binding: dict[str, Any], payload: dict[str, Any] + ) -> dict[str, Any]: + key = derive_ordinal_key(self.master_key, int(binding["worker_ordinal"])) + return sign_envelope(key, kind, binding, payload) + @staticmethod def _binding(record: dict[str, Any]) -> dict[str, Any]: return {name: record[name] for name in ( @@ -182,20 +161,18 @@ class Coordinator: )} def poll(self, envelope: dict[str, Any]) -> dict[str, Any]: - verified = verify_envelope(self.key, envelope, expected_kind="poll") + verified = self._verify(envelope, "poll") record = self.store.offer(int(verified["worker_ordinal"])) if record is None: binding = {**self._binding({ "board": "", "task_id": "", "run_id": "", "worker_ordinal": verified["worker_ordinal"], "attempt": 0, })} - return sign_envelope(self.key, "ack", binding, {"assignment": None}) - return sign_envelope( - self.key, "assignment", self._binding(record), record["payload"] - ) + return self._sign("ack", binding, {"assignment": None}) + return self._sign("assignment", self._binding(record), record["payload"]) def heartbeat(self, envelope: dict[str, Any]) -> dict[str, Any]: - verified = verify_envelope(self.key, envelope, expected_kind="heartbeat") + verified = self._verify(envelope, "heartbeat") accepted, duplicate = self.store.heartbeat(verified) from hermes_cli import kanban_db @@ -204,10 +181,13 @@ class Coordinator: with self._kanban_lock, kanban_db.scoped_current_board(verified["board"]): connection = kanban_db.connect(board=verified["board"]) try: + run_id = canonical_run_id(verified["run_id"]) + if run_id is None: + raise ProtocolError("worker run ID is not a canonical SQLite run") alive = kanban_db.heartbeat_worker( connection, verified["task_id"], note=note or "distributed worker active", - expected_run_id=verified["run_id"], + expected_run_id=run_id, ) if not alive: raise ProtocolError("Kanban run no longer owns this worker") @@ -215,18 +195,18 @@ class Coordinator: _append_activity(kanban_db, verified) finally: connection.close() - return sign_envelope( - self.key, "ack", self._binding(verified), + return self._sign( + "ack", self._binding(verified), {"accepted": accepted, "duplicate": duplicate}, ) def result(self, envelope: dict[str, Any]) -> dict[str, Any]: - verified = verify_envelope(self.key, envelope, expected_kind="result") + verified = self._verify(envelope, "result") record, duplicate = self.store.accept_result(verified) if record.get("state") == "result": self.finalize(record) - return sign_envelope( - self.key, "ack", self._binding(verified), + return self._sign( + "ack", self._binding(verified), {"accepted": True, "duplicate": duplicate}, ) @@ -235,6 +215,10 @@ class Coordinator: from hermes_cli import kanban_db binding = self._binding(record) + run_id = canonical_run_id(binding["run_id"]) + if run_id is None: + self.store.finalize(binding, "stale") + return payload = record.get("result") if not isinstance(payload, dict): raise ProtocolError("result payload must be an object") @@ -245,7 +229,10 @@ class Coordinator: connection = kanban_db.connect(board=binding["board"]) try: task = kanban_db.get_task(connection, binding["task_id"]) - if task is None or _task_value(task, "current_run_id") != binding["run_id"]: + current_run_id = canonical_run_id( + _task_value(task, "current_run_id", None) + ) if task is not None else None + if current_run_id is None or current_run_id != run_id: self.store.finalize(binding, "stale") return _append_terminal_activity(kanban_db, record) @@ -273,7 +260,7 @@ class Coordinator: connection, binding["task_id"], result=json.dumps(structured, sort_keys=True), summary=str(structured.get("summary") or "Completed"), - metadata=metadata, expected_run_id=binding["run_id"], + metadata=metadata, expected_run_id=run_id, ) else: reason = problem or "; ".join(map(str, structured.get("blockers", []))) @@ -281,7 +268,7 @@ class Coordinator: changed = kanban_db.block_task( connection, binding["task_id"], reason=reason, kind="transient" if payload.get("capacity_failure") else "capability", - expected_run_id=binding["run_id"], + expected_run_id=run_id, ) self.store.finalize(binding, "finalized" if changed else "stale") finally: @@ -294,6 +281,37 @@ class Coordinator: except (OSError, sqlite3.Error) as error: print(f"result recovery deferred: {error}", file=sys.stderr, flush=True) + def expire_leases(self) -> None: + """Fence dead attempts; after bounded retries, surface and release the run.""" + from hermes_cli import kanban_db + + for record in self.store.expire_leases(): + if record.get("state") != "lease_failed": + continue + binding = self._binding(record) + run_id = canonical_run_id(binding["run_id"]) + if run_id is None: + self.store.finalize(binding, "stale") + continue + with self._kanban_lock, kanban_db.scoped_current_board(binding["board"]): + connection = kanban_db.connect(board=binding["board"]) + try: + changed = kanban_db.block_task( + connection, + binding["task_id"], + reason=( + "Distributed worker lease expired after " + f"{binding['attempt']} fenced attempts" + ), + kind="transient", + expected_run_id=run_id, + ) + self.store.finalize( + binding, "finalized" if changed else "stale" + ) + finally: + connection.close() + def reconcile(self) -> None: """Recover the narrow claim/assignment crash gaps without double execution.""" from hermes_cli import kanban_db @@ -309,7 +327,12 @@ class Coordinator: connection = kanban_db.connect(board=record["board"]) try: task = kanban_db.get_task(connection, record["task_id"]) - current = str(_task_value(task, "current_run_id", "") or "") if task else "" + current_run_id = ( + canonical_run_id(_task_value(task, "current_run_id", None)) + if task + else None + ) + current = str(current_run_id or "") status = str(_task_value(task, "status", "") or "") if task else "" finally: connection.close() @@ -321,7 +344,7 @@ class Coordinator: if not ordinals: return for raw_board in kanban_db.list_boards(include_archived=False): - board = cli_lane_runner._board_slug(raw_board) + board = cli_lane_dispatch._board_slug(raw_board) if not board: continue with kanban_db.scoped_current_board(board): @@ -330,7 +353,10 @@ class Coordinator: tasks = kanban_db.list_tasks(connection) for task in tasks: task_id = str(_task_value(task, "id", "") or "") - run_id = str(_task_value(task, "current_run_id", "") or "") + raw_run_id = canonical_run_id( + _task_value(task, "current_run_id", None) + ) + run_id = str(raw_run_id or "") assignee = str(_task_value(task, "assignee", "") or "") if ( not ordinals @@ -338,12 +364,12 @@ class Coordinator: or not run_id or str(_task_value(task, "status", "")) != "running" or not assignee.startswith("cli-") + or not distributed_workspace_eligible(task) or (board, task_id, run_id) in active_runs ): continue try: - payload = assignment_payload(kanban_db, connection, task) - kanban_db.set_branch_name(connection, task_id, payload["branch"]) + payload = assignment_payload(kanban_db, connection, task, board) except Exception as error: kanban_db.block_task( connection, task_id, @@ -351,7 +377,7 @@ class Coordinator: "Distributed assignment recovery failed: " f"{type(error).__name__}: {error}" ), - kind="capability", expected_run_id=run_id, + kind="capability", expected_run_id=raw_run_id, ) continue ordinal = ordinals.pop(0) @@ -373,9 +399,13 @@ class Coordinator: ordinals = self.store.available_ordinals() if not ordinals: return - claimed = cli_lane_runner.claim_ready(set(), len(ordinals)) + claimed = cli_lane_dispatch.claim_ready( + set(), len(ordinals), + lambda _board, task: distributed_workspace_eligible(task), + ) for ordinal, (board, task_id) in zip(ordinals, claimed, strict=False): run_id = "" + database_run_id: int | None = None try: with kanban_db.scoped_current_board(board): connection = kanban_db.connect(board=board) @@ -383,9 +413,29 @@ class Coordinator: task = kanban_db.get_task(connection, task_id) if task is None: continue - run_id = str(_task_value(task, "current_run_id", "") or "") - payload = assignment_payload(kanban_db, connection, task) - kanban_db.set_branch_name(connection, task_id, payload["branch"]) + if not distributed_workspace_eligible(task): + raw_run = canonical_run_id( + _task_value(task, "current_run_id", None) + ) + if raw_run is not None: + kanban_db.block_task( + connection, task_id, + reason=( + "Distributed claim fenced because an existing " + "workspace is owned by the local lane" + ), + kind="capability", + expected_run_id=raw_run, + ) + continue + raw_run = canonical_run_id( + _task_value(task, "current_run_id", None) + ) + if raw_run is None: + raise ProjectPolicyError("claimed task has no canonical run ID") + database_run_id = raw_run + run_id = str(raw_run) + payload = assignment_payload(kanban_db, connection, task, board) finally: connection.close() binding = { @@ -400,92 +450,24 @@ class Coordinator: try: kanban_db.block_task( connection, task_id, reason=reason, kind="capability", - expected_run_id=run_id or None, + expected_run_id=database_run_id, ) finally: connection.close() def handler_factory(coordinator: Coordinator) -> type[BaseHTTPRequestHandler]: - class Handler(BaseHTTPRequestHandler): - server_version = "hermes-execution-pool/1" + """Compatibility export for tests and the mounted coordinator entry point.""" + from execution_pool_server import handler_factory as factory - def _reply(self, status: int, value: dict[str, Any]) -> None: - body = canonical_json(value) - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.send_header("Cache-Control", "no-store") - self.end_headers() - self.wfile.write(body) - - def do_GET(self) -> None: # noqa: N802 - if self.path != "/ready": - self._reply(404, {"error": "not found"}) - return - try: - coordinator.store.available_ordinals() - self._reply(200, {"ready": True}) - except (OSError, sqlite3.Error): - self._reply(503, {"ready": False}) - - def do_POST(self) -> None: # noqa: N802 - try: - length = int(self.headers.get("Content-Length", "0")) - if length <= 0 or length > MAX_WIRE_BYTES: - raise ProtocolError("invalid content length") - envelope = parse_wire(self.rfile.read(length)) - routes = { - "/v1/poll": coordinator.poll, - "/v1/heartbeat": coordinator.heartbeat, - "/v1/result": coordinator.result, - } - if self.path not in routes: - self._reply(404, {"error": "not found"}) - return - self._reply(200, routes[self.path](envelope)) - except ProtocolError as error: - self._reply(409, {"error": str(error)}) - except Exception as error: - print(f"pool request failed: {type(error).__name__}: {error}", file=sys.stderr, flush=True) - self._reply(503, {"error": "coordinator unavailable"}) - - def log_message(self, _format: str, *_arguments: Any) -> None: - return - - return Handler + return factory(coordinator) def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--once", action="store_true") - args = parser.parse_args() - key = read_key(KEY_PATH) - store = PoolStore(STATE_ROOT / "assignments.db") - coordinator = Coordinator(key, store) - coordinator.recover_results() - coordinator.reconcile() - coordinator.dispatch() - if args.once: - return 0 - server = BoundedHTTPServer( - ("0.0.0.0", PORT), handler_factory(coordinator), max_workers=8 - ) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - while True: - for operation in ( - coordinator.recover_results, - coordinator.reconcile, - coordinator.dispatch, - lambda: store.garbage_collect(RETENTION_SECONDS), - ): - try: - operation() - except Exception as error: - print(f"pool maintenance deferred: {type(error).__name__}: {error}", file=sys.stderr, flush=True) - time.sleep(5) + from execution_pool_server import run + + return run(Coordinator) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint raise SystemExit(main()) diff --git a/services/hermes/scripts/execution_pool_project.py b/services/hermes/scripts/execution_pool_project.py new file mode 100644 index 00000000..d4b6f382 --- /dev/null +++ b/services/hermes/scripts/execution_pool_project.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Canonical Atlas project and Git-ref policy for distributed execution.""" + +from __future__ import annotations + +import json +import os +import re +import stat +import subprocess +from pathlib import Path +from typing import Any + + +DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data")) +PROJECT_ROOT = DATA_ROOT / "workspace/projects" +BOARD_ROOT = DATA_ROOT / "kanban/boards" +ATLAS_REPO = re.compile( + r"https://scm\.bstein\.dev/atlas/(?P[A-Za-z0-9][A-Za-z0-9_.-]{0,99})\.git\Z" +) +SAFE_PREFIXES = frozenset( + {"feature", "fix", "chore", "docs", "test", "refactor", "wt", "review", "hermes", "handoff"} +) +IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z") +GIT = "/usr/bin/git" +MAX_BOARD_BYTES = 64 * 1024 + + +class ProjectPolicyError(ValueError): + """Canonical project metadata or a requested ref failed closed.""" + + +def _task_value(task: Any, name: str, default: Any = None) -> Any: + return getattr(task, name, default) + + +def _run_git(workdir: Path, *arguments: str) -> str: + completed = subprocess.run( + [GIT, "-C", str(workdir), *arguments], + check=False, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=30, + env={ + "HOME": "/nonexistent", + "PATH": "/usr/bin:/bin", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + }, + ) + if completed.returncode: + raise ProjectPolicyError("canonical project Git metadata is unavailable") + return completed.stdout.strip() + + +def validate_branch(value: object, *, feature: bool) -> str: + """Validate a complete local branch name with Git and a reviewed namespace.""" + if not isinstance(value, str) or not value or len(value) > 200: + raise ProjectPolicyError("branch name exceeds the safe limit") + if not value.isascii() or value.startswith("-"): + raise ProjectPolicyError("branch name must be canonical ASCII") + prefix = value.split("/", 1)[0] + if feature and ("/" not in value or prefix not in SAFE_PREFIXES): + raise ProjectPolicyError("task branch is outside the reviewed namespace") + completed = subprocess.run( + [GIT, "check-ref-format", f"refs/heads/{value}"], + check=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + env={"PATH": "/usr/bin:/bin", "GIT_CONFIG_NOSYSTEM": "1"}, + ) + if completed.returncode: + raise ProjectPolicyError("branch name is not a valid Git ref") + return value + + +def _read_board(board: str) -> dict[str, Any]: + if not IDENTIFIER.fullmatch(board): + raise ProjectPolicyError("board slug is invalid") + path = BOARD_ROOT / board / "board.json" + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= MAX_BOARD_BYTES: + raise ProjectPolicyError("board registry entry is not a bounded regular file") + raw = os.read(descriptor, MAX_BOARD_BYTES + 1) + finally: + os.close(descriptor) + try: + value = json.loads(raw) + except (UnicodeError, json.JSONDecodeError) as error: + raise ProjectPolicyError("board registry entry is malformed") from error + if not isinstance(value, dict) or value.get("slug") != board or value.get("archived") is True: + raise ProjectPolicyError("board registry identity is unavailable or archived") + return value + + +def resolve_project(board: str) -> tuple[str, str, Path]: + """Resolve repo and base exclusively from the canonical board registry. + + A board may be created before its primary checkout. In that case the + Atlas repository naming contract and ``main`` are the registry defaults; + once a checkout exists, its credential-free origin and remote HEAD must + agree with that identity. This keeps a missing checkout from silently + routing work to some other project's repository. + """ + entry = _read_board(board) + raw_workdir = entry.get("default_workdir") + if not isinstance(raw_workdir, str) or not Path(raw_workdir).is_absolute(): + raise ProjectPolicyError("board default_workdir is missing") + workdir = Path(raw_workdir).resolve(strict=False) + project_root = PROJECT_ROOT.resolve(strict=True) + try: + workdir.relative_to(project_root) + except ValueError as error: + raise ProjectPolicyError("board workdir is outside the Atlas project registry") from error + remote = f"https://scm.bstein.dev/atlas/{board}.git" + if not ATLAS_REPO.fullmatch(remote): + raise ProjectPolicyError("board repository identity is invalid") + if not workdir.exists(): + return remote, "main", workdir + if not workdir.is_dir(): + raise ProjectPolicyError("board checkout is not a directory") + checkout_remote = _run_git(workdir, "remote", "get-url", "origin") + if "@" in checkout_remote or checkout_remote != remote: + raise ProjectPolicyError("board origin disagrees with the Atlas registry") + try: + base = _run_git(workdir, "symbolic-ref", "--short", "refs/remotes/origin/HEAD") + if not base.startswith("origin/"): + raise ProjectPolicyError("origin HEAD is not canonical") + base = base.removeprefix("origin/") + except ProjectPolicyError: + base = "main" + return remote, validate_branch(base, feature=False), workdir + + +def resolve_assignment(board: str, task: Any) -> tuple[str, str, str]: + """Resolve the exact repo/base and safe task branch for one board task.""" + repo, base, _workdir = resolve_project(board) + task_id = str(_task_value(task, "id", "") or "") + if not IDENTIFIER.fullmatch(task_id): + raise ProjectPolicyError("task identity is invalid") + branch = str(_task_value(task, "branch_name", "") or f"wt/{task_id}") + return repo, validate_branch(branch, feature=True), base + + +def distributed_workspace_eligible(task: Any) -> bool: + """Only pathless tasks migrate; an existing worktree stays with its local owner.""" + return not str(_task_value(task, "workspace_path", "") or "").strip() diff --git a/services/hermes/scripts/execution_pool_protocol.py b/services/hermes/scripts/execution_pool_protocol.py index 8659a9ba..599279be 100644 --- a/services/hermes/scripts/execution_pool_protocol.py +++ b/services/hermes/scripts/execution_pool_protocol.py @@ -22,6 +22,7 @@ MAX_WIRE_BYTES = 64 * 1024 MAX_ACTIVITY_BYTES = 12 * 1024 MAX_CLOCK_SKEW = 30 MAX_ENVELOPE_LIFETIME = 300 +PROTOCOL_VERSION = 2 IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") KINDS = frozenset({"poll", "assignment", "heartbeat", "result", "ack"}) @@ -64,9 +65,7 @@ class BoundedHTTPServer(ThreadingHTTPServer): def canonical_json(value: Any) -> bytes: """Encode one value deterministically for digests and signatures.""" - return json.dumps( - value, ensure_ascii=False, separators=(",", ":"), sort_keys=True - ).encode("utf-8") + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") def atomic_json(path: Path, value: dict[str, Any], mode: int = 0o600) -> None: @@ -95,6 +94,24 @@ def payload_digest(payload: Any) -> str: return hashlib.sha256(canonical_json(payload)).hexdigest() +def derive_ordinal_key(master: bytes, ordinal: int) -> bytes: + """Derive one cryptographically isolated worker authority from the pool root.""" + if ordinal not in range(3) or not 32 <= len(master) <= 4096: + raise ProtocolError("pool key derivation input is invalid") + context = f"hermes-execution-pool-v2:worker:{ordinal}".encode() + return hmac.new(master, context, hashlib.sha256).hexdigest().encode() + + +def envelope_ordinal(envelope: Any) -> int: + """Read only the bounded ordinal needed to select a verification key.""" + if not isinstance(envelope, dict): + raise ProtocolError("invalid pool message") + value = envelope.get("worker_ordinal") + if type(value) is not int or value not in range(3): + raise ProtocolError("worker binding is outside the pool") + return value + + def read_key(path: Path) -> bytes: """Read a private regular file without following a final symlink.""" flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) @@ -115,7 +132,9 @@ def read_key(path: Path) -> bytes: def _identifier(name: str, value: Any, *, allow_empty: bool = False) -> str: - text = str(value or "") + if not isinstance(value, str): + raise ProtocolError(f"invalid {name}") + text = value if allow_empty and not text: return text if not IDENTIFIER.fullmatch(text): @@ -137,7 +156,7 @@ def sign_envelope( current = int(time.time()) if now is None else int(now) lifetime = max(1, min(int(lifetime), MAX_ENVELOPE_LIFETIME)) envelope = { - "version": 1, + "version": PROTOCOL_VERSION, "kind": kind, "board": str(binding.get("board") or ""), "task_id": str(binding.get("task_id") or ""), @@ -169,14 +188,14 @@ def verify_envelope( """Verify structure, lifetime, digest, and HMAC before using a message.""" if not isinstance(envelope, dict) or len(canonical_json(envelope)) > MAX_WIRE_BYTES: raise ProtocolError("invalid or oversized pool message") - required = { - "version", "kind", "board", "task_id", "run_id", "worker_ordinal", - "attempt", "delivery_id", "issued_at", "expires_at", "payload_digest", - "payload", "signature", - } - if set(envelope) != required or envelope.get("version") != 1: - raise ProtocolError("pool message fields do not match version 1") - kind = str(envelope["kind"]) + required = {"version", "kind", "board", "task_id", "run_id", "worker_ordinal", "attempt", "delivery_id", "issued_at", "expires_at", "payload_digest", "payload", "signature"} + if set(envelope) != required or envelope.get("version") != PROTOCOL_VERSION: + raise ProtocolError( + f"pool message fields do not match version {PROTOCOL_VERSION}" + ) + kind = envelope["kind"] + if not isinstance(kind, str): + raise ProtocolError("unexpected message kind") if kind not in KINDS or (expected_kind and kind != expected_kind): raise ProtocolError("unexpected message kind") empty_binding = kind in {"poll", "ack"} @@ -184,13 +203,10 @@ def verify_envelope( _identifier("task_id", envelope["task_id"], allow_empty=empty_binding) _identifier("run_id", envelope["run_id"], allow_empty=empty_binding) _identifier("delivery_id", envelope["delivery_id"]) - try: - ordinal = int(envelope["worker_ordinal"]) - attempt = int(envelope["attempt"]) - issued = int(envelope["issued_at"]) - expires = int(envelope["expires_at"]) - except (TypeError, ValueError) as error: - raise ProtocolError("invalid numeric binding") from error + numeric = tuple(envelope[name] for name in ("worker_ordinal", "attempt", "issued_at", "expires_at")) + if any(type(value) is not int for value in numeric): + raise ProtocolError("invalid numeric binding") + ordinal, attempt, issued, expires = numeric if ordinal not in range(3) or attempt < 0: raise ProtocolError("worker binding is outside the pool") current = int(time.time()) if now is None else int(now) @@ -374,6 +390,8 @@ class PoolStore: row = self._matching(connection, envelope) if row["state"] not in {"assigned", "running"}: raise ProtocolError("assignment is no longer running") + if row["state"] == "running" and float(row["lease_until"]) < now: + raise ProtocolError("assignment lease expired") if duplicate and duplicate[0] != delivery_digest: raise ProtocolError("delivery identifier was reused") if not duplicate: @@ -401,6 +419,8 @@ class PoolStore: return self._record(row) or {}, True if row["state"] not in {"assigned", "running"}: raise ProtocolError("assignment cannot accept a result") + if row["state"] == "running" and float(row["lease_until"]) < now: + raise ProtocolError("assignment lease expired") connection.execute( """UPDATE assignments SET state='result',result_digest=?,result_json=?,updated_at=? WHERE board=? AND task_id=? AND run_id=?""", @@ -416,18 +436,53 @@ class PoolStore: def pending_results(self) -> list[dict[str, Any]]: with self._connect() as connection: - rows = connection.execute( - "SELECT * FROM assignments WHERE state='result' ORDER BY updated_at" - ).fetchall() + rows = connection.execute("SELECT * FROM assignments WHERE state='result' ORDER BY updated_at").fetchall() return [self._record(row) or {} for row in rows] + def expire_leases( + self, *, now: float | None = None, max_attempts: int = 3 + ) -> list[dict[str, Any]]: + """Fence expired attempts and re-offer or terminally release their ordinals.""" + current = time.time() if now is None else float(now) + maximum = max(1, min(int(max_attempts), 10)) + changed: list[dict[str, Any]] = [] + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + rows = connection.execute( + """SELECT * FROM assignments WHERE state='running' + AND lease_until > 0 AND lease_until < ? ORDER BY updated_at""", + (current,), + ).fetchall() + for row in rows: + if int(row["attempt"]) >= maximum: + state, attempt = "lease_failed", int(row["attempt"]) + else: + state, attempt = "assigned", int(row["attempt"]) + 1 + connection.execute( + """UPDATE assignments SET state=?,attempt=?,lease_until=0, + last_heartbeat=0,updated_at=? WHERE board=? AND task_id=? AND run_id=? + AND attempt=? AND state='running'""", + ( + state, attempt, current, row["board"], row["task_id"], + row["run_id"], row["attempt"], + ), + ) + updated = connection.execute( + "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?", + (row["board"], row["task_id"], row["run_id"]), + ).fetchone() + if updated is not None: + changed.append(self._record(updated) or {}) + connection.commit() + return changed + def finalize(self, binding: dict[str, Any], state: str) -> None: if state not in {"finalized", "stale"}: raise ProtocolError("invalid terminal assignment state") with self._lock, self._connect() as connection: connection.execute( - "UPDATE assignments SET state=?,updated_at=? WHERE board=? AND task_id=? AND run_id=?", - (state, time.time(), binding["board"], binding["task_id"], binding["run_id"]), + "UPDATE assignments SET state=?,updated_at=? WHERE board=? AND task_id=? AND run_id=? AND worker_ordinal=? AND attempt=?", + (state, time.time(), *(binding[name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt"))), ) def garbage_collect(self, retention_seconds: int) -> int: diff --git a/services/hermes/scripts/execution_pool_scm.py b/services/hermes/scripts/execution_pool_scm.py index 74a488b1..6badfbb9 100644 --- a/services/hermes/scripts/execution_pool_scm.py +++ b/services/hermes/scripts/execution_pool_scm.py @@ -1,153 +1,97 @@ #!/usr/bin/env python3 -"""Ordinal-scoped Atlas SCM boundary; the model container never gets its token.""" +"""Assignment-bound Git gate routed exclusively through the PR14 SCM broker.""" from __future__ import annotations -import configparser import json import os import re -import shutil import stat import subprocess import threading -import urllib.error import urllib.parse -import urllib.request -import uuid -from http.server import BaseHTTPRequestHandler from pathlib import Path from typing import Any -from execution_pool_protocol import ( - MAX_WIRE_BYTES, - BoundedHTTPServer, - ProtocolError, - parse_wire, - read_key, - verify_envelope, -) +import scm_broker_client +from execution_pool_project import ATLAS_REPO, validate_branch +from execution_pool_protocol import ProtocolError, atomic_json, verify_envelope WORKSPACE_ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace")) SCM_ROOT = Path(os.environ.get("HERMES_SCM_STATE_ROOT", "/scm-state")) -KEY_PATH = Path(os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/pool-access/execution-pool-key")) -TOKEN_PATH = Path(os.environ.get("HERMES_GITEA_TOKEN_FILE", "/vault/secrets/gitea-token")) -USERNAME_PATH = Path(os.environ.get("HERMES_GITEA_USERNAME_FILE", "/vault/secrets/gitea-username")) -ASKPASS = os.environ.get( - "HERMES_GITEA_ASKPASS", "/opt/coordinator/execution_pool_askpass.sh" -) ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1")) -PORT = int(os.environ.get("HERMES_SCM_BOUNDARY_PORT", "9008")) -REPO = re.compile(r"^https://scm\.bstein\.dev/atlas/([A-Za-z0-9_.-]+)\.git$") -BRANCH = re.compile(r"^(?:feature|fix|chore|docs|test|refactor)/[A-Za-z0-9][A-Za-z0-9._/-]{0,119}$") -MAX_BUNDLE_BYTES = 128 * 1024 * 1024 +BROKER_ORIGIN = scm_broker_client.BROKER_ORIGIN.rstrip("/") +MAX_STATUS_BYTES = 4 * 1024 * 1024 -def _private_text(path: Path) -> str: - descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) - try: - value = os.read(descriptor, 65537).decode("utf-8").strip() - finally: - os.close(descriptor) - if not value or len(value) > 65536: - raise ProtocolError(f"SCM credential is missing or invalid: {path.name}") - return value - - -def _git_env(authenticated: bool) -> dict[str, str]: - """Expose credential paths only to explicit boundary-owned network calls.""" - sensitive = { - "GIT_ASKPASS", "SSH_ASKPASS", "HERMES_SCM_PASSWORD_FILE", - "HERMES_SCM_USERNAME_FILE", +def _git_environment() -> dict[str, str]: + """Run Git without credentials, prompts, ambient config, or hook execution.""" + return { + "HOME": "/nonexistent", + "PATH": "/usr/bin:/bin", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", } - environment = { - name: value for name, value in os.environ.items() - if name not in sensitive and not name.startswith("GIT_CONFIG_") - } - environment["GIT_TERMINAL_PROMPT"] = "0" - if authenticated: - environment.update( - { - "GIT_ASKPASS": ASKPASS, - "HERMES_SCM_PASSWORD_FILE": str(TOKEN_PATH), - "HERMES_SCM_USERNAME_FILE": str(USERNAME_PATH), - "GIT_CONFIG_COUNT": "1", - "GIT_CONFIG_KEY_0": ( - "url.http://gitea.gitea.svc.cluster.local:3000/.insteadOf" - ), - "GIT_CONFIG_VALUE_0": "https://scm.bstein.dev/", - } - ) - return environment -def _run( - *arguments: str, - cwd: Path | None = None, - timeout: int = 300, - authenticated: bool = False, -) -> str: +def _run(*arguments: str, cwd: Path | None = None, timeout: int = 300) -> str: + command = [ + "/usr/bin/git", + "-c", + "core.hooksPath=/dev/null", + "-c", + "core.fsmonitor=false", + *arguments, + ] completed = subprocess.run( - list(arguments), cwd=cwd, env=_git_env(authenticated), stdin=subprocess.DEVNULL, - text=True, capture_output=True, timeout=timeout, check=False, + command, + cwd=cwd, + env=_git_environment(), + stdin=subprocess.DEVNULL, + text=True, + capture_output=True, + timeout=timeout, + check=False, ) if completed.returncode: message = (completed.stderr or completed.stdout or "SCM operation failed")[-2000:] raise RuntimeError(message.strip()) + if len(completed.stdout.encode()) > MAX_STATUS_BYTES: + raise ProtocolError("SCM command output exceeds the safe limit") return completed.stdout.strip() -def _regular_text(path: Path, limit: int, encoding: str = "utf-8") -> str: - """Read bounded model-controlled metadata without following or blocking.""" - flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0) - try: - descriptor = os.open(path, flags) - except OSError as error: - raise ProtocolError(f"workspace metadata is unavailable: {path.name}") from error - try: - info = os.fstat(descriptor) - if not stat.S_ISREG(info.st_mode) or info.st_size > limit: - raise ProtocolError(f"workspace metadata is invalid: {path.name}") - raw = os.read(descriptor, limit + 1) - if len(raw) > limit: - raise ProtocolError(f"workspace metadata is oversized: {path.name}") - return raw.decode(encoding) - except UnicodeError as error: - raise ProtocolError(f"workspace metadata is malformed: {path.name}") from error - finally: - os.close(descriptor) - - def _binding(envelope: dict[str, Any]) -> tuple[dict[str, Any], str, str, str]: if envelope["kind"] != "assignment" or int(envelope["worker_ordinal"]) != ORDINAL: raise ProtocolError("assignment does not belong to this worker ordinal") - payload = envelope["payload"] + payload = envelope.get("payload") if not isinstance(payload, dict): raise ProtocolError("assignment payload must be an object") repo = str(payload.get("repo_url") or "") - branch = str(payload.get("branch") or "") - base_branch = str(payload.get("base_branch") or "main") - match = REPO.fullmatch(repo) - if ( - not match - or not BRANCH.fullmatch(branch) - or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,119}", base_branch) - or ".." in branch - or "//" in branch - or ".." in base_branch - or "//" in base_branch - ): - raise ProtocolError("assignment is outside the Atlas SCM policy") - run_name = f"{envelope['task_id']}-{envelope['run_id']}" - return payload, repo, branch, match.group(1) + ":" + run_name + match = ATLAS_REPO.fullmatch(repo) + if not match: + raise ProtocolError("assignment repository is outside Atlas") + try: + branch = validate_branch(payload.get("branch"), feature=True) + base = validate_branch(payload.get("base_branch"), feature=False) + except ValueError as error: + raise ProtocolError(str(error)) from error + return payload, match.group("repo"), branch, base + + +def _broker_repo(repo: str) -> str: + return f"{BROKER_ORIGIN}/git/atlas/{repo}.git" def workspace_path(envelope: dict[str, Any]) -> Path: - """Derive a contained path; no caller-provided filesystem path is accepted.""" - parts = (str(envelope["board"]), str(envelope["task_id"]), str(envelope["run_id"])) - if any(not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", part) for part in parts): + """Derive a private ordinal path; no caller-provided path is accepted.""" + parts = tuple(str(envelope[name]) for name in ("board", "task_id", "run_id")) + identifier = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z") + if any(not identifier.fullmatch(part) for part in parts): raise ProtocolError("assignment path binding is invalid") + if WORKSPACE_ROOT.is_symlink(): + raise ProtocolError("workspace root must not be a symlink") workspace_root = WORKSPACE_ROOT.resolve() root = WORKSPACE_ROOT / "runs" if root.is_symlink(): @@ -156,112 +100,77 @@ def workspace_path(envelope: dict[str, Any]) -> Path: root = root.resolve() root.relative_to(workspace_root) candidate = root.joinpath(*parts) - if candidate.is_symlink(): - raise ProtocolError("workspace must not be a symlink") current = root for part in parts[:-1]: - current = current / part + current /= part if current.is_symlink(): raise ProtocolError("workspace parent must not be a symlink") current.mkdir(mode=0o700, exist_ok=True) - if current.is_symlink(): - raise ProtocolError("workspace parent must not be a symlink") - try: - candidate.resolve(strict=False).relative_to(root) - except ValueError as error: - raise ProtocolError("workspace escaped its ordinal root") from error + if candidate.is_symlink(): + raise ProtocolError("workspace must not be a symlink") + candidate.resolve(strict=False).relative_to(root) return candidate -def _workspace_identity(destination: Path, repo: str, branch: str) -> str: - """Read identity as data; never execute Git in a model-controlled checkout.""" - git_dir = destination / ".git" - if git_dir.is_symlink() or not git_dir.is_dir(): - raise ProtocolError("workspace Git metadata must be a private directory") - raw_config = _regular_text(git_dir / "config", 64 * 1024) - parser = configparser.ConfigParser(interpolation=None, strict=True) +def _regular_text(path: Path, maximum: int) -> str: + descriptor = os.open( + path, os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0) + ) try: - parser.read_string(raw_config) - origin = parser.get('remote "origin"', "url") - except (configparser.Error, KeyError, UnicodeError) as error: - raise ProtocolError("workspace Git config cannot prove its origin") from error - if origin != repo: - raise ProtocolError("durable workspace origin does not match assignment") - head = _regular_text(git_dir / "HEAD", 4096).strip() - if head != f"ref: refs/heads/{branch}": - raise ProtocolError("durable workspace branch does not match assignment") - ref_path = git_dir / "refs/heads" / Path(branch) - if ref_path.is_symlink(): - raise ProtocolError("workspace branch ref must not be a symlink") - if ref_path.exists(): - commit = _regular_text(ref_path, 128, "ascii").strip() - else: - packed = _regular_text(git_dir / "packed-refs", 1024 * 1024, "ascii") - matches = [ - line.split(" ", 1)[0] - for line in packed.splitlines() - if line.endswith(f" refs/heads/{branch}") - ] - if len(matches) != 1: - raise ProtocolError("workspace branch ref is unavailable") - commit = matches[0] - if not re.fullmatch(r"[0-9a-f]{40,64}", commit): - raise ProtocolError("workspace branch ref is invalid") - return commit + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or info.st_size > maximum: + raise ProtocolError("private SCM state is invalid") + raw = os.read(descriptor, maximum + 1) + finally: + os.close(descriptor) + try: + return raw.decode().strip() + except UnicodeError as error: + raise ProtocolError("private SCM state is malformed") from error -def _private_repo(envelope: dict[str, Any]) -> Path: +def _state_path(envelope: dict[str, Any]) -> Path: + if SCM_ROOT.is_symlink(): + raise ProtocolError("private SCM root must not be a symlink") root = SCM_ROOT.resolve() root.mkdir(mode=0o700, parents=True, exist_ok=True) - name = f"{envelope['board']}-{envelope['task_id']}-{envelope['run_id']}.git" - path = root / name - path.resolve(strict=False).relative_to(root) + name = "-".join( + str(envelope[name]) for name in ("board", "task_id", "run_id") + ) + path = root / f"{name}.json" if path.is_symlink(): raise ProtocolError("private SCM state must not be a symlink") + path.resolve(strict=False).relative_to(root) return path -def _copy_bundle(source: Path, destination: Path) -> None: - """Copy a bounded regular bundle into boundary-private storage.""" - source_fd = os.open( - source, os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0) - ) - temporary = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.tmp") - destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True) - target_fd = -1 - try: - info = os.fstat(source_fd) - if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= MAX_BUNDLE_BYTES: - raise ProtocolError("submission bundle is empty, oversized, or not regular") - target_fd = os.open( - temporary, - os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), - 0o600, - ) - remaining = info.st_size - while remaining: - chunk = os.read(source_fd, min(1024 * 1024, remaining)) - if not chunk: - raise ProtocolError("submission bundle ended early") - view = memoryview(chunk) - while view: - view = view[os.write(target_fd, view) :] - remaining -= len(chunk) - os.fsync(target_fd) - except Exception: - temporary.unlink(missing_ok=True) - raise - finally: - os.close(source_fd) - if target_fd >= 0: - os.close(target_fd) - try: - os.replace(temporary, destination) - finally: - temporary.unlink(missing_ok=True) +def _workspace_identity(destination: Path, repo: str, branch: str) -> str: + """Validate the checkout using bounded Git plumbing with all hooks disabled.""" + if destination.is_symlink() or not (destination / ".git").is_dir(): + raise ProtocolError("workspace Git metadata is unavailable") + ref_path = destination / ".git/refs/heads" + for part in branch.split("/"): + ref_path /= part + if ref_path.is_symlink(): + raise ProtocolError("workspace branch ref must not be a symlink") + origin = _run("remote", "get-url", "origin", cwd=destination) + if origin != f"https://scm.bstein.dev/atlas/{repo}.git": + raise ProtocolError("workspace origin does not match assignment") + broker = _run("remote", "get-url", "hermes-broker", cwd=destination) + if broker != _broker_repo(repo): + raise ProtocolError("workspace broker remote does not match assignment") + current = _run("symbolic-ref", "--short", "HEAD", cwd=destination) + if current != branch: + raise ProtocolError("workspace branch does not match assignment") + head = _run("rev-parse", "--verify", "HEAD", cwd=destination) + if not re.fullmatch(r"[0-9a-f]{40,64}", head): + raise ProtocolError("workspace HEAD is invalid") + return head class Boundary: + """The only process allowed to turn model output into an SCM/result handoff.""" + def __init__(self, key: bytes): self.key = key self.lock = threading.RLock() @@ -270,181 +179,103 @@ class Boundary: return verify_envelope(self.key, raw, expected_kind="assignment") def checkout(self, envelope: dict[str, Any]) -> dict[str, Any]: - """Clone/fetch exactly one assigned branch into its private durable path.""" - payload, repo, branch, _ = _binding(envelope) - base_branch = str(payload.get("base_branch") or "main") + _payload, repo, branch, base = _binding(envelope) destination = workspace_path(envelope) + state_path = _state_path(envelope) with self.lock: if (destination / ".git").exists(): - return { - "workspace": str(destination), "preserved_dirty_state": True, - "baseline_sha": _workspace_identity(destination, repo, branch), - } + _workspace_identity(destination, repo, branch) + state = json.loads(_regular_text(state_path, 16 * 1024)) + baseline = state.get("baseline_sha") if isinstance(state, dict) else None + if not isinstance(baseline, str) or not re.fullmatch( + r"[0-9a-f]{40,64}", baseline + ): + raise ProtocolError("private SCM baseline is unavailable") + return {"workspace": str(destination), "baseline_sha": baseline} if destination.exists() and any(destination.iterdir()): raise ProtocolError("workspace is non-empty and unmanaged") destination.parent.mkdir(parents=True, exist_ok=True) + broker = _broker_repo(repo) try: _run( - "git", "clone", "--single-branch", "--branch", branch, - "--no-tags", repo, str(destination), timeout=900, - authenticated=True, + "clone", "--single-branch", "--branch", branch, "--no-tags", + broker, str(destination), timeout=900, ) except RuntimeError as error: if destination.exists() and any(destination.iterdir()): - raise RuntimeError( - "assigned branch clone failed and left state for review" + raise ProtocolError( + "failed branch checkout left unmanaged workspace state" ) from error + if destination.exists(): + destination.rmdir() _run( - "git", "clone", "--single-branch", "--branch", base_branch, - "--no-tags", repo, str(destination), timeout=900, - authenticated=True, + "clone", "--single-branch", "--branch", base, "--no-tags", + broker, str(destination), timeout=900, ) - _run("git", "checkout", "-b", branch, cwd=destination) - _run("git", "config", "user.name", "Hermes Execution Worker", cwd=destination) - _run("git", "config", "user.email", "hermes@bstein.dev", cwd=destination) - return { - "workspace": str(destination), "preserved_dirty_state": False, - "baseline_sha": _run("git", "rev-parse", "HEAD", cwd=destination), - } - - def submit(self, envelope: dict[str, Any], request: dict[str, Any]) -> dict[str, Any]: - """Push only the assignment branch and create/reuse its draft pull request.""" - _payload, repo, branch, repo_binding = _binding(envelope) - repo_name, _ = repo_binding.split(":", 1) - destination = workspace_path(envelope) - title = str(request.get("title") or f"Hermes task {envelope['task_id']}").strip()[:240] - body = str(request.get("body") or "Automated draft from Hermes execution pool.")[:12000] - if not title or not (destination / ".git").exists(): - raise ProtocolError("submission workspace or title is invalid") - with self.lock: - _workspace_identity(destination, repo, branch) - private = _private_repo(envelope) - bundle = private.with_suffix(".bundle") - _copy_bundle(destination / ".git/hermes-submit.bundle", bundle) - if not private.exists(): - _run("git", "init", "--bare", str(private)) - _run("git", "--git-dir", str(private), "remote", "add", "origin", repo) - else: - _run("git", "--git-dir", str(private), "remote", "set-url", "origin", repo) - try: - _run( - "git", "--git-dir", str(private), "fetch", "origin", - f"+refs/heads/{branch}:refs/remotes/origin/{branch}", timeout=900, - authenticated=True, - ) - except RuntimeError as error: - if "couldn't find remote ref" not in str(error).lower(): - raise + _run("checkout", "-b", branch, cwd=destination) _run( - "git", "--git-dir", str(private), "fetch", str(bundle), - "HEAD:refs/pool/candidate", timeout=900, + "remote", "set-url", "origin", + f"https://scm.bstein.dev/atlas/{repo}.git", cwd=destination, ) - _run( - "git", "--git-dir", str(private), "push", "origin", - f"refs/pool/candidate:refs/heads/{branch}", timeout=900, - authenticated=True, + _run("remote", "add", "hermes-broker", broker, cwd=destination) + _run("config", "user.name", "Hermes Execution Worker", cwd=destination) + _run("config", "user.email", "hermes@bstein.dev", cwd=destination) + baseline = _workspace_identity(destination, repo, branch) + atomic_json( + state_path, + {"baseline_sha": baseline, "repo": repo, "branch": branch}, ) - pull = self._draft_pull( - repo_name, branch, str(_payload.get("base_branch") or "main"), title, body - ) - bundle.unlink(missing_ok=True) - shutil.rmtree(private) - return {"workspace": str(destination), "branch": branch, "pull_request": pull} + return {"workspace": str(destination), "baseline_sha": baseline} @staticmethod - def _api(path: str, data: dict[str, Any] | None = None) -> Any: - token = _private_text(TOKEN_PATH) - request = urllib.request.Request( - "http://gitea.gitea.svc.cluster.local:3000" + path, - data=json.dumps(data, separators=(",", ":")).encode() if data else None, - method="POST" if data else "GET", - headers={ - "Authorization": f"token {token}", "Accept": "application/json", - "Content-Type": "application/json", "User-Agent": "hermes-scm-boundary/1", - }, + def _draft(repo: str, branch: str, base: str, head: str, title: str, body: str) -> str: + query = urllib.parse.urlencode( + {"state": "open", "head": f"atlas:{branch}", "limit": 10} + ) + existing = json.loads( + scm_broker_client.read(f"/api/v1/repos/atlas/{repo}/pulls?{query}") ) - with urllib.request.urlopen(request, timeout=30) as response: - body = response.read(1024 * 1024 + 1) - if len(body) > 1024 * 1024: - raise ProtocolError("Gitea response exceeds the SCM boundary limit") - value = json.loads(body) - if not isinstance(value, (dict, list)): - raise ProtocolError("Gitea response has an invalid shape") - return value - - def _draft_pull( - self, repo: str, branch: str, base_branch: str, title: str, body: str - ) -> str: - query = urllib.parse.urlencode({"state": "open", "head": f"atlas:{branch}", "limit": 10}) - existing = self._api(f"/api/v1/repos/atlas/{repo}/pulls?{query}") if isinstance(existing, list) and existing: return str(existing[0].get("html_url") or "") - created = self._api( - f"/api/v1/repos/atlas/{repo}/pulls", - { - "base": base_branch, - "head": branch, "title": title, "body": body, "draft": True, - }, + created = json.loads( + scm_broker_client.create_draft( + repo, + base=base, + head=branch, + head_sha=head, + title=title, + body=body, + ) ) return str(created.get("html_url") or "") - def garbage_collect(self, envelope: dict[str, Any]) -> dict[str, Any]: - """Online requests never delete model-controlled or private SCM state.""" - return {"removed": False, "eligible": False, "workspace": str(workspace_path(envelope))} - - -def handler_factory(boundary: Boundary) -> type[BaseHTTPRequestHandler]: - class Handler(BaseHTTPRequestHandler): - server_version = "hermes-scm-boundary/1" - - def _reply(self, status: int, value: dict[str, Any]) -> None: - body = json.dumps(value, separators=(",", ":"), sort_keys=True).encode() - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.send_header("Cache-Control", "no-store") - self.end_headers() - self.wfile.write(body) - - def do_GET(self) -> None: # noqa: N802 - self._reply(200, {"ready": True}) if self.path == "/ready" else self._reply(404, {"error": "not found"}) - - def do_POST(self) -> None: # noqa: N802 - try: - length = int(self.headers.get("Content-Length", "0")) - request = parse_wire(self.rfile.read(length)) if 0 < length <= MAX_WIRE_BYTES else None - if not request or set(request) - {"operation", "assignment", "title", "body"}: - raise ProtocolError("invalid SCM request") - envelope = boundary.verify(request.get("assignment")) - operations = { - "checkout": lambda: boundary.checkout(envelope), - "submit": lambda: boundary.submit(envelope, request), - "gc-check": lambda: boundary.garbage_collect(envelope), - } - operation = str(request.get("operation") or "") - if operation not in operations: - raise ProtocolError("unsupported SCM operation") - self._reply(200, operations[operation]()) - except (ProtocolError, RuntimeError, OSError, urllib.error.URLError) as error: - self._reply(409, {"error": str(error)[:2000]}) - - def log_message(self, _format: str, *_arguments: Any) -> None: - return - - return Handler - - -def main() -> int: - if ORDINAL not in range(3): - raise SystemExit("HERMES_WORKER_ORDINAL must be 0, 1, or 2") - key = read_key(KEY_PATH) - _private_text(TOKEN_PATH) - server = BoundedHTTPServer( - ("0.0.0.0", PORT), handler_factory(Boundary(key)), max_workers=4 - ) - server.serve_forever() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) + def submit(self, envelope: dict[str, Any], request: dict[str, Any]) -> dict[str, Any]: + """Enforce clean/committed state, broker push, and reviewed draft creation.""" + _payload, repo, branch, base = _binding(envelope) + destination = workspace_path(envelope) + title = str(request.get("title") or f"Hermes task {envelope['task_id']}").strip() + body = str(request.get("body") or "Automated Hermes draft.") + if not title or len(title.encode()) > 512 or len(body.encode()) > 32 * 1024: + raise ProtocolError("pull-request metadata exceeds the safe limit") + with self.lock: + head = _workspace_identity(destination, repo, branch) + state = json.loads(_regular_text(_state_path(envelope), 16 * 1024)) + baseline = state.get("baseline_sha") if isinstance(state, dict) else "" + if not isinstance(baseline, str) or not re.fullmatch( + r"[0-9a-f]{40,64}", baseline + ): + raise ProtocolError("private SCM baseline is unavailable") + status = _run( + "status", "--porcelain=v1", "--untracked-files=all", cwd=destination + ) + if status: + raise ProtocolError("workspace has uncommitted or untracked files") + ahead = int(_run("rev-list", "--count", f"{baseline}..{head}", cwd=destination)) + if ahead <= 0: + return {"workspace": str(destination), "branch": branch, "pull_request": ""} + _run( + "push", "hermes-broker", f"HEAD:refs/heads/{branch}", + cwd=destination, timeout=900, + ) + pull = self._draft(repo, branch, base, head, title, body) + return {"workspace": str(destination), "branch": branch, "pull_request": pull} diff --git a/services/hermes/scripts/execution_pool_server.py b/services/hermes/scripts/execution_pool_server.py new file mode 100644 index 00000000..b2a01713 --- /dev/null +++ b/services/hermes/scripts/execution_pool_server.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Versioned HTTP and maintenance loop for the execution-pool coordinator.""" + +from __future__ import annotations + +import argparse +import os +import sqlite3 +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler +from pathlib import Path +from typing import Any + +from execution_pool_protocol import ( + MAX_WIRE_BYTES, + PROTOCOL_VERSION, + BoundedHTTPServer, + PoolStore, + ProtocolError, + canonical_json, + parse_wire, + read_key, +) + + +DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data")) +STATE_ROOT = DATA_ROOT / "execution-pool" +KEY_PATH = Path( + os.environ.get( + "HERMES_EXECUTION_POOL_KEY_FILE", "/runtime-access/execution-pool-key" + ) +) +PORT = int(os.environ.get("HERMES_EXECUTION_POOL_PORT", "9007")) +RETENTION_SECONDS = int( + os.environ.get("HERMES_EXECUTION_POOL_RETENTION_SECONDS", "1209600") +) + + +def handler_factory(coordinator: Any) -> type[BaseHTTPRequestHandler]: + class Handler(BaseHTTPRequestHandler): + server_version = f"hermes-execution-pool/{PROTOCOL_VERSION}" + + def _reply(self, status: int, value: dict[str, Any]) -> None: + body = canonical_json(value) + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 + if self.path != "/ready": + self._reply(404, {"error": "not found"}) + return + try: + coordinator.store.available_ordinals() + self._reply(200, {"ready": True, "version": PROTOCOL_VERSION}) + except (OSError, sqlite3.Error): + self._reply(503, {"ready": False, "version": PROTOCOL_VERSION}) + + def do_POST(self) -> None: # noqa: N802 + try: + length = int(self.headers.get("Content-Length", "0")) + if length <= 0 or length > MAX_WIRE_BYTES: + raise ProtocolError("invalid content length") + envelope = parse_wire(self.rfile.read(length)) + routes = { + "/v1/poll": coordinator.poll, + "/v1/heartbeat": coordinator.heartbeat, + "/v1/result": coordinator.result, + } + if self.path not in routes: + self._reply(404, {"error": "not found"}) + return + self._reply(200, routes[self.path](envelope)) + except ProtocolError as error: + self._reply(409, {"error": str(error)}) + except Exception as error: + print( + f"pool request failed: {type(error).__name__}: {error}", + file=sys.stderr, + flush=True, + ) + self._reply(503, {"error": "coordinator unavailable"}) + + def log_message(self, _format: str, *_arguments: Any) -> None: + return + + return Handler + + +def run(coordinator_type: type[Any]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--once", action="store_true") + args = parser.parse_args() + store = PoolStore(STATE_ROOT / "assignments.db") + coordinator = coordinator_type(read_key(KEY_PATH), store) + for operation in ( + coordinator.expire_leases, + coordinator.recover_results, + coordinator.reconcile, + coordinator.dispatch, + ): + operation() + if args.once: + return 0 + server = BoundedHTTPServer( + ("0.0.0.0", PORT), handler_factory(coordinator), max_workers=8 + ) + threading.Thread(target=server.serve_forever, daemon=True).start() + while True: + for operation in ( + coordinator.expire_leases, + coordinator.recover_results, + coordinator.reconcile, + coordinator.dispatch, + lambda: store.garbage_collect(RETENTION_SECONDS), + ): + try: + operation() + except Exception as error: + print( + f"pool maintenance deferred: {type(error).__name__}: {error}", + file=sys.stderr, + flush=True, + ) + time.sleep(5) diff --git a/services/hermes/scripts/execution_pool_worker.py b/services/hermes/scripts/execution_pool_worker.py index be3e4da5..c98dfd56 100644 --- a/services/hermes/scripts/execution_pool_worker.py +++ b/services/hermes/scripts/execution_pool_worker.py @@ -25,9 +25,11 @@ from execution_pool_protocol import ( ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace")) -CLIENT = os.environ.get("HERMES_EXECUTION_CLIENT_URL", "http://127.0.0.1:9009").rstrip("/") -SCM = os.environ.get("HERMES_SCM_BOUNDARY_URL", "http://127.0.0.1:9008").rstrip("/") ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1")) +CLIENT = os.environ.get( + "HERMES_EXECUTION_CLIENT_URL", + f"http://hermes-execution-mediator-{ORDINAL}.hermes.svc.cluster.local:9009", +).rstrip("/") NODE = os.environ.get("HERMES_WORKER_NODE", "unknown")[:128] RUN_PART = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") RETENTION_SECONDS = int(os.environ.get("HERMES_WORKER_RETENTION_SECONDS", "1209600")) @@ -59,19 +61,15 @@ def _poll() -> dict[str, Any] | None: assignment = _client("poll").get("assignment") if assignment is None: return None - if not isinstance(assignment, dict) or int(assignment["worker_ordinal"]) != ORDINAL: + if ( + not isinstance(assignment, dict) + or int(assignment["worker_ordinal"]) != ORDINAL + or int(assignment.get("protocol_version", 0)) != 2 + ): raise ProtocolError("local boundary returned a foreign assignment") return assignment -def _scm(operation: str, assignment: dict[str, Any], **values: Any) -> dict[str, Any]: - request = {"operation": operation, "assignment": assignment, **values} - response = _post(f"{SCM}/v1/scm", request, timeout=900) - if response.get("error"): - raise RuntimeError(str(response["error"])) - return response - - def _binding(assignment: dict[str, Any]) -> dict[str, Any]: return {name: assignment[name] for name in ( "board", "task_id", "run_id", "worker_ordinal", "attempt" @@ -82,7 +80,10 @@ def _state_path(assignment: dict[str, Any]) -> Path: parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id")) if any(not RUN_PART.fullmatch(part) for part in parts): raise ProtocolError("assignment state path is invalid") - root = (ROOT / "session-state").resolve() + configured_root = ROOT / "session-state" + if configured_root.is_symlink(): + raise ProtocolError("assignment state root must not be a symlink") + root = configured_root.resolve() path = root.joinpath(*parts).with_suffix(".json") path.parent.mkdir(parents=True, exist_ok=True) if path.is_symlink() or any(parent.is_symlink() for parent in path.parents if parent != root.parent): @@ -261,13 +262,12 @@ def execute(assignment: dict[str, Any]) -> None: if not isinstance(payload, dict): raise ProtocolError("assignment payload is invalid") _bind_provider_sessions(assignment) - checkout = _scm("checkout", assignment) - workspace = Path(str(checkout.get("workspace") or "")).resolve(strict=True) + workspace = Path(str(assignment.get("workspace") or "")).resolve(strict=True) workspace.relative_to((ROOT / "runs").resolve()) state_file = _state_path(assignment) state = cli_lane_runner.load_json(state_file) state.update({**binding, "node": NODE, "workspace": str(workspace)}) - state.setdefault("baseline_sha", str(checkout.get("baseline_sha") or "")) + state.setdefault("baseline_sha", str(assignment.get("baseline_sha") or "")) atomic_json(state_file, state) log_path = state_file.with_suffix(".log") offset = 0 @@ -339,21 +339,7 @@ def execute(assignment: dict[str, Any]) -> None: baseline = str(state.get("baseline_sha") or "") if not re.fullmatch(r"[0-9a-f]{40,64}", baseline): raise RuntimeError("durable SCM baseline is missing or invalid") - ahead = int(_git(workspace, "rev-list", "--count", f"{baseline}..HEAD") or "0") - if ahead: - fresh = _refresh_assignment(binding) - bundle = workspace / ".git/hermes-submit.bundle" - bundle.unlink(missing_ok=True) - _git(workspace, "bundle", "create", str(bundle), "HEAD") - submission = _scm( - "submit", fresh, - title=str(structured.get("summary") or f"Hermes task {binding['task_id']}")[:240], - body=json.dumps(structured, indent=2, sort_keys=True)[:12000], - ) - pull = str(submission.get("pull_request") or "") - if pull: - structured["artifacts"].append(pull) - bundle.unlink(missing_ok=True) + _refresh_assignment(binding) structured = _bounded_result(structured) activity, _ = _read_activity(log_path, offset) result_payload = { @@ -368,13 +354,53 @@ def execute(assignment: dict[str, Any]) -> None: }, "final_activity": activity, } - response = _client("result", binding=binding, payload=result_payload) + response = _client( + "finish", + binding=binding, + payload=result_payload, + title=str(structured.get("summary") or f"Hermes task {binding['task_id']}")[:240], + body=json.dumps(structured, indent=2, sort_keys=True)[:12000], + ) if not response.get("ack", {}).get("accepted"): raise ProtocolError("coordinator did not accept the terminal result") state["terminal_at"] = time.time() atomic_json(state_file, state) +def report_exception(assignment: dict[str, Any], error: Exception) -> bool: + """Surface one worker exception as an exact-run transient result.""" + binding = _binding(assignment) + detail = f"{type(error).__name__}: {error}"[:1500] + structured = { + "status": "blocked", + "summary": "Distributed worker failed before a safe handoff.", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": [detail], + } + payload = { + "structured": structured, + "returncode": 1, + # This is an execution-infrastructure failure, not an objective capability + # verdict. The exact-run result releases the ordinal for another task. + "capacity_failure": True, + "node": NODE, + "route": {}, + "provider_sessions": {}, + "final_activity": detail, + } + try: + return bool( + _client("finish", binding=binding, payload=payload).get("ack", {}).get( + "accepted" + ) + ) + except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError): + return False + + def readiness() -> None: if ORDINAL not in range(3): raise ProtocolError("worker ordinal must be 0, 1, or 2") @@ -397,6 +423,7 @@ def readiness() -> None: def main() -> int: readiness() while True: + assignment = None try: garbage_collect() assignment = _poll() @@ -405,9 +432,14 @@ def main() -> int: continue execute(assignment) except (OSError, RuntimeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as error: - print(f"worker {ORDINAL} deferred: {type(error).__name__}: {error}", flush=True) + surfaced = assignment is not None and report_exception(assignment, error) + print( + f"worker {ORDINAL} {'surfaced' if surfaced else 'deferred'}: " + f"{type(error).__name__}: {error}", + flush=True, + ) time.sleep(10) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint raise SystemExit(main()) diff --git a/services/hermes/scripts/stage_runtime_access.py b/services/hermes/scripts/stage_runtime_access.py index f9eb6940..2d998dd8 100644 --- a/services/hermes/scripts/stage_runtime_access.py +++ b/services/hermes/scripts/stage_runtime_access.py @@ -4,8 +4,11 @@ from __future__ import annotations import argparse +import hashlib +import hmac import json import os +import stat import uuid from pathlib import Path @@ -14,24 +17,42 @@ VAULT_ROOT = Path("/vault/secrets") RUNTIME_ROOT = Path("/runtime-access") PERSISTENT_HOME = Path("/opt/data/home") WORKER_ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace")) +PROVIDER_ACCESS_ROOT = Path( + os.environ.get("HERMES_PROVIDER_ACCESS_ROOT", "/provider-access") +) POOL_ACCESS_ROOT = Path(os.environ.get("HERMES_POOL_ACCESS_ROOT", "/pool-access")) -SCM_ACCESS_ROOT = Path(os.environ.get("HERMES_SCM_ACCESS_ROOT", "/scm-access")) OWNER_UID = 10000 OWNER_GID = 10000 def _owned_directory(path: Path) -> None: """Create one private directory owned by the unprivileged Hermes user.""" + if path.is_symlink(): + raise RuntimeError(f"private directory {path.name} must not be a symlink") path.mkdir(mode=0o700, parents=True, exist_ok=True) + if path.is_symlink() or not path.is_dir(): + raise RuntimeError(f"private directory {path.name} is invalid") path.chmod(0o700) os.chown(path, OWNER_UID, OWNER_GID) -def _copy_secret(source_name: str, destination: Path) -> str: - """Copy a non-empty Vault projection without logging its value.""" - value = (VAULT_ROOT / source_name).read_text(encoding="utf-8").strip() +def _read_secret(source_name: str) -> str: + path = VAULT_ROOT / source_name + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= 1024 * 1024: + raise RuntimeError(f"Vault projection {source_name} is invalid") + raw = os.read(descriptor, info.st_size + 1) + finally: + os.close(descriptor) + value = raw.decode("utf-8").strip() if not value: raise RuntimeError(f"Vault projection {source_name} is empty") + return value + + +def _write_secret(destination: Path, value: str) -> None: temporary = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.tmp") flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) descriptor = os.open(temporary, flags, 0o600) @@ -48,26 +69,65 @@ def _copy_secret(source_name: str, destination: Path) -> str: os.replace(temporary, destination) finally: temporary.unlink(missing_ok=True) + + +def _copy_secret(source_name: str, destination: Path) -> str: + """Copy a non-empty Vault projection without logging its value.""" + value = _read_secret(source_name) + _write_secret(destination, value) return value +def _validate_json_value(value: str, source_name: str, required: tuple[str, ...]) -> None: + """Validate a provider credential shape without exposing its value.""" + try: + document = json.loads(value) + except json.JSONDecodeError as error: + raise RuntimeError(f"credential {source_name} is not valid JSON") from error + current = document + for key in required: + if not isinstance(current, dict) or key not in current: + raise RuntimeError(f"credential {source_name} has an invalid shape") + current = current[key] + if not isinstance(current, str) or not current: + raise RuntimeError(f"credential {source_name} has an empty credential") + + def _validated_json(source_name: str, destination: Path, required: tuple[str, ...]) -> None: """Stage one credential document after checking its expected shape.""" value = _copy_secret(source_name, destination) try: - document = json.loads(value) - except json.JSONDecodeError: + _validate_json_value(value, source_name, required) + except RuntimeError: destination.unlink(missing_ok=True) raise - current = document - for key in required: - if not isinstance(current, dict) or key not in current: - destination.unlink(missing_ok=True) - raise RuntimeError(f"Vault projection {source_name} has an invalid shape") - current = current[key] - if not isinstance(current, str) or not current: - destination.unlink(missing_ok=True) - raise RuntimeError(f"Vault projection {source_name} has an empty credential") + + +def _bootstrap_json( + source_name: str, destination: Path, required: tuple[str, ...] +) -> None: + """Initialize one ordinal credential once, then preserve provider refreshes.""" + if destination.is_symlink(): + raise RuntimeError(f"durable credential {destination.name} must not be a symlink") + try: + descriptor = os.open( + destination, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + ) + except FileNotFoundError: + _validated_json(source_name, destination, required) + return + try: + info = os.fstat(descriptor) + if ( + not stat.S_ISREG(info.st_mode) + or info.st_mode & 0o077 + or not 0 < info.st_size <= 1024 * 1024 + ): + raise RuntimeError(f"durable credential {destination.name} is unsafe") + value = os.read(descriptor, info.st_size + 1).decode("utf-8").strip() + finally: + os.close(descriptor) + _validate_json_value(value, destination.name, required) def _link_noncredential_state( @@ -190,36 +250,51 @@ def _durable_link(runtime: Path, durable: Path, name: str) -> None: def stage_execution_worker() -> None: - """Stage only subscription/channel auth and ordinal-private session state.""" + """Bootstrap and preserve ordinal-private subscription refresh ownership.""" provider_state = WORKER_ROOT / "provider-state" for path in ( - RUNTIME_ROOT, - RUNTIME_ROOT / "claude", - RUNTIME_ROOT / "codex", - POOL_ACCESS_ROOT, - SCM_ACCESS_ROOT, provider_state, + PROVIDER_ACCESS_ROOT, + PROVIDER_ACCESS_ROOT / "claude", + PROVIDER_ACCESS_ROOT / "codex", ): _owned_directory(path) - _copy_secret("execution-pool-key", POOL_ACCESS_ROOT / "execution-pool-key") - _copy_secret("gitea-token", SCM_ACCESS_ROOT / "gitea-token") - _copy_secret("gitea-username", SCM_ACCESS_ROOT / "gitea-username") - _validated_json( - "claude-credentials", - RUNTIME_ROOT / "claude" / ".credentials.json", + ordinal = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1")) + if ordinal not in range(3): + raise RuntimeError("execution worker ordinal is invalid") + _bootstrap_json( + f"claude-credentials-{ordinal}", + PROVIDER_ACCESS_ROOT / "claude" / ".credentials.json", ("claudeAiOauth", "refreshToken"), ) - _validated_json( - "codex-auth", - RUNTIME_ROOT / "codex" / "auth.json", + _bootstrap_json( + f"codex-auth-{ordinal}", + PROVIDER_ACCESS_ROOT / "codex" / "auth.json", ("tokens", "refresh_token"), ) - _write_empty_auth_store() + + +def stage_execution_mediator() -> None: + """Derive one ordinal HMAC authority in the isolated mediator Pod only.""" + _owned_directory(POOL_ACCESS_ROOT) + ordinal = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1")) + if ordinal not in range(3): + raise RuntimeError("execution mediator ordinal is invalid") + master = _read_secret("execution-pool-key").encode() + derived = hmac.new( + master, + f"hermes-execution-pool-v2:worker:{ordinal}".encode(), + hashlib.sha256, + ).hexdigest() + _write_secret(POOL_ACCESS_ROOT / "execution-pool-key", derived) def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("mode", choices=("agent", "chat", "triage", "execution-worker")) + parser.add_argument( + "mode", + choices=("agent", "chat", "triage", "execution-worker", "execution-mediator"), + ) args = parser.parse_args() if args.mode == "agent": stage_agent() @@ -227,8 +302,10 @@ def main() -> int: stage_chat() elif args.mode == "triage": stage_triage() - else: + elif args.mode == "execution-worker": stage_execution_worker() + else: + stage_execution_mediator() print("Runtime access staged from Vault.", flush=True) return 0 diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index 943e0c0d..d6baabc5 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -224,9 +224,7 @@ write_policy_and_role "outline" "outline" "outline-vault" \ write_policy_and_role "planka" "planka" "planka-vault" \ "planka/* shared/postmark-relay" "" write_policy_and_role "hermes-execution-worker" "hermes" "hermes-execution-worker" \ - "hermes/agent-tokens hermes/developer-gitea" "" -write_policy_and_role "hermes-execution-credential-sync" "hermes" "hermes-execution-worker" \ - "" "hermes/agent-tokens" + "hermes/agent-tokens" "" write_policy_and_role "bstein-dev-home" "bstein-dev-home" "bstein-dev-home,bstein-dev-home-vault-sync" \ "portal/* shared/chat-ai-keys-runtime shared/portal-e2e-client shared/postmark-relay mailu/mailu-initial-account-secret shared/harbor-pull" "" write_policy_and_role "gitea" "gitea" "gitea-vault" \ diff --git a/testing/quality_contract.json b/testing/quality_contract.json index e8793d84..a919251f 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -20,6 +20,13 @@ "ci/scripts/supply_chain_report.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", + "services/hermes/scripts/execution_pool_client.py", + "services/hermes/scripts/execution_pool_coordinator.py", + "services/hermes/scripts/execution_pool_project.py", + "services/hermes/scripts/execution_pool_protocol.py", + "services/hermes/scripts/execution_pool_scm.py", + "services/hermes/scripts/execution_pool_server.py", + "services/hermes/scripts/execution_pool_worker.py", "testing/__init__.py", "testing/quality_contract.py", "testing/quality_docs.py", @@ -43,6 +50,13 @@ "services/comms/scripts/tests", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", + "services/hermes/scripts/execution_pool_client.py", + "services/hermes/scripts/execution_pool_coordinator.py", + "services/hermes/scripts/execution_pool_project.py", + "services/hermes/scripts/execution_pool_protocol.py", + "services/hermes/scripts/execution_pool_scm.py", + "services/hermes/scripts/execution_pool_server.py", + "services/hermes/scripts/execution_pool_worker.py", "testing/tests", "testing" ], @@ -58,6 +72,7 @@ "coverage_sources": [ "ci/scripts", "scripts.render.dashboards_render_atlas", + "services/hermes/scripts", "services/mailu/scripts", "testing" ], @@ -124,6 +139,7 @@ "ci/tests/**/*.py", "scripts/tests/**/*.py", "services/*/scripts/tests/**/*.py", + "services/hermes/scripts/execution_pool_*.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py" ], @@ -174,6 +190,13 @@ "ci/scripts/supply_chain_report.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", + "services/hermes/scripts/execution_pool_client.py", + "services/hermes/scripts/execution_pool_coordinator.py", + "services/hermes/scripts/execution_pool_project.py", + "services/hermes/scripts/execution_pool_protocol.py", + "services/hermes/scripts/execution_pool_scm.py", + "services/hermes/scripts/execution_pool_server.py", + "services/hermes/scripts/execution_pool_worker.py", "testing/quality_contract.py", "testing/quality_docs.py", "testing/quality_hygiene.py", diff --git a/testing/tests/test_hermes_agent_access.py b/testing/tests/test_hermes_agent_access.py index d4138256..a932c56c 100644 --- a/testing/tests/test_hermes_agent_access.py +++ b/testing/tests/test_hermes_agent_access.py @@ -78,15 +78,16 @@ def test_owner_agent_has_pinned_dedicated_node_ssh_access(): "secretProviderClass": "hermes-node-ssh-access" } reconciler = pod["containers"][0]["args"][0] - assert "cat /vault/secrets/node-ssh-public-key" in reconciler - assert "grep -qxF" in reconciler - assert "for user in atlas oceanus" in reconciler - assert "/host-etc/passwd" in reconciler - assert "chown \"${uid}:${gid}\"" in reconciler - host_passwd = next( - item for item in pod["volumes"] if item["name"] == "host-passwd" - ) - assert host_passwd["hostPath"] == {"path": "/etc/passwd", "type": "File"} + assert "/opt/node-hardener/node_account_hardening.py" in reconciler + assert "--public-key-file /vault/secrets/node-ssh-public-key" in reconciler + assert "grep -qxF" not in reconciler + host_etc = next(item for item in pod["volumes"] if item["name"] == "host-etc") + assert host_etc["hostPath"] == {"path": "/etc", "type": "Directory"} + hardener = next(item for item in pod["volumes"] if item["name"] == "coordinator") + assert hardener["configMap"] == { + "name": "hermes-node-account-hardener", + "defaultMode": 0o555, + } def test_owner_agent_tracks_no_ssh_identity_or_host_key_material(): @@ -155,13 +156,16 @@ def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog(): rbac = [ item - for item in yaml.safe_load_all((HERMES / "agent-rbac.yaml").read_text()) + for item in yaml.safe_load_all((HERMES / "rbac.yaml").read_text()) if item ] - binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding") - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} - ] + bindings = [item for item in rbac if item["kind"] == "ClusterRoleBinding"] + assert bindings + assert all( + subject["name"] not in {"hermes-agent", "hermes-switchyard"} + for binding in bindings + for subject in binding["subjects"] + ) def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): diff --git a/testing/tests/test_hermes_agent_security.py b/testing/tests/test_hermes_agent_security.py index cf23a2d0..f3410035 100644 --- a/testing/tests/test_hermes_agent_security.py +++ b/testing/tests/test_hermes_agent_security.py @@ -182,21 +182,54 @@ def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces() "ports": [{"protocol": "TCP", "port": 9010}], }, ] - assert isolation["spec"]["egress"] == [{}] - - -def test_owner_agent_has_cluster_admin_kubernetes_context(): - config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) - assert config["current-context"] == "atlas-owner" - assert config["contexts"][0]["context"]["namespace"] == "default" - rbac_path = HERMES / "agent-rbac.yaml" - documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] - binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding") - assert binding["roleRef"] == { - "apiGroup": "rbac.authorization.k8s.io", - "kind": "ClusterRole", - "name": "cluster-admin", - } - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} + egress = isolation["spec"]["egress"] + assert {} not in egress + broker = next( + rule + for rule in egress + if rule.get("ports") == [{"protocol": "TCP", "port": 9081}] + ) + assert broker["to"] == [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "hermes-scm"} + }, + "podSelector": {"matchLabels": {"app": "hermes-scm-broker"}}, + } ] + namespace_rule = next( + rule + for rule in egress + if rule.get("to", [{}])[0].get("namespaceSelector", {}).get( + "matchExpressions" + ) + ) + assert namespace_rule["to"][0]["namespaceSelector"]["matchExpressions"] == [ + { + "key": "kubernetes.io/metadata.name", + "operator": "NotIn", + "values": ["gitea", "hermes-scm"], + } + ] + public = next( + rule + for rule in egress + if rule.get("to", [{}])[0].get("ipBlock", {}).get("cidr") == "0.0.0.0/0" + ) + assert "192.168.0.0/16" in public["to"][0]["ipBlock"]["except"] + + +def test_owner_agent_has_observer_kubernetes_context_without_cluster_admin(): + config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) + assert config["current-context"] == "atlas-observer" + assert config["contexts"][0]["context"]["namespace"] == "default" + rbac_path = HERMES / "rbac.yaml" + documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] + bindings = [item for item in documents if item["kind"] == "ClusterRoleBinding"] + assert bindings + assert all(binding["roleRef"]["name"] != "cluster-admin" for binding in bindings) + assert all( + subject["name"] != "hermes-agent" + for binding in bindings + for subject in binding["subjects"] + ) diff --git a/testing/tests/test_hermes_chat_config.py b/testing/tests/test_hermes_chat_config.py index 7dd0e21c..c964e3e1 100644 --- a/testing/tests/test_hermes_chat_config.py +++ b/testing/tests/test_hermes_chat_config.py @@ -56,16 +56,17 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): assert "Atlas organization has private visibility" in instructions assert "may be public or private" in instructions assert "do not infer a\nrepository's visibility" in instructions - assert "already supplied through `GIT_ASKPASS`" in instructions + assert "Atlas Git access is supplied by the isolated SCM\nbroker" in soul assert "Never call `kanban_show` without a known, non-empty task ID" in instructions assert "bounded ad-hoc inspection and acceptance checks may" in instructions assert "load implementation or TDD skills" in instructions assert "Never call `kanban_show` without\na known, non-empty task ID" in soul assert "must load a skill only when its workflow\nmaterially applies" in soul - assert "runtime-only `GIT_ASKPASS`" in soul - assert "`scm.bstein.dev` is Gitea, not GitHub" in instructions - assert "Never load or follow a GitHub/`gh`" in instructions - assert "/opt/coordinator/gitea_api.py METHOD /api/v1/..." in instructions + assert "no repository token is present in this pod" in soul + assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions + assert "Never load or follow a\nGitHub/`gh`" in instructions + assert "use `/opt/scm/gitea_api.py`" in instructions + assert "never bypass the client with raw HTTP" in instructions assert "JENKINS_BASE_URL" in instructions assert "Do not\nuse `git reset --hard`" in instructions rendered = (HERMES / "agent-deployment.yaml").read_text() diff --git a/testing/tests/test_hermes_chat_provider_auth.py b/testing/tests/test_hermes_chat_provider_auth.py index 53bca9b6..abfb920c 100644 --- a/testing/tests/test_hermes_chat_provider_auth.py +++ b/testing/tests/test_hermes_chat_provider_auth.py @@ -3,7 +3,6 @@ from __future__ import annotations import base64 -import importlib.util import json import sys import time @@ -13,9 +12,6 @@ from types import ModuleType import pytest from testing.tests.test_hermes_chat_support import ( - HERMES, - ROOT, - _documents, _load_broker_module, ) @@ -455,181 +451,3 @@ def test_codex_broker_refreshes_and_persists_first_party_oauth( # refresh token or create a second billing/authentication path. assert module._access_token() == live assert len(calls) == 1 - - -def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch): - """Claude traffic must use the native first-party CLI subscription lane.""" - module = _load_broker_module( - "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch - ) - monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak") - monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak") - monkeypatch.setattr( - module, - "resolve_route", - lambda route: "claude-fable-5" if "/fable/" in route else route, - ) - - model, effort = module._route( - "route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}} - ) - - assert (model, effort) == ("claude-fable-5", "xhigh") - assert "ANTHROPIC_API_KEY" not in module._claude_environment() - assert "CLAUDE_API_KEY" not in module._claude_environment() - assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted") -def test_codex_native_health_overrides_historical_router_errors( - tmp_path: Path, monkeypatch -): - """Fresh first-party health is authoritative over old Switchyard probes.""" - plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" - spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - health_path = tmp_path / "codex.json" - health_path.write_text( - json.dumps( - { - "state": "available", - "authenticated": True, - "transport": "codex-chatgpt-subscription", - } - ) - ) - monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) - monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") - monkeypatch.setattr( - module, - "_get_json", - lambda url: {"status": "ok"} - if url.endswith("/health") - else { - "models": { - "route/codex/terra/medium": { - "calls": 1, - "errors": 99, - "total_tokens": 12, - } - } - }, - ) - monkeypatch.setattr(module, "_codex_account", lambda: {}) - monkeypatch.setattr(module, "_claude_account", lambda: {}) - - codex = module.provider_status_payload()["providers"]["codex"] - - assert codex["errors"] == 99 - assert codex["state"] == "available" - assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" - -def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): - """API-created workers must persist the originating Hermes session.""" - module_path = HERMES / "scripts" / "patch_api_server_sessions.py" - spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - source = tmp_path / "api_server.py" - destination = tmp_path / "patched.py" - source.write_text( - "prefix\n" - + module.BEFORE - + "middle\n" - + module.RUNS_BEFORE - + "run body\n" - + module.RUN_CLOSE_BEFORE - + module.RESPONSES_SESSION_BEFORE - + module.EVENT_CALLBACK_SIGNATURE_BEFORE - + "callback docstring and push helper\n" - + module.EVENT_CALLBACK_BODY_BEFORE - + "tool start body\n" - + module.EVENT_CALLBACK_END_BEFORE - + module.EVENT_CALLBACK_CALL_BEFORE - + module.RUN_SWEEP_BEFORE - + "suffix\n", - encoding="utf-8", - ) - - module.patch(source, destination) - patched = destination.read_text(encoding="utf-8") - - assert "X-Hermes-Parent-Session-Id" in patched - assert "parent_session_id=parent_session_id" in patched - assert "Parent session not found" in patched - assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched - assert "user_message.startswith(default_prefixes)" in patched - assert "session_parent_conflict" in patched - assert "X-Hermes-Conversation-Platform" in patched - assert "X-Hermes-Conversation-Title" in patched - assert 'conversation_platform != "telegram"' in patched - assert "db.record_gateway_session_peer(" in patched - assert 'display_name="Telegram"' in patched - assert "db.reopen_session(session_id)" in patched - assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched - assert "def _record_run_activity(" in patched - assert '"_thinking": "Hermes is reasoning"' in patched - assert '"run.started": "Worker started"' in patched - assert '"run.completed": "Worker completed"' in patched - assert '"reasoning.available": "Hermes finished a reasoning step"' in patched - assert '"subagent.progress": "Nested worker progress"' in patched - assert "redact_sensitive_text" in patched - assert 'getattr(os, "O_NOFOLLOW", 0)' in patched - assert "os.fchmod(fd, 0o600)" in patched - assert "session_id=session_id" in patched - assert 'self._record_run_activity(session_id, "run.started")' in patched - assert 'detail = tool_name if event_type in {' in patched - assert 'if event_type == "subagent.tool"' in patched - assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched - assert 'heartbeats.get(session_id, 0.0)' in patched - assert '"subagent.thinking",' in patched - assert "Stream retention and run lifetime are separate" in patched - assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched - assert patched.index("terminal_status = self._run_statuses") < patched.index( - "self._active_run_tasks.pop(run_id, None)" - ) - -def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): - """Thin brokers stay small while native Claude runs beside owner auth.""" - dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers").read_text() - assert "httpx==0.28.1" in dockerfile - assert "worker_route_broker.py" in dockerfile - assert "routing_catalog.py" in dockerfile - - deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] - containers = { - container["name"]: container - for container in deployment["spec"]["template"]["spec"]["containers"] - } - expected = ( - "registry.bstein.dev/bstein/hermes-switchyard-brokers@" - "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" - ) - assert containers["worker-route-broker"]["image"] == expected - assert containers["classifier-broker"]["image"] == expected - assert "claude-oauth-broker" not in containers - - agent = _documents(HERMES / "agent-deployment.yaml")[0] - agent_containers = { - container["name"]: container - for container in agent["spec"]["template"]["spec"]["containers"] - } - for container_name in ("hermes", "terminal"): - container = agent_containers[container_name] - environment = {item["name"]: item["value"] for item in container["env"]} - mounts = {item["name"]: item for item in container["volumeMounts"]} - assert environment["HERMES_ROUTING_CATALOG_PATH"] == "/routing-catalog/catalog.json" - assert environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" - assert environment["HERMES_CLAUDE_HEALTH_PATH"] == "/opt/data/provider-health/claude.json" - assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" - assert mounts["routing-catalog"]["readOnly"] is True - codex = agent_containers["codex-broker"] - codex_environment = {item["name"]: item["value"] for item in codex["env"]} - assert codex_environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" - claude = agent_containers["claude-broker"] - assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") - assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] - assert any( - mount["name"] == "home" and mount["mountPath"] == "/opt/data" - for mount in claude["volumeMounts"] - ) diff --git a/testing/tests/test_hermes_chat_provider_runtime.py b/testing/tests/test_hermes_chat_provider_runtime.py new file mode 100644 index 00000000..ebafe2ad --- /dev/null +++ b/testing/tests/test_hermes_chat_provider_runtime.py @@ -0,0 +1,196 @@ +"""Native provider health, API lineage, and runtime-image contracts.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, + _load_broker_module, +) + + +def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch): + """Claude traffic must use the native first-party CLI subscription lane.""" + module = _load_broker_module( + "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch + ) + monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak") + monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak") + monkeypatch.setattr( + module, + "resolve_route", + lambda route: "claude-fable-5" if "/fable/" in route else route, + ) + + model, effort = module._route( + "route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}} + ) + + assert (model, effort) == ("claude-fable-5", "xhigh") + assert "ANTHROPIC_API_KEY" not in module._claude_environment() + assert "CLAUDE_API_KEY" not in module._claude_environment() + assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted") + + +def test_codex_native_health_overrides_historical_router_errors( + tmp_path: Path, monkeypatch +): + """Fresh first-party health is authoritative over old Switchyard probes.""" + plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" + spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + health_path = tmp_path / "codex.json" + health_path.write_text( + json.dumps( + { + "state": "available", + "authenticated": True, + "transport": "codex-chatgpt-subscription", + } + ) + ) + monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) + monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") + monkeypatch.setattr( + module, + "_get_json", + lambda url: {"status": "ok"} + if url.endswith("/health") + else { + "models": { + "route/codex/terra/medium": { + "calls": 1, + "errors": 99, + "total_tokens": 12, + } + } + }, + ) + monkeypatch.setattr(module, "_codex_account", lambda: {}) + monkeypatch.setattr(module, "_claude_account", lambda: {}) + + codex = module.provider_status_payload()["providers"]["codex"] + + assert codex["errors"] == 99 + assert codex["state"] == "available" + assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" + + +def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): + """API-created workers must persist the originating Hermes session.""" + module_path = HERMES / "scripts" / "patch_api_server_sessions.py" + spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + source = tmp_path / "api_server.py" + destination = tmp_path / "patched.py" + source.write_text( + "prefix\n" + + module.BEFORE + + "middle\n" + + module.RUNS_BEFORE + + "run body\n" + + module.RUN_CLOSE_BEFORE + + module.RESPONSES_SESSION_BEFORE + + module.EVENT_CALLBACK_SIGNATURE_BEFORE + + "callback docstring and push helper\n" + + module.EVENT_CALLBACK_BODY_BEFORE + + "tool start body\n" + + module.EVENT_CALLBACK_END_BEFORE + + module.EVENT_CALLBACK_CALL_BEFORE + + module.RUN_SWEEP_BEFORE + + "suffix\n", + encoding="utf-8", + ) + + module.patch(source, destination) + patched = destination.read_text(encoding="utf-8") + + assert "X-Hermes-Parent-Session-Id" in patched + assert "parent_session_id=parent_session_id" in patched + assert "Parent session not found" in patched + assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched + assert "user_message.startswith(default_prefixes)" in patched + assert "session_parent_conflict" in patched + assert "X-Hermes-Conversation-Platform" in patched + assert "X-Hermes-Conversation-Title" in patched + assert 'conversation_platform != "telegram"' in patched + assert "db.record_gateway_session_peer(" in patched + assert 'display_name="Telegram"' in patched + assert "db.reopen_session(session_id)" in patched + assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched + assert "def _record_run_activity(" in patched + assert '"_thinking": "Hermes is reasoning"' in patched + assert '"run.started": "Worker started"' in patched + assert '"run.completed": "Worker completed"' in patched + assert '"reasoning.available": "Hermes finished a reasoning step"' in patched + assert '"subagent.progress": "Nested worker progress"' in patched + assert "redact_sensitive_text" in patched + assert 'getattr(os, "O_NOFOLLOW", 0)' in patched + assert "os.fchmod(fd, 0o600)" in patched + assert "session_id=session_id" in patched + assert 'self._record_run_activity(session_id, "run.started")' in patched + assert 'detail = tool_name if event_type in {' in patched + assert 'if event_type == "subagent.tool"' in patched + assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched + assert 'heartbeats.get(session_id, 0.0)' in patched + assert '"subagent.thinking",' in patched + assert "Stream retention and run lifetime are separate" in patched + assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched + assert patched.index("terminal_status = self._run_statuses") < patched.index( + "self._active_run_tasks.pop(run_id, None)" + ) + + +def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): + """Thin brokers stay small while native Claude runs beside owner auth.""" + dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers").read_text() + assert "httpx==0.28.1" in dockerfile + assert "worker_route_broker.py" in dockerfile + assert "routing_catalog.py" in dockerfile + + deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] + containers = { + container["name"]: container + for container in deployment["spec"]["template"]["spec"]["containers"] + } + expected = ( + "registry.bstein.dev/bstein/hermes-switchyard-brokers@" + "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" + ) + assert containers["worker-route-broker"]["image"] == expected + assert containers["classifier-broker"]["image"] == expected + assert "claude-oauth-broker" not in containers + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + agent_containers = { + container["name"]: container + for container in agent["spec"]["template"]["spec"]["containers"] + } + for container_name in ("hermes", "terminal"): + container = agent_containers[container_name] + environment = {item["name"]: item["value"] for item in container["env"]} + mounts = {item["name"]: item for item in container["volumeMounts"]} + assert environment["HERMES_ROUTING_CATALOG_PATH"] == "/routing-catalog/catalog.json" + assert environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" + assert environment["HERMES_CLAUDE_HEALTH_PATH"] == "/opt/data/provider-health/claude.json" + assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" + assert mounts["routing-catalog"]["readOnly"] is True + codex = agent_containers["codex-broker"] + codex_environment = {item["name"]: item["value"] for item in codex["env"]} + assert codex_environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" + claude = agent_containers["claude-broker"] + assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") + assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] + assert any( + mount["name"] == "home" and mount["mountPath"] == "/opt/data" + for mount in claude["volumeMounts"] + ) diff --git a/testing/tests/test_hermes_chat_voice.py b/testing/tests/test_hermes_chat_voice.py index 180a17e4..e04f3908 100644 --- a/testing/tests/test_hermes_chat_voice.py +++ b/testing/tests/test_hermes_chat_voice.py @@ -254,11 +254,14 @@ def test_chat_image_generation_uses_private_owner_broker(): vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() assert ( '"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram ' - 'hermes/developer-keycloak hermes/developer-gitea ' - 'hermes/developer-harbor hermes/developer-jenkins ' + 'hermes/developer-keycloak hermes/developer-harbor hermes/developer-jenkins ' 'hermes/developer-ssh"' in vault_policy ) + assert ( + 'write_policy_and_role "hermes-scm-broker" "hermes-scm" ' + '"hermes-scm-broker"' in vault_policy + ) assert ( 'write_policy_and_role "hermes-node-ssh" "hermes" ' '"hermes-node-ssh-access"' in vault_policy diff --git a/testing/tests/test_hermes_cli_lanes_configuration.py b/testing/tests/test_hermes_cli_lanes_configuration.py index bf1807f4..8489be89 100644 --- a/testing/tests/test_hermes_cli_lanes_configuration.py +++ b/testing/tests/test_hermes_cli_lanes_configuration.py @@ -20,6 +20,8 @@ from testing.tests.test_hermes_cli_lanes_support import ( policy, ) +import cli_lane_execution as lane_execution + def test_workspace_preparation_failure_durably_blocks_the_claim( tmp_path: Path, monkeypatch @@ -43,7 +45,7 @@ def test_workspace_preparation_failure_durably_blocks_the_claim( ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) monkeypatch.setattr( - lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json" + lane_execution, "state_path", lambda _board, _task_id: tmp_path / "state.json" ) lanes.execute_claim("cassandra", "t_bad_worktree") @@ -102,21 +104,21 @@ def test_restart_provider_change_includes_explicit_workspace_handoff( block_task=lambda *_args, **_kwargs: None, ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: state_file) + monkeypatch.setattr(lane_execution, "state_path", lambda _board, _task_id: state_file) monkeypatch.setattr( - lanes, + lane_execution, "select_route", lambda *_args, **_kwargs: lanes.Route( "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () ), ) monkeypatch.setattr( - lanes, + lane_execution, "git_handoff", lambda _workspace, output: f"HANDOFF:{output}", ) monkeypatch.setattr( - lanes, + lane_execution, "run_provider", lambda _route, prompt, *_args, **_kwargs: ( prompts.append(prompt) diff --git a/testing/tests/test_hermes_cli_lanes_kanban.py b/testing/tests/test_hermes_cli_lanes_kanban.py deleted file mode 100644 index 279b74c5..00000000 --- a/testing/tests/test_hermes_cli_lanes_kanban.py +++ /dev/null @@ -1,392 +0,0 @@ -"""Kanban claim and lifecycle contracts for Hermes CLI lanes.""" - -from __future__ import annotations - -import sys -from contextlib import nullcontext -from pathlib import Path -from types import SimpleNamespace - -import pytest - -from testing.tests.test_hermes_cli_lanes_support import ( - lanes, -) - - -def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch): - task = SimpleNamespace(id="t_auto", assignee=None, status="ready") - assigned = [] - - class Connection: - def close(self): - return None - - def assign_task(_conn, task_id, profile): - assigned.append((task_id, profile)) - task.assignee = profile - return True - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [{"slug": "cassandra"}], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - assign_task=assign_task, - get_task=lambda _conn, _task_id: task, - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")] - assert assigned == [("t_auto", "cli-auto")] - - -def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes( - monkeypatch, capsys -): - class CorruptBoardError(Exception): - pass - - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def close(self): - return None - - def connect(*, board): - if board == "cassandra": - raise CorruptBoardError("integrity_check failed") - return Connection() - - fake_db = SimpleNamespace( - KanbanDbCorruptError=CorruptBoardError, - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err - - -def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys): - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def __init__(self, board): - self.board = board - - def close(self): - return None - - def recompute_ready(connection): - if connection.board == "cassandra": - raise lanes.sqlite3.OperationalError("disk I/O error") - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(board), - recompute_ready=recompute_ready, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - error = capsys.readouterr().err - assert "temporarily skipping Kanban board 'cassandra'" in error - assert "storage OperationalError: disk I/O error" in error - - -def test_board_call_retries_storage_faults_on_fresh_connections(): - connections = [] - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - attempts = [] - - def operation(_connection): - attempts.append(1) - if len(attempts) < 3: - raise lanes.sqlite3.OperationalError("disk I/O error") - return "healthy" - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - ) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes._board_call(fake_db, "cassandra", operation) == "healthy" - assert len(connections) == 3 - assert all(connection.closed for connection in connections) - - -@pytest.mark.parametrize( - ("result", "expected_action"), - [ - (lanes.ProcessResult(0, "plain text only", None, False), "block"), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "done", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q"], - "artifacts": ["reports/result.json"], - "blockers": [], - }, - False, - ), - "complete", - ), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "The full test suite is still running.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q — in progress"], - "artifacts": [], - "blockers": [], - }, - False, - ), - "block", - ), - ], -) -def test_claim_requires_structured_evidence_and_surfaces_artifacts( - tmp_path: Path, - monkeypatch, - result, - expected_action, -): - task = SimpleNamespace( - id="t_worker", - current_run_id=4, - assignee="cli-auto", - max_runtime_seconds=60, - ) - calls = [] - heartbeats = [] - connections = [] - artifact = tmp_path / "reports/result.json" - artifact.parent.mkdir() - artifact.write_text("{}\n", encoding="utf-8") - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "bounded objective", - heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: ( - heartbeats.append((note, expected_run_id)) or True - ), - add_comment=lambda *_args: None, - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr( - lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json" - ) - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ), - ) - - def run_provider(*args, **_kwargs): - assert connections[0].closed - before_heartbeat = len(connections) - assert args[6]("working") is True - assert len(connections) == before_heartbeat + 1 - assert connections[-1].closed - return result - - monkeypatch.setattr(lanes, "run_provider", run_provider) - - lanes.execute_claim("cassandra", "t_worker") - - assert calls[0][0] == expected_action - assert heartbeats == [("working", 4)] - if expected_action == "complete": - assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)] - assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"] - else: - assert calls[0][1]["kind"] == "capability" - assert all(connection.closed for connection in connections) - - -def test_goal_card_continues_after_local_judge_rejects_progress( - tmp_path: Path, - monkeypatch, -): - task = SimpleNamespace( - id="t_goal", - current_run_id=12, - assignee="cli-auto", - max_runtime_seconds=300, - goal_mode=True, - goal_max_turns=3, - ) - calls = [] - comments = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.", - heartbeat_worker=lambda *_args, **_kwargs: True, - add_comment=lambda _conn, _task_id, _author, body: comments.append(body), - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr( - lanes, - "state_path", - lambda _board, _task_id: tmp_path / "state.json", - ) - claude_low = lanes.Route( - "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () - ) - codex_low = lanes.Route( - "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, () - ) - codex_xhigh = lanes.Route( - "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, () - ) - route_calls = [] - - def select_route(_prompt, assignee, **kwargs): - route_calls.append((assignee, kwargs)) - if assignee == "cli-codex-low": - return codex_low - if len(route_calls) == 1: - return claude_low - return codex_xhigh - - monkeypatch.setattr(lanes, "select_route", select_route) - monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) - reports = [ - lanes.ProcessResult(1, "authentication expired", None, True), - lanes.ProcessResult( - 0, - "first turn", - { - "status": "completed", - "summary": "Focused tests passed.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest focused: passed"], - "artifacts": [], - "blockers": [], - }, - False, - ), - lanes.ProcessResult( - 0, - "second turn", - { - "status": "completed", - "summary": "Full tests passed; commit pushed and remote HEAD verified.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest full: passed"], - "artifacts": [], - "blockers": [], - }, - False, - ), - ] - monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) - verdicts = iter( - [ - (False, "commit, push, and remote verification are missing"), - (True, "all explicit acceptance criteria have evidence"), - ] - ) - judge_contexts = [] - - def judge_goal_completion(objective, *_args, **_kwargs): - judge_contexts.append(objective) - return next(verdicts) - - monkeypatch.setattr( - lanes.cli_lane_goal, - "judge_goal_completion", - judge_goal_completion, - ) - - lanes.execute_claim("cassandra", "t_goal") - - assert calls[0][0] == "complete" - assert calls[0][1]["metadata"]["goal_turn"] == 2 - assert any( - "Goal completion rejected; continuing turn 2/3" in item for item in comments - ) - assert any( - "Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments - ) - assert route_calls[2][1]["exclude_provider"] == "claude" - assert "prior rejected reports" in judge_contexts[1] - assert "commit, push, and remote verification are missing" in judge_contexts[1] - assert reports == [] diff --git a/testing/tests/test_hermes_coordinator_boards.py b/testing/tests/test_hermes_coordinator_boards.py index aa63943f..c18bb7fb 100644 --- a/testing/tests/test_hermes_coordinator_boards.py +++ b/testing/tests/test_hermes_coordinator_boards.py @@ -141,7 +141,7 @@ def test_cassandra_sync_repairs_existing_worktree( assert all("GITEA_TOKEN" not in environment for environment in environments) def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch): - """Missing credentials skip only the fetch, not the local origin repair.""" + """The credential-isolated broker needs no model-facing token to fetch.""" workspace = tmp_path / "cassandra" (workspace / ".git").mkdir(parents=True) monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) @@ -156,8 +156,11 @@ def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch monkeypatch.setattr(coordinator.subprocess, "run", run) state = coordinator.sync_cassandra_repo({}) - assert state == "ready; fetch skipped until Gitea token is configured" - assert [command[-3] for command in commands] == ["remote", "set-url"] + assert state == "ready" + assert len(commands) == 3 + assert commands[0][-3:] == ["remote", "get-url", "origin"] + assert commands[1][-4:-1] == ["remote", "set-url", "origin"] + assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): """A project switch moves queued work without relocating active evidence.""" diff --git a/testing/tests/test_hermes_execution_pool.py b/testing/tests/test_hermes_execution_pool.py index 0ee44945..8b9966d5 100644 --- a/testing/tests/test_hermes_execution_pool.py +++ b/testing/tests/test_hermes_execution_pool.py @@ -3,7 +3,6 @@ from __future__ import annotations import json -import os import subprocess import sys import threading @@ -17,7 +16,7 @@ import yaml ROOT = Path(__file__).parents[2] HERMES = ROOT / "services/hermes" SCRIPTS = HERMES / "scripts" -sys.path.insert(0, str(SCRIPTS)) +sys.path[:0] = [str(SCRIPTS), str(HERMES / "scm-common/scripts")] import execution_pool_coordinator as coordinator # noqa: E402 import execution_pool_client as pool_client # noqa: E402 @@ -132,13 +131,13 @@ def test_key_requires_private_regular_file_and_rejects_symlink(tmp_path): def test_local_signing_boundary_rejects_foreign_or_unassigned_result(monkeypatch): monkeypatch.setattr(pool_client, "ORDINAL", 0) boundary = pool_client.ClientBoundary(KEY) - request = {"binding": binding(), "payload": {"returncode": 0}} + request = {"binding": binding(), "payload": {"note": "active"}} with pytest.raises(protocol.ProtocolError, match="does not own"): - boundary.forward("result", request) + boundary.heartbeat(request) boundary.current = binding() request["binding"] = binding(worker_ordinal=1) with pytest.raises(protocol.ProtocolError, match="does not own"): - boundary.forward("result", request) + boundary.heartbeat(request) def test_simultaneous_claim_materialization_has_one_winner(tmp_path): @@ -282,8 +281,8 @@ def test_scm_workspace_is_ordinal_contained_and_rejects_symlink(tmp_path, monkey def test_scm_boundary_rejects_other_repo_branch_and_ordinal(monkeypatch): monkeypatch.setattr(scm, "ORDINAL", 0) for changes, message in ( - ({"repo_url": "https://evil.example/atlas/titan-iac.git"}, "SCM policy"), - ({"branch": "main"}, "SCM policy"), + ({"repo_url": "https://evil.example/atlas/titan-iac.git"}, "outside Atlas"), + ({"branch": "main"}, "reviewed namespace"), ): signed = protocol.sign_envelope( KEY, "assignment", binding(), assignment_payload(**changes) @@ -297,68 +296,17 @@ def test_scm_boundary_rejects_other_repo_branch_and_ordinal(monkeypatch): scm._binding(signed) -def test_scm_identity_is_data_only_and_supports_packed_refs(tmp_path): - checkout = tmp_path / "checkout" - subprocess.run(["git", "init", "-q", str(checkout)], check=True) - subprocess.run( - ["git", "-C", str(checkout), "remote", "add", "origin", - "https://scm.bstein.dev/atlas/titan-iac.git"], check=True, - ) - subprocess.run( - ["git", "-C", str(checkout), "checkout", "-qb", "feature/hermes-safe-pool"], - check=True, - ) - subprocess.run(["git", "-C", str(checkout), "config", "user.email", "a@b.c"], check=True) - subprocess.run(["git", "-C", str(checkout), "config", "user.name", "Test"], check=True) - (checkout / "tracked").write_text("safe\n") - subprocess.run(["git", "-C", str(checkout), "add", "tracked"], check=True) - subprocess.run(["git", "-C", str(checkout), "commit", "-qm", "initial"], check=True) - expected = subprocess.check_output( - ["git", "-C", str(checkout), "rev-parse", "HEAD"], text=True - ).strip() - subprocess.run(["git", "-C", str(checkout), "pack-refs", "--all"], check=True) - assert scm._workspace_identity( - checkout, "https://scm.bstein.dev/atlas/titan-iac.git", - "feature/hermes-safe-pool", - ) == expected - (checkout / ".git/refs/heads/feature").mkdir(parents=True) - (checkout / ".git/refs/heads/feature/hermes-safe-pool").symlink_to("/etc/passwd") - with pytest.raises(protocol.ProtocolError, match="symlink"): - scm._workspace_identity( - checkout, "https://scm.bstein.dev/atlas/titan-iac.git", - "feature/hermes-safe-pool", - ) - - -def test_scm_bundle_copy_is_nofollow_bounded_and_fetchable(tmp_path, monkeypatch): - repo = tmp_path / "repo" - private = tmp_path / "private.git" - subprocess.run(["git", "init", "-q", str(repo)], check=True) - subprocess.run(["git", "-C", str(repo), "config", "user.email", "a@b.c"], check=True) - subprocess.run(["git", "-C", str(repo), "config", "user.name", "Test"], check=True) - (repo / "tracked").write_text("safe\n") - subprocess.run(["git", "-C", str(repo), "add", "tracked"], check=True) - subprocess.run(["git", "-C", str(repo), "commit", "-qm", "initial"], check=True) - bundle = repo / ".git/hermes-submit.bundle" - subprocess.run(["git", "-C", str(repo), "bundle", "create", str(bundle), "HEAD"], check=True) - copied = tmp_path / "copied.bundle" - scm._copy_bundle(bundle, copied) - subprocess.run(["git", "init", "--bare", "-q", str(private)], check=True) - subprocess.run( - ["git", "--git-dir", str(private), "fetch", str(copied), - "HEAD:refs/pool/candidate"], check=True, - ) - link = tmp_path / "bundle-link" - link.symlink_to(bundle) - with pytest.raises(OSError): - scm._copy_bundle(link, tmp_path / "rejected") - fifo = tmp_path / "bundle-fifo" - os.mkfifo(fifo) - with pytest.raises(protocol.ProtocolError, match="not regular"): - scm._copy_bundle(fifo, tmp_path / "rejected-fifo") - monkeypatch.setattr(scm, "MAX_BUNDLE_BYTES", 1) - with pytest.raises(protocol.ProtocolError, match="oversized"): - scm._copy_bundle(bundle, tmp_path / "rejected-large") +def test_scm_git_process_has_no_credential_or_ambient_configuration(): + environment = scm._git_environment() + assert environment == { + "HOME": "/nonexistent", + "PATH": "/usr/bin:/bin", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + } + assert scm._broker_repo("titan-iac").startswith("http://hermes-scm-broker.") + source = (SCRIPTS / "execution_pool_scm.py").read_text() + assert "GITEA_TOKEN" not in source and "GIT_ASKPASS" not in source def test_activity_is_bounded_sanitized_and_nofollow(tmp_path): @@ -396,10 +344,12 @@ def test_three_node_statefulset_contract_and_cross_worker_isolation(): assert stateful["spec"]["replicas"] == 3 assert stateful["spec"]["podManagementPolicy"] == "Parallel" assert pod["automountServiceAccountToken"] is False - assert len(stateful["spec"]["volumeClaimTemplates"]) == 1 - claim = stateful["spec"]["volumeClaimTemplates"][0] - assert claim["metadata"]["name"] == "workspace" - assert claim["spec"]["accessModes"] == ["ReadWriteOnce"] + claims = { + item["metadata"]["name"]: item + for item in stateful["spec"]["volumeClaimTemplates"] + } + assert set(claims) == {"workspace", "provider-access"} + assert all(item["spec"]["accessModes"] == ["ReadWriteOnce"] for item in claims.values()) anti = pod["affinity"]["podAntiAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"] assert anti[0]["topologyKey"] == "kubernetes.io/hostname" spread = pod["topologySpreadConstraints"][0] @@ -409,17 +359,33 @@ def test_three_node_statefulset_contract_and_cross_worker_isolation(): def test_worker_placement_prefers_accelerators_and_preserves_exclusions(): stateful = _documents(HERMES / "execution-worker-statefulset.yaml")[0] + pod = stateful["spec"]["template"]["spec"] + assert pod["priorityClassName"] == "scavenger" affinity = stateful["spec"]["template"]["spec"]["affinity"]["nodeAffinity"] terms = affinity["requiredDuringSchedulingIgnoredDuringExecution"]["nodeSelectorTerms"] - accelerator = {item["key"]: item for item in terms[0]["matchExpressions"]} - assert accelerator["kubernetes.io/hostname"]["values"] == ["titan-20", "titan-21"] - assert accelerator["node-role.kubernetes.io/accelerator"]["operator"] == "Exists" - fallback = {item["key"]: item for item in terms[1]["matchExpressions"]} - assert fallback["hardware"]["values"] == ["rpi5"] - assert "node-role.kubernetes.io/worker" not in fallback - excluded = set(fallback["kubernetes.io/hostname"]["values"]) - assert {"titan-04", "titan-08", "titan-13", "titan-14", "titan-17", "titan-18", "titan-19", "titan-22", "titan-24"} <= excluded + required = {item["key"]: item for item in terms[0]["matchExpressions"]} + assert required["node-role.kubernetes.io/worker"]["values"] == ["true"] + excluded = set(required["kubernetes.io/hostname"]["values"]) + assert {"titan-04", "titan-13", "titan-14", "titan-17", "titan-18", "titan-19", "titan-22", "titan-24"} <= excluded assert affinity["preferredDuringSchedulingIgnoredDuringExecution"][0]["weight"] == 100 + assert affinity["preferredDuringSchedulingIgnoredDuringExecution"][1]["weight"] == 50 + worker_container = pod["containers"][0] + assert worker_container["resources"] == { + "requests": {"cpu": "5m", "memory": "128Mi", "ephemeral-storage": "1Gi"}, + "limits": {"cpu": "2", "memory": "4Gi", "ephemeral-storage": "8Gi"}, + } + mediators = [ + item + for item in _documents(HERMES / "execution-mediator.yaml") + if item["kind"] == "Deployment" + ] + assert all( + item["spec"]["template"]["spec"]["containers"][0]["resources"][ + "requests" + ] + == {"cpu": "2m", "memory": "64Mi"} + for item in mediators + ) def test_model_worker_has_no_scm_or_cluster_credential_mount(): @@ -432,17 +398,26 @@ def test_model_worker_has_no_scm_or_cluster_credential_mount(): mounts = {item["mountPath"] for item in worker["volumeMounts"]} assert "/vault/secrets" not in mounts assert "/pool-access" not in mounts + assert "/provider-access" in mounts assert not any(item["name"] == "vault-auth-token" for item in worker["volumeMounts"]) assert pod["metadata"]["annotations"]["vault.hashicorp.com/agent-inject-containers"] == "stage-worker-access" - assert any(item["name"] == "vault-auth-token" for item in containers["credential-sync"]["volumeMounts"]) - for boundary_name in ("execution-client", "scm-boundary"): - assert any( - item["mountPath"] == "/pool-access" - for item in containers[boundary_name]["volumeMounts"] - ) - scm_mounts = {item["mountPath"] for item in containers["scm-boundary"]["volumeMounts"]} - assert "/scm-access" in scm_mounts - assert "/vault/secrets" not in scm_mounts + assert set(containers) == {"execution-worker"} + assert "execution-pool-key" not in str(pod["metadata"]["annotations"]) + mediators = [ + item for item in _documents(HERMES / "execution-mediator.yaml") + if item["kind"] == "Deployment" + ] + assert len(mediators) == 3 + for mediator in mediators: + spec = mediator["spec"]["template"]["spec"] + assert spec["automountServiceAccountToken"] is False + privileged = spec["containers"][0] + assert {mount["mountPath"] for mount in privileged["volumeMounts"]} >= { + "/pool-access", "/scm-state", "/opt/scm", "/workspace" + } + assert "/provider-access" not in { + mount["mountPath"] for mount in privileged["volumeMounts"] + } def test_worker_service_account_has_no_kubernetes_permissions(): @@ -471,12 +446,13 @@ def test_coordinator_remains_single_state_owner_and_workers_do_not_mount_home(): def test_worker_protocol_preserves_switchyard_fallback_and_visible_evidence(): source = (SCRIPTS / "execution_pool_worker.py").read_text() coordinator_source = (SCRIPTS / "execution_pool_coordinator.py").read_text() + server_source = (SCRIPTS / "execution_pool_server.py").read_text() assert "cli_lane_runner.select_route" in source assert 'alternate = "claude" if route.provider == "codex" else "codex"' in source assert "codex_thread_id" in source and "claude_session_id" in source assert "final_activity" in source assert "worker_ordinal" in coordinator_source and "provider_sessions" in coordinator_source - assert "coordinator.reconcile()" in coordinator_source + assert "coordinator.reconcile" in server_source def test_retention_gc_removes_only_clean_terminal_workspace(tmp_path, monkeypatch): diff --git a/testing/tests/test_hermes_execution_pool_assignment.py b/testing/tests/test_hermes_execution_pool_assignment.py index cf1d1a3e..d2fd91bd 100644 --- a/testing/tests/test_hermes_execution_pool_assignment.py +++ b/testing/tests/test_hermes_execution_pool_assignment.py @@ -13,9 +13,10 @@ import yaml SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" HERMES = SCRIPTS.parent -sys.path.insert(0, str(SCRIPTS)) +sys.path[:0] = [str(SCRIPTS), str(HERMES / "scm-common/scripts")] import execution_pool_coordinator as coordinator # noqa: E402 +import execution_pool_project as project # noqa: E402 import execution_pool_protocol as protocol # noqa: E402 import execution_pool_scm as scm # noqa: E402 import execution_pool_worker as worker # noqa: E402 @@ -26,34 +27,34 @@ KEY = b"k" * 32 def test_new_task_uses_explicit_reviewed_atlas_default(monkeypatch): monkeypatch.setattr( - coordinator, - "DEFAULT_REPO", - "https://scm.bstein.dev/atlas/titan-iac.git", + project, + "resolve_project", + lambda board: ( + f"https://scm.bstein.dev/atlas/{board}.git", "main", Path("/unused") + ), ) task = SimpleNamespace(id="t_deadbeef", workspace_path="", branch_name="") - assert coordinator.resolve_scm(task) == ( + assert project.resolve_assignment("titan-iac", task) == ( "https://scm.bstein.dev/atlas/titan-iac.git", - "feature/hermes-t_deadbeef", + "wt/t_deadbeef", "main", ) -def test_legacy_local_workspace_fails_closed_without_running_git(tmp_path): +def test_legacy_local_workspace_remains_owned_by_local_lane(tmp_path): task = SimpleNamespace( id="t_deadbeef", workspace_path=str(tmp_path), branch_name="feature/safe" ) - with pytest.raises(RuntimeError, match="preserve or commit"): - coordinator.resolve_scm(task) + assert project.distributed_workspace_eligible(task) is False def test_local_git_environment_excludes_credential_boundary_paths(): - local = scm._git_env(False) - authenticated = scm._git_env(True) + local = scm._git_environment() assert "HERMES_SCM_PASSWORD_FILE" not in local assert "GIT_ASKPASS" not in local - assert authenticated["HERMES_SCM_PASSWORD_FILE"] == str(scm.TOKEN_PATH) + assert "GITEA_TOKEN" not in local def test_finalized_duplicate_result_is_acknowledged_without_refinalizing(tmp_path): @@ -63,7 +64,8 @@ def test_finalized_duplicate_result_is_acknowledged_without_refinalizing(tmp_pat } store = protocol.PoolStore(tmp_path / "pool.db") store.add(binding, {"context": "safe"}) - result = protocol.sign_envelope(KEY, "result", binding, {"structured": {}}) + ordinal_key = protocol.derive_ordinal_key(KEY, 0) + result = protocol.sign_envelope(ordinal_key, "result", binding, {"structured": {}}) store.accept_result(result) store.finalize(binding, "finalized") pool = coordinator.Coordinator(KEY, store) @@ -117,9 +119,13 @@ def test_internal_http_server_rejects_work_above_its_bound(monkeypatch): def test_additive_patch_replaces_local_lane_without_touching_base_deployment(): patch = yaml.safe_load((HERMES / "execution-coordinator-patch.yaml").read_text()) containers = patch["spec"]["template"]["spec"]["containers"] - deleted = next(item for item in containers if item["name"] == "cli-lane-runner") + local = next(item for item in containers if item["name"] == "cli-lane-runner") pool = next(item for item in containers if item["name"] == "execution-pool-coordinator") - assert deleted["$patch"] == "delete" + environment = {item["name"]: item["value"] for item in local["env"]} + assert environment == { + "HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY": "true", + "HERMES_CLI_LANE_CONCURRENCY": "1", + } assert pool["resources"]["requests"] == {"cpu": "50m", "memory": "128Mi"} access = next(item for item in pool["volumeMounts"] if item["name"] == "runtime-access") assert access["subPath"] == "execution-pool-key" and access["readOnly"] is True @@ -132,27 +138,71 @@ def test_additive_network_policies_expose_only_worker_pool_and_switchyard_ports( policies = {item["metadata"]["name"]: item for item in documents} assert set(policies) == { "hermes-execution-worker-isolation", + "hermes-execution-mediator-isolation", "hermes-execution-pool-ingress", "hermes-execution-switchyard-ingress", + *(f"hermes-execution-worker-mediator-{ordinal}" for ordinal in range(3)), + *(f"hermes-execution-mediator-worker-{ordinal}" for ordinal in range(3)), } assert policies["hermes-execution-pool-ingress"]["spec"]["ingress"][0]["ports"] == [ {"protocol": "TCP", "port": 9007} ] + pool_source = policies["hermes-execution-pool-ingress"]["spec"]["ingress"] + assert "hermes-execution-mediator" in str(pool_source) + worker_egress = policies["hermes-execution-worker-isolation"]["spec"]["egress"] + assert "hermes-scm-broker" not in str(worker_egress) + assert "hermes-execution-mediator" not in str(worker_egress) + for ordinal in range(3): + worker_policy = policies[f"hermes-execution-worker-mediator-{ordinal}"] + mediator_policy = policies[f"hermes-execution-mediator-worker-{ordinal}"] + assert worker_policy["spec"]["podSelector"]["matchLabels"][ + "apps.kubernetes.io/pod-index" + ] == str(ordinal) + assert worker_policy["spec"]["egress"][0]["to"][0]["podSelector"][ + "matchLabels" + ]["pool-ordinal"] == str(ordinal) + assert mediator_policy["spec"]["ingress"][0]["from"][0]["podSelector"][ + "matchLabels" + ]["apps.kubernetes.io/pod-index"] == str(ordinal) @pytest.mark.parametrize( - "repo,base", + "branch", [ - ("https://evil.example/atlas/titan-iac.git", "main"), - ("https://token@scm.bstein.dev/atlas/titan-iac.git", "main"), - ("https://scm.bstein.dev/atlas/titan-iac.git", "../main"), + "../main", + "main", + "feature/../../main", ], ) -def test_assignment_rejects_unreviewed_repo_or_base(repo, base): +def test_assignment_rejects_unreviewed_task_branch(monkeypatch, branch): + monkeypatch.setattr( + project, + "resolve_project", + lambda _board: ( + "https://scm.bstein.dev/atlas/titan-iac.git", "main", Path("/unused") + ), + ) task = SimpleNamespace( - id="t_deadbeef", workspace_path="", branch_name="feature/safe", - repo_url=repo, base_branch=base, + id="t_deadbeef", workspace_path="", branch_name=branch, + repo_url="https://evil.example/atlas/other.git", base_branch="../main", ) - with pytest.raises(RuntimeError, match="outside"): - coordinator.resolve_scm(task) + with pytest.raises(project.ProjectPolicyError): + project.resolve_assignment("titan-iac", task) + + +def test_registry_authority_ignores_unreviewed_task_repo_metadata(monkeypatch): + monkeypatch.setattr( + project, + "resolve_project", + lambda _board: ( + "https://scm.bstein.dev/atlas/metis.git", "main", Path("/unused") + ), + ) + task = SimpleNamespace( + id="t_deadbeef", branch_name="review/t_deadbeef", + repo_url="https://evil.example/atlas/other.git", base_branch="../main", + ) + assert project.resolve_assignment("metis", task)[:2] == ( + "https://scm.bstein.dev/atlas/metis.git", "review/t_deadbeef" + ) diff --git a/testing/tests/test_hermes_execution_pool_coordinator_v2.py b/testing/tests/test_hermes_execution_pool_coordinator_v2.py new file mode 100644 index 00000000..fa679b0d --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_coordinator_v2.py @@ -0,0 +1,293 @@ +"""Exact-run coordinator finalization, lease, and ownership contracts.""" + +from __future__ import annotations + +import json +import sys +import types +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + +import execution_pool_coordinator as coordinator # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 + + +MASTER = b"k" * 32 +STRUCTURED = { + "status": "completed", + "summary": "Completed safely.", + "changed_files": ["safe.py"], + "tests_run": ["pytest"], + "artifacts": [], + "findings": [], + "blockers": [], +} + + +def binding(**changes): + value = { + "board": "metis", "task_id": "t_deadbeef", "run_id": "23", + "worker_ordinal": 0, "attempt": 1, + } + value.update(changes) + return value + + +def assignment_payload(**changes): + value = { + "context": "objective", + "assignee": "cli-auto", + "repo_url": "https://scm.bstein.dev/atlas/metis.git", + "branch": "wt/t_deadbeef", + "base_branch": "main", + "max_runtime_seconds": 3600, + "deadline_unix": 10_000_000_000, + } + value.update(changes) + return value + + +def signed(kind, exact_binding, payload): + key = protocol.derive_ordinal_key(MASTER, exact_binding["worker_ordinal"]) + return protocol.sign_envelope(key, kind, exact_binding, payload) + + +class Scope: + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + +class Connection: + def close(self): + return None + + +def install_kanban(monkeypatch, tasks, tmp_path, boards=None): + module = types.ModuleType("kanban_db") + module.tasks = {task.id: task for task in tasks} + module.completed = [] + module.blocked = [] + module.heartbeats = [] + module.reclaimed = [] + module.branches = [] + module.scoped_current_board = lambda _board: Scope() + module.connect = lambda board=None: Connection() + module.get_task = lambda _connection, task_id: module.tasks.get(task_id) + module.list_tasks = lambda _connection: list(module.tasks.values()) + module.list_boards = lambda include_archived=False: boards or ["metis"] + module.build_worker_context = lambda _connection, task_id: f"objective {task_id}" + module.worker_log_path = lambda task_id, board=None: str( + tmp_path / board / f"{task_id}.log" + ) + + def complete(_connection, task_id, **values): + module.completed.append((task_id, values)) + module.tasks[task_id].status = "completed" + return True + + def block(_connection, task_id, **values): + module.blocked.append((task_id, values)) + module.tasks[task_id].status = "blocked" + return True + + def heartbeat(_connection, task_id, **values): + module.heartbeats.append((task_id, values)) + return True + + module.complete_task = complete + module.block_task = block + module.heartbeat_worker = heartbeat + module.reclaim_task = lambda _c, task_id, **values: module.reclaimed.append( + (task_id, values) + ) + package = types.ModuleType("hermes_cli") + package.kanban_db = module + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.kanban_db", module) + return module + + +def task(**changes): + value = { + "id": "t_deadbeef", + "status": "running", + "current_run_id": 23, + "assignee": "cli-auto", + "workspace_path": "", + "branch_name": "wt/t_deadbeef", + "max_runtime_seconds": 3600, + "goal_mode": False, + "goal_max_turns": 1, + } + value.update(changes) + return SimpleNamespace(**value) + + +def store_with_assignment(tmp_path, exact_binding=None): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(exact_binding or binding(), assignment_payload()) + return store + + +def test_assignment_payload_is_bounded_registry_derived_and_typed(monkeypatch): + monkeypatch.setattr( + coordinator, + "resolve_assignment", + lambda board, _task: ( + f"https://scm.bstein.dev/atlas/{board}.git", "wt/t_deadbeef", "main" + ), + ) + kanban = SimpleNamespace( + build_worker_context=lambda _connection, _task_id: {"safe": True} + ) + item = task(max_runtime_seconds=1, goal_max_turns=99) + value = coordinator.assignment_payload(kanban, object(), item, "metis") + assert json.loads(value["context"]) == {"safe": True} + assert value["max_runtime_seconds"] == 60 + assert value["goal_max_turns"] == 12 + assert value["repo_url"].endswith("/metis.git") + kanban.build_worker_context = lambda *_a: "x" * (32 * 1024 + 1) + with pytest.raises(RuntimeError, match="32KiB"): + coordinator.assignment_payload(kanban, object(), item, "metis") + + +def test_poll_uses_per_ordinal_key_for_empty_and_active_assignment(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + poll_binding = binding(board="", task_id="", run_id="", attempt=0) + request = signed("poll", poll_binding, {"ready": True}) + empty = pool.poll(request) + assert protocol.verify_envelope( + protocol.derive_ordinal_key(MASTER, 0), empty, expected_kind="ack" + )["payload"] == {"assignment": None} + store.add(binding(), assignment_payload()) + offered = protocol.verify_envelope( + protocol.derive_ordinal_key(MASTER, 0), pool.poll(request), + expected_kind="assignment", + ) + assert offered["run_id"] == "23" + wrong_key = protocol.derive_ordinal_key(MASTER, 1) + with pytest.raises(protocol.ProtocolError, match="authentication"): + pool.poll(protocol.sign_envelope(wrong_key, "poll", poll_binding, {})) + + +def test_finalize_canonicalizes_integer_run_and_reconcile_does_not_reexecute( + tmp_path, monkeypatch +): + live = task() + kanban = install_kanban(monkeypatch, [live], tmp_path) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + envelope = signed( + "result", + binding(), + { + "structured": dict(STRUCTURED), + "returncode": 0, + "capacity_failure": False, + "node": "titan-05", + "route": {}, + "provider_sessions": {}, + "final_activity": "done\n", + }, + ) + ack = pool.result(envelope) + assert protocol.verify_envelope( + protocol.derive_ordinal_key(MASTER, 0), ack, expected_kind="ack" + )["payload"] == {"accepted": True, "duplicate": False} + assert kanban.completed[0][1]["expected_run_id"] == 23 + row = store._connect().execute("SELECT state FROM assignments").fetchone() + assert row[0] == "finalized" and store.available_ordinals() == [0, 1, 2] + pool.reconcile() + assert store.active_assignments() == [] + + +def test_finalize_fences_stale_run_and_blocks_failed_result_exactly(tmp_path, monkeypatch): + stale_task = task(current_run_id=24) + kanban = install_kanban(monkeypatch, [stale_task], tmp_path) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + pool.result(signed("result", binding(), {"structured": dict(STRUCTURED), "returncode": 0})) + assert not kanban.completed + assert store._connect().execute("SELECT state FROM assignments").fetchone()[0] == "stale" + + second = binding(run_id="25", worker_ordinal=1) + failed_task = task(current_run_id=25) + kanban.tasks[failed_task.id] = failed_task + store.add(second, assignment_payload()) + blocked = {**STRUCTURED, "status": "blocked", "blockers": ["capacity"]} + pool.result( + signed( + "result", second, + {"structured": blocked, "returncode": 1, "capacity_failure": True}, + ) + ) + assert kanban.blocked[-1][1]["expected_run_id"] == 25 + assert kanban.blocked[-1][1]["kind"] == "transient" + + +def test_finalize_rejects_invalid_payload_and_noncanonical_run(tmp_path, monkeypatch): + install_kanban(monkeypatch, [task()], tmp_path) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + with pytest.raises(protocol.ProtocolError, match="payload"): + pool.finalize({**binding(), "result": []}) + bad = binding(run_id="run-bad", worker_ordinal=1) + store.add(bad, assignment_payload()) + pool.finalize({**bad, "result": {"structured": {}}}) + states = dict(store._connect().execute("SELECT run_id,state FROM assignments")) + assert states["run-bad"] == "stale" + + +def test_heartbeat_uses_integer_exact_api_and_surfaces_activity(tmp_path, monkeypatch): + kanban = install_kanban(monkeypatch, [task()], tmp_path) + store = store_with_assignment(tmp_path) + store.offer(0) + pool = coordinator.Coordinator(MASTER, store) + request = signed( + "heartbeat", binding(), + {"note": "active", "activity": "safe output\n"}, + ) + ack = pool.heartbeat(request) + assert kanban.heartbeats[0][1]["expected_run_id"] == 23 + assert (tmp_path / "metis/t_deadbeef.log").read_text() == "safe output\n" + assert ack["kind"] == "ack" + kanban.heartbeat_worker = lambda *_a, **_k: False + with pytest.raises(protocol.ProtocolError, match="no longer owns"): + pool.heartbeat(signed("heartbeat", binding(), {})) + + +def test_recovery_defers_io_errors_and_lease_failure_releases_ordinal( + tmp_path, monkeypatch, capsys +): + kanban = install_kanban(monkeypatch, [task()], tmp_path) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + monkeypatch.setattr(store, "pending_results", lambda: [binding()]) + monkeypatch.setattr(pool, "finalize", lambda _record: (_ for _ in ()).throw(OSError("busy"))) + pool.recover_results() + assert "recovery deferred" in capsys.readouterr().err + + monkeypatch.undo() + kanban = install_kanban(monkeypatch, [task()], tmp_path) + store = protocol.PoolStore(tmp_path / "leases.db") + exact = binding(attempt=3) + store.add(exact, assignment_payload()) + store.offer(0) + with store._connect() as connection: + connection.execute("UPDATE assignments SET lease_until=1") + pool = coordinator.Coordinator(MASTER, store) + pool.expire_leases() + assert kanban.blocked[0][1]["expected_run_id"] == 23 + assert kanban.blocked[0][1]["kind"] == "transient" + assert store.available_ordinals() == [0, 1, 2] diff --git a/testing/tests/test_hermes_execution_pool_dispatch_v2.py b/testing/tests/test_hermes_execution_pool_dispatch_v2.py new file mode 100644 index 00000000..57dd0f74 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_dispatch_v2.py @@ -0,0 +1,410 @@ +"""Coordinator dispatch migration and versioned server contracts.""" + +from __future__ import annotations + +import sqlite3 +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +SCM_SCRIPTS = ROOT / "services/hermes/scm-common/scripts" +sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)] + +import execution_pool_coordinator as coordinator # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_server as server # noqa: E402 +from testing.tests.test_hermes_execution_pool_coordinator_v2 import ( # noqa: E402 + MASTER, + assignment_payload, + binding, + install_kanban, + store_with_assignment, + task, +) +from testing.tests.test_hermes_execution_pool_mediator import http_request # noqa: E402 + + +def test_reconcile_fences_old_run_then_recovers_current_pathless_run( + tmp_path, monkeypatch +): + item = task(current_run_id=24) + install_kanban(monkeypatch, [item], tmp_path) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + monkeypatch.setattr( + coordinator, "assignment_payload", + lambda _db, _connection, _task, _board: assignment_payload(), + ) + pool.reconcile() + rows = store._connect().execute( + "SELECT run_id,state,worker_ordinal FROM assignments ORDER BY run_id" + ).fetchall() + assert [tuple(row) for row in rows] == [ + ("23", "stale", 0), ("24", "assigned", 0) + ] + + +def test_reconcile_preserves_owned_workspace_and_exactly_blocks_registry_failure( + tmp_path, monkeypatch +): + owned = task(id="t_owned", current_run_id=30, workspace_path="/owned") + broken = task(id="t_broken", current_run_id=31) + kanban = install_kanban(monkeypatch, [owned, broken], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + + def prepare(_db, _connection, item, _board): + if item.id == "t_broken": + raise RuntimeError("registry unavailable") + return assignment_payload() + + monkeypatch.setattr(coordinator, "assignment_payload", prepare) + pool.reconcile() + assert store.active_assignments() == [] + assert kanban.blocked[0][0] == "t_broken" + assert kanban.blocked[0][1]["expected_run_id"] == 31 + assert all(call[0] != "t_owned" for call in kanban.blocked) + + +def test_dispatch_claims_only_pathless_task_with_safe_assignment_branch( + tmp_path, monkeypatch +): + item = task(status="running") + install_kanban(monkeypatch, [item], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + observed = [] + + def claim(active, limit, eligible): + observed.append((active, limit, eligible("metis", item))) + return [("metis", item.id)] + + monkeypatch.setattr(coordinator.cli_lane_dispatch, "claim_ready", claim) + monkeypatch.setattr( + coordinator, "assignment_payload", + lambda *_a: assignment_payload(branch="wt/t_deadbeef"), + ) + pool.dispatch() + assert observed == [(set(), 3, True)] + record = store.active_assignments()[0] + assert record["run_id"] == "23" + assert record["payload"]["branch"] == "wt/t_deadbeef" + + +def test_dispatch_incompatible_unversioned_claim_api_fails_closed( + tmp_path, monkeypatch +): + owned = task(workspace_path="/opt/data/workspace/live") + kanban = install_kanban(monkeypatch, [owned], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + calls = [] + + def old_claim(active, limit): + calls.append((active, limit)) + return [("metis", owned.id)] + + monkeypatch.setattr(coordinator.cli_lane_dispatch, "claim_ready", old_claim) + with pytest.raises(TypeError): + pool.dispatch() + assert calls == [] + assert store.active_assignments() == [] + assert kanban.reclaimed == [] + + +def test_dispatch_workspace_ownership_race_is_exactly_fenced( + tmp_path, monkeypatch +): + owned = task(workspace_path="/opt/data/workspace/live") + kanban = install_kanban(monkeypatch, [owned], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + monkeypatch.setattr( + coordinator.cli_lane_dispatch, + "claim_ready", + lambda *_a: [("metis", owned.id)], + ) + pool.dispatch() + assert store.active_assignments() == [] + assert kanban.blocked == [ + ( + owned.id, + { + "reason": ( + "Distributed claim fenced because an existing workspace " + "is owned by the local lane" + ), + "kind": "capability", + "expected_run_id": 23, + }, + ) + ] + + +def test_dispatch_preparation_failure_is_surfaced_and_full_pool_does_not_claim( + tmp_path, monkeypatch +): + broken = task(current_run_id=None) + kanban = install_kanban(monkeypatch, [broken], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + monkeypatch.setattr( + coordinator.cli_lane_dispatch, + "claim_ready", + lambda *_a: [("metis", broken.id)], + ) + pool.dispatch() + assert kanban.blocked[0][1]["expected_run_id"] is None + assert "canonical run ID" in kanban.blocked[0][1]["reason"] + + full = protocol.PoolStore(tmp_path / "full.db") + for ordinal in range(3): + full.add( + binding(task_id=f"t_{ordinal}", run_id=str(ordinal + 1), worker_ordinal=ordinal), + assignment_payload(), + ) + pool = coordinator.Coordinator(MASTER, full) + monkeypatch.setattr( + coordinator.cli_lane_dispatch, + "claim_ready", + lambda *_a: (_ for _ in ()).throw(AssertionError("must not claim")), + ) + pool.dispatch() + + +def test_lease_expiry_with_noncanonical_run_is_released_as_stale(tmp_path, monkeypatch): + install_kanban(monkeypatch, [task()], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + exact = binding(run_id="bad-run", attempt=3) + store.add(exact, assignment_payload()) + store.offer(0) + with store._connect() as connection: + connection.execute("UPDATE assignments SET lease_until=1") + coordinator.Coordinator(MASTER, store).expire_leases() + assert store._connect().execute("SELECT state FROM assignments").fetchone()[0] == "stale" + + +def test_terminal_activity_marker_prevents_duplicate_append(tmp_path): + class Kanban: + worker_log_path = staticmethod( + lambda task_id, board=None: str(tmp_path / f"{board}-{task_id}.log") + ) + + record = { + **binding(), + "result_digest": "a" * 64, + "result": {"final_activity": "terminal evidence\n"}, + } + coordinator._append_terminal_activity(Kanban, record) + coordinator._append_terminal_activity(Kanban, record) + text = (tmp_path / "metis-t_deadbeef.log").read_text() + assert text.count("execution-pool-result") == 1 + empty = {**record, "result": {"final_activity": ""}} + coordinator._append_terminal_activity(Kanban, empty) + + +def test_activity_and_terminal_log_reject_invalid_or_symlink_targets(tmp_path): + class Kanban: + worker_log_path = staticmethod(lambda *_a, **_k: str(tmp_path / "worker.log")) + + with pytest.raises(protocol.ProtocolError, match="payload"): + coordinator._append_activity(Kanban, {**binding(), "payload": []}) + coordinator._append_activity(Kanban, {**binding(), "payload": {"activity": ""}}) + (tmp_path / "worker.log").symlink_to(tmp_path / "target") + with pytest.raises(protocol.ProtocolError, match="symlink"): + coordinator._append_terminal_activity( + Kanban, + { + **binding(), "result_digest": "a" * 64, + "result": {"final_activity": "terminal"}, + }, + ) + + +def test_heartbeat_bad_run_duplicate_and_unstructured_result_paths( + tmp_path, monkeypatch +): + item = task() + kanban = install_kanban(monkeypatch, [item], tmp_path) + bad_binding = binding(run_id="bad-run") + bad_store = protocol.PoolStore(tmp_path / "bad.db") + bad_store.add(bad_binding, assignment_payload()) + bad_store.offer(0) + pool = coordinator.Coordinator(MASTER, bad_store) + key = protocol.derive_ordinal_key(MASTER, 0) + with pytest.raises(protocol.ProtocolError, match="canonical"): + pool.heartbeat(protocol.sign_envelope(key, "heartbeat", bad_binding, {})) + + store = store_with_assignment(tmp_path / "duplicate") + store.offer(0) + pool = coordinator.Coordinator(MASTER, store) + request = protocol.sign_envelope( + key, "heartbeat", binding(), {"activity": "once"}, + delivery_id="same-heartbeat", + ) + pool.heartbeat(request) + pool.heartbeat(request) + assert (tmp_path / "metis/t_deadbeef.log").read_text() == "once" + + second = binding(run_id="24", worker_ordinal=1) + item.current_run_id = 24 + store.add(second, assignment_payload()) + pool.result( + protocol.sign_envelope( + protocol.derive_ordinal_key(MASTER, 1), "result", second, + {"structured": [], "returncode": 1}, + ) + ) + assert kanban.blocked[-1][1]["expected_run_id"] == 24 + + +def test_reconcile_and_dispatch_cover_empty_error_and_missing_task_paths( + tmp_path, monkeypatch +): + item = task() + kanban = install_kanban(monkeypatch, [item], tmp_path, boards=["", "metis"]) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + original_connect = kanban.connect + kanban.list_boards = lambda include_archived=False: [] + kanban.connect = lambda board=None: (_ for _ in ()).throw(OSError("busy")) + pool.reconcile() + kanban.connect = original_connect + kanban.list_boards = lambda include_archived=False: [""] + pool.reconcile() + + full = SimpleNamespace( + active_assignments=lambda: [], available_ordinals=lambda: [] + ) + coordinator.Coordinator(MASTER, full).reconcile() + + empty = protocol.PoolStore(tmp_path / "missing.db") + monkeypatch.setattr( + coordinator.cli_lane_dispatch, "claim_ready", lambda *_a: [("metis", "missing")] + ) + coordinator.Coordinator(MASTER, empty).dispatch() + + +def test_nonterminal_lease_expiry_is_reoffered_without_kanban_block(tmp_path, monkeypatch): + kanban = install_kanban(monkeypatch, [task()], tmp_path) + store = store_with_assignment(tmp_path) + store.offer(0) + with store._connect() as connection: + connection.execute("UPDATE assignments SET lease_until=1") + coordinator.Coordinator(MASTER, store).expire_leases() + assert kanban.blocked == [] + assert store.active_assignments()[0]["attempt"] == 2 + + +class HTTPStore: + def __init__(self, fail=False): + self.fail = fail + + def available_ordinals(self): + if self.fail: + raise sqlite3.Error("busy") + return [0, 1, 2] + + +class HTTPCoordinator: + def __init__(self, fail_ready=False): + self.store = HTTPStore(fail_ready) + + @staticmethod + def poll(value): + return {"route": "poll", "value": value} + + @staticmethod + def heartbeat(_value): + raise protocol.ProtocolError("stale") + + @staticmethod + def result(_value): + raise RuntimeError("storage") + + +def test_versioned_server_routes_readiness_and_errors(): + handler = server.handler_factory(HTTPCoordinator()) + assert http_request(handler, "/ready") == (200, {"ready": True, "version": 2}) + assert http_request(handler, "/missing")[0] == 404 + body = protocol.canonical_json({"safe": True}) + assert http_request(handler, "/v1/poll", body=body) == ( + 200, {"route": "poll", "value": {"safe": True}} + ) + assert http_request(handler, "/v1/heartbeat", body=body)[0] == 409 + assert http_request(handler, "/v1/result", body=body)[0] == 503 + assert http_request(handler, "/unknown", body=body)[0] == 404 + assert http_request(handler, "/v1/poll", body=b"")[0] == 409 + assert http_request(server.handler_factory(HTTPCoordinator(True)), "/ready")[0] == 503 + + +class RunCoordinator: + instances = [] + + def __init__(self, key, store): + self.key = key + self.store = store + self.calls = [] + self.dispatch_count = 0 + self.__class__.instances.append(self) + + def expire_leases(self): + self.calls.append("expire") + + def recover_results(self): + self.calls.append("recover") + + def reconcile(self): + self.calls.append("reconcile") + + def dispatch(self): + self.calls.append("dispatch") + self.dispatch_count += 1 + if self.dispatch_count > 1: + raise RuntimeError("deferred") + + +def test_server_once_and_maintenance_loop_are_versioned_and_resilient( + tmp_path, monkeypatch, capsys +): + monkeypatch.setattr(server, "STATE_ROOT", tmp_path) + monkeypatch.setattr(server, "read_key", lambda _path: MASTER) + monkeypatch.setattr(sys, "argv", ["pool", "--once"]) + assert server.run(RunCoordinator) == 0 + assert RunCoordinator.instances[-1].calls == [ + "expire", "recover", "reconcile", "dispatch" + ] + + started = [] + + class FakeServer: + def __init__(self, address, _handler, max_workers): + started.append((address, max_workers)) + + def serve_forever(self): + return + + monkeypatch.setattr(sys, "argv", ["pool"]) + monkeypatch.setattr(server, "BoundedHTTPServer", FakeServer) + monkeypatch.setattr( + server.time, + "sleep", + lambda _seconds: (_ for _ in ()).throw(StopIteration()), + ) + with pytest.raises(StopIteration): + server.run(RunCoordinator) + assert started == [(("0.0.0.0", server.PORT), 8)] + assert "maintenance deferred" in capsys.readouterr().err + + +def test_coordinator_compatibility_exports_delegate_to_server(monkeypatch): + marker = object() + monkeypatch.setattr(server, "handler_factory", lambda value: (marker, value)) + assert coordinator.handler_factory(marker) == (marker, marker) + monkeypatch.setattr(server, "run", lambda value: 17 if value is coordinator.Coordinator else 0) + assert coordinator.main() == 17 diff --git a/testing/tests/test_hermes_execution_pool_mediator.py b/testing/tests/test_hermes_execution_pool_mediator.py new file mode 100644 index 00000000..cf66abf0 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_mediator.py @@ -0,0 +1,448 @@ +"""Isolated mediator, SCM gate, and model-facing API contracts.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import threading +import urllib.error +import urllib.request +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +SCM_SCRIPTS = ROOT / "services/hermes/scm-common/scripts" +sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)] + +import execution_pool_client as client # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_scm as scm # noqa: E402 + + +KEY = b"d" * 64 +RESULT = { + "status": "completed", + "summary": "Completed safely.", + "changed_files": ["safe.py"], + "tests_run": ["pytest"], + "artifacts": [], + "findings": [], + "blockers": [], +} + + +def binding(**changes): + value = { + "board": "metis", "task_id": "t_deadbeef", "run_id": "42", + "worker_ordinal": 0, "attempt": 1, + } + value.update(changes) + return value + + +def payload(**changes): + value = { + "context": "safe objective", + "repo_url": "https://scm.bstein.dev/atlas/metis.git", + "branch": "wt/t_deadbeef", + "base_branch": "main", + } + value.update(changes) + return value + + +def assignment(**payload_changes): + return protocol.sign_envelope( + KEY, "assignment", binding(), payload(**payload_changes) + ) + + +class Response: + def __init__(self, body): + self.body = body + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, _size=-1): + return self.body + + +class FakeSCM: + def __init__(self): + self.submitted = [] + + def checkout(self, _envelope): + return {"workspace": "/workspace/run", "baseline_sha": "a" * 40} + + def submit(self, envelope, request): + self.submitted.append((envelope, request)) + return {"pull_request": "https://scm.bstein.dev/atlas/metis/pulls/7"} + + +def test_result_schema_validation_rejects_every_unsafe_shape(): + invalid = [ + None, + {}, + {"structured": []}, + {"structured": {**RESULT, "extra": True}}, + {"structured": {**RESULT, "status": "unknown"}}, + {"structured": {**RESULT, "summary": ""}}, + {"structured": {**RESULT, "tests_run": "pytest"}}, + {"structured": {**RESULT, "tests_run": [7]}}, + ] + for value in invalid: + with pytest.raises(protocol.ProtocolError): + client._validate_result(value) + assert client._validate_result({"structured": dict(RESULT)})["structured"] == RESULT + + +def test_client_post_verifies_response_and_bounds_body(monkeypatch): + boundary = client.ClientBoundary(KEY, FakeSCM()) + ack = protocol.sign_envelope(KEY, "ack", binding(), {"accepted": True}) + monkeypatch.setattr(client.urllib.request, "urlopen", lambda *_a, **_k: Response(protocol.canonical_json(ack))) + assert boundary._post("/v1/result", ack)["kind"] == "ack" + monkeypatch.setattr( + client.urllib.request, + "urlopen", + lambda *_a, **_k: Response(b"x" * (protocol.MAX_WIRE_BYTES + 1)), + ) + with pytest.raises(protocol.ProtocolError, match="exceeds"): + boundary._post("/v1/result", ack) + + +def test_poll_materializes_only_ordinal_owned_assignment(monkeypatch): + monkeypatch.setattr(client, "ORDINAL", 0) + boundary = client.ClientBoundary(KEY, FakeSCM()) + boundary._post = lambda *_a: protocol.sign_envelope( + KEY, + "ack", + binding(board="", task_id="", run_id="", attempt=0), + {"assignment": None}, + ) + assert boundary.poll() == {"assignment": None} + + boundary._post = lambda *_a: assignment() + result = boundary.poll()["assignment"] + assert result["workspace"] == "/workspace/run" + assert result["protocol_version"] == 2 + assert "signature" not in result + + boundary._post = lambda *_a: protocol.sign_envelope( + KEY, "assignment", binding(worker_ordinal=1), payload() + ) + with pytest.raises(protocol.ProtocolError, match="foreign"): + boundary.poll() + + +def test_heartbeat_and_finish_require_exact_current_binding(monkeypatch): + fake_scm = FakeSCM() + boundary = client.ClientBoundary(KEY, fake_scm) + current = assignment() + boundary.current = current + + def post(path, envelope): + kind = "heartbeat" if path.endswith("heartbeat") else "result" + protocol.verify_envelope(KEY, envelope, expected_kind=kind) + return protocol.sign_envelope( + KEY, "ack", binding(), {"accepted": True, "duplicate": False} + ) + + boundary._post = post + assert boundary.heartbeat( + {"binding": binding(), "payload": {"note": "active"}} + )["ack"]["accepted"] + request = { + "binding": binding(), + "payload": {"structured": dict(RESULT), "returncode": 0}, + "title": "Safe change", + "body": "Evidence", + } + finished = boundary.finish(request) + assert finished["ack"]["accepted"] and boundary.current is None + assert fake_scm.submitted + assert finished["structured"]["artifacts"] == [ + "https://scm.bstein.dev/atlas/metis/pulls/7" + ] + + boundary.current = current + with pytest.raises(protocol.ProtocolError, match="heartbeat payload"): + boundary.heartbeat({"binding": binding(), "payload": "bad"}) + with pytest.raises(protocol.ProtocolError, match="object"): + boundary._current_for(None) + with pytest.raises(protocol.ProtocolError, match="does not own"): + boundary._current_for(binding(attempt=2)) + + +def test_noncompleted_finish_never_invokes_scm_and_bad_ack_is_rejected(): + fake_scm = FakeSCM() + boundary = client.ClientBoundary(KEY, fake_scm) + boundary.current = assignment() + bad = protocol.sign_envelope(KEY, "ack", binding(attempt=2), {"accepted": True}) + boundary._post = lambda *_a: bad + blocked = {**RESULT, "status": "blocked", "blockers": ["capacity"]} + with pytest.raises(protocol.ProtocolError, match="binding changed"): + boundary.finish( + {"binding": binding(), "payload": {"structured": blocked, "returncode": 1}} + ) + assert fake_scm.submitted == [] + + boundary._post = lambda *_a: protocol.sign_envelope( + KEY, "result", binding(), {"accepted": True} + ) + with pytest.raises(protocol.ProtocolError, match="binding changed"): + boundary.heartbeat({"binding": binding(), "payload": {}}) + + +def http_request(handler, path, *, body=None): + server = protocol.BoundedHTTPServer(("127.0.0.1", 0), handler, max_workers=2) + thread = threading.Thread(target=server.handle_request) + thread.start() + request = urllib.request.Request( + f"http://127.0.0.1:{server.server_port}{path}", + data=body, + method="POST" if body is not None else "GET", + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(request, timeout=3) as response: + return response.status, json.loads(response.read()) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read()) + finally: + thread.join(timeout=3) + server.server_close() + + +def test_model_api_exposes_only_gated_state_machine_operations(): + class Boundary: + poll = staticmethod(lambda: {"assignment": None}) + heartbeat = staticmethod(lambda request: {"heartbeat": request["payload"]}) + finish = staticmethod(lambda request: {"finish": request["payload"]}) + + handler = client.handler_factory(Boundary()) + assert http_request(handler, "/ready") == ( + 200, {"protocol_version": 2, "ready": True} + ) + assert http_request(handler, "/missing")[0] == 404 + for operation in ("poll", "heartbeat", "finish"): + body = protocol.canonical_json( + {"operation": operation, "binding": binding(), "payload": {}} + ) + assert http_request(handler, "/v1/client", body=body)[0] == 200 + for value in ( + {"operation": "bypass"}, + {"operation": "poll", "authority": "steal"}, + ): + status, response = http_request( + handler, "/v1/client", body=protocol.canonical_json(value) + ) + assert status == 409 and response.get("error") + + +def test_client_main_validates_ordinal_and_starts_bounded_server(monkeypatch): + monkeypatch.setattr(client, "ORDINAL", -1) + with pytest.raises(SystemExit, match="ORDINAL"): + client.main() + started = [] + + class Server: + def __init__(self, address, _handler, max_workers): + started.append((address, max_workers)) + + def serve_forever(self): + return + + monkeypatch.setattr(client, "ORDINAL", 2) + monkeypatch.setattr(client, "read_key", lambda _path: KEY) + monkeypatch.setattr(client, "BoundedHTTPServer", Server) + monkeypatch.setattr(client, "SCMBoundary", lambda _key: FakeSCM()) + assert client.main() == 0 + assert started == [(("0.0.0.0", client.PORT), 4)] + + +def init_checkout(path, branch="wt/t_deadbeef"): + subprocess.run(["git", "init", "-q", str(path)], check=True) + commands = [ + ("remote", "add", "origin", "https://scm.bstein.dev/atlas/metis.git"), + ( + "remote", + "add", + "hermes-broker", + "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/" + "git/atlas/metis.git", + ), + ("checkout", "-qb", branch), + ("config", "user.email", "test@example.com"), + ("config", "user.name", "Test"), + ] + for command in commands: + subprocess.run(["git", "-C", str(path), *command], check=True) + (path / "tracked").write_text("safe\n") + subprocess.run(["git", "-C", str(path), "add", "tracked"], check=True) + subprocess.run(["git", "-C", str(path), "commit", "-qm", "initial"], check=True) + return subprocess.check_output( + ["git", "-C", str(path), "rev-parse", "HEAD"], text=True + ).strip() + + +def test_scm_run_and_binding_validation(tmp_path, monkeypatch): + checkout = tmp_path / "checkout" + head = init_checkout(checkout) + assert scm._run("rev-parse", "HEAD", cwd=checkout) == head + with pytest.raises(RuntimeError): + scm._run("rev-parse", "missing", cwd=checkout) + monkeypatch.setattr(scm, "MAX_STATUS_BYTES", 1) + with pytest.raises(protocol.ProtocolError, match="output"): + scm._run("rev-parse", "HEAD", cwd=checkout) + + monkeypatch.setattr(scm, "ORDINAL", 0) + assert scm._binding(assignment())[1:] == ("metis", "wt/t_deadbeef", "main") + for envelope in ( + {**assignment(), "kind": "result"}, + {**assignment(), "payload": []}, + assignment(repo_url="https://evil.example/metis.git"), + assignment(branch="main"), + assignment(base_branch="../main"), + protocol.sign_envelope(KEY, "assignment", binding(worker_ordinal=1), payload()), + ): + with pytest.raises(protocol.ProtocolError): + scm._binding(envelope) + + +def test_scm_private_paths_and_text_are_bounded(tmp_path, monkeypatch): + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + target = scm.workspace_path(assignment()) + assert target.name == "42" + assert scm._state_path(assignment()).name == "metis-t_deadbeef-42.json" + bad = {**assignment(), "task_id": "../bad"} + with pytest.raises(protocol.ProtocolError, match="binding"): + scm.workspace_path(bad) + state = tmp_path / "regular" + state.write_text("safe") + assert scm._regular_text(state, 10) == "safe" + with pytest.raises(protocol.ProtocolError, match="invalid"): + scm._regular_text(state, 1) + binary = tmp_path / "binary" + binary.write_bytes(b"\xff") + with pytest.raises(protocol.ProtocolError, match="malformed"): + scm._regular_text(binary, 10) + linked_state = tmp_path / "linked-state" + linked_state.mkdir() + (tmp_path / "state-link").symlink_to(linked_state, target_is_directory=True) + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state-link") + with pytest.raises(protocol.ProtocolError): + scm._state_path(assignment()) + + +def test_boundary_verification_existing_checkout_and_missing_baseline(tmp_path, monkeypatch): + workspace = tmp_path / "workspace" + state_root = tmp_path / "state" + monkeypatch.setattr(scm, "WORKSPACE_ROOT", workspace) + monkeypatch.setattr(scm, "SCM_ROOT", state_root) + monkeypatch.setattr(scm, "ORDINAL", 0) + boundary = scm.Boundary(KEY) + assert boundary.verify(assignment())["kind"] == "assignment" + destination = scm.workspace_path(assignment()) + baseline = init_checkout(destination) + protocol.atomic_json( + scm._state_path(assignment()), + {"baseline_sha": baseline, "repo": "metis", "branch": "wt/t_deadbeef"}, + ) + assert boundary.checkout(assignment())["baseline_sha"] == baseline + scm._state_path(assignment()).write_text("{}") + with pytest.raises(protocol.ProtocolError, match="baseline"): + boundary.checkout(assignment()) + + +@pytest.mark.parametrize("feature_exists", [True, False]) +def test_new_checkout_uses_broker_and_safe_base_fallback( + tmp_path, monkeypatch, feature_exists +): + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + monkeypatch.setattr(scm, "ORDINAL", 0) + calls = [] + + def run(*arguments, cwd=None, timeout=300): + calls.append((arguments, cwd, timeout)) + if arguments[0] == "clone": + branch = arguments[arguments.index("--branch") + 1] + destination = Path(arguments[-1]) + if branch == "wt/t_deadbeef" and not feature_exists: + raise RuntimeError("missing branch") + (destination / ".git").mkdir(parents=True) + return "" + + monkeypatch.setattr(scm, "_run", run) + monkeypatch.setattr(scm, "_workspace_identity", lambda *_a: "a" * 40) + result = scm.Boundary(KEY).checkout(assignment()) + assert result["baseline_sha"] == "a" * 40 + assert any("hermes-scm-broker" in str(call) for call in calls) + if not feature_exists: + assert any(call[0][0] == "checkout" for call in calls) + + +def test_failed_clone_never_deletes_unmanaged_state(tmp_path, monkeypatch): + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + monkeypatch.setattr(scm, "ORDINAL", 0) + + def fail(*arguments, **_kwargs): + destination = Path(arguments[-1]) + destination.mkdir(parents=True, exist_ok=True) + (destination / "preserved").write_text("owner data") + raise RuntimeError("clone failed") + + monkeypatch.setattr(scm, "_run", fail) + with pytest.raises(protocol.ProtocolError, match="unmanaged"): + scm.Boundary(KEY).checkout(assignment()) + assert (scm.workspace_path(assignment()) / "preserved").read_text() == "owner data" + + +def test_draft_reuse_create_and_submit_gates(tmp_path, monkeypatch): + existing = json.dumps([{"html_url": "https://scm/pulls/1"}]).encode() + monkeypatch.setattr(scm.scm_broker_client, "read", lambda _path: existing) + assert scm.Boundary._draft("metis", "wt/task", "main", "a" * 40, "t", "b") == "https://scm/pulls/1" + monkeypatch.setattr(scm.scm_broker_client, "read", lambda _path: b"[]") + monkeypatch.setattr( + scm.scm_broker_client, "create_draft", + lambda *_a, **_k: json.dumps({"html_url": "https://scm/pulls/2"}).encode(), + ) + assert scm.Boundary._draft("metis", "wt/task", "main", "a" * 40, "t", "b") == "https://scm/pulls/2" + + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + monkeypatch.setattr(scm, "ORDINAL", 0) + destination = scm.workspace_path(assignment()) + destination.mkdir(parents=True) + state = scm._state_path(assignment()) + protocol.atomic_json(state, {"baseline_sha": "a" * 40}) + boundary = scm.Boundary(KEY) + monkeypatch.setattr(scm, "_workspace_identity", lambda *_a: "b" * 40) + outputs = {"status": "", "rev-list": "1", "push": ""} + + def run(*arguments, **_kwargs): + return outputs.get(arguments[0], "") + + monkeypatch.setattr(scm, "_run", run) + monkeypatch.setattr(boundary, "_draft", lambda *_a: "https://scm/pulls/3") + result = boundary.submit(assignment(), {"title": "safe", "body": "evidence"}) + assert result["pull_request"] == "https://scm/pulls/3" + outputs["status"] = "?? untracked" + with pytest.raises(protocol.ProtocolError, match="uncommitted"): + boundary.submit(assignment(), {}) + outputs["status"] = "" + outputs["rev-list"] = "0" + assert boundary.submit(assignment(), {})["pull_request"] == "" + with pytest.raises(protocol.ProtocolError, match="metadata"): + boundary.submit(assignment(), {"title": "x" * 513, "body": "x"}) + diff --git a/testing/tests/test_hermes_execution_pool_project.py b/testing/tests/test_hermes_execution_pool_project.py new file mode 100644 index 00000000..16c57427 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_project.py @@ -0,0 +1,211 @@ +"""Canonical board-registry and Git-ref assignment contracts.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + +import execution_pool_project as project # noqa: E402 + + +def git(*arguments: str, cwd: Path | None = None) -> str: + return subprocess.check_output( + ["git", *arguments], cwd=cwd, text=True, stderr=subprocess.DEVNULL + ).strip() + + +def registry(tmp_path, monkeypatch, board="metis", remote=None, base=None): + projects = tmp_path / "projects" + boards = tmp_path / "boards" + checkout = projects / board + checkout.mkdir(parents=True) + git("init", "-q", cwd=checkout) + git( + "remote", "add", "origin", + remote or f"https://scm.bstein.dev/atlas/{board}.git", + cwd=checkout, + ) + if base: + git( + "symbolic-ref", "refs/remotes/origin/HEAD", f"refs/remotes/origin/{base}", + cwd=checkout, + ) + entry = boards / board + entry.mkdir(parents=True) + (entry / "board.json").write_text( + json.dumps({"slug": board, "default_workdir": str(checkout)}) + ) + monkeypatch.setattr(project, "PROJECT_ROOT", projects) + monkeypatch.setattr(project, "BOARD_ROOT", boards) + return checkout, entry / "board.json" + + +@pytest.mark.parametrize( + "branch", + [ + "feature/safe", "fix/safe", "chore/safe", "docs/safe", "test/safe", + "refactor/safe", "wt/t_deadbeef", "review/t_deadbeef", "hermes/safe", + "handoff/safe", + ], +) +def test_every_established_feature_namespace_is_accepted(branch): + assert project.validate_branch(branch, feature=True) == branch + + +@pytest.mark.parametrize( + "branch", + ["main", "unknown/safe", "../main", "feature/../../main", "feature/.hidden", "-bad", "é/safe", "x" * 201], +) +def test_ref_validation_denies_traversal_and_unreviewed_namespaces(branch): + with pytest.raises(project.ProjectPolicyError): + project.validate_branch(branch, feature=True) + + +def test_canonical_registry_resolves_repo_base_and_default_branch(tmp_path, monkeypatch): + checkout, _ = registry(tmp_path, monkeypatch, base="trunk") + assert project.resolve_project("metis") == ( + "https://scm.bstein.dev/atlas/metis.git", "trunk", checkout.resolve() + ) + task = SimpleNamespace(id="t_deadbeef", branch_name="") + assert project.resolve_assignment("metis", task) == ( + "https://scm.bstein.dev/atlas/metis.git", "wt/t_deadbeef", "trunk" + ) + + +@pytest.mark.parametrize("board", ["cassandra", "metis", "soteria", "titan-iac"]) +def test_every_atlas_project_uses_its_own_registry_checkout(tmp_path, monkeypatch, board): + registry(tmp_path, monkeypatch, board=board) + repo, base, _ = project.resolve_project(board) + assert repo == f"https://scm.bstein.dev/atlas/{board}.git" + assert base == "main" + + +def test_registered_project_without_checkout_keeps_its_own_repo_identity( + tmp_path, monkeypatch +): + checkout, _ = registry(tmp_path, monkeypatch, board="metis") + shutil.rmtree(checkout) + assert project.resolve_project("metis") == ( + "https://scm.bstein.dev/atlas/metis.git", + "main", + checkout.resolve(), + ) + + +def test_registry_rejects_project_slug_too_long_for_an_atlas_repo( + tmp_path, monkeypatch +): + board = "m" * 101 + checkout, _ = registry(tmp_path, monkeypatch, board=board) + shutil.rmtree(checkout) + with pytest.raises(project.ProjectPolicyError, match="repository identity"): + project.resolve_project(board) + + +def test_noncanonical_remote_head_falls_back_to_main(tmp_path, monkeypatch): + checkout, _ = registry(tmp_path, monkeypatch) + real_run = project._run_git + + def run_git(workdir, *arguments): + if arguments[0] == "symbolic-ref": + return "heads/not-origin" + return real_run(workdir, *arguments) + + monkeypatch.setattr(project, "_run_git", run_git) + assert project.resolve_project("metis") == ( + "https://scm.bstein.dev/atlas/metis.git", + "main", + checkout.resolve(), + ) + + +def test_registry_rejects_slug_traversal_symlink_and_malformed_documents(tmp_path, monkeypatch): + _, board_file = registry(tmp_path, monkeypatch) + with pytest.raises(project.ProjectPolicyError, match="slug"): + project._read_board("../metis") + board_file.write_text("not-json") + with pytest.raises(project.ProjectPolicyError, match="malformed"): + project._read_board("metis") + board_file.write_text(json.dumps({"slug": "other", "default_workdir": "/tmp"})) + with pytest.raises(project.ProjectPolicyError, match="identity"): + project._read_board("metis") + board_file.unlink() + board_file.symlink_to("/etc/passwd") + with pytest.raises(OSError): + project._read_board("metis") + + +def test_registry_rejects_oversized_archived_and_non_regular_entry(tmp_path, monkeypatch): + _, board_file = registry(tmp_path, monkeypatch) + board_file.write_text("x" * (project.MAX_BOARD_BYTES + 1)) + with pytest.raises(project.ProjectPolicyError, match="bounded"): + project._read_board("metis") + board_file.write_text(json.dumps({"slug": "metis", "archived": True})) + with pytest.raises(project.ProjectPolicyError, match="archived"): + project._read_board("metis") + board_file.unlink() + board_file.mkdir() + with pytest.raises(project.ProjectPolicyError, match="regular"): + project._read_board("metis") + + +@pytest.mark.parametrize( + "remote", + [ + "https://token@scm.bstein.dev/atlas/metis.git", + "https://evil.example/atlas/metis.git", + "ssh://git@scm.bstein.dev/atlas/metis.git", + ], +) +def test_registry_rejects_credentialed_or_non_atlas_origin(tmp_path, monkeypatch, remote): + registry(tmp_path, monkeypatch, remote=remote) + with pytest.raises(project.ProjectPolicyError, match="origin"): + project.resolve_project("metis") + + +def test_registry_rejects_missing_or_outside_workdir(tmp_path, monkeypatch): + checkout, board_file = registry(tmp_path, monkeypatch) + board_file.write_text(json.dumps({"slug": "metis"})) + with pytest.raises(project.ProjectPolicyError, match="default_workdir"): + project.resolve_project("metis") + outside = tmp_path / "outside" + checkout.rename(outside) + board_file.write_text( + json.dumps({"slug": "metis", "default_workdir": str(outside)}) + ) + with pytest.raises(project.ProjectPolicyError, match="outside"): + project.resolve_project("metis") + + +def test_registry_rejects_non_directory_checkout(tmp_path, monkeypatch): + checkout, _ = registry(tmp_path, monkeypatch) + shutil.rmtree(checkout) + checkout.write_text("not a checkout") + with pytest.raises(project.ProjectPolicyError, match="not a directory"): + project.resolve_project("metis") + + +def test_git_metadata_failure_and_invalid_task_identity_fail_closed(tmp_path, monkeypatch): + checkout, _ = registry(tmp_path, monkeypatch) + with pytest.raises(project.ProjectPolicyError, match="metadata"): + project._run_git(checkout, "remote", "get-url", "missing") + with pytest.raises(project.ProjectPolicyError, match="identity"): + project.resolve_assignment("metis", SimpleNamespace(id="../../bad")) + + +def test_workspace_migration_boundary_is_explicit(): + assert project.distributed_workspace_eligible(SimpleNamespace(workspace_path="")) + assert project.distributed_workspace_eligible(SimpleNamespace()) + assert not project.distributed_workspace_eligible( + SimpleNamespace(workspace_path="/opt/data/workspace/live") + ) diff --git a/testing/tests/test_hermes_execution_pool_protocol_v2.py b/testing/tests/test_hermes_execution_pool_protocol_v2.py new file mode 100644 index 00000000..aea60727 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_protocol_v2.py @@ -0,0 +1,241 @@ +"""Version-2 protocol reliability and exact-attempt fencing contracts.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import sys +import threading +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + + +SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + +import execution_pool_protocol as protocol # noqa: E402 + + +MASTER = b"m" * 32 + + +def binding(**changes): + value = { + "board": "metis", + "task_id": "t_deadbeef", + "run_id": "42", + "worker_ordinal": 0, + "attempt": 1, + } + value.update(changes) + return value + + +def resign(envelope, key=MASTER): + unsigned = dict(envelope) + unsigned.pop("signature", None) + envelope["signature"] = hmac.new( + key, protocol.canonical_json(unsigned), hashlib.sha256 + ).hexdigest() + return envelope + + +def test_atomic_json_is_private_durable_and_rejects_symlink_parent(tmp_path): + target = tmp_path / "state/value.json" + protocol.atomic_json(target, {"safe": True}) + assert json.loads(target.read_text()) == {"safe": True} + assert target.stat().st_mode & 0o777 == 0o600 + + outside = tmp_path / "outside" + outside.mkdir() + (tmp_path / "linked").symlink_to(outside, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="directory"): + protocol.atomic_json(tmp_path / "linked/value.json", {"safe": False}) + + +def test_ordinal_derivation_and_selector_fail_closed(): + keys = [protocol.derive_ordinal_key(MASTER, ordinal) for ordinal in range(3)] + assert len(set(keys)) == 3 + for master, ordinal in ((b"short", 0), (MASTER, -1), (MASTER, 3)): + with pytest.raises(protocol.ProtocolError, match="derivation"): + protocol.derive_ordinal_key(master, ordinal) + for value in (None, [], {}, {"worker_ordinal": True}, {"worker_ordinal": 3}): + with pytest.raises(protocol.ProtocolError): + protocol.envelope_ordinal(value) + + +def test_key_reader_bounds_content_and_type(tmp_path): + for content in (b"x" * 31, b"x" * 4097): + path = tmp_path / f"key-{len(content)}" + path.write_bytes(content) + path.chmod(0o600) + with pytest.raises(protocol.ProtocolError, match="length"): + protocol.read_key(path) + directory = tmp_path / "directory" + directory.mkdir(mode=0o700) + with pytest.raises(protocol.ProtocolError, match="regular"): + protocol.read_key(directory) + + +def test_envelope_rejects_kind_fields_numeric_lifetime_and_digest(): + with pytest.raises(protocol.ProtocolError, match="unsupported"): + protocol.sign_envelope(MASTER, "admin", binding(), {}) + valid = protocol.sign_envelope(MASTER, "heartbeat", binding(), {}) + + extra = {**valid, "extra": True} + with pytest.raises(protocol.ProtocolError, match="fields"): + protocol.verify_envelope(MASTER, extra) + wrong_version = resign({**valid, "version": 1}) + with pytest.raises(protocol.ProtocolError, match="version"): + protocol.verify_envelope(MASTER, wrong_version) + with pytest.raises(protocol.ProtocolError, match="unexpected"): + protocol.verify_envelope(MASTER, valid, expected_kind="result") + + numeric = resign({**valid, "attempt": "not-a-number"}) + with pytest.raises(protocol.ProtocolError, match="numeric"): + protocol.verify_envelope(MASTER, numeric) + numeric_text = resign({**valid, "attempt": "1"}) + with pytest.raises(protocol.ProtocolError, match="numeric"): + protocol.verify_envelope(MASTER, numeric_text) + typed_identifier = resign({**valid, "run_id": 42}) + with pytest.raises(protocol.ProtocolError, match="run_id"): + protocol.verify_envelope(MASTER, typed_identifier) + lifetime = resign({**valid, "expires_at": valid["issued_at"]}) + with pytest.raises(protocol.ProtocolError, match="lifetime"): + protocol.verify_envelope(MASTER, lifetime) + digest = resign({**valid, "payload_digest": "0" * 64}) + with pytest.raises(protocol.ProtocolError, match="digest"): + protocol.verify_envelope(MASTER, digest) + with pytest.raises(protocol.ProtocolError, match="oversized"): + protocol.verify_envelope(MASTER, {"value": "x" * protocol.MAX_WIRE_BYTES}) + + +def test_wire_parser_rejects_non_object_and_empty(): + for body, message in ((b"", "empty"), (b"[]", "object")): + with pytest.raises(protocol.ProtocolError, match=message): + protocol.parse_wire(body) + + +def test_bounded_http_server_sets_timeout_and_releases_slot(): + handled = threading.Event() + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 + handled.set() + self.send_response(204) + self.end_headers() + + def log_message(self, *_args): + return + + server = protocol.BoundedHTTPServer(("127.0.0.1", 0), Handler, max_workers=1) + thread = threading.Thread(target=server.handle_request) + thread.start() + with urllib.request.urlopen( + f"http://127.0.0.1:{server.server_port}/", timeout=3 + ) as response: + assert response.status == 204 + thread.join(timeout=3) + assert handled.is_set() + acquired = False + for _ in range(100): + acquired = server._slots.acquire(blocking=False) + if acquired: + break + threading.Event().wait(0.01) + assert acquired + server._slots.release() + server.server_close() + + +def test_bounded_server_releases_slot_when_thread_dispatch_raises(monkeypatch): + server = protocol.BoundedHTTPServer( + ("127.0.0.1", 0), BaseHTTPRequestHandler, max_workers=1 + ) + + def explode(*_args): + raise RuntimeError("dispatch failed") + + monkeypatch.setattr(ThreadingHTTPServer, "process_request", explode) + with pytest.raises(RuntimeError, match="dispatch"): + server.process_request(object(), ("127.0.0.1", 1)) + assert server._slots.acquire(blocking=False) + server._slots.release() + server.server_close() + + +def test_store_fences_ordinal_attempt_state_and_expired_lease(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db", lease_seconds=60) + assert protocol.PoolStore._record(None) is None + store.add(binding(), {"context": "safe"}) + store.offer(0) + + foreign = protocol.sign_envelope( + MASTER, "heartbeat", binding(worker_ordinal=1), {"note": "foreign"} + ) + with pytest.raises(protocol.ProtocolError, match="ordinal"): + store.heartbeat(foreign) + + with store._connect() as connection: + connection.execute("UPDATE assignments SET lease_until=1") + expired = protocol.sign_envelope(MASTER, "heartbeat", binding(), {"note": "late"}) + with pytest.raises(protocol.ProtocolError, match="lease expired"): + store.heartbeat(expired) + terminal = protocol.sign_envelope(MASTER, "result", binding(), {"structured": {}}) + with pytest.raises(protocol.ProtocolError, match="lease expired"): + store.accept_result(terminal) + + +def test_store_pending_result_invalid_state_and_exact_finalize(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding(), {"context": "safe"}) + result = protocol.sign_envelope(MASTER, "result", binding(), {"structured": {}}) + record, duplicate = store.accept_result(result) + assert duplicate is False and record["state"] == "result" + assert store.pending_results()[0]["result"] == {"structured": {}} + with pytest.raises(protocol.ProtocolError, match="terminal"): + store.finalize(binding(), "invalid") + store.finalize(binding(attempt=2), "stale") + assert store.pending_results()[0]["state"] == "result" + store.finalize(binding(), "finalized") + heartbeat = protocol.sign_envelope(MASTER, "heartbeat", binding(), {}) + with pytest.raises(protocol.ProtocolError, match="running"): + store.heartbeat(heartbeat) + + +def test_expired_attempt_is_reoffered_then_terminally_released(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding(), {"context": "safe"}) + store.offer(0) + changed = store.expire_leases(now=10_000_000_000, max_attempts=2) + assert changed[0]["state"] == "assigned" and changed[0]["attempt"] == 2 + stale = protocol.sign_envelope(MASTER, "heartbeat", binding(), {}) + with pytest.raises(protocol.ProtocolError, match="attempt is stale"): + store.heartbeat(stale) + + offered = store.offer(0) + assert offered and offered["attempt"] == 2 + changed = store.expire_leases(now=10_000_000_001, max_attempts=2) + assert changed[0]["state"] == "lease_failed" + assert store.available_ordinals() == [0, 1, 2] + + +def test_store_garbage_collection_removes_old_terminal_and_deliveries(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding(), {"context": "safe"}) + heartbeat = protocol.sign_envelope( + MASTER, "heartbeat", binding(), {}, delivery_id="old-delivery" + ) + store.heartbeat(heartbeat) + store.finalize(binding(), "stale") + with store._connect() as connection: + connection.execute("UPDATE assignments SET updated_at=1") + connection.execute("UPDATE deliveries SET received_at=1") + assert store.garbage_collect(3600) == 1 + assert store.active_assignments() == [] + with store._connect() as connection: + assert connection.execute("SELECT count(*) FROM deliveries").fetchone()[0] == 0 diff --git a/testing/tests/test_hermes_execution_pool_scm_tampering.py b/testing/tests/test_hermes_execution_pool_scm_tampering.py new file mode 100644 index 00000000..50bfe9e8 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_scm_tampering.py @@ -0,0 +1,200 @@ +"""SCM checkout ownership and private-baseline corruption probes.""" + +from __future__ import annotations + +import sys +import subprocess +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +sys.path[:0] = [ + str(ROOT / "services/hermes/scripts"), + str(ROOT / "services/hermes/scm-common/scripts"), +] + +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_scm as scm # noqa: E402 +from testing.tests.test_hermes_execution_pool_mediator import ( # noqa: E402 + KEY, + assignment, + init_checkout, +) + + +def test_checkout_and_submit_reject_unmanaged_or_corrupt_private_state( + tmp_path, monkeypatch +): + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + monkeypatch.setattr(scm, "ORDINAL", 0) + destination = scm.workspace_path(assignment()) + destination.mkdir(parents=True) + (destination / "owner-file").write_text("preserve") + boundary = scm.Boundary(KEY) + with pytest.raises(protocol.ProtocolError, match="non-empty"): + boundary.checkout(assignment()) + + (destination / "owner-file").unlink() + + def clone_with_empty_failure(*arguments, **_kwargs): + if arguments[0] == "clone" and arguments[4] == "wt/t_deadbeef": + destination.mkdir(parents=True, exist_ok=True) + raise RuntimeError("missing") + if arguments[0] == "clone": + (destination / ".git").mkdir(parents=True) + return "" + + monkeypatch.setattr(scm, "_run", clone_with_empty_failure) + monkeypatch.setattr(scm, "_workspace_identity", lambda *_a: "a" * 40) + assert boundary.checkout(assignment())["baseline_sha"] == "a" * 40 + + protocol.atomic_json(scm._state_path(assignment()), {"baseline_sha": "bad"}) + with pytest.raises(protocol.ProtocolError, match="baseline"): + boundary.submit(assignment(), {}) + + +def test_scm_identity_is_data_only_and_supports_packed_refs(tmp_path): + checkout = tmp_path / "checkout" + subprocess.run(["git", "init", "-q", str(checkout)], check=True) + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "add", "origin", + "https://scm.bstein.dev/atlas/titan-iac.git", + ], + check=True, + ) + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "add", "hermes-broker", + scm._broker_repo("titan-iac"), + ], + check=True, + ) + subprocess.run( + [ + "git", "-C", str(checkout), "checkout", "-qb", + "feature/hermes-safe-pool", + ], + check=True, + ) + subprocess.run( + ["git", "-C", str(checkout), "config", "user.email", "a@b.c"], + check=True, + ) + subprocess.run( + ["git", "-C", str(checkout), "config", "user.name", "Test"], + check=True, + ) + (checkout / "tracked").write_text("safe\n") + subprocess.run(["git", "-C", str(checkout), "add", "tracked"], check=True) + subprocess.run( + ["git", "-C", str(checkout), "commit", "-qm", "initial"], check=True + ) + expected = subprocess.check_output( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], text=True + ).strip() + subprocess.run( + ["git", "-C", str(checkout), "pack-refs", "--all"], check=True + ) + assert scm._workspace_identity( + checkout, + "titan-iac", + "feature/hermes-safe-pool", + ) == expected + (checkout / ".git/refs/heads/feature").mkdir(parents=True) + (checkout / ".git/refs/heads/feature/hermes-safe-pool").symlink_to( + "/etc/passwd" + ) + with pytest.raises(protocol.ProtocolError, match="symlink"): + scm._workspace_identity( + checkout, + "titan-iac", + "feature/hermes-safe-pool", + ) + + +def test_scm_paths_and_identity_reject_symlink_and_git_tampering( + tmp_path, monkeypatch +): + workspace = tmp_path / "workspace" + workspace.mkdir() + workspace_link = tmp_path / "workspace-link" + workspace_link.symlink_to(workspace, target_is_directory=True) + monkeypatch.setattr(scm, "WORKSPACE_ROOT", workspace_link) + with pytest.raises(protocol.ProtocolError, match="root"): + scm.workspace_path(assignment()) + + monkeypatch.setattr(scm, "WORKSPACE_ROOT", workspace) + (workspace / "runs").mkdir() + (workspace / "runs/metis").symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="parent"): + scm.workspace_path(assignment()) + + state = tmp_path / "state" + state.mkdir() + monkeypatch.setattr(scm, "SCM_ROOT", state) + expected_state = state / "metis-t_deadbeef-42.json" + expected_state.symlink_to("/etc/passwd") + with pytest.raises(protocol.ProtocolError, match="state"): + scm._state_path(assignment()) + + checkout = tmp_path / "checkout" + checkout.mkdir() + with pytest.raises(protocol.ProtocolError, match="metadata"): + scm._workspace_identity(checkout, "metis", "wt/t_deadbeef") + init_checkout(checkout) + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "set-url", "origin", + "https://evil.example/repo.git", + ], + check=True, + ) + with pytest.raises(protocol.ProtocolError, match="origin"): + scm._workspace_identity(checkout, "metis", "wt/t_deadbeef") + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "set-url", "origin", + "https://scm.bstein.dev/atlas/metis.git", + ], + check=True, + ) + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "set-url", "hermes-broker", + "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/" + "git/atlas/soteria.git", + ], + check=True, + ) + with pytest.raises(protocol.ProtocolError, match="broker remote"): + scm._workspace_identity(checkout, "metis", "wt/t_deadbeef") + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "set-url", "hermes-broker", + "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/" + "git/atlas/metis.git", + ], + check=True, + ) + with pytest.raises(protocol.ProtocolError, match="branch"): + scm._workspace_identity(checkout, "metis", "review/other") + monkeypatch.setattr( + scm, + "_run", + lambda *arguments, **_kwargs: { + "remote": ( + "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/" + "git/atlas/metis.git" + if arguments[-1] == "hermes-broker" + else "https://scm.bstein.dev/atlas/metis.git" + ), + "symbolic-ref": "wt/t_deadbeef", + "rev-parse": "invalid", + }[arguments[0]], + ) + with pytest.raises(protocol.ProtocolError, match="HEAD"): + scm._workspace_identity(checkout, "metis", "wt/t_deadbeef") diff --git a/testing/tests/test_hermes_execution_pool_worker_execute.py b/testing/tests/test_hermes_execution_pool_worker_execute.py new file mode 100644 index 00000000..05b5add4 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_worker_execute.py @@ -0,0 +1,246 @@ +"""Worker execution, fallback, terminal handoff, and exception contracts.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + +from cli_lane_config import ProcessResult, Route # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_worker as worker # noqa: E402 +from testing.tests.test_hermes_execution_pool_worker_v2 import assignment # noqa: E402 + + +def route(provider="codex", effort="high"): + return Route( + provider=provider, + model=f"{provider}-model", + effort=effort, + profile="agent", + classifier="test", + reason="unit", + latency_ms=1, + fallback_chain=(provider,), + ) + + +def completed_result(**changes): + structured = { + "status": "completed", + "summary": "Completed safely.", + "changed_files": ["safe.py"], + "tests_run": ["pytest"], + "artifacts": [], + "findings": [], + "blockers": [], + } + value = ProcessResult( + returncode=0, + output="done", + structured=structured, + capacity_failure=False, + ) + for name, replacement in changes.items(): + setattr(value, name, replacement) + return value + + +def prepare_execute(tmp_path, monkeypatch, *, item=None, results=None, git_status=""): + worker_root = tmp_path / "worker" + workspace = worker_root / "runs/metis/t_deadbeef/23" + workspace.mkdir(parents=True) + exact = item or assignment( + workspace=str(workspace), + payload={ + "context": "safe objective", + "assignee": "cli-auto", + "deadline_unix": 10_000_000_000, + "max_runtime_seconds": 3600, + }, + ) + monkeypatch.setattr(worker, "ROOT", worker_root) + monkeypatch.setattr(worker, "ORDINAL", 0) + monkeypatch.setattr(worker, "NODE", "titan-05") + monkeypatch.setattr(worker, "_bind_provider_sessions", lambda _a: None) + monkeypatch.setattr(worker.cli_lane_runner, "load_json", lambda _path: {}) + routes = [route("codex"), route("claude")] + route_calls = [] + + def select_route(*arguments, **keywords): + route_calls.append((arguments, keywords)) + return routes[min(len(route_calls) - 1, len(routes) - 1)] + + monkeypatch.setattr(worker.cli_lane_runner, "select_route", select_route) + monkeypatch.setattr(worker.cli_lane_runner, "fresh_unavailable_provider", lambda: "claude") + monkeypatch.setattr(worker.cli_lane_runner, "git_handoff", lambda *_a: "\nhandoff") + outcomes = list(results or [completed_result()]) + provider_calls = [] + + def run_provider(*arguments): + provider_calls.append(arguments) + return outcomes.pop(0) + + monkeypatch.setattr(worker.cli_lane_runner, "run_provider", run_provider) + client_calls = [] + + def client(operation, **values): + client_calls.append((operation, values)) + return {"ack": {"accepted": True}} + + monkeypatch.setattr(worker, "_client", client) + monkeypatch.setattr(worker, "_git", lambda *_a: git_status) + refreshed = [] + monkeypatch.setattr(worker, "_refresh_assignment", lambda exact: refreshed.append(exact)) + return exact, workspace, client_calls, provider_calls, route_calls, refreshed + + +def test_execute_completed_clean_result_refreshes_and_finishes_exact_run( + tmp_path, monkeypatch +): + exact, _workspace, calls, providers, routes, refreshed = prepare_execute( + tmp_path, monkeypatch + ) + worker.execute(exact) + assert len(providers) == 1 and len(routes) == 1 + assert refreshed == [worker._binding(exact)] + operations = [name for name, _values in calls] + assert operations == ["heartbeat", "finish"] + finish = calls[-1][1]["payload"] + assert finish["structured"]["status"] == "completed" + assert finish["node"] == "titan-05" + state = json.loads(worker._state_path(exact).read_text()) + assert state["terminal_at"] > 0 and state["baseline_sha"] == "a" * 40 + + +def test_execute_capacity_fallback_changes_provider_and_preserves_handoff( + tmp_path, monkeypatch +): + first = completed_result(capacity_failure=True, output="capacity") + second = completed_result() + exact, _workspace, calls, providers, routes, _refreshed = prepare_execute( + tmp_path, monkeypatch, results=[first, second] + ) + worker.execute(exact) + assert len(providers) == 2 and len(routes) == 2 + assert providers[1][0].provider == "claude" + assert providers[1][1].endswith("handoff") + heartbeats = [value for operation, value in calls if operation == "heartbeat"] + assert any("fallback=codex->claude" in value["payload"]["note"] for value in heartbeats) + + +def test_execute_dirty_workspace_downgrades_completed_result(tmp_path, monkeypatch): + exact, _workspace, calls, _providers, _routes, refreshed = prepare_execute( + tmp_path, monkeypatch, git_status="?? untracked" + ) + worker.execute(exact) + structured = calls[-1][1]["payload"]["structured"] + assert structured["status"] == "incomplete" + assert "uncommitted or untracked" in structured["blockers"][0] + assert refreshed == [] + + +def test_execute_fills_missing_lists_for_failed_provider_result(tmp_path, monkeypatch): + failed = ProcessResult( + returncode=1, + output="failed", + structured={"status": "blocked", "summary": "Provider failed."}, + capacity_failure=False, + ) + exact, _workspace, calls, _providers, _routes, _refreshed = prepare_execute( + tmp_path, monkeypatch, results=[failed] + ) + worker.execute(exact) + structured = calls[-1][1]["payload"]["structured"] + for name in ("changed_files", "tests_run", "artifacts", "findings", "blockers"): + assert structured[name] == [] + + +def test_execute_rejects_payload_lease_baseline_and_terminal_ack_failures( + tmp_path, monkeypatch +): + exact, *_ = prepare_execute(tmp_path, monkeypatch) + with pytest.raises(protocol.ProtocolError, match="payload"): + worker.execute({**exact, "payload": []}) + + monkeypatch.setattr(worker, "_client", lambda *_a, **_k: {"ack": {"accepted": False}}) + with pytest.raises(protocol.ProtocolError, match="lease"): + worker.execute(exact) + + exact, *_ = prepare_execute(tmp_path / "baseline", monkeypatch) + exact["baseline_sha"] = "invalid" + with pytest.raises(RuntimeError, match="baseline"): + worker.execute(exact) + + exact, *_ = prepare_execute(tmp_path / "ack", monkeypatch) + calls = [] + + def client(operation, **_values): + calls.append(operation) + return {"ack": {"accepted": operation != "finish"}} + + monkeypatch.setattr(worker, "_client", client) + with pytest.raises(protocol.ProtocolError, match="terminal"): + worker.execute(exact) + + +def test_execute_heartbeat_transport_failure_loses_lease(tmp_path, monkeypatch): + exact, *_ = prepare_execute(tmp_path, monkeypatch) + monkeypatch.setattr( + worker, "_client", lambda *_a, **_k: (_ for _ in ()).throw(ValueError("offline")) + ) + with pytest.raises(protocol.ProtocolError, match="lease"): + worker.execute(exact) + + +def test_report_exception_is_transient_exact_and_transport_safe(monkeypatch): + calls = [] + + def client(operation, **values): + calls.append((operation, values)) + return {"ack": {"accepted": True}} + + monkeypatch.setattr(worker, "_client", client) + assert worker.report_exception(assignment(), RuntimeError("boom")) + payload = calls[0][1]["payload"] + assert payload["capacity_failure"] is True + assert payload["structured"]["status"] == "blocked" + assert payload["structured"]["blockers"] == ["RuntimeError: boom"] + monkeypatch.setattr( + worker, "_client", lambda *_a, **_k: (_ for _ in ()).throw(OSError("offline")) + ) + assert worker.report_exception(assignment(), OSError("boom")) is False + + +def test_main_idle_and_exception_paths_do_not_spin_silently(monkeypatch, capsys): + monkeypatch.setattr(worker, "readiness", lambda: None) + monkeypatch.setattr(worker, "garbage_collect", lambda: 0) + monkeypatch.setattr(worker, "_poll", lambda: None) + monkeypatch.setattr( + worker.time, "sleep", lambda _seconds: (_ for _ in ()).throw(SystemExit("stop")) + ) + with pytest.raises(SystemExit, match="stop"): + worker.main() + + exact = assignment() + monkeypatch.setattr(worker, "_poll", lambda: exact) + monkeypatch.setattr(worker, "execute", lambda _a: (_ for _ in ()).throw(RuntimeError("boom"))) + monkeypatch.setattr(worker, "report_exception", lambda _a, _e: True) + with pytest.raises(SystemExit, match="stop"): + worker.main() + assert "surfaced" in capsys.readouterr().out + + monkeypatch.setattr( + worker, "garbage_collect", lambda: (_ for _ in ()).throw(RuntimeError("gc")) + ) + monkeypatch.setattr(worker, "report_exception", lambda *_a: False) + with pytest.raises(SystemExit, match="stop"): + worker.main() + assert "deferred" in capsys.readouterr().out diff --git a/testing/tests/test_hermes_execution_pool_worker_v2.py b/testing/tests/test_hermes_execution_pool_worker_v2.py new file mode 100644 index 00000000..a1984e56 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_worker_v2.py @@ -0,0 +1,292 @@ +"""Model worker path, evidence, retention, and readiness contracts.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_worker as worker # noqa: E402 + + +def assignment(**changes): + value = { + "board": "metis", "task_id": "t_deadbeef", "run_id": "23", + "worker_ordinal": 0, "attempt": 1, "protocol_version": 2, + "workspace": "/workspace/runs/metis/t_deadbeef/23", + "baseline_sha": "a" * 40, + "payload": {"context": "safe objective", "assignee": "cli-auto"}, + } + value.update(changes) + return value + + +class Response: + def __init__(self, value): + self.value = value + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, _size=-1): + return self.value + + +def test_post_client_and_poll_validate_every_response_boundary(monkeypatch): + monkeypatch.setattr( + worker.urllib.request, "urlopen", + lambda *_a, **_k: Response(json.dumps({"safe": True}).encode()), + ) + assert worker._post("http://mediator", {}) == {"safe": True} + monkeypatch.setattr( + worker.urllib.request, "urlopen", + lambda *_a, **_k: Response(b"x" * (64 * 1024 + 1)), + ) + with pytest.raises(protocol.ProtocolError, match="wire"): + worker._post("http://mediator", {}) + monkeypatch.setattr( + worker.urllib.request, "urlopen", lambda *_a, **_k: Response(b"[]") + ) + with pytest.raises(protocol.ProtocolError, match="object"): + worker._post("http://mediator", {}) + + monkeypatch.setattr(worker, "_post", lambda *_a, **_k: {"error": "denied"}) + with pytest.raises(protocol.ProtocolError, match="denied"): + worker._client("poll") + monkeypatch.setattr(worker, "_post", lambda *_a, **_k: {"safe": True}) + assert worker._client("poll") == {"safe": True} + monkeypatch.setattr(worker, "ORDINAL", 0) + monkeypatch.setattr(worker, "_client", lambda *_a, **_k: {"assignment": None}) + assert worker._poll() is None + for value in ([], assignment(worker_ordinal=1), assignment(protocol_version=1)): + monkeypatch.setattr(worker, "_client", lambda *_a, value=value, **_k: {"assignment": value}) + with pytest.raises(protocol.ProtocolError, match="foreign"): + worker._poll() + monkeypatch.setattr(worker, "_client", lambda *_a, **_k: {"assignment": assignment()}) + assert worker._poll()["task_id"] == "t_deadbeef" + assert worker._binding(assignment()) == { + "board": "metis", "task_id": "t_deadbeef", "run_id": "23", + "worker_ordinal": 0, "attempt": 1, + } + + +def test_state_path_rejects_traversal_root_and_leaf_symlinks(tmp_path, monkeypatch): + monkeypatch.setattr(worker, "ROOT", tmp_path) + path = worker._state_path(assignment()) + assert path == tmp_path / "session-state/metis/t_deadbeef/23.json" + with pytest.raises(protocol.ProtocolError, match="invalid"): + worker._state_path(assignment(task_id="../bad")) + + state_root = tmp_path / "session-state" + outside = tmp_path / "outside" + path.parent.rmdir() + path.parent.parent.rmdir() + state_root.rmdir() + outside.mkdir() + state_root.symlink_to(outside, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="root"): + worker._state_path(assignment()) + state_root.unlink() + path = worker._state_path(assignment()) + path.symlink_to("/etc/passwd") + with pytest.raises(protocol.ProtocolError, match="symlink"): + worker._state_path(assignment()) + + +def prepare_provider_roots(tmp_path, monkeypatch): + worker_root = tmp_path / "worker" + data_root = tmp_path / "data" + codex = tmp_path / "runtime/codex" + claude = tmp_path / "runtime/claude" + (worker_root / "provider-state").mkdir(parents=True) + data_root.mkdir() + codex.mkdir(parents=True) + claude.mkdir(parents=True) + monkeypatch.setattr(worker, "ROOT", worker_root) + monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root) + monkeypatch.setenv("CODEX_HOME", str(codex)) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(claude)) + return worker_root, data_root, codex, claude + + +def test_provider_session_binding_replaces_only_owned_symlinks(tmp_path, monkeypatch): + worker_root, data_root, codex, claude = prepare_provider_roots(tmp_path, monkeypatch) + old = tmp_path / "old" + old.mkdir() + (data_root / "home").symlink_to(old, target_is_directory=True) + (codex / "sessions").symlink_to(old, target_is_directory=True) + worker._bind_provider_sessions(assignment()) + assert "metis/t_deadbeef/23" in str((data_root / "home").resolve()) + assert "metis/t_deadbeef/23" in str((codex / "sessions").resolve()) + settings = worker_root / "provider-state/metis/t_deadbeef/23/home/.claude/settings.json" + assert json.loads(settings.read_text()) == {} + worker._bind_provider_sessions(assignment()) + + with pytest.raises(protocol.ProtocolError, match="binding"): + worker._bind_provider_sessions(assignment(run_id="../bad")) + monkeypatch.setattr(worker, "ROOT", tmp_path / "missing") + with pytest.raises(protocol.ProtocolError, match="unavailable"): + worker._bind_provider_sessions(assignment()) + + +def test_provider_session_binding_rejects_durable_and_runtime_tampering( + tmp_path, monkeypatch +): + worker_root, data_root, codex, _claude = prepare_provider_roots(tmp_path, monkeypatch) + run_parent = worker_root / "provider-state/metis" + run_parent.symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="session path"): + worker._bind_provider_sessions(assignment()) + run_parent.unlink() + home = worker_root / "provider-state/metis/t_deadbeef/23/home" + home.parent.mkdir(parents=True) + home.symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="HOME"): + worker._bind_provider_sessions(assignment()) + home.unlink() + (data_root / "home").write_text("not-owned") + with pytest.raises(protocol.ProtocolError, match="task-bound"): + worker._bind_provider_sessions(assignment()) + (data_root / "home").unlink() + (codex / "sessions").write_text("not-owned") + with pytest.raises(protocol.ProtocolError, match="not a symlink"): + worker._bind_provider_sessions(assignment()) + (codex / "sessions").unlink() + durable_sessions = ( + worker_root / "provider-state/metis/t_deadbeef/23/codex/sessions" + ) + durable_sessions.rmdir() + durable_sessions.symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="session path"): + worker._bind_provider_sessions(assignment()) + + +def test_prompt_activity_git_and_result_bounding(tmp_path, monkeypatch): + text = worker._prompt("objective", tmp_path, worker._binding(assignment())) + assert "no Kubernetes identity" in text and "objective" in text + missing = tmp_path / "missing.log" + assert worker._read_activity(missing, 7) == ("", 7) + log = tmp_path / "worker.log" + log.write_text("abc") + assert worker._read_activity(log, 0) == ("abc", 3) + assert worker._read_activity(log, 99) == ("abc", 3) + fifo = tmp_path / "fifo" + os.mkfifo(fifo) + with pytest.raises(protocol.ProtocolError, match="regular"): + worker._read_activity(fifo, 0) + + repo = tmp_path / "repo" + subprocess.run(["git", "init", "-q", str(repo)], check=True) + assert worker._git(repo, "status", "--porcelain") == "" + with pytest.raises(RuntimeError, match="ambiguous"): + worker._git(repo, "rev-parse", "missing") + + value = { + "status": "completed", "summary": "s" * 20_000, + "changed_files": ["x" * 3000] * 100, + "tests_run": "not-list", "artifacts": [], "findings": [], "blockers": [], + } + bounded = worker._bounded_result(value) + assert len(protocol.canonical_json(bounded)) <= 32 * 1024 + assert bounded["tests_run"] == [] and len(bounded["summary"]) <= 8000 + + +def test_refresh_assignment_requires_exact_binding(monkeypatch): + exact = worker._binding(assignment()) + monkeypatch.setattr(worker, "_poll", lambda: assignment()) + assert worker._refresh_assignment(exact)["run_id"] == "23" + for value in (None, assignment(attempt=2)): + monkeypatch.setattr(worker, "_poll", lambda value=value: value) + with pytest.raises(protocol.ProtocolError, match="changed"): + worker._refresh_assignment(exact) + + +def write_gc_state(root, task_id, workspace, terminal_at): + path = root / f"session-state/metis/{task_id}/23.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"terminal_at": terminal_at, "workspace": str(workspace)})) + return path + + +def init_clean_repo(path): + subprocess.run(["git", "init", "-q", str(path)], check=True) + subprocess.run(["git", "-C", str(path), "config", "user.email", "a@b.c"], check=True) + subprocess.run(["git", "-C", str(path), "config", "user.name", "Test"], check=True) + (path / "tracked").write_text("safe") + subprocess.run(["git", "-C", str(path), "add", "tracked"], check=True) + subprocess.run(["git", "-C", str(path), "commit", "-qm", "initial"], check=True) + + +def test_retention_skips_young_outside_dirty_and_symlink_workspaces(tmp_path, monkeypatch): + monkeypatch.setattr(worker, "ROOT", tmp_path) + monkeypatch.setattr(worker, "RETENTION_SECONDS", 3600) + run_root = tmp_path / "runs/metis" + clean = run_root / "clean/23" + dirty = run_root / "dirty/23" + clean.mkdir(parents=True) + dirty.mkdir(parents=True) + init_clean_repo(clean) + init_clean_repo(dirty) + (dirty / "untracked").write_text("dirty") + now = 10_000 + write_gc_state(tmp_path, "clean", clean, 1) + dirty_state = write_gc_state(tmp_path, "dirty", dirty, 1) + young = write_gc_state(tmp_path, "young", clean, now) + outside = write_gc_state(tmp_path, "outside", tmp_path / "missing", 1) + assert worker.garbage_collect(now=now) == 1 + assert not clean.exists() and dirty.exists() + assert dirty_state.exists() and young.exists() and outside.exists() + + link = run_root / "link/23" + link.parent.mkdir() + link.symlink_to(dirty, target_is_directory=True) + link_state = write_gc_state(tmp_path, "link", link, 1) + assert worker.garbage_collect(now=now) == 0 + assert link_state.exists() + + +def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monkeypatch): + worker_root = tmp_path / "worker" + data_root = tmp_path / "data" + codex = tmp_path / "codex" + claude = tmp_path / "claude" + for path in (worker_root, worker_root / "provider-state", data_root, codex, claude): + path.mkdir(parents=True, exist_ok=True) + (codex / "auth.json").write_text("{}") + (claude / ".credentials.json").write_text("{}") + schema = tmp_path / "schema/result.json" + monkeypatch.setattr(worker, "ORDINAL", 0) + monkeypatch.setattr(worker, "ROOT", worker_root) + monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root) + monkeypatch.setattr(worker.cli_lane_runner, "RESULT_SCHEMA_PATH", schema) + monkeypatch.setenv("CODEX_HOME", str(codex)) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(claude)) + polled = [] + monkeypatch.setattr(worker, "_poll", lambda: polled.append(True)) + worker.readiness() + assert polled and json.loads(schema.read_text()) == worker.cli_lane_runner.RESULT_SCHEMA + + monkeypatch.setattr(worker, "ORDINAL", 3) + with pytest.raises(protocol.ProtocolError, match="ordinal"): + worker.readiness() + monkeypatch.setattr(worker, "ORDINAL", 0) + monkeypatch.setattr(worker, "ROOT", tmp_path / "missing") + with pytest.raises(protocol.ProtocolError, match="path"): + worker.readiness() + monkeypatch.setattr(worker, "ROOT", worker_root) + (codex / "auth.json").unlink() + with pytest.raises(protocol.ProtocolError, match="credential"): + worker.readiness() diff --git a/testing/tests/test_hermes_gitea_pr_integration.py b/testing/tests/test_hermes_gitea_pr_integration.py index ee482cfe..b9f44bc4 100644 --- a/testing/tests/test_hermes_gitea_pr_integration.py +++ b/testing/tests/test_hermes_gitea_pr_integration.py @@ -142,7 +142,7 @@ def test_flux_manifest_isolates_vault_token_in_separate_broker_only(): ) ) boundary = common["configMapGenerator"][0] - assert boundary["name"] == "hermes-scm-boundary" + assert boundary["name"] == "hermes-scm-boundary-v2" assert "gitea_api.py=scripts/gitea_api.py" in boundary["files"] assert "gitea_api_policy.py=scripts/gitea_api_policy.py" in boundary["files"] assert "scm_broker_client.py=scripts/scm_broker_client.py" in boundary["files"] diff --git a/testing/tests/test_hermes_node_account_privilege_audit.py b/testing/tests/test_hermes_node_account_privilege_audit.py index 3370fe4f..a8762e54 100644 --- a/testing/tests/test_hermes_node_account_privilege_audit.py +++ b/testing/tests/test_hermes_node_account_privilege_audit.py @@ -2,6 +2,7 @@ from __future__ import annotations +import shutil import subprocess from pathlib import Path @@ -115,8 +116,11 @@ def test_visudo_valid_numeric_and_alias_grants_fail_closed( monkeypatch.setattr(module, "ACCOUNT_UID", 1200) policy = module.HOST_ETC / "sudoers" policy.write_text(value, encoding="utf-8") + visudo = shutil.which("visudo") + if visudo is None: + pytest.skip("visudo is not installed in this test environment") validation = subprocess.run( - ["/usr/bin/visudo", "-c", "-f", str(policy)], + [visudo, "-c", "-f", str(policy)], check=False, capture_output=True, text=True, diff --git a/testing/tests/test_hermes_runtime_access.py b/testing/tests/test_hermes_runtime_access.py index abb64f40..21054868 100644 --- a/testing/tests/test_hermes_runtime_access.py +++ b/testing/tests/test_hermes_runtime_access.py @@ -3,6 +3,8 @@ from __future__ import annotations import importlib.util +import hashlib +import hmac import json import sys import urllib.request @@ -205,41 +207,51 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp assert auth == {"version": 1, "providers": {}, "credential_pool": {}} -def test_execution_worker_separates_channel_key_and_durable_sessions( +def test_execution_worker_and_mediator_separate_credentials_and_hmac( tmp_path: Path, monkeypatch ): stage = _load("stage_runtime_access") vault = tmp_path / "vault" runtime = tmp_path / "runtime" worker = tmp_path / "worker" + provider_access = tmp_path / "provider-access" pool_access = tmp_path / "pool-access" - scm_access = tmp_path / "scm-access" vault.mkdir() - (vault / "execution-pool-key").write_text("e" * 64) - (vault / "claude-credentials").write_text( + (vault / "claude-credentials-1").write_text( json.dumps({"claudeAiOauth": {"refreshToken": "claude-refresh"}}) ) - (vault / "codex-auth").write_text( + (vault / "codex-auth-1").write_text( json.dumps({"tokens": {"refresh_token": "codex-refresh"}}) ) - (vault / "gitea-token").write_text("test-gitea-token") - (vault / "gitea-username").write_text("hermes-automation") monkeypatch.setattr(stage, "VAULT_ROOT", vault) monkeypatch.setattr(stage, "RUNTIME_ROOT", runtime) monkeypatch.setattr(stage, "WORKER_ROOT", worker) + monkeypatch.setattr(stage, "PROVIDER_ACCESS_ROOT", provider_access) monkeypatch.setattr(stage, "POOL_ACCESS_ROOT", pool_access) - monkeypatch.setattr(stage, "SCM_ACCESS_ROOT", scm_access) + monkeypatch.setenv("HERMES_WORKER_ORDINAL", "1") monkeypatch.setattr(stage.os, "chown", lambda *_args: None) monkeypatch.setattr(stage.os, "fchown", lambda *_args: None) stage.stage_execution_worker() - assert (pool_access / "execution-pool-key").read_text().strip() == "e" * 64 + assert not pool_access.exists() assert not (runtime / "execution-pool-key").exists() - assert (scm_access / "gitea-token").read_text().strip() == "test-gitea-token" assert not (runtime / "codex/sessions").exists() assert not (runtime / "claude/projects").exists() - assert (runtime / "codex/auth.json").stat().st_mode & 0o777 == 0o600 + assert (provider_access / "codex/auth.json").stat().st_mode & 0o777 == 0o600 + refreshed = {"tokens": {"refresh_token": "provider-rotated"}} + (provider_access / "codex/auth.json").write_text(json.dumps(refreshed)) + (provider_access / "codex/auth.json").chmod(0o600) + stage.stage_execution_worker() + assert json.loads((provider_access / "codex/auth.json").read_text()) == refreshed + + master = "e" * 64 + (vault / "execution-pool-key").write_text(master) + stage.stage_execution_mediator() + expected = hmac.new( + master.encode(), b"hermes-execution-pool-v2:worker:1", hashlib.sha256 + ).hexdigest() + assert (pool_access / "execution-pool-key").read_text().strip() == expected def test_execution_worker_fails_closed_without_channel_credential( @@ -248,19 +260,14 @@ def test_execution_worker_fails_closed_without_channel_credential( stage = _load("stage_runtime_access") vault = tmp_path / "vault" vault.mkdir() - (vault / "claude-credentials").write_text( + (vault / "claude-credentials-0").write_text( json.dumps({"claudeAiOauth": {"refreshToken": "claude-refresh"}}) ) - (vault / "codex-auth").write_text( - json.dumps({"tokens": {"refresh_token": "codex-refresh"}}) - ) - (vault / "gitea-token").write_text("test-gitea-token") - (vault / "gitea-username").write_text("hermes-automation") monkeypatch.setattr(stage, "VAULT_ROOT", vault) monkeypatch.setattr(stage, "RUNTIME_ROOT", tmp_path / "runtime") monkeypatch.setattr(stage, "WORKER_ROOT", tmp_path / "worker") - monkeypatch.setattr(stage, "POOL_ACCESS_ROOT", tmp_path / "pool-access") - monkeypatch.setattr(stage, "SCM_ACCESS_ROOT", tmp_path / "scm-access") + monkeypatch.setattr(stage, "PROVIDER_ACCESS_ROOT", tmp_path / "provider-access") + monkeypatch.setenv("HERMES_WORKER_ORDINAL", "0") monkeypatch.setattr(stage.os, "chown", lambda *_args: None) monkeypatch.setattr(stage.os, "fchown", lambda *_args: None) @@ -268,6 +275,21 @@ def test_execution_worker_fails_closed_without_channel_credential( stage.stage_execution_worker() +def test_execution_worker_rejects_durable_credential_symlink( + tmp_path: Path, monkeypatch +): + stage = _load("stage_runtime_access") + provider = tmp_path / "provider" + provider.mkdir() + (provider / "claude").symlink_to(tmp_path, target_is_directory=True) + monkeypatch.setattr(stage, "WORKER_ROOT", tmp_path / "worker") + monkeypatch.setattr(stage, "PROVIDER_ACCESS_ROOT", provider) + monkeypatch.setenv("HERMES_WORKER_ORDINAL", "0") + monkeypatch.setattr(stage.os, "chown", lambda *_args: None) + with pytest.raises(RuntimeError, match="symlink"): + stage.stage_execution_worker() + + def test_invalid_runtime_json_is_removed(tmp_path: Path, monkeypatch): stage = _load("stage_runtime_access") vault = tmp_path / "vault" @@ -281,7 +303,7 @@ def test_invalid_runtime_json_is_removed(tmp_path: Path, monkeypatch): try: stage._validated_json("credential", destination, ("token",)) - except json.JSONDecodeError: + except RuntimeError: pass else: raise AssertionError("invalid credential JSON should fail staging") diff --git a/testing/tests/test_hermes_scm_broker.py b/testing/tests/test_hermes_scm_broker.py index 2a0fad44..8f1b2ca9 100644 --- a/testing/tests/test_hermes_scm_broker.py +++ b/testing/tests/test_hermes_scm_broker.py @@ -48,7 +48,7 @@ def test_broker_metadata_boundary_reuses_explicit_read_allowlist(): api.authorize_request("GET", path, None) -def test_receive_pack_allows_only_new_namespaced_feature_branch(): +def test_receive_pack_allows_only_new_branches_in_reviewed_namespaces(): broker = _load("scm_broker") zero = b"0" * 40 commit = b"1" * 40 @@ -57,9 +57,18 @@ def test_receive_pack_allows_only_new_namespaced_feature_branch(): _receive_command(zero, commit, b"refs/heads/hermes/focused-fix"), "runtime-sentinel", ) + broker._validate_receive_pack( + _receive_command(zero, commit, b"refs/heads/wt/t_deadbeef"), + "runtime-sentinel", + ) + broker._validate_receive_pack( + _receive_command(zero, commit, b"refs/heads/review/t_deadbeef"), + "runtime-sentinel", + ) for old, new, ref in ( (commit, b"2" * 40, b"refs/heads/hermes/focused-fix"), (commit, zero, b"refs/heads/hermes/focused-fix"), + (commit, commit, b"refs/heads/hermes/focused-fix"), (zero, commit, b"refs/heads/main"), (zero, commit, b"refs/heads/master"), (zero, commit, b"refs/tags/release"),