From 7a55b259bf1224d9bf19f5ec294f3c917fb4bfc8 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 16:31:15 +0000 Subject: [PATCH] hermes: add the fenced three-node distributed execution pool Three fenced worker Pods claim Hermes Kanban runs through a coordinator that owns every state transition, with per-ordinal HMAC authority, a mediated broker-only SCM path, and durable per-ordinal workspaces. Content is the reviewed head of PR #18 (689bcb6e) with PR 16's and PR 19's contributions removed: they were merged in only to validate co-existence and are not prerequisites, so this branch no longer carries them as ancestors. Only PR 14 and PR 15 remain, because the broker boundary and the cli_lane_* decomposition are load-bearing for two of the fixed P0 boundaries. Co-Authored-By: Claude Opus 5 --- ci/scripts/semgrep_report.py | 2 +- .../applications/hermes/kustomization.yaml | 4 + ...test_dashboards_render_atlas_drilldowns.py | 2 +- services/hermes-scm-broker/deployment.yaml | 3 +- services/hermes-scm-broker/networkpolicy.yaml | 6 + services/hermes/NOTES.md | 26 + services/hermes/agent-deployment.yaml | 2 +- .../hermes/execution-coordinator-patch.yaml | 66 +++ services/hermes/execution-mediator.yaml | 425 +++++++++++++++ .../execution-worker-networkpolicy.yaml | 206 ++++++++ services/hermes/execution-worker-rbac.yaml | 12 + .../hermes/execution-worker-statefulset.yaml | 264 ++++++++++ services/hermes/kustomization.yaml | 37 ++ services/hermes/scm-common/kustomization.yaml | 2 +- .../hermes/scm-common/scripts/scm_broker.py | 41 +- services/hermes/scripts/cli_lane_dispatch.py | 25 +- services/hermes/scripts/cli_lane_evidence.py | 15 +- services/hermes/scripts/cli_lane_execution.py | 26 +- .../hermes/scripts/cli_lane_finalization.py | 1 + .../hermes/scripts/cli_lane_quarantine.py | 2 + services/hermes/scripts/cli_lane_retention.py | 9 + .../hermes/scripts/execution_pool_client.py | 221 ++++++++ .../scripts/execution_pool_coordinator.py | 473 +++++++++++++++++ .../hermes/scripts/execution_pool_project.py | 152 ++++++ .../hermes/scripts/execution_pool_protocol.py | 496 ++++++++++++++++++ services/hermes/scripts/execution_pool_scm.py | 281 ++++++++++ .../hermes/scripts/execution_pool_server.py | 130 +++++ .../hermes/scripts/execution_pool_worker.py | 445 ++++++++++++++++ .../hermes/scripts/stage_runtime_access.py | 177 ++++++- services/hermes/service.yaml | 31 ++ .../vault/scripts/vault_k8s_auth_configure.sh | 2 + testing/tests/test_hermes_agent_access.py | 32 +- testing/tests/test_hermes_agent_security.py | 67 ++- testing/tests/test_hermes_auto_router.py | 1 + testing/tests/test_hermes_chat_config.py | 11 +- .../test_hermes_chat_provider_runtime.py | 196 +++++++ testing/tests/test_hermes_chat_voice.py | 7 +- testing/tests/test_hermes_cli_capabilities.py | 8 +- .../tests/test_hermes_cli_dispatch_runtime.py | 37 ++ .../tests/test_hermes_cli_evidence_edges.py | 35 ++ .../tests/test_hermes_cli_execution_edges.py | 15 +- testing/tests/test_hermes_cli_fallback.py | 4 + .../test_hermes_cli_finalization_edges.py | 8 + .../test_hermes_cli_foundation_coverage.py | 60 ++- .../test_hermes_cli_lanes_configuration.py | 12 +- testing/tests/test_hermes_cli_lanes_kanban.py | 392 -------------- .../tests/test_hermes_cli_provider_edges.py | 67 ++- .../tests/test_hermes_cli_records_edges.py | 11 + .../tests/test_hermes_cli_recovery_edges.py | 34 ++ .../tests/test_hermes_cli_retention_edges.py | 46 ++ .../tests/test_hermes_coordinator_boards.py | 9 +- testing/tests/test_hermes_execution_pool.py | 476 +++++++++++++++++ .../test_hermes_execution_pool_assignment.py | 208 ++++++++ ...st_hermes_execution_pool_coordinator_v2.py | 293 +++++++++++ .../test_hermes_execution_pool_dispatch_v2.py | 410 +++++++++++++++ .../test_hermes_execution_pool_mediator.py | 447 ++++++++++++++++ .../test_hermes_execution_pool_project.py | 211 ++++++++ .../test_hermes_execution_pool_protocol_v2.py | 241 +++++++++ ...est_hermes_execution_pool_scm_tampering.py | 200 +++++++ ...st_hermes_execution_pool_worker_execute.py | 246 +++++++++ .../test_hermes_execution_pool_worker_v2.py | 292 +++++++++++ .../tests/test_hermes_gitea_pr_integration.py | 2 +- ...est_hermes_node_account_privilege_audit.py | 6 +- testing/tests/test_hermes_runtime_access.py | 88 +++- .../test_hermes_runtime_stage_coverage.py | 96 +++- testing/tests/test_hermes_scm_broker.py | 11 +- 66 files changed, 7344 insertions(+), 519 deletions(-) create mode 100644 services/hermes/execution-coordinator-patch.yaml create mode 100644 services/hermes/execution-mediator.yaml create mode 100644 services/hermes/execution-worker-networkpolicy.yaml create mode 100644 services/hermes/execution-worker-rbac.yaml create mode 100644 services/hermes/execution-worker-statefulset.yaml create mode 100644 services/hermes/scripts/execution_pool_client.py create mode 100644 services/hermes/scripts/execution_pool_coordinator.py create mode 100644 services/hermes/scripts/execution_pool_project.py create mode 100644 services/hermes/scripts/execution_pool_protocol.py create mode 100644 services/hermes/scripts/execution_pool_scm.py create mode 100644 services/hermes/scripts/execution_pool_server.py create mode 100644 services/hermes/scripts/execution_pool_worker.py create mode 100644 testing/tests/test_hermes_chat_provider_runtime.py delete mode 100644 testing/tests/test_hermes_cli_lanes_kanban.py create mode 100644 testing/tests/test_hermes_execution_pool.py create mode 100644 testing/tests/test_hermes_execution_pool_assignment.py create mode 100644 testing/tests/test_hermes_execution_pool_coordinator_v2.py create mode 100644 testing/tests/test_hermes_execution_pool_dispatch_v2.py create mode 100644 testing/tests/test_hermes_execution_pool_mediator.py create mode 100644 testing/tests/test_hermes_execution_pool_project.py create mode 100644 testing/tests/test_hermes_execution_pool_protocol_v2.py create mode 100644 testing/tests/test_hermes_execution_pool_scm_tampering.py create mode 100644 testing/tests/test_hermes_execution_pool_worker_execute.py create mode 100644 testing/tests/test_hermes_execution_pool_worker_v2.py diff --git a/ci/scripts/semgrep_report.py b/ci/scripts/semgrep_report.py index b4ed5a2d..2408f3e9 100644 --- a/ci/scripts/semgrep_report.py +++ b/ci/scripts/semgrep_report.py @@ -52,7 +52,7 @@ def build_report( results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else [] errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else [] - severity_counts = {severity: 0 for severity in SEVERITIES} + severity_counts = dict.fromkeys(SEVERITIES, 0) blocking_findings = 0 for finding in results: severity = _finding_severity(finding) diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index df7a5560..084e6074 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -30,6 +30,10 @@ spec: kind: Deployment name: hermes-agent namespace: hermes + - apiVersion: apps/v1 + kind: StatefulSet + name: hermes-execution-worker + namespace: hermes - apiVersion: apps/v1 kind: DaemonSet name: hermes-node-ssh-access diff --git a/scripts/tests/test_dashboards_render_atlas_drilldowns.py b/scripts/tests/test_dashboards_render_atlas_drilldowns.py index 08871b71..904379c9 100644 --- a/scripts/tests/test_dashboards_render_atlas_drilldowns.py +++ b/scripts/tests/test_dashboards_render_atlas_drilldowns.py @@ -1,6 +1,6 @@ """Detailed Atlas Jobs dashboard collapse and drilldown contracts.""" -from scripts.tests.test_dashboards_render_atlas import load_module +from scripts.tests.test_dashboard_render_support import load_module def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): diff --git a/services/hermes-scm-broker/deployment.yaml b/services/hermes-scm-broker/deployment.yaml index 649ada1d..ba62f451 100644 --- a/services/hermes-scm-broker/deployment.yaml +++ b/services/hermes-scm-broker/deployment.yaml @@ -17,6 +17,7 @@ spec: labels: app: hermes-scm-broker annotations: + ai.bstein.dev/config-rev: scm-boundary-v2 vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-scm-broker vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea @@ -91,7 +92,7 @@ spec: volumes: - name: broker-code configMap: - name: hermes-scm-boundary + name: hermes-scm-boundary-v2 defaultMode: 0555 - name: tmp emptyDir: diff --git a/services/hermes-scm-broker/networkpolicy.yaml b/services/hermes-scm-broker/networkpolicy.yaml index 8187e333..9c1a0542 100644 --- a/services/hermes-scm-broker/networkpolicy.yaml +++ b/services/hermes-scm-broker/networkpolicy.yaml @@ -17,6 +17,12 @@ spec: podSelector: matchLabels: app: hermes-agent + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hermes + podSelector: + matchLabels: + app: hermes-execution-mediator ports: - {protocol: TCP, port: 9081} egress: diff --git a/services/hermes/NOTES.md b/services/hermes/NOTES.md index c70e528a..400bf44f 100644 --- a/services/hermes/NOTES.md +++ b/services/hermes/NOTES.md @@ -257,6 +257,32 @@ Use this short explanation: leading different real incidents, improving the skill after failures, and teaching the architecture without prompts. +## Distributed execution-pool rollout boundary + +- Do not reconcile the pool until the approved credential owner has provisioned + six distinct Vault fields: `execution_worker_{0,1,2}_claude_credentials_json` + and `execution_worker_{0,1,2}_codex_auth_json`. Each ordinal needs an + independent account or refresh-token lineage; copying one rotating refresh + token into multiple fields recreates the lost-update failure this design + prevents. +- Worker credential files live on separate ordinal-owned RWO `provider-access` + claims. Provider refresh updates these durable private copies across Pod + restarts; they are never synchronized back to Vault. Rotate one bootstrap + credential at a time through the reviewed Vault workflow and reinitialize + only that ordinal after human approval. +- Existing task worktrees remain on the single local owner lane. Only tasks + without `workspace_path` enter the distributed pool, where each ordinal owns + one RWO checkout. Repository and base-branch identity comes from the canonical + board registry rather than task-supplied metadata. +- Model Pods have no pool key, broker mount, broker egress, or Kubernetes token. + Ordinal mediator Deployments are separate network identities colocated with + the matching workspace PVC; they alone authenticate exact-run results and + reach the PR14 SCM broker. +- A hashed execution-pool ConfigMap, protocol-version readiness checks, and + versioned SCM boundary name make code/config changes controlled rollouts. A + rollout is still a human-reviewed operation; this repository change does not + reconcile or deploy it. + ## Your shortest path to fluency 1. Explain the request diagram without looking. diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index ccbc36ea..ed0533e7 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -1191,7 +1191,7 @@ spec: - {key: openai.yaml, path: agents/openai.yaml} - name: scm-boundary configMap: - name: hermes-scm-boundary + name: hermes-scm-boundary-v2 defaultMode: 0555 - name: image-policy configMap: diff --git a/services/hermes/execution-coordinator-patch.yaml b/services/hermes/execution-coordinator-patch.yaml new file mode 100644 index 00000000..ec9ab80b --- /dev/null +++ b/services/hermes/execution-coordinator-patch.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-agent + namespace: hermes +spec: + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600" + vault.hashicorp.com/agent-inject-template-execution-pool-key: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }} + {{- end }} + spec: + containers: + - name: cli-lane-runner + env: + - {name: HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY, value: "true"} + - {name: HERMES_CLI_LANE_CONCURRENCY, value: "1"} + - name: execution-pool-coordinator + image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 + imagePullPolicy: IfNotPresent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_coordinator.py] + env: + - {name: HERMES_HOME, value: /opt/data} + - {name: HOME, value: /opt/data/home} + - {name: PYTHONPATH, value: /opt/hermes} + - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key} + - {name: PATH, value: /opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin} + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: home, mountPath: /opt/data} + - {name: runtime-access, mountPath: /pool-access/execution-pool-key, subPath: execution-pool-key, readOnly: true} + - {name: execution-pool-code, mountPath: /opt/coordinator, readOnly: true} + - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} + - {name: tmp, mountPath: /tmp} + ports: + - {name: execution-pool, containerPort: 9007, protocol: TCP} + startupProbe: + httpGet: {path: /ready, port: execution-pool} + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: {path: /ready, port: execution-pool} + periodSeconds: 10 + livenessProbe: + httpGet: {path: /ready, port: execution-pool} + initialDelaySeconds: 30 + periodSeconds: 30 + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {cpu: 500m, memory: 512Mi} + volumes: + - name: execution-pool-code + configMap: + name: hermes-execution-pool + defaultMode: 0555 diff --git a/services/hermes/execution-mediator.yaml b/services/hermes/execution-mediator.yaml new file mode 100644 index 00000000..d41ee9f9 --- /dev/null +++ b/services/hermes/execution-mediator.yaml @@ -0,0 +1,425 @@ +# Privileged HMAC and SCM mediation runs outside the model Pods. Each mediator is +# colocated with exactly one ordinal's RWO workspace but has a distinct network +# identity, private HMAC derivation, and durable integrity state. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: hermes-execution-mediator-state-0, namespace: hermes} +spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: {requests: {storage: 1Gi}} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: hermes-execution-mediator-state-1, namespace: hermes} +spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: {requests: {storage: 1Gi}} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: hermes-execution-mediator-state-2, namespace: hermes} +spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: {requests: {storage: 1Gi}} +--- +apiVersion: v1 +kind: Service +metadata: {name: hermes-execution-mediator-0, namespace: hermes} +spec: + selector: {app: hermes-execution-mediator, pool-ordinal: "0"} + ports: [{name: mediator, port: 9009, targetPort: mediator}] +--- +apiVersion: v1 +kind: Service +metadata: {name: hermes-execution-mediator-1, namespace: hermes} +spec: + selector: {app: hermes-execution-mediator, pool-ordinal: "1"} + ports: [{name: mediator, port: 9009, targetPort: mediator}] +--- +apiVersion: v1 +kind: Service +metadata: {name: hermes-execution-mediator-2, namespace: hermes} +spec: + selector: {app: hermes-execution-mediator, pool-ordinal: "2"} + ports: [{name: mediator, port: 9009, targetPort: mediator}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-execution-mediator-0 + namespace: hermes + labels: {app: hermes-execution-mediator, pool-ordinal: "0"} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"} + template: + metadata: + labels: {app: hermes-execution-mediator, pool-ordinal: "0"} + annotations: + ai.bstein.dev/config-rev: execution-pool-v2-mediated + ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-execution-worker + vault.hashicorp.com/agent-inject-containers: stage-mediator-access + vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token + vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600" + vault.hashicorp.com/agent-inject-template-execution-pool-key: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 2m + vault.hashicorp.com/agent-requests-mem: 16Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi + spec: + serviceAccountName: hermes-execution-worker + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: {type: RuntimeDefault} + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + statefulset.kubernetes.io/pod-name: hermes-execution-worker-0 + topologyKey: kubernetes.io/hostname + initContainers: + - name: stage-mediator-access + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator] + env: + - {name: HERMES_WORKER_ORDINAL, value: "0"} + - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: pool-access, mountPath: /pool-access} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + resources: + requests: {cpu: 2m, memory: 16Mi} + limits: {cpu: 100m, memory: 64Mi} + containers: + - name: execution-mediator + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py] + env: + - {name: HERMES_WORKER_ORDINAL, value: "0"} + - {name: HERMES_WORKER_ROOT, value: /workspace} + - {name: HERMES_SCM_STATE_ROOT, value: /scm-state} + - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key} + - {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes} + ports: [{name: mediator, containerPort: 9009, protocol: TCP}] + startupProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: workspace, mountPath: /workspace} + - {name: scm-state, mountPath: /scm-state} + - {name: pool-access, mountPath: /pool-access, readOnly: true} + - {name: scm-broker-client, mountPath: /opt/scm, readOnly: true} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 2m, memory: 64Mi} + limits: {cpu: 250m, memory: 256Mi} + volumes: + - name: workspace + persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-0} + - name: scm-state + persistentVolumeClaim: {claimName: hermes-execution-mediator-state-0} + - name: pool-access + emptyDir: {medium: Memory, sizeLimit: 1Mi} + - name: scm-broker-client + configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555} + - name: coordinator + configMap: {name: hermes-execution-pool, defaultMode: 0555} + - name: tmp + emptyDir: {sizeLimit: 64Mi} + - name: vault-auth-token + projected: + defaultMode: 0600 + sources: + - serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token} + - configMap: + name: kube-root-ca.crt + items: [{key: ca.crt, path: ca.crt}] + - downwardAPI: + items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-execution-mediator-1 + namespace: hermes + labels: {app: hermes-execution-mediator, pool-ordinal: "1"} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"} + template: + metadata: + labels: {app: hermes-execution-mediator, pool-ordinal: "1"} + annotations: + ai.bstein.dev/config-rev: execution-pool-v2-mediated + ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-execution-worker + vault.hashicorp.com/agent-inject-containers: stage-mediator-access + vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token + vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600" + vault.hashicorp.com/agent-inject-template-execution-pool-key: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 2m + vault.hashicorp.com/agent-requests-mem: 16Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi + spec: + serviceAccountName: hermes-execution-worker + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: {type: RuntimeDefault} + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + statefulset.kubernetes.io/pod-name: hermes-execution-worker-1 + topologyKey: kubernetes.io/hostname + initContainers: + - name: stage-mediator-access + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator] + env: + - {name: HERMES_WORKER_ORDINAL, value: "1"} + - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: pool-access, mountPath: /pool-access} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + resources: + requests: {cpu: 2m, memory: 16Mi} + limits: {cpu: 100m, memory: 64Mi} + containers: + - name: execution-mediator + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py] + env: + - {name: HERMES_WORKER_ORDINAL, value: "1"} + - {name: HERMES_WORKER_ROOT, value: /workspace} + - {name: HERMES_SCM_STATE_ROOT, value: /scm-state} + - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key} + - {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes} + ports: [{name: mediator, containerPort: 9009, protocol: TCP}] + startupProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: workspace, mountPath: /workspace} + - {name: scm-state, mountPath: /scm-state} + - {name: pool-access, mountPath: /pool-access, readOnly: true} + - {name: scm-broker-client, mountPath: /opt/scm, readOnly: true} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 2m, memory: 64Mi} + limits: {cpu: 250m, memory: 256Mi} + volumes: + - name: workspace + persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-1} + - name: scm-state + persistentVolumeClaim: {claimName: hermes-execution-mediator-state-1} + - name: pool-access + emptyDir: {medium: Memory, sizeLimit: 1Mi} + - name: scm-broker-client + configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555} + - name: coordinator + configMap: {name: hermes-execution-pool, defaultMode: 0555} + - name: tmp + emptyDir: {sizeLimit: 64Mi} + - name: vault-auth-token + projected: + defaultMode: 0600 + sources: + - serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token} + - configMap: + name: kube-root-ca.crt + items: [{key: ca.crt, path: ca.crt}] + - downwardAPI: + items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-execution-mediator-2 + namespace: hermes + labels: {app: hermes-execution-mediator, pool-ordinal: "2"} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"} + template: + metadata: + labels: {app: hermes-execution-mediator, pool-ordinal: "2"} + annotations: + ai.bstein.dev/config-rev: execution-pool-v2-mediated + ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-execution-worker + vault.hashicorp.com/agent-inject-containers: stage-mediator-access + vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token + vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600" + vault.hashicorp.com/agent-inject-template-execution-pool-key: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 2m + vault.hashicorp.com/agent-requests-mem: 16Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi + spec: + serviceAccountName: hermes-execution-worker + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: {type: RuntimeDefault} + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + statefulset.kubernetes.io/pod-name: hermes-execution-worker-2 + topologyKey: kubernetes.io/hostname + initContainers: + - name: stage-mediator-access + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator] + env: + - {name: HERMES_WORKER_ORDINAL, value: "2"} + - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: pool-access, mountPath: /pool-access} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + resources: + requests: {cpu: 2m, memory: 16Mi} + limits: {cpu: 100m, memory: 64Mi} + containers: + - name: execution-mediator + image: registry.bstein.dev/bstein/hermes-agent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py] + env: + - {name: HERMES_WORKER_ORDINAL, value: "2"} + - {name: HERMES_WORKER_ROOT, value: /workspace} + - {name: HERMES_SCM_STATE_ROOT, value: /scm-state} + - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key} + - {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes} + ports: [{name: mediator, containerPort: 9009, protocol: TCP}] + startupProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: {path: /ready, port: mediator} + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: workspace, mountPath: /workspace} + - {name: scm-state, mountPath: /scm-state} + - {name: pool-access, mountPath: /pool-access, readOnly: true} + - {name: scm-broker-client, mountPath: /opt/scm, readOnly: true} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 2m, memory: 64Mi} + limits: {cpu: 250m, memory: 256Mi} + volumes: + - name: workspace + persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-2} + - name: scm-state + persistentVolumeClaim: {claimName: hermes-execution-mediator-state-2} + - name: pool-access + emptyDir: {medium: Memory, sizeLimit: 1Mi} + - name: scm-broker-client + configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555} + - name: coordinator + configMap: {name: hermes-execution-pool, defaultMode: 0555} + - name: tmp + emptyDir: {sizeLimit: 64Mi} + - name: vault-auth-token + projected: + defaultMode: 0600 + sources: + - serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token} + - configMap: + name: kube-root-ca.crt + items: [{key: ca.crt, path: ca.crt}] + - downwardAPI: + items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}] diff --git a/services/hermes/execution-worker-networkpolicy.yaml b/services/hermes/execution-worker-networkpolicy.yaml new file mode 100644 index 00000000..93e46be8 --- /dev/null +++ b/services/hermes/execution-worker-networkpolicy.yaml @@ -0,0 +1,206 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-worker-isolation + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-execution-worker + policyTypes: [Ingress, Egress] + ingress: [] + egress: + - to: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: kube-system} + podSelector: + matchLabels: {k8s-app: kube-dns} + ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}] + - to: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: vault} + podSelector: + matchLabels: {app: vault} + ports: [{protocol: TCP, port: 8200}] + - to: + - podSelector: + matchLabels: {app: hermes-switchyard} + ports: [{protocol: TCP, port: 9005}] + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 127.0.0.0/8 + - 169.254.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 224.0.0.0/4 + ports: [{protocol: TCP, port: 443}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-worker-mediator-0 + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "0" + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"} + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-worker-mediator-1 + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "1" + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"} + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-worker-mediator-2 + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "2" + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"} + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-pool-ingress + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-agent} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: {app: hermes-execution-mediator} + ports: [{protocol: TCP, port: 9007}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-mediator-isolation + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-execution-mediator} + policyTypes: [Ingress, Egress] + ingress: [] + egress: + - to: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: kube-system} + podSelector: + matchLabels: {k8s-app: kube-dns} + ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}] + - to: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: vault} + podSelector: + matchLabels: {app: vault} + ports: [{protocol: TCP, port: 8200}] + - to: + - podSelector: + matchLabels: {app: hermes-agent} + ports: [{protocol: TCP, port: 9007}] + - to: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: hermes-scm} + podSelector: + matchLabels: {app: hermes-scm-broker} + ports: [{protocol: TCP, port: 9081}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-mediator-worker-0 + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "0" + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-mediator-worker-1 + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "1" + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-mediator-worker-2 + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-execution-worker + apps.kubernetes.io/pod-index: "2" + ports: [{protocol: TCP, port: 9009}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-execution-switchyard-ingress + namespace: hermes +spec: + podSelector: + matchLabels: {app: hermes-switchyard} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: {app: hermes-execution-worker} + ports: [{protocol: TCP, port: 9005}] diff --git a/services/hermes/execution-worker-rbac.yaml b/services/hermes/execution-worker-rbac.yaml new file mode 100644 index 00000000..89ef915e --- /dev/null +++ b/services/hermes/execution-worker-rbac.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-execution-worker + namespace: hermes + labels: + app.kubernetes.io/name: hermes-execution-worker + app.kubernetes.io/part-of: hermes +automountServiceAccountToken: false +# Intentionally no RoleBinding or ClusterRoleBinding. The projected, bounded +# token is mounted only into Vault-facing containers and grants no Kubernetes +# API verbs to the model-facing execution worker. diff --git a/services/hermes/execution-worker-statefulset.yaml b/services/hermes/execution-worker-statefulset.yaml new file mode 100644 index 00000000..6794f4da --- /dev/null +++ b/services/hermes/execution-worker-statefulset.yaml @@ -0,0 +1,264 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hermes-execution-worker + namespace: hermes + labels: + app: hermes-execution-worker +spec: + serviceName: hermes-execution-worker + replicas: 3 + podManagementPolicy: Parallel + revisionHistoryLimit: 2 + selector: + matchLabels: + app: hermes-execution-worker + updateStrategy: + type: RollingUpdate + rollingUpdate: + partition: 0 + template: + metadata: + labels: + app: hermes-execution-worker + app.kubernetes.io/name: hermes-execution-worker + app.kubernetes.io/part-of: hermes + annotations: + ai.bstein.dev/role: fenced-execution-only + ai.bstein.dev/scm-boundary: mediated-pr14-broker-with-completion-gates + ai.bstein.dev/model-policy: Switchyard AUTO with cross-provider fallback + ai.bstein.dev/storage: separate durable RWO workspace and OAuth refresh ownership per ordinal + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-execution-worker + vault.hashicorp.com/agent-inject-containers: stage-worker-access + vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token + vault.hashicorp.com/agent-inject-secret-claude-credentials-0: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-claude-credentials-0: "0600" + vault.hashicorp.com/agent-inject-template-claude-credentials-0: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.execution_worker_0_claude_credentials_json }} + {{- end }} + vault.hashicorp.com/agent-inject-secret-claude-credentials-1: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-claude-credentials-1: "0600" + vault.hashicorp.com/agent-inject-template-claude-credentials-1: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.execution_worker_1_claude_credentials_json }} + {{- end }} + vault.hashicorp.com/agent-inject-secret-claude-credentials-2: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-claude-credentials-2: "0600" + vault.hashicorp.com/agent-inject-template-claude-credentials-2: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.execution_worker_2_claude_credentials_json }} + {{- end }} + vault.hashicorp.com/agent-inject-secret-codex-auth-0: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-codex-auth-0: "0600" + vault.hashicorp.com/agent-inject-template-codex-auth-0: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.execution_worker_0_codex_auth_json }} + {{- end }} + vault.hashicorp.com/agent-inject-secret-codex-auth-1: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-codex-auth-1: "0600" + vault.hashicorp.com/agent-inject-template-codex-auth-1: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.execution_worker_1_codex_auth_json }} + {{- end }} + vault.hashicorp.com/agent-inject-secret-codex-auth-2: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-perms-codex-auth-2: "0600" + vault.hashicorp.com/agent-inject-template-codex-auth-2: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.execution_worker_2_codex_auth_json }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 2m + vault.hashicorp.com/agent-requests-mem: 16Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi + spec: + serviceAccountName: hermes-execution-worker + priorityClassName: scavenger + automountServiceAccountToken: false + enableServiceLinks: false + terminationGracePeriodSeconds: 30 + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - {key: kubernetes.io/arch, operator: In, values: [arm64]} + - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]} + - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]} + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - {key: node-role.kubernetes.io/accelerator, operator: Exists} + - weight: 50 + preference: + matchExpressions: + - {key: hardware, operator: In, values: [rpi5]} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: hermes-execution-worker + topologyKey: kubernetes.io/hostname + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: hermes-execution-worker + initContainers: + - name: stage-worker-access + image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 + imagePullPolicy: IfNotPresent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-worker] + env: + - {name: HERMES_WORKER_ROOT, value: /workspace} + - {name: HERMES_PROVIDER_ACCESS_ROOT, value: /provider-access} + - name: HERMES_WORKER_ORDINAL + valueFrom: + fieldRef: + fieldPath: metadata.labels['apps.kubernetes.io/pod-index'] + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: workspace, mountPath: /workspace} + - {name: provider-access, mountPath: /provider-access} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + resources: + requests: {cpu: 2m, memory: 32Mi} + limits: {cpu: 100m, memory: 64Mi} + - name: install-provider-clis + image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 + imagePullPolicy: IfNotPresent + command: [/bin/sh, -ec] + args: + - | + tools=/worker-data/tools + mkdir -p "${tools}/bin" + if [ ! -f "${tools}/.cli-versions-0.147.0-2.1.226" ]; then + npm install --global --omit=dev --no-audit --no-fund --prefix "${tools}" \ + @openai/codex@0.147.0 @anthropic-ai/claude-code@2.1.226 + touch "${tools}/.cli-versions-0.147.0-2.1.226" + fi + test -x "${tools}/bin/codex" + test -x "${tools}/bin/claude" + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: tools, mountPath: /worker-data/tools} + resources: + requests: {cpu: 5m, memory: 64Mi} + limits: {cpu: "1", memory: 1Gi} + containers: + - name: execution-worker + image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 + imagePullPolicy: IfNotPresent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_worker.py] + env: + - {name: HERMES_HOME, value: /worker-data} + - {name: HERMES_WORKER_ROOT, value: /workspace} + - {name: HOME, value: /worker-data/home} + - {name: CODEX_HOME, value: /provider-access/codex} + - {name: CLAUDE_CONFIG_DIR, value: /provider-access/claude} + - {name: HERMES_AUTO_ROUTER_PROFILE, value: agent} + - {name: PYTHONPATH, value: /opt/hermes} + - {name: PATH, value: /worker-data/tools/bin:/opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin} + - name: HERMES_WORKER_ORDINAL + valueFrom: + fieldRef: + fieldPath: metadata.labels['apps.kubernetes.io/pod-index'] + - name: HERMES_WORKER_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + startupProbe: + exec: + command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json && test -w /provider-access/claude/.credentials.json"] + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + exec: + command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json"] + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: workspace, mountPath: /workspace} + - {name: worker-data, mountPath: /worker-data} + - {name: tools, mountPath: /worker-data/tools, readOnly: true} + - {name: provider-access, mountPath: /provider-access} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 5m, memory: 128Mi, ephemeral-storage: 1Gi} + limits: {cpu: "2", memory: 4Gi, ephemeral-storage: 8Gi} + volumes: + - name: worker-data + emptyDir: {sizeLimit: 128Mi} + - name: tools + emptyDir: {sizeLimit: 1Gi} + - name: coordinator + configMap: + name: hermes-execution-pool + defaultMode: 0555 + - name: tmp + emptyDir: {sizeLimit: 2Gi} + - name: vault-auth-token + projected: + defaultMode: 0600 + sources: + - serviceAccountToken: + audience: vault + expirationSeconds: 3600 + path: token + - configMap: + name: kube-root-ca.crt + items: + - {key: ca.crt, path: ca.crt} + - downwardAPI: + items: + - {path: namespace, fieldRef: {fieldPath: metadata.namespace}} + volumeClaimTemplates: + - metadata: + name: workspace + labels: + app: hermes-execution-worker + spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: + requests: + storage: 30Gi + - metadata: + name: provider-access + labels: + app: hermes-execution-worker + spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: + requests: + storage: 1Gi diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 2c2efe77..6becbaf6 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -41,6 +41,13 @@ resources: - oauth2-proxy.yaml - agent-certificate.yaml - agent-ingress.yaml + - execution-worker-rbac.yaml + - execution-worker-statefulset.yaml + - execution-mediator.yaml + - execution-worker-networkpolicy.yaml + +patches: + - path: execution-coordinator-patch.yaml configMapGenerator: - name: hermes-chat-oauth-templates @@ -55,6 +62,36 @@ configMapGenerator: - OPERATOR-RUNBOOK.md=NOTES.md options: disableNameSuffixHash: true + - name: hermes-execution-pool + namespace: hermes + files: + - cli_lane_board.py=scripts/cli_lane_board.py + - cli_lane_capabilities.py=scripts/cli_lane_capabilities.py + - cli_lane_config.py=scripts/cli_lane_config.py + - cli_lane_dispatch.py=scripts/cli_lane_dispatch.py + - cli_lane_evidence.py=scripts/cli_lane_evidence.py + - cli_lane_execution.py=scripts/cli_lane_execution.py + - cli_lane_files.py=scripts/cli_lane_files.py + - cli_lane_finalization.py=scripts/cli_lane_finalization.py + - cli_lane_goal.py=scripts/cli_lane_goal.py + - cli_lane_prompt.py=scripts/cli_lane_prompt.py + - cli_lane_provider.py=scripts/cli_lane_provider.py + - cli_lane_quarantine.py=scripts/cli_lane_quarantine.py + - cli_lane_records.py=scripts/cli_lane_records.py + - cli_lane_recovery.py=scripts/cli_lane_recovery.py + - cli_lane_retention.py=scripts/cli_lane_retention.py + - cli_lane_routing.py=scripts/cli_lane_routing.py + - cli_lane_runner.py=scripts/cli_lane_runner.py + - execution_pool_protocol.py=scripts/execution_pool_protocol.py + - execution_pool_project.py=scripts/execution_pool_project.py + - execution_pool_coordinator.py=scripts/execution_pool_coordinator.py + - execution_pool_server.py=scripts/execution_pool_server.py + - execution_pool_client.py=scripts/execution_pool_client.py + - execution_pool_worker.py=scripts/execution_pool_worker.py + - execution_pool_scm.py=scripts/execution_pool_scm.py + - gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py + - scm_broker_client.py=scm-common/scripts/scm_broker_client.py + - stage_runtime_access.py=scripts/stage_runtime_access.py - name: hermes-coordinator namespace: hermes files: diff --git a/services/hermes/scm-common/kustomization.yaml b/services/hermes/scm-common/kustomization.yaml index 17dc1da6..f2d222b5 100644 --- a/services/hermes/scm-common/kustomization.yaml +++ b/services/hermes/scm-common/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization configMapGenerator: - - name: hermes-scm-boundary + - name: hermes-scm-boundary-v2 files: - gitea_api.py=scripts/gitea_api.py - gitea_api_policy.py=scripts/gitea_api_policy.py diff --git a/services/hermes/scm-common/scripts/scm_broker.py b/services/hermes/scm-common/scripts/scm_broker.py index a0c4043c..75d6d508 100644 --- a/services/hermes/scm-common/scripts/scm_broker.py +++ b/services/hermes/scm-common/scripts/scm_broker.py @@ -7,7 +7,6 @@ import argparse import base64 import json import re -import socket import tempfile import time import urllib.error @@ -45,7 +44,8 @@ GIT_PATH_RE = re.compile( r"(?Pinfo/refs|git-upload-pack|git-receive-pack)\Z" ) FEATURE_REF_RE = re.compile( - r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z" + r"refs/heads/(?:(?:feature|fix|chore|docs|test|refactor|wt|review|hermes|handoff)/" + r"[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z" ) @@ -96,7 +96,7 @@ def _spool_bounded( """Copy a fixed-length exchange through a bounded-memory disk spool.""" if not 0 <= length <= maximum: raise PolicyError("SCM request exceeds the safe size limit") - spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns lifetime max_size=SPOOL_MEMORY_LIMIT, dir="/tmp" ) remaining = length @@ -228,7 +228,7 @@ def _receive_prefix(body: bytes | BinaryIO) -> bytes: def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None: - """Permit only creation of new, namespaced feature branches.""" + """Permit only creation of new branches in reviewed namespaces.""" body = _receive_prefix(body) _reject_credential_bytes(body, token, "Git request") position = 0 @@ -247,13 +247,11 @@ def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None: command = body[position + 4 : position + size].rstrip(b"\n") command = command.split(b"\x00", 1)[0] fields = command.split(b" ") - if len(fields) != 3 or not all( - re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2] - ): + if len(fields) != 3 or not all(re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2]): raise PolicyError("Git receive-pack ref command is invalid") old_sha, new_sha, raw_ref = fields if old_sha != ZERO_SHA or new_sha == ZERO_SHA: - raise PolicyError("Git broker permits only new feature-branch creation") + raise PolicyError("Git broker permits only new namespaced branch creation") try: ref = raw_ref.decode("ascii") except UnicodeDecodeError as exc: @@ -333,9 +331,7 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): if len(self.headers.get_all("Content-Length", [])) > 1: raise PolicyError("SCM request has duplicate Content-Length") - def _stream( - self, status: int, content_type: str, body: BinaryIO, length: int - ) -> None: + def _stream(self, status: int, content_type: str, body: BinaryIO, length: int) -> None: self.send_response(status) self.send_header("Content-Type", content_type) self.send_header("Content-Length", str(length)) @@ -391,13 +387,7 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): self._stream(200, expected, body, length) finally: body.close() - except ( - OSError, - PolicyError, - socket.timeout, - urllib.error.URLError, - ValueError, - ): + except (TimeoutError, OSError, PolicyError, urllib.error.URLError, ValueError): self._reject() def do_POST(self) -> None: @@ -409,9 +399,9 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): else: self._git_rpc() except ( + TimeoutError, OSError, PolicyError, - socket.timeout, urllib.error.URLError, ValueError, json.JSONDecodeError, @@ -427,9 +417,7 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): result = read(data["path"], token=token) else: expected = {"base", "body", "head", "head_sha", "repo", "title"} - if set(data) != expected or not all( - isinstance(data[key], str) for key in expected - ): + if set(data) != expected or not all(isinstance(data[key], str) for key in expected): raise PolicyError("draft request fields are invalid") result = create_draft(token=token, **data) # type: ignore[arg-type] if token.encode("utf-8") in result: @@ -438,15 +426,10 @@ class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler): def _git_rpc(self) -> None: repo, operation, service = _git_target(self.path) - if ( - operation not in {"git-upload-pack", "git-receive-pack"} - or service != operation - ): + if operation not in {"git-upload-pack", "git-receive-pack"} or service != operation: raise PolicyError("Git RPC operation is outside the allowlist") expected_request = f"application/x-{service}-request" - if self.headers.get_content_type() != expected_request or self.headers.get( - "Transfer-Encoding" - ): + if self.headers.get_content_type() != expected_request or self.headers.get("Transfer-Encoding"): raise PolicyError("Git RPC request type is invalid") length = _content_length(self.headers, MAX_GIT_REQUEST) token = read_token() diff --git a/services/hermes/scripts/cli_lane_dispatch.py b/services/hermes/scripts/cli_lane_dispatch.py index 8a740dbf..0a059911 100644 --- a/services/hermes/scripts/cli_lane_dispatch.py +++ b/services/hermes/scripts/cli_lane_dispatch.py @@ -7,6 +7,7 @@ import concurrent.futures import os import sys import time +from collections.abc import Callable from typing import Any from cli_lane_board import _external, _record_board_access_error, _task_value @@ -24,6 +25,16 @@ from cli_lane_recovery import _has_pending_finalization, recover_pending_finaliz from cli_lane_retention import maybe_gc_lane_artifacts +OWNED_WORKSPACES_ONLY = os.environ.get( + "HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY", "" +).strip().lower() in {"1", "true", "yes", "on"} + + +def _owns_local_workspace(task: Any) -> bool: + """Keep legacy/dirty worktree tasks on their existing single-host owner.""" + return bool(str(_task_value(task, "workspace_path", "") or "").strip()) + + def _board_slug(board: Any) -> str: if isinstance(board, dict): return str(board.get("slug") or board.get("id") or "") @@ -59,7 +70,11 @@ def recover_orphans() -> None: continue try: for task in kanban_db.list_tasks(conn): - if _external(task) and str(_task_value(task, "status", "")) == "running": + if ( + _external(task) + and str(_task_value(task, "status", "")) == "running" + and (not OWNED_WORKSPACES_ONLY or _owns_local_workspace(task)) + ): task_id = str(_task_value(task, "id")) run_id = _task_value(task, "current_run_id", None) if not isinstance(run_id, int): @@ -78,7 +93,11 @@ def recover_orphans() -> None: finally: conn.close() -def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str]]: +def claim_ready( + active: set[tuple[str, str]], + limit: int, + eligible: Callable[[str, Any], bool] | None = None, +) -> list[tuple[str, str]]: """Atomically claim external ready tasks across all non-archived boards.""" from hermes_cli import kanban_db @@ -113,6 +132,8 @@ def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str or (board, task_id) in active or not assignee.startswith(EXTERNAL_PREFIX) or str(_task_value(task, "status", "")) != "ready" + or (OWNED_WORKSPACES_ONLY and not _owns_local_workspace(task)) + or (eligible is not None and not eligible(board, task)) ): continue try: diff --git a/services/hermes/scripts/cli_lane_evidence.py b/services/hermes/scripts/cli_lane_evidence.py index d85abb0e..529c1fd7 100644 --- a/services/hermes/scripts/cli_lane_evidence.py +++ b/services/hermes/scripts/cli_lane_evidence.py @@ -56,8 +56,12 @@ def _retire_terminal_entry( board_descriptor: int, quarantine_descriptor: int, authority_name: str | None = None, + source_descriptor: int | None = None, ) -> str: """Retire only the inode previously inspected; preserve any replacement.""" + pinned = os.fstat(source_descriptor) if source_descriptor is not None else source_stat + if pinned.st_dev != source_stat.st_dev or pinned.st_ino != source_stat.st_ino: + return "replacement" try: current = os.stat( path.name, @@ -66,7 +70,7 @@ def _retire_terminal_entry( ) except FileNotFoundError: return "missing" - if current.st_dev != source_stat.st_dev or current.st_ino != source_stat.st_ino: + if current.st_dev != pinned.st_dev or current.st_ino != pinned.st_ino: return "replacement" # A replacement can itself be staged repeatedly while recovery races a # writer. Keep every generation bound to the original canonical pending @@ -76,7 +80,7 @@ def _retire_terminal_entry( (authority_name or path.name).encode("utf-8") ).hexdigest()[:16] source_digest = hashlib.sha256( - f"{source_stat.st_dev:x}\0{source_stat.st_ino:x}".encode() + f"{pinned.st_dev:x}\0{pinned.st_ino:x}".encode() ).hexdigest()[:16] for sequence in range(32): staging = f".retire.{path_digest}.{source_digest}.{sequence}" @@ -97,7 +101,7 @@ def _retire_terminal_entry( dir_fd=quarantine_descriptor, follow_symlinks=False, ) - if staged.st_dev == source_stat.st_dev and staged.st_ino == source_stat.st_ino: + if staged.st_dev == pinned.st_dev and staged.st_ino == pinned.st_ino: os.unlink(staging, dir_fd=quarantine_descriptor) os.fsync(quarantine_descriptor) os.fsync(board_descriptor) @@ -132,6 +136,7 @@ def _write_json_noreplace(path: Path, value: dict[str, Any]) -> bool: temporary = f".{path.name}.{uuid.uuid4().hex}.tmp" temporary_stat = None descriptor = None + cleanup_descriptor = None try: descriptor = os.open(temporary, create_flags, 0o600, dir_fd=directory) temporary_stat = os.fstat(descriptor) @@ -143,6 +148,7 @@ def _write_json_noreplace(path: Path, value: dict[str, Any]) -> bool: written = os.fstat(stream.fileno()) if stat.S_IMODE(written.st_mode) != 0o600 or written.st_nlink != 1: raise OSError("terminal evidence is not private and singly linked") + cleanup_descriptor = os.dup(stream.fileno()) try: _rename_noreplace( temporary, @@ -164,7 +170,10 @@ def _write_json_noreplace(path: Path, value: dict[str, Any]) -> bool: temporary_stat, board_descriptor=directory, quarantine_descriptor=directory, + source_descriptor=cleanup_descriptor, ) + if cleanup_descriptor is not None: + os.close(cleanup_descriptor) os.close(directory) def _terminal_document_digest(document: dict[str, Any]) -> str: diff --git a/services/hermes/scripts/cli_lane_execution.py b/services/hermes/scripts/cli_lane_execution.py index 46a3d7b0..59344a4d 100644 --- a/services/hermes/scripts/cli_lane_execution.py +++ b/services/hermes/scripts/cli_lane_execution.py @@ -30,6 +30,12 @@ from cli_lane_recovery import _has_pending_finalization from cli_lane_routing import fresh_unavailable_provider, select_route +def _goal_rejections(state: dict[str, object]) -> list[object]: + """Return only bounded list-shaped prior judge evidence.""" + history = state.get("goal_rejections", []) + return history if isinstance(history, list) else [] + + def execute_claim(board: str, task_id: str) -> None: """Execute one already-claimed task and commit its outcome to Kanban.""" from hermes_cli import kanban_db @@ -260,16 +266,12 @@ def execute_claim(board: str, task_id: str) -> None: and goal_mode ): heartbeat("local goal-completion judge active") - rejection_history = state.get("goal_rejections", []) - if not isinstance(rejection_history, list): - rejection_history = [] - judge_context = context - if goal_turn > 1 or rejection_history: - judge_context += ( - "\n\nAuthoritative Hermes goal-controller evidence: " - f"current turn {goal_turn}/{goal_max_turns}; prior rejected " - f"reports: {json.dumps(rejection_history[-5:])}." - ) + rejection_history = _goal_rejections(state) + judge_context = context + ( + "\n\nAuthoritative Hermes goal-controller evidence: " + f"current turn {goal_turn}/{goal_max_turns}; prior rejected " + f"reports: {json.dumps(rejection_history[-5:])}." + ) accepted, judge_reason = cli_lane_goal.judge_goal_completion( judge_context, structured, @@ -338,9 +340,7 @@ def execute_claim(board: str, task_id: str) -> None: and deadline - time.monotonic() > 30 ) if can_continue: - rejection_history = state.get("goal_rejections", []) - if not isinstance(rejection_history, list): - rejection_history = [] + rejection_history = _goal_rejections(state) state["goal_rejections"] = [ *rejection_history[-4:], completion_problem, diff --git a/services/hermes/scripts/cli_lane_finalization.py b/services/hermes/scripts/cli_lane_finalization.py index ef39ffc5..b5468296 100644 --- a/services/hermes/scripts/cli_lane_finalization.py +++ b/services/hermes/scripts/cli_lane_finalization.py @@ -89,6 +89,7 @@ def _retire_snapshot( board_descriptor=snapshot.directory_descriptor, quarantine_descriptor=snapshot.directory_descriptor, authority_name=authority_name, + source_descriptor=snapshot.descriptor, ) def _retire_snapshot_after_db(path: Path, snapshot: TerminalSnapshot) -> str: diff --git a/services/hermes/scripts/cli_lane_quarantine.py b/services/hermes/scripts/cli_lane_quarantine.py index 8c599184..8de6d3ea 100644 --- a/services/hermes/scripts/cli_lane_quarantine.py +++ b/services/hermes/scripts/cli_lane_quarantine.py @@ -180,6 +180,7 @@ def _quarantine_terminal( source_stat, board_descriptor=board_descriptor, quarantine_descriptor=quarantine_descriptor, + source_descriptor=source_descriptor, ) except OSError as error: # A bad quarantine target must not make an attacker-controlled pending @@ -196,6 +197,7 @@ def _quarantine_terminal( if quarantine_descriptor is not None else board_descriptor ), + source_descriptor=source_descriptor, ) except OSError: retirement = "deferred" diff --git a/services/hermes/scripts/cli_lane_retention.py b/services/hermes/scripts/cli_lane_retention.py index c88df653..9d1b9245 100644 --- a/services/hermes/scripts/cli_lane_retention.py +++ b/services/hermes/scripts/cli_lane_retention.py @@ -50,18 +50,27 @@ def _unlink_artifact_if_same(path: Path, observed: os.stat_result) -> bool: directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) directory_flags |= getattr(os, "O_NOFOLLOW", 0) descriptor = None + source_descriptor = None try: descriptor = os.open(path.parent, directory_flags) + file_flags = getattr(os, "O_PATH", os.O_RDONLY) | getattr(os, "O_NOFOLLOW", 0) + source_descriptor = os.open(path.name, file_flags, dir_fd=descriptor) + pinned = os.fstat(source_descriptor) + if pinned.st_dev != observed.st_dev or pinned.st_ino != observed.st_ino: + return False outcome = _retire_terminal_entry( path, observed, board_descriptor=descriptor, quarantine_descriptor=descriptor, + source_descriptor=source_descriptor, ) return outcome == "retired" except OSError: return False finally: + if source_descriptor is not None: + os.close(source_descriptor) if descriptor is not None: os.close(descriptor) diff --git a/services/hermes/scripts/execution_pool_client.py b/services/hermes/scripts/execution_pool_client.py new file mode 100644 index 00000000..a1dc6622 --- /dev/null +++ b/services/hermes/scripts/execution_pool_client.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Ordinal-local signing and SCM mediation boundary for a model worker.""" + +from __future__ import annotations + +import json +import os +import threading +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler +from pathlib import Path +from typing import Any + +import cli_lane_goal +from execution_pool_protocol import ( + MAX_WIRE_BYTES, + PROTOCOL_VERSION, + BoundedHTTPServer, + ProtocolError, + canonical_json, + parse_wire, + read_key, + sign_envelope, + verify_envelope, +) +from execution_pool_scm import Boundary as SCMBoundary + + +KEY_PATH = Path( + os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/pool-access/execution-pool-key") +) +COORDINATOR = os.environ.get( + "HERMES_EXECUTION_POOL_URL", + "http://hermes-execution-pool.hermes.svc.cluster.local:9007", +).rstrip("/") +ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1")) +PORT = int(os.environ.get("HERMES_EXECUTION_CLIENT_PORT", "9009")) +RESULT_FIELDS = frozenset( + {"status", "summary", "changed_files", "tests_run", "artifacts", "findings", "blockers"} +) + + +def _binding(value: dict[str, Any]) -> dict[str, Any]: + return { + name: value[name] + for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt") + } + + +def _validate_result(payload: Any) -> dict[str, Any]: + if not isinstance(payload, dict): + raise ProtocolError("terminal result payload must be an object") + structured = payload.get("structured") + if not isinstance(structured, dict) or set(structured) != RESULT_FIELDS: + raise ProtocolError("terminal result fields do not match the reviewed schema") + if structured.get("status") not in cli_lane_goal.RESULT_STATUSES: + raise ProtocolError("terminal result status is invalid") + if not isinstance(structured.get("summary"), str) or not structured["summary"].strip(): + raise ProtocolError("terminal result summary is required") + for name in RESULT_FIELDS - {"status", "summary"}: + value = structured.get(name) + if not isinstance(value, list) or any(not isinstance(item, str) for item in value): + raise ProtocolError(f"terminal result {name} must be a text list") + return payload + + +class ClientBoundary: + """Keep HMAC and SCM authority outside the model-facing container.""" + + def __init__(self, key: bytes, scm: SCMBoundary | None = None): + self.key = key + self.scm = scm or SCMBoundary(key) + self.current: dict[str, Any] | None = None + self.lock = threading.RLock() + + def _post(self, path: str, envelope: dict[str, Any]) -> dict[str, Any]: + request = urllib.request.Request( + COORDINATOR + path, + data=canonical_json(envelope), + method="POST", + headers={"Content-Type": "application/json", "Cache-Control": "no-store"}, + ) + with urllib.request.urlopen(request, timeout=60) as response: + body = response.read(MAX_WIRE_BYTES + 1) + if len(body) > MAX_WIRE_BYTES: + raise ProtocolError("coordinator response exceeds the wire limit") + return verify_envelope(self.key, json.loads(body)) + + def poll(self) -> dict[str, Any]: + poll_binding = { + "board": "", + "task_id": "", + "run_id": "", + "worker_ordinal": ORDINAL, + "attempt": 0, + } + response = self._post( + "/v1/poll", + sign_envelope(self.key, "poll", poll_binding, {"ready": True}), + ) + with self.lock: + if response["kind"] == "ack": + self.current = None + return {"assignment": None} + if response["kind"] != "assignment" or response["worker_ordinal"] != ORDINAL: + raise ProtocolError("coordinator returned a foreign assignment") + checkout = self.scm.checkout(response) + self.current = response + assignment = { + **_binding(response), + "payload": response["payload"], + "workspace": checkout["workspace"], + "baseline_sha": checkout["baseline_sha"], + "protocol_version": PROTOCOL_VERSION, + } + return {"assignment": assignment} + + def _current_for(self, supplied: Any) -> tuple[dict[str, Any], dict[str, Any]]: + if not isinstance(supplied, dict): + raise ProtocolError("local request binding must be an object") + if self.current is None or supplied != _binding(self.current): + raise ProtocolError("local request does not own the current assignment") + return self.current, dict(supplied) + + def heartbeat(self, request: dict[str, Any]) -> dict[str, Any]: + payload = request.get("payload") + if not isinstance(payload, dict): + raise ProtocolError("heartbeat payload must be an object") + with self.lock: + _assignment, binding = self._current_for(request.get("binding")) + response = self._post( + "/v1/heartbeat", + sign_envelope(self.key, "heartbeat", binding, payload), + ) + if response["kind"] != "ack" or _binding(response) != binding: + raise ProtocolError("coordinator acknowledgement binding changed") + return {"ack": response["payload"]} + + def finish(self, request: dict[str, Any]) -> dict[str, Any]: + payload = _validate_result(request.get("payload")) + with self.lock: + assignment, binding = self._current_for(request.get("binding")) + structured = payload["structured"] + if structured["status"] == "completed" and int(payload.get("returncode", 1)) == 0: + submission = self.scm.submit(assignment, request) + pull = str(submission.get("pull_request") or "") + if pull and pull not in structured["artifacts"]: + structured["artifacts"].append(pull) + response = self._post( + "/v1/result", sign_envelope(self.key, "result", binding, payload) + ) + if response["kind"] != "ack" or _binding(response) != binding: + raise ProtocolError("coordinator acknowledgement binding changed") + if response["payload"].get("accepted"): + self.current = None + return {"ack": response["payload"], "structured": structured} + + +def handler_factory(boundary: ClientBoundary) -> type[BaseHTTPRequestHandler]: + class Handler(BaseHTTPRequestHandler): + server_version = f"hermes-execution-mediator/{PROTOCOL_VERSION}" + + def _reply(self, status: int, value: dict[str, Any]) -> None: + body = canonical_json(value) + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 + value = {"ready": True, "protocol_version": PROTOCOL_VERSION} + self._reply(200, value) if self.path == "/ready" else self._reply( + 404, {"error": "not found"} + ) + + def do_POST(self) -> None: # noqa: N802 + try: + length = int(self.headers.get("Content-Length", "0")) + request = ( + parse_wire(self.rfile.read(length)) + if 0 < length <= MAX_WIRE_BYTES + else None + ) + allowed = {"operation", "binding", "payload", "title", "body"} + if not request or set(request) - allowed: + raise ProtocolError("invalid local mediator request") + operation = str(request.get("operation") or "") + routes = { + "poll": boundary.poll, + "heartbeat": lambda: boundary.heartbeat(request), + "finish": lambda: boundary.finish(request), + } + if operation not in routes: + raise ProtocolError("unsupported local mediator operation") + self._reply(200, routes[operation]()) + except (ProtocolError, OSError, ValueError, urllib.error.URLError) as error: + self._reply(409, {"error": str(error)[:2000]}) + + def log_message(self, _format: str, *_arguments: Any) -> None: + return + + return Handler + + +def main() -> int: + if ORDINAL not in range(3): + raise SystemExit("HERMES_WORKER_ORDINAL must be 0, 1, or 2") + key = read_key(KEY_PATH) + BoundedHTTPServer( + ("0.0.0.0", PORT), + handler_factory(ClientBoundary(key)), + max_workers=4, + ).serve_forever() + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint + raise SystemExit(main()) diff --git a/services/hermes/scripts/execution_pool_coordinator.py b/services/hermes/scripts/execution_pool_coordinator.py new file mode 100644 index 00000000..6fc0f5a4 --- /dev/null +++ b/services/hermes/scripts/execution_pool_coordinator.py @@ -0,0 +1,473 @@ +#!/usr/bin/env python3 +"""Coordinator-only Kanban claim/finalize bridge for three Hermes workers.""" + +from __future__ import annotations + +import json +import os +import re +import sqlite3 +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler +from pathlib import Path +from typing import Any + +import cli_lane_goal +import cli_lane_dispatch +from cli_lane_config import canonical_run_id +from execution_pool_project import ( + ProjectPolicyError, + distributed_workspace_eligible, + resolve_assignment, +) +from execution_pool_protocol import ( + MAX_ACTIVITY_BYTES, + PoolStore, + ProtocolError, + derive_ordinal_key, + envelope_ordinal, + sign_envelope, + verify_envelope, +) + +REDACTIONS = ( + re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/-]{12,}"), + re.compile( + r"(?i)(authorization|token|secret|password|api[_-]?key|" + r"refresh[_-]?token|access[_-]?token)[\"']?\s*[:=]\s*[\"']?[^\s\"',;}]+" + ), + re.compile(r"\b(?:sk-ant-|sk-proj-|rt_)[A-Za-z0-9_-]{12,}\b"), + re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"), + re.compile(r"(?i)https://[^/@\s]+@scm\.bstein\.dev"), +) + + +def _task_value(task: Any, name: str, default: Any = None) -> Any: + return getattr(task, name, default) + + +def resolve_scm(task: Any, board: str = "titan-iac") -> tuple[str, str, str]: + """Resolve SCM only through the canonical board/project registry.""" + return resolve_assignment(board, task) + + +def assignment_payload( + kanban_db: Any, connection: Any, task: Any, board: str = "titan-iac" +) -> dict[str, Any]: + context = kanban_db.build_worker_context(connection, str(_task_value(task, "id"))) + if not isinstance(context, str): + context = json.dumps(context, default=str, sort_keys=True) + encoded = context.encode("utf-8") + if len(encoded) > 32 * 1024: + raise RuntimeError("Kanban worker context exceeds the 32KiB assignment limit") + repo_url, branch, base_branch = resolve_scm(task, board) + runtime = int(_task_value(task, "max_runtime_seconds", 0) or 12 * 60 * 60) + runtime = max(60, min(runtime, 12 * 60 * 60)) + return { + "context": context, + "assignee": str(_task_value(task, "assignee", "cli-auto") or "cli-auto"), + "repo_url": repo_url, + "branch": branch, + "base_branch": base_branch, + "max_runtime_seconds": runtime, + "deadline_unix": int(time.time()) + runtime, + "goal_mode": bool(_task_value(task, "goal_mode", False)), + "goal_max_turns": max(1, min(int(_task_value(task, "goal_max_turns", 1) or 1), 12)), + } + + +def sanitize_activity(value: Any) -> str: + """Bound and redact worker output before it reaches the shared UI log.""" + text = str(value or "").replace("\x00", "") + text = "".join(character for character in text if character in "\n\t" or ord(character) >= 32) + for pattern in REDACTIONS: + text = pattern.sub("[REDACTED]", text) + return text.encode("utf-8")[:MAX_ACTIVITY_BYTES].decode("utf-8", "ignore") + + +def _append_activity(kanban_db: Any, envelope: dict[str, Any]) -> None: + payload = envelope["payload"] + if not isinstance(payload, dict): + raise ProtocolError("heartbeat payload must be an object") + activity = sanitize_activity(payload.get("activity")) + if not activity: + return + path = Path(kanban_db.worker_log_path(envelope["task_id"], board=envelope["board"])) + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_symlink(): + raise ProtocolError("Kanban activity log must not be a symlink") + flags = os.O_APPEND | os.O_CREAT | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags, 0o600) + try: + os.write(descriptor, activity.encode("utf-8")) + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _append_terminal_activity(kanban_db: Any, record: dict[str, Any]) -> None: + """Append one terminal batch exactly once across result retries/restarts.""" + payload = record.get("result") + activity = payload.get("final_activity") if isinstance(payload, dict) else "" + activity = sanitize_activity(activity) + if not activity: + return + marker = f"\n[execution-pool-result:{record['result_digest']}]\n" + path = Path(kanban_db.worker_log_path(record["task_id"], board=record["board"])) + if path.is_symlink(): + raise ProtocolError("Kanban activity log must not be a symlink") + if path.exists(): + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + size = os.fstat(descriptor).st_size + os.lseek(descriptor, max(0, size - 64 * 1024), os.SEEK_SET) + if marker.encode() in os.read(descriptor, 64 * 1024): + return + finally: + os.close(descriptor) + _append_activity( + kanban_db, + {**record, "payload": {"activity": marker + activity}}, + ) + + +class Coordinator: + """Own all transitions between durable assignments and Hermes Kanban.""" + + def __init__(self, key: bytes, store: PoolStore): + self.master_key = key + self.store = store + self._kanban_lock = threading.RLock() + + def _key(self, envelope: dict[str, Any]) -> bytes: + return derive_ordinal_key(self.master_key, envelope_ordinal(envelope)) + + def _verify(self, envelope: dict[str, Any], kind: str) -> dict[str, Any]: + return verify_envelope(self._key(envelope), envelope, expected_kind=kind) + + def _sign( + self, kind: str, binding: dict[str, Any], payload: dict[str, Any] + ) -> dict[str, Any]: + key = derive_ordinal_key(self.master_key, int(binding["worker_ordinal"])) + return sign_envelope(key, kind, binding, payload) + + @staticmethod + def _binding(record: dict[str, Any]) -> dict[str, Any]: + return {name: record[name] for name in ( + "board", "task_id", "run_id", "worker_ordinal", "attempt" + )} + + def poll(self, envelope: dict[str, Any]) -> dict[str, Any]: + verified = self._verify(envelope, "poll") + record = self.store.offer(int(verified["worker_ordinal"])) + if record is None: + binding = {**self._binding({ + "board": "", "task_id": "", "run_id": "", + "worker_ordinal": verified["worker_ordinal"], "attempt": 0, + })} + return self._sign("ack", binding, {"assignment": None}) + return self._sign("assignment", self._binding(record), record["payload"]) + + def heartbeat(self, envelope: dict[str, Any]) -> dict[str, Any]: + verified = self._verify(envelope, "heartbeat") + accepted, duplicate = self.store.heartbeat(verified) + from hermes_cli import kanban_db + + payload = verified["payload"] + note = sanitize_activity(payload.get("note") if isinstance(payload, dict) else "")[:1000] + with self._kanban_lock, kanban_db.scoped_current_board(verified["board"]): + connection = kanban_db.connect(board=verified["board"]) + try: + run_id = canonical_run_id(verified["run_id"]) + if run_id is None: + raise ProtocolError("worker run ID is not a canonical SQLite run") + alive = kanban_db.heartbeat_worker( + connection, verified["task_id"], + note=note or "distributed worker active", + expected_run_id=run_id, + ) + if not alive: + raise ProtocolError("Kanban run no longer owns this worker") + if not duplicate: + _append_activity(kanban_db, verified) + finally: + connection.close() + return self._sign( + "ack", self._binding(verified), + {"accepted": accepted, "duplicate": duplicate}, + ) + + def result(self, envelope: dict[str, Any]) -> dict[str, Any]: + verified = self._verify(envelope, "result") + record, duplicate = self.store.accept_result(verified) + if record.get("state") == "result": + self.finalize(record) + return self._sign( + "ack", self._binding(verified), + {"accepted": True, "duplicate": duplicate}, + ) + + def finalize(self, record: dict[str, Any]) -> None: + """Apply a result only if this exact run remains authoritative.""" + from hermes_cli import kanban_db + + binding = self._binding(record) + run_id = canonical_run_id(binding["run_id"]) + if run_id is None: + self.store.finalize(binding, "stale") + return + payload = record.get("result") + if not isinstance(payload, dict): + raise ProtocolError("result payload must be an object") + structured = payload.get("structured") + if not isinstance(structured, dict): + structured = {} + with self._kanban_lock, kanban_db.scoped_current_board(binding["board"]): + connection = kanban_db.connect(board=binding["board"]) + try: + task = kanban_db.get_task(connection, binding["task_id"]) + current_run_id = canonical_run_id( + _task_value(task, "current_run_id", None) + ) if task is not None else None + if current_run_id is None or current_run_id != run_id: + self.store.finalize(binding, "stale") + return + _append_terminal_activity(kanban_db, record) + metadata = { + "executor": "distributed-execution-pool", + "worker_ordinal": binding["worker_ordinal"], + "attempt": binding["attempt"], + "node": str(payload.get("node") or "unknown")[:128], + "route": payload.get("route") if isinstance(payload.get("route"), dict) else {}, + "provider_sessions": payload.get("provider_sessions") + if isinstance(payload.get("provider_sessions"), dict) else {}, + "changed_files": structured.get("changed_files", []), + "tests_run": structured.get("tests_run", []), + "artifacts": structured.get("artifacts", []), + "findings": structured.get("findings", []), + "blockers": structured.get("blockers", []), + } + problem = cli_lane_goal.unfinished_result_reason(structured) + if ( + structured.get("status") == "completed" + and int(payload.get("returncode", 1)) == 0 + and problem is None + ): + changed = kanban_db.complete_task( + connection, binding["task_id"], + result=json.dumps(structured, sort_keys=True), + summary=str(structured.get("summary") or "Completed"), + metadata=metadata, expected_run_id=run_id, + ) + else: + reason = problem or "; ".join(map(str, structured.get("blockers", []))) + reason = reason or str(structured.get("summary") or "worker failed") + changed = kanban_db.block_task( + connection, binding["task_id"], reason=reason, + kind="transient" if payload.get("capacity_failure") else "capability", + expected_run_id=run_id, + ) + self.store.finalize(binding, "finalized" if changed else "stale") + finally: + connection.close() + + def recover_results(self) -> None: + for record in self.store.pending_results(): + try: + self.finalize(record) + except (OSError, sqlite3.Error) as error: + print(f"result recovery deferred: {error}", file=sys.stderr, flush=True) + + def expire_leases(self) -> None: + """Fence dead attempts; after bounded retries, surface and release the run.""" + from hermes_cli import kanban_db + + for record in self.store.expire_leases(): + if record.get("state") != "lease_failed": + continue + binding = self._binding(record) + run_id = canonical_run_id(binding["run_id"]) + if run_id is None: + self.store.finalize(binding, "stale") + continue + with self._kanban_lock, kanban_db.scoped_current_board(binding["board"]): + connection = kanban_db.connect(board=binding["board"]) + try: + changed = kanban_db.block_task( + connection, + binding["task_id"], + reason=( + "Distributed worker lease expired after " + f"{binding['attempt']} fenced attempts" + ), + kind="transient", + expected_run_id=run_id, + ) + self.store.finalize( + binding, "finalized" if changed else "stale" + ) + finally: + connection.close() + + def reconcile(self) -> None: + """Recover the narrow claim/assignment crash gaps without double execution.""" + from hermes_cli import kanban_db + + active = self.store.active_assignments() + active_runs = { + (record["board"], record["task_id"], record["run_id"]) + for record in active + } + for record in active: + try: + with kanban_db.scoped_current_board(record["board"]): + connection = kanban_db.connect(board=record["board"]) + try: + task = kanban_db.get_task(connection, record["task_id"]) + current_run_id = ( + canonical_run_id(_task_value(task, "current_run_id", None)) + if task + else None + ) + current = str(current_run_id or "") + status = str(_task_value(task, "status", "") or "") if task else "" + finally: + connection.close() + if current != record["run_id"] or status != "running": + self.store.finalize(record, "stale") + except (OSError, sqlite3.Error): + continue + ordinals = self.store.available_ordinals() + if not ordinals: + return + for raw_board in kanban_db.list_boards(include_archived=False): + board = cli_lane_dispatch._board_slug(raw_board) + if not board: + continue + with kanban_db.scoped_current_board(board): + connection = kanban_db.connect(board=board) + try: + tasks = kanban_db.list_tasks(connection) + for task in tasks: + task_id = str(_task_value(task, "id", "") or "") + raw_run_id = canonical_run_id( + _task_value(task, "current_run_id", None) + ) + run_id = str(raw_run_id or "") + assignee = str(_task_value(task, "assignee", "") or "") + if ( + not ordinals + or not task_id + or not run_id + or str(_task_value(task, "status", "")) != "running" + or not assignee.startswith("cli-") + or not distributed_workspace_eligible(task) + or (board, task_id, run_id) in active_runs + ): + continue + try: + payload = assignment_payload(kanban_db, connection, task, board) + except Exception as error: + kanban_db.block_task( + connection, task_id, + reason=( + "Distributed assignment recovery failed: " + f"{type(error).__name__}: {error}" + ), + kind="capability", expected_run_id=raw_run_id, + ) + continue + ordinal = ordinals.pop(0) + self.store.add( + { + "board": board, "task_id": task_id, + "run_id": run_id, "worker_ordinal": ordinal, + "attempt": 1, + }, + payload, + ) + finally: + connection.close() + + def dispatch(self) -> None: + """Claim up to the free ordinal count and materialize assignments.""" + from hermes_cli import kanban_db + + ordinals = self.store.available_ordinals() + if not ordinals: + return + claimed = cli_lane_dispatch.claim_ready( + set(), len(ordinals), + lambda _board, task: distributed_workspace_eligible(task), + ) + for ordinal, (board, task_id) in zip(ordinals, claimed, strict=False): + run_id = "" + database_run_id: int | None = None + try: + with kanban_db.scoped_current_board(board): + connection = kanban_db.connect(board=board) + try: + task = kanban_db.get_task(connection, task_id) + if task is None: + continue + if not distributed_workspace_eligible(task): + raw_run = canonical_run_id( + _task_value(task, "current_run_id", None) + ) + if raw_run is not None: + kanban_db.block_task( + connection, task_id, + reason=( + "Distributed claim fenced because an existing " + "workspace is owned by the local lane" + ), + kind="capability", + expected_run_id=raw_run, + ) + continue + raw_run = canonical_run_id( + _task_value(task, "current_run_id", None) + ) + if raw_run is None: + raise ProjectPolicyError("claimed task has no canonical run ID") + database_run_id = raw_run + run_id = str(raw_run) + payload = assignment_payload(kanban_db, connection, task, board) + finally: + connection.close() + binding = { + "board": board, "task_id": task_id, "run_id": run_id, + "worker_ordinal": ordinal, "attempt": 1, + } + self.store.add(binding, payload) + except Exception as error: + reason = f"Distributed assignment preparation failed: {type(error).__name__}: {error}" + with kanban_db.scoped_current_board(board): + connection = kanban_db.connect(board=board) + try: + kanban_db.block_task( + connection, task_id, reason=reason, kind="capability", + expected_run_id=database_run_id, + ) + finally: + connection.close() + + +def handler_factory(coordinator: Coordinator) -> type[BaseHTTPRequestHandler]: + """Compatibility export for tests and the mounted coordinator entry point.""" + from execution_pool_server import handler_factory as factory + + return factory(coordinator) + + +def main() -> int: + from execution_pool_server import run + + return run(Coordinator) + + +if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint + raise SystemExit(main()) diff --git a/services/hermes/scripts/execution_pool_project.py b/services/hermes/scripts/execution_pool_project.py new file mode 100644 index 00000000..d4b6f382 --- /dev/null +++ b/services/hermes/scripts/execution_pool_project.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Canonical Atlas project and Git-ref policy for distributed execution.""" + +from __future__ import annotations + +import json +import os +import re +import stat +import subprocess +from pathlib import Path +from typing import Any + + +DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data")) +PROJECT_ROOT = DATA_ROOT / "workspace/projects" +BOARD_ROOT = DATA_ROOT / "kanban/boards" +ATLAS_REPO = re.compile( + r"https://scm\.bstein\.dev/atlas/(?P[A-Za-z0-9][A-Za-z0-9_.-]{0,99})\.git\Z" +) +SAFE_PREFIXES = frozenset( + {"feature", "fix", "chore", "docs", "test", "refactor", "wt", "review", "hermes", "handoff"} +) +IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z") +GIT = "/usr/bin/git" +MAX_BOARD_BYTES = 64 * 1024 + + +class ProjectPolicyError(ValueError): + """Canonical project metadata or a requested ref failed closed.""" + + +def _task_value(task: Any, name: str, default: Any = None) -> Any: + return getattr(task, name, default) + + +def _run_git(workdir: Path, *arguments: str) -> str: + completed = subprocess.run( + [GIT, "-C", str(workdir), *arguments], + check=False, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=30, + env={ + "HOME": "/nonexistent", + "PATH": "/usr/bin:/bin", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + }, + ) + if completed.returncode: + raise ProjectPolicyError("canonical project Git metadata is unavailable") + return completed.stdout.strip() + + +def validate_branch(value: object, *, feature: bool) -> str: + """Validate a complete local branch name with Git and a reviewed namespace.""" + if not isinstance(value, str) or not value or len(value) > 200: + raise ProjectPolicyError("branch name exceeds the safe limit") + if not value.isascii() or value.startswith("-"): + raise ProjectPolicyError("branch name must be canonical ASCII") + prefix = value.split("/", 1)[0] + if feature and ("/" not in value or prefix not in SAFE_PREFIXES): + raise ProjectPolicyError("task branch is outside the reviewed namespace") + completed = subprocess.run( + [GIT, "check-ref-format", f"refs/heads/{value}"], + check=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + env={"PATH": "/usr/bin:/bin", "GIT_CONFIG_NOSYSTEM": "1"}, + ) + if completed.returncode: + raise ProjectPolicyError("branch name is not a valid Git ref") + return value + + +def _read_board(board: str) -> dict[str, Any]: + if not IDENTIFIER.fullmatch(board): + raise ProjectPolicyError("board slug is invalid") + path = BOARD_ROOT / board / "board.json" + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= MAX_BOARD_BYTES: + raise ProjectPolicyError("board registry entry is not a bounded regular file") + raw = os.read(descriptor, MAX_BOARD_BYTES + 1) + finally: + os.close(descriptor) + try: + value = json.loads(raw) + except (UnicodeError, json.JSONDecodeError) as error: + raise ProjectPolicyError("board registry entry is malformed") from error + if not isinstance(value, dict) or value.get("slug") != board or value.get("archived") is True: + raise ProjectPolicyError("board registry identity is unavailable or archived") + return value + + +def resolve_project(board: str) -> tuple[str, str, Path]: + """Resolve repo and base exclusively from the canonical board registry. + + A board may be created before its primary checkout. In that case the + Atlas repository naming contract and ``main`` are the registry defaults; + once a checkout exists, its credential-free origin and remote HEAD must + agree with that identity. This keeps a missing checkout from silently + routing work to some other project's repository. + """ + entry = _read_board(board) + raw_workdir = entry.get("default_workdir") + if not isinstance(raw_workdir, str) or not Path(raw_workdir).is_absolute(): + raise ProjectPolicyError("board default_workdir is missing") + workdir = Path(raw_workdir).resolve(strict=False) + project_root = PROJECT_ROOT.resolve(strict=True) + try: + workdir.relative_to(project_root) + except ValueError as error: + raise ProjectPolicyError("board workdir is outside the Atlas project registry") from error + remote = f"https://scm.bstein.dev/atlas/{board}.git" + if not ATLAS_REPO.fullmatch(remote): + raise ProjectPolicyError("board repository identity is invalid") + if not workdir.exists(): + return remote, "main", workdir + if not workdir.is_dir(): + raise ProjectPolicyError("board checkout is not a directory") + checkout_remote = _run_git(workdir, "remote", "get-url", "origin") + if "@" in checkout_remote or checkout_remote != remote: + raise ProjectPolicyError("board origin disagrees with the Atlas registry") + try: + base = _run_git(workdir, "symbolic-ref", "--short", "refs/remotes/origin/HEAD") + if not base.startswith("origin/"): + raise ProjectPolicyError("origin HEAD is not canonical") + base = base.removeprefix("origin/") + except ProjectPolicyError: + base = "main" + return remote, validate_branch(base, feature=False), workdir + + +def resolve_assignment(board: str, task: Any) -> tuple[str, str, str]: + """Resolve the exact repo/base and safe task branch for one board task.""" + repo, base, _workdir = resolve_project(board) + task_id = str(_task_value(task, "id", "") or "") + if not IDENTIFIER.fullmatch(task_id): + raise ProjectPolicyError("task identity is invalid") + branch = str(_task_value(task, "branch_name", "") or f"wt/{task_id}") + return repo, validate_branch(branch, feature=True), base + + +def distributed_workspace_eligible(task: Any) -> bool: + """Only pathless tasks migrate; an existing worktree stays with its local owner.""" + return not str(_task_value(task, "workspace_path", "") or "").strip() diff --git a/services/hermes/scripts/execution_pool_protocol.py b/services/hermes/scripts/execution_pool_protocol.py new file mode 100644 index 00000000..599279be --- /dev/null +++ b/services/hermes/scripts/execution_pool_protocol.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +"""Authenticated, bounded, restart-safe Hermes execution-pool protocol.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import os +import re +import sqlite3 +import stat +import threading +import time +import uuid +from http.server import ThreadingHTTPServer +from pathlib import Path +from typing import Any + + +MAX_WIRE_BYTES = 64 * 1024 +MAX_ACTIVITY_BYTES = 12 * 1024 +MAX_CLOCK_SKEW = 30 +MAX_ENVELOPE_LIFETIME = 300 +PROTOCOL_VERSION = 2 +IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") +KINDS = frozenset({"poll", "assignment", "heartbeat", "result", "ack"}) + + +class ProtocolError(ValueError): + """A request failed the authenticated pool contract.""" + + +class BoundedHTTPServer(ThreadingHTTPServer): + """Bound concurrent requests and slow clients on internal pool channels.""" + + daemon_threads = True + request_queue_size = 8 + + def __init__(self, *args: Any, max_workers: int = 8, **kwargs: Any): + self._slots = threading.BoundedSemaphore(max(1, min(max_workers, 16))) + super().__init__(*args, **kwargs) + + def get_request(self) -> tuple[Any, Any]: + request, address = super().get_request() + request.settimeout(15) + return request, address + + def process_request(self, request: Any, client_address: Any) -> None: + if not self._slots.acquire(blocking=False): + self.shutdown_request(request) + return + try: + super().process_request(request, client_address) + except Exception: + self._slots.release() + raise + + def process_request_thread(self, request: Any, client_address: Any) -> None: + try: + super().process_request_thread(request, client_address) + finally: + self._slots.release() + + +def canonical_json(value: Any) -> bytes: + """Encode one value deterministically for digests and signatures.""" + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") + + +def atomic_json(path: Path, value: dict[str, Any], mode: int = 0o600) -> None: + """Durably replace one bounded pool document without following symlinks.""" + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + if path.parent.is_symlink(): + raise ProtocolError("pool state directory must not be a symlink") + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(temporary, flags, mode) + try: + remaining = memoryview(json.dumps(value, indent=2, sort_keys=True).encode() + b"\n") + while remaining: + remaining = remaining[os.write(descriptor, remaining) :] + os.fsync(descriptor) + os.fchmod(descriptor, mode) + finally: + os.close(descriptor) + try: + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def payload_digest(payload: Any) -> str: + return hashlib.sha256(canonical_json(payload)).hexdigest() + + +def derive_ordinal_key(master: bytes, ordinal: int) -> bytes: + """Derive one cryptographically isolated worker authority from the pool root.""" + if ordinal not in range(3) or not 32 <= len(master) <= 4096: + raise ProtocolError("pool key derivation input is invalid") + context = f"hermes-execution-pool-v2:worker:{ordinal}".encode() + return hmac.new(master, context, hashlib.sha256).hexdigest().encode() + + +def envelope_ordinal(envelope: Any) -> int: + """Read only the bounded ordinal needed to select a verification key.""" + if not isinstance(envelope, dict): + raise ProtocolError("invalid pool message") + value = envelope.get("worker_ordinal") + if type(value) is not int or value not in range(3): + raise ProtocolError("worker binding is outside the pool") + return value + + +def read_key(path: Path) -> bytes: + """Read a private regular file without following a final symlink.""" + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as error: + raise ProtocolError(f"pool key is unavailable: {error}") from error + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077: + raise ProtocolError("pool key must be a private regular file") + value = os.read(descriptor, 4097).strip() + finally: + os.close(descriptor) + if len(value) < 32 or len(value) > 4096: + raise ProtocolError("pool key length is outside the safe range") + return value + + +def _identifier(name: str, value: Any, *, allow_empty: bool = False) -> str: + if not isinstance(value, str): + raise ProtocolError(f"invalid {name}") + text = value + if allow_empty and not text: + return text + if not IDENTIFIER.fullmatch(text): + raise ProtocolError(f"invalid {name}") + return text + + +def sign_envelope( + key: bytes, + kind: str, + binding: dict[str, Any], + payload: Any, + *, + now: int | None = None, + lifetime: int = 120, + delivery_id: str | None = None, +) -> dict[str, Any]: + """Bind a message to the exact Kanban run, ordinal, and attempt.""" + current = int(time.time()) if now is None else int(now) + lifetime = max(1, min(int(lifetime), MAX_ENVELOPE_LIFETIME)) + envelope = { + "version": PROTOCOL_VERSION, + "kind": kind, + "board": str(binding.get("board") or ""), + "task_id": str(binding.get("task_id") or ""), + "run_id": str(binding.get("run_id") or ""), + "worker_ordinal": int(binding.get("worker_ordinal", -1)), + "attempt": int(binding.get("attempt", 0)), + "delivery_id": delivery_id or str(uuid.uuid4()), + "issued_at": current, + "expires_at": current + lifetime, + "payload_digest": payload_digest(payload), + "payload": payload, + } + if kind not in KINDS: + raise ProtocolError("unsupported message kind") + unsigned = canonical_json(envelope) + if len(unsigned) > MAX_WIRE_BYTES: + raise ProtocolError("pool message exceeds the wire limit") + envelope["signature"] = hmac.new(key, unsigned, hashlib.sha256).hexdigest() + return envelope + + +def verify_envelope( + key: bytes, + envelope: Any, + *, + expected_kind: str | None = None, + now: int | None = None, +) -> dict[str, Any]: + """Verify structure, lifetime, digest, and HMAC before using a message.""" + if not isinstance(envelope, dict) or len(canonical_json(envelope)) > MAX_WIRE_BYTES: + raise ProtocolError("invalid or oversized pool message") + required = {"version", "kind", "board", "task_id", "run_id", "worker_ordinal", "attempt", "delivery_id", "issued_at", "expires_at", "payload_digest", "payload", "signature"} + if set(envelope) != required or envelope.get("version") != PROTOCOL_VERSION: + raise ProtocolError( + f"pool message fields do not match version {PROTOCOL_VERSION}" + ) + kind = envelope["kind"] + if not isinstance(kind, str): + raise ProtocolError("unexpected message kind") + if kind not in KINDS or (expected_kind and kind != expected_kind): + raise ProtocolError("unexpected message kind") + empty_binding = kind in {"poll", "ack"} + _identifier("board", envelope["board"], allow_empty=empty_binding) + _identifier("task_id", envelope["task_id"], allow_empty=empty_binding) + _identifier("run_id", envelope["run_id"], allow_empty=empty_binding) + _identifier("delivery_id", envelope["delivery_id"]) + numeric = tuple(envelope[name] for name in ("worker_ordinal", "attempt", "issued_at", "expires_at")) + if any(type(value) is not int for value in numeric): + raise ProtocolError("invalid numeric binding") + ordinal, attempt, issued, expires = numeric + if ordinal not in range(3) or attempt < 0: + raise ProtocolError("worker binding is outside the pool") + current = int(time.time()) if now is None else int(now) + if issued > current + MAX_CLOCK_SKEW or expires < current - MAX_CLOCK_SKEW: + raise ProtocolError("pool message is outside its validity window") + if expires <= issued or expires - issued > MAX_ENVELOPE_LIFETIME: + raise ProtocolError("pool message lifetime is invalid") + if envelope["payload_digest"] != payload_digest(envelope["payload"]): + raise ProtocolError("pool payload digest mismatch") + signature = str(envelope["signature"]) + unsigned = dict(envelope) + unsigned.pop("signature") + expected = hmac.new(key, canonical_json(unsigned), hashlib.sha256).hexdigest() + if not hmac.compare_digest(signature, expected): + raise ProtocolError("pool message authentication failed") + return envelope + + +def parse_wire(body: bytes) -> dict[str, Any]: + if not body or len(body) > MAX_WIRE_BYTES: + raise ProtocolError("empty or oversized request") + try: + value = json.loads(body) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ProtocolError("malformed JSON request") from error + if not isinstance(value, dict): + raise ProtocolError("request must be a JSON object") + return value + + +class PoolStore: + """Coordinator-owned durable assignments; never stores provider secrets.""" + + def __init__(self, path: Path, lease_seconds: int = 90): + self.path = path + self.lease_seconds = max(60, min(int(lease_seconds), 600)) + self._lock = threading.RLock() + path.parent.mkdir(parents=True, exist_ok=True) + self._initialize() + + def _connect(self) -> sqlite3.Connection: + connection = sqlite3.connect(self.path, timeout=10, isolation_level=None) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA journal_mode=WAL") + connection.execute("PRAGMA synchronous=FULL") + connection.execute("PRAGMA busy_timeout=10000") + return connection + + def _initialize(self) -> None: + with self._connect() as connection: + connection.executescript( + """ + CREATE TABLE IF NOT EXISTS assignments ( + board TEXT NOT NULL, task_id TEXT NOT NULL, run_id TEXT NOT NULL, + worker_ordinal INTEGER NOT NULL CHECK(worker_ordinal BETWEEN 0 AND 2), + attempt INTEGER NOT NULL, assignment_digest TEXT NOT NULL, + payload_json TEXT NOT NULL, state TEXT NOT NULL, + lease_until REAL NOT NULL DEFAULT 0, last_heartbeat REAL NOT NULL DEFAULT 0, + result_digest TEXT, result_json TEXT, created_at REAL NOT NULL, + updated_at REAL NOT NULL, PRIMARY KEY(board, task_id, run_id) + ); + CREATE UNIQUE INDEX IF NOT EXISTS one_live_assignment_per_worker + ON assignments(worker_ordinal) WHERE state IN ('assigned','running','result'); + CREATE TABLE IF NOT EXISTS deliveries ( + delivery_id TEXT PRIMARY KEY, kind TEXT NOT NULL, digest TEXT NOT NULL, + received_at REAL NOT NULL + ); + """ + ) + + @staticmethod + def _record(row: sqlite3.Row | None) -> dict[str, Any] | None: + if row is None: + return None + value = dict(row) + value["payload"] = json.loads(value.pop("payload_json")) + if value.get("result_json"): + value["result"] = json.loads(value["result_json"]) + return value + + def add(self, binding: dict[str, Any], payload: dict[str, Any]) -> bool: + """Create exactly one assignment for a claimed run and free ordinal.""" + now = time.time() + digest = payload_digest(payload) + values = ( + binding["board"], binding["task_id"], binding["run_id"], + binding["worker_ordinal"], binding["attempt"], digest, + canonical_json(payload).decode(), "assigned", now, now, + ) + with self._lock, self._connect() as connection: + try: + connection.execute("BEGIN IMMEDIATE") + connection.execute( + """INSERT INTO assignments + (board,task_id,run_id,worker_ordinal,attempt,assignment_digest, + payload_json,state,created_at,updated_at) + VALUES (?,?,?,?,?,?,?,?,?,?)""", + values, + ) + connection.commit() + return True + except sqlite3.IntegrityError as error: + connection.rollback() + existing = connection.execute( + """SELECT assignment_digest,worker_ordinal,attempt FROM assignments + WHERE board=? AND task_id=? AND run_id=?""", + values[:3], + ).fetchone() + if existing and tuple(existing) == ( + digest, binding["worker_ordinal"], binding["attempt"] + ): + return False + if existing: + raise ProtocolError("conflicting duplicate assignment") from error + raise ProtocolError("worker ordinal already has a live assignment") from error + + def available_ordinals(self) -> list[int]: + with self._connect() as connection: + rows = connection.execute( + "SELECT worker_ordinal FROM assignments WHERE state IN ('assigned','running','result')" + ).fetchall() + occupied = {int(row[0]) for row in rows} + return [ordinal for ordinal in range(3) if ordinal not in occupied] + + def active_assignments(self) -> list[dict[str, Any]]: + with self._connect() as connection: + rows = connection.execute( + "SELECT * FROM assignments WHERE state IN ('assigned','running','result')" + ).fetchall() + return [self._record(row) or {} for row in rows] + + def offer(self, ordinal: int) -> dict[str, Any] | None: + """Return the ordinal's durable assignment, preserving restart identity.""" + now = time.time() + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + """SELECT * FROM assignments WHERE worker_ordinal=? + AND state IN ('assigned','running') ORDER BY created_at LIMIT 1""", + (ordinal,), + ).fetchone() + if row is not None: + connection.execute( + """UPDATE assignments SET state='running',lease_until=?,last_heartbeat=?,updated_at=? + WHERE board=? AND task_id=? AND run_id=?""", + (now + self.lease_seconds, now, now, row["board"], row["task_id"], row["run_id"]), + ) + connection.commit() + return self._record(row) + + def _matching(self, connection: sqlite3.Connection, envelope: dict[str, Any]) -> sqlite3.Row: + row = connection.execute( + "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?", + (envelope["board"], envelope["task_id"], envelope["run_id"]), + ).fetchone() + if row is None: + raise ProtocolError("assignment is unknown or stale") + if int(row["worker_ordinal"]) != int(envelope["worker_ordinal"]): + raise ProtocolError("worker ordinal does not own this assignment") + if int(row["attempt"]) != int(envelope["attempt"]): + raise ProtocolError("assignment attempt is stale") + return row + + def heartbeat(self, envelope: dict[str, Any]) -> tuple[bool, bool]: + now = time.time() + delivery_digest = payload_digest( + { + name: envelope[name] + for name in ( + "kind", "board", "task_id", "run_id", "worker_ordinal", + "attempt", "payload_digest", + ) + } + ) + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + duplicate = connection.execute( + "SELECT digest FROM deliveries WHERE delivery_id=?", + (envelope["delivery_id"],), + ).fetchone() + row = self._matching(connection, envelope) + if row["state"] not in {"assigned", "running"}: + raise ProtocolError("assignment is no longer running") + if row["state"] == "running" and float(row["lease_until"]) < now: + raise ProtocolError("assignment lease expired") + if duplicate and duplicate[0] != delivery_digest: + raise ProtocolError("delivery identifier was reused") + if not duplicate: + connection.execute( + "INSERT INTO deliveries VALUES (?,?,?,?)", + (envelope["delivery_id"], "heartbeat", delivery_digest, now), + ) + connection.execute( + "UPDATE assignments SET state='running',lease_until=?,last_heartbeat=?,updated_at=? WHERE board=? AND task_id=? AND run_id=?", + (now + self.lease_seconds, now, now, envelope["board"], envelope["task_id"], envelope["run_id"]), + ) + connection.commit() + return True, bool(duplicate) + + def accept_result(self, envelope: dict[str, Any]) -> tuple[dict[str, Any], bool]: + now = time.time() + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + row = self._matching(connection, envelope) + digest = envelope["payload_digest"] + if row["result_digest"]: + if row["result_digest"] != digest: + raise ProtocolError("conflicting result for completed delivery") + connection.rollback() + return self._record(row) or {}, True + if row["state"] not in {"assigned", "running"}: + raise ProtocolError("assignment cannot accept a result") + if row["state"] == "running" and float(row["lease_until"]) < now: + raise ProtocolError("assignment lease expired") + connection.execute( + """UPDATE assignments SET state='result',result_digest=?,result_json=?,updated_at=? + WHERE board=? AND task_id=? AND run_id=?""", + (digest, canonical_json(envelope["payload"]).decode(), now, + envelope["board"], envelope["task_id"], envelope["run_id"]), + ) + connection.commit() + row = connection.execute( + "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?", + (envelope["board"], envelope["task_id"], envelope["run_id"]), + ).fetchone() + return self._record(row) or {}, False + + def pending_results(self) -> list[dict[str, Any]]: + with self._connect() as connection: + rows = connection.execute("SELECT * FROM assignments WHERE state='result' ORDER BY updated_at").fetchall() + return [self._record(row) or {} for row in rows] + + def expire_leases( + self, *, now: float | None = None, max_attempts: int = 3 + ) -> list[dict[str, Any]]: + """Fence expired attempts and re-offer or terminally release their ordinals.""" + current = time.time() if now is None else float(now) + maximum = max(1, min(int(max_attempts), 10)) + changed: list[dict[str, Any]] = [] + with self._lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + rows = connection.execute( + """SELECT * FROM assignments WHERE state='running' + AND lease_until > 0 AND lease_until < ? ORDER BY updated_at""", + (current,), + ).fetchall() + for row in rows: + if int(row["attempt"]) >= maximum: + state, attempt = "lease_failed", int(row["attempt"]) + else: + state, attempt = "assigned", int(row["attempt"]) + 1 + connection.execute( + """UPDATE assignments SET state=?,attempt=?,lease_until=0, + last_heartbeat=0,updated_at=? WHERE board=? AND task_id=? AND run_id=? + AND attempt=? AND state='running'""", + ( + state, attempt, current, row["board"], row["task_id"], + row["run_id"], row["attempt"], + ), + ) + updated = connection.execute( + "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?", + (row["board"], row["task_id"], row["run_id"]), + ).fetchone() + if updated is not None: + changed.append(self._record(updated) or {}) + connection.commit() + return changed + + def finalize(self, binding: dict[str, Any], state: str) -> None: + if state not in {"finalized", "stale"}: + raise ProtocolError("invalid terminal assignment state") + with self._lock, self._connect() as connection: + connection.execute( + "UPDATE assignments SET state=?,updated_at=? WHERE board=? AND task_id=? AND run_id=? AND worker_ordinal=? AND attempt=?", + (state, time.time(), *(binding[name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt"))), + ) + + def garbage_collect(self, retention_seconds: int) -> int: + cutoff = time.time() - max(3600, retention_seconds) + with self._lock, self._connect() as connection: + cursor = connection.execute( + "DELETE FROM assignments WHERE state IN ('finalized','stale') AND updated_at < ?", + (cutoff,), + ) + connection.execute("DELETE FROM deliveries WHERE received_at < ?", (cutoff,)) + return int(cursor.rowcount) diff --git a/services/hermes/scripts/execution_pool_scm.py b/services/hermes/scripts/execution_pool_scm.py new file mode 100644 index 00000000..6badfbb9 --- /dev/null +++ b/services/hermes/scripts/execution_pool_scm.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""Assignment-bound Git gate routed exclusively through the PR14 SCM broker.""" + +from __future__ import annotations + +import json +import os +import re +import stat +import subprocess +import threading +import urllib.parse +from pathlib import Path +from typing import Any + +import scm_broker_client +from execution_pool_project import ATLAS_REPO, validate_branch +from execution_pool_protocol import ProtocolError, atomic_json, verify_envelope + + +WORKSPACE_ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace")) +SCM_ROOT = Path(os.environ.get("HERMES_SCM_STATE_ROOT", "/scm-state")) +ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1")) +BROKER_ORIGIN = scm_broker_client.BROKER_ORIGIN.rstrip("/") +MAX_STATUS_BYTES = 4 * 1024 * 1024 + + +def _git_environment() -> dict[str, str]: + """Run Git without credentials, prompts, ambient config, or hook execution.""" + return { + "HOME": "/nonexistent", + "PATH": "/usr/bin:/bin", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + } + + +def _run(*arguments: str, cwd: Path | None = None, timeout: int = 300) -> str: + command = [ + "/usr/bin/git", + "-c", + "core.hooksPath=/dev/null", + "-c", + "core.fsmonitor=false", + *arguments, + ] + completed = subprocess.run( + command, + cwd=cwd, + env=_git_environment(), + stdin=subprocess.DEVNULL, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + if completed.returncode: + message = (completed.stderr or completed.stdout or "SCM operation failed")[-2000:] + raise RuntimeError(message.strip()) + if len(completed.stdout.encode()) > MAX_STATUS_BYTES: + raise ProtocolError("SCM command output exceeds the safe limit") + return completed.stdout.strip() + + +def _binding(envelope: dict[str, Any]) -> tuple[dict[str, Any], str, str, str]: + if envelope["kind"] != "assignment" or int(envelope["worker_ordinal"]) != ORDINAL: + raise ProtocolError("assignment does not belong to this worker ordinal") + payload = envelope.get("payload") + if not isinstance(payload, dict): + raise ProtocolError("assignment payload must be an object") + repo = str(payload.get("repo_url") or "") + match = ATLAS_REPO.fullmatch(repo) + if not match: + raise ProtocolError("assignment repository is outside Atlas") + try: + branch = validate_branch(payload.get("branch"), feature=True) + base = validate_branch(payload.get("base_branch"), feature=False) + except ValueError as error: + raise ProtocolError(str(error)) from error + return payload, match.group("repo"), branch, base + + +def _broker_repo(repo: str) -> str: + return f"{BROKER_ORIGIN}/git/atlas/{repo}.git" + + +def workspace_path(envelope: dict[str, Any]) -> Path: + """Derive a private ordinal path; no caller-provided path is accepted.""" + parts = tuple(str(envelope[name]) for name in ("board", "task_id", "run_id")) + identifier = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z") + if any(not identifier.fullmatch(part) for part in parts): + raise ProtocolError("assignment path binding is invalid") + if WORKSPACE_ROOT.is_symlink(): + raise ProtocolError("workspace root must not be a symlink") + workspace_root = WORKSPACE_ROOT.resolve() + root = WORKSPACE_ROOT / "runs" + if root.is_symlink(): + raise ProtocolError("workspace run root must not be a symlink") + root.mkdir(mode=0o700, parents=True, exist_ok=True) + root = root.resolve() + root.relative_to(workspace_root) + candidate = root.joinpath(*parts) + current = root + for part in parts[:-1]: + current /= part + if current.is_symlink(): + raise ProtocolError("workspace parent must not be a symlink") + current.mkdir(mode=0o700, exist_ok=True) + if candidate.is_symlink(): + raise ProtocolError("workspace must not be a symlink") + candidate.resolve(strict=False).relative_to(root) + return candidate + + +def _regular_text(path: Path, maximum: int) -> str: + descriptor = os.open( + path, os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0) + ) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or info.st_size > maximum: + raise ProtocolError("private SCM state is invalid") + raw = os.read(descriptor, maximum + 1) + finally: + os.close(descriptor) + try: + return raw.decode().strip() + except UnicodeError as error: + raise ProtocolError("private SCM state is malformed") from error + + +def _state_path(envelope: dict[str, Any]) -> Path: + if SCM_ROOT.is_symlink(): + raise ProtocolError("private SCM root must not be a symlink") + root = SCM_ROOT.resolve() + root.mkdir(mode=0o700, parents=True, exist_ok=True) + name = "-".join( + str(envelope[name]) for name in ("board", "task_id", "run_id") + ) + path = root / f"{name}.json" + if path.is_symlink(): + raise ProtocolError("private SCM state must not be a symlink") + path.resolve(strict=False).relative_to(root) + return path + + +def _workspace_identity(destination: Path, repo: str, branch: str) -> str: + """Validate the checkout using bounded Git plumbing with all hooks disabled.""" + if destination.is_symlink() or not (destination / ".git").is_dir(): + raise ProtocolError("workspace Git metadata is unavailable") + ref_path = destination / ".git/refs/heads" + for part in branch.split("/"): + ref_path /= part + if ref_path.is_symlink(): + raise ProtocolError("workspace branch ref must not be a symlink") + origin = _run("remote", "get-url", "origin", cwd=destination) + if origin != f"https://scm.bstein.dev/atlas/{repo}.git": + raise ProtocolError("workspace origin does not match assignment") + broker = _run("remote", "get-url", "hermes-broker", cwd=destination) + if broker != _broker_repo(repo): + raise ProtocolError("workspace broker remote does not match assignment") + current = _run("symbolic-ref", "--short", "HEAD", cwd=destination) + if current != branch: + raise ProtocolError("workspace branch does not match assignment") + head = _run("rev-parse", "--verify", "HEAD", cwd=destination) + if not re.fullmatch(r"[0-9a-f]{40,64}", head): + raise ProtocolError("workspace HEAD is invalid") + return head + + +class Boundary: + """The only process allowed to turn model output into an SCM/result handoff.""" + + def __init__(self, key: bytes): + self.key = key + self.lock = threading.RLock() + + def verify(self, raw: Any) -> dict[str, Any]: + return verify_envelope(self.key, raw, expected_kind="assignment") + + def checkout(self, envelope: dict[str, Any]) -> dict[str, Any]: + _payload, repo, branch, base = _binding(envelope) + destination = workspace_path(envelope) + state_path = _state_path(envelope) + with self.lock: + if (destination / ".git").exists(): + _workspace_identity(destination, repo, branch) + state = json.loads(_regular_text(state_path, 16 * 1024)) + baseline = state.get("baseline_sha") if isinstance(state, dict) else None + if not isinstance(baseline, str) or not re.fullmatch( + r"[0-9a-f]{40,64}", baseline + ): + raise ProtocolError("private SCM baseline is unavailable") + return {"workspace": str(destination), "baseline_sha": baseline} + if destination.exists() and any(destination.iterdir()): + raise ProtocolError("workspace is non-empty and unmanaged") + destination.parent.mkdir(parents=True, exist_ok=True) + broker = _broker_repo(repo) + try: + _run( + "clone", "--single-branch", "--branch", branch, "--no-tags", + broker, str(destination), timeout=900, + ) + except RuntimeError as error: + if destination.exists() and any(destination.iterdir()): + raise ProtocolError( + "failed branch checkout left unmanaged workspace state" + ) from error + if destination.exists(): + destination.rmdir() + _run( + "clone", "--single-branch", "--branch", base, "--no-tags", + broker, str(destination), timeout=900, + ) + _run("checkout", "-b", branch, cwd=destination) + _run( + "remote", "set-url", "origin", + f"https://scm.bstein.dev/atlas/{repo}.git", cwd=destination, + ) + _run("remote", "add", "hermes-broker", broker, cwd=destination) + _run("config", "user.name", "Hermes Execution Worker", cwd=destination) + _run("config", "user.email", "hermes@bstein.dev", cwd=destination) + baseline = _workspace_identity(destination, repo, branch) + atomic_json( + state_path, + {"baseline_sha": baseline, "repo": repo, "branch": branch}, + ) + return {"workspace": str(destination), "baseline_sha": baseline} + + @staticmethod + def _draft(repo: str, branch: str, base: str, head: str, title: str, body: str) -> str: + query = urllib.parse.urlencode( + {"state": "open", "head": f"atlas:{branch}", "limit": 10} + ) + existing = json.loads( + scm_broker_client.read(f"/api/v1/repos/atlas/{repo}/pulls?{query}") + ) + if isinstance(existing, list) and existing: + return str(existing[0].get("html_url") or "") + created = json.loads( + scm_broker_client.create_draft( + repo, + base=base, + head=branch, + head_sha=head, + title=title, + body=body, + ) + ) + return str(created.get("html_url") or "") + + def submit(self, envelope: dict[str, Any], request: dict[str, Any]) -> dict[str, Any]: + """Enforce clean/committed state, broker push, and reviewed draft creation.""" + _payload, repo, branch, base = _binding(envelope) + destination = workspace_path(envelope) + title = str(request.get("title") or f"Hermes task {envelope['task_id']}").strip() + body = str(request.get("body") or "Automated Hermes draft.") + if not title or len(title.encode()) > 512 or len(body.encode()) > 32 * 1024: + raise ProtocolError("pull-request metadata exceeds the safe limit") + with self.lock: + head = _workspace_identity(destination, repo, branch) + state = json.loads(_regular_text(_state_path(envelope), 16 * 1024)) + baseline = state.get("baseline_sha") if isinstance(state, dict) else "" + if not isinstance(baseline, str) or not re.fullmatch( + r"[0-9a-f]{40,64}", baseline + ): + raise ProtocolError("private SCM baseline is unavailable") + status = _run( + "status", "--porcelain=v1", "--untracked-files=all", cwd=destination + ) + if status: + raise ProtocolError("workspace has uncommitted or untracked files") + ahead = int(_run("rev-list", "--count", f"{baseline}..{head}", cwd=destination)) + if ahead <= 0: + return {"workspace": str(destination), "branch": branch, "pull_request": ""} + _run( + "push", "hermes-broker", f"HEAD:refs/heads/{branch}", + cwd=destination, timeout=900, + ) + pull = self._draft(repo, branch, base, head, title, body) + return {"workspace": str(destination), "branch": branch, "pull_request": pull} diff --git a/services/hermes/scripts/execution_pool_server.py b/services/hermes/scripts/execution_pool_server.py new file mode 100644 index 00000000..b2a01713 --- /dev/null +++ b/services/hermes/scripts/execution_pool_server.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Versioned HTTP and maintenance loop for the execution-pool coordinator.""" + +from __future__ import annotations + +import argparse +import os +import sqlite3 +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler +from pathlib import Path +from typing import Any + +from execution_pool_protocol import ( + MAX_WIRE_BYTES, + PROTOCOL_VERSION, + BoundedHTTPServer, + PoolStore, + ProtocolError, + canonical_json, + parse_wire, + read_key, +) + + +DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data")) +STATE_ROOT = DATA_ROOT / "execution-pool" +KEY_PATH = Path( + os.environ.get( + "HERMES_EXECUTION_POOL_KEY_FILE", "/runtime-access/execution-pool-key" + ) +) +PORT = int(os.environ.get("HERMES_EXECUTION_POOL_PORT", "9007")) +RETENTION_SECONDS = int( + os.environ.get("HERMES_EXECUTION_POOL_RETENTION_SECONDS", "1209600") +) + + +def handler_factory(coordinator: Any) -> type[BaseHTTPRequestHandler]: + class Handler(BaseHTTPRequestHandler): + server_version = f"hermes-execution-pool/{PROTOCOL_VERSION}" + + def _reply(self, status: int, value: dict[str, Any]) -> None: + body = canonical_json(value) + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 + if self.path != "/ready": + self._reply(404, {"error": "not found"}) + return + try: + coordinator.store.available_ordinals() + self._reply(200, {"ready": True, "version": PROTOCOL_VERSION}) + except (OSError, sqlite3.Error): + self._reply(503, {"ready": False, "version": PROTOCOL_VERSION}) + + def do_POST(self) -> None: # noqa: N802 + try: + length = int(self.headers.get("Content-Length", "0")) + if length <= 0 or length > MAX_WIRE_BYTES: + raise ProtocolError("invalid content length") + envelope = parse_wire(self.rfile.read(length)) + routes = { + "/v1/poll": coordinator.poll, + "/v1/heartbeat": coordinator.heartbeat, + "/v1/result": coordinator.result, + } + if self.path not in routes: + self._reply(404, {"error": "not found"}) + return + self._reply(200, routes[self.path](envelope)) + except ProtocolError as error: + self._reply(409, {"error": str(error)}) + except Exception as error: + print( + f"pool request failed: {type(error).__name__}: {error}", + file=sys.stderr, + flush=True, + ) + self._reply(503, {"error": "coordinator unavailable"}) + + def log_message(self, _format: str, *_arguments: Any) -> None: + return + + return Handler + + +def run(coordinator_type: type[Any]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--once", action="store_true") + args = parser.parse_args() + store = PoolStore(STATE_ROOT / "assignments.db") + coordinator = coordinator_type(read_key(KEY_PATH), store) + for operation in ( + coordinator.expire_leases, + coordinator.recover_results, + coordinator.reconcile, + coordinator.dispatch, + ): + operation() + if args.once: + return 0 + server = BoundedHTTPServer( + ("0.0.0.0", PORT), handler_factory(coordinator), max_workers=8 + ) + threading.Thread(target=server.serve_forever, daemon=True).start() + while True: + for operation in ( + coordinator.expire_leases, + coordinator.recover_results, + coordinator.reconcile, + coordinator.dispatch, + lambda: store.garbage_collect(RETENTION_SECONDS), + ): + try: + operation() + except Exception as error: + print( + f"pool maintenance deferred: {type(error).__name__}: {error}", + file=sys.stderr, + flush=True, + ) + time.sleep(5) diff --git a/services/hermes/scripts/execution_pool_worker.py b/services/hermes/scripts/execution_pool_worker.py new file mode 100644 index 00000000..c98dfd56 --- /dev/null +++ b/services/hermes/scripts/execution_pool_worker.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +"""Run one fenced Hermes assignment on an ordinal-scoped durable workspace.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import stat +import subprocess +import time +import urllib.error +import urllib.request +from dataclasses import asdict +from pathlib import Path +from typing import Any + +import cli_lane_runner +from execution_pool_protocol import ( + ProtocolError, + atomic_json, + canonical_json, +) + + +ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace")) +ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1")) +CLIENT = os.environ.get( + "HERMES_EXECUTION_CLIENT_URL", + f"http://hermes-execution-mediator-{ORDINAL}.hermes.svc.cluster.local:9009", +).rstrip("/") +NODE = os.environ.get("HERMES_WORKER_NODE", "unknown")[:128] +RUN_PART = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") +RETENTION_SECONDS = int(os.environ.get("HERMES_WORKER_RETENTION_SECONDS", "1209600")) + + +def _post(url: str, value: dict[str, Any], timeout: int = 60) -> dict[str, Any]: + request = urllib.request.Request( + url, data=canonical_json(value), method="POST", + headers={"Content-Type": "application/json", "Cache-Control": "no-store"}, + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + body = response.read(64 * 1024 + 1) + if len(body) > 64 * 1024: + raise ProtocolError("server response exceeds the wire limit") + document = json.loads(body) + if not isinstance(document, dict): + raise ProtocolError("server response is not an object") + return document + + +def _client(operation: str, **values: Any) -> dict[str, Any]: + response = _post(f"{CLIENT}/v1/client", {"operation": operation, **values}) + if response.get("error"): + raise ProtocolError(str(response["error"])) + return response + + +def _poll() -> dict[str, Any] | None: + assignment = _client("poll").get("assignment") + if assignment is None: + return None + if ( + not isinstance(assignment, dict) + or int(assignment["worker_ordinal"]) != ORDINAL + or int(assignment.get("protocol_version", 0)) != 2 + ): + raise ProtocolError("local boundary returned a foreign assignment") + return assignment + + +def _binding(assignment: dict[str, Any]) -> dict[str, Any]: + return {name: assignment[name] for name in ( + "board", "task_id", "run_id", "worker_ordinal", "attempt" + )} + + +def _state_path(assignment: dict[str, Any]) -> Path: + parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id")) + if any(not RUN_PART.fullmatch(part) for part in parts): + raise ProtocolError("assignment state path is invalid") + configured_root = ROOT / "session-state" + if configured_root.is_symlink(): + raise ProtocolError("assignment state root must not be a symlink") + root = configured_root.resolve() + path = root.joinpath(*parts).with_suffix(".json") + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_symlink() or any(parent.is_symlink() for parent in path.parents if parent != root.parent): + raise ProtocolError("assignment state path contains a symlink") + path.resolve(strict=False).relative_to(root) + return path + + +def _bind_provider_sessions(assignment: dict[str, Any]) -> None: + """Attach provider session directories to this exact durable task/run.""" + parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id")) + if any(not RUN_PART.fullmatch(part) for part in parts): + raise ProtocolError("provider session binding is invalid") + configured_root = ROOT / "provider-state" + if configured_root.is_symlink() or not configured_root.is_dir(): + raise ProtocolError("durable provider state root is unavailable") + provider_root = configured_root.resolve() + run_root = provider_root + for part in parts: + run_root /= part + if run_root.is_symlink(): + raise ProtocolError("provider session path contains a symlink") + run_root.mkdir(mode=0o700, exist_ok=True) + run_root.resolve().relative_to(provider_root) + home = run_root / "home" + if home.is_symlink(): + raise ProtocolError("provider HOME must not be a durable symlink") + home.mkdir(mode=0o700, parents=True, exist_ok=True) + runtime_home = cli_lane_runner.DATA_ROOT / "home" + if runtime_home.is_symlink(): + runtime_home.unlink() + elif runtime_home.exists(): + raise ProtocolError("provider HOME is not a task-bound symlink") + runtime_home.symlink_to(home) + (home / ".claude").mkdir(mode=0o700, exist_ok=True) + settings = home / ".claude/settings.json" + if not settings.exists(): + atomic_json(settings, {}, 0o600) + bindings = { + Path(os.environ.get("CODEX_HOME", "/runtime-access/codex")) / "sessions": + run_root / "codex/sessions", + Path(os.environ.get("CLAUDE_CONFIG_DIR", "/runtime-access/claude")) / "projects": + run_root / "claude/projects", + Path(os.environ.get("CLAUDE_CONFIG_DIR", "/runtime-access/claude")) / "session-env": + run_root / "claude/session-env", + Path(os.environ.get("CLAUDE_CONFIG_DIR", "/runtime-access/claude")) / "todos": + run_root / "claude/todos", + } + for runtime, durable in bindings.items(): + current = run_root + for part in durable.relative_to(run_root).parts: + current /= part + if current.is_symlink(): + raise ProtocolError("provider session path contains a symlink") + current.mkdir(mode=0o700, exist_ok=True) + current.resolve().relative_to(provider_root) + if runtime.is_symlink(): + runtime.unlink() + elif runtime.exists(): + raise ProtocolError(f"provider session path is not a symlink: {runtime.name}") + runtime.symlink_to(durable) + + +def _prompt(context: str, workspace: Path, binding: dict[str, Any]) -> str: + return f"""You are a durable coding worker managed by Hermes Kanban. + +Work only on this objective and its acceptance criteria: + +{context} + +Workspace: {workspace} +Run binding: board={binding['board']} task={binding['task_id']} run={binding['run_id']} worker={ORDINAL} + +Operate autonomously only inside this private assigned checkout. Inspect before editing, +preserve unrelated and untracked files, and fail closed rather than overwrite state you do +not understand. You have no Kubernetes identity and no SCM credential. Do not attempt to +read Secrets, mutate workloads, use exec/attach/port-forward, reach node roots, or bypass +the reviewed SCM boundary. Commit intended changes locally on the assigned feature branch; +the worker boundary handles the bounded push and draft pull request after validation. +Switchyard owns provider/model/effort selection and cross-provider fallback. + +Return a final JSON object matching the supplied schema. Use status=incomplete when work, +tests, commits, or verification remain. Use blocked only for a concrete task obstacle. +Completed must have no blockers. List changed files, tests, artifacts, findings, and blockers. +""" + + +def _read_activity(log_path: Path, offset: int) -> tuple[str, int]: + flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(log_path, flags) + except FileNotFoundError: + return "", offset + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode): + raise ProtocolError("worker log must be a regular file") + if info.st_size < offset: + offset = 0 + os.lseek(descriptor, offset, os.SEEK_SET) + value = os.read(descriptor, 12 * 1024) + next_offset = os.lseek(descriptor, 0, os.SEEK_CUR) + finally: + os.close(descriptor) + return value.decode("utf-8", "replace"), next_offset + + +def _git(workspace: Path, *arguments: str) -> str: + completed = subprocess.run( + ["git", "-C", str(workspace), *arguments], stdin=subprocess.DEVNULL, + text=True, capture_output=True, timeout=30, check=False, + env={**os.environ, "GIT_TERMINAL_PROMPT": "0", "GIT_ASKPASS": "/bin/false"}, + ) + if completed.returncode: + raise RuntimeError((completed.stderr or "Git inspection failed")[-1000:]) + return completed.stdout.strip() + + +def _bounded_result(value: dict[str, Any]) -> dict[str, Any]: + """Keep terminal evidence useful while fitting the authenticated wire cap.""" + bounded: dict[str, Any] = {} + for name in ("status", "summary"): + bounded[name] = str(value.get(name) or "")[:8000] + for name in ("changed_files", "tests_run", "artifacts", "findings", "blockers"): + items = value.get(name) + bounded[name] = [str(item)[:2000] for item in items[:100]] if isinstance(items, list) else [] + while len(canonical_json(bounded)) > 32 * 1024: + longest = max( + (name for name in bounded if isinstance(bounded[name], list) and bounded[name]), + key=lambda name: len(canonical_json(bounded[name])), + default="", + ) + if longest: + bounded[longest].pop() + else: + bounded["summary"] = bounded["summary"][: len(bounded["summary"]) // 2] + return bounded + + +def _refresh_assignment(binding: dict[str, Any]) -> dict[str, Any]: + fresh = _poll() + if fresh is None or _binding(fresh) != binding: + raise ProtocolError("assignment changed before SCM submission") + return fresh + + +def garbage_collect(now: float | None = None) -> int: + """Remove only clean, terminal, assignment-derived workspaces after retention.""" + current = time.time() if now is None else now + state_root = (ROOT / "session-state").resolve() + run_root = (ROOT / "runs").resolve() + removed = 0 + for state_file in state_root.glob("*/*/*.json") if state_root.is_dir() else (): + state = cli_lane_runner.load_json(state_file) + terminal_at = float(state.get("terminal_at") or 0) + if terminal_at <= 0 or current - terminal_at < max(3600, RETENTION_SECONDS): + continue + workspace = Path(str(state.get("workspace") or "")) + try: + workspace.resolve(strict=True).relative_to(run_root) + except (OSError, RuntimeError, ValueError): + continue + if workspace.is_symlink() or _git( + workspace, "status", "--porcelain=v1", "--untracked-files=all" + ): + continue + shutil.rmtree(workspace) + state_file.unlink() + removed += 1 + return removed + + +def execute(assignment: dict[str, Any]) -> None: + binding = _binding(assignment) + payload = assignment["payload"] + if not isinstance(payload, dict): + raise ProtocolError("assignment payload is invalid") + _bind_provider_sessions(assignment) + workspace = Path(str(assignment.get("workspace") or "")).resolve(strict=True) + workspace.relative_to((ROOT / "runs").resolve()) + state_file = _state_path(assignment) + state = cli_lane_runner.load_json(state_file) + state.update({**binding, "node": NODE, "workspace": str(workspace)}) + state.setdefault("baseline_sha", str(assignment.get("baseline_sha") or "")) + atomic_json(state_file, state) + log_path = state_file.with_suffix(".log") + offset = 0 + latest_route: dict[str, Any] = {} + + def heartbeat(note: str) -> bool: + nonlocal offset + try: + activity, offset = _read_activity(log_path, offset) + response = _client( + "heartbeat", binding=binding, + payload={ + "note": f"worker={ORDINAL} node={NODE} {note}"[:1000], + "activity": activity, "route": latest_route, + }, + ) + return bool(response.get("ack", {}).get("accepted")) + except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError): + return False + + context = str(payload.get("context") or "") + assignee = str(payload.get("assignee") or "cli-auto") + route = cli_lane_runner.select_route( + context, assignee, + exclude_provider=cli_lane_runner.fresh_unavailable_provider() + if assignee == "cli-auto" else None, + ) + latest_route = asdict(route) + if not heartbeat( + f"route={route.provider}/{route.model}/{route.effort}; assignment accepted" + ): + raise ProtocolError("assignment lease was lost before provider startup") + deadline = min( + int(payload.get("deadline_unix") or int(time.time()) + 60), + int(time.time()) + int(payload.get("max_runtime_seconds") or 60), + ) + result = cli_lane_runner.run_provider( + route, _prompt(context, workspace, binding), workspace, state, state_file, + log_path, heartbeat, max(1, deadline - int(time.time())), + ) + if result.capacity_failure and deadline - time.time() > 30: + alternate = "claude" if route.provider == "codex" else "codex" + fallback = cli_lane_runner.select_route( + context + "\nThe first provider failed from capacity or authentication.", + f"cli-{alternate}-{route.effort}", + ) + latest_route = asdict(fallback) + heartbeat(f"fallback={route.provider}->{fallback.provider}") + result = cli_lane_runner.run_provider( + fallback, + _prompt(context, workspace, binding) + + cli_lane_runner.git_handoff(workspace, result.output), + workspace, state, state_file, log_path, heartbeat, + max(1, deadline - int(time.time())), + ) + route = fallback + structured = dict(result.structured or {}) + for name in ("changed_files", "tests_run", "artifacts", "findings", "blockers"): + if not isinstance(structured.get(name), list): + structured[name] = [] + if structured.get("status") == "completed" and result.returncode == 0: + status = _git(workspace, "status", "--porcelain=v1", "--untracked-files=all") + if status: + structured["status"] = "incomplete" + structured["blockers"].append( + "Worker left uncommitted or untracked files; SCM submission failed closed." + ) + else: + baseline = str(state.get("baseline_sha") or "") + if not re.fullmatch(r"[0-9a-f]{40,64}", baseline): + raise RuntimeError("durable SCM baseline is missing or invalid") + _refresh_assignment(binding) + structured = _bounded_result(structured) + activity, _ = _read_activity(log_path, offset) + result_payload = { + "structured": structured, + "returncode": result.returncode, + "capacity_failure": result.capacity_failure, + "node": NODE, + "route": asdict(route), + "provider_sessions": { + "codex_thread_id": state.get("codex_thread_id"), + "claude_session_id": state.get("claude_session_id"), + }, + "final_activity": activity, + } + response = _client( + "finish", + binding=binding, + payload=result_payload, + title=str(structured.get("summary") or f"Hermes task {binding['task_id']}")[:240], + body=json.dumps(structured, indent=2, sort_keys=True)[:12000], + ) + if not response.get("ack", {}).get("accepted"): + raise ProtocolError("coordinator did not accept the terminal result") + state["terminal_at"] = time.time() + atomic_json(state_file, state) + + +def report_exception(assignment: dict[str, Any], error: Exception) -> bool: + """Surface one worker exception as an exact-run transient result.""" + binding = _binding(assignment) + detail = f"{type(error).__name__}: {error}"[:1500] + structured = { + "status": "blocked", + "summary": "Distributed worker failed before a safe handoff.", + "changed_files": [], + "tests_run": [], + "artifacts": [], + "findings": [], + "blockers": [detail], + } + payload = { + "structured": structured, + "returncode": 1, + # This is an execution-infrastructure failure, not an objective capability + # verdict. The exact-run result releases the ordinal for another task. + "capacity_failure": True, + "node": NODE, + "route": {}, + "provider_sessions": {}, + "final_activity": detail, + } + try: + return bool( + _client("finish", binding=binding, payload=payload).get("ack", {}).get( + "accepted" + ) + ) + except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError): + return False + + +def readiness() -> None: + if ORDINAL not in range(3): + raise ProtocolError("worker ordinal must be 0, 1, or 2") + for path in (ROOT, ROOT / "provider-state", cli_lane_runner.DATA_ROOT): + if not path.is_dir() or not os.access(path, os.W_OK): + raise ProtocolError(f"durable worker path is not writable: {path}") + for credential in ( + Path(os.environ.get("CODEX_HOME", "/runtime-access/codex")) / "auth.json", + Path(os.environ.get("CLAUDE_CONFIG_DIR", "/runtime-access/claude")) / ".credentials.json", + ): + if not credential.is_file() or not os.access(credential, os.W_OK): + raise ProtocolError(f"subscription credential is unavailable or read-only: {credential.name}") + cli_lane_runner.RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True) + atomic_json( + cli_lane_runner.RESULT_SCHEMA_PATH, cli_lane_runner.RESULT_SCHEMA, 0o644 + ) + _poll() + + +def main() -> int: + readiness() + while True: + assignment = None + try: + garbage_collect() + assignment = _poll() + if assignment is None: + time.sleep(5) + continue + execute(assignment) + except (OSError, RuntimeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as error: + surfaced = assignment is not None and report_exception(assignment, error) + print( + f"worker {ORDINAL} {'surfaced' if surfaced else 'deferred'}: " + f"{type(error).__name__}: {error}", + flush=True, + ) + time.sleep(10) + + +if __name__ == "__main__": # pragma: no cover - exercised by the container entrypoint + raise SystemExit(main()) diff --git a/services/hermes/scripts/stage_runtime_access.py b/services/hermes/scripts/stage_runtime_access.py index 07423d70..2d998dd8 100644 --- a/services/hermes/scripts/stage_runtime_access.py +++ b/services/hermes/scripts/stage_runtime_access.py @@ -4,53 +4,130 @@ from __future__ import annotations import argparse +import hashlib +import hmac import json import os +import stat +import uuid from pathlib import Path VAULT_ROOT = Path("/vault/secrets") RUNTIME_ROOT = Path("/runtime-access") PERSISTENT_HOME = Path("/opt/data/home") +WORKER_ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace")) +PROVIDER_ACCESS_ROOT = Path( + os.environ.get("HERMES_PROVIDER_ACCESS_ROOT", "/provider-access") +) +POOL_ACCESS_ROOT = Path(os.environ.get("HERMES_POOL_ACCESS_ROOT", "/pool-access")) OWNER_UID = 10000 OWNER_GID = 10000 def _owned_directory(path: Path) -> None: """Create one private directory owned by the unprivileged Hermes user.""" + if path.is_symlink(): + raise RuntimeError(f"private directory {path.name} must not be a symlink") path.mkdir(mode=0o700, parents=True, exist_ok=True) + if path.is_symlink() or not path.is_dir(): + raise RuntimeError(f"private directory {path.name} is invalid") path.chmod(0o700) os.chown(path, OWNER_UID, OWNER_GID) -def _copy_secret(source_name: str, destination: Path) -> str: - """Copy a non-empty Vault projection without logging its value.""" - value = (VAULT_ROOT / source_name).read_text(encoding="utf-8").strip() +def _read_secret(source_name: str) -> str: + path = VAULT_ROOT / source_name + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= 1024 * 1024: + raise RuntimeError(f"Vault projection {source_name} is invalid") + raw = os.read(descriptor, info.st_size + 1) + finally: + os.close(descriptor) + value = raw.decode("utf-8").strip() if not value: raise RuntimeError(f"Vault projection {source_name} is empty") - destination.write_text(value + "\n", encoding="utf-8") - destination.chmod(0o600) - os.chown(destination, OWNER_UID, OWNER_GID) return value +def _write_secret(destination: Path, value: str) -> None: + temporary = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.tmp") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(temporary, flags, 0o600) + try: + remaining = memoryview((value + "\n").encode()) + while remaining: + remaining = remaining[os.write(descriptor, remaining) :] + os.fsync(descriptor) + os.fchmod(descriptor, 0o600) + os.fchown(descriptor, OWNER_UID, OWNER_GID) + finally: + os.close(descriptor) + try: + os.replace(temporary, destination) + finally: + temporary.unlink(missing_ok=True) + + +def _copy_secret(source_name: str, destination: Path) -> str: + """Copy a non-empty Vault projection without logging its value.""" + value = _read_secret(source_name) + _write_secret(destination, value) + return value + + +def _validate_json_value(value: str, source_name: str, required: tuple[str, ...]) -> None: + """Validate a provider credential shape without exposing its value.""" + try: + document = json.loads(value) + except json.JSONDecodeError as error: + raise RuntimeError(f"credential {source_name} is not valid JSON") from error + current = document + for key in required: + if not isinstance(current, dict) or key not in current: + raise RuntimeError(f"credential {source_name} has an invalid shape") + current = current[key] + if not isinstance(current, str) or not current: + raise RuntimeError(f"credential {source_name} has an empty credential") + + def _validated_json(source_name: str, destination: Path, required: tuple[str, ...]) -> None: """Stage one credential document after checking its expected shape.""" value = _copy_secret(source_name, destination) try: - document = json.loads(value) - except json.JSONDecodeError: + _validate_json_value(value, source_name, required) + except RuntimeError: destination.unlink(missing_ok=True) raise - current = document - for key in required: - if not isinstance(current, dict) or key not in current: - destination.unlink(missing_ok=True) - raise RuntimeError(f"Vault projection {source_name} has an invalid shape") - current = current[key] - if not isinstance(current, str) or not current: - destination.unlink(missing_ok=True) - raise RuntimeError(f"Vault projection {source_name} has an empty credential") + + +def _bootstrap_json( + source_name: str, destination: Path, required: tuple[str, ...] +) -> None: + """Initialize one ordinal credential once, then preserve provider refreshes.""" + if destination.is_symlink(): + raise RuntimeError(f"durable credential {destination.name} must not be a symlink") + try: + descriptor = os.open( + destination, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + ) + except FileNotFoundError: + _validated_json(source_name, destination, required) + return + try: + info = os.fstat(descriptor) + if ( + not stat.S_ISREG(info.st_mode) + or info.st_mode & 0o077 + or not 0 < info.st_size <= 1024 * 1024 + ): + raise RuntimeError(f"durable credential {destination.name} is unsafe") + value = os.read(descriptor, info.st_size + 1).decode("utf-8").strip() + finally: + os.close(descriptor) + _validate_json_value(value, destination.name, required) def _link_noncredential_state( @@ -107,6 +184,7 @@ def stage_agent() -> None: _owned_directory(path) for name in ( "agent-api-key", + "execution-pool-key", "chat-relay-key", "node-ssh-private-key", "node-ssh-known-hosts", @@ -157,16 +235,77 @@ def stage_triage() -> None: _write_empty_auth_store() +def _durable_link(runtime: Path, durable: Path, name: str) -> None: + """Link one provider state path only after rejecting replacement symlinks.""" + durable.mkdir(mode=0o700, parents=True, exist_ok=True) + durable.chmod(0o700) + os.chown(durable, OWNER_UID, OWNER_GID) + target = runtime / name + if target.is_symlink() and target.resolve(strict=False) != durable.resolve(): + target.unlink() + elif target.exists() and not target.is_symlink(): + raise RuntimeError(f"refusing to replace provider state path: {target}") + if not target.exists(): + target.symlink_to(durable) + + +def stage_execution_worker() -> None: + """Bootstrap and preserve ordinal-private subscription refresh ownership.""" + provider_state = WORKER_ROOT / "provider-state" + for path in ( + provider_state, + PROVIDER_ACCESS_ROOT, + PROVIDER_ACCESS_ROOT / "claude", + PROVIDER_ACCESS_ROOT / "codex", + ): + _owned_directory(path) + ordinal = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1")) + if ordinal not in range(3): + raise RuntimeError("execution worker ordinal is invalid") + _bootstrap_json( + f"claude-credentials-{ordinal}", + PROVIDER_ACCESS_ROOT / "claude" / ".credentials.json", + ("claudeAiOauth", "refreshToken"), + ) + _bootstrap_json( + f"codex-auth-{ordinal}", + PROVIDER_ACCESS_ROOT / "codex" / "auth.json", + ("tokens", "refresh_token"), + ) + + +def stage_execution_mediator() -> None: + """Derive one ordinal HMAC authority in the isolated mediator Pod only.""" + _owned_directory(POOL_ACCESS_ROOT) + ordinal = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1")) + if ordinal not in range(3): + raise RuntimeError("execution mediator ordinal is invalid") + master = _read_secret("execution-pool-key").encode() + derived = hmac.new( + master, + f"hermes-execution-pool-v2:worker:{ordinal}".encode(), + hashlib.sha256, + ).hexdigest() + _write_secret(POOL_ACCESS_ROOT / "execution-pool-key", derived) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("mode", choices=("agent", "chat", "triage")) + parser.add_argument( + "mode", + choices=("agent", "chat", "triage", "execution-worker", "execution-mediator"), + ) args = parser.parse_args() if args.mode == "agent": stage_agent() elif args.mode == "chat": stage_chat() - else: + elif args.mode == "triage": stage_triage() + elif args.mode == "execution-worker": + stage_execution_worker() + else: + stage_execution_mediator() print("Runtime access staged from Vault.", flush=True) return 0 diff --git a/services/hermes/service.yaml b/services/hermes/service.yaml index 0fc8d939..3903b5f9 100644 --- a/services/hermes/service.yaml +++ b/services/hermes/service.yaml @@ -117,6 +117,37 @@ spec: --- apiVersion: v1 kind: Service +metadata: + name: hermes-execution-worker + namespace: hermes + labels: + app: hermes-execution-worker +spec: + clusterIP: None + publishNotReadyAddresses: true + selector: + app: hermes-execution-worker +--- +apiVersion: v1 +kind: Service +metadata: + name: hermes-execution-pool + namespace: hermes + labels: + app: hermes-agent +spec: + type: ClusterIP + publishNotReadyAddresses: true + selector: + app: hermes-agent + ports: + - name: http + port: 9007 + targetPort: execution-pool + protocol: TCP +--- +apiVersion: v1 +kind: Service metadata: name: hermes-local-image namespace: hermes diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index 612391c9..d6baabc5 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -223,6 +223,8 @@ write_policy_and_role "outline" "outline" "outline-vault" \ "outline/* shared/postmark-relay" "" write_policy_and_role "planka" "planka" "planka-vault" \ "planka/* shared/postmark-relay" "" +write_policy_and_role "hermes-execution-worker" "hermes" "hermes-execution-worker" \ + "hermes/agent-tokens" "" write_policy_and_role "bstein-dev-home" "bstein-dev-home" "bstein-dev-home,bstein-dev-home-vault-sync" \ "portal/* shared/chat-ai-keys-runtime shared/portal-e2e-client shared/postmark-relay mailu/mailu-initial-account-secret shared/harbor-pull" "" write_policy_and_role "gitea" "gitea" "gitea-vault" \ diff --git a/testing/tests/test_hermes_agent_access.py b/testing/tests/test_hermes_agent_access.py index d4138256..a932c56c 100644 --- a/testing/tests/test_hermes_agent_access.py +++ b/testing/tests/test_hermes_agent_access.py @@ -78,15 +78,16 @@ def test_owner_agent_has_pinned_dedicated_node_ssh_access(): "secretProviderClass": "hermes-node-ssh-access" } reconciler = pod["containers"][0]["args"][0] - assert "cat /vault/secrets/node-ssh-public-key" in reconciler - assert "grep -qxF" in reconciler - assert "for user in atlas oceanus" in reconciler - assert "/host-etc/passwd" in reconciler - assert "chown \"${uid}:${gid}\"" in reconciler - host_passwd = next( - item for item in pod["volumes"] if item["name"] == "host-passwd" - ) - assert host_passwd["hostPath"] == {"path": "/etc/passwd", "type": "File"} + assert "/opt/node-hardener/node_account_hardening.py" in reconciler + assert "--public-key-file /vault/secrets/node-ssh-public-key" in reconciler + assert "grep -qxF" not in reconciler + host_etc = next(item for item in pod["volumes"] if item["name"] == "host-etc") + assert host_etc["hostPath"] == {"path": "/etc", "type": "Directory"} + hardener = next(item for item in pod["volumes"] if item["name"] == "coordinator") + assert hardener["configMap"] == { + "name": "hermes-node-account-hardener", + "defaultMode": 0o555, + } def test_owner_agent_tracks_no_ssh_identity_or_host_key_material(): @@ -155,13 +156,16 @@ def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog(): rbac = [ item - for item in yaml.safe_load_all((HERMES / "agent-rbac.yaml").read_text()) + for item in yaml.safe_load_all((HERMES / "rbac.yaml").read_text()) if item ] - binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding") - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} - ] + bindings = [item for item in rbac if item["kind"] == "ClusterRoleBinding"] + assert bindings + assert all( + subject["name"] not in {"hermes-agent", "hermes-switchyard"} + for binding in bindings + for subject in binding["subjects"] + ) def test_switchyard_active_state_uses_a_relocatable_rwx_claim(): diff --git a/testing/tests/test_hermes_agent_security.py b/testing/tests/test_hermes_agent_security.py index cf23a2d0..f3410035 100644 --- a/testing/tests/test_hermes_agent_security.py +++ b/testing/tests/test_hermes_agent_security.py @@ -182,21 +182,54 @@ def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces() "ports": [{"protocol": "TCP", "port": 9010}], }, ] - assert isolation["spec"]["egress"] == [{}] - - -def test_owner_agent_has_cluster_admin_kubernetes_context(): - config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) - assert config["current-context"] == "atlas-owner" - assert config["contexts"][0]["context"]["namespace"] == "default" - rbac_path = HERMES / "agent-rbac.yaml" - documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] - binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding") - assert binding["roleRef"] == { - "apiGroup": "rbac.authorization.k8s.io", - "kind": "ClusterRole", - "name": "cluster-admin", - } - assert binding["subjects"] == [ - {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"} + egress = isolation["spec"]["egress"] + assert {} not in egress + broker = next( + rule + for rule in egress + if rule.get("ports") == [{"protocol": "TCP", "port": 9081}] + ) + assert broker["to"] == [ + { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "hermes-scm"} + }, + "podSelector": {"matchLabels": {"app": "hermes-scm-broker"}}, + } ] + namespace_rule = next( + rule + for rule in egress + if rule.get("to", [{}])[0].get("namespaceSelector", {}).get( + "matchExpressions" + ) + ) + assert namespace_rule["to"][0]["namespaceSelector"]["matchExpressions"] == [ + { + "key": "kubernetes.io/metadata.name", + "operator": "NotIn", + "values": ["gitea", "hermes-scm"], + } + ] + public = next( + rule + for rule in egress + if rule.get("to", [{}])[0].get("ipBlock", {}).get("cidr") == "0.0.0.0/0" + ) + assert "192.168.0.0/16" in public["to"][0]["ipBlock"]["except"] + + +def test_owner_agent_has_observer_kubernetes_context_without_cluster_admin(): + config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text()) + assert config["current-context"] == "atlas-observer" + assert config["contexts"][0]["context"]["namespace"] == "default" + rbac_path = HERMES / "rbac.yaml" + documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item] + bindings = [item for item in documents if item["kind"] == "ClusterRoleBinding"] + assert bindings + assert all(binding["roleRef"]["name"] != "cluster-admin" for binding in bindings) + assert all( + subject["name"] != "hermes-agent" + for binding in bindings + for subject in binding["subjects"] + ) diff --git a/testing/tests/test_hermes_auto_router.py b/testing/tests/test_hermes_auto_router.py index c0a31103..05af3b01 100644 --- a/testing/tests/test_hermes_auto_router.py +++ b/testing/tests/test_hermes_auto_router.py @@ -255,6 +255,7 @@ def test_provider_status_separates_observed_activity_from_plan_quota(monkeypatch monkeypatch.setattr(module, "_claude_account", lambda: { "authenticated": True, "plan": "max", "quota_reported": False, }) + monkeypatch.setattr(module, "_fresh_health", lambda _path: {}) payload = module.provider_status_payload() diff --git a/testing/tests/test_hermes_chat_config.py b/testing/tests/test_hermes_chat_config.py index 7dd0e21c..c964e3e1 100644 --- a/testing/tests/test_hermes_chat_config.py +++ b/testing/tests/test_hermes_chat_config.py @@ -56,16 +56,17 @@ def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle(): assert "Atlas organization has private visibility" in instructions assert "may be public or private" in instructions assert "do not infer a\nrepository's visibility" in instructions - assert "already supplied through `GIT_ASKPASS`" in instructions + assert "Atlas Git access is supplied by the isolated SCM\nbroker" in soul assert "Never call `kanban_show` without a known, non-empty task ID" in instructions assert "bounded ad-hoc inspection and acceptance checks may" in instructions assert "load implementation or TDD skills" in instructions assert "Never call `kanban_show` without\na known, non-empty task ID" in soul assert "must load a skill only when its workflow\nmaterially applies" in soul - assert "runtime-only `GIT_ASKPASS`" in soul - assert "`scm.bstein.dev` is Gitea, not GitHub" in instructions - assert "Never load or follow a GitHub/`gh`" in instructions - assert "/opt/coordinator/gitea_api.py METHOD /api/v1/..." in instructions + assert "no repository token is present in this pod" in soul + assert "`scm.bstein.dev` is Forgejo/Gitea, not GitHub" in instructions + assert "Never load or follow a\nGitHub/`gh`" in instructions + assert "use `/opt/scm/gitea_api.py`" in instructions + assert "never bypass the client with raw HTTP" in instructions assert "JENKINS_BASE_URL" in instructions assert "Do not\nuse `git reset --hard`" in instructions rendered = (HERMES / "agent-deployment.yaml").read_text() diff --git a/testing/tests/test_hermes_chat_provider_runtime.py b/testing/tests/test_hermes_chat_provider_runtime.py new file mode 100644 index 00000000..ebafe2ad --- /dev/null +++ b/testing/tests/test_hermes_chat_provider_runtime.py @@ -0,0 +1,196 @@ +"""Native provider health, API lineage, and runtime-image contracts.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +from testing.tests.test_hermes_chat_support import ( + HERMES, + ROOT, + _documents, + _load_broker_module, +) + + +def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch): + """Claude traffic must use the native first-party CLI subscription lane.""" + module = _load_broker_module( + "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch + ) + monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak") + monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak") + monkeypatch.setattr( + module, + "resolve_route", + lambda route: "claude-fable-5" if "/fable/" in route else route, + ) + + model, effort = module._route( + "route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}} + ) + + assert (model, effort) == ("claude-fable-5", "xhigh") + assert "ANTHROPIC_API_KEY" not in module._claude_environment() + assert "CLAUDE_API_KEY" not in module._claude_environment() + assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted") + + +def test_codex_native_health_overrides_historical_router_errors( + tmp_path: Path, monkeypatch +): + """Fresh first-party health is authoritative over old Switchyard probes.""" + plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py" + spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + health_path = tmp_path / "codex.json" + health_path.write_text( + json.dumps( + { + "state": "available", + "authenticated": True, + "transport": "codex-chatgpt-subscription", + } + ) + ) + monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path) + monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json") + monkeypatch.setattr( + module, + "_get_json", + lambda url: {"status": "ok"} + if url.endswith("/health") + else { + "models": { + "route/codex/terra/medium": { + "calls": 1, + "errors": 99, + "total_tokens": 12, + } + } + }, + ) + monkeypatch.setattr(module, "_codex_account", lambda: {}) + monkeypatch.setattr(module, "_claude_account", lambda: {}) + + codex = module.provider_status_payload()["providers"]["codex"] + + assert codex["errors"] == 99 + assert codex["state"] == "available" + assert codex["native_health"]["transport"] == "codex-chatgpt-subscription" + + +def test_api_session_patch_accepts_parent_lineage(tmp_path: Path): + """API-created workers must persist the originating Hermes session.""" + module_path = HERMES / "scripts" / "patch_api_server_sessions.py" + spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + source = tmp_path / "api_server.py" + destination = tmp_path / "patched.py" + source.write_text( + "prefix\n" + + module.BEFORE + + "middle\n" + + module.RUNS_BEFORE + + "run body\n" + + module.RUN_CLOSE_BEFORE + + module.RESPONSES_SESSION_BEFORE + + module.EVENT_CALLBACK_SIGNATURE_BEFORE + + "callback docstring and push helper\n" + + module.EVENT_CALLBACK_BODY_BEFORE + + "tool start body\n" + + module.EVENT_CALLBACK_END_BEFORE + + module.EVENT_CALLBACK_CALL_BEFORE + + module.RUN_SWEEP_BEFORE + + "suffix\n", + encoding="utf-8", + ) + + module.patch(source, destination) + patched = destination.read_text(encoding="utf-8") + + assert "X-Hermes-Parent-Session-Id" in patched + assert "parent_session_id=parent_session_id" in patched + assert "Parent session not found" in patched + assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched + assert "user_message.startswith(default_prefixes)" in patched + assert "session_parent_conflict" in patched + assert "X-Hermes-Conversation-Platform" in patched + assert "X-Hermes-Conversation-Title" in patched + assert 'conversation_platform != "telegram"' in patched + assert "db.record_gateway_session_peer(" in patched + assert 'display_name="Telegram"' in patched + assert "db.reopen_session(session_id)" in patched + assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched + assert "def _record_run_activity(" in patched + assert '"_thinking": "Hermes is reasoning"' in patched + assert '"run.started": "Worker started"' in patched + assert '"run.completed": "Worker completed"' in patched + assert '"reasoning.available": "Hermes finished a reasoning step"' in patched + assert '"subagent.progress": "Nested worker progress"' in patched + assert "redact_sensitive_text" in patched + assert 'getattr(os, "O_NOFOLLOW", 0)' in patched + assert "os.fchmod(fd, 0o600)" in patched + assert "session_id=session_id" in patched + assert 'self._record_run_activity(session_id, "run.started")' in patched + assert 'detail = tool_name if event_type in {' in patched + assert 'if event_type == "subagent.tool"' in patched + assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched + assert 'heartbeats.get(session_id, 0.0)' in patched + assert '"subagent.thinking",' in patched + assert "Stream retention and run lifetime are separate" in patched + assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched + assert patched.index("terminal_status = self._run_statuses") < patched.index( + "self._active_run_tasks.pop(run_id, None)" + ) + + +def test_switchyard_brokers_and_native_claude_lane_use_the_right_images(): + """Thin brokers stay small while native Claude runs beside owner auth.""" + dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers").read_text() + assert "httpx==0.28.1" in dockerfile + assert "worker_route_broker.py" in dockerfile + assert "routing_catalog.py" in dockerfile + + deployment = _documents(HERMES / "switchyard-deployment.yaml")[0] + containers = { + container["name"]: container + for container in deployment["spec"]["template"]["spec"]["containers"] + } + expected = ( + "registry.bstein.dev/bstein/hermes-switchyard-brokers@" + "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083" + ) + assert containers["worker-route-broker"]["image"] == expected + assert containers["classifier-broker"]["image"] == expected + assert "claude-oauth-broker" not in containers + + agent = _documents(HERMES / "agent-deployment.yaml")[0] + agent_containers = { + container["name"]: container + for container in agent["spec"]["template"]["spec"]["containers"] + } + for container_name in ("hermes", "terminal"): + container = agent_containers[container_name] + environment = {item["name"]: item["value"] for item in container["env"]} + mounts = {item["name"]: item for item in container["volumeMounts"]} + assert environment["HERMES_ROUTING_CATALOG_PATH"] == "/routing-catalog/catalog.json" + assert environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" + assert environment["HERMES_CLAUDE_HEALTH_PATH"] == "/opt/data/provider-health/claude.json" + assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog" + assert mounts["routing-catalog"]["readOnly"] is True + codex = agent_containers["codex-broker"] + codex_environment = {item["name"]: item["value"] for item in codex["env"]} + assert codex_environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json" + claude = agent_containers["claude-broker"] + assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@") + assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0] + assert any( + mount["name"] == "home" and mount["mountPath"] == "/opt/data" + for mount in claude["volumeMounts"] + ) diff --git a/testing/tests/test_hermes_chat_voice.py b/testing/tests/test_hermes_chat_voice.py index 180a17e4..e04f3908 100644 --- a/testing/tests/test_hermes_chat_voice.py +++ b/testing/tests/test_hermes_chat_voice.py @@ -254,11 +254,14 @@ def test_chat_image_generation_uses_private_owner_broker(): vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() assert ( '"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram ' - 'hermes/developer-keycloak hermes/developer-gitea ' - 'hermes/developer-harbor hermes/developer-jenkins ' + 'hermes/developer-keycloak hermes/developer-harbor hermes/developer-jenkins ' 'hermes/developer-ssh"' in vault_policy ) + assert ( + 'write_policy_and_role "hermes-scm-broker" "hermes-scm" ' + '"hermes-scm-broker"' in vault_policy + ) assert ( 'write_policy_and_role "hermes-node-ssh" "hermes" ' '"hermes-node-ssh-access"' in vault_policy diff --git a/testing/tests/test_hermes_cli_capabilities.py b/testing/tests/test_hermes_cli_capabilities.py index adc6e3ca..ddb67492 100644 --- a/testing/tests/test_hermes_cli_capabilities.py +++ b/testing/tests/test_hermes_cli_capabilities.py @@ -93,6 +93,9 @@ def test_capability_detection_requires_explicit_safety_keywords(): assert lanes.detect_kanban_capabilities(variadic).ready is False assert lanes._explicit_keyword(object(), "expected_run_id") is False + missing_completion = lanes.KanbanCapabilities(False, True, True) + assert missing_completion.deferred_features == ("exact-run-completion",) + def test_startup_health_moves_from_bounded_deferred_to_ready( tmp_path: Path, @@ -117,9 +120,10 @@ def test_startup_health_moves_from_bounded_deferred_to_ready( assert "compatibility deferred" in capsys.readouterr().err new_db = _db(patched=True) - new = lanes.kanban_capabilities(new_db) - lanes.initialize_kanban_capabilities(new_db, health_path=health) + lanes.kanban_capabilities(new_db) + new = lanes.initialize_kanban_capabilities(new_db, health_path=health) assert new.ready is True + assert lanes.kanban_capabilities(new_db) is new assert lanes.runtime_health(health)["state"] == "ready" diff --git a/testing/tests/test_hermes_cli_dispatch_runtime.py b/testing/tests/test_hermes_cli_dispatch_runtime.py index 74684105..8355b8c0 100644 --- a/testing/tests/test_hermes_cli_dispatch_runtime.py +++ b/testing/tests/test_hermes_cli_dispatch_runtime.py @@ -13,6 +13,8 @@ from testing.tests.test_hermes_cli_support import lanes def test_board_slug_and_connection_failure_paths(monkeypatch): + assert lanes._owns_local_workspace(SimpleNamespace(workspace_path=" /workspace ")) + assert not lanes._owns_local_workspace(SimpleNamespace(workspace_path="")) assert lanes._board_slug(SimpleNamespace(slug="cassandra")) == "cassandra" assert lanes._board_slug(SimpleNamespace(id="fallback")) == "fallback" failures = [] @@ -100,6 +102,41 @@ def test_claim_ready_handles_zero_limit_empty_boards_and_claim_errors(monkeypatc assert lanes.claim_ready(set(), 1) == [] +def test_dispatch_skips_unhealthy_ineligible_and_unclaimed_entries(monkeypatch): + class Connection: + def __init__(self, board): + self.board = board + + def close(self): + return None + + tasks = [ + SimpleNamespace(id="t_ineligible", status="ready", assignee="cli-auto"), + SimpleNamespace(id="t_unclaimed", status="ready", assignee="cli-auto"), + ] + db = SimpleNamespace( + list_boards=lambda **_kwargs: [ + {"slug": "offline"}, + {"slug": "cassandra"}, + ], + scoped_current_board=lambda _board: nullcontext(), + connect=lambda board: (_ for _ in ()).throw(OSError("offline")) + if board == "offline" + else Connection(board), + recompute_ready=lambda _conn: None, + list_tasks=lambda _conn: tasks, + claim_task=lambda *_args, **_kwargs: None, + ) + monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db)) + assert lanes.claim_ready( + set(), + 2, + eligible=lambda _board, task: task.id == "t_unclaimed", + ) == [] + monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0) + lanes.recover_orphans() + + class _Future: def __init__(self): self.polls = 0 diff --git a/testing/tests/test_hermes_cli_evidence_edges.py b/testing/tests/test_hermes_cli_evidence_edges.py index c28ff43f..6a954864 100644 --- a/testing/tests/test_hermes_cli_evidence_edges.py +++ b/testing/tests/test_hermes_cli_evidence_edges.py @@ -44,6 +44,7 @@ def test_retirement_reports_missing_replacement_collision_and_restoration( path = tmp_path / "pending" path.write_text("old", encoding="utf-8") source = path.stat() + source_descriptor = os.open(path, os.O_RDONLY) directory = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: path.unlink() @@ -52,6 +53,7 @@ def test_retirement_reports_missing_replacement_collision_and_restoration( source, board_descriptor=directory, quarantine_descriptor=directory, + source_descriptor=source_descriptor, ) == "missing" path.write_text("new", encoding="utf-8") @@ -60,8 +62,11 @@ def test_retirement_reports_missing_replacement_collision_and_restoration( source, board_descriptor=directory, quarantine_descriptor=directory, + source_descriptor=source_descriptor, ) == "replacement" + os.close(source_descriptor) + source_descriptor = os.open(path, os.O_RDONLY) replacement = path.stat() monkeypatch.setattr( lanes, @@ -73,8 +78,10 @@ def test_retirement_reports_missing_replacement_collision_and_restoration( replacement, board_descriptor=directory, quarantine_descriptor=directory, + source_descriptor=source_descriptor, ) == "collision" finally: + os.close(source_descriptor) os.close(directory) @@ -82,6 +89,7 @@ def test_retirement_preserves_a_postcheck_replacement(tmp_path: Path, monkeypatc path = tmp_path / "pending" path.write_text("old", encoding="utf-8") source = path.stat() + source_descriptor = os.open(path, os.O_RDONLY) directory = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) calls = [] @@ -102,8 +110,10 @@ def test_retirement_preserves_a_postcheck_replacement(tmp_path: Path, monkeypatc source, board_descriptor=directory, quarantine_descriptor=directory, + source_descriptor=source_descriptor, ) == "replacement" finally: + os.close(source_descriptor) os.close(directory) assert path.read_text(encoding="utf-8") == "replacement" @@ -133,6 +143,31 @@ def test_evidence_writer_bounds_payload_and_cleans_failed_temporary( monkeypatch.setattr(lanes, "_retire_terminal_entry", real_retire) +def test_evidence_writer_closes_unwrapped_and_rejects_unsafe_temporary_files( + tmp_path: Path, + monkeypatch, +): + destination = tmp_path / "evidence.json" + real_fdopen = lanes.os.fdopen + opened = [] + + def fail_fdopen(descriptor, *_args, **_kwargs): + opened.append(descriptor) + raise OSError("fdopen failed") + + monkeypatch.setattr(lanes.os, "fdopen", fail_fdopen) + with pytest.raises(OSError, match="fdopen failed"): + lanes._write_json_noreplace(destination, {"value": "bounded"}) + with pytest.raises(OSError): + os.fstat(opened[0]) + + monkeypatch.setattr(lanes.os, "fdopen", real_fdopen) + with monkeypatch.context() as unsafe: + unsafe.setattr(lanes.stat, "S_IMODE", lambda _mode: 0o644) + with pytest.raises(OSError, match="not private"): + lanes._write_json_noreplace(destination, {"value": "bounded"}) + + def test_evidence_paths_and_collision_validation_fail_closed( tmp_path: Path, monkeypatch, diff --git a/testing/tests/test_hermes_cli_execution_edges.py b/testing/tests/test_hermes_cli_execution_edges.py index 07b9035d..889218db 100644 --- a/testing/tests/test_hermes_cli_execution_edges.py +++ b/testing/tests/test_hermes_cli_execution_edges.py @@ -16,6 +16,13 @@ class _Connection: return None +def test_goal_rejection_history_is_list_bounded(): + execution = __import__("cli_lane_execution") + history = ["first"] + assert execution._goal_rejections({"goal_rejections": history}) is history + assert execution._goal_rejections({"goal_rejections": "invalid"}) == [] + + def test_missing_claim_is_a_noop(tmp_path: Path, monkeypatch): db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), @@ -64,12 +71,18 @@ def test_transient_callback_storage_errors_do_not_kill_provider( def provider(*args, **_kwargs): assert args[6]("still working") is True - return lanes.ProcessResult(1, "failed", None, False) + return lanes.ProcessResult( + 1, + "failed", + {"status": "incomplete", "blockers": ["synthetic provider failure"]}, + False, + ) monkeypatch.setattr(lanes, "run_provider", provider) lanes.execute_claim("cassandra", "t_callbacks") assert blocks and blocks[0]["kind"] == "capability" + assert blocks[0]["reason"] == "synthetic provider failure" def test_restart_handoff_survives_missing_prior_log(tmp_path: Path, monkeypatch): diff --git a/testing/tests/test_hermes_cli_fallback.py b/testing/tests/test_hermes_cli_fallback.py index e2a8141e..728c8fe7 100644 --- a/testing/tests/test_hermes_cli_fallback.py +++ b/testing/tests/test_hermes_cli_fallback.py @@ -53,6 +53,10 @@ def test_goal_card_continues_after_local_judge_rejects_progress( ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") + lanes.atomic_json( + lanes.state_path("cassandra", "t_goal"), + {"goal_rejections": ["prior incomplete report"]}, + ) claude_low = lanes.Route( "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () ) diff --git a/testing/tests/test_hermes_cli_finalization_edges.py b/testing/tests/test_hermes_cli_finalization_edges.py index 0f2c2a2c..f8210243 100644 --- a/testing/tests/test_hermes_cli_finalization_edges.py +++ b/testing/tests/test_hermes_cli_finalization_edges.py @@ -159,6 +159,14 @@ def test_terminal_finalizer_closes_supplied_invalid_snapshots( monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") path = lanes._terminal_path(lanes.state_path("board", "task"), 8) + lanes.atomic_json(path, {}) empty = _Snapshot(None) assert lanes._finalize_terminal_record(_db(None), path, snapshot=empty) == "invalid" assert empty.closed == 1 + + finalization = __import__("cli_lane_finalization") + direct = _Snapshot(None) + assert finalization._finalize_terminal_record( + _db(None), path, snapshot=direct + ) == "invalid" + assert direct.closed == 1 diff --git a/testing/tests/test_hermes_cli_foundation_coverage.py b/testing/tests/test_hermes_cli_foundation_coverage.py index 71f0c44a..83fda691 100644 --- a/testing/tests/test_hermes_cli_foundation_coverage.py +++ b/testing/tests/test_hermes_cli_foundation_coverage.py @@ -3,6 +3,8 @@ from __future__ import annotations import json +import os +import runpy import sqlite3 from contextlib import nullcontext from pathlib import Path @@ -11,7 +13,7 @@ from urllib.error import URLError import pytest -from testing.tests.test_hermes_cli_support import _completed_result, lanes +from testing.tests.test_hermes_cli_support import SCRIPTS, _completed_result, lanes def test_board_context_serializes_objects_and_storage_failure_is_bounded( @@ -93,6 +95,26 @@ def test_json_and_terminal_path_helpers_fail_closed(tmp_path: Path, monkeypatch) ) assert lanes._terminal_evidence_identity(missing_evidence) is None + with pytest.raises(ValueError, match="positive SQLite"): + lanes.TerminalIdentity("board", "task", 0, "pending") + + +def test_recovery_snapshot_close_is_idempotent_after_descriptors_are_released( + tmp_path: Path, +): + directory = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + snapshot = lanes.TerminalRecoverySnapshot( + None, + tmp_path.stat(), + None, + directory, + b"", + "synthetic", + ) + snapshot.close() + snapshot.close() + assert snapshot.directory_descriptor == -1 + def test_goal_helpers_cover_compaction_and_deterministic_failures(): compacted = lanes.cli_lane_goal._bounded("a" * 100, 40) @@ -122,6 +144,9 @@ def test_goal_helpers_cover_compaction_and_deterministic_failures(): ) assert accepted is False assert reason + assert lanes.cli_lane_goal.unfinished_result_reason( + {"status": "completed", "summary": "done", "tests_run": "not-a-list"} + ) is None def test_prompt_artifact_and_json_helpers_reject_unsafe_inputs( @@ -147,6 +172,31 @@ def test_prompt_artifact_and_json_helpers_reject_unsafe_inputs( assert lanes._extract_json('prefix {"status":"blocked"} suffix') == { "status": "blocked" } + assert lanes._extract_json('{"status":"unknown"}') is None + + state_file = workspace / "state.json" + state = {} + assert lanes._event_payload( + "codex", + json.dumps({"type": "thread.started", "result": {"status": "blocked"}}), + state, + state_file, + ) == {"status": "blocked"} + assert lanes._event_payload( + "claude", + json.dumps({"session_id": "session-2", "text": "not-json"}), + state, + state_file, + ) is None + assert state["claude_session_id"] == "session-2" + + prompt_module = __import__("cli_lane_prompt") + assert prompt_module.workspace_artifacts( + workspace, [str(directory), str(artifact)] + ) == [str(artifact.resolve())] + assert prompt_module._event_payload( + "other", '{"item": {}}', {}, state_file + ) is None def test_provider_event_parsing_persists_sessions_and_nested_results( @@ -205,3 +255,11 @@ def test_switchyard_network_failure_is_explicit(): URLError("offline") ), ) + + +def test_runner_main_guard_delegates_to_dispatch(monkeypatch): + dispatch = __import__("cli_lane_dispatch") + monkeypatch.setattr(dispatch, "main", lambda: 17) + with pytest.raises(SystemExit) as raised: + runpy.run_path(str(SCRIPTS / "cli_lane_runner.py"), run_name="__main__") + assert raised.value.code == 17 diff --git a/testing/tests/test_hermes_cli_lanes_configuration.py b/testing/tests/test_hermes_cli_lanes_configuration.py index bf1807f4..8489be89 100644 --- a/testing/tests/test_hermes_cli_lanes_configuration.py +++ b/testing/tests/test_hermes_cli_lanes_configuration.py @@ -20,6 +20,8 @@ from testing.tests.test_hermes_cli_lanes_support import ( policy, ) +import cli_lane_execution as lane_execution + def test_workspace_preparation_failure_durably_blocks_the_claim( tmp_path: Path, monkeypatch @@ -43,7 +45,7 @@ def test_workspace_preparation_failure_durably_blocks_the_claim( ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) monkeypatch.setattr( - lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json" + lane_execution, "state_path", lambda _board, _task_id: tmp_path / "state.json" ) lanes.execute_claim("cassandra", "t_bad_worktree") @@ -102,21 +104,21 @@ def test_restart_provider_change_includes_explicit_workspace_handoff( block_task=lambda *_args, **_kwargs: None, ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: state_file) + monkeypatch.setattr(lane_execution, "state_path", lambda _board, _task_id: state_file) monkeypatch.setattr( - lanes, + lane_execution, "select_route", lambda *_args, **_kwargs: lanes.Route( "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () ), ) monkeypatch.setattr( - lanes, + lane_execution, "git_handoff", lambda _workspace, output: f"HANDOFF:{output}", ) monkeypatch.setattr( - lanes, + lane_execution, "run_provider", lambda _route, prompt, *_args, **_kwargs: ( prompts.append(prompt) diff --git a/testing/tests/test_hermes_cli_lanes_kanban.py b/testing/tests/test_hermes_cli_lanes_kanban.py deleted file mode 100644 index 279b74c5..00000000 --- a/testing/tests/test_hermes_cli_lanes_kanban.py +++ /dev/null @@ -1,392 +0,0 @@ -"""Kanban claim and lifecycle contracts for Hermes CLI lanes.""" - -from __future__ import annotations - -import sys -from contextlib import nullcontext -from pathlib import Path -from types import SimpleNamespace - -import pytest - -from testing.tests.test_hermes_cli_lanes_support import ( - lanes, -) - - -def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch): - task = SimpleNamespace(id="t_auto", assignee=None, status="ready") - assigned = [] - - class Connection: - def close(self): - return None - - def assign_task(_conn, task_id, profile): - assigned.append((task_id, profile)) - task.assignee = profile - return True - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [{"slug": "cassandra"}], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - assign_task=assign_task, - get_task=lambda _conn, _task_id: task, - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - - assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")] - assert assigned == [("t_auto", "cli-auto")] - - -def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes( - monkeypatch, capsys -): - class CorruptBoardError(Exception): - pass - - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def close(self): - return None - - def connect(*, board): - if board == "cassandra": - raise CorruptBoardError("integrity_check failed") - return Connection() - - fake_db = SimpleNamespace( - KanbanDbCorruptError=CorruptBoardError, - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - recompute_ready=lambda _conn: None, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err - - -def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys): - task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready") - - class Connection: - def __init__(self, board): - self.board = board - - def close(self): - return None - - def recompute_ready(connection): - if connection.board == "cassandra": - raise lanes.sqlite3.OperationalError("disk I/O error") - - fake_db = SimpleNamespace( - list_boards=lambda include_archived=False: [ - {"slug": "cassandra"}, - {"slug": "healthy"}, - ], - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(board), - recompute_ready=recompute_ready, - list_tasks=lambda _conn: [task], - claim_task=lambda _conn, _task_id, **_kwargs: task, - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")] - error = capsys.readouterr().err - assert "temporarily skipping Kanban board 'cassandra'" in error - assert "storage OperationalError: disk I/O error" in error - - -def test_board_call_retries_storage_faults_on_fresh_connections(): - connections = [] - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - attempts = [] - - def operation(_connection): - attempts.append(1) - if len(attempts) < 3: - raise lanes.sqlite3.OperationalError("disk I/O error") - return "healthy" - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - ) - lanes.BOARD_CORRUPTION_ERRORS.clear() - - assert lanes._board_call(fake_db, "cassandra", operation) == "healthy" - assert len(connections) == 3 - assert all(connection.closed for connection in connections) - - -@pytest.mark.parametrize( - ("result", "expected_action"), - [ - (lanes.ProcessResult(0, "plain text only", None, False), "block"), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "done", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q"], - "artifacts": ["reports/result.json"], - "blockers": [], - }, - False, - ), - "complete", - ), - ( - lanes.ProcessResult( - 0, - "", - { - "status": "completed", - "summary": "The full test suite is still running.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest -q — in progress"], - "artifacts": [], - "blockers": [], - }, - False, - ), - "block", - ), - ], -) -def test_claim_requires_structured_evidence_and_surfaces_artifacts( - tmp_path: Path, - monkeypatch, - result, - expected_action, -): - task = SimpleNamespace( - id="t_worker", - current_run_id=4, - assignee="cli-auto", - max_runtime_seconds=60, - ) - calls = [] - heartbeats = [] - connections = [] - artifact = tmp_path / "reports/result.json" - artifact.parent.mkdir() - artifact.write_text("{}\n", encoding="utf-8") - - class Connection: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def connect(*, board): - assert board == "cassandra" - connection = Connection() - connections.append(connection) - return connection - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=connect, - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "bounded objective", - heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: ( - heartbeats.append((note, expected_run_id)) or True - ), - add_comment=lambda *_args: None, - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr( - lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json" - ) - monkeypatch.setattr( - lanes, - "select_route", - lambda *_args, **_kwargs: lanes.Route( - "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, () - ), - ) - - def run_provider(*args, **_kwargs): - assert connections[0].closed - before_heartbeat = len(connections) - assert args[6]("working") is True - assert len(connections) == before_heartbeat + 1 - assert connections[-1].closed - return result - - monkeypatch.setattr(lanes, "run_provider", run_provider) - - lanes.execute_claim("cassandra", "t_worker") - - assert calls[0][0] == expected_action - assert heartbeats == [("working", 4)] - if expected_action == "complete": - assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)] - assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"] - else: - assert calls[0][1]["kind"] == "capability" - assert all(connection.closed for connection in connections) - - -def test_goal_card_continues_after_local_judge_rejects_progress( - tmp_path: Path, - monkeypatch, -): - task = SimpleNamespace( - id="t_goal", - current_run_id=12, - assignee="cli-auto", - max_runtime_seconds=300, - goal_mode=True, - goal_max_turns=3, - ) - calls = [] - comments = [] - - class Connection: - def close(self): - return None - - fake_db = SimpleNamespace( - scoped_current_board=lambda _board: nullcontext(), - connect=lambda board: Connection(), - get_task=lambda _conn, _task_id: task, - worker_log_path=lambda _task_id, board: tmp_path / "worker.log", - _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"), - set_branch_name=lambda *_args: None, - set_workspace_path=lambda *_args: None, - build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.", - heartbeat_worker=lambda *_args, **_kwargs: True, - add_comment=lambda _conn, _task_id, _author, body: comments.append(body), - complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)), - block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)), - ) - monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) - monkeypatch.setattr( - lanes, - "state_path", - lambda _board, _task_id: tmp_path / "state.json", - ) - claude_low = lanes.Route( - "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () - ) - codex_low = lanes.Route( - "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, () - ) - codex_xhigh = lanes.Route( - "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, () - ) - route_calls = [] - - def select_route(_prompt, assignee, **kwargs): - route_calls.append((assignee, kwargs)) - if assignee == "cli-codex-low": - return codex_low - if len(route_calls) == 1: - return claude_low - return codex_xhigh - - monkeypatch.setattr(lanes, "select_route", select_route) - monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) - reports = [ - lanes.ProcessResult(1, "authentication expired", None, True), - lanes.ProcessResult( - 0, - "first turn", - { - "status": "completed", - "summary": "Focused tests passed.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest focused: passed"], - "artifacts": [], - "blockers": [], - }, - False, - ), - lanes.ProcessResult( - 0, - "second turn", - { - "status": "completed", - "summary": "Full tests passed; commit pushed and remote HEAD verified.", - "changed_files": ["src/a.py"], - "tests_run": ["pytest full: passed"], - "artifacts": [], - "blockers": [], - }, - False, - ), - ] - monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0)) - verdicts = iter( - [ - (False, "commit, push, and remote verification are missing"), - (True, "all explicit acceptance criteria have evidence"), - ] - ) - judge_contexts = [] - - def judge_goal_completion(objective, *_args, **_kwargs): - judge_contexts.append(objective) - return next(verdicts) - - monkeypatch.setattr( - lanes.cli_lane_goal, - "judge_goal_completion", - judge_goal_completion, - ) - - lanes.execute_claim("cassandra", "t_goal") - - assert calls[0][0] == "complete" - assert calls[0][1]["metadata"]["goal_turn"] == 2 - assert any( - "Goal completion rejected; continuing turn 2/3" in item for item in comments - ) - assert any( - "Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments - ) - assert route_calls[2][1]["exclude_provider"] == "claude" - assert "prior rejected reports" in judge_contexts[1] - assert "commit, push, and remote verification are missing" in judge_contexts[1] - assert reports == [] diff --git a/testing/tests/test_hermes_cli_provider_edges.py b/testing/tests/test_hermes_cli_provider_edges.py index 21e57453..6f6f88a3 100644 --- a/testing/tests/test_hermes_cli_provider_edges.py +++ b/testing/tests/test_hermes_cli_provider_edges.py @@ -44,6 +44,26 @@ def test_stream_process_enforces_runtime_and_lease_boundaries( assert lease_lost.returncode != 0 +def test_stream_process_bounds_retained_line_history(tmp_path: Path): + result = lanes.stream_process( + [ + sys.executable, + "-c", + "import sys; [sys.stdout.write(f'{n}\\n') for n in range(4010)]", + ], + provider="codex", + cwd=tmp_path, + env=dict(os.environ), + log_path=tmp_path / "bounded.log", + state={}, + state_file=tmp_path / "state.json", + heartbeat=lambda _note: True, + max_runtime=60, + ) + assert result.returncode == 0 + assert "4009" in result.output + + def test_process_identity_and_signal_failures_are_bounded(monkeypatch): assert lanes._process_record(999999999) is None monkeypatch.setattr( @@ -63,9 +83,26 @@ def test_process_identity_and_signal_failures_are_bounded(monkeypatch): ) lanes._signal_worker_tree(10, {11: (12, 13)}, signal.SIGTERM) + signals = [] + monkeypatch.setattr(lanes, "_process_identity_matches", lambda *_args: False) + monkeypatch.setattr(lanes.os, "killpg", lambda pid, sig: signals.append((pid, sig))) + monkeypatch.setattr( + lanes.os, + "kill", + lambda *_args: (_ for _ in ()).throw(AssertionError("stale PID signaled")), + ) + lanes._signal_worker_tree(20, {21: (22, 23)}, signal.SIGKILL) + assert signals == [(20, signal.SIGKILL)] + def test_descendant_snapshot_follows_multiple_generations(monkeypatch): - entries = [Path("/proc/self"), Path("/proc/10"), Path("/proc/11"), Path("/proc/12")] + entries = [ + Path("/proc/self"), + Path("/proc/10"), + Path("/proc/11"), + Path("/proc/12"), + Path("/proc/13"), + ] monkeypatch.setattr(lanes.Path, "iterdir", lambda _path: iter(entries)) records = { 10: (1, 10, 100), @@ -76,6 +113,34 @@ def test_descendant_snapshot_follows_multiple_generations(monkeypatch): assert lanes._descendant_processes(10) == {11: (11, 101), 12: (12, 102)} +def test_claude_missing_unstarted_session_does_not_mark_started( + tmp_path: Path, + monkeypatch, +): + state = {"claude_session_id": "synthetic-session"} + state_file = tmp_path / "state.json" + monkeypatch.setattr( + lanes, + "stream_process", + lambda *_args, **_kwargs: lanes.ProcessResult( + 1, lanes.NO_CLAUDE_SESSION, None, False + ), + ) + route = lanes.Route("claude", "model", "high", "profile", "test", "test", 1, ()) + result = lanes.run_provider( + route, + "work", + tmp_path, + state, + state_file, + tmp_path / "log", + lambda _note: True, + 60, + ) + assert result.returncode == 1 + assert "claude_started" not in state + + def test_terminate_escalates_after_term_timeout(monkeypatch): class Process: pid = 321 diff --git a/testing/tests/test_hermes_cli_records_edges.py b/testing/tests/test_hermes_cli_records_edges.py index 3362466a..fed9e872 100644 --- a/testing/tests/test_hermes_cli_records_edges.py +++ b/testing/tests/test_hermes_cli_records_edges.py @@ -78,6 +78,17 @@ def test_small_snapshot_rejects_path_replaced_after_read( assert lanes._open_small_json_snapshot(path) is None +def test_small_snapshot_rejects_a_reader_that_exceeds_its_bound( + tmp_path: Path, + monkeypatch, +): + path = tmp_path / "record" + path.write_text("{}", encoding="utf-8") + monkeypatch.setattr(lanes, "MAX_TERMINAL_RECORD_BYTES", 8) + monkeypatch.setattr(lanes, "_read_bounded", lambda *_args: b"x" * 9) + assert lanes._open_small_json_snapshot(path) is None + + def test_recovery_snapshot_classifies_nonregular_and_open_failure( tmp_path: Path, monkeypatch, diff --git a/testing/tests/test_hermes_cli_recovery_edges.py b/testing/tests/test_hermes_cli_recovery_edges.py index 27ba6773..46687826 100644 --- a/testing/tests/test_hermes_cli_recovery_edges.py +++ b/testing/tests/test_hermes_cli_recovery_edges.py @@ -87,6 +87,9 @@ def test_staged_authority_rejects_outside_nested_and_unknown_state( document["kanban_state"] = "unknown" assert lanes._staged_terminal_authority(staged, _Snapshot(document)) is None + valid = _pending_terminal_record("board", "task", 5, "accepted") + assert lanes._staged_terminal_authority(staged, _Snapshot(valid)) is None + def test_restore_staged_authority_handles_every_first_writer_state( tmp_path: Path, @@ -296,3 +299,34 @@ def test_pending_guard_rejects_invalid_run_and_inaccessible_or_bad_artifacts( monkeypatch.setattr(Path, "stat", denied) assert lanes._has_pending_finalization("board", "task", 12) is False + + +def test_pending_guard_scans_invalid_prepared_and_staged_candidates( + tmp_path: Path, + monkeypatch, +): + monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes") + base = lanes.state_path("board", "task") + pending = lanes._terminal_path(base, 13) + pending.parent.mkdir(parents=True) + pending.write_text("{}", encoding="utf-8") + (pending.parent / "task.run-13.terminal.prepared-bad.json").write_text( + "{}", encoding="utf-8" + ) + (pending.parent / f"task.run-13.terminal.prepared-{'a' * 32}.json").write_text( + "{}", encoding="utf-8" + ) + staged = [ + pending.parent / f".retire.{'b' * 16}.{'c' * 16}.0", + pending.parent / f".retire.{'d' * 16}.{'e' * 16}.0", + ] + for path in staged: + path.write_text("{}", encoding="utf-8") + snapshots = iter((None, _Snapshot({}))) + monkeypatch.setattr( + lanes, + "_open_terminal_recovery_snapshot", + lambda _path: next(snapshots), + ) + monkeypatch.setattr(lanes, "_staged_terminal_authority", lambda *_args: None) + assert lanes._has_pending_finalization("board", "task", 13) is False diff --git a/testing/tests/test_hermes_cli_retention_edges.py b/testing/tests/test_hermes_cli_retention_edges.py index f7deb6a2..50f3cb19 100644 --- a/testing/tests/test_hermes_cli_retention_edges.py +++ b/testing/tests/test_hermes_cli_retention_edges.py @@ -87,3 +87,49 @@ def test_gc_skips_symlink_board_and_disappearing_candidate( monkeypatch.setattr(Path, "stat", disappear) assert lanes.gc_lane_artifacts() == 0 + + +def test_gc_counts_classified_quarantine_and_bounds_unlink_deferrals( + tmp_path: Path, + monkeypatch, +): + missing = tmp_path / "missing" + assert lanes._unlink_artifact_if_same(missing, tmp_path.stat()) is False + + root = tmp_path / "lanes" + board = root / "board" + board.mkdir(parents=True) + classified = board / "classified.candidate-1.json" + classified.write_text("{}", encoding="utf-8") + monkeypatch.setattr(lanes, "STATE_ROOT", root) + monkeypatch.setattr( + lanes, + "_artifact_gc_candidates", + lambda _board: [classified], + ) + monkeypatch.setattr( + lanes, + "_quarantine_invalid_retained_terminal", + lambda _path: (True, True), + ) + assert lanes.gc_lane_artifacts() == 1 + + staged = board / f".retire.{'a' * 16}.{'b' * 16}.0" + staged.write_text("{}", encoding="utf-8") + monkeypatch.setattr( + lanes, + "_artifact_gc_candidates", + lambda _board: [staged], + ) + monkeypatch.setattr( + lanes, + "_quarantine_invalid_retained_terminal", + lambda _path: (False, False), + ) + monkeypatch.setattr(lanes, "_open_terminal_recovery_snapshot", lambda _path: None) + monkeypatch.setattr(lanes, "_unlink_artifact_if_same", lambda *_args: False) + assert lanes.gc_lane_artifacts( + max_age_seconds=-1, + max_count=0, + max_bytes=-1, + ) == 0 diff --git a/testing/tests/test_hermes_coordinator_boards.py b/testing/tests/test_hermes_coordinator_boards.py index aa63943f..c18bb7fb 100644 --- a/testing/tests/test_hermes_coordinator_boards.py +++ b/testing/tests/test_hermes_coordinator_boards.py @@ -141,7 +141,7 @@ def test_cassandra_sync_repairs_existing_worktree( assert all("GITEA_TOKEN" not in environment for environment in environments) def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch): - """Missing credentials skip only the fetch, not the local origin repair.""" + """The credential-isolated broker needs no model-facing token to fetch.""" workspace = tmp_path / "cassandra" (workspace / ".git").mkdir(parents=True) monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) @@ -156,8 +156,11 @@ def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch monkeypatch.setattr(coordinator.subprocess, "run", run) state = coordinator.sync_cassandra_repo({}) - assert state == "ready; fetch skipped until Gitea token is configured" - assert [command[-3] for command in commands] == ["remote", "set-url"] + assert state == "ready" + assert len(commands) == 3 + assert commands[0][-3:] == ["remote", "get-url", "origin"] + assert commands[1][-4:-1] == ["remote", "set-url", "origin"] + assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): """A project switch moves queued work without relocating active evidence.""" diff --git a/testing/tests/test_hermes_execution_pool.py b/testing/tests/test_hermes_execution_pool.py new file mode 100644 index 00000000..8b9966d5 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool.py @@ -0,0 +1,476 @@ +"""Adversarial contracts for the fenced three-node Hermes execution pool.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import threading +import time +from pathlib import Path + +import pytest +import yaml + + +ROOT = Path(__file__).parents[2] +HERMES = ROOT / "services/hermes" +SCRIPTS = HERMES / "scripts" +sys.path[:0] = [str(SCRIPTS), str(HERMES / "scm-common/scripts")] + +import execution_pool_coordinator as coordinator # noqa: E402 +import execution_pool_client as pool_client # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_scm as scm # noqa: E402 +import execution_pool_worker as worker # noqa: E402 + + +KEY = b"k" * 32 + + +def binding(**values): + result = { + "board": "atlas", + "task_id": "t_deadbeef", + "run_id": "run-1234", + "worker_ordinal": 0, + "attempt": 1, + } + result.update(values) + return result + + +def assignment_payload(**values): + result = { + "context": "Implement the bounded objective.", + "assignee": "cli-auto", + "repo_url": "https://scm.bstein.dev/atlas/titan-iac.git", + "branch": "feature/hermes-safe-pool", + "base_branch": "main", + "max_runtime_seconds": 3600, + "deadline_unix": int(time.time()) + 3600, + } + result.update(values) + return result + + +def test_envelope_binds_run_ordinal_attempt_and_digest(): + signed = protocol.sign_envelope(KEY, "heartbeat", binding(), {"note": "active"}) + verified = protocol.verify_envelope(KEY, signed, expected_kind="heartbeat") + + assert verified["payload_digest"] == protocol.payload_digest({"note": "active"}) + for name, value in binding().items(): + assert verified[name] == value + + signed["attempt"] = 2 + with pytest.raises(protocol.ProtocolError, match="authentication"): + protocol.verify_envelope(KEY, signed) + + +def test_empty_poll_acknowledgement_is_authenticated(): + empty = { + "board": "", "task_id": "", "run_id": "", + "worker_ordinal": 2, "attempt": 0, + } + signed = protocol.sign_envelope(KEY, "ack", empty, {"assignment": None}) + assert protocol.verify_envelope(KEY, signed, expected_kind="ack")["payload"] == { + "assignment": None + } + + +@pytest.mark.parametrize( + "mutation,error", + [ + ({"board": "../atlas"}, "invalid board"), + ({"worker_ordinal": 3}, "outside the pool"), + ({"expires_at": 1}, "validity window"), + ], +) +def test_malformed_or_traversal_bindings_fail_closed(mutation, error): + signed = protocol.sign_envelope(KEY, "heartbeat", binding(), {"note": "active"}) + signed.update(mutation) + unsigned = dict(signed) + unsigned.pop("signature") + import hashlib + import hmac + + signed["signature"] = hmac.new( + KEY, protocol.canonical_json(unsigned), hashlib.sha256 + ).hexdigest() + with pytest.raises(protocol.ProtocolError, match=error): + protocol.verify_envelope(KEY, signed) + + +def test_oversized_and_malformed_payloads_are_rejected(): + with pytest.raises(protocol.ProtocolError, match="wire limit"): + protocol.sign_envelope( + KEY, "result", binding(), {"output": "x" * protocol.MAX_WIRE_BYTES} + ) + with pytest.raises(protocol.ProtocolError, match="malformed"): + protocol.parse_wire(b"{not-json") + with pytest.raises(protocol.ProtocolError, match="oversized"): + protocol.parse_wire(b"x" * (protocol.MAX_WIRE_BYTES + 1)) + + +def test_key_requires_private_regular_file_and_rejects_symlink(tmp_path): + key = tmp_path / "key" + key.write_bytes(KEY) + key.chmod(0o644) + with pytest.raises(protocol.ProtocolError, match="private"): + protocol.read_key(key) + key.chmod(0o600) + assert protocol.read_key(key) == KEY + link = tmp_path / "link" + link.symlink_to(key) + with pytest.raises(protocol.ProtocolError, match="unavailable"): + protocol.read_key(link) + with pytest.raises(protocol.ProtocolError, match="unavailable"): + protocol.read_key(tmp_path / "missing") + + +def test_local_signing_boundary_rejects_foreign_or_unassigned_result(monkeypatch): + monkeypatch.setattr(pool_client, "ORDINAL", 0) + boundary = pool_client.ClientBoundary(KEY) + request = {"binding": binding(), "payload": {"note": "active"}} + with pytest.raises(protocol.ProtocolError, match="does not own"): + boundary.heartbeat(request) + boundary.current = binding() + request["binding"] = binding(worker_ordinal=1) + with pytest.raises(protocol.ProtocolError, match="does not own"): + boundary.heartbeat(request) + + +def test_simultaneous_claim_materialization_has_one_winner(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + barrier = threading.Barrier(8) + outcomes = [] + + def add(index): + barrier.wait() + try: + outcome = store.add( + binding(task_id=f"t_{index}", run_id=f"run-{index}"), + assignment_payload(), + ) + except protocol.ProtocolError: + outcome = False + outcomes.append(outcome) + + threads = [threading.Thread(target=add, args=(index,)) for index in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert outcomes.count(True) == 1 + assert store.available_ordinals() == [1, 2] + + +def test_duplicate_assignment_is_idempotent_but_conflict_is_rejected(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + assert store.add(binding(), assignment_payload()) is True + assert store.add(binding(), assignment_payload()) is False + with pytest.raises(protocol.ProtocolError, match="conflicting"): + store.add(binding(), assignment_payload(context="different")) + with pytest.raises(protocol.ProtocolError, match="conflicting"): + store.add(binding(worker_ordinal=1), assignment_payload()) + + +def test_duplicate_heartbeat_and_result_are_idempotent(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding(), assignment_payload()) + store.offer(0) + heartbeat = protocol.sign_envelope( + KEY, "heartbeat", binding(), {"note": "active"}, delivery_id="delivery-1" + ) + assert store.heartbeat(heartbeat) == (True, False) + assert store.heartbeat(heartbeat) == (True, True) + result = protocol.sign_envelope( + KEY, "result", binding(), {"structured": {"status": "completed"}} + ) + _, duplicate = store.accept_result(result) + assert duplicate is False + _, duplicate = store.accept_result(result) + assert duplicate is True + + +def test_reused_delivery_and_stale_attempt_cannot_cross_runs(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding(), assignment_payload()) + first = protocol.sign_envelope( + KEY, "heartbeat", binding(), {"note": "one"}, delivery_id="delivery-1" + ) + store.heartbeat(first) + reused = protocol.sign_envelope( + KEY, "heartbeat", binding(), {"note": "two"}, delivery_id="delivery-1" + ) + with pytest.raises(protocol.ProtocolError, match="reused"): + store.heartbeat(reused) + store.finalize(binding(), "finalized") + store.add(binding(run_id="other-run"), assignment_payload()) + cross_run = protocol.sign_envelope( + KEY, "heartbeat", binding(run_id="other-run"), {"note": "one"}, + delivery_id="delivery-1", + ) + with pytest.raises(protocol.ProtocolError, match="reused"): + store.heartbeat(cross_run) + stale = protocol.sign_envelope( + KEY, "result", binding(attempt=2), {"structured": {"status": "completed"}} + ) + with pytest.raises(protocol.ProtocolError, match="attempt is stale"): + store.accept_result(stale) + replacement = protocol.sign_envelope( + KEY, + "result", + binding(run_id="replacement-run"), + {"structured": {"status": "completed"}}, + ) + with pytest.raises(protocol.ProtocolError, match="unknown or stale"): + store.accept_result(replacement) + + +def test_restart_heartbeat_loss_and_node_replacement_reuse_durable_assignment(tmp_path): + database = tmp_path / "pool.db" + first = protocol.PoolStore(database) + first.add(binding(), assignment_payload()) + offered = first.offer(0) + assert offered and offered["attempt"] == 1 + # Reopening the SQLite ledger models coordinator restart. An expired lease + # is deliberately not reassigned across ordinals; the StatefulSet replaces + # ordinal 0 with the same PVC and provider session state. + restarted = protocol.PoolStore(database) + with restarted._connect() as connection: + connection.execute("UPDATE assignments SET lease_until=0") + resumed = restarted.offer(0) + assert resumed and resumed["run_id"] == "run-1234" + assert resumed["attempt"] == 1 + assert restarted.offer(1) is None + assert len(restarted.active_assignments()) == 1 + + +def test_result_conflict_never_overwrites_first_result(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding(), assignment_payload()) + first = protocol.sign_envelope(KEY, "result", binding(), {"returncode": 0}) + store.accept_result(first) + conflict = protocol.sign_envelope(KEY, "result", binding(), {"returncode": 1}) + with pytest.raises(protocol.ProtocolError, match="conflicting result"): + store.accept_result(conflict) + + +def test_scm_workspace_is_ordinal_contained_and_rejects_symlink(tmp_path, monkeypatch): + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path) + monkeypatch.setattr(scm, "ORDINAL", 0) + signed = protocol.sign_envelope(KEY, "assignment", binding(), assignment_payload()) + path = scm.workspace_path(signed) + assert path == tmp_path / "runs/atlas/t_deadbeef/run-1234" + path.parent.mkdir(parents=True, exist_ok=True) + path.symlink_to(tmp_path / "elsewhere", target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="symlink"): + scm.workspace_path(signed) + root_case = tmp_path / "root-case" + root_case.mkdir() + root = tmp_path / "other-root" + root.mkdir() + (root_case / "runs").symlink_to(root, target_is_directory=True) + monkeypatch.setattr(scm, "WORKSPACE_ROOT", root_case) + with pytest.raises(protocol.ProtocolError, match="run root"): + scm.workspace_path(signed) + + +def test_scm_boundary_rejects_other_repo_branch_and_ordinal(monkeypatch): + monkeypatch.setattr(scm, "ORDINAL", 0) + for changes, message in ( + ({"repo_url": "https://evil.example/atlas/titan-iac.git"}, "outside Atlas"), + ({"branch": "main"}, "reviewed namespace"), + ): + signed = protocol.sign_envelope( + KEY, "assignment", binding(), assignment_payload(**changes) + ) + with pytest.raises(protocol.ProtocolError, match=message): + scm._binding(signed) + signed = protocol.sign_envelope( + KEY, "assignment", binding(worker_ordinal=1), assignment_payload() + ) + with pytest.raises(protocol.ProtocolError, match="ordinal"): + scm._binding(signed) + + +def test_scm_git_process_has_no_credential_or_ambient_configuration(): + environment = scm._git_environment() + assert environment == { + "HOME": "/nonexistent", + "PATH": "/usr/bin:/bin", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + } + assert scm._broker_repo("titan-iac").startswith("http://hermes-scm-broker.") + source = (SCRIPTS / "execution_pool_scm.py").read_text() + assert "GITEA_TOKEN" not in source and "GIT_ASKPASS" not in source + + +def test_activity_is_bounded_sanitized_and_nofollow(tmp_path): + text = coordinator.sanitize_activity( + 'route=codex token=super-secret\nAuthorization: Bearer abcdefghijklmnop\n' + '"refresh_token":"sk-ant-oat01-abcdefghijklmnopqrstuvwxyz"' + ) + assert "super-secret" not in text + assert "abcdefghijklmnop" not in text + assert "abcdefghijklmnopqrstuvwxyz" not in text + assert len(text.encode()) <= protocol.MAX_ACTIVITY_BYTES + + class FakeKanban: + @staticmethod + def worker_log_path(_task, board): + assert board == "atlas" + return str(tmp_path / "worker.log") + + envelope = {**binding(), "payload": {"activity": "visible worker activity\n"}} + coordinator._append_activity(FakeKanban, envelope) + assert (tmp_path / "worker.log").read_text() == "visible worker activity\n" + (tmp_path / "worker.log").unlink() + (tmp_path / "worker.log").symlink_to(tmp_path / "target") + with pytest.raises(protocol.ProtocolError, match="symlink"): + coordinator._append_activity(FakeKanban, envelope) + + +def _documents(path): + return [item for item in yaml.safe_load_all(path.read_text()) if item] + + +def test_three_node_statefulset_contract_and_cross_worker_isolation(): + stateful = _documents(HERMES / "execution-worker-statefulset.yaml")[0] + pod = stateful["spec"]["template"]["spec"] + assert stateful["spec"]["replicas"] == 3 + assert stateful["spec"]["podManagementPolicy"] == "Parallel" + assert pod["automountServiceAccountToken"] is False + claims = { + item["metadata"]["name"]: item + for item in stateful["spec"]["volumeClaimTemplates"] + } + assert set(claims) == {"workspace", "provider-access"} + assert all(item["spec"]["accessModes"] == ["ReadWriteOnce"] for item in claims.values()) + anti = pod["affinity"]["podAntiAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"] + assert anti[0]["topologyKey"] == "kubernetes.io/hostname" + spread = pod["topologySpreadConstraints"][0] + assert spread["whenUnsatisfiable"] == "DoNotSchedule" + assert spread["maxSkew"] == 1 + + +def test_worker_placement_prefers_accelerators_and_preserves_exclusions(): + stateful = _documents(HERMES / "execution-worker-statefulset.yaml")[0] + pod = stateful["spec"]["template"]["spec"] + assert pod["priorityClassName"] == "scavenger" + affinity = stateful["spec"]["template"]["spec"]["affinity"]["nodeAffinity"] + terms = affinity["requiredDuringSchedulingIgnoredDuringExecution"]["nodeSelectorTerms"] + required = {item["key"]: item for item in terms[0]["matchExpressions"]} + assert required["node-role.kubernetes.io/worker"]["values"] == ["true"] + excluded = set(required["kubernetes.io/hostname"]["values"]) + assert {"titan-04", "titan-13", "titan-14", "titan-17", "titan-18", "titan-19", "titan-22", "titan-24"} <= excluded + assert affinity["preferredDuringSchedulingIgnoredDuringExecution"][0]["weight"] == 100 + assert affinity["preferredDuringSchedulingIgnoredDuringExecution"][1]["weight"] == 50 + worker_container = pod["containers"][0] + assert worker_container["resources"] == { + "requests": {"cpu": "5m", "memory": "128Mi", "ephemeral-storage": "1Gi"}, + "limits": {"cpu": "2", "memory": "4Gi", "ephemeral-storage": "8Gi"}, + } + mediators = [ + item + for item in _documents(HERMES / "execution-mediator.yaml") + if item["kind"] == "Deployment" + ] + assert all( + item["spec"]["template"]["spec"]["containers"][0]["resources"][ + "requests" + ] + == {"cpu": "2m", "memory": "64Mi"} + for item in mediators + ) + + +def test_model_worker_has_no_scm_or_cluster_credential_mount(): + stateful = _documents(HERMES / "execution-worker-statefulset.yaml")[0] + pod = stateful["spec"]["template"] + containers = {item["name"]: item for item in pod["spec"]["containers"]} + worker = containers["execution-worker"] + environment = {item["name"] for item in worker["env"]} + assert not {"GITEA_TOKEN", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "KUBECONFIG"} & environment + mounts = {item["mountPath"] for item in worker["volumeMounts"]} + assert "/vault/secrets" not in mounts + assert "/pool-access" not in mounts + assert "/provider-access" in mounts + assert not any(item["name"] == "vault-auth-token" for item in worker["volumeMounts"]) + assert pod["metadata"]["annotations"]["vault.hashicorp.com/agent-inject-containers"] == "stage-worker-access" + assert set(containers) == {"execution-worker"} + assert "execution-pool-key" not in str(pod["metadata"]["annotations"]) + mediators = [ + item for item in _documents(HERMES / "execution-mediator.yaml") + if item["kind"] == "Deployment" + ] + assert len(mediators) == 3 + for mediator in mediators: + spec = mediator["spec"]["template"]["spec"] + assert spec["automountServiceAccountToken"] is False + privileged = spec["containers"][0] + assert {mount["mountPath"] for mount in privileged["volumeMounts"]} >= { + "/pool-access", "/scm-state", "/opt/scm", "/workspace" + } + assert "/provider-access" not in { + mount["mountPath"] for mount in privileged["volumeMounts"] + } + + +def test_worker_service_account_has_no_kubernetes_permissions(): + documents = _documents(HERMES / "execution-worker-rbac.yaml") + assert len(documents) == 1 + account = documents[0] + assert account["kind"] == "ServiceAccount" + assert account["automountServiceAccountToken"] is False + + +def test_coordinator_remains_single_state_owner_and_workers_do_not_mount_home(): + agent = _documents(HERMES / "agent-deployment.yaml")[0] + assert agent["kind"] == "Deployment" + assert agent["spec"]["replicas"] == 1 + assert agent["spec"]["strategy"]["type"] == "Recreate" + assert any( + item["name"] == "home" + and item["persistentVolumeClaim"]["claimName"] == "hermes-agent-home" + for item in agent["spec"]["template"]["spec"]["volumes"] + ) + worker_text = (HERMES / "execution-worker-statefulset.yaml").read_text() + assert "hermes-agent-home" not in worker_text + assert "kanban.db" not in worker_text + + +def test_worker_protocol_preserves_switchyard_fallback_and_visible_evidence(): + source = (SCRIPTS / "execution_pool_worker.py").read_text() + coordinator_source = (SCRIPTS / "execution_pool_coordinator.py").read_text() + server_source = (SCRIPTS / "execution_pool_server.py").read_text() + assert "cli_lane_runner.select_route" in source + assert 'alternate = "claude" if route.provider == "codex" else "codex"' in source + assert "codex_thread_id" in source and "claude_session_id" in source + assert "final_activity" in source + assert "worker_ordinal" in coordinator_source and "provider_sessions" in coordinator_source + assert "coordinator.reconcile" in server_source + + +def test_retention_gc_removes_only_clean_terminal_workspace(tmp_path, monkeypatch): + monkeypatch.setattr(worker, "ROOT", tmp_path) + monkeypatch.setattr(worker, "RETENTION_SECONDS", 3600) + workspace = tmp_path / "runs/atlas/t_deadbeef/run-1234" + workspace.mkdir(parents=True) + subprocess.run(["git", "init", "-q", str(workspace)], check=True) + subprocess.run(["git", "-C", str(workspace), "config", "user.email", "test@example.com"], check=True) + subprocess.run(["git", "-C", str(workspace), "config", "user.name", "Test"], check=True) + (workspace / "tracked").write_text("safe\n") + subprocess.run(["git", "-C", str(workspace), "add", "tracked"], check=True) + subprocess.run(["git", "-C", str(workspace), "commit", "-qm", "initial"], check=True) + state_file = tmp_path / "session-state/atlas/t_deadbeef/run-1234.json" + state_file.parent.mkdir(parents=True) + state_file.write_text( + json.dumps({"terminal_at": time.time() - 7200, "workspace": str(workspace)}) + ) + assert worker.garbage_collect() == 1 + assert not workspace.exists() + assert not state_file.exists() diff --git a/testing/tests/test_hermes_execution_pool_assignment.py b/testing/tests/test_hermes_execution_pool_assignment.py new file mode 100644 index 00000000..d2fd91bd --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_assignment.py @@ -0,0 +1,208 @@ +"""SCM assignment tests that keep coordinator credentials out of worktrees.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from http.server import BaseHTTPRequestHandler +from types import SimpleNamespace + +import pytest +import yaml + + +SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" +HERMES = SCRIPTS.parent +sys.path[:0] = [str(SCRIPTS), str(HERMES / "scm-common/scripts")] + +import execution_pool_coordinator as coordinator # noqa: E402 +import execution_pool_project as project # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_scm as scm # noqa: E402 +import execution_pool_worker as worker # noqa: E402 + + +KEY = b"k" * 32 + + +def test_new_task_uses_explicit_reviewed_atlas_default(monkeypatch): + monkeypatch.setattr( + project, + "resolve_project", + lambda board: ( + f"https://scm.bstein.dev/atlas/{board}.git", "main", Path("/unused") + ), + ) + task = SimpleNamespace(id="t_deadbeef", workspace_path="", branch_name="") + + assert project.resolve_assignment("titan-iac", task) == ( + "https://scm.bstein.dev/atlas/titan-iac.git", + "wt/t_deadbeef", + "main", + ) + + +def test_legacy_local_workspace_remains_owned_by_local_lane(tmp_path): + task = SimpleNamespace( + id="t_deadbeef", workspace_path=str(tmp_path), branch_name="feature/safe" + ) + + assert project.distributed_workspace_eligible(task) is False + + +def test_local_git_environment_excludes_credential_boundary_paths(): + local = scm._git_environment() + assert "HERMES_SCM_PASSWORD_FILE" not in local + assert "GIT_ASKPASS" not in local + assert "GITEA_TOKEN" not in local + + +def test_finalized_duplicate_result_is_acknowledged_without_refinalizing(tmp_path): + binding = { + "board": "atlas", "task_id": "t_deadbeef", "run_id": "run-1", + "worker_ordinal": 0, "attempt": 1, + } + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding, {"context": "safe"}) + ordinal_key = protocol.derive_ordinal_key(KEY, 0) + result = protocol.sign_envelope(ordinal_key, "result", binding, {"structured": {}}) + store.accept_result(result) + store.finalize(binding, "finalized") + pool = coordinator.Coordinator(KEY, store) + called = [] + pool.finalize = called.append + + assert pool.result(result)["payload"]["duplicate"] is True + assert called == [] + + +def test_provider_sessions_are_bound_to_exact_task_run(tmp_path, monkeypatch): + runtime = tmp_path / "runtime" + (runtime / "codex").mkdir(parents=True) + (runtime / "claude").mkdir(parents=True) + monkeypatch.setattr(worker, "ROOT", tmp_path / "worker") + (tmp_path / "worker/provider-state").mkdir(parents=True) + monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", tmp_path / "worker-data") + (tmp_path / "worker-data").mkdir() + monkeypatch.setenv("CODEX_HOME", str(runtime / "codex")) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(runtime / "claude")) + assignment = {"board": "atlas", "task_id": "t_deadbeef", "run_id": "run-1"} + + worker._bind_provider_sessions(assignment) + + for path in ( + runtime / "codex/sessions", runtime / "claude/projects", + runtime / "claude/session-env", runtime / "claude/todos", + ): + assert path.is_symlink() + assert "atlas/t_deadbeef/run-1" in str(path.resolve()) + assert (tmp_path / "worker-data/home").is_symlink() + assert "atlas/t_deadbeef/run-1" in str((tmp_path / "worker-data/home").resolve()) + + +def test_internal_http_server_rejects_work_above_its_bound(monkeypatch): + server = protocol.BoundedHTTPServer( + ("127.0.0.1", 0), BaseHTTPRequestHandler, max_workers=1 + ) + rejected = [] + monkeypatch.setattr(server, "shutdown_request", rejected.append) + assert server._slots.acquire(blocking=False) + try: + marker = object() + server.process_request(marker, ("127.0.0.1", 1)) + assert rejected == [marker] + finally: + server._slots.release() + server.server_close() + + +def test_additive_patch_replaces_local_lane_without_touching_base_deployment(): + patch = yaml.safe_load((HERMES / "execution-coordinator-patch.yaml").read_text()) + containers = patch["spec"]["template"]["spec"]["containers"] + local = next(item for item in containers if item["name"] == "cli-lane-runner") + pool = next(item for item in containers if item["name"] == "execution-pool-coordinator") + environment = {item["name"]: item["value"] for item in local["env"]} + assert environment == { + "HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY": "true", + "HERMES_CLI_LANE_CONCURRENCY": "1", + } + assert pool["resources"]["requests"] == {"cpu": "50m", "memory": "128Mi"} + access = next(item for item in pool["volumeMounts"] if item["name"] == "runtime-access") + assert access["subPath"] == "execution-pool-key" and access["readOnly"] is True + + +def test_additive_network_policies_expose_only_worker_pool_and_switchyard_ports(): + documents = list(yaml.safe_load_all( + (HERMES / "execution-worker-networkpolicy.yaml").read_text() + )) + policies = {item["metadata"]["name"]: item for item in documents} + assert set(policies) == { + "hermes-execution-worker-isolation", + "hermes-execution-mediator-isolation", + "hermes-execution-pool-ingress", + "hermes-execution-switchyard-ingress", + *(f"hermes-execution-worker-mediator-{ordinal}" for ordinal in range(3)), + *(f"hermes-execution-mediator-worker-{ordinal}" for ordinal in range(3)), + } + assert policies["hermes-execution-pool-ingress"]["spec"]["ingress"][0]["ports"] == [ + {"protocol": "TCP", "port": 9007} + ] + pool_source = policies["hermes-execution-pool-ingress"]["spec"]["ingress"] + assert "hermes-execution-mediator" in str(pool_source) + worker_egress = policies["hermes-execution-worker-isolation"]["spec"]["egress"] + assert "hermes-scm-broker" not in str(worker_egress) + assert "hermes-execution-mediator" not in str(worker_egress) + for ordinal in range(3): + worker_policy = policies[f"hermes-execution-worker-mediator-{ordinal}"] + mediator_policy = policies[f"hermes-execution-mediator-worker-{ordinal}"] + assert worker_policy["spec"]["podSelector"]["matchLabels"][ + "apps.kubernetes.io/pod-index" + ] == str(ordinal) + assert worker_policy["spec"]["egress"][0]["to"][0]["podSelector"][ + "matchLabels" + ]["pool-ordinal"] == str(ordinal) + assert mediator_policy["spec"]["ingress"][0]["from"][0]["podSelector"][ + "matchLabels" + ]["apps.kubernetes.io/pod-index"] == str(ordinal) + + +@pytest.mark.parametrize( + "branch", + [ + "../main", + "main", + "feature/../../main", + ], +) +def test_assignment_rejects_unreviewed_task_branch(monkeypatch, branch): + monkeypatch.setattr( + project, + "resolve_project", + lambda _board: ( + "https://scm.bstein.dev/atlas/titan-iac.git", "main", Path("/unused") + ), + ) + task = SimpleNamespace( + id="t_deadbeef", workspace_path="", branch_name=branch, + repo_url="https://evil.example/atlas/other.git", base_branch="../main", + ) + + with pytest.raises(project.ProjectPolicyError): + project.resolve_assignment("titan-iac", task) + + +def test_registry_authority_ignores_unreviewed_task_repo_metadata(monkeypatch): + monkeypatch.setattr( + project, + "resolve_project", + lambda _board: ( + "https://scm.bstein.dev/atlas/metis.git", "main", Path("/unused") + ), + ) + task = SimpleNamespace( + id="t_deadbeef", branch_name="review/t_deadbeef", + repo_url="https://evil.example/atlas/other.git", base_branch="../main", + ) + assert project.resolve_assignment("metis", task)[:2] == ( + "https://scm.bstein.dev/atlas/metis.git", "review/t_deadbeef" + ) diff --git a/testing/tests/test_hermes_execution_pool_coordinator_v2.py b/testing/tests/test_hermes_execution_pool_coordinator_v2.py new file mode 100644 index 00000000..fa679b0d --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_coordinator_v2.py @@ -0,0 +1,293 @@ +"""Exact-run coordinator finalization, lease, and ownership contracts.""" + +from __future__ import annotations + +import json +import sys +import types +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + +import execution_pool_coordinator as coordinator # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 + + +MASTER = b"k" * 32 +STRUCTURED = { + "status": "completed", + "summary": "Completed safely.", + "changed_files": ["safe.py"], + "tests_run": ["pytest"], + "artifacts": [], + "findings": [], + "blockers": [], +} + + +def binding(**changes): + value = { + "board": "metis", "task_id": "t_deadbeef", "run_id": "23", + "worker_ordinal": 0, "attempt": 1, + } + value.update(changes) + return value + + +def assignment_payload(**changes): + value = { + "context": "objective", + "assignee": "cli-auto", + "repo_url": "https://scm.bstein.dev/atlas/metis.git", + "branch": "wt/t_deadbeef", + "base_branch": "main", + "max_runtime_seconds": 3600, + "deadline_unix": 10_000_000_000, + } + value.update(changes) + return value + + +def signed(kind, exact_binding, payload): + key = protocol.derive_ordinal_key(MASTER, exact_binding["worker_ordinal"]) + return protocol.sign_envelope(key, kind, exact_binding, payload) + + +class Scope: + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + +class Connection: + def close(self): + return None + + +def install_kanban(monkeypatch, tasks, tmp_path, boards=None): + module = types.ModuleType("kanban_db") + module.tasks = {task.id: task for task in tasks} + module.completed = [] + module.blocked = [] + module.heartbeats = [] + module.reclaimed = [] + module.branches = [] + module.scoped_current_board = lambda _board: Scope() + module.connect = lambda board=None: Connection() + module.get_task = lambda _connection, task_id: module.tasks.get(task_id) + module.list_tasks = lambda _connection: list(module.tasks.values()) + module.list_boards = lambda include_archived=False: boards or ["metis"] + module.build_worker_context = lambda _connection, task_id: f"objective {task_id}" + module.worker_log_path = lambda task_id, board=None: str( + tmp_path / board / f"{task_id}.log" + ) + + def complete(_connection, task_id, **values): + module.completed.append((task_id, values)) + module.tasks[task_id].status = "completed" + return True + + def block(_connection, task_id, **values): + module.blocked.append((task_id, values)) + module.tasks[task_id].status = "blocked" + return True + + def heartbeat(_connection, task_id, **values): + module.heartbeats.append((task_id, values)) + return True + + module.complete_task = complete + module.block_task = block + module.heartbeat_worker = heartbeat + module.reclaim_task = lambda _c, task_id, **values: module.reclaimed.append( + (task_id, values) + ) + package = types.ModuleType("hermes_cli") + package.kanban_db = module + monkeypatch.setitem(sys.modules, "hermes_cli", package) + monkeypatch.setitem(sys.modules, "hermes_cli.kanban_db", module) + return module + + +def task(**changes): + value = { + "id": "t_deadbeef", + "status": "running", + "current_run_id": 23, + "assignee": "cli-auto", + "workspace_path": "", + "branch_name": "wt/t_deadbeef", + "max_runtime_seconds": 3600, + "goal_mode": False, + "goal_max_turns": 1, + } + value.update(changes) + return SimpleNamespace(**value) + + +def store_with_assignment(tmp_path, exact_binding=None): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(exact_binding or binding(), assignment_payload()) + return store + + +def test_assignment_payload_is_bounded_registry_derived_and_typed(monkeypatch): + monkeypatch.setattr( + coordinator, + "resolve_assignment", + lambda board, _task: ( + f"https://scm.bstein.dev/atlas/{board}.git", "wt/t_deadbeef", "main" + ), + ) + kanban = SimpleNamespace( + build_worker_context=lambda _connection, _task_id: {"safe": True} + ) + item = task(max_runtime_seconds=1, goal_max_turns=99) + value = coordinator.assignment_payload(kanban, object(), item, "metis") + assert json.loads(value["context"]) == {"safe": True} + assert value["max_runtime_seconds"] == 60 + assert value["goal_max_turns"] == 12 + assert value["repo_url"].endswith("/metis.git") + kanban.build_worker_context = lambda *_a: "x" * (32 * 1024 + 1) + with pytest.raises(RuntimeError, match="32KiB"): + coordinator.assignment_payload(kanban, object(), item, "metis") + + +def test_poll_uses_per_ordinal_key_for_empty_and_active_assignment(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + poll_binding = binding(board="", task_id="", run_id="", attempt=0) + request = signed("poll", poll_binding, {"ready": True}) + empty = pool.poll(request) + assert protocol.verify_envelope( + protocol.derive_ordinal_key(MASTER, 0), empty, expected_kind="ack" + )["payload"] == {"assignment": None} + store.add(binding(), assignment_payload()) + offered = protocol.verify_envelope( + protocol.derive_ordinal_key(MASTER, 0), pool.poll(request), + expected_kind="assignment", + ) + assert offered["run_id"] == "23" + wrong_key = protocol.derive_ordinal_key(MASTER, 1) + with pytest.raises(protocol.ProtocolError, match="authentication"): + pool.poll(protocol.sign_envelope(wrong_key, "poll", poll_binding, {})) + + +def test_finalize_canonicalizes_integer_run_and_reconcile_does_not_reexecute( + tmp_path, monkeypatch +): + live = task() + kanban = install_kanban(monkeypatch, [live], tmp_path) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + envelope = signed( + "result", + binding(), + { + "structured": dict(STRUCTURED), + "returncode": 0, + "capacity_failure": False, + "node": "titan-05", + "route": {}, + "provider_sessions": {}, + "final_activity": "done\n", + }, + ) + ack = pool.result(envelope) + assert protocol.verify_envelope( + protocol.derive_ordinal_key(MASTER, 0), ack, expected_kind="ack" + )["payload"] == {"accepted": True, "duplicate": False} + assert kanban.completed[0][1]["expected_run_id"] == 23 + row = store._connect().execute("SELECT state FROM assignments").fetchone() + assert row[0] == "finalized" and store.available_ordinals() == [0, 1, 2] + pool.reconcile() + assert store.active_assignments() == [] + + +def test_finalize_fences_stale_run_and_blocks_failed_result_exactly(tmp_path, monkeypatch): + stale_task = task(current_run_id=24) + kanban = install_kanban(monkeypatch, [stale_task], tmp_path) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + pool.result(signed("result", binding(), {"structured": dict(STRUCTURED), "returncode": 0})) + assert not kanban.completed + assert store._connect().execute("SELECT state FROM assignments").fetchone()[0] == "stale" + + second = binding(run_id="25", worker_ordinal=1) + failed_task = task(current_run_id=25) + kanban.tasks[failed_task.id] = failed_task + store.add(second, assignment_payload()) + blocked = {**STRUCTURED, "status": "blocked", "blockers": ["capacity"]} + pool.result( + signed( + "result", second, + {"structured": blocked, "returncode": 1, "capacity_failure": True}, + ) + ) + assert kanban.blocked[-1][1]["expected_run_id"] == 25 + assert kanban.blocked[-1][1]["kind"] == "transient" + + +def test_finalize_rejects_invalid_payload_and_noncanonical_run(tmp_path, monkeypatch): + install_kanban(monkeypatch, [task()], tmp_path) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + with pytest.raises(protocol.ProtocolError, match="payload"): + pool.finalize({**binding(), "result": []}) + bad = binding(run_id="run-bad", worker_ordinal=1) + store.add(bad, assignment_payload()) + pool.finalize({**bad, "result": {"structured": {}}}) + states = dict(store._connect().execute("SELECT run_id,state FROM assignments")) + assert states["run-bad"] == "stale" + + +def test_heartbeat_uses_integer_exact_api_and_surfaces_activity(tmp_path, monkeypatch): + kanban = install_kanban(monkeypatch, [task()], tmp_path) + store = store_with_assignment(tmp_path) + store.offer(0) + pool = coordinator.Coordinator(MASTER, store) + request = signed( + "heartbeat", binding(), + {"note": "active", "activity": "safe output\n"}, + ) + ack = pool.heartbeat(request) + assert kanban.heartbeats[0][1]["expected_run_id"] == 23 + assert (tmp_path / "metis/t_deadbeef.log").read_text() == "safe output\n" + assert ack["kind"] == "ack" + kanban.heartbeat_worker = lambda *_a, **_k: False + with pytest.raises(protocol.ProtocolError, match="no longer owns"): + pool.heartbeat(signed("heartbeat", binding(), {})) + + +def test_recovery_defers_io_errors_and_lease_failure_releases_ordinal( + tmp_path, monkeypatch, capsys +): + kanban = install_kanban(monkeypatch, [task()], tmp_path) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + monkeypatch.setattr(store, "pending_results", lambda: [binding()]) + monkeypatch.setattr(pool, "finalize", lambda _record: (_ for _ in ()).throw(OSError("busy"))) + pool.recover_results() + assert "recovery deferred" in capsys.readouterr().err + + monkeypatch.undo() + kanban = install_kanban(monkeypatch, [task()], tmp_path) + store = protocol.PoolStore(tmp_path / "leases.db") + exact = binding(attempt=3) + store.add(exact, assignment_payload()) + store.offer(0) + with store._connect() as connection: + connection.execute("UPDATE assignments SET lease_until=1") + pool = coordinator.Coordinator(MASTER, store) + pool.expire_leases() + assert kanban.blocked[0][1]["expected_run_id"] == 23 + assert kanban.blocked[0][1]["kind"] == "transient" + assert store.available_ordinals() == [0, 1, 2] diff --git a/testing/tests/test_hermes_execution_pool_dispatch_v2.py b/testing/tests/test_hermes_execution_pool_dispatch_v2.py new file mode 100644 index 00000000..57dd0f74 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_dispatch_v2.py @@ -0,0 +1,410 @@ +"""Coordinator dispatch migration and versioned server contracts.""" + +from __future__ import annotations + +import sqlite3 +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +SCM_SCRIPTS = ROOT / "services/hermes/scm-common/scripts" +sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)] + +import execution_pool_coordinator as coordinator # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_server as server # noqa: E402 +from testing.tests.test_hermes_execution_pool_coordinator_v2 import ( # noqa: E402 + MASTER, + assignment_payload, + binding, + install_kanban, + store_with_assignment, + task, +) +from testing.tests.test_hermes_execution_pool_mediator import http_request # noqa: E402 + + +def test_reconcile_fences_old_run_then_recovers_current_pathless_run( + tmp_path, monkeypatch +): + item = task(current_run_id=24) + install_kanban(monkeypatch, [item], tmp_path) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + monkeypatch.setattr( + coordinator, "assignment_payload", + lambda _db, _connection, _task, _board: assignment_payload(), + ) + pool.reconcile() + rows = store._connect().execute( + "SELECT run_id,state,worker_ordinal FROM assignments ORDER BY run_id" + ).fetchall() + assert [tuple(row) for row in rows] == [ + ("23", "stale", 0), ("24", "assigned", 0) + ] + + +def test_reconcile_preserves_owned_workspace_and_exactly_blocks_registry_failure( + tmp_path, monkeypatch +): + owned = task(id="t_owned", current_run_id=30, workspace_path="/owned") + broken = task(id="t_broken", current_run_id=31) + kanban = install_kanban(monkeypatch, [owned, broken], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + + def prepare(_db, _connection, item, _board): + if item.id == "t_broken": + raise RuntimeError("registry unavailable") + return assignment_payload() + + monkeypatch.setattr(coordinator, "assignment_payload", prepare) + pool.reconcile() + assert store.active_assignments() == [] + assert kanban.blocked[0][0] == "t_broken" + assert kanban.blocked[0][1]["expected_run_id"] == 31 + assert all(call[0] != "t_owned" for call in kanban.blocked) + + +def test_dispatch_claims_only_pathless_task_with_safe_assignment_branch( + tmp_path, monkeypatch +): + item = task(status="running") + install_kanban(monkeypatch, [item], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + observed = [] + + def claim(active, limit, eligible): + observed.append((active, limit, eligible("metis", item))) + return [("metis", item.id)] + + monkeypatch.setattr(coordinator.cli_lane_dispatch, "claim_ready", claim) + monkeypatch.setattr( + coordinator, "assignment_payload", + lambda *_a: assignment_payload(branch="wt/t_deadbeef"), + ) + pool.dispatch() + assert observed == [(set(), 3, True)] + record = store.active_assignments()[0] + assert record["run_id"] == "23" + assert record["payload"]["branch"] == "wt/t_deadbeef" + + +def test_dispatch_incompatible_unversioned_claim_api_fails_closed( + tmp_path, monkeypatch +): + owned = task(workspace_path="/opt/data/workspace/live") + kanban = install_kanban(monkeypatch, [owned], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + calls = [] + + def old_claim(active, limit): + calls.append((active, limit)) + return [("metis", owned.id)] + + monkeypatch.setattr(coordinator.cli_lane_dispatch, "claim_ready", old_claim) + with pytest.raises(TypeError): + pool.dispatch() + assert calls == [] + assert store.active_assignments() == [] + assert kanban.reclaimed == [] + + +def test_dispatch_workspace_ownership_race_is_exactly_fenced( + tmp_path, monkeypatch +): + owned = task(workspace_path="/opt/data/workspace/live") + kanban = install_kanban(monkeypatch, [owned], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + monkeypatch.setattr( + coordinator.cli_lane_dispatch, + "claim_ready", + lambda *_a: [("metis", owned.id)], + ) + pool.dispatch() + assert store.active_assignments() == [] + assert kanban.blocked == [ + ( + owned.id, + { + "reason": ( + "Distributed claim fenced because an existing workspace " + "is owned by the local lane" + ), + "kind": "capability", + "expected_run_id": 23, + }, + ) + ] + + +def test_dispatch_preparation_failure_is_surfaced_and_full_pool_does_not_claim( + tmp_path, monkeypatch +): + broken = task(current_run_id=None) + kanban = install_kanban(monkeypatch, [broken], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + pool = coordinator.Coordinator(MASTER, store) + monkeypatch.setattr( + coordinator.cli_lane_dispatch, + "claim_ready", + lambda *_a: [("metis", broken.id)], + ) + pool.dispatch() + assert kanban.blocked[0][1]["expected_run_id"] is None + assert "canonical run ID" in kanban.blocked[0][1]["reason"] + + full = protocol.PoolStore(tmp_path / "full.db") + for ordinal in range(3): + full.add( + binding(task_id=f"t_{ordinal}", run_id=str(ordinal + 1), worker_ordinal=ordinal), + assignment_payload(), + ) + pool = coordinator.Coordinator(MASTER, full) + monkeypatch.setattr( + coordinator.cli_lane_dispatch, + "claim_ready", + lambda *_a: (_ for _ in ()).throw(AssertionError("must not claim")), + ) + pool.dispatch() + + +def test_lease_expiry_with_noncanonical_run_is_released_as_stale(tmp_path, monkeypatch): + install_kanban(monkeypatch, [task()], tmp_path) + store = protocol.PoolStore(tmp_path / "pool.db") + exact = binding(run_id="bad-run", attempt=3) + store.add(exact, assignment_payload()) + store.offer(0) + with store._connect() as connection: + connection.execute("UPDATE assignments SET lease_until=1") + coordinator.Coordinator(MASTER, store).expire_leases() + assert store._connect().execute("SELECT state FROM assignments").fetchone()[0] == "stale" + + +def test_terminal_activity_marker_prevents_duplicate_append(tmp_path): + class Kanban: + worker_log_path = staticmethod( + lambda task_id, board=None: str(tmp_path / f"{board}-{task_id}.log") + ) + + record = { + **binding(), + "result_digest": "a" * 64, + "result": {"final_activity": "terminal evidence\n"}, + } + coordinator._append_terminal_activity(Kanban, record) + coordinator._append_terminal_activity(Kanban, record) + text = (tmp_path / "metis-t_deadbeef.log").read_text() + assert text.count("execution-pool-result") == 1 + empty = {**record, "result": {"final_activity": ""}} + coordinator._append_terminal_activity(Kanban, empty) + + +def test_activity_and_terminal_log_reject_invalid_or_symlink_targets(tmp_path): + class Kanban: + worker_log_path = staticmethod(lambda *_a, **_k: str(tmp_path / "worker.log")) + + with pytest.raises(protocol.ProtocolError, match="payload"): + coordinator._append_activity(Kanban, {**binding(), "payload": []}) + coordinator._append_activity(Kanban, {**binding(), "payload": {"activity": ""}}) + (tmp_path / "worker.log").symlink_to(tmp_path / "target") + with pytest.raises(protocol.ProtocolError, match="symlink"): + coordinator._append_terminal_activity( + Kanban, + { + **binding(), "result_digest": "a" * 64, + "result": {"final_activity": "terminal"}, + }, + ) + + +def test_heartbeat_bad_run_duplicate_and_unstructured_result_paths( + tmp_path, monkeypatch +): + item = task() + kanban = install_kanban(monkeypatch, [item], tmp_path) + bad_binding = binding(run_id="bad-run") + bad_store = protocol.PoolStore(tmp_path / "bad.db") + bad_store.add(bad_binding, assignment_payload()) + bad_store.offer(0) + pool = coordinator.Coordinator(MASTER, bad_store) + key = protocol.derive_ordinal_key(MASTER, 0) + with pytest.raises(protocol.ProtocolError, match="canonical"): + pool.heartbeat(protocol.sign_envelope(key, "heartbeat", bad_binding, {})) + + store = store_with_assignment(tmp_path / "duplicate") + store.offer(0) + pool = coordinator.Coordinator(MASTER, store) + request = protocol.sign_envelope( + key, "heartbeat", binding(), {"activity": "once"}, + delivery_id="same-heartbeat", + ) + pool.heartbeat(request) + pool.heartbeat(request) + assert (tmp_path / "metis/t_deadbeef.log").read_text() == "once" + + second = binding(run_id="24", worker_ordinal=1) + item.current_run_id = 24 + store.add(second, assignment_payload()) + pool.result( + protocol.sign_envelope( + protocol.derive_ordinal_key(MASTER, 1), "result", second, + {"structured": [], "returncode": 1}, + ) + ) + assert kanban.blocked[-1][1]["expected_run_id"] == 24 + + +def test_reconcile_and_dispatch_cover_empty_error_and_missing_task_paths( + tmp_path, monkeypatch +): + item = task() + kanban = install_kanban(monkeypatch, [item], tmp_path, boards=["", "metis"]) + store = store_with_assignment(tmp_path) + pool = coordinator.Coordinator(MASTER, store) + original_connect = kanban.connect + kanban.list_boards = lambda include_archived=False: [] + kanban.connect = lambda board=None: (_ for _ in ()).throw(OSError("busy")) + pool.reconcile() + kanban.connect = original_connect + kanban.list_boards = lambda include_archived=False: [""] + pool.reconcile() + + full = SimpleNamespace( + active_assignments=lambda: [], available_ordinals=lambda: [] + ) + coordinator.Coordinator(MASTER, full).reconcile() + + empty = protocol.PoolStore(tmp_path / "missing.db") + monkeypatch.setattr( + coordinator.cli_lane_dispatch, "claim_ready", lambda *_a: [("metis", "missing")] + ) + coordinator.Coordinator(MASTER, empty).dispatch() + + +def test_nonterminal_lease_expiry_is_reoffered_without_kanban_block(tmp_path, monkeypatch): + kanban = install_kanban(monkeypatch, [task()], tmp_path) + store = store_with_assignment(tmp_path) + store.offer(0) + with store._connect() as connection: + connection.execute("UPDATE assignments SET lease_until=1") + coordinator.Coordinator(MASTER, store).expire_leases() + assert kanban.blocked == [] + assert store.active_assignments()[0]["attempt"] == 2 + + +class HTTPStore: + def __init__(self, fail=False): + self.fail = fail + + def available_ordinals(self): + if self.fail: + raise sqlite3.Error("busy") + return [0, 1, 2] + + +class HTTPCoordinator: + def __init__(self, fail_ready=False): + self.store = HTTPStore(fail_ready) + + @staticmethod + def poll(value): + return {"route": "poll", "value": value} + + @staticmethod + def heartbeat(_value): + raise protocol.ProtocolError("stale") + + @staticmethod + def result(_value): + raise RuntimeError("storage") + + +def test_versioned_server_routes_readiness_and_errors(): + handler = server.handler_factory(HTTPCoordinator()) + assert http_request(handler, "/ready") == (200, {"ready": True, "version": 2}) + assert http_request(handler, "/missing")[0] == 404 + body = protocol.canonical_json({"safe": True}) + assert http_request(handler, "/v1/poll", body=body) == ( + 200, {"route": "poll", "value": {"safe": True}} + ) + assert http_request(handler, "/v1/heartbeat", body=body)[0] == 409 + assert http_request(handler, "/v1/result", body=body)[0] == 503 + assert http_request(handler, "/unknown", body=body)[0] == 404 + assert http_request(handler, "/v1/poll", body=b"")[0] == 409 + assert http_request(server.handler_factory(HTTPCoordinator(True)), "/ready")[0] == 503 + + +class RunCoordinator: + instances = [] + + def __init__(self, key, store): + self.key = key + self.store = store + self.calls = [] + self.dispatch_count = 0 + self.__class__.instances.append(self) + + def expire_leases(self): + self.calls.append("expire") + + def recover_results(self): + self.calls.append("recover") + + def reconcile(self): + self.calls.append("reconcile") + + def dispatch(self): + self.calls.append("dispatch") + self.dispatch_count += 1 + if self.dispatch_count > 1: + raise RuntimeError("deferred") + + +def test_server_once_and_maintenance_loop_are_versioned_and_resilient( + tmp_path, monkeypatch, capsys +): + monkeypatch.setattr(server, "STATE_ROOT", tmp_path) + monkeypatch.setattr(server, "read_key", lambda _path: MASTER) + monkeypatch.setattr(sys, "argv", ["pool", "--once"]) + assert server.run(RunCoordinator) == 0 + assert RunCoordinator.instances[-1].calls == [ + "expire", "recover", "reconcile", "dispatch" + ] + + started = [] + + class FakeServer: + def __init__(self, address, _handler, max_workers): + started.append((address, max_workers)) + + def serve_forever(self): + return + + monkeypatch.setattr(sys, "argv", ["pool"]) + monkeypatch.setattr(server, "BoundedHTTPServer", FakeServer) + monkeypatch.setattr( + server.time, + "sleep", + lambda _seconds: (_ for _ in ()).throw(StopIteration()), + ) + with pytest.raises(StopIteration): + server.run(RunCoordinator) + assert started == [(("0.0.0.0", server.PORT), 8)] + assert "maintenance deferred" in capsys.readouterr().err + + +def test_coordinator_compatibility_exports_delegate_to_server(monkeypatch): + marker = object() + monkeypatch.setattr(server, "handler_factory", lambda value: (marker, value)) + assert coordinator.handler_factory(marker) == (marker, marker) + monkeypatch.setattr(server, "run", lambda value: 17 if value is coordinator.Coordinator else 0) + assert coordinator.main() == 17 diff --git a/testing/tests/test_hermes_execution_pool_mediator.py b/testing/tests/test_hermes_execution_pool_mediator.py new file mode 100644 index 00000000..e5c6c996 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_mediator.py @@ -0,0 +1,447 @@ +"""Isolated mediator, SCM gate, and model-facing API contracts.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import threading +import urllib.error +import urllib.request +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +SCM_SCRIPTS = ROOT / "services/hermes/scm-common/scripts" +sys.path[:0] = [str(SCRIPTS), str(SCM_SCRIPTS)] + +import execution_pool_client as client # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_scm as scm # noqa: E402 + + +KEY = b"d" * 64 +RESULT = { + "status": "completed", + "summary": "Completed safely.", + "changed_files": ["safe.py"], + "tests_run": ["pytest"], + "artifacts": [], + "findings": [], + "blockers": [], +} + + +def binding(**changes): + value = { + "board": "metis", "task_id": "t_deadbeef", "run_id": "42", + "worker_ordinal": 0, "attempt": 1, + } + value.update(changes) + return value + + +def payload(**changes): + value = { + "context": "safe objective", + "repo_url": "https://scm.bstein.dev/atlas/metis.git", + "branch": "wt/t_deadbeef", + "base_branch": "main", + } + value.update(changes) + return value + + +def assignment(**payload_changes): + return protocol.sign_envelope( + KEY, "assignment", binding(), payload(**payload_changes) + ) + + +class Response: + def __init__(self, body): + self.body = body + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, _size=-1): + return self.body + + +class FakeSCM: + def __init__(self): + self.submitted = [] + + def checkout(self, _envelope): + return {"workspace": "/workspace/run", "baseline_sha": "a" * 40} + + def submit(self, envelope, request): + self.submitted.append((envelope, request)) + return {"pull_request": "https://scm.bstein.dev/atlas/metis/pulls/7"} + + +def test_result_schema_validation_rejects_every_unsafe_shape(): + invalid = [ + None, + {}, + {"structured": []}, + {"structured": {**RESULT, "extra": True}}, + {"structured": {**RESULT, "status": "unknown"}}, + {"structured": {**RESULT, "summary": ""}}, + {"structured": {**RESULT, "tests_run": "pytest"}}, + {"structured": {**RESULT, "tests_run": [7]}}, + ] + for value in invalid: + with pytest.raises(protocol.ProtocolError): + client._validate_result(value) + assert client._validate_result({"structured": dict(RESULT)})["structured"] == RESULT + + +def test_client_post_verifies_response_and_bounds_body(monkeypatch): + boundary = client.ClientBoundary(KEY, FakeSCM()) + ack = protocol.sign_envelope(KEY, "ack", binding(), {"accepted": True}) + monkeypatch.setattr(client.urllib.request, "urlopen", lambda *_a, **_k: Response(protocol.canonical_json(ack))) + assert boundary._post("/v1/result", ack)["kind"] == "ack" + monkeypatch.setattr( + client.urllib.request, + "urlopen", + lambda *_a, **_k: Response(b"x" * (protocol.MAX_WIRE_BYTES + 1)), + ) + with pytest.raises(protocol.ProtocolError, match="exceeds"): + boundary._post("/v1/result", ack) + + +def test_poll_materializes_only_ordinal_owned_assignment(monkeypatch): + monkeypatch.setattr(client, "ORDINAL", 0) + boundary = client.ClientBoundary(KEY, FakeSCM()) + boundary._post = lambda *_a: protocol.sign_envelope( + KEY, + "ack", + binding(board="", task_id="", run_id="", attempt=0), + {"assignment": None}, + ) + assert boundary.poll() == {"assignment": None} + + boundary._post = lambda *_a: assignment() + result = boundary.poll()["assignment"] + assert result["workspace"] == "/workspace/run" + assert result["protocol_version"] == 2 + assert "signature" not in result + + boundary._post = lambda *_a: protocol.sign_envelope( + KEY, "assignment", binding(worker_ordinal=1), payload() + ) + with pytest.raises(protocol.ProtocolError, match="foreign"): + boundary.poll() + + +def test_heartbeat_and_finish_require_exact_current_binding(monkeypatch): + fake_scm = FakeSCM() + boundary = client.ClientBoundary(KEY, fake_scm) + current = assignment() + boundary.current = current + + def post(path, envelope): + kind = "heartbeat" if path.endswith("heartbeat") else "result" + protocol.verify_envelope(KEY, envelope, expected_kind=kind) + return protocol.sign_envelope( + KEY, "ack", binding(), {"accepted": True, "duplicate": False} + ) + + boundary._post = post + assert boundary.heartbeat( + {"binding": binding(), "payload": {"note": "active"}} + )["ack"]["accepted"] + request = { + "binding": binding(), + "payload": {"structured": dict(RESULT), "returncode": 0}, + "title": "Safe change", + "body": "Evidence", + } + finished = boundary.finish(request) + assert finished["ack"]["accepted"] and boundary.current is None + assert fake_scm.submitted + assert finished["structured"]["artifacts"] == [ + "https://scm.bstein.dev/atlas/metis/pulls/7" + ] + + boundary.current = current + with pytest.raises(protocol.ProtocolError, match="heartbeat payload"): + boundary.heartbeat({"binding": binding(), "payload": "bad"}) + with pytest.raises(protocol.ProtocolError, match="object"): + boundary._current_for(None) + with pytest.raises(protocol.ProtocolError, match="does not own"): + boundary._current_for(binding(attempt=2)) + + +def test_noncompleted_finish_never_invokes_scm_and_bad_ack_is_rejected(): + fake_scm = FakeSCM() + boundary = client.ClientBoundary(KEY, fake_scm) + boundary.current = assignment() + bad = protocol.sign_envelope(KEY, "ack", binding(attempt=2), {"accepted": True}) + boundary._post = lambda *_a: bad + blocked = {**RESULT, "status": "blocked", "blockers": ["capacity"]} + with pytest.raises(protocol.ProtocolError, match="binding changed"): + boundary.finish( + {"binding": binding(), "payload": {"structured": blocked, "returncode": 1}} + ) + assert fake_scm.submitted == [] + + boundary._post = lambda *_a: protocol.sign_envelope( + KEY, "result", binding(), {"accepted": True} + ) + with pytest.raises(protocol.ProtocolError, match="binding changed"): + boundary.heartbeat({"binding": binding(), "payload": {}}) + + +def http_request(handler, path, *, body=None): + server = protocol.BoundedHTTPServer(("127.0.0.1", 0), handler, max_workers=2) + thread = threading.Thread(target=server.handle_request) + thread.start() + request = urllib.request.Request( + f"http://127.0.0.1:{server.server_port}{path}", + data=body, + method="POST" if body is not None else "GET", + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(request, timeout=3) as response: + return response.status, json.loads(response.read()) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read()) + finally: + thread.join(timeout=3) + server.server_close() + + +def test_model_api_exposes_only_gated_state_machine_operations(): + class Boundary: + poll = staticmethod(lambda: {"assignment": None}) + heartbeat = staticmethod(lambda request: {"heartbeat": request["payload"]}) + finish = staticmethod(lambda request: {"finish": request["payload"]}) + + handler = client.handler_factory(Boundary()) + assert http_request(handler, "/ready") == ( + 200, {"protocol_version": 2, "ready": True} + ) + assert http_request(handler, "/missing")[0] == 404 + for operation in ("poll", "heartbeat", "finish"): + body = protocol.canonical_json( + {"operation": operation, "binding": binding(), "payload": {}} + ) + assert http_request(handler, "/v1/client", body=body)[0] == 200 + for value in ( + {"operation": "bypass"}, + {"operation": "poll", "authority": "steal"}, + ): + status, response = http_request( + handler, "/v1/client", body=protocol.canonical_json(value) + ) + assert status == 409 and response.get("error") + + +def test_client_main_validates_ordinal_and_starts_bounded_server(monkeypatch): + monkeypatch.setattr(client, "ORDINAL", -1) + with pytest.raises(SystemExit, match="ORDINAL"): + client.main() + started = [] + + class Server: + def __init__(self, address, _handler, max_workers): + started.append((address, max_workers)) + + def serve_forever(self): + return + + monkeypatch.setattr(client, "ORDINAL", 2) + monkeypatch.setattr(client, "read_key", lambda _path: KEY) + monkeypatch.setattr(client, "BoundedHTTPServer", Server) + monkeypatch.setattr(client, "SCMBoundary", lambda _key: FakeSCM()) + assert client.main() == 0 + assert started == [(("0.0.0.0", client.PORT), 4)] + + +def init_checkout(path, branch="wt/t_deadbeef"): + subprocess.run(["git", "init", "-q", str(path)], check=True) + commands = [ + ("remote", "add", "origin", "https://scm.bstein.dev/atlas/metis.git"), + ( + "remote", + "add", + "hermes-broker", + "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/" + "git/atlas/metis.git", + ), + ("checkout", "-qb", branch), + ("config", "user.email", "test@example.com"), + ("config", "user.name", "Test"), + ] + for command in commands: + subprocess.run(["git", "-C", str(path), *command], check=True) + (path / "tracked").write_text("safe\n") + subprocess.run(["git", "-C", str(path), "add", "tracked"], check=True) + subprocess.run(["git", "-C", str(path), "commit", "-qm", "initial"], check=True) + return subprocess.check_output( + ["git", "-C", str(path), "rev-parse", "HEAD"], text=True + ).strip() + + +def test_scm_run_and_binding_validation(tmp_path, monkeypatch): + checkout = tmp_path / "checkout" + head = init_checkout(checkout) + assert scm._run("rev-parse", "HEAD", cwd=checkout) == head + with pytest.raises(RuntimeError): + scm._run("rev-parse", "missing", cwd=checkout) + monkeypatch.setattr(scm, "MAX_STATUS_BYTES", 1) + with pytest.raises(protocol.ProtocolError, match="output"): + scm._run("rev-parse", "HEAD", cwd=checkout) + + monkeypatch.setattr(scm, "ORDINAL", 0) + assert scm._binding(assignment())[1:] == ("metis", "wt/t_deadbeef", "main") + for envelope in ( + {**assignment(), "kind": "result"}, + {**assignment(), "payload": []}, + assignment(repo_url="https://evil.example/metis.git"), + assignment(branch="main"), + assignment(base_branch="../main"), + protocol.sign_envelope(KEY, "assignment", binding(worker_ordinal=1), payload()), + ): + with pytest.raises(protocol.ProtocolError): + scm._binding(envelope) + + +def test_scm_private_paths_and_text_are_bounded(tmp_path, monkeypatch): + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + target = scm.workspace_path(assignment()) + assert target.name == "42" + assert scm._state_path(assignment()).name == "metis-t_deadbeef-42.json" + bad = {**assignment(), "task_id": "../bad"} + with pytest.raises(protocol.ProtocolError, match="binding"): + scm.workspace_path(bad) + state = tmp_path / "regular" + state.write_text("safe") + assert scm._regular_text(state, 10) == "safe" + with pytest.raises(protocol.ProtocolError, match="invalid"): + scm._regular_text(state, 1) + binary = tmp_path / "binary" + binary.write_bytes(b"\xff") + with pytest.raises(protocol.ProtocolError, match="malformed"): + scm._regular_text(binary, 10) + linked_state = tmp_path / "linked-state" + linked_state.mkdir() + (tmp_path / "state-link").symlink_to(linked_state, target_is_directory=True) + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state-link") + with pytest.raises(protocol.ProtocolError): + scm._state_path(assignment()) + + +def test_boundary_verification_existing_checkout_and_missing_baseline(tmp_path, monkeypatch): + workspace = tmp_path / "workspace" + state_root = tmp_path / "state" + monkeypatch.setattr(scm, "WORKSPACE_ROOT", workspace) + monkeypatch.setattr(scm, "SCM_ROOT", state_root) + monkeypatch.setattr(scm, "ORDINAL", 0) + boundary = scm.Boundary(KEY) + assert boundary.verify(assignment())["kind"] == "assignment" + destination = scm.workspace_path(assignment()) + baseline = init_checkout(destination) + protocol.atomic_json( + scm._state_path(assignment()), + {"baseline_sha": baseline, "repo": "metis", "branch": "wt/t_deadbeef"}, + ) + assert boundary.checkout(assignment())["baseline_sha"] == baseline + scm._state_path(assignment()).write_text("{}") + with pytest.raises(protocol.ProtocolError, match="baseline"): + boundary.checkout(assignment()) + + +@pytest.mark.parametrize("feature_exists", [True, False]) +def test_new_checkout_uses_broker_and_safe_base_fallback( + tmp_path, monkeypatch, feature_exists +): + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + monkeypatch.setattr(scm, "ORDINAL", 0) + calls = [] + + def run(*arguments, cwd=None, timeout=300): + calls.append((arguments, cwd, timeout)) + if arguments[0] == "clone": + branch = arguments[arguments.index("--branch") + 1] + destination = Path(arguments[-1]) + if branch == "wt/t_deadbeef" and not feature_exists: + raise RuntimeError("missing branch") + (destination / ".git").mkdir(parents=True) + return "" + + monkeypatch.setattr(scm, "_run", run) + monkeypatch.setattr(scm, "_workspace_identity", lambda *_a: "a" * 40) + result = scm.Boundary(KEY).checkout(assignment()) + assert result["baseline_sha"] == "a" * 40 + assert any("hermes-scm-broker" in str(call) for call in calls) + if not feature_exists: + assert any(call[0][0] == "checkout" for call in calls) + + +def test_failed_clone_never_deletes_unmanaged_state(tmp_path, monkeypatch): + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + monkeypatch.setattr(scm, "ORDINAL", 0) + + def fail(*arguments, **_kwargs): + destination = Path(arguments[-1]) + destination.mkdir(parents=True, exist_ok=True) + (destination / "preserved").write_text("owner data") + raise RuntimeError("clone failed") + + monkeypatch.setattr(scm, "_run", fail) + with pytest.raises(protocol.ProtocolError, match="unmanaged"): + scm.Boundary(KEY).checkout(assignment()) + assert (scm.workspace_path(assignment()) / "preserved").read_text() == "owner data" + + +def test_draft_reuse_create_and_submit_gates(tmp_path, monkeypatch): + existing = json.dumps([{"html_url": "https://scm/pulls/1"}]).encode() + monkeypatch.setattr(scm.scm_broker_client, "read", lambda _path: existing) + assert scm.Boundary._draft("metis", "wt/task", "main", "a" * 40, "t", "b") == "https://scm/pulls/1" + monkeypatch.setattr(scm.scm_broker_client, "read", lambda _path: b"[]") + monkeypatch.setattr( + scm.scm_broker_client, "create_draft", + lambda *_a, **_k: json.dumps({"html_url": "https://scm/pulls/2"}).encode(), + ) + assert scm.Boundary._draft("metis", "wt/task", "main", "a" * 40, "t", "b") == "https://scm/pulls/2" + + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + monkeypatch.setattr(scm, "ORDINAL", 0) + destination = scm.workspace_path(assignment()) + destination.mkdir(parents=True) + state = scm._state_path(assignment()) + protocol.atomic_json(state, {"baseline_sha": "a" * 40}) + boundary = scm.Boundary(KEY) + monkeypatch.setattr(scm, "_workspace_identity", lambda *_a: "b" * 40) + outputs = {"status": "", "rev-list": "1", "push": ""} + + def run(*arguments, **_kwargs): + return outputs.get(arguments[0], "") + + monkeypatch.setattr(scm, "_run", run) + monkeypatch.setattr(boundary, "_draft", lambda *_a: "https://scm/pulls/3") + result = boundary.submit(assignment(), {"title": "safe", "body": "evidence"}) + assert result["pull_request"] == "https://scm/pulls/3" + outputs["status"] = "?? untracked" + with pytest.raises(protocol.ProtocolError, match="uncommitted"): + boundary.submit(assignment(), {}) + outputs["status"] = "" + outputs["rev-list"] = "0" + assert boundary.submit(assignment(), {})["pull_request"] == "" + with pytest.raises(protocol.ProtocolError, match="metadata"): + boundary.submit(assignment(), {"title": "x" * 513, "body": "x"}) diff --git a/testing/tests/test_hermes_execution_pool_project.py b/testing/tests/test_hermes_execution_pool_project.py new file mode 100644 index 00000000..16c57427 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_project.py @@ -0,0 +1,211 @@ +"""Canonical board-registry and Git-ref assignment contracts.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + +import execution_pool_project as project # noqa: E402 + + +def git(*arguments: str, cwd: Path | None = None) -> str: + return subprocess.check_output( + ["git", *arguments], cwd=cwd, text=True, stderr=subprocess.DEVNULL + ).strip() + + +def registry(tmp_path, monkeypatch, board="metis", remote=None, base=None): + projects = tmp_path / "projects" + boards = tmp_path / "boards" + checkout = projects / board + checkout.mkdir(parents=True) + git("init", "-q", cwd=checkout) + git( + "remote", "add", "origin", + remote or f"https://scm.bstein.dev/atlas/{board}.git", + cwd=checkout, + ) + if base: + git( + "symbolic-ref", "refs/remotes/origin/HEAD", f"refs/remotes/origin/{base}", + cwd=checkout, + ) + entry = boards / board + entry.mkdir(parents=True) + (entry / "board.json").write_text( + json.dumps({"slug": board, "default_workdir": str(checkout)}) + ) + monkeypatch.setattr(project, "PROJECT_ROOT", projects) + monkeypatch.setattr(project, "BOARD_ROOT", boards) + return checkout, entry / "board.json" + + +@pytest.mark.parametrize( + "branch", + [ + "feature/safe", "fix/safe", "chore/safe", "docs/safe", "test/safe", + "refactor/safe", "wt/t_deadbeef", "review/t_deadbeef", "hermes/safe", + "handoff/safe", + ], +) +def test_every_established_feature_namespace_is_accepted(branch): + assert project.validate_branch(branch, feature=True) == branch + + +@pytest.mark.parametrize( + "branch", + ["main", "unknown/safe", "../main", "feature/../../main", "feature/.hidden", "-bad", "é/safe", "x" * 201], +) +def test_ref_validation_denies_traversal_and_unreviewed_namespaces(branch): + with pytest.raises(project.ProjectPolicyError): + project.validate_branch(branch, feature=True) + + +def test_canonical_registry_resolves_repo_base_and_default_branch(tmp_path, monkeypatch): + checkout, _ = registry(tmp_path, monkeypatch, base="trunk") + assert project.resolve_project("metis") == ( + "https://scm.bstein.dev/atlas/metis.git", "trunk", checkout.resolve() + ) + task = SimpleNamespace(id="t_deadbeef", branch_name="") + assert project.resolve_assignment("metis", task) == ( + "https://scm.bstein.dev/atlas/metis.git", "wt/t_deadbeef", "trunk" + ) + + +@pytest.mark.parametrize("board", ["cassandra", "metis", "soteria", "titan-iac"]) +def test_every_atlas_project_uses_its_own_registry_checkout(tmp_path, monkeypatch, board): + registry(tmp_path, monkeypatch, board=board) + repo, base, _ = project.resolve_project(board) + assert repo == f"https://scm.bstein.dev/atlas/{board}.git" + assert base == "main" + + +def test_registered_project_without_checkout_keeps_its_own_repo_identity( + tmp_path, monkeypatch +): + checkout, _ = registry(tmp_path, monkeypatch, board="metis") + shutil.rmtree(checkout) + assert project.resolve_project("metis") == ( + "https://scm.bstein.dev/atlas/metis.git", + "main", + checkout.resolve(), + ) + + +def test_registry_rejects_project_slug_too_long_for_an_atlas_repo( + tmp_path, monkeypatch +): + board = "m" * 101 + checkout, _ = registry(tmp_path, monkeypatch, board=board) + shutil.rmtree(checkout) + with pytest.raises(project.ProjectPolicyError, match="repository identity"): + project.resolve_project(board) + + +def test_noncanonical_remote_head_falls_back_to_main(tmp_path, monkeypatch): + checkout, _ = registry(tmp_path, monkeypatch) + real_run = project._run_git + + def run_git(workdir, *arguments): + if arguments[0] == "symbolic-ref": + return "heads/not-origin" + return real_run(workdir, *arguments) + + monkeypatch.setattr(project, "_run_git", run_git) + assert project.resolve_project("metis") == ( + "https://scm.bstein.dev/atlas/metis.git", + "main", + checkout.resolve(), + ) + + +def test_registry_rejects_slug_traversal_symlink_and_malformed_documents(tmp_path, monkeypatch): + _, board_file = registry(tmp_path, monkeypatch) + with pytest.raises(project.ProjectPolicyError, match="slug"): + project._read_board("../metis") + board_file.write_text("not-json") + with pytest.raises(project.ProjectPolicyError, match="malformed"): + project._read_board("metis") + board_file.write_text(json.dumps({"slug": "other", "default_workdir": "/tmp"})) + with pytest.raises(project.ProjectPolicyError, match="identity"): + project._read_board("metis") + board_file.unlink() + board_file.symlink_to("/etc/passwd") + with pytest.raises(OSError): + project._read_board("metis") + + +def test_registry_rejects_oversized_archived_and_non_regular_entry(tmp_path, monkeypatch): + _, board_file = registry(tmp_path, monkeypatch) + board_file.write_text("x" * (project.MAX_BOARD_BYTES + 1)) + with pytest.raises(project.ProjectPolicyError, match="bounded"): + project._read_board("metis") + board_file.write_text(json.dumps({"slug": "metis", "archived": True})) + with pytest.raises(project.ProjectPolicyError, match="archived"): + project._read_board("metis") + board_file.unlink() + board_file.mkdir() + with pytest.raises(project.ProjectPolicyError, match="regular"): + project._read_board("metis") + + +@pytest.mark.parametrize( + "remote", + [ + "https://token@scm.bstein.dev/atlas/metis.git", + "https://evil.example/atlas/metis.git", + "ssh://git@scm.bstein.dev/atlas/metis.git", + ], +) +def test_registry_rejects_credentialed_or_non_atlas_origin(tmp_path, monkeypatch, remote): + registry(tmp_path, monkeypatch, remote=remote) + with pytest.raises(project.ProjectPolicyError, match="origin"): + project.resolve_project("metis") + + +def test_registry_rejects_missing_or_outside_workdir(tmp_path, monkeypatch): + checkout, board_file = registry(tmp_path, monkeypatch) + board_file.write_text(json.dumps({"slug": "metis"})) + with pytest.raises(project.ProjectPolicyError, match="default_workdir"): + project.resolve_project("metis") + outside = tmp_path / "outside" + checkout.rename(outside) + board_file.write_text( + json.dumps({"slug": "metis", "default_workdir": str(outside)}) + ) + with pytest.raises(project.ProjectPolicyError, match="outside"): + project.resolve_project("metis") + + +def test_registry_rejects_non_directory_checkout(tmp_path, monkeypatch): + checkout, _ = registry(tmp_path, monkeypatch) + shutil.rmtree(checkout) + checkout.write_text("not a checkout") + with pytest.raises(project.ProjectPolicyError, match="not a directory"): + project.resolve_project("metis") + + +def test_git_metadata_failure_and_invalid_task_identity_fail_closed(tmp_path, monkeypatch): + checkout, _ = registry(tmp_path, monkeypatch) + with pytest.raises(project.ProjectPolicyError, match="metadata"): + project._run_git(checkout, "remote", "get-url", "missing") + with pytest.raises(project.ProjectPolicyError, match="identity"): + project.resolve_assignment("metis", SimpleNamespace(id="../../bad")) + + +def test_workspace_migration_boundary_is_explicit(): + assert project.distributed_workspace_eligible(SimpleNamespace(workspace_path="")) + assert project.distributed_workspace_eligible(SimpleNamespace()) + assert not project.distributed_workspace_eligible( + SimpleNamespace(workspace_path="/opt/data/workspace/live") + ) diff --git a/testing/tests/test_hermes_execution_pool_protocol_v2.py b/testing/tests/test_hermes_execution_pool_protocol_v2.py new file mode 100644 index 00000000..aea60727 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_protocol_v2.py @@ -0,0 +1,241 @@ +"""Version-2 protocol reliability and exact-attempt fencing contracts.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import sys +import threading +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + + +SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + +import execution_pool_protocol as protocol # noqa: E402 + + +MASTER = b"m" * 32 + + +def binding(**changes): + value = { + "board": "metis", + "task_id": "t_deadbeef", + "run_id": "42", + "worker_ordinal": 0, + "attempt": 1, + } + value.update(changes) + return value + + +def resign(envelope, key=MASTER): + unsigned = dict(envelope) + unsigned.pop("signature", None) + envelope["signature"] = hmac.new( + key, protocol.canonical_json(unsigned), hashlib.sha256 + ).hexdigest() + return envelope + + +def test_atomic_json_is_private_durable_and_rejects_symlink_parent(tmp_path): + target = tmp_path / "state/value.json" + protocol.atomic_json(target, {"safe": True}) + assert json.loads(target.read_text()) == {"safe": True} + assert target.stat().st_mode & 0o777 == 0o600 + + outside = tmp_path / "outside" + outside.mkdir() + (tmp_path / "linked").symlink_to(outside, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="directory"): + protocol.atomic_json(tmp_path / "linked/value.json", {"safe": False}) + + +def test_ordinal_derivation_and_selector_fail_closed(): + keys = [protocol.derive_ordinal_key(MASTER, ordinal) for ordinal in range(3)] + assert len(set(keys)) == 3 + for master, ordinal in ((b"short", 0), (MASTER, -1), (MASTER, 3)): + with pytest.raises(protocol.ProtocolError, match="derivation"): + protocol.derive_ordinal_key(master, ordinal) + for value in (None, [], {}, {"worker_ordinal": True}, {"worker_ordinal": 3}): + with pytest.raises(protocol.ProtocolError): + protocol.envelope_ordinal(value) + + +def test_key_reader_bounds_content_and_type(tmp_path): + for content in (b"x" * 31, b"x" * 4097): + path = tmp_path / f"key-{len(content)}" + path.write_bytes(content) + path.chmod(0o600) + with pytest.raises(protocol.ProtocolError, match="length"): + protocol.read_key(path) + directory = tmp_path / "directory" + directory.mkdir(mode=0o700) + with pytest.raises(protocol.ProtocolError, match="regular"): + protocol.read_key(directory) + + +def test_envelope_rejects_kind_fields_numeric_lifetime_and_digest(): + with pytest.raises(protocol.ProtocolError, match="unsupported"): + protocol.sign_envelope(MASTER, "admin", binding(), {}) + valid = protocol.sign_envelope(MASTER, "heartbeat", binding(), {}) + + extra = {**valid, "extra": True} + with pytest.raises(protocol.ProtocolError, match="fields"): + protocol.verify_envelope(MASTER, extra) + wrong_version = resign({**valid, "version": 1}) + with pytest.raises(protocol.ProtocolError, match="version"): + protocol.verify_envelope(MASTER, wrong_version) + with pytest.raises(protocol.ProtocolError, match="unexpected"): + protocol.verify_envelope(MASTER, valid, expected_kind="result") + + numeric = resign({**valid, "attempt": "not-a-number"}) + with pytest.raises(protocol.ProtocolError, match="numeric"): + protocol.verify_envelope(MASTER, numeric) + numeric_text = resign({**valid, "attempt": "1"}) + with pytest.raises(protocol.ProtocolError, match="numeric"): + protocol.verify_envelope(MASTER, numeric_text) + typed_identifier = resign({**valid, "run_id": 42}) + with pytest.raises(protocol.ProtocolError, match="run_id"): + protocol.verify_envelope(MASTER, typed_identifier) + lifetime = resign({**valid, "expires_at": valid["issued_at"]}) + with pytest.raises(protocol.ProtocolError, match="lifetime"): + protocol.verify_envelope(MASTER, lifetime) + digest = resign({**valid, "payload_digest": "0" * 64}) + with pytest.raises(protocol.ProtocolError, match="digest"): + protocol.verify_envelope(MASTER, digest) + with pytest.raises(protocol.ProtocolError, match="oversized"): + protocol.verify_envelope(MASTER, {"value": "x" * protocol.MAX_WIRE_BYTES}) + + +def test_wire_parser_rejects_non_object_and_empty(): + for body, message in ((b"", "empty"), (b"[]", "object")): + with pytest.raises(protocol.ProtocolError, match=message): + protocol.parse_wire(body) + + +def test_bounded_http_server_sets_timeout_and_releases_slot(): + handled = threading.Event() + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 + handled.set() + self.send_response(204) + self.end_headers() + + def log_message(self, *_args): + return + + server = protocol.BoundedHTTPServer(("127.0.0.1", 0), Handler, max_workers=1) + thread = threading.Thread(target=server.handle_request) + thread.start() + with urllib.request.urlopen( + f"http://127.0.0.1:{server.server_port}/", timeout=3 + ) as response: + assert response.status == 204 + thread.join(timeout=3) + assert handled.is_set() + acquired = False + for _ in range(100): + acquired = server._slots.acquire(blocking=False) + if acquired: + break + threading.Event().wait(0.01) + assert acquired + server._slots.release() + server.server_close() + + +def test_bounded_server_releases_slot_when_thread_dispatch_raises(monkeypatch): + server = protocol.BoundedHTTPServer( + ("127.0.0.1", 0), BaseHTTPRequestHandler, max_workers=1 + ) + + def explode(*_args): + raise RuntimeError("dispatch failed") + + monkeypatch.setattr(ThreadingHTTPServer, "process_request", explode) + with pytest.raises(RuntimeError, match="dispatch"): + server.process_request(object(), ("127.0.0.1", 1)) + assert server._slots.acquire(blocking=False) + server._slots.release() + server.server_close() + + +def test_store_fences_ordinal_attempt_state_and_expired_lease(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db", lease_seconds=60) + assert protocol.PoolStore._record(None) is None + store.add(binding(), {"context": "safe"}) + store.offer(0) + + foreign = protocol.sign_envelope( + MASTER, "heartbeat", binding(worker_ordinal=1), {"note": "foreign"} + ) + with pytest.raises(protocol.ProtocolError, match="ordinal"): + store.heartbeat(foreign) + + with store._connect() as connection: + connection.execute("UPDATE assignments SET lease_until=1") + expired = protocol.sign_envelope(MASTER, "heartbeat", binding(), {"note": "late"}) + with pytest.raises(protocol.ProtocolError, match="lease expired"): + store.heartbeat(expired) + terminal = protocol.sign_envelope(MASTER, "result", binding(), {"structured": {}}) + with pytest.raises(protocol.ProtocolError, match="lease expired"): + store.accept_result(terminal) + + +def test_store_pending_result_invalid_state_and_exact_finalize(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding(), {"context": "safe"}) + result = protocol.sign_envelope(MASTER, "result", binding(), {"structured": {}}) + record, duplicate = store.accept_result(result) + assert duplicate is False and record["state"] == "result" + assert store.pending_results()[0]["result"] == {"structured": {}} + with pytest.raises(protocol.ProtocolError, match="terminal"): + store.finalize(binding(), "invalid") + store.finalize(binding(attempt=2), "stale") + assert store.pending_results()[0]["state"] == "result" + store.finalize(binding(), "finalized") + heartbeat = protocol.sign_envelope(MASTER, "heartbeat", binding(), {}) + with pytest.raises(protocol.ProtocolError, match="running"): + store.heartbeat(heartbeat) + + +def test_expired_attempt_is_reoffered_then_terminally_released(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding(), {"context": "safe"}) + store.offer(0) + changed = store.expire_leases(now=10_000_000_000, max_attempts=2) + assert changed[0]["state"] == "assigned" and changed[0]["attempt"] == 2 + stale = protocol.sign_envelope(MASTER, "heartbeat", binding(), {}) + with pytest.raises(protocol.ProtocolError, match="attempt is stale"): + store.heartbeat(stale) + + offered = store.offer(0) + assert offered and offered["attempt"] == 2 + changed = store.expire_leases(now=10_000_000_001, max_attempts=2) + assert changed[0]["state"] == "lease_failed" + assert store.available_ordinals() == [0, 1, 2] + + +def test_store_garbage_collection_removes_old_terminal_and_deliveries(tmp_path): + store = protocol.PoolStore(tmp_path / "pool.db") + store.add(binding(), {"context": "safe"}) + heartbeat = protocol.sign_envelope( + MASTER, "heartbeat", binding(), {}, delivery_id="old-delivery" + ) + store.heartbeat(heartbeat) + store.finalize(binding(), "stale") + with store._connect() as connection: + connection.execute("UPDATE assignments SET updated_at=1") + connection.execute("UPDATE deliveries SET received_at=1") + assert store.garbage_collect(3600) == 1 + assert store.active_assignments() == [] + with store._connect() as connection: + assert connection.execute("SELECT count(*) FROM deliveries").fetchone()[0] == 0 diff --git a/testing/tests/test_hermes_execution_pool_scm_tampering.py b/testing/tests/test_hermes_execution_pool_scm_tampering.py new file mode 100644 index 00000000..50bfe9e8 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_scm_tampering.py @@ -0,0 +1,200 @@ +"""SCM checkout ownership and private-baseline corruption probes.""" + +from __future__ import annotations + +import sys +import subprocess +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +sys.path[:0] = [ + str(ROOT / "services/hermes/scripts"), + str(ROOT / "services/hermes/scm-common/scripts"), +] + +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_scm as scm # noqa: E402 +from testing.tests.test_hermes_execution_pool_mediator import ( # noqa: E402 + KEY, + assignment, + init_checkout, +) + + +def test_checkout_and_submit_reject_unmanaged_or_corrupt_private_state( + tmp_path, monkeypatch +): + monkeypatch.setattr(scm, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(scm, "SCM_ROOT", tmp_path / "state") + monkeypatch.setattr(scm, "ORDINAL", 0) + destination = scm.workspace_path(assignment()) + destination.mkdir(parents=True) + (destination / "owner-file").write_text("preserve") + boundary = scm.Boundary(KEY) + with pytest.raises(protocol.ProtocolError, match="non-empty"): + boundary.checkout(assignment()) + + (destination / "owner-file").unlink() + + def clone_with_empty_failure(*arguments, **_kwargs): + if arguments[0] == "clone" and arguments[4] == "wt/t_deadbeef": + destination.mkdir(parents=True, exist_ok=True) + raise RuntimeError("missing") + if arguments[0] == "clone": + (destination / ".git").mkdir(parents=True) + return "" + + monkeypatch.setattr(scm, "_run", clone_with_empty_failure) + monkeypatch.setattr(scm, "_workspace_identity", lambda *_a: "a" * 40) + assert boundary.checkout(assignment())["baseline_sha"] == "a" * 40 + + protocol.atomic_json(scm._state_path(assignment()), {"baseline_sha": "bad"}) + with pytest.raises(protocol.ProtocolError, match="baseline"): + boundary.submit(assignment(), {}) + + +def test_scm_identity_is_data_only_and_supports_packed_refs(tmp_path): + checkout = tmp_path / "checkout" + subprocess.run(["git", "init", "-q", str(checkout)], check=True) + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "add", "origin", + "https://scm.bstein.dev/atlas/titan-iac.git", + ], + check=True, + ) + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "add", "hermes-broker", + scm._broker_repo("titan-iac"), + ], + check=True, + ) + subprocess.run( + [ + "git", "-C", str(checkout), "checkout", "-qb", + "feature/hermes-safe-pool", + ], + check=True, + ) + subprocess.run( + ["git", "-C", str(checkout), "config", "user.email", "a@b.c"], + check=True, + ) + subprocess.run( + ["git", "-C", str(checkout), "config", "user.name", "Test"], + check=True, + ) + (checkout / "tracked").write_text("safe\n") + subprocess.run(["git", "-C", str(checkout), "add", "tracked"], check=True) + subprocess.run( + ["git", "-C", str(checkout), "commit", "-qm", "initial"], check=True + ) + expected = subprocess.check_output( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], text=True + ).strip() + subprocess.run( + ["git", "-C", str(checkout), "pack-refs", "--all"], check=True + ) + assert scm._workspace_identity( + checkout, + "titan-iac", + "feature/hermes-safe-pool", + ) == expected + (checkout / ".git/refs/heads/feature").mkdir(parents=True) + (checkout / ".git/refs/heads/feature/hermes-safe-pool").symlink_to( + "/etc/passwd" + ) + with pytest.raises(protocol.ProtocolError, match="symlink"): + scm._workspace_identity( + checkout, + "titan-iac", + "feature/hermes-safe-pool", + ) + + +def test_scm_paths_and_identity_reject_symlink_and_git_tampering( + tmp_path, monkeypatch +): + workspace = tmp_path / "workspace" + workspace.mkdir() + workspace_link = tmp_path / "workspace-link" + workspace_link.symlink_to(workspace, target_is_directory=True) + monkeypatch.setattr(scm, "WORKSPACE_ROOT", workspace_link) + with pytest.raises(protocol.ProtocolError, match="root"): + scm.workspace_path(assignment()) + + monkeypatch.setattr(scm, "WORKSPACE_ROOT", workspace) + (workspace / "runs").mkdir() + (workspace / "runs/metis").symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="parent"): + scm.workspace_path(assignment()) + + state = tmp_path / "state" + state.mkdir() + monkeypatch.setattr(scm, "SCM_ROOT", state) + expected_state = state / "metis-t_deadbeef-42.json" + expected_state.symlink_to("/etc/passwd") + with pytest.raises(protocol.ProtocolError, match="state"): + scm._state_path(assignment()) + + checkout = tmp_path / "checkout" + checkout.mkdir() + with pytest.raises(protocol.ProtocolError, match="metadata"): + scm._workspace_identity(checkout, "metis", "wt/t_deadbeef") + init_checkout(checkout) + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "set-url", "origin", + "https://evil.example/repo.git", + ], + check=True, + ) + with pytest.raises(protocol.ProtocolError, match="origin"): + scm._workspace_identity(checkout, "metis", "wt/t_deadbeef") + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "set-url", "origin", + "https://scm.bstein.dev/atlas/metis.git", + ], + check=True, + ) + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "set-url", "hermes-broker", + "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/" + "git/atlas/soteria.git", + ], + check=True, + ) + with pytest.raises(protocol.ProtocolError, match="broker remote"): + scm._workspace_identity(checkout, "metis", "wt/t_deadbeef") + subprocess.run( + [ + "git", "-C", str(checkout), "remote", "set-url", "hermes-broker", + "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/" + "git/atlas/metis.git", + ], + check=True, + ) + with pytest.raises(protocol.ProtocolError, match="branch"): + scm._workspace_identity(checkout, "metis", "review/other") + monkeypatch.setattr( + scm, + "_run", + lambda *arguments, **_kwargs: { + "remote": ( + "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/" + "git/atlas/metis.git" + if arguments[-1] == "hermes-broker" + else "https://scm.bstein.dev/atlas/metis.git" + ), + "symbolic-ref": "wt/t_deadbeef", + "rev-parse": "invalid", + }[arguments[0]], + ) + with pytest.raises(protocol.ProtocolError, match="HEAD"): + scm._workspace_identity(checkout, "metis", "wt/t_deadbeef") diff --git a/testing/tests/test_hermes_execution_pool_worker_execute.py b/testing/tests/test_hermes_execution_pool_worker_execute.py new file mode 100644 index 00000000..05b5add4 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_worker_execute.py @@ -0,0 +1,246 @@ +"""Worker execution, fallback, terminal handoff, and exception contracts.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + +from cli_lane_config import ProcessResult, Route # noqa: E402 +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_worker as worker # noqa: E402 +from testing.tests.test_hermes_execution_pool_worker_v2 import assignment # noqa: E402 + + +def route(provider="codex", effort="high"): + return Route( + provider=provider, + model=f"{provider}-model", + effort=effort, + profile="agent", + classifier="test", + reason="unit", + latency_ms=1, + fallback_chain=(provider,), + ) + + +def completed_result(**changes): + structured = { + "status": "completed", + "summary": "Completed safely.", + "changed_files": ["safe.py"], + "tests_run": ["pytest"], + "artifacts": [], + "findings": [], + "blockers": [], + } + value = ProcessResult( + returncode=0, + output="done", + structured=structured, + capacity_failure=False, + ) + for name, replacement in changes.items(): + setattr(value, name, replacement) + return value + + +def prepare_execute(tmp_path, monkeypatch, *, item=None, results=None, git_status=""): + worker_root = tmp_path / "worker" + workspace = worker_root / "runs/metis/t_deadbeef/23" + workspace.mkdir(parents=True) + exact = item or assignment( + workspace=str(workspace), + payload={ + "context": "safe objective", + "assignee": "cli-auto", + "deadline_unix": 10_000_000_000, + "max_runtime_seconds": 3600, + }, + ) + monkeypatch.setattr(worker, "ROOT", worker_root) + monkeypatch.setattr(worker, "ORDINAL", 0) + monkeypatch.setattr(worker, "NODE", "titan-05") + monkeypatch.setattr(worker, "_bind_provider_sessions", lambda _a: None) + monkeypatch.setattr(worker.cli_lane_runner, "load_json", lambda _path: {}) + routes = [route("codex"), route("claude")] + route_calls = [] + + def select_route(*arguments, **keywords): + route_calls.append((arguments, keywords)) + return routes[min(len(route_calls) - 1, len(routes) - 1)] + + monkeypatch.setattr(worker.cli_lane_runner, "select_route", select_route) + monkeypatch.setattr(worker.cli_lane_runner, "fresh_unavailable_provider", lambda: "claude") + monkeypatch.setattr(worker.cli_lane_runner, "git_handoff", lambda *_a: "\nhandoff") + outcomes = list(results or [completed_result()]) + provider_calls = [] + + def run_provider(*arguments): + provider_calls.append(arguments) + return outcomes.pop(0) + + monkeypatch.setattr(worker.cli_lane_runner, "run_provider", run_provider) + client_calls = [] + + def client(operation, **values): + client_calls.append((operation, values)) + return {"ack": {"accepted": True}} + + monkeypatch.setattr(worker, "_client", client) + monkeypatch.setattr(worker, "_git", lambda *_a: git_status) + refreshed = [] + monkeypatch.setattr(worker, "_refresh_assignment", lambda exact: refreshed.append(exact)) + return exact, workspace, client_calls, provider_calls, route_calls, refreshed + + +def test_execute_completed_clean_result_refreshes_and_finishes_exact_run( + tmp_path, monkeypatch +): + exact, _workspace, calls, providers, routes, refreshed = prepare_execute( + tmp_path, monkeypatch + ) + worker.execute(exact) + assert len(providers) == 1 and len(routes) == 1 + assert refreshed == [worker._binding(exact)] + operations = [name for name, _values in calls] + assert operations == ["heartbeat", "finish"] + finish = calls[-1][1]["payload"] + assert finish["structured"]["status"] == "completed" + assert finish["node"] == "titan-05" + state = json.loads(worker._state_path(exact).read_text()) + assert state["terminal_at"] > 0 and state["baseline_sha"] == "a" * 40 + + +def test_execute_capacity_fallback_changes_provider_and_preserves_handoff( + tmp_path, monkeypatch +): + first = completed_result(capacity_failure=True, output="capacity") + second = completed_result() + exact, _workspace, calls, providers, routes, _refreshed = prepare_execute( + tmp_path, monkeypatch, results=[first, second] + ) + worker.execute(exact) + assert len(providers) == 2 and len(routes) == 2 + assert providers[1][0].provider == "claude" + assert providers[1][1].endswith("handoff") + heartbeats = [value for operation, value in calls if operation == "heartbeat"] + assert any("fallback=codex->claude" in value["payload"]["note"] for value in heartbeats) + + +def test_execute_dirty_workspace_downgrades_completed_result(tmp_path, monkeypatch): + exact, _workspace, calls, _providers, _routes, refreshed = prepare_execute( + tmp_path, monkeypatch, git_status="?? untracked" + ) + worker.execute(exact) + structured = calls[-1][1]["payload"]["structured"] + assert structured["status"] == "incomplete" + assert "uncommitted or untracked" in structured["blockers"][0] + assert refreshed == [] + + +def test_execute_fills_missing_lists_for_failed_provider_result(tmp_path, monkeypatch): + failed = ProcessResult( + returncode=1, + output="failed", + structured={"status": "blocked", "summary": "Provider failed."}, + capacity_failure=False, + ) + exact, _workspace, calls, _providers, _routes, _refreshed = prepare_execute( + tmp_path, monkeypatch, results=[failed] + ) + worker.execute(exact) + structured = calls[-1][1]["payload"]["structured"] + for name in ("changed_files", "tests_run", "artifacts", "findings", "blockers"): + assert structured[name] == [] + + +def test_execute_rejects_payload_lease_baseline_and_terminal_ack_failures( + tmp_path, monkeypatch +): + exact, *_ = prepare_execute(tmp_path, monkeypatch) + with pytest.raises(protocol.ProtocolError, match="payload"): + worker.execute({**exact, "payload": []}) + + monkeypatch.setattr(worker, "_client", lambda *_a, **_k: {"ack": {"accepted": False}}) + with pytest.raises(protocol.ProtocolError, match="lease"): + worker.execute(exact) + + exact, *_ = prepare_execute(tmp_path / "baseline", monkeypatch) + exact["baseline_sha"] = "invalid" + with pytest.raises(RuntimeError, match="baseline"): + worker.execute(exact) + + exact, *_ = prepare_execute(tmp_path / "ack", monkeypatch) + calls = [] + + def client(operation, **_values): + calls.append(operation) + return {"ack": {"accepted": operation != "finish"}} + + monkeypatch.setattr(worker, "_client", client) + with pytest.raises(protocol.ProtocolError, match="terminal"): + worker.execute(exact) + + +def test_execute_heartbeat_transport_failure_loses_lease(tmp_path, monkeypatch): + exact, *_ = prepare_execute(tmp_path, monkeypatch) + monkeypatch.setattr( + worker, "_client", lambda *_a, **_k: (_ for _ in ()).throw(ValueError("offline")) + ) + with pytest.raises(protocol.ProtocolError, match="lease"): + worker.execute(exact) + + +def test_report_exception_is_transient_exact_and_transport_safe(monkeypatch): + calls = [] + + def client(operation, **values): + calls.append((operation, values)) + return {"ack": {"accepted": True}} + + monkeypatch.setattr(worker, "_client", client) + assert worker.report_exception(assignment(), RuntimeError("boom")) + payload = calls[0][1]["payload"] + assert payload["capacity_failure"] is True + assert payload["structured"]["status"] == "blocked" + assert payload["structured"]["blockers"] == ["RuntimeError: boom"] + monkeypatch.setattr( + worker, "_client", lambda *_a, **_k: (_ for _ in ()).throw(OSError("offline")) + ) + assert worker.report_exception(assignment(), OSError("boom")) is False + + +def test_main_idle_and_exception_paths_do_not_spin_silently(monkeypatch, capsys): + monkeypatch.setattr(worker, "readiness", lambda: None) + monkeypatch.setattr(worker, "garbage_collect", lambda: 0) + monkeypatch.setattr(worker, "_poll", lambda: None) + monkeypatch.setattr( + worker.time, "sleep", lambda _seconds: (_ for _ in ()).throw(SystemExit("stop")) + ) + with pytest.raises(SystemExit, match="stop"): + worker.main() + + exact = assignment() + monkeypatch.setattr(worker, "_poll", lambda: exact) + monkeypatch.setattr(worker, "execute", lambda _a: (_ for _ in ()).throw(RuntimeError("boom"))) + monkeypatch.setattr(worker, "report_exception", lambda _a, _e: True) + with pytest.raises(SystemExit, match="stop"): + worker.main() + assert "surfaced" in capsys.readouterr().out + + monkeypatch.setattr( + worker, "garbage_collect", lambda: (_ for _ in ()).throw(RuntimeError("gc")) + ) + monkeypatch.setattr(worker, "report_exception", lambda *_a: False) + with pytest.raises(SystemExit, match="stop"): + worker.main() + assert "deferred" in capsys.readouterr().out diff --git a/testing/tests/test_hermes_execution_pool_worker_v2.py b/testing/tests/test_hermes_execution_pool_worker_v2.py new file mode 100644 index 00000000..a1984e56 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_worker_v2.py @@ -0,0 +1,292 @@ +"""Model worker path, evidence, retention, and readiness contracts.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "services/hermes/scripts" +sys.path.insert(0, str(SCRIPTS)) + +import execution_pool_protocol as protocol # noqa: E402 +import execution_pool_worker as worker # noqa: E402 + + +def assignment(**changes): + value = { + "board": "metis", "task_id": "t_deadbeef", "run_id": "23", + "worker_ordinal": 0, "attempt": 1, "protocol_version": 2, + "workspace": "/workspace/runs/metis/t_deadbeef/23", + "baseline_sha": "a" * 40, + "payload": {"context": "safe objective", "assignee": "cli-auto"}, + } + value.update(changes) + return value + + +class Response: + def __init__(self, value): + self.value = value + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, _size=-1): + return self.value + + +def test_post_client_and_poll_validate_every_response_boundary(monkeypatch): + monkeypatch.setattr( + worker.urllib.request, "urlopen", + lambda *_a, **_k: Response(json.dumps({"safe": True}).encode()), + ) + assert worker._post("http://mediator", {}) == {"safe": True} + monkeypatch.setattr( + worker.urllib.request, "urlopen", + lambda *_a, **_k: Response(b"x" * (64 * 1024 + 1)), + ) + with pytest.raises(protocol.ProtocolError, match="wire"): + worker._post("http://mediator", {}) + monkeypatch.setattr( + worker.urllib.request, "urlopen", lambda *_a, **_k: Response(b"[]") + ) + with pytest.raises(protocol.ProtocolError, match="object"): + worker._post("http://mediator", {}) + + monkeypatch.setattr(worker, "_post", lambda *_a, **_k: {"error": "denied"}) + with pytest.raises(protocol.ProtocolError, match="denied"): + worker._client("poll") + monkeypatch.setattr(worker, "_post", lambda *_a, **_k: {"safe": True}) + assert worker._client("poll") == {"safe": True} + monkeypatch.setattr(worker, "ORDINAL", 0) + monkeypatch.setattr(worker, "_client", lambda *_a, **_k: {"assignment": None}) + assert worker._poll() is None + for value in ([], assignment(worker_ordinal=1), assignment(protocol_version=1)): + monkeypatch.setattr(worker, "_client", lambda *_a, value=value, **_k: {"assignment": value}) + with pytest.raises(protocol.ProtocolError, match="foreign"): + worker._poll() + monkeypatch.setattr(worker, "_client", lambda *_a, **_k: {"assignment": assignment()}) + assert worker._poll()["task_id"] == "t_deadbeef" + assert worker._binding(assignment()) == { + "board": "metis", "task_id": "t_deadbeef", "run_id": "23", + "worker_ordinal": 0, "attempt": 1, + } + + +def test_state_path_rejects_traversal_root_and_leaf_symlinks(tmp_path, monkeypatch): + monkeypatch.setattr(worker, "ROOT", tmp_path) + path = worker._state_path(assignment()) + assert path == tmp_path / "session-state/metis/t_deadbeef/23.json" + with pytest.raises(protocol.ProtocolError, match="invalid"): + worker._state_path(assignment(task_id="../bad")) + + state_root = tmp_path / "session-state" + outside = tmp_path / "outside" + path.parent.rmdir() + path.parent.parent.rmdir() + state_root.rmdir() + outside.mkdir() + state_root.symlink_to(outside, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="root"): + worker._state_path(assignment()) + state_root.unlink() + path = worker._state_path(assignment()) + path.symlink_to("/etc/passwd") + with pytest.raises(protocol.ProtocolError, match="symlink"): + worker._state_path(assignment()) + + +def prepare_provider_roots(tmp_path, monkeypatch): + worker_root = tmp_path / "worker" + data_root = tmp_path / "data" + codex = tmp_path / "runtime/codex" + claude = tmp_path / "runtime/claude" + (worker_root / "provider-state").mkdir(parents=True) + data_root.mkdir() + codex.mkdir(parents=True) + claude.mkdir(parents=True) + monkeypatch.setattr(worker, "ROOT", worker_root) + monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root) + monkeypatch.setenv("CODEX_HOME", str(codex)) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(claude)) + return worker_root, data_root, codex, claude + + +def test_provider_session_binding_replaces_only_owned_symlinks(tmp_path, monkeypatch): + worker_root, data_root, codex, claude = prepare_provider_roots(tmp_path, monkeypatch) + old = tmp_path / "old" + old.mkdir() + (data_root / "home").symlink_to(old, target_is_directory=True) + (codex / "sessions").symlink_to(old, target_is_directory=True) + worker._bind_provider_sessions(assignment()) + assert "metis/t_deadbeef/23" in str((data_root / "home").resolve()) + assert "metis/t_deadbeef/23" in str((codex / "sessions").resolve()) + settings = worker_root / "provider-state/metis/t_deadbeef/23/home/.claude/settings.json" + assert json.loads(settings.read_text()) == {} + worker._bind_provider_sessions(assignment()) + + with pytest.raises(protocol.ProtocolError, match="binding"): + worker._bind_provider_sessions(assignment(run_id="../bad")) + monkeypatch.setattr(worker, "ROOT", tmp_path / "missing") + with pytest.raises(protocol.ProtocolError, match="unavailable"): + worker._bind_provider_sessions(assignment()) + + +def test_provider_session_binding_rejects_durable_and_runtime_tampering( + tmp_path, monkeypatch +): + worker_root, data_root, codex, _claude = prepare_provider_roots(tmp_path, monkeypatch) + run_parent = worker_root / "provider-state/metis" + run_parent.symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="session path"): + worker._bind_provider_sessions(assignment()) + run_parent.unlink() + home = worker_root / "provider-state/metis/t_deadbeef/23/home" + home.parent.mkdir(parents=True) + home.symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="HOME"): + worker._bind_provider_sessions(assignment()) + home.unlink() + (data_root / "home").write_text("not-owned") + with pytest.raises(protocol.ProtocolError, match="task-bound"): + worker._bind_provider_sessions(assignment()) + (data_root / "home").unlink() + (codex / "sessions").write_text("not-owned") + with pytest.raises(protocol.ProtocolError, match="not a symlink"): + worker._bind_provider_sessions(assignment()) + (codex / "sessions").unlink() + durable_sessions = ( + worker_root / "provider-state/metis/t_deadbeef/23/codex/sessions" + ) + durable_sessions.rmdir() + durable_sessions.symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(protocol.ProtocolError, match="session path"): + worker._bind_provider_sessions(assignment()) + + +def test_prompt_activity_git_and_result_bounding(tmp_path, monkeypatch): + text = worker._prompt("objective", tmp_path, worker._binding(assignment())) + assert "no Kubernetes identity" in text and "objective" in text + missing = tmp_path / "missing.log" + assert worker._read_activity(missing, 7) == ("", 7) + log = tmp_path / "worker.log" + log.write_text("abc") + assert worker._read_activity(log, 0) == ("abc", 3) + assert worker._read_activity(log, 99) == ("abc", 3) + fifo = tmp_path / "fifo" + os.mkfifo(fifo) + with pytest.raises(protocol.ProtocolError, match="regular"): + worker._read_activity(fifo, 0) + + repo = tmp_path / "repo" + subprocess.run(["git", "init", "-q", str(repo)], check=True) + assert worker._git(repo, "status", "--porcelain") == "" + with pytest.raises(RuntimeError, match="ambiguous"): + worker._git(repo, "rev-parse", "missing") + + value = { + "status": "completed", "summary": "s" * 20_000, + "changed_files": ["x" * 3000] * 100, + "tests_run": "not-list", "artifacts": [], "findings": [], "blockers": [], + } + bounded = worker._bounded_result(value) + assert len(protocol.canonical_json(bounded)) <= 32 * 1024 + assert bounded["tests_run"] == [] and len(bounded["summary"]) <= 8000 + + +def test_refresh_assignment_requires_exact_binding(monkeypatch): + exact = worker._binding(assignment()) + monkeypatch.setattr(worker, "_poll", lambda: assignment()) + assert worker._refresh_assignment(exact)["run_id"] == "23" + for value in (None, assignment(attempt=2)): + monkeypatch.setattr(worker, "_poll", lambda value=value: value) + with pytest.raises(protocol.ProtocolError, match="changed"): + worker._refresh_assignment(exact) + + +def write_gc_state(root, task_id, workspace, terminal_at): + path = root / f"session-state/metis/{task_id}/23.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"terminal_at": terminal_at, "workspace": str(workspace)})) + return path + + +def init_clean_repo(path): + subprocess.run(["git", "init", "-q", str(path)], check=True) + subprocess.run(["git", "-C", str(path), "config", "user.email", "a@b.c"], check=True) + subprocess.run(["git", "-C", str(path), "config", "user.name", "Test"], check=True) + (path / "tracked").write_text("safe") + subprocess.run(["git", "-C", str(path), "add", "tracked"], check=True) + subprocess.run(["git", "-C", str(path), "commit", "-qm", "initial"], check=True) + + +def test_retention_skips_young_outside_dirty_and_symlink_workspaces(tmp_path, monkeypatch): + monkeypatch.setattr(worker, "ROOT", tmp_path) + monkeypatch.setattr(worker, "RETENTION_SECONDS", 3600) + run_root = tmp_path / "runs/metis" + clean = run_root / "clean/23" + dirty = run_root / "dirty/23" + clean.mkdir(parents=True) + dirty.mkdir(parents=True) + init_clean_repo(clean) + init_clean_repo(dirty) + (dirty / "untracked").write_text("dirty") + now = 10_000 + write_gc_state(tmp_path, "clean", clean, 1) + dirty_state = write_gc_state(tmp_path, "dirty", dirty, 1) + young = write_gc_state(tmp_path, "young", clean, now) + outside = write_gc_state(tmp_path, "outside", tmp_path / "missing", 1) + assert worker.garbage_collect(now=now) == 1 + assert not clean.exists() and dirty.exists() + assert dirty_state.exists() and young.exists() and outside.exists() + + link = run_root / "link/23" + link.parent.mkdir() + link.symlink_to(dirty, target_is_directory=True) + link_state = write_gc_state(tmp_path, "link", link, 1) + assert worker.garbage_collect(now=now) == 0 + assert link_state.exists() + + +def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monkeypatch): + worker_root = tmp_path / "worker" + data_root = tmp_path / "data" + codex = tmp_path / "codex" + claude = tmp_path / "claude" + for path in (worker_root, worker_root / "provider-state", data_root, codex, claude): + path.mkdir(parents=True, exist_ok=True) + (codex / "auth.json").write_text("{}") + (claude / ".credentials.json").write_text("{}") + schema = tmp_path / "schema/result.json" + monkeypatch.setattr(worker, "ORDINAL", 0) + monkeypatch.setattr(worker, "ROOT", worker_root) + monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root) + monkeypatch.setattr(worker.cli_lane_runner, "RESULT_SCHEMA_PATH", schema) + monkeypatch.setenv("CODEX_HOME", str(codex)) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(claude)) + polled = [] + monkeypatch.setattr(worker, "_poll", lambda: polled.append(True)) + worker.readiness() + assert polled and json.loads(schema.read_text()) == worker.cli_lane_runner.RESULT_SCHEMA + + monkeypatch.setattr(worker, "ORDINAL", 3) + with pytest.raises(protocol.ProtocolError, match="ordinal"): + worker.readiness() + monkeypatch.setattr(worker, "ORDINAL", 0) + monkeypatch.setattr(worker, "ROOT", tmp_path / "missing") + with pytest.raises(protocol.ProtocolError, match="path"): + worker.readiness() + monkeypatch.setattr(worker, "ROOT", worker_root) + (codex / "auth.json").unlink() + with pytest.raises(protocol.ProtocolError, match="credential"): + worker.readiness() diff --git a/testing/tests/test_hermes_gitea_pr_integration.py b/testing/tests/test_hermes_gitea_pr_integration.py index ee482cfe..b9f44bc4 100644 --- a/testing/tests/test_hermes_gitea_pr_integration.py +++ b/testing/tests/test_hermes_gitea_pr_integration.py @@ -142,7 +142,7 @@ def test_flux_manifest_isolates_vault_token_in_separate_broker_only(): ) ) boundary = common["configMapGenerator"][0] - assert boundary["name"] == "hermes-scm-boundary" + assert boundary["name"] == "hermes-scm-boundary-v2" assert "gitea_api.py=scripts/gitea_api.py" in boundary["files"] assert "gitea_api_policy.py=scripts/gitea_api_policy.py" in boundary["files"] assert "scm_broker_client.py=scripts/scm_broker_client.py" in boundary["files"] diff --git a/testing/tests/test_hermes_node_account_privilege_audit.py b/testing/tests/test_hermes_node_account_privilege_audit.py index 3370fe4f..a8762e54 100644 --- a/testing/tests/test_hermes_node_account_privilege_audit.py +++ b/testing/tests/test_hermes_node_account_privilege_audit.py @@ -2,6 +2,7 @@ from __future__ import annotations +import shutil import subprocess from pathlib import Path @@ -115,8 +116,11 @@ def test_visudo_valid_numeric_and_alias_grants_fail_closed( monkeypatch.setattr(module, "ACCOUNT_UID", 1200) policy = module.HOST_ETC / "sudoers" policy.write_text(value, encoding="utf-8") + visudo = shutil.which("visudo") + if visudo is None: + pytest.skip("visudo is not installed in this test environment") validation = subprocess.run( - ["/usr/bin/visudo", "-c", "-f", str(policy)], + [visudo, "-c", "-f", str(policy)], check=False, capture_output=True, text=True, diff --git a/testing/tests/test_hermes_runtime_access.py b/testing/tests/test_hermes_runtime_access.py index 1ffb6ac7..21054868 100644 --- a/testing/tests/test_hermes_runtime_access.py +++ b/testing/tests/test_hermes_runtime_access.py @@ -3,6 +3,8 @@ from __future__ import annotations import importlib.util +import hashlib +import hmac import json import sys import urllib.request @@ -172,6 +174,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp ) values = { "agent-api-key": "agent-key", + "execution-pool-key": "e" * 64, "chat-relay-key": "relay-key", "node-ssh-private-key": "private-key", "node-ssh-config": "host-config", @@ -204,6 +207,89 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp assert auth == {"version": 1, "providers": {}, "credential_pool": {}} +def test_execution_worker_and_mediator_separate_credentials_and_hmac( + tmp_path: Path, monkeypatch +): + stage = _load("stage_runtime_access") + vault = tmp_path / "vault" + runtime = tmp_path / "runtime" + worker = tmp_path / "worker" + provider_access = tmp_path / "provider-access" + pool_access = tmp_path / "pool-access" + vault.mkdir() + (vault / "claude-credentials-1").write_text( + json.dumps({"claudeAiOauth": {"refreshToken": "claude-refresh"}}) + ) + (vault / "codex-auth-1").write_text( + json.dumps({"tokens": {"refresh_token": "codex-refresh"}}) + ) + monkeypatch.setattr(stage, "VAULT_ROOT", vault) + monkeypatch.setattr(stage, "RUNTIME_ROOT", runtime) + monkeypatch.setattr(stage, "WORKER_ROOT", worker) + monkeypatch.setattr(stage, "PROVIDER_ACCESS_ROOT", provider_access) + monkeypatch.setattr(stage, "POOL_ACCESS_ROOT", pool_access) + monkeypatch.setenv("HERMES_WORKER_ORDINAL", "1") + monkeypatch.setattr(stage.os, "chown", lambda *_args: None) + monkeypatch.setattr(stage.os, "fchown", lambda *_args: None) + + stage.stage_execution_worker() + + assert not pool_access.exists() + assert not (runtime / "execution-pool-key").exists() + assert not (runtime / "codex/sessions").exists() + assert not (runtime / "claude/projects").exists() + assert (provider_access / "codex/auth.json").stat().st_mode & 0o777 == 0o600 + refreshed = {"tokens": {"refresh_token": "provider-rotated"}} + (provider_access / "codex/auth.json").write_text(json.dumps(refreshed)) + (provider_access / "codex/auth.json").chmod(0o600) + stage.stage_execution_worker() + assert json.loads((provider_access / "codex/auth.json").read_text()) == refreshed + + master = "e" * 64 + (vault / "execution-pool-key").write_text(master) + stage.stage_execution_mediator() + expected = hmac.new( + master.encode(), b"hermes-execution-pool-v2:worker:1", hashlib.sha256 + ).hexdigest() + assert (pool_access / "execution-pool-key").read_text().strip() == expected + + +def test_execution_worker_fails_closed_without_channel_credential( + tmp_path: Path, monkeypatch +): + stage = _load("stage_runtime_access") + vault = tmp_path / "vault" + vault.mkdir() + (vault / "claude-credentials-0").write_text( + json.dumps({"claudeAiOauth": {"refreshToken": "claude-refresh"}}) + ) + monkeypatch.setattr(stage, "VAULT_ROOT", vault) + monkeypatch.setattr(stage, "RUNTIME_ROOT", tmp_path / "runtime") + monkeypatch.setattr(stage, "WORKER_ROOT", tmp_path / "worker") + monkeypatch.setattr(stage, "PROVIDER_ACCESS_ROOT", tmp_path / "provider-access") + monkeypatch.setenv("HERMES_WORKER_ORDINAL", "0") + monkeypatch.setattr(stage.os, "chown", lambda *_args: None) + monkeypatch.setattr(stage.os, "fchown", lambda *_args: None) + + with pytest.raises(FileNotFoundError): + stage.stage_execution_worker() + + +def test_execution_worker_rejects_durable_credential_symlink( + tmp_path: Path, monkeypatch +): + stage = _load("stage_runtime_access") + provider = tmp_path / "provider" + provider.mkdir() + (provider / "claude").symlink_to(tmp_path, target_is_directory=True) + monkeypatch.setattr(stage, "WORKER_ROOT", tmp_path / "worker") + monkeypatch.setattr(stage, "PROVIDER_ACCESS_ROOT", provider) + monkeypatch.setenv("HERMES_WORKER_ORDINAL", "0") + monkeypatch.setattr(stage.os, "chown", lambda *_args: None) + with pytest.raises(RuntimeError, match="symlink"): + stage.stage_execution_worker() + + def test_invalid_runtime_json_is_removed(tmp_path: Path, monkeypatch): stage = _load("stage_runtime_access") vault = tmp_path / "vault" @@ -217,7 +303,7 @@ def test_invalid_runtime_json_is_removed(tmp_path: Path, monkeypatch): try: stage._validated_json("credential", destination, ("token",)) - except json.JSONDecodeError: + except RuntimeError: pass else: raise AssertionError("invalid credential JSON should fail staging") diff --git a/testing/tests/test_hermes_runtime_stage_coverage.py b/testing/tests/test_hermes_runtime_stage_coverage.py index 004853c2..3ca8e496 100644 --- a/testing/tests/test_hermes_runtime_stage_coverage.py +++ b/testing/tests/test_hermes_runtime_stage_coverage.py @@ -39,6 +39,21 @@ def test_private_directory_and_secret_copy_validate_mode_and_content( assert (runtime / "value").read_text(encoding="utf-8") == "staged\n" +def test_private_directory_and_vault_projection_reject_invalid_entries( + tmp_path, monkeypatch +): + module, vault, runtime, _home = _stage(tmp_path, monkeypatch) + checks = iter((False, True)) + with monkeypatch.context() as invalid_directory: + invalid_directory.setattr(Path, "is_symlink", lambda _path: next(checks)) + with pytest.raises(RuntimeError, match="is invalid"): + module._owned_directory(runtime) + + (vault / "invalid").write_bytes(b"") + with pytest.raises(RuntimeError, match="projection invalid is invalid"): + module._read_secret("invalid") + + @pytest.mark.parametrize( ("document", "required", "match"), [ @@ -107,6 +122,77 @@ def test_node_ssh_config_rejects_nul_and_removes_partial_file(tmp_path, monkeypa assert not (runtime / "node-ssh-config").exists() +def test_durable_credentials_reject_links_and_unsafe_existing_files( + tmp_path, monkeypatch +): + module, vault, runtime, _home = _stage(tmp_path, monkeypatch) + runtime.mkdir() + (vault / "credential").write_text( + json.dumps({"tokens": {"refresh_token": "synthetic"}}), + encoding="utf-8", + ) + destination = runtime / "credential.json" + destination.symlink_to(vault / "credential") + with pytest.raises(RuntimeError, match="must not be a symlink"): + module._bootstrap_json("credential", destination, ("tokens", "refresh_token")) + + destination.unlink() + destination.write_text( + json.dumps({"tokens": {"refresh_token": "durable"}}), + encoding="utf-8", + ) + destination.chmod(0o644) + with pytest.raises(RuntimeError, match="is unsafe"): + module._bootstrap_json("credential", destination, ("tokens", "refresh_token")) + + +def test_durable_link_creates_repairs_preserves_and_rejects_targets( + tmp_path, monkeypatch +): + module, _vault, _runtime, _home = _stage(tmp_path, monkeypatch) + monkeypatch.setattr(module.os, "chown", lambda *_args: None) + runtime = tmp_path / "worker" + runtime.mkdir() + durable = tmp_path / "provider" + + module._durable_link(runtime, durable, "state") + target = runtime / "state" + assert target.is_symlink() and target.resolve() == durable.resolve() + module._durable_link(runtime, durable, "state") + assert target.is_symlink() + + target.unlink() + wrong = tmp_path / "wrong" + wrong.mkdir() + target.symlink_to(wrong) + module._durable_link(runtime, durable, "state") + assert target.resolve() == durable.resolve() + + target.unlink() + target.write_text("collision", encoding="utf-8") + with pytest.raises(RuntimeError, match="refusing to replace"): + module._durable_link(runtime, durable, "state") + + +@pytest.mark.parametrize( + ("function", "match"), + [ + ("stage_execution_worker", "worker ordinal is invalid"), + ("stage_execution_mediator", "mediator ordinal is invalid"), + ], +) +def test_execution_staging_rejects_invalid_ordinal( + tmp_path, monkeypatch, function, match +): + module, _vault, _runtime, _home = _stage(tmp_path, monkeypatch) + monkeypatch.setattr(module, "WORKER_ROOT", tmp_path / "worker") + monkeypatch.setattr(module, "PROVIDER_ACCESS_ROOT", tmp_path / "provider") + monkeypatch.setattr(module, "POOL_ACCESS_ROOT", tmp_path / "pool") + monkeypatch.setenv("HERMES_WORKER_ORDINAL", "9") + with pytest.raises(RuntimeError, match=match): + getattr(module, function)() + + @pytest.mark.parametrize( ("mode", "secret_name"), [("chat", "chat-relay-key"), ("triage", "triage-api-key")], @@ -124,13 +210,21 @@ def test_isolated_tenant_staging_exposes_only_required_secret( ) -@pytest.mark.parametrize("mode", ["agent", "chat", "triage"]) +@pytest.mark.parametrize( + "mode", ["agent", "chat", "triage", "execution-worker", "execution-mediator"] +) def test_main_dispatches_each_mode(mode, monkeypatch, capsys): module = _load("stage_runtime_access") calls = [] monkeypatch.setattr(module, "stage_agent", lambda: calls.append("agent")) monkeypatch.setattr(module, "stage_chat", lambda: calls.append("chat")) monkeypatch.setattr(module, "stage_triage", lambda: calls.append("triage")) + monkeypatch.setattr( + module, "stage_execution_worker", lambda: calls.append("execution-worker") + ) + monkeypatch.setattr( + module, "stage_execution_mediator", lambda: calls.append("execution-mediator") + ) monkeypatch.setattr(sys, "argv", ["stage-runtime-access", mode]) assert module.main() == 0 assert calls == [mode] diff --git a/testing/tests/test_hermes_scm_broker.py b/testing/tests/test_hermes_scm_broker.py index 2a0fad44..8f1b2ca9 100644 --- a/testing/tests/test_hermes_scm_broker.py +++ b/testing/tests/test_hermes_scm_broker.py @@ -48,7 +48,7 @@ def test_broker_metadata_boundary_reuses_explicit_read_allowlist(): api.authorize_request("GET", path, None) -def test_receive_pack_allows_only_new_namespaced_feature_branch(): +def test_receive_pack_allows_only_new_branches_in_reviewed_namespaces(): broker = _load("scm_broker") zero = b"0" * 40 commit = b"1" * 40 @@ -57,9 +57,18 @@ def test_receive_pack_allows_only_new_namespaced_feature_branch(): _receive_command(zero, commit, b"refs/heads/hermes/focused-fix"), "runtime-sentinel", ) + broker._validate_receive_pack( + _receive_command(zero, commit, b"refs/heads/wt/t_deadbeef"), + "runtime-sentinel", + ) + broker._validate_receive_pack( + _receive_command(zero, commit, b"refs/heads/review/t_deadbeef"), + "runtime-sentinel", + ) for old, new, ref in ( (commit, b"2" * 40, b"refs/heads/hermes/focused-fix"), (commit, zero, b"refs/heads/hermes/focused-fix"), + (commit, commit, b"refs/heads/hermes/focused-fix"), (zero, commit, b"refs/heads/main"), (zero, commit, b"refs/heads/master"), (zero, commit, b"refs/tags/release"),