diff --git a/services/hermes/execution-worker-statefulset.yaml b/services/hermes/execution-worker-statefulset.yaml index e885e985..8a7c905e 100644 --- a/services/hermes/execution-worker-statefulset.yaml +++ b/services/hermes/execution-worker-statefulset.yaml @@ -26,9 +26,9 @@ spec: annotations: ai.bstein.dev/role: fenced-execution-only ai.bstein.dev/scm-boundary: mediated-pr14-broker-with-completion-gates - ai.bstein.dev/model-policy: Switchyard AUTO with cross-provider fallback - ai.bstein.dev/storage: durable workspace and ordinal-private Codex refresh state - ai.bstein.dev/config-rev: "20260822-claude-setup-token-v1" + ai.bstein.dev/model-policy: Switchyard effort selection on the Claude subscription lane + ai.bstein.dev/storage: durable workspace and ordinal-private provider session state + ai.bstein.dev/config-rev: "20260822-claude-setup-token-v2" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-execution-worker vault.hashicorp.com/agent-inject-containers: stage-worker-access @@ -39,24 +39,6 @@ spec: {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} {{ .Data.data.claude_oauth_token }} {{- end }} - vault.hashicorp.com/agent-inject-secret-codex-auth-0: kv/data/atlas/hermes/agent-tokens - vault.hashicorp.com/agent-inject-perms-codex-auth-0: "0600" - vault.hashicorp.com/agent-inject-template-codex-auth-0: | - {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} - {{ .Data.data.execution_worker_0_codex_auth_json }} - {{- end }} - vault.hashicorp.com/agent-inject-secret-codex-auth-1: kv/data/atlas/hermes/agent-tokens - vault.hashicorp.com/agent-inject-perms-codex-auth-1: "0600" - vault.hashicorp.com/agent-inject-template-codex-auth-1: | - {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} - {{ .Data.data.execution_worker_1_codex_auth_json }} - {{- end }} - vault.hashicorp.com/agent-inject-secret-codex-auth-2: kv/data/atlas/hermes/agent-tokens - vault.hashicorp.com/agent-inject-perms-codex-auth-2: "0600" - vault.hashicorp.com/agent-inject-template-codex-auth-2: | - {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} - {{ .Data.data.execution_worker_2_codex_auth_json }} - {{- end }} vault.hashicorp.com/agent-pre-populate-only: "true" vault.hashicorp.com/agent-init-first: "true" vault.hashicorp.com/agent-requests-cpu: 2m @@ -181,6 +163,7 @@ spec: - {name: CLAUDE_CODE_OAUTH_TOKEN_FILE, value: /claude-oauth-access/token} - {name: HERMES_CLAUDE_BIN, value: /opt/coordinator/claude_oauth_exec} - {name: HERMES_CLAUDE_NATIVE_BIN, value: /worker-data/tools/bin/claude} + - {name: HERMES_EXECUTION_DISABLED_PROVIDER, value: codex} - {name: HERMES_AUTO_ROUTER_PROFILE, value: agent} - {name: PYTHONPATH, value: /opt/hermes} - {name: PATH, value: /worker-data/tools/bin:/opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin} @@ -194,12 +177,12 @@ spec: fieldPath: spec.nodeName startupProbe: exec: - command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json && test -r /claude-oauth-access/token"] + command: [/bin/sh, -ec, "test -w /workspace && test -r /claude-oauth-access/token"] periodSeconds: 5 failureThreshold: 60 readinessProbe: exec: - command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json && test -r /claude-oauth-access/token"] + command: [/bin/sh, -ec, "test -w /workspace && test -r /claude-oauth-access/token"] periodSeconds: 10 securityContext: allowPrivilegeEscalation: false @@ -274,7 +257,7 @@ spec: resources: requests: storage: 2Gi - # Codex refresh state and Claude session state remain ordinal-private. + # Claude session state remains ordinal-private and survives Pod replacement. - metadata: name: provider-access labels: diff --git a/services/hermes/scripts/execution_pool_worker.py b/services/hermes/scripts/execution_pool_worker.py index e00a2eee..29d09483 100644 --- a/services/hermes/scripts/execution_pool_worker.py +++ b/services/hermes/scripts/execution_pool_worker.py @@ -33,6 +33,7 @@ CLIENT = os.environ.get( NODE = os.environ.get("HERMES_WORKER_NODE", "unknown")[:128] RUN_PART = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") RETENTION_SECONDS = int(os.environ.get("HERMES_WORKER_RETENTION_SECONDS", "1209600")) +DISABLED_PROVIDER = os.environ.get("HERMES_EXECUTION_DISABLED_PROVIDER", "").strip() def _post(url: str, value: dict[str, Any], timeout: int = 60) -> dict[str, Any]: @@ -290,10 +291,13 @@ def execute(assignment: dict[str, Any]) -> None: context = str(payload.get("context") or "") assignee = str(payload.get("assignee") or "cli-auto") + excluded = None + if assignee == "cli-auto": + excluded = DISABLED_PROVIDER or cli_lane_runner.fresh_unavailable_provider() route = cli_lane_runner.select_route( - context, assignee, - exclude_provider=cli_lane_runner.fresh_unavailable_provider() - if assignee == "cli-auto" else None, + context, + assignee, + exclude_provider=excluded, ) latest_route = asdict(route) if not heartbeat( @@ -310,6 +314,11 @@ def execute(assignment: dict[str, Any]) -> None: ) if result.capacity_failure and deadline - time.time() > 30: alternate = "claude" if route.provider == "codex" else "codex" + if alternate == DISABLED_PROVIDER: + alternate = "" + else: + alternate = "" + if alternate: fallback = cli_lane_runner.select_route( context + "\nThe first provider failed from capacity or authentication.", f"cli-{alternate}-{route.effort}", @@ -408,22 +417,14 @@ def readiness() -> None: if not path.is_dir() or not os.access(path, os.W_OK): raise ProtocolError(f"durable worker path is not writable: {path}") credentials = ( - ( - Path(os.environ.get("CODEX_HOME", "/runtime-access/codex")) - / "auth.json", - os.W_OK, - ), - ( - Path( - os.environ.get( - "CLAUDE_CODE_OAUTH_TOKEN_FILE", "/claude-oauth-access/token" - ) - ), - os.R_OK, + Path( + os.environ.get( + "CLAUDE_CODE_OAUTH_TOKEN_FILE", "/claude-oauth-access/token" + ) ), ) - for credential, access in credentials: - if not credential.is_file() or not os.access(credential, access): + for credential in credentials: + if not credential.is_file() or not os.access(credential, os.R_OK): raise ProtocolError(f"subscription credential is unavailable or read-only: {credential.name}") cli_lane_runner.RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True) atomic_json( diff --git a/services/hermes/scripts/stage_runtime_access.py b/services/hermes/scripts/stage_runtime_access.py index 06dd5b9d..3d5ad1b0 100644 --- a/services/hermes/scripts/stage_runtime_access.py +++ b/services/hermes/scripts/stage_runtime_access.py @@ -291,7 +291,7 @@ def _durable_link(runtime: Path, durable: Path, name: str) -> None: def stage_execution_worker() -> None: - """Stage the shared Claude setup token and ordinal-private Codex refresh state.""" + """Stage the shared Claude setup token into one isolated worker Pod.""" provider_state = WORKER_ROOT / "provider-state" for path in ( provider_state, @@ -308,11 +308,6 @@ def stage_execution_worker() -> None: "claude-oauth-token", CLAUDE_OAUTH_ACCESS_ROOT / "token", ) - _bootstrap_json( - f"codex-auth-{ordinal}", - PROVIDER_ACCESS_ROOT / "codex" / "auth.json", - ("tokens", "refresh_token"), - ) def stage_execution_mediator() -> None: diff --git a/testing/tests/test_hermes_execution_pool_worker_v2.py b/testing/tests/test_hermes_execution_pool_worker_v2.py index 80a2c153..7bef1f84 100644 --- a/testing/tests/test_hermes_execution_pool_worker_v2.py +++ b/testing/tests/test_hermes_execution_pool_worker_v2.py @@ -261,11 +261,9 @@ def test_retention_skips_young_outside_dirty_and_symlink_workspaces(tmp_path, mo def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monkeypatch): worker_root = tmp_path / "worker" data_root = tmp_path / "data" - codex = tmp_path / "codex" claude_token = tmp_path / "claude-oauth/token" - for path in (worker_root, worker_root / "provider-state", data_root, codex): + for path in (worker_root, worker_root / "provider-state", data_root): path.mkdir(parents=True, exist_ok=True) - (codex / "auth.json").write_text("{}") claude_token.parent.mkdir() claude_token.write_text("setup-token") schema = tmp_path / "schema/result.json" @@ -273,7 +271,6 @@ def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monke monkeypatch.setattr(worker, "ROOT", worker_root) monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root) monkeypatch.setattr(worker.cli_lane_runner, "RESULT_SCHEMA_PATH", schema) - monkeypatch.setenv("CODEX_HOME", str(codex)) monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN_FILE", str(claude_token)) polled = [] monkeypatch.setattr(worker, "_poll", lambda: polled.append(True)) @@ -288,6 +285,6 @@ def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monke with pytest.raises(protocol.ProtocolError, match="path"): worker.readiness() monkeypatch.setattr(worker, "ROOT", worker_root) - (codex / "auth.json").unlink() + claude_token.unlink() with pytest.raises(protocol.ProtocolError, match="credential"): worker.readiness() diff --git a/testing/tests/test_hermes_runtime_access.py b/testing/tests/test_hermes_runtime_access.py index 4c2a126e..3a80feb7 100644 --- a/testing/tests/test_hermes_runtime_access.py +++ b/testing/tests/test_hermes_runtime_access.py @@ -150,7 +150,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp assert auth == {"version": 1, "providers": {}, "credential_pool": {}} -def test_execution_worker_and_mediator_separate_credentials_and_hmac( +def test_execution_worker_and_mediator_separate_claude_token_and_hmac( tmp_path: Path, monkeypatch ): stage = _load("stage_runtime_access") @@ -162,9 +162,6 @@ def test_execution_worker_and_mediator_separate_credentials_and_hmac( pool_access = tmp_path / "pool-access" vault.mkdir() (vault / "claude-oauth-token").write_text("claude-setup-token") - (vault / "codex-auth-1").write_text( - json.dumps({"tokens": {"refresh_token": "codex-refresh"}}) - ) monkeypatch.setattr(stage, "VAULT_ROOT", vault) monkeypatch.setattr(stage, "RUNTIME_ROOT", runtime) monkeypatch.setattr(stage, "WORKER_ROOT", worker) @@ -185,12 +182,10 @@ def test_execution_worker_and_mediator_separate_credentials_and_hmac( "claude-setup-token" ) assert not (provider_access / "claude/.credentials.json").exists() - assert (provider_access / "codex/auth.json").stat().st_mode & 0o777 == 0o600 - refreshed = {"tokens": {"refresh_token": "provider-rotated"}} - (provider_access / "codex/auth.json").write_text(json.dumps(refreshed)) - (provider_access / "codex/auth.json").chmod(0o600) stage.stage_execution_worker() - assert json.loads((provider_access / "codex/auth.json").read_text()) == refreshed + assert (claude_oauth_access / "token").read_text().strip() == ( + "claude-setup-token" + ) master = "e" * 64 (vault / "execution-pool-key").write_text(master) @@ -207,7 +202,6 @@ def test_execution_worker_fails_closed_without_channel_credential( stage = _load("stage_runtime_access") vault = tmp_path / "vault" vault.mkdir() - (vault / "claude-oauth-token").write_text("claude-setup-token") monkeypatch.setattr(stage, "VAULT_ROOT", vault) monkeypatch.setattr(stage, "RUNTIME_ROOT", tmp_path / "runtime") monkeypatch.setattr(stage, "WORKER_ROOT", tmp_path / "worker") @@ -398,6 +392,8 @@ def test_claude_clients_use_the_vault_staged_long_lived_token(): assert worker_environment["HERMES_CLAUDE_NATIVE_BIN"] == ( "/worker-data/tools/bin/claude" ) + assert worker_environment["HERMES_EXECUTION_DISABLED_PROVIDER"] == "codex" + assert not any("codex-auth" in key for key in worker_annotations) oauth_volume = next( item for item in worker_pod["volumes"] if item["name"] == "claude-oauth-access" )