hermes: keep distributed workers on Claude
This commit is contained in:
jenkins
parent
5136a0026e
commit
69ad76bebc
@ -26,9 +26,9 @@ spec:
|
||||
annotations:
|
||||
ai.bstein.dev/role: fenced-execution-only
|
||||
ai.bstein.dev/scm-boundary: mediated-pr14-broker-with-completion-gates
|
||||
ai.bstein.dev/model-policy: Switchyard AUTO with cross-provider fallback
|
||||
ai.bstein.dev/storage: durable workspace and ordinal-private Codex refresh state
|
||||
ai.bstein.dev/config-rev: "20260822-claude-setup-token-v1"
|
||||
ai.bstein.dev/model-policy: Switchyard effort selection on the Claude subscription lane
|
||||
ai.bstein.dev/storage: durable workspace and ordinal-private provider session state
|
||||
ai.bstein.dev/config-rev: "20260822-claude-setup-token-v2"
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/role: hermes-execution-worker
|
||||
vault.hashicorp.com/agent-inject-containers: stage-worker-access
|
||||
@ -39,24 +39,6 @@ spec:
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ .Data.data.claude_oauth_token }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-codex-auth-0: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-perms-codex-auth-0: "0600"
|
||||
vault.hashicorp.com/agent-inject-template-codex-auth-0: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ .Data.data.execution_worker_0_codex_auth_json }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-codex-auth-1: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-perms-codex-auth-1: "0600"
|
||||
vault.hashicorp.com/agent-inject-template-codex-auth-1: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ .Data.data.execution_worker_1_codex_auth_json }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-codex-auth-2: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-perms-codex-auth-2: "0600"
|
||||
vault.hashicorp.com/agent-inject-template-codex-auth-2: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ .Data.data.execution_worker_2_codex_auth_json }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-pre-populate-only: "true"
|
||||
vault.hashicorp.com/agent-init-first: "true"
|
||||
vault.hashicorp.com/agent-requests-cpu: 2m
|
||||
@ -181,6 +163,7 @@ spec:
|
||||
- {name: CLAUDE_CODE_OAUTH_TOKEN_FILE, value: /claude-oauth-access/token}
|
||||
- {name: HERMES_CLAUDE_BIN, value: /opt/coordinator/claude_oauth_exec}
|
||||
- {name: HERMES_CLAUDE_NATIVE_BIN, value: /worker-data/tools/bin/claude}
|
||||
- {name: HERMES_EXECUTION_DISABLED_PROVIDER, value: codex}
|
||||
- {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
|
||||
- {name: PYTHONPATH, value: /opt/hermes}
|
||||
- {name: PATH, value: /worker-data/tools/bin:/opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
|
||||
@ -194,12 +177,12 @@ spec:
|
||||
fieldPath: spec.nodeName
|
||||
startupProbe:
|
||||
exec:
|
||||
command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json && test -r /claude-oauth-access/token"]
|
||||
command: [/bin/sh, -ec, "test -w /workspace && test -r /claude-oauth-access/token"]
|
||||
periodSeconds: 5
|
||||
failureThreshold: 60
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json && test -r /claude-oauth-access/token"]
|
||||
command: [/bin/sh, -ec, "test -w /workspace && test -r /claude-oauth-access/token"]
|
||||
periodSeconds: 10
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
@ -274,7 +257,7 @@ spec:
|
||||
resources:
|
||||
requests:
|
||||
storage: 2Gi
|
||||
# Codex refresh state and Claude session state remain ordinal-private.
|
||||
# Claude session state remains ordinal-private and survives Pod replacement.
|
||||
- metadata:
|
||||
name: provider-access
|
||||
labels:
|
||||
|
||||
@ -33,6 +33,7 @@ CLIENT = os.environ.get(
|
||||
NODE = os.environ.get("HERMES_WORKER_NODE", "unknown")[:128]
|
||||
RUN_PART = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
|
||||
RETENTION_SECONDS = int(os.environ.get("HERMES_WORKER_RETENTION_SECONDS", "1209600"))
|
||||
DISABLED_PROVIDER = os.environ.get("HERMES_EXECUTION_DISABLED_PROVIDER", "").strip()
|
||||
|
||||
|
||||
def _post(url: str, value: dict[str, Any], timeout: int = 60) -> dict[str, Any]:
|
||||
@ -290,10 +291,13 @@ def execute(assignment: dict[str, Any]) -> None:
|
||||
|
||||
context = str(payload.get("context") or "")
|
||||
assignee = str(payload.get("assignee") or "cli-auto")
|
||||
excluded = None
|
||||
if assignee == "cli-auto":
|
||||
excluded = DISABLED_PROVIDER or cli_lane_runner.fresh_unavailable_provider()
|
||||
route = cli_lane_runner.select_route(
|
||||
context, assignee,
|
||||
exclude_provider=cli_lane_runner.fresh_unavailable_provider()
|
||||
if assignee == "cli-auto" else None,
|
||||
context,
|
||||
assignee,
|
||||
exclude_provider=excluded,
|
||||
)
|
||||
latest_route = asdict(route)
|
||||
if not heartbeat(
|
||||
@ -310,6 +314,11 @@ def execute(assignment: dict[str, Any]) -> None:
|
||||
)
|
||||
if result.capacity_failure and deadline - time.time() > 30:
|
||||
alternate = "claude" if route.provider == "codex" else "codex"
|
||||
if alternate == DISABLED_PROVIDER:
|
||||
alternate = ""
|
||||
else:
|
||||
alternate = ""
|
||||
if alternate:
|
||||
fallback = cli_lane_runner.select_route(
|
||||
context + "\nThe first provider failed from capacity or authentication.",
|
||||
f"cli-{alternate}-{route.effort}",
|
||||
@ -408,22 +417,14 @@ def readiness() -> None:
|
||||
if not path.is_dir() or not os.access(path, os.W_OK):
|
||||
raise ProtocolError(f"durable worker path is not writable: {path}")
|
||||
credentials = (
|
||||
(
|
||||
Path(os.environ.get("CODEX_HOME", "/runtime-access/codex"))
|
||||
/ "auth.json",
|
||||
os.W_OK,
|
||||
),
|
||||
(
|
||||
Path(
|
||||
os.environ.get(
|
||||
"CLAUDE_CODE_OAUTH_TOKEN_FILE", "/claude-oauth-access/token"
|
||||
)
|
||||
),
|
||||
os.R_OK,
|
||||
Path(
|
||||
os.environ.get(
|
||||
"CLAUDE_CODE_OAUTH_TOKEN_FILE", "/claude-oauth-access/token"
|
||||
)
|
||||
),
|
||||
)
|
||||
for credential, access in credentials:
|
||||
if not credential.is_file() or not os.access(credential, access):
|
||||
for credential in credentials:
|
||||
if not credential.is_file() or not os.access(credential, os.R_OK):
|
||||
raise ProtocolError(f"subscription credential is unavailable or read-only: {credential.name}")
|
||||
cli_lane_runner.RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic_json(
|
||||
|
||||
@ -291,7 +291,7 @@ def _durable_link(runtime: Path, durable: Path, name: str) -> None:
|
||||
|
||||
|
||||
def stage_execution_worker() -> None:
|
||||
"""Stage the shared Claude setup token and ordinal-private Codex refresh state."""
|
||||
"""Stage the shared Claude setup token into one isolated worker Pod."""
|
||||
provider_state = WORKER_ROOT / "provider-state"
|
||||
for path in (
|
||||
provider_state,
|
||||
@ -308,11 +308,6 @@ def stage_execution_worker() -> None:
|
||||
"claude-oauth-token",
|
||||
CLAUDE_OAUTH_ACCESS_ROOT / "token",
|
||||
)
|
||||
_bootstrap_json(
|
||||
f"codex-auth-{ordinal}",
|
||||
PROVIDER_ACCESS_ROOT / "codex" / "auth.json",
|
||||
("tokens", "refresh_token"),
|
||||
)
|
||||
|
||||
|
||||
def stage_execution_mediator() -> None:
|
||||
|
||||
@ -261,11 +261,9 @@ def test_retention_skips_young_outside_dirty_and_symlink_workspaces(tmp_path, mo
|
||||
def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monkeypatch):
|
||||
worker_root = tmp_path / "worker"
|
||||
data_root = tmp_path / "data"
|
||||
codex = tmp_path / "codex"
|
||||
claude_token = tmp_path / "claude-oauth/token"
|
||||
for path in (worker_root, worker_root / "provider-state", data_root, codex):
|
||||
for path in (worker_root, worker_root / "provider-state", data_root):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
(codex / "auth.json").write_text("{}")
|
||||
claude_token.parent.mkdir()
|
||||
claude_token.write_text("setup-token")
|
||||
schema = tmp_path / "schema/result.json"
|
||||
@ -273,7 +271,6 @@ def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monke
|
||||
monkeypatch.setattr(worker, "ROOT", worker_root)
|
||||
monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root)
|
||||
monkeypatch.setattr(worker.cli_lane_runner, "RESULT_SCHEMA_PATH", schema)
|
||||
monkeypatch.setenv("CODEX_HOME", str(codex))
|
||||
monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN_FILE", str(claude_token))
|
||||
polled = []
|
||||
monkeypatch.setattr(worker, "_poll", lambda: polled.append(True))
|
||||
@ -288,6 +285,6 @@ def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monke
|
||||
with pytest.raises(protocol.ProtocolError, match="path"):
|
||||
worker.readiness()
|
||||
monkeypatch.setattr(worker, "ROOT", worker_root)
|
||||
(codex / "auth.json").unlink()
|
||||
claude_token.unlink()
|
||||
with pytest.raises(protocol.ProtocolError, match="credential"):
|
||||
worker.readiness()
|
||||
|
||||
@ -150,7 +150,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp
|
||||
assert auth == {"version": 1, "providers": {}, "credential_pool": {}}
|
||||
|
||||
|
||||
def test_execution_worker_and_mediator_separate_credentials_and_hmac(
|
||||
def test_execution_worker_and_mediator_separate_claude_token_and_hmac(
|
||||
tmp_path: Path, monkeypatch
|
||||
):
|
||||
stage = _load("stage_runtime_access")
|
||||
@ -162,9 +162,6 @@ def test_execution_worker_and_mediator_separate_credentials_and_hmac(
|
||||
pool_access = tmp_path / "pool-access"
|
||||
vault.mkdir()
|
||||
(vault / "claude-oauth-token").write_text("claude-setup-token")
|
||||
(vault / "codex-auth-1").write_text(
|
||||
json.dumps({"tokens": {"refresh_token": "codex-refresh"}})
|
||||
)
|
||||
monkeypatch.setattr(stage, "VAULT_ROOT", vault)
|
||||
monkeypatch.setattr(stage, "RUNTIME_ROOT", runtime)
|
||||
monkeypatch.setattr(stage, "WORKER_ROOT", worker)
|
||||
@ -185,12 +182,10 @@ def test_execution_worker_and_mediator_separate_credentials_and_hmac(
|
||||
"claude-setup-token"
|
||||
)
|
||||
assert not (provider_access / "claude/.credentials.json").exists()
|
||||
assert (provider_access / "codex/auth.json").stat().st_mode & 0o777 == 0o600
|
||||
refreshed = {"tokens": {"refresh_token": "provider-rotated"}}
|
||||
(provider_access / "codex/auth.json").write_text(json.dumps(refreshed))
|
||||
(provider_access / "codex/auth.json").chmod(0o600)
|
||||
stage.stage_execution_worker()
|
||||
assert json.loads((provider_access / "codex/auth.json").read_text()) == refreshed
|
||||
assert (claude_oauth_access / "token").read_text().strip() == (
|
||||
"claude-setup-token"
|
||||
)
|
||||
|
||||
master = "e" * 64
|
||||
(vault / "execution-pool-key").write_text(master)
|
||||
@ -207,7 +202,6 @@ def test_execution_worker_fails_closed_without_channel_credential(
|
||||
stage = _load("stage_runtime_access")
|
||||
vault = tmp_path / "vault"
|
||||
vault.mkdir()
|
||||
(vault / "claude-oauth-token").write_text("claude-setup-token")
|
||||
monkeypatch.setattr(stage, "VAULT_ROOT", vault)
|
||||
monkeypatch.setattr(stage, "RUNTIME_ROOT", tmp_path / "runtime")
|
||||
monkeypatch.setattr(stage, "WORKER_ROOT", tmp_path / "worker")
|
||||
@ -398,6 +392,8 @@ def test_claude_clients_use_the_vault_staged_long_lived_token():
|
||||
assert worker_environment["HERMES_CLAUDE_NATIVE_BIN"] == (
|
||||
"/worker-data/tools/bin/claude"
|
||||
)
|
||||
assert worker_environment["HERMES_EXECUTION_DISABLED_PROVIDER"] == "codex"
|
||||
assert not any("codex-auth" in key for key in worker_annotations)
|
||||
oauth_volume = next(
|
||||
item for item in worker_pod["volumes"] if item["name"] == "claude-oauth-access"
|
||||
)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user