diff --git a/services/hermes/execution-worker-statefulset.yaml b/services/hermes/execution-worker-statefulset.yaml index 9207a806..29ca9338 100644 --- a/services/hermes/execution-worker-statefulset.yaml +++ b/services/hermes/execution-worker-statefulset.yaml @@ -111,6 +111,25 @@ spec: resources: requests: {cpu: 2m, memory: 32Mi} limits: {cpu: 100m, memory: 64Mi} + - name: install-worker-go + image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 + imagePullPolicy: IfNotPresent + command: [/bin/sh, -ec, "timeout 300 /bin/sh /opt/coordinator/install_worker_go.sh"] + env: + - {name: HERMES_WORKER_TOOLS_DIR, value: /worker-data/tools} + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: {type: RuntimeDefault} + volumeMounts: + - {name: tools, mountPath: /worker-data/tools} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + resources: + requests: {cpu: 5m, memory: 64Mi} + limits: {cpu: "1", memory: 256Mi} - name: install-provider-clis image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 imagePullPolicy: IfNotPresent diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index da75eff9..df238ff9 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -167,6 +167,7 @@ configMapGenerator: - hermes_stt_client.py=scripts/hermes_stt_client.py - image_broker.py=scripts/image_broker.py - install_agent_tools.sh=scripts/install_agent_tools.sh + - install_worker_go.sh=scripts/install_worker_go.sh - jenkins_build_evidence.py=scripts/jenkins_build_evidence.py - jenkins_image_build_trigger.py=scripts/jenkins_image_build_trigger.py - hermes_image_release_status.py=scripts/hermes_image_release_status.py diff --git a/services/hermes/scripts/execution_pool_worker.py b/services/hermes/scripts/execution_pool_worker.py index ca6f8bbc..99b9ac20 100644 --- a/services/hermes/scripts/execution_pool_worker.py +++ b/services/hermes/scripts/execution_pool_worker.py @@ -429,6 +429,12 @@ def report_exception(assignment: dict[str, Any], error: Exception) -> bool: ) except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError): return False +def _require_executable(path: Path, provider: str) -> None: + """Fail startup when an enabled provider cannot execute its configured binary.""" + if not path.is_file() or not os.access(path, os.X_OK): + raise ProtocolError(f"configured provider executable is unavailable: {provider}") + + def readiness() -> None: if ORDINAL not in range(3): raise ProtocolError("worker ordinal must be 0, 1, or 2") @@ -445,6 +451,13 @@ def readiness() -> None: for credential in credentials: if not credential.is_file() or not os.access(credential, os.R_OK): raise ProtocolError(f"subscription credential is unavailable or read-only: {credential.name}") + if DISABLED_PROVIDER != "codex": + _require_executable(cli_lane_runner.CODEX_BIN, "codex") + if DISABLED_PROVIDER != "claude": + _require_executable(cli_lane_runner.CLAUDE_BIN, "claude") + native = os.environ.get("HERMES_CLAUDE_NATIVE_BIN", "").strip() + if native: + _require_executable(Path(native), "claude-native") cli_lane_runner.RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True) atomic_json( cli_lane_runner.RESULT_SCHEMA_PATH, cli_lane_runner.RESULT_SCHEMA, 0o644 diff --git a/services/hermes/scripts/install_worker_go.sh b/services/hermes/scripts/install_worker_go.sh new file mode 100755 index 00000000..d411467b --- /dev/null +++ b/services/hermes/scripts/install_worker_go.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# Install only the pinned Go toolchain on one worker's durable tools claim. +set -eu + +tools=${HERMES_WORKER_TOOLS_DIR:-/worker-data/tools} +bin=${tools}/bin +version=1.26.5 + +case "$(uname -m)" in + aarch64) + dl_arch=arm64 + go_sha=fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49 + ;; + x86_64) + dl_arch=amd64 + go_sha=5c2c3b16caefa1d968a94c1daca04a7ca301a496d9b086e17ad77bb81393f053 + ;; + *) + echo "unsupported architecture for worker Go toolchain" >&2 + exit 1 + ;; +esac + +root=${tools}/go-${version}-${dl_arch} +marker=${tools}/.worker-go-toolchain-${version}-${dl_arch} +expected="go version go${version} linux/${dl_arch}" +mkdir -p "${bin}" + +verify() { + test -x "${root}/bin/go" && test -x "${root}/bin/gofmt" || return 1 + test "$(timeout 15 "${root}/bin/go" version)" = "${expected}" || return 1 + probe=$(mktemp "${tools}/.worker-gofmt.XXXXXX") || return 1 + printf 'package main\nfunc main(){}\n' > "${probe}" + timeout 15 "${root}/bin/gofmt" -w "${probe}" || { rm -f "${probe}"; return 1; } + rm -f "${probe}" +} + +publish() { + ln -sfn "../go-${version}-${dl_arch}/bin/go" "${bin}/go" + ln -sfn "../go-${version}-${dl_arch}/bin/gofmt" "${bin}/gofmt" + test -x "${bin}/go" && test -x "${bin}/gofmt" +} + +if ! verify; then + rm -f "${marker}" + rm -rf "${root}" + work=$(mktemp -d "${tools}/.worker-go.XXXXXX") + trap 'rm -rf "${work}"' 0 HUP INT TERM + archive=${work}/go.tar.gz + curl -fsSL --connect-timeout 15 --max-time 90 --retry 2 --retry-delay 2 \ + -o "${archive}" "https://go.dev/dl/go${version}.linux-${dl_arch}.tar.gz" + printf '%s %s\n' "${go_sha}" "${archive}" | sha256sum -c - + tar -xzf "${archive}" -C "${work}" + test -d "${work}/go" + mv "${work}/go" "${root}" +fi + +verify +publish +touch "${marker}" diff --git a/testing/tests/test_hermes_execution_pool_worker_go.py b/testing/tests/test_hermes_execution_pool_worker_go.py new file mode 100644 index 00000000..4acd8255 --- /dev/null +++ b/testing/tests/test_hermes_execution_pool_worker_go.py @@ -0,0 +1,101 @@ +"""Pinned Go bootstrap contracts for isolated Hermes execution workers.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest +import yaml + + +ROOT = Path(__file__).parents[2] +HERMES = ROOT / "services/hermes" +INSTALLER = HERMES / "scripts/install_worker_go.sh" + + +def _write_executable(path: Path, text: str) -> None: + path.write_text(text) + path.chmod(0o755) + + +def _run_installer(tmp_path: Path, architecture: str, download_arch: str) -> Path: + """Run the installer against fake verified archive tools without network access.""" + tools, mock = tmp_path / "tools", tmp_path / "mock" + mock.mkdir() + curl_log = tmp_path / "curl.log" + _write_executable(mock / "uname", '#!/bin/sh\nprintf "%s\\n" "$TEST_ARCH"\n') + _write_executable( + mock / "curl", + """#!/bin/sh +last="" +destination="" +previous="" +for argument in "$@"; do + if [ "$previous" = "-o" ]; then destination=$argument; fi + previous=$argument + last=$argument +done +printf "%s" "$last" > "$CURL_LOG" +printf archive > "$destination" +""", + ) + _write_executable(mock / "sha256sum", '#!/bin/sh\ncat >/dev/null\n') + _write_executable( + mock / "tar", + '''#!/bin/sh +while [ "$#" -gt 0 ]; do + if [ "$1" = "-C" ]; then work=$2; shift 2; continue; fi + shift +done +mkdir -p "$work/go/bin" +printf '#!/bin/sh\\nprintf "go version go1.26.5 linux/%%s\\\\n" "$TEST_DL_ARCH"\\n' > "$work/go/bin/go" +printf '#!/bin/sh\\n[ "$1" = "-w" ] && [ -f "$2" ]\\n' > "$work/go/bin/gofmt" +chmod 0755 "$work/go/bin/go" "$work/go/bin/gofmt" +''', + ) + root = tools / f"go-1.26.5-{download_arch}" + (root / "bin").mkdir(parents=True) + _write_executable(root / "bin/go", "#!/bin/sh\nexit 1\n") + environment = { + **os.environ, + "PATH": f"{mock}:/usr/bin:/bin", + "HERMES_WORKER_TOOLS_DIR": str(tools), + "TEST_ARCH": architecture, + "TEST_DL_ARCH": download_arch, + "CURL_LOG": str(curl_log), + } + subprocess.run(["/bin/sh", str(INSTALLER)], env=environment, check=True) + assert curl_log.read_text().endswith(f"go1.26.5.linux-{download_arch}.tar.gz") + assert (tools / f".worker-go-toolchain-1.26.5-{download_arch}").is_file() + assert subprocess.check_output( + [tools / "bin/go", "version"], text=True, env=environment + ).strip() == f"go version go1.26.5 linux/{download_arch}" + subprocess.run([tools / "bin/gofmt", "-w", str(tmp_path / "formatted.go")], check=False) + return tools + + +@pytest.mark.parametrize(("architecture", "download_arch"), [("aarch64", "arm64"), ("x86_64", "amd64")]) +def test_worker_go_installer_recovers_incomplete_arch_cache(tmp_path, architecture, download_arch): + """An incomplete cached tree is replaced only with the verified native archive.""" + tools = _run_installer(tmp_path, architecture, download_arch) + assert (tools / f"go-1.26.5-{download_arch}/bin/gofmt").is_file() + + +def test_worker_manifest_installs_only_pinned_go_on_writable_tools_claim(): + """The worker gets Go before model binaries without operator credentials or mounts.""" + document = yaml.safe_load(HERMES.joinpath("execution-worker-statefulset.yaml").read_text()) + init = next(item for item in document["spec"]["template"]["spec"]["initContainers"] if item["name"] == "install-worker-go") + assert init["command"] == [ + "/bin/sh", "-ec", "timeout 300 /bin/sh /opt/coordinator/install_worker_go.sh" + ] + assert {mount["name"] for mount in init["volumeMounts"]} == {"tools", "coordinator"} + assert not next(mount for mount in init["volumeMounts"] if mount["name"] == "tools").get("readOnly", False) + assert next(mount for mount in init["volumeMounts"] if mount["name"] == "coordinator")["readOnly"] is True + script = INSTALLER.read_text() + assert "sha256sum -c -" in script and "go${version}.linux-${dl_arch}.tar.gz" in script + assert "version=1.26.5" in script + assert 'timeout 15 "${root}/bin/go" version' in script + assert 'timeout 15 "${root}/bin/gofmt" -w' in script + assert "--connect-timeout 15 --max-time 90 --retry 2 --retry-delay 2" in script diff --git a/testing/tests/test_hermes_execution_pool_worker_v2.py b/testing/tests/test_hermes_execution_pool_worker_v2.py index 76306ff8..584b3c61 100644 --- a/testing/tests/test_hermes_execution_pool_worker_v2.py +++ b/testing/tests/test_hermes_execution_pool_worker_v2.py @@ -319,16 +319,34 @@ def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monke claude_token.parent.mkdir() claude_token.write_text("setup-token") schema = tmp_path / "schema/result.json" + claude_bin = tmp_path / "tools/claude" + native_claude = tmp_path / "tools/claude-native" + codex_bin = tmp_path / "tools/codex" + claude_bin.parent.mkdir() + for binary in (claude_bin, native_claude): + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o755) monkeypatch.setattr(worker, "ORDINAL", 0) monkeypatch.setattr(worker, "ROOT", worker_root) monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root) monkeypatch.setattr(worker.cli_lane_runner, "RESULT_SCHEMA_PATH", schema) + monkeypatch.setattr(worker.cli_lane_runner, "CLAUDE_BIN", claude_bin) + monkeypatch.setattr(worker.cli_lane_runner, "CODEX_BIN", codex_bin) + monkeypatch.setattr(worker, "DISABLED_PROVIDER", "codex") + monkeypatch.setenv("HERMES_CLAUDE_NATIVE_BIN", str(native_claude)) monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN_FILE", str(claude_token)) monkeypatch.setattr( worker, "_poll", lambda: pytest.fail("readiness must not poll the mediator") ) worker.readiness() assert json.loads(schema.read_text()) == worker.cli_lane_runner.RESULT_SCHEMA + monkeypatch.setattr(worker, "DISABLED_PROVIDER", "") + with pytest.raises(protocol.ProtocolError, match="provider executable.*codex"): + worker.readiness() + codex_bin.write_text("#!/bin/sh\nexit 0\n") + codex_bin.chmod(0o755) + worker.readiness() + monkeypatch.setattr(worker, "DISABLED_PROVIDER", "codex") monkeypatch.setattr(worker, "ORDINAL", 3) with pytest.raises(protocol.ProtocolError, match="ordinal"): @@ -338,6 +356,11 @@ def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monke with pytest.raises(protocol.ProtocolError, match="path"): worker.readiness() monkeypatch.setattr(worker, "ROOT", worker_root) + native_claude.unlink() + with pytest.raises(protocol.ProtocolError, match="provider executable.*claude-native"): + worker.readiness() + native_claude.write_text("#!/bin/sh\nexit 0\n") + native_claude.chmod(0o755) claude_token.unlink() with pytest.raises(protocol.ProtocolError, match="credential"): worker.readiness()