hermes: equip workers with verified Go tools before accepting work

services/hermes/execution-worker-statefulset.yaml

@@ -111,6 +111,25 @@ spec:
           resources:
             requests: {cpu: 2m, memory: 32Mi}
             limits: {cpu: 100m, memory: 64Mi}
+        - name: install-worker-go
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
+          imagePullPolicy: IfNotPresent
+          command: [/bin/sh, -ec, "timeout 300 /bin/sh /opt/coordinator/install_worker_go.sh"]
+          env:
+            - {name: HERMES_WORKER_TOOLS_DIR, value: /worker-data/tools}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            runAsNonRoot: true
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: tools, mountPath: /worker-data/tools}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+          resources:
+            requests: {cpu: 5m, memory: 64Mi}
+            limits: {cpu: "1", memory: 256Mi}
         - name: install-provider-clis
           image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
           imagePullPolicy: IfNotPresent

services/hermes/kustomization.yaml

@@ -167,6 +167,7 @@ configMapGenerator:
       - hermes_stt_client.py=scripts/hermes_stt_client.py
       - image_broker.py=scripts/image_broker.py
       - install_agent_tools.sh=scripts/install_agent_tools.sh
+      - install_worker_go.sh=scripts/install_worker_go.sh
       - jenkins_build_evidence.py=scripts/jenkins_build_evidence.py
       - jenkins_image_build_trigger.py=scripts/jenkins_image_build_trigger.py
       - hermes_image_release_status.py=scripts/hermes_image_release_status.py

services/hermes/scripts/execution_pool_worker.py

@@ -429,6 +429,12 @@ def report_exception(assignment: dict[str, Any], error: Exception) -> bool:
         )
     except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError):
         return False
+def _require_executable(path: Path, provider: str) -> None:
+    """Fail startup when an enabled provider cannot execute its configured binary."""
+    if not path.is_file() or not os.access(path, os.X_OK):
+        raise ProtocolError(f"configured provider executable is unavailable: {provider}")
+
+
 def readiness() -> None:
     if ORDINAL not in range(3):
         raise ProtocolError("worker ordinal must be 0, 1, or 2")
@@ -445,6 +451,13 @@ def readiness() -> None:
     for credential in credentials:
         if not credential.is_file() or not os.access(credential, os.R_OK):
             raise ProtocolError(f"subscription credential is unavailable or read-only: {credential.name}")
+    if DISABLED_PROVIDER != "codex":
+        _require_executable(cli_lane_runner.CODEX_BIN, "codex")
+    if DISABLED_PROVIDER != "claude":
+        _require_executable(cli_lane_runner.CLAUDE_BIN, "claude")
+        native = os.environ.get("HERMES_CLAUDE_NATIVE_BIN", "").strip()
+        if native:
+            _require_executable(Path(native), "claude-native")
     cli_lane_runner.RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
     atomic_json(
         cli_lane_runner.RESULT_SCHEMA_PATH, cli_lane_runner.RESULT_SCHEMA, 0o644

services/hermes/scripts/install_worker_go.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+# Install only the pinned Go toolchain on one worker's durable tools claim.
+set -eu
+
+tools=${HERMES_WORKER_TOOLS_DIR:-/worker-data/tools}
+bin=${tools}/bin
+version=1.26.5
+
+case "$(uname -m)" in
+  aarch64)
+    dl_arch=arm64
+    go_sha=fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49
+    ;;
+  x86_64)
+    dl_arch=amd64
+    go_sha=5c2c3b16caefa1d968a94c1daca04a7ca301a496d9b086e17ad77bb81393f053
+    ;;
+  *)
+    echo "unsupported architecture for worker Go toolchain" >&2
+    exit 1
+    ;;
+esac
+
+root=${tools}/go-${version}-${dl_arch}
+marker=${tools}/.worker-go-toolchain-${version}-${dl_arch}
+expected="go version go${version} linux/${dl_arch}"
+mkdir -p "${bin}"
+
+verify() {
+  test -x "${root}/bin/go" && test -x "${root}/bin/gofmt" || return 1
+  test "$(timeout 15 "${root}/bin/go" version)" = "${expected}" || return 1
+  probe=$(mktemp "${tools}/.worker-gofmt.XXXXXX") || return 1
+  printf 'package main\nfunc main(){}\n' > "${probe}"
+  timeout 15 "${root}/bin/gofmt" -w "${probe}" || { rm -f "${probe}"; return 1; }
+  rm -f "${probe}"
+}
+
+publish() {
+  ln -sfn "../go-${version}-${dl_arch}/bin/go" "${bin}/go"
+  ln -sfn "../go-${version}-${dl_arch}/bin/gofmt" "${bin}/gofmt"
+  test -x "${bin}/go" && test -x "${bin}/gofmt"
+}
+
+if ! verify; then
+  rm -f "${marker}"
+  rm -rf "${root}"
+  work=$(mktemp -d "${tools}/.worker-go.XXXXXX")
+  trap 'rm -rf "${work}"' 0 HUP INT TERM
+  archive=${work}/go.tar.gz
+  curl -fsSL --connect-timeout 15 --max-time 90 --retry 2 --retry-delay 2 \
+    -o "${archive}" "https://go.dev/dl/go${version}.linux-${dl_arch}.tar.gz"
+  printf '%s  %s\n' "${go_sha}" "${archive}" | sha256sum -c -
+  tar -xzf "${archive}" -C "${work}"
+  test -d "${work}/go"
+  mv "${work}/go" "${root}"
+fi
+
+verify
+publish
+touch "${marker}"

testing/tests/test_hermes_execution_pool_worker_go.py

@@ -0,0 +1,101 @@
+"""Pinned Go bootstrap contracts for isolated Hermes execution workers."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+from pathlib import Path
+
+import pytest
+import yaml
+
+
+ROOT = Path(__file__).parents[2]
+HERMES = ROOT / "services/hermes"
+INSTALLER = HERMES / "scripts/install_worker_go.sh"
+
+
+def _write_executable(path: Path, text: str) -> None:
+    path.write_text(text)
+    path.chmod(0o755)
+
+
+def _run_installer(tmp_path: Path, architecture: str, download_arch: str) -> Path:
+    """Run the installer against fake verified archive tools without network access."""
+    tools, mock = tmp_path / "tools", tmp_path / "mock"
+    mock.mkdir()
+    curl_log = tmp_path / "curl.log"
+    _write_executable(mock / "uname", '#!/bin/sh\nprintf "%s\\n" "$TEST_ARCH"\n')
+    _write_executable(
+        mock / "curl",
+        """#!/bin/sh
+last=""
+destination=""
+previous=""
+for argument in "$@"; do
+  if [ "$previous" = "-o" ]; then destination=$argument; fi
+  previous=$argument
+  last=$argument
+done
+printf "%s" "$last" > "$CURL_LOG"
+printf archive > "$destination"
+""",
+    )
+    _write_executable(mock / "sha256sum", '#!/bin/sh\ncat >/dev/null\n')
+    _write_executable(
+        mock / "tar",
+        '''#!/bin/sh
+while [ "$#" -gt 0 ]; do
+  if [ "$1" = "-C" ]; then work=$2; shift 2; continue; fi
+  shift
+done
+mkdir -p "$work/go/bin"
+printf '#!/bin/sh\\nprintf "go version go1.26.5 linux/%%s\\\\n" "$TEST_DL_ARCH"\\n' > "$work/go/bin/go"
+printf '#!/bin/sh\\n[ "$1" = "-w" ] && [ -f "$2" ]\\n' > "$work/go/bin/gofmt"
+chmod 0755 "$work/go/bin/go" "$work/go/bin/gofmt"
+''',
+    )
+    root = tools / f"go-1.26.5-{download_arch}"
+    (root / "bin").mkdir(parents=True)
+    _write_executable(root / "bin/go", "#!/bin/sh\nexit 1\n")
+    environment = {
+        **os.environ,
+        "PATH": f"{mock}:/usr/bin:/bin",
+        "HERMES_WORKER_TOOLS_DIR": str(tools),
+        "TEST_ARCH": architecture,
+        "TEST_DL_ARCH": download_arch,
+        "CURL_LOG": str(curl_log),
+    }
+    subprocess.run(["/bin/sh", str(INSTALLER)], env=environment, check=True)
+    assert curl_log.read_text().endswith(f"go1.26.5.linux-{download_arch}.tar.gz")
+    assert (tools / f".worker-go-toolchain-1.26.5-{download_arch}").is_file()
+    assert subprocess.check_output(
+        [tools / "bin/go", "version"], text=True, env=environment
+    ).strip() == f"go version go1.26.5 linux/{download_arch}"
+    subprocess.run([tools / "bin/gofmt", "-w", str(tmp_path / "formatted.go")], check=False)
+    return tools
+
+
+@pytest.mark.parametrize(("architecture", "download_arch"), [("aarch64", "arm64"), ("x86_64", "amd64")])
+def test_worker_go_installer_recovers_incomplete_arch_cache(tmp_path, architecture, download_arch):
+    """An incomplete cached tree is replaced only with the verified native archive."""
+    tools = _run_installer(tmp_path, architecture, download_arch)
+    assert (tools / f"go-1.26.5-{download_arch}/bin/gofmt").is_file()
+
+
+def test_worker_manifest_installs_only_pinned_go_on_writable_tools_claim():
+    """The worker gets Go before model binaries without operator credentials or mounts."""
+    document = yaml.safe_load(HERMES.joinpath("execution-worker-statefulset.yaml").read_text())
+    init = next(item for item in document["spec"]["template"]["spec"]["initContainers"] if item["name"] == "install-worker-go")
+    assert init["command"] == [
+        "/bin/sh", "-ec", "timeout 300 /bin/sh /opt/coordinator/install_worker_go.sh"
+    ]
+    assert {mount["name"] for mount in init["volumeMounts"]} == {"tools", "coordinator"}
+    assert not next(mount for mount in init["volumeMounts"] if mount["name"] == "tools").get("readOnly", False)
+    assert next(mount for mount in init["volumeMounts"] if mount["name"] == "coordinator")["readOnly"] is True
+    script = INSTALLER.read_text()
+    assert "sha256sum -c -" in script and "go${version}.linux-${dl_arch}.tar.gz" in script
+    assert "version=1.26.5" in script
+    assert 'timeout 15 "${root}/bin/go" version' in script
+    assert 'timeout 15 "${root}/bin/gofmt" -w' in script
+    assert "--connect-timeout 15 --max-time 90 --retry 2 --retry-delay 2" in script

testing/tests/test_hermes_execution_pool_worker_v2.py

@@ -319,16 +319,34 @@ def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monke
     claude_token.parent.mkdir()
     claude_token.write_text("setup-token")
     schema = tmp_path / "schema/result.json"
+    claude_bin = tmp_path / "tools/claude"
+    native_claude = tmp_path / "tools/claude-native"
+    codex_bin = tmp_path / "tools/codex"
+    claude_bin.parent.mkdir()
+    for binary in (claude_bin, native_claude):
+        binary.write_text("#!/bin/sh\nexit 0\n")
+        binary.chmod(0o755)
     monkeypatch.setattr(worker, "ORDINAL", 0)
     monkeypatch.setattr(worker, "ROOT", worker_root)
     monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root)
     monkeypatch.setattr(worker.cli_lane_runner, "RESULT_SCHEMA_PATH", schema)
+    monkeypatch.setattr(worker.cli_lane_runner, "CLAUDE_BIN", claude_bin)
+    monkeypatch.setattr(worker.cli_lane_runner, "CODEX_BIN", codex_bin)
+    monkeypatch.setattr(worker, "DISABLED_PROVIDER", "codex")
+    monkeypatch.setenv("HERMES_CLAUDE_NATIVE_BIN", str(native_claude))
     monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN_FILE", str(claude_token))
     monkeypatch.setattr(
         worker, "_poll", lambda: pytest.fail("readiness must not poll the mediator")
     )
     worker.readiness()
     assert json.loads(schema.read_text()) == worker.cli_lane_runner.RESULT_SCHEMA
+    monkeypatch.setattr(worker, "DISABLED_PROVIDER", "")
+    with pytest.raises(protocol.ProtocolError, match="provider executable.*codex"):
+        worker.readiness()
+    codex_bin.write_text("#!/bin/sh\nexit 0\n")
+    codex_bin.chmod(0o755)
+    worker.readiness()
+    monkeypatch.setattr(worker, "DISABLED_PROVIDER", "codex")
 
     monkeypatch.setattr(worker, "ORDINAL", 3)
     with pytest.raises(protocol.ProtocolError, match="ordinal"):
@@ -338,6 +356,11 @@ def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monke
     with pytest.raises(protocol.ProtocolError, match="path"):
         worker.readiness()
     monkeypatch.setattr(worker, "ROOT", worker_root)
+    native_claude.unlink()
+    with pytest.raises(protocol.ProtocolError, match="provider executable.*claude-native"):
+        worker.readiness()
+    native_claude.write_text("#!/bin/sh\nexit 0\n")
+    native_claude.chmod(0o755)
     claude_token.unlink()
     with pytest.raises(protocol.ProtocolError, match="credential"):
         worker.readiness()