hermes: equip workers with verified Go tools before accepting work
This commit is contained in:
jenkins
parent
d66af9dbd4
commit
3cdcb2a2df
@ -111,6 +111,25 @@ spec:
|
||||
resources:
|
||||
requests: {cpu: 2m, memory: 32Mi}
|
||||
limits: {cpu: 100m, memory: 64Mi}
|
||||
- name: install-worker-go
|
||||
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [/bin/sh, -ec, "timeout 300 /bin/sh /opt/coordinator/install_worker_go.sh"]
|
||||
env:
|
||||
- {name: HERMES_WORKER_TOOLS_DIR, value: /worker-data/tools}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: [ALL]}
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10000
|
||||
runAsGroup: 10000
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
volumeMounts:
|
||||
- {name: tools, mountPath: /worker-data/tools}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
resources:
|
||||
requests: {cpu: 5m, memory: 64Mi}
|
||||
limits: {cpu: "1", memory: 256Mi}
|
||||
- name: install-provider-clis
|
||||
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
|
||||
imagePullPolicy: IfNotPresent
|
||||
|
||||
@ -167,6 +167,7 @@ configMapGenerator:
|
||||
- hermes_stt_client.py=scripts/hermes_stt_client.py
|
||||
- image_broker.py=scripts/image_broker.py
|
||||
- install_agent_tools.sh=scripts/install_agent_tools.sh
|
||||
- install_worker_go.sh=scripts/install_worker_go.sh
|
||||
- jenkins_build_evidence.py=scripts/jenkins_build_evidence.py
|
||||
- jenkins_image_build_trigger.py=scripts/jenkins_image_build_trigger.py
|
||||
- hermes_image_release_status.py=scripts/hermes_image_release_status.py
|
||||
|
||||
@ -429,6 +429,12 @@ def report_exception(assignment: dict[str, Any], error: Exception) -> bool:
|
||||
)
|
||||
except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError):
|
||||
return False
|
||||
def _require_executable(path: Path, provider: str) -> None:
|
||||
"""Fail startup when an enabled provider cannot execute its configured binary."""
|
||||
if not path.is_file() or not os.access(path, os.X_OK):
|
||||
raise ProtocolError(f"configured provider executable is unavailable: {provider}")
|
||||
|
||||
|
||||
def readiness() -> None:
|
||||
if ORDINAL not in range(3):
|
||||
raise ProtocolError("worker ordinal must be 0, 1, or 2")
|
||||
@ -445,6 +451,13 @@ def readiness() -> None:
|
||||
for credential in credentials:
|
||||
if not credential.is_file() or not os.access(credential, os.R_OK):
|
||||
raise ProtocolError(f"subscription credential is unavailable or read-only: {credential.name}")
|
||||
if DISABLED_PROVIDER != "codex":
|
||||
_require_executable(cli_lane_runner.CODEX_BIN, "codex")
|
||||
if DISABLED_PROVIDER != "claude":
|
||||
_require_executable(cli_lane_runner.CLAUDE_BIN, "claude")
|
||||
native = os.environ.get("HERMES_CLAUDE_NATIVE_BIN", "").strip()
|
||||
if native:
|
||||
_require_executable(Path(native), "claude-native")
|
||||
cli_lane_runner.RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic_json(
|
||||
cli_lane_runner.RESULT_SCHEMA_PATH, cli_lane_runner.RESULT_SCHEMA, 0o644
|
||||
|
||||
60
services/hermes/scripts/install_worker_go.sh
Executable file
60
services/hermes/scripts/install_worker_go.sh
Executable file
@ -0,0 +1,60 @@
|
||||
#!/bin/sh
|
||||
# Install only the pinned Go toolchain on one worker's durable tools claim.
|
||||
set -eu
|
||||
|
||||
tools=${HERMES_WORKER_TOOLS_DIR:-/worker-data/tools}
|
||||
bin=${tools}/bin
|
||||
version=1.26.5
|
||||
|
||||
case "$(uname -m)" in
|
||||
aarch64)
|
||||
dl_arch=arm64
|
||||
go_sha=fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49
|
||||
;;
|
||||
x86_64)
|
||||
dl_arch=amd64
|
||||
go_sha=5c2c3b16caefa1d968a94c1daca04a7ca301a496d9b086e17ad77bb81393f053
|
||||
;;
|
||||
*)
|
||||
echo "unsupported architecture for worker Go toolchain" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
root=${tools}/go-${version}-${dl_arch}
|
||||
marker=${tools}/.worker-go-toolchain-${version}-${dl_arch}
|
||||
expected="go version go${version} linux/${dl_arch}"
|
||||
mkdir -p "${bin}"
|
||||
|
||||
verify() {
|
||||
test -x "${root}/bin/go" && test -x "${root}/bin/gofmt" || return 1
|
||||
test "$(timeout 15 "${root}/bin/go" version)" = "${expected}" || return 1
|
||||
probe=$(mktemp "${tools}/.worker-gofmt.XXXXXX") || return 1
|
||||
printf 'package main\nfunc main(){}\n' > "${probe}"
|
||||
timeout 15 "${root}/bin/gofmt" -w "${probe}" || { rm -f "${probe}"; return 1; }
|
||||
rm -f "${probe}"
|
||||
}
|
||||
|
||||
publish() {
|
||||
ln -sfn "../go-${version}-${dl_arch}/bin/go" "${bin}/go"
|
||||
ln -sfn "../go-${version}-${dl_arch}/bin/gofmt" "${bin}/gofmt"
|
||||
test -x "${bin}/go" && test -x "${bin}/gofmt"
|
||||
}
|
||||
|
||||
if ! verify; then
|
||||
rm -f "${marker}"
|
||||
rm -rf "${root}"
|
||||
work=$(mktemp -d "${tools}/.worker-go.XXXXXX")
|
||||
trap 'rm -rf "${work}"' 0 HUP INT TERM
|
||||
archive=${work}/go.tar.gz
|
||||
curl -fsSL --connect-timeout 15 --max-time 90 --retry 2 --retry-delay 2 \
|
||||
-o "${archive}" "https://go.dev/dl/go${version}.linux-${dl_arch}.tar.gz"
|
||||
printf '%s %s\n' "${go_sha}" "${archive}" | sha256sum -c -
|
||||
tar -xzf "${archive}" -C "${work}"
|
||||
test -d "${work}/go"
|
||||
mv "${work}/go" "${root}"
|
||||
fi
|
||||
|
||||
verify
|
||||
publish
|
||||
touch "${marker}"
|
||||
101
testing/tests/test_hermes_execution_pool_worker_go.py
Normal file
101
testing/tests/test_hermes_execution_pool_worker_go.py
Normal file
@ -0,0 +1,101 @@
|
||||
"""Pinned Go bootstrap contracts for isolated Hermes execution workers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
HERMES = ROOT / "services/hermes"
|
||||
INSTALLER = HERMES / "scripts/install_worker_go.sh"
|
||||
|
||||
|
||||
def _write_executable(path: Path, text: str) -> None:
|
||||
path.write_text(text)
|
||||
path.chmod(0o755)
|
||||
|
||||
|
||||
def _run_installer(tmp_path: Path, architecture: str, download_arch: str) -> Path:
|
||||
"""Run the installer against fake verified archive tools without network access."""
|
||||
tools, mock = tmp_path / "tools", tmp_path / "mock"
|
||||
mock.mkdir()
|
||||
curl_log = tmp_path / "curl.log"
|
||||
_write_executable(mock / "uname", '#!/bin/sh\nprintf "%s\\n" "$TEST_ARCH"\n')
|
||||
_write_executable(
|
||||
mock / "curl",
|
||||
"""#!/bin/sh
|
||||
last=""
|
||||
destination=""
|
||||
previous=""
|
||||
for argument in "$@"; do
|
||||
if [ "$previous" = "-o" ]; then destination=$argument; fi
|
||||
previous=$argument
|
||||
last=$argument
|
||||
done
|
||||
printf "%s" "$last" > "$CURL_LOG"
|
||||
printf archive > "$destination"
|
||||
""",
|
||||
)
|
||||
_write_executable(mock / "sha256sum", '#!/bin/sh\ncat >/dev/null\n')
|
||||
_write_executable(
|
||||
mock / "tar",
|
||||
'''#!/bin/sh
|
||||
while [ "$#" -gt 0 ]; do
|
||||
if [ "$1" = "-C" ]; then work=$2; shift 2; continue; fi
|
||||
shift
|
||||
done
|
||||
mkdir -p "$work/go/bin"
|
||||
printf '#!/bin/sh\\nprintf "go version go1.26.5 linux/%%s\\\\n" "$TEST_DL_ARCH"\\n' > "$work/go/bin/go"
|
||||
printf '#!/bin/sh\\n[ "$1" = "-w" ] && [ -f "$2" ]\\n' > "$work/go/bin/gofmt"
|
||||
chmod 0755 "$work/go/bin/go" "$work/go/bin/gofmt"
|
||||
''',
|
||||
)
|
||||
root = tools / f"go-1.26.5-{download_arch}"
|
||||
(root / "bin").mkdir(parents=True)
|
||||
_write_executable(root / "bin/go", "#!/bin/sh\nexit 1\n")
|
||||
environment = {
|
||||
**os.environ,
|
||||
"PATH": f"{mock}:/usr/bin:/bin",
|
||||
"HERMES_WORKER_TOOLS_DIR": str(tools),
|
||||
"TEST_ARCH": architecture,
|
||||
"TEST_DL_ARCH": download_arch,
|
||||
"CURL_LOG": str(curl_log),
|
||||
}
|
||||
subprocess.run(["/bin/sh", str(INSTALLER)], env=environment, check=True)
|
||||
assert curl_log.read_text().endswith(f"go1.26.5.linux-{download_arch}.tar.gz")
|
||||
assert (tools / f".worker-go-toolchain-1.26.5-{download_arch}").is_file()
|
||||
assert subprocess.check_output(
|
||||
[tools / "bin/go", "version"], text=True, env=environment
|
||||
).strip() == f"go version go1.26.5 linux/{download_arch}"
|
||||
subprocess.run([tools / "bin/gofmt", "-w", str(tmp_path / "formatted.go")], check=False)
|
||||
return tools
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("architecture", "download_arch"), [("aarch64", "arm64"), ("x86_64", "amd64")])
|
||||
def test_worker_go_installer_recovers_incomplete_arch_cache(tmp_path, architecture, download_arch):
|
||||
"""An incomplete cached tree is replaced only with the verified native archive."""
|
||||
tools = _run_installer(tmp_path, architecture, download_arch)
|
||||
assert (tools / f"go-1.26.5-{download_arch}/bin/gofmt").is_file()
|
||||
|
||||
|
||||
def test_worker_manifest_installs_only_pinned_go_on_writable_tools_claim():
|
||||
"""The worker gets Go before model binaries without operator credentials or mounts."""
|
||||
document = yaml.safe_load(HERMES.joinpath("execution-worker-statefulset.yaml").read_text())
|
||||
init = next(item for item in document["spec"]["template"]["spec"]["initContainers"] if item["name"] == "install-worker-go")
|
||||
assert init["command"] == [
|
||||
"/bin/sh", "-ec", "timeout 300 /bin/sh /opt/coordinator/install_worker_go.sh"
|
||||
]
|
||||
assert {mount["name"] for mount in init["volumeMounts"]} == {"tools", "coordinator"}
|
||||
assert not next(mount for mount in init["volumeMounts"] if mount["name"] == "tools").get("readOnly", False)
|
||||
assert next(mount for mount in init["volumeMounts"] if mount["name"] == "coordinator")["readOnly"] is True
|
||||
script = INSTALLER.read_text()
|
||||
assert "sha256sum -c -" in script and "go${version}.linux-${dl_arch}.tar.gz" in script
|
||||
assert "version=1.26.5" in script
|
||||
assert 'timeout 15 "${root}/bin/go" version' in script
|
||||
assert 'timeout 15 "${root}/bin/gofmt" -w' in script
|
||||
assert "--connect-timeout 15 --max-time 90 --retry 2 --retry-delay 2" in script
|
||||
@ -319,16 +319,34 @@ def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monke
|
||||
claude_token.parent.mkdir()
|
||||
claude_token.write_text("setup-token")
|
||||
schema = tmp_path / "schema/result.json"
|
||||
claude_bin = tmp_path / "tools/claude"
|
||||
native_claude = tmp_path / "tools/claude-native"
|
||||
codex_bin = tmp_path / "tools/codex"
|
||||
claude_bin.parent.mkdir()
|
||||
for binary in (claude_bin, native_claude):
|
||||
binary.write_text("#!/bin/sh\nexit 0\n")
|
||||
binary.chmod(0o755)
|
||||
monkeypatch.setattr(worker, "ORDINAL", 0)
|
||||
monkeypatch.setattr(worker, "ROOT", worker_root)
|
||||
monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root)
|
||||
monkeypatch.setattr(worker.cli_lane_runner, "RESULT_SCHEMA_PATH", schema)
|
||||
monkeypatch.setattr(worker.cli_lane_runner, "CLAUDE_BIN", claude_bin)
|
||||
monkeypatch.setattr(worker.cli_lane_runner, "CODEX_BIN", codex_bin)
|
||||
monkeypatch.setattr(worker, "DISABLED_PROVIDER", "codex")
|
||||
monkeypatch.setenv("HERMES_CLAUDE_NATIVE_BIN", str(native_claude))
|
||||
monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN_FILE", str(claude_token))
|
||||
monkeypatch.setattr(
|
||||
worker, "_poll", lambda: pytest.fail("readiness must not poll the mediator")
|
||||
)
|
||||
worker.readiness()
|
||||
assert json.loads(schema.read_text()) == worker.cli_lane_runner.RESULT_SCHEMA
|
||||
monkeypatch.setattr(worker, "DISABLED_PROVIDER", "")
|
||||
with pytest.raises(protocol.ProtocolError, match="provider executable.*codex"):
|
||||
worker.readiness()
|
||||
codex_bin.write_text("#!/bin/sh\nexit 0\n")
|
||||
codex_bin.chmod(0o755)
|
||||
worker.readiness()
|
||||
monkeypatch.setattr(worker, "DISABLED_PROVIDER", "codex")
|
||||
|
||||
monkeypatch.setattr(worker, "ORDINAL", 3)
|
||||
with pytest.raises(protocol.ProtocolError, match="ordinal"):
|
||||
@ -338,6 +356,11 @@ def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monke
|
||||
with pytest.raises(protocol.ProtocolError, match="path"):
|
||||
worker.readiness()
|
||||
monkeypatch.setattr(worker, "ROOT", worker_root)
|
||||
native_claude.unlink()
|
||||
with pytest.raises(protocol.ProtocolError, match="provider executable.*claude-native"):
|
||||
worker.readiness()
|
||||
native_claude.write_text("#!/bin/sh\nexit 0\n")
|
||||
native_claude.chmod(0o755)
|
||||
claude_token.unlink()
|
||||
with pytest.raises(protocol.ProtocolError, match="credential"):
|
||||
worker.readiness()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user