From 12a6d2c4f54c88fcf7177024e42eee8fe12b05a4 Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 25 Aug 2026 17:37:57 -0300 Subject: [PATCH] hermes(agent): make runtime tooling install architecture-aware The hermes-agent installs its CLI toolchain at runtime into the shared /opt/data/tools Longhorn volume, but every download hardcoded arm64. On the amd64 node titan-22 that left configure-agent-clients failing with "Missing optional dependency @openai/codex-linux-x64" and the operator toolchain fetching arm64 binaries, so the pod churned. Detect the running node's arch (uname -m; fail closed on anything but aarch64/x86_64) and resolve every asset per-arch: - install-agent-tools init script (agent-deployment.yaml): ttyd and kubectl download the arch-correct asset with the arch-correct sha256 (real ttyd 1.7.7 x86_64 and kubectl v1.33.3 amd64 checksums added; the arm64 ones kept). The npm CLI stamp is now arch-specific (.cli-versions--${arch}) so a fresh arch re-runs npm install and pulls its own native optional deps; npm keeps both arches' packages. - install_agent_tools.sh: flux/helm/kustomize/jq/yq/gh/vault/sops/age/ k9s/terraform/go URLs, tarball subdirs (helm linux-${arch}, gh dir), and checksums are all arch-resolved with both arches pinned. Stamps and the Go tree are arch-specific, and an active-arch marker forces a republish of the single-arch ${bin} binaries when the pod moves between arches on the shared volume. Single fetch/verify helper kept. Tests updated to assert the arch-aware form (both arches' Go checksums, ${dl_arch} templating) instead of the arm64-only literal. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf --- services/hermes/agent-deployment.yaml | 40 +++++- .../hermes/scripts/install_agent_tools.sh | 127 +++++++++++++----- .../test_hermes_agent_runtime_patches.py | 11 +- .../tests/test_hermes_cli_lanes_toolchain.py | 7 +- 4 files changed, 145 insertions(+), 40 deletions(-) diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 9fce2b2c..eb8094f5 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -394,25 +394,53 @@ spec: set -eu tools=/opt/data/tools mkdir -p "${tools}/bin" + # The tools volume is shared across whatever node the pod lands + # on, so this install must resolve the running node's arch instead + # of a hardcoded one. Fail closed on anything but the two arches we + # publish images for. + arch="$(uname -m)" + case "${arch}" in + aarch64) + ttyd_asset=ttyd.aarch64 + ttyd_sha=b38acadd89d1d396a0f5649aa52c539edbad07f4bc7348b27b4f4b7219dd4165 + kubectl_arch=arm64 + kubectl_sha=3d514dbae5dc8c09f773df0ef0f5d449dfad05b3aca5c96b13565f886df345fd + ;; + x86_64) + ttyd_asset=ttyd.x86_64 + ttyd_sha=8a217c968aba172e0dbf3f34447218dc015bc4d5e59bf51db2f2cd12b7be4f55 + kubectl_arch=amd64 + kubectl_sha=2fcf65c64f352742dc253a25a7c95617c2aba79843d1b74e585c69fe4884afb0 + ;; + *) + echo "unsupported architecture for agent tools: ${arch}" >&2 + exit 1 + ;; + esac ttyd_version="$("${tools}/bin/ttyd" --version 2>/dev/null || true)" case "${ttyd_version}" in *1.7.7*) ttyd_ready=1 ;; *) ttyd_ready=0 ;; esac if [ "${ttyd_ready}" != "1" ]; then - curl -fsSL -o "${tools}/bin/ttyd.tmp" https://github.com/tsl0922/ttyd/releases/download/1.7.7/ttyd.aarch64 - printf '%s %s\n' b38acadd89d1d396a0f5649aa52c539edbad07f4bc7348b27b4f4b7219dd4165 "${tools}/bin/ttyd.tmp" | sha256sum -c - + curl -fsSL -o "${tools}/bin/ttyd.tmp" "https://github.com/tsl0922/ttyd/releases/download/1.7.7/${ttyd_asset}" + printf '%s %s\n' "${ttyd_sha}" "${tools}/bin/ttyd.tmp" | sha256sum -c - chmod 0755 "${tools}/bin/ttyd.tmp" mv "${tools}/bin/ttyd.tmp" "${tools}/bin/ttyd" fi - if [ ! -f "${tools}/.cli-versions-0.147.0-2.1.226" ]; then + # The CLI stamp is arch-specific: npm only materializes the current + # arch's native optional deps (e.g. @openai/codex-linux-x64), so a + # fresh arch must re-run the install even though the shared volume + # already holds the other arch's node_modules. npm keeps both + # arches' optional packages, so this never clobbers the sibling arch. + if [ ! -f "${tools}/.cli-versions-0.147.0-2.1.226-${arch}" ]; then npm install --global --omit=dev --no-audit --no-fund --prefix "${tools}" \ @openai/codex@0.147.0 \ @anthropic-ai/claude-code@2.1.226 - touch "${tools}/.cli-versions-0.147.0-2.1.226" + touch "${tools}/.cli-versions-0.147.0-2.1.226-${arch}" fi kubectl_version="$("${tools}/bin/kubectl" version --client --output=json 2>/dev/null || true)" case "${kubectl_version}" in *\"gitVersion\":\"v1.33.3\"*) kubectl_ready=1 ;; *) kubectl_ready=0 ;; esac if [ "${kubectl_ready}" != "1" ]; then - curl -fsSL -o "${tools}/bin/kubectl.tmp" https://dl.k8s.io/release/v1.33.3/bin/linux/arm64/kubectl - printf '%s %s\n' 3d514dbae5dc8c09f773df0ef0f5d449dfad05b3aca5c96b13565f886df345fd "${tools}/bin/kubectl.tmp" | sha256sum -c - + curl -fsSL -o "${tools}/bin/kubectl.tmp" "https://dl.k8s.io/release/v1.33.3/bin/linux/${kubectl_arch}/kubectl" + printf '%s %s\n' "${kubectl_sha}" "${tools}/bin/kubectl.tmp" | sha256sum -c - chmod 0755 "${tools}/bin/kubectl.tmp" mv "${tools}/bin/kubectl.tmp" "${tools}/bin/kubectl" fi diff --git a/services/hermes/scripts/install_agent_tools.sh b/services/hermes/scripts/install_agent_tools.sh index 3cf4ba5a..0a86a827 100755 --- a/services/hermes/scripts/install_agent_tools.sh +++ b/services/hermes/scripts/install_agent_tools.sh @@ -1,15 +1,77 @@ #!/bin/sh -# Install the pinned ARM64 operator toolchain on the persistent owner volume. +# Install the pinned operator toolchain on the persistent owner volume. +# The /opt/data/tools volume is shared across whatever node the pod lands on, +# so every download resolves the running node's architecture instead of a +# hardcoded one and fails closed on anything but the two arches we build for. set -eu tools=${HERMES_AGENT_TOOLS_DIR:-/opt/data/tools} python=${HERMES_AGENT_PYTHON:-/opt/hermes/.venv/bin/python} bin=${tools}/bin -stamp=${tools}/.operator-cli-versions-2.7.0-3.18.6-5.7.1-1.8.1-4.47.1-2.78.0-2.0.4-3.13.3-1.3.1-0.51.0-1.15.8 -go_stamp=${tools}/.go-toolchain-1.26.5 + +arch="$(uname -m)" +case "${arch}" in + aarch64) dl_arch=arm64 ;; + x86_64) dl_arch=amd64 ;; + *) + echo "unsupported architecture for operator toolchain: ${arch}" >&2 + exit 1 + ;; +esac + +# Per-arch pinned checksums. Keep the arm64 values already proven in +# production; the amd64 values were taken from each project's published +# checksum manifest (or by hashing the release asset directly). +case "${dl_arch}" in + arm64) + go_sha=fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49 + flux_sha=758703b8cd96be98f1ea23b7bd3ff2ae13d90a23467d8aba8b83adbe335c6854 + helm_sha=5b8e00b6709caab466cbbb0bc29ee09059b8dc9417991dd04b497530e49b1737 + kustomize_sha=4261a040217df3bd6896597c3986d1465925726e4f22a945304b5233a4dcdbda + jq_sha=6bc62f25981328edd3cfcfe6fe51b073f2d7e7710d7ef7fcdac28d4e384fc3d4 + yq_sha=b7f7c991abe262b0c6f96bbcb362f8b35429cefd59c8b4c2daa4811f1e9df599 + gh_sha=9e3ca75b227a5503f6ef92c4b8b6dbf94e34bfdd8069ac0f16b8739856ebba7b + vault_sha=87bb68fdd04ca90cd4cf54f8cd783a037fbf860b73d85e6697f6129dac49c683 + sops_sha=53b0abacd38ef1b12a66d6c100956691b9cefce018d91f81e73ddf7438b94d77 + age_sha=c6878a324421b69e3e20b00ba17c04bc5c6dab0030cfe55bf8f68fa8d9e9093a + k9s_sha=3ee05c82e5f9198928a4e86133608ba6a2c10a2244d6a7789e820f78319d640c + terraform_sha=8891e9dcedc9e3b8950bc6af9d4d8af1f4cfade3062f53b9dc403a89f6ce8c9c + ;; + amd64) + go_sha=5c2c3b16caefa1d968a94c1daca04a7ca301a496d9b086e17ad77bb81393f053 + flux_sha=81ee173d47fd6df1015a1c82d083f6242c95e67badc0a33c00cb9d8a64233d14 + helm_sha=3f43c0aa57243852dd542493a0f54f1396c0bc8ec7296bbb2c01e802010819ce + kustomize_sha=ea375e7372f9aa029129d4b2d16c66b7750b7f1213c4f66f910d981c895818d8 + jq_sha=020468de7539ce70ef1bceaf7cde2e8c4f2ca6c3afb84642aabc5c97d9fc2a0d + yq_sha=0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb + gh_sha=ac309f70c5d6b122c82e6138ce82cb65ca5d8595cc09d11751fbc4e3907e1a05 + vault_sha=7429e7d85f8ef29df063701c49420f7984a0ae2c8511c026cc75edfbbb2df387 + sops_sha=e5bec3346a873ae91d871550f3e698c1aad962aff462a080e40f25fde17fef6b + age_sha=bdc69c09cbdd6cf8b1f333d372a1f58247b3a33146406333e30c0f26e8f51377 + k9s_sha=c3752ad51a5a4015a113819c4eeb6e55a4d0e4b8e652494797532f6fc8161dd7 + terraform_sha=d25ce7b6902013ad905db3d2eab0be4cd905887fe88b81a6171b8d5503c31f3d + ;; +esac + +# Stamps are arch-specific so a fresh arch re-installs its own native binaries +# instead of trusting the sibling arch's completion marker on the shared volume. +stamp=${tools}/.operator-cli-versions-2.7.0-3.18.6-5.7.1-1.8.1-4.47.1-2.78.0-2.0.4-3.13.3-1.3.1-0.51.0-1.15.8-${dl_arch} +go_stamp=${tools}/.go-toolchain-1.26.5-${dl_arch} +go_root=${tools}/go-1.26.5-${dl_arch} +active_arch_file=${tools}/.operator-active-arch mkdir -p "${bin}" +# The published ${bin} binaries (moved, single-arch files) reflect exactly one +# arch at a time. When the pod moves to the other arch, re-publish this arch's +# binaries even though its download may already be cached: clearing the stamps +# forces the republish while the arch-specific ${go_root} keeps Go from being +# re-downloaded. ttyd/kubectl next door already self-heal via version probes. +if [ "$(cat "${active_arch_file}" 2>/dev/null || true)" != "${dl_arch}" ]; then + rm -f "${stamp}" "${go_stamp}" +fi + if [ -f "${stamp}" ] && [ -f "${go_stamp}" ]; then + printf '%s' "${dl_arch}" > "${active_arch_file}" exit 0 fi @@ -32,11 +94,10 @@ publish() { } if [ ! -f "${go_stamp}" ]; then - go_root=${tools}/go-1.26.5 if [ ! -e "${go_root}" ]; then fetch \ - https://go.dev/dl/go1.26.5.linux-arm64.tar.gz \ - fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49 \ + "https://go.dev/dl/go1.26.5.linux-${dl_arch}.tar.gz" \ + "${go_sha}" \ "${work}/go.tar.gz" tar -xzf "${work}/go.tar.gz" -C "${work}" mv "${work}/go" "${go_root}" @@ -45,76 +106,77 @@ if [ ! -f "${go_stamp}" ]; then echo "incomplete Go toolchain path exists: ${go_root}" >&2 exit 1 fi - ln -sfn "../go-1.26.5/bin/go" "${bin}/go" - ln -sfn "../go-1.26.5/bin/gofmt" "${bin}/gofmt" + ln -sfn "../go-1.26.5-${dl_arch}/bin/go" "${bin}/go" + ln -sfn "../go-1.26.5-${dl_arch}/bin/gofmt" "${bin}/gofmt" touch "${go_stamp}" fi if [ -f "${stamp}" ]; then + printf '%s' "${dl_arch}" > "${active_arch_file}" exit 0 fi fetch \ - https://github.com/fluxcd/flux2/releases/download/v2.7.0/flux_2.7.0_linux_arm64.tar.gz \ - 758703b8cd96be98f1ea23b7bd3ff2ae13d90a23467d8aba8b83adbe335c6854 \ + "https://github.com/fluxcd/flux2/releases/download/v2.7.0/flux_2.7.0_linux_${dl_arch}.tar.gz" \ + "${flux_sha}" \ "${work}/flux.tar.gz" mkdir "${work}/flux" tar -xzf "${work}/flux.tar.gz" -C "${work}/flux" publish "${work}/flux/flux" flux fetch \ - https://get.helm.sh/helm-v3.18.6-linux-arm64.tar.gz \ - 5b8e00b6709caab466cbbb0bc29ee09059b8dc9417991dd04b497530e49b1737 \ + "https://get.helm.sh/helm-v3.18.6-linux-${dl_arch}.tar.gz" \ + "${helm_sha}" \ "${work}/helm.tar.gz" mkdir "${work}/helm" tar -xzf "${work}/helm.tar.gz" -C "${work}/helm" -publish "${work}/helm/linux-arm64/helm" helm +publish "${work}/helm/linux-${dl_arch}/helm" helm fetch \ - https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.7.1/kustomize_v5.7.1_linux_arm64.tar.gz \ - 4261a040217df3bd6896597c3986d1465925726e4f22a945304b5233a4dcdbda \ + "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.7.1/kustomize_v5.7.1_linux_${dl_arch}.tar.gz" \ + "${kustomize_sha}" \ "${work}/kustomize.tar.gz" mkdir "${work}/kustomize" tar -xzf "${work}/kustomize.tar.gz" -C "${work}/kustomize" publish "${work}/kustomize/kustomize" kustomize fetch \ - https://github.com/jqlang/jq/releases/download/jq-1.8.1/jq-linux-arm64 \ - 6bc62f25981328edd3cfcfe6fe51b073f2d7e7710d7ef7fcdac28d4e384fc3d4 \ + "https://github.com/jqlang/jq/releases/download/jq-1.8.1/jq-linux-${dl_arch}" \ + "${jq_sha}" \ "${work}/jq" publish "${work}/jq" jq fetch \ - https://github.com/mikefarah/yq/releases/download/v4.47.1/yq_linux_arm64 \ - b7f7c991abe262b0c6f96bbcb362f8b35429cefd59c8b4c2daa4811f1e9df599 \ + "https://github.com/mikefarah/yq/releases/download/v4.47.1/yq_linux_${dl_arch}" \ + "${yq_sha}" \ "${work}/yq" publish "${work}/yq" yq fetch \ - https://github.com/cli/cli/releases/download/v2.78.0/gh_2.78.0_linux_arm64.tar.gz \ - 9e3ca75b227a5503f6ef92c4b8b6dbf94e34bfdd8069ac0f16b8739856ebba7b \ + "https://github.com/cli/cli/releases/download/v2.78.0/gh_2.78.0_linux_${dl_arch}.tar.gz" \ + "${gh_sha}" \ "${work}/gh.tar.gz" mkdir "${work}/gh" tar -xzf "${work}/gh.tar.gz" -C "${work}/gh" -publish "${work}/gh/gh_2.78.0_linux_arm64/bin/gh" gh +publish "${work}/gh/gh_2.78.0_linux_${dl_arch}/bin/gh" gh fetch \ - https://releases.hashicorp.com/vault/2.0.4/vault_2.0.4_linux_arm64.zip \ - 87bb68fdd04ca90cd4cf54f8cd783a037fbf860b73d85e6697f6129dac49c683 \ + "https://releases.hashicorp.com/vault/2.0.4/vault_2.0.4_linux_${dl_arch}.zip" \ + "${vault_sha}" \ "${work}/vault.zip" mkdir "${work}/vault" "${python}" -m zipfile -e "${work}/vault.zip" "${work}/vault" publish "${work}/vault/vault" vault fetch \ - https://github.com/getsops/sops/releases/download/v3.13.3/sops-v3.13.3.linux.arm64 \ - 53b0abacd38ef1b12a66d6c100956691b9cefce018d91f81e73ddf7438b94d77 \ + "https://github.com/getsops/sops/releases/download/v3.13.3/sops-v3.13.3.linux.${dl_arch}" \ + "${sops_sha}" \ "${work}/sops" publish "${work}/sops" sops fetch \ - https://github.com/FiloSottile/age/releases/download/v1.3.1/age-v1.3.1-linux-arm64.tar.gz \ - c6878a324421b69e3e20b00ba17c04bc5c6dab0030cfe55bf8f68fa8d9e9093a \ + "https://github.com/FiloSottile/age/releases/download/v1.3.1/age-v1.3.1-linux-${dl_arch}.tar.gz" \ + "${age_sha}" \ "${work}/age.tar.gz" mkdir "${work}/age" tar -xzf "${work}/age.tar.gz" -C "${work}/age" @@ -123,19 +185,20 @@ for name in age age-inspect age-keygen age-plugin-batchpass; do done fetch \ - https://github.com/derailed/k9s/releases/download/v0.51.0/k9s_Linux_arm64.tar.gz \ - 3ee05c82e5f9198928a4e86133608ba6a2c10a2244d6a7789e820f78319d640c \ + "https://github.com/derailed/k9s/releases/download/v0.51.0/k9s_Linux_${dl_arch}.tar.gz" \ + "${k9s_sha}" \ "${work}/k9s.tar.gz" mkdir "${work}/k9s" tar -xzf "${work}/k9s.tar.gz" -C "${work}/k9s" publish "${work}/k9s/k9s" k9s fetch \ - https://releases.hashicorp.com/terraform/1.15.8/terraform_1.15.8_linux_arm64.zip \ - 8891e9dcedc9e3b8950bc6af9d4d8af1f4cfade3062f53b9dc403a89f6ce8c9c \ + "https://releases.hashicorp.com/terraform/1.15.8/terraform_1.15.8_linux_${dl_arch}.zip" \ + "${terraform_sha}" \ "${work}/terraform.zip" mkdir "${work}/terraform" "${python}" -m zipfile -e "${work}/terraform.zip" "${work}/terraform" publish "${work}/terraform/terraform" terraform touch "${stamp}" +printf '%s' "${dl_arch}" > "${active_arch_file}" diff --git a/testing/tests/test_hermes_agent_runtime_patches.py b/testing/tests/test_hermes_agent_runtime_patches.py index a7ad1bad..b3b07013 100644 --- a/testing/tests/test_hermes_agent_runtime_patches.py +++ b/testing/tests/test_hermes_agent_runtime_patches.py @@ -37,11 +37,20 @@ def test_owner_agent_installs_the_pinned_operator_toolchain(): "gofmt", ]: assert value in script - assert "go1.26.5.linux-arm64.tar.gz" in script + # The toolchain install resolves the running node's arch instead of a + # hardcoded one, so the Go URL is templated on ${dl_arch} and both arches' + # pinned Go checksums are present. + assert "go1.26.5.linux-${dl_arch}.tar.gz" in script + assert "dl_arch=arm64" in script + assert "dl_arch=amd64" in script assert ( "fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49" in script ) + assert ( + "5c2c3b16caefa1d968a94c1daca04a7ca301a496d9b086e17ad77bb81393f053" + in script + ) assert script.count("sha256sum -c -") == 1 deployment = _agent_deployment() diff --git a/testing/tests/test_hermes_cli_lanes_toolchain.py b/testing/tests/test_hermes_cli_lanes_toolchain.py index d7741a24..98b77d06 100644 --- a/testing/tests/test_hermes_cli_lanes_toolchain.py +++ b/testing/tests/test_hermes_cli_lanes_toolchain.py @@ -99,8 +99,13 @@ def test_owner_agent_installs_the_pinned_operator_toolchain(): "gofmt", ]: assert value in script - assert "go1.26.5.linux-arm64.tar.gz" in script + # Arch-aware: the Go URL is templated on ${dl_arch} and both arches' pinned + # Go checksums are present so amd64 nodes install native binaries. + assert "go1.26.5.linux-${dl_arch}.tar.gz" in script + assert "dl_arch=arm64" in script + assert "dl_arch=amd64" in script assert "fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49" in script + assert "5c2c3b16caefa1d968a94c1daca04a7ca301a496d9b086e17ad77bb81393f053" in script assert script.count("sha256sum -c -") == 1 deployment = _agent_deployment()