From 9c516b98087a2a5178b6e87605a20add1abe8bb0 Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 25 Aug 2026 18:17:35 -0300 Subject: [PATCH] build(hermes-webui): multi-arch image (arm64 + amd64) Make registry.bstein.dev/bstein/hermes-webui a linux/amd64 + linux/arm64 manifest list so the agent pod's `hux` sidecar (which runs the webui image) can schedule onto the amd64 node titan-22. Reuses the hermes-agent multi-arch pattern already on main. - Dockerfile.hermes-webui: repoint both FROMs to multi-arch, internal sources. The upstream WebUI base (ghcr sha256:a83a3893..., already a multi-arch OCI index) is now pulled from the in-cluster Harbor mirror; the agent base moves from the retired arm64-only leaf (81970563) to the multi-arch agent index (a68d1c4d). Kaniko selects the matching arch leaf per build node. - services/harbor/hermes-webui-base-mirror-job.yaml: new suspended, operator-run skopeo `copy --all` Job mirroring the upstream WebUI base index into Harbor's `mirror` project (modeled on hermes-agent-base-mirror-job.yaml; reuses the generic ensure-project helper). Wired into the harbor kustomization. - Jenkinsfile.hermes-webui-image: arm64 leg (titan-20) + amd64 leg (titan-24, hostname+arch pin, toleration Exists, resource-capped, own checkout scm) + Combine multi-arch index stage; per-arch evidence archived alongside the index. - hermes_multiarch_combine.py: generalize the destination pattern/component to serve both hermes-agent and hermes-webui (fail-closed to just those two). - Tests updated to the two-arch topology (two legs, combine, both FROM bases, the mirror Job, twelve archived evidence files). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf --- ci/Jenkinsfile.hermes-webui-image | 204 +++++++++++++++++- ci/scripts/hermes_multiarch_combine.py | 7 +- dockerfiles/Dockerfile.hermes-webui | 21 +- .../harbor/hermes-webui-base-mirror-job.yaml | 152 +++++++++++++ services/harbor/kustomization.yaml | 4 + .../tests/test_hermes_multiarch_combine.py | 45 ++++ testing/tests/test_hermes_webui_release.py | 162 +++++++++++++- 7 files changed, 582 insertions(+), 13 deletions(-) create mode 100644 services/harbor/hermes-webui-base-mirror-job.yaml diff --git a/ci/Jenkinsfile.hermes-webui-image b/ci/Jenkinsfile.hermes-webui-image index a82570bf..bf9053c2 100644 --- a/ci/Jenkinsfile.hermes-webui-image +++ b/ci/Jenkinsfile.hermes-webui-image @@ -205,6 +205,7 @@ spec: testing/tests/test_hermes_hux_runtime_vendor_parity.py \ testing/tests/test_hermes_hux_delivery.py \ testing/tests/test_hermes_oci_promote.py \ + testing/tests/test_hermes_multiarch_combine.py \ testing/tests/test_hermes_image_automation.py HUX_BACKEND_TESTS="$(find testing/tests -maxdepth 1 -type f \ -name 'test_hermes_hux_*.py' \ @@ -239,7 +240,7 @@ spec: } } } - stage('Build and publish without a daemon') { + stage('Build arm64 leg without a daemon') { steps { container('kaniko') { withCredentials([usernamePassword( @@ -251,7 +252,7 @@ spec: set -eu set +x config_path=/kaniko/.docker/config.json - destination="$(cat build/hermes-webui.destination)" + destination="$(cat build/hermes-webui.destination)-arm64" source_revision="$(cat build/hermes-webui.source-revision)" umask 077 auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')" @@ -267,19 +268,206 @@ spec: --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-webui" \ --destination="${destination}" \ --build-arg="HERMES_WEBUI_RELEASE_ID=git-${source_revision}-build-${BUILD_NUMBER}" \ - --digest-file="${WORKSPACE}/build/hermes-webui.digest" \ - --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-webui.image" \ + --digest-file="${WORKSPACE}/build/hermes-webui-arm64.digest" \ + --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-webui-arm64.image" \ --label="org.opencontainers.image.revision=${source_revision}" \ --label="org.opencontainers.image.source=https://scm.bstein.dev/atlas/titan-iac" \ --label="org.opencontainers.image.title=hermes-webui" \ --cleanup \ --push-retry=3 - /busybox/chmod 644 build/hermes-webui.digest build/hermes-webui.image + /busybox/chmod 644 build/hermes-webui-arm64.digest build/hermes-webui-arm64.image ''' } } } } + stage('Build amd64 leg without a daemon') { + agent { + kubernetes { + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + atlas.bstein.dev/workload: hermes-webui-image-builder-amd64 +spec: + serviceAccountName: hermes-image-builder + automountServiceAccountToken: false + enableServiceLinks: false + restartPolicy: Never + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + # titan-24 is an accelerator node (not a general worker) that co-hosts the + # out-of-cluster Sui validator. Pin the disposable amd64 build to it by + # hostname + arch ONLY — do NOT require node-role worker, so titan-24 is never + # opened to general cluster scheduling. The toleration + tight caps below keep + # this off the validator's back. + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/hostname: titan-24 + tolerations: + # titan-24 co-hosts the out-of-cluster Sui validator; tolerate whatever + # PreferNoSchedule/NoSchedule guard taint the node carries so the pinned + # build lands, and rely on the tight resource caps below (not scheduling + # priority) to keep the disposable build from starving the validator. + - operator: Exists + imagePullSecrets: + - name: harbor-bstein-robot + containers: + - name: jnlp + image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 25m + memory: 128Mi + limits: + cpu: 250m + memory: 384Mi + - name: kaniko + image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e + command: ["/busybox/sh", "-c"] + args: ["/busybox/sleep 99d"] + tty: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"] + privileged: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 100m + memory: 512Mi + ephemeral-storage: 10Gi + limits: + cpu: "1500m" + memory: 3Gi + ephemeral-storage: 20Gi +""" + } + } + steps { + // This amd64 leg runs on its own fresh pod (titan-24), so it must check + // out the SCM itself before the reviewed-revision git boundary check — + // otherwise `git rev-parse origin/main` fails with "not a git repository". + checkout scm + container('jnlp') { + sh ''' + set -eu + mkdir -p build + test "${PUBLISH_IMAGE}" = "true" + test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES WEBUI" + case "${EXPECTED_SOURCE_REVISION}" in + *[!0-9a-f]*|'') + echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2 + exit 2 + ;; + esac + test "${#EXPECTED_SOURCE_REVISION}" -eq 40 + main_revision="$(git rev-parse HEAD)" + test "${main_revision}" = "$(git rev-parse origin/main)" + git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}" + git checkout --detach "${EXPECTED_SOURCE_REVISION}" + actual_revision="$(git rev-parse HEAD)" + test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}" + test -z "$(git status --porcelain)" + test -f dockerfiles/Dockerfile.hermes-webui + case "${BUILD_NUMBER}" in + ''|0*|*[!0-9]*) + echo "BUILD_NUMBER must be a positive decimal integer" >&2 + exit 2 + ;; + esac + printf '%s\n' \ + "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \ + > build/hermes-webui.destination + printf '%s\n' "${actual_revision}" > build/hermes-webui.source-revision + ''' + } + container('kaniko') { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh '''#!/busybox/sh + set -eu + set +x + config_path=/kaniko/.docker/config.json + destination="$(cat build/hermes-webui.destination)-amd64" + source_revision="$(cat build/hermes-webui.source-revision)" + umask 077 + auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')" + /busybox/mkdir -p /kaniko/.docker + /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}" + unset HARBOR_USER HARBOR_PASSWORD auth + trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM + umask 022 + /kaniko/executor \ + --registry-mirror=harbor-core.harbor.svc.cluster.local \ + --insecure-registry=harbor-core.harbor.svc.cluster.local \ + --context="dir://${WORKSPACE}" \ + --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-webui" \ + --destination="${destination}" \ + --build-arg="HERMES_WEBUI_RELEASE_ID=git-${source_revision}-build-${BUILD_NUMBER}" \ + --digest-file="${WORKSPACE}/build/hermes-webui-amd64.digest" \ + --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-webui-amd64.image" \ + --label="org.opencontainers.image.revision=${source_revision}" \ + --label="org.opencontainers.image.source=https://scm.bstein.dev/atlas/titan-iac" \ + --label="org.opencontainers.image.title=hermes-webui" \ + --cleanup \ + --push-retry=3 + /busybox/chmod 644 build/hermes-webui-amd64.digest build/hermes-webui-amd64.image + ''' + } + } + stash( + name: 'hermes-webui-amd64-evidence', + includes: 'build/hermes-webui-amd64.digest,build/hermes-webui-amd64.image' + ) + } + } + stage('Combine multi-arch index') { + steps { + unstash 'hermes-webui-amd64-evidence' + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-webui.destination)" + source_revision="$(cat build/hermes-webui.source-revision)" + python3 ci/scripts/hermes_multiarch_combine.py \ + --destination "${destination}" \ + --source-revision "${source_revision}" \ + --build-number "${BUILD_NUMBER}" \ + --arm64-digest-file build/hermes-webui-arm64.digest \ + --arm64-image-file build/hermes-webui-arm64.image \ + --amd64-digest-file build/hermes-webui-amd64.digest \ + --amd64-image-file build/hermes-webui-amd64.image \ + --digest-file build/hermes-webui.digest \ + --image-file build/hermes-webui.image + test -s build/hermes-webui.digest + test -s build/hermes-webui.image + ''' + } + } + } stage('Render reviewed Flux handoff') { steps { withCredentials([usernamePassword( @@ -316,6 +504,10 @@ spec: build/hermes-webui.digest \ build/hermes-webui.image \ build/hermes-webui.source-revision \ + build/hermes-webui-arm64.digest \ + build/hermes-webui-arm64.image \ + build/hermes-webui-amd64.digest \ + build/hermes-webui-amd64.image \ build/hermes-webui-release/hermes-chat-statefulset.yaml \ build/hermes-webui-release/hermes-dashboard-deployment.yaml \ build/hermes-webui-release/hermes-webui-image.json \ @@ -336,7 +528,7 @@ spec: --output-dir build/hermes-webui-release ''' archiveArtifacts( - artifacts: 'build/hermes-webui.destination,build/hermes-webui.digest,build/hermes-webui.image,build/hermes-webui.source-revision,build/hermes-webui-release/hermes-chat-statefulset.yaml,build/hermes-webui-release/hermes-dashboard-deployment.yaml,build/hermes-webui-release/hermes-webui-image.json,build/hermes-webui-release/hermes-webui-image-update.patch', + artifacts: 'build/hermes-webui.destination,build/hermes-webui.digest,build/hermes-webui.image,build/hermes-webui.source-revision,build/hermes-webui-arm64.digest,build/hermes-webui-arm64.image,build/hermes-webui-amd64.digest,build/hermes-webui-amd64.image,build/hermes-webui-release/hermes-chat-statefulset.yaml,build/hermes-webui-release/hermes-dashboard-deployment.yaml,build/hermes-webui-release/hermes-webui-image.json,build/hermes-webui-release/hermes-webui-image-update.patch', allowEmptyArchive: false, fingerprint: true ) diff --git a/ci/scripts/hermes_multiarch_combine.py b/ci/scripts/hermes_multiarch_combine.py index c5dbc300..644a6e50 100644 --- a/ci/scripts/hermes_multiarch_combine.py +++ b/ci/scripts/hermes_multiarch_combine.py @@ -40,8 +40,11 @@ from typing import Any, Callable REGISTRY_ORIGIN = "https://registry.bstein.dev" # The final (arch-less) Flux-visible tag; identical contract to the promoter. +# Both Hermes images that the multi-arch pipelines publish share the exact same +# arch-less final-tag contract; the repository name is the only difference and is +# captured here so the combiner stays fail-closed to just these two components. DESTINATION_PATTERN = re.compile( - r"^registry\.bstein\.dev/bstein/hermes-agent:" + r"^registry\.bstein\.dev/bstein/(?Phermes-agent|hermes-webui):" r"git-(?P[0-9a-f]{40})-build-(?P[1-9][0-9]*)$" ) DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") @@ -224,7 +227,7 @@ def combine_multiarch_index( raise ValueError("invalid multi-arch destination") if set(arch_digests) != set(ARCHITECTURES): raise ValueError("expected exactly the arm64 and amd64 per-arch digests") - component = "hermes-agent" + component = match.group("component") index_tag = destination.rsplit(":", 1)[1] authorization = _authorization(username, password) diff --git a/dockerfiles/Dockerfile.hermes-webui b/dockerfiles/Dockerfile.hermes-webui index 5ebf296b..cc4775be 100644 --- a/dockerfiles/Dockerfile.hermes-webui +++ b/dockerfiles/Dockerfile.hermes-webui @@ -1,8 +1,25 @@ # syntax=docker/dockerfile:1 # dockerfiles/Dockerfile.hermes-webui -FROM ghcr.io/nesquena/hermes-webui@sha256:a83a3893111dcb250e7aa7aa657d3d6f4570b0e2fd00d9b7569246fc5e7339b2 AS webui +# +# Both FROM bases below are multi-arch (linux/amd64 + linux/arm64). Kaniko builds +# one native leaf per node arch (arm64 on titan-20, amd64 on titan-24) and each +# leaf selects the matching arch from these indexes; ci/scripts/hermes_multiarch_combine.py +# then binds the two leaves into one manifest list. This is what lets the agent +# pod's `hux` sidecar (which runs this image) schedule onto the amd64 node titan-22. +# +# The upstream WebUI base is a multi-arch OCI index +# (sha256:a83a3893... -> amd64 sha256:54fd4990..., arm64 sha256:9094ae6a...). It is +# mirrored digest-for-digest into the in-cluster Harbor `mirror` project by +# services/harbor/hermes-webui-base-mirror-job.yaml so the build never depends on +# ghcr.io egress (flaky from build pods). Bump this digest and BOTH args in that +# Job together, then an operator re-runs the (suspended) Job once. +FROM harbor-core.harbor.svc.cluster.local/mirror/hermes-webui@sha256:a83a3893111dcb250e7aa7aa657d3d6f4570b0e2fd00d9b7569246fc5e7339b2 AS webui -FROM registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 +# Layer each arch's WebUI on the matching leaf of the multi-arch Hermes agent index +# (sha256:a68d1c4d... -> amd64 sha256:c89ac4bc..., arm64 sha256:572854cb...), a Docker +# manifest list already in Harbor's bstein project. Kaniko's --registry-mirror pulls +# it internally through harbor-core. +FROM registry.bstein.dev/bstein/hermes-agent@sha256:a68d1c4d5517cc5e6719661e77be4f18d4964b07a85dcebaf6f62368646e4e6c ARG HERMES_WEBUI_RELEASE_ID diff --git a/services/harbor/hermes-webui-base-mirror-job.yaml b/services/harbor/hermes-webui-base-mirror-job.yaml new file mode 100644 index 00000000..c991e459 --- /dev/null +++ b/services/harbor/hermes-webui-base-mirror-job.yaml @@ -0,0 +1,152 @@ +# services/harbor/hermes-webui-base-mirror-job.yaml +# +# One-shot mirror of the upstream Hermes WebUI base image INDEX from ghcr.io into +# the in-cluster Harbor "mirror" project, so the reviewed image build +# (dockerfiles/Dockerfile.hermes-webui) pulls its `AS webui` FROM base internally +# with no ghcr.io fallback (ghcr/docker.io egress is IPv6-broken/flaky from build +# pods, which is why the agent base was mirrored the same way). +# +# The upstream digest sha256:a83a3893... is ALREADY a multi-arch OCI index +# (amd64 sha256:54fd4990..., arm64 sha256:9094ae6a...). `skopeo copy --all` is +# content-addressed, so it reproduces the identical index and both arch leaves in +# Harbor -- the build stays digest-pinned AND multi-arch. +# +# Kept `suspend: true` exactly like hermes-agent-base-mirror-job.yaml: it needs +# egress to ghcr.io and is only run deliberately, once per base-digest bump. To +# run it, an operator clears suspend (or `kubectl create job --from`) AFTER +# updating the digest in BOTH args below and in the Dockerfile `AS webui` FROM. +apiVersion: batch/v1 +kind: Job +metadata: + name: harbor-hermes-webui-base-mirror-1 + namespace: harbor +spec: + suspend: true + backoffLimit: 2 + activeDeadlineSeconds: 1800 + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + # This Job has an app init container (ensure-project) that reads the + # injected secret, so Vault's init must run FIRST — otherwise it is + # appended after ensure-project and the secret file isn't written yet. + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-run-as-user: "65532" + vault.hashicorp.com/agent-run-as-group: "65532" + vault.hashicorp.com/role: harbor-policy-bootstrap + vault.hashicorp.com/agent-inject-secret-harbor-admin-password: kv/data/atlas/harbor/harbor-core + vault.hashicorp.com/agent-inject-template-harbor-admin-password: | + {{- with secret "kv/data/atlas/harbor/harbor-core" -}} + {{ .Data.data.harbor_admin_password }} + {{- end -}} + spec: + serviceAccountName: harbor-policy-bootstrap + enableServiceLinks: false + restartPolicy: Never + nodeSelector: + hardware: rpi5 + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-04, titan-14, titan-18, titan-19, titan-24] + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + initContainers: + # Ensure the public "mirror" project exists before skopeo tries to push + # into it (Harbor only auto-creates repositories inside an existing + # project). Reuses the exact same generic ensure-project helper and + # Vault-injected admin password as the sibling agent base-mirror Job -- + # no new credential and no new project is introduced. + - name: ensure-project + image: docker.io/library/python@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757 + imagePullPolicy: IfNotPresent + command: [python3, /scripts/harbor_mirror_project_ensure.py] + env: + - name: HARBOR_API_ORIGIN + value: http://harbor-core.harbor.svc.cluster.local/api/v2.0 + - name: HARBOR_ADMIN_PASSWORD_FILE + value: /vault/secrets/harbor-admin-password + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: tmp + mountPath: /tmp + resources: + requests: {cpu: 25m, memory: 32Mi} + limits: {cpu: 250m, memory: 128Mi} + containers: + # skopeo copies the WHOLE multi-arch index (--all) straight from ghcr.io + # to the internal Harbor registry. Source is the public upstream image, so + # no source credential is needed; the digest is asserted on both ends so a + # drifted upstream tag cannot be mirrored. + - name: mirror + image: quay.io/skopeo/stable@sha256:94f5c5e26997e2e78c234ec9abf19a391c234b39eb22e6d1210d0b527c97dcc8 + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] + args: + - | + set -eu + src="docker://ghcr.io/nesquena/hermes-webui@sha256:a83a3893111dcb250e7aa7aa657d3d6f4570b0e2fd00d9b7569246fc5e7339b2" + # Push to the EXTERNAL Harbor endpoint (valid TLS) — the same path + # kaniko already publishes to — to avoid harbor-core's internal + # HTTP/token-realm quirks. The image lands in the same Harbor + # 'mirror' project, so kaniko still pulls it internally via harbor-core. + dst="docker://registry.bstein.dev/mirror/hermes-webui@sha256:a83a3893111dcb250e7aa7aa657d3d6f4570b0e2fd00d9b7569246fc5e7339b2" + pw="$(cat /vault/secrets/harbor-admin-password)" + exec skopeo copy --all \ + --src-tls-verify=true \ + --dest-tls-verify=true \ + --dest-creds "admin:${pw}" \ + "${src}" "${dst}" + env: + - name: HOME + value: /tmp + - name: TMPDIR + value: /tmp + - name: XDG_RUNTIME_DIR + value: /tmp + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: tmp + mountPath: /tmp + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {cpu: "1", memory: 1Gi} + volumes: + - name: scripts + configMap: + name: harbor-hermes-webui-base-mirror-script + defaultMode: 0555 + - name: tmp + emptyDir: {} diff --git a/services/harbor/kustomization.yaml b/services/harbor/kustomization.yaml index 97b3fffc..a70789d4 100644 --- a/services/harbor/kustomization.yaml +++ b/services/harbor/kustomization.yaml @@ -17,6 +17,7 @@ resources: - hermes-webui-immutability-job.yaml - hermes-chat-router-immutability-job.yaml - hermes-agent-base-mirror-job.yaml + - hermes-webui-base-mirror-job.yaml - bootstrap-jobs/cassandra-registry-ensure-job.yaml - image.yaml configMapGenerator: @@ -36,3 +37,6 @@ configMapGenerator: - name: harbor-hermes-agent-base-mirror-script files: - harbor_mirror_project_ensure.py=scripts/harbor_mirror_project_ensure.py + - name: harbor-hermes-webui-base-mirror-script + files: + - harbor_mirror_project_ensure.py=scripts/harbor_mirror_project_ensure.py diff --git a/testing/tests/test_hermes_multiarch_combine.py b/testing/tests/test_hermes_multiarch_combine.py index 9887b07e..3dbdf9f7 100644 --- a/testing/tests/test_hermes_multiarch_combine.py +++ b/testing/tests/test_hermes_multiarch_combine.py @@ -383,3 +383,48 @@ def test_cli_rejects_mismatched_per_arch_evidence_pair( ) assert module.main() == 1 assert "does not match" in json.loads(capsys.readouterr().out)["error"] + + +# The combiner serves both Hermes multi-arch images; only the repository name in +# the destination differs. hermes-webui must route to its own registry paths. +WEBUI_DESTINATION = ( + f"registry.bstein.dev/bstein/hermes-webui:git-{REVISION}-build-{BUILD}" +) + + +def test_destination_pattern_captures_both_components() -> None: + """The fail-closed pattern accepts exactly hermes-agent and hermes-webui.""" + module = _load() + agent = module.DESTINATION_PATTERN.fullmatch(DESTINATION) + webui = module.DESTINATION_PATTERN.fullmatch(WEBUI_DESTINATION) + assert agent is not None and agent.group("component") == "hermes-agent" + assert webui is not None and webui.group("component") == "hermes-webui" + # A third, unexpected repository is still rejected. + assert ( + module.DESTINATION_PATTERN.fullmatch( + f"registry.bstein.dev/bstein/hermes-other:git-{REVISION}-build-{BUILD}" + ) + is None + ) + + +def test_combines_webui_leaves_under_the_webui_component() -> None: + """A hermes-webui destination re-reads leaves from the webui registry paths.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + registry = Registry(module, amd64, arm64) + result = module.combine_multiarch_index( + destination=WEBUI_DESTINATION, + arch_digests={"amd64": amd64["digest"], "arm64": arm64["digest"]}, + username="robot", + password="private", + opener=registry, + ) + assert result["component"] == "hermes-webui" + assert result["result"] == "published" + # Every registry call the combiner made must be scoped to the webui repo, and + # never leak into the agent repo. + assert registry.calls, "combiner made no registry calls" + for _method, url in registry.calls: + assert "/v2/bstein/hermes-webui/" in url + assert "hermes-agent" not in url diff --git a/testing/tests/test_hermes_webui_release.py b/testing/tests/test_hermes_webui_release.py index e8bb1b75..506b67f9 100644 --- a/testing/tests/test_hermes_webui_release.py +++ b/testing/tests/test_hermes_webui_release.py @@ -579,17 +579,173 @@ def test_webui_policy_rejects_disabled_duplicate_or_truncated_rules() -> None: policy.list_rules(_FakePolicyClient([(200, b"[]", {"X-Total-Count": "1"})])) -def test_pipeline_archives_exact_eight_files_before_release() -> None: - """Release cannot pass with missing digest, workload, or metadata evidence.""" +def test_pipeline_archives_exact_twelve_files_before_release() -> None: + """Release cannot pass with missing digest, workload, or metadata evidence. + + The multi-arch topology archives the four arch-less index files, the four + per-arch leg leaves (arm64 + amd64 digest/image), and the four rendered Flux + handoff files -- twelve exact paths, no globs. + """ source = PIPELINE.read_text(encoding="utf-8") evidence = source.split("stage('Verify and archive release evidence')", 1)[1] evidence = evidence.split("stage('Publish Flux release tag')", 1)[0] archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0].split(",") - assert len(archive) == len(set(archive)) == 8 + assert len(archive) == len(set(archive)) == 12 assert all("*" not in path for path in archive) assert "find build -type f" in evidence assert "allowEmptyArchive: false" in evidence assert "build/hermes-webui.source-revision" in archive + for arch in ("arm64", "amd64"): + assert f"build/hermes-webui-{arch}.digest" in archive + assert f"build/hermes-webui-{arch}.image" in archive assert "hermes-chat-statefulset.yaml" in evidence assert "hermes-dashboard-deployment.yaml" in evidence assert " post {" not in source + + +# --- Multi-arch (linux/amd64 + linux/arm64) topology ----------------------- +# +# The WebUI image must be multi-arch so the agent pod's `hux` sidecar (which runs +# this image) can schedule onto the amd64 node titan-22. These tests lock the two +# legs + combine, the two internal multi-arch FROM bases, and the Harbor mirror +# Job that feeds the upstream WebUI base internally. + +MIRROR_JOB = ROOT / "services/harbor/hermes-webui-base-mirror-job.yaml" +HARBOR_KUSTOMIZATION = ROOT / "services/harbor/kustomization.yaml" +WEBUI_BASE_DIGEST = ( + "sha256:a83a3893111dcb250e7aa7aa657d3d6f4570b0e2fd00d9b7569246fc5e7339b2" +) +AGENT_MULTIARCH_INDEX_DIGEST = ( + "sha256:a68d1c4d5517cc5e6719661e77be4f18d4964b07a85dcebaf6f62368646e4e6c" +) +AGENT_SINGLE_ARCH_DIGEST = ( + "sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107" +) + + +def _amd64_pod_spec() -> dict: + """Parse the nested amd64 build pod (the second inline pod template).""" + source = PIPELINE.read_text(encoding="utf-8") + blocks = source.split('yaml """')[1:] + for block in blocks: + body = block.split('"""', 1)[0] + doc = yaml.safe_load(body) + labels = doc.get("metadata", {}).get("labels", {}) + if str(labels.get("atlas.bstein.dev/workload", "")).endswith("-amd64"): + return doc["spec"] + raise AssertionError("no amd64 build pod template found in the pipeline") + + +def test_pipeline_builds_two_arch_legs_and_combines_one_index() -> None: + """The pipeline builds arm64 + amd64 legs and binds them into one index.""" + source = PIPELINE.read_text(encoding="utf-8") + assert "stage('Build arm64 leg without a daemon')" in source + assert "stage('Build amd64 leg without a daemon')" in source + assert "stage('Combine multi-arch index')" in source + # Each leg publishes an arch-suffixed candidate tag and its own evidence. + assert 'destination="$(cat build/hermes-webui.destination)-arm64"' in source + assert 'destination="$(cat build/hermes-webui.destination)-amd64"' in source + for arch in ("arm64", "amd64"): + assert f"build/hermes-webui-{arch}.digest" in source + assert f"build/hermes-webui-{arch}.image" in source + # Both legs must still stamp the WebUI release id (no arch drift in args). + assert ( + '--build-arg="HERMES_WEBUI_RELEASE_ID=' + 'git-${source_revision}-build-${BUILD_NUMBER}"' + ) in source + # The combiner reuses the shared, fail-closed manifest-list assembler. + assert "ci/scripts/hermes_multiarch_combine.py" in source + assert "--arm64-digest-file build/hermes-webui-arm64.digest" in source + assert "--amd64-digest-file build/hermes-webui-amd64.digest" in source + assert "--digest-file build/hermes-webui.digest" in source + # The amd64 leg runs on its own fresh pod, so it re-checks out and re-runs the + # exact release boundary before building. + assert source.count("checkout scm") >= 2 + assert source.count('test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES WEBUI"') >= 2 + # The final archived evidence covers the index AND both per-arch leaves. + for evidence in ( + "build/hermes-webui-arm64.digest", + "build/hermes-webui-arm64.image", + "build/hermes-webui-amd64.digest", + "build/hermes-webui-amd64.image", + ): + assert source.count(evidence) >= 3 # build, expected list, archive list + + +def test_amd64_leg_is_pinned_to_titan_24_without_worker_role() -> None: + """The disposable amd64 leg targets the accelerator node by hostname only.""" + spec = _amd64_pod_spec() + assert spec["serviceAccountName"] == "hermes-image-builder" + assert spec["automountServiceAccountToken"] is False + assert spec["nodeSelector"] == { + "kubernetes.io/arch": "amd64", + "kubernetes.io/hostname": "titan-24", + } + # titan-24 is an accelerator, not a general worker: never require the worker + # role label, and tolerate its guard taint so the pinned build lands. + assert "node-role.kubernetes.io/worker" not in spec["nodeSelector"] + assert spec["tolerations"] == [{"operator": "Exists"}] + containers = {item["name"]: item for item in spec["containers"]} + assert "kaniko" in containers + for container in containers.values(): + sc = container["securityContext"] + assert sc["allowPrivilegeEscalation"] is False + assert sc["capabilities"]["drop"] == ["ALL"] + # Tight caps keep the disposable build off the co-hosted validator's back. + assert containers["kaniko"]["resources"]["limits"]["memory"] == "3Gi" + + +def test_dockerfile_bases_are_multiarch_and_pulled_internally() -> None: + """Both FROM bases are multi-arch indexes sourced from inside the cluster.""" + source = DOCKERFILE.read_text(encoding="utf-8") + # Upstream WebUI base now comes from the in-cluster Harbor mirror, not ghcr. + assert ( + "FROM harbor-core.harbor.svc.cluster.local/mirror/hermes-webui@" + f"{WEBUI_BASE_DIGEST} AS webui" + ) in source + assert "ghcr.io/nesquena" not in source + # The agent base is the multi-arch index, not the retired single-arch leaf. + assert ( + f"FROM registry.bstein.dev/bstein/hermes-agent@{AGENT_MULTIARCH_INDEX_DIGEST}" + in source + ) + assert AGENT_SINGLE_ARCH_DIGEST not in source + + +def test_webui_base_mirror_job_is_suspended_and_digest_pinned() -> None: + """A suspended, digest-pinned skopeo Job mirrors the WebUI base into Harbor.""" + job = yaml.safe_load(MIRROR_JOB.read_text(encoding="utf-8")) + assert job["kind"] == "Job" + assert job["metadata"]["name"] == "harbor-hermes-webui-base-mirror-1" + assert job["metadata"]["namespace"] == "harbor" + # Operator-run only: it needs ghcr egress and must never fire automatically. + assert job["spec"]["suspend"] is True + pod = job["spec"]["template"]["spec"] + mirror = next(c for c in pod["containers"] if c["name"] == "mirror") + args = "\n".join(mirror["args"]) + assert "skopeo copy --all" in args + assert f"docker://ghcr.io/nesquena/hermes-webui@{WEBUI_BASE_DIGEST}" in args + assert ( + f"docker://registry.bstein.dev/mirror/hermes-webui@{WEBUI_BASE_DIGEST}" in args + ) + # Vault must init before the ensure-project init container reads the secret. + annotations = job["spec"]["template"]["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/agent-init-first"] == "true" + init = next(c for c in pod["initContainers"] if c["name"] == "ensure-project") + assert init["command"] == ["python3", "/scripts/harbor_mirror_project_ensure.py"] + script_volume = next(v for v in pod["volumes"] if v["name"] == "scripts") + assert ( + script_volume["configMap"]["name"] == "harbor-hermes-webui-base-mirror-script" + ) + + +def test_webui_base_mirror_job_is_wired_into_harbor_kustomization() -> None: + """Flux applies the mirror Job and mounts its ensure-project script.""" + kustomization = yaml.safe_load(HARBOR_KUSTOMIZATION.read_text(encoding="utf-8")) + assert "hermes-webui-base-mirror-job.yaml" in kustomization["resources"] + generators = {g["name"]: g for g in kustomization["configMapGenerator"]} + assert "harbor-hermes-webui-base-mirror-script" in generators + assert ( + "harbor_mirror_project_ensure.py=scripts/harbor_mirror_project_ensure.py" + in generators["harbor-hermes-webui-base-mirror-script"]["files"] + )