From ff2c003b4b08840323f8d855f4b4fbd0371bb207 Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 25 Aug 2026 11:05:29 -0300 Subject: [PATCH] build(hermes-agent): multi-arch image via two native kaniko legs Repoint the hermes-agent base FROM at the upstream multi-arch OCI INDEX digest (tag v2026.7.7.2, revision 9de9c25f) whose arm64 leaf is byte-for-byte the previously pinned single-arch base, so the arm64 build is unchanged while the same reviewed version now also resolves an amd64 leaf. Kaniko selects the matching leaf per build platform. Rework the release pipeline to build both arches natively and promote a multi-arch image without switching off kaniko or weakening any existing security assertion: - Keep the arm64 kaniko leg on the unchanged rpi5 coordinating pod; it now pushes an arch-suffixed candidate tag (...-build--arm64). - Add a second native amd64 kaniko leg on a titan-24-pinned, tolerating, resource-capped pod (ceiling strictly below the arm64 leg) that independently re-verifies the reviewed revision and stashes its leaf evidence (...-build--amd64). - Add ci/scripts/hermes_multiarch_combine.py: a pure-python, fail-closed combiner that re-reads each per-arch leaf from the registry, proves its digest AND its config architecture, assembles a Docker manifest LIST (already inside the promote allow-list), refuses to overwrite an existing final tag, publishes the arch-less ...-build- tag, and re-verifies the registry resolved the exact index referencing exactly the two leaves. It emits the index digest in the SAME digest-file/image-file format the single-arch step produced, so render/verify-evidence/hermes_oci_promote.py promote the INDEX with no change to those scripts. Tests: add test_hermes_multiarch_combine.py (full hash/verification chain); strengthen the image-builder suites for the two-arch topology (both kaniko legs carry the reviewed heredoc-compat build-arg; amd64 leg pinned+capped+ boundary-checked; combine stage wiring; expanded evidence archive) without weakening the arm64-leg assertions. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf --- ci/Jenkinsfile.hermes-agent-image | 191 ++++++++- ci/scripts/hermes_multiarch_combine.py | 390 ++++++++++++++++++ dockerfiles/Dockerfile.hermes-agent | 12 +- testing/tests/test_hermes_image_builder.py | 3 +- .../test_hermes_image_builder_fresh_review.py | 96 ++++- .../tests/test_hermes_multiarch_combine.py | 385 +++++++++++++++++ 6 files changed, 1066 insertions(+), 11 deletions(-) create mode 100644 ci/scripts/hermes_multiarch_combine.py create mode 100644 testing/tests/test_hermes_multiarch_combine.py diff --git a/ci/Jenkinsfile.hermes-agent-image b/ci/Jenkinsfile.hermes-agent-image index 76a32000..3f7ea910 100644 --- a/ci/Jenkinsfile.hermes-agent-image +++ b/ci/Jenkinsfile.hermes-agent-image @@ -188,6 +188,7 @@ spec: testing/tests/test_hermes_image_builder_coverage.py \ testing/tests/test_hermes_image_builder_fresh_review.py \ testing/tests/test_hermes_oci_promote.py \ + testing/tests/test_hermes_multiarch_combine.py \ testing/tests/test_hermes_image_automation.py ''' } @@ -213,7 +214,7 @@ spec: } } } - stage('Build and publish without a daemon') { + stage('Build arm64 leg without a daemon') { steps { container('kaniko') { withCredentials([usernamePassword( @@ -225,7 +226,7 @@ spec: set -eu set +x config_path=/kaniko/.docker/config.json - destination="$(cat build/hermes-agent.destination)" + destination="$(cat build/hermes-agent.destination)-arm64" source_revision="$(cat build/hermes-agent.source-revision)" umask 077 auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')" @@ -240,18 +241,192 @@ spec: --context="dir://${WORKSPACE}" \ --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \ --destination="${destination}" \ - --digest-file="${WORKSPACE}/build/hermes-agent.digest" \ - --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \ + --digest-file="${WORKSPACE}/build/hermes-agent-arm64.digest" \ + --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-arm64.image" \ --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \ --label="org.opencontainers.image.revision=${source_revision}" \ --cleanup \ --push-retry=3 - /busybox/chmod 644 build/hermes-agent.digest build/hermes-agent.image + /busybox/chmod 644 build/hermes-agent-arm64.digest build/hermes-agent-arm64.image ''' } } } } + stage('Build amd64 leg without a daemon') { + agent { + kubernetes { + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + atlas.bstein.dev/workload: hermes-agent-image-builder-amd64 +spec: + serviceAccountName: hermes-image-builder + automountServiceAccountToken: false + enableServiceLinks: false + restartPolicy: Never + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/hostname: titan-24 + node-role.kubernetes.io/worker: "true" + tolerations: + # titan-24 co-hosts the out-of-cluster Sui validator; tolerate whatever + # PreferNoSchedule/NoSchedule guard taint the node carries so the pinned + # build lands, and rely on the tight resource caps below (not scheduling + # priority) to keep the disposable build from starving the validator. + - operator: Exists + imagePullSecrets: + - name: harbor-bstein-robot + containers: + - name: jnlp + image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 25m + memory: 128Mi + limits: + cpu: 250m + memory: 384Mi + - name: kaniko + image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e + command: ["/busybox/sh", "-c"] + args: ["/busybox/sleep 99d"] + tty: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"] + privileged: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + cpu: "1500m" + memory: 3Gi +""" + } + } + steps { + container('jnlp') { + sh ''' + set -eu + mkdir -p build + test "${PUBLISH_IMAGE}" = "true" + test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT" + case "${EXPECTED_SOURCE_REVISION}" in + *[!0-9a-f]*|'') + echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2 + exit 2 + ;; + esac + test "${#EXPECTED_SOURCE_REVISION}" -eq 40 + main_revision="$(git rev-parse origin/main)" + git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}" + git checkout --detach "${EXPECTED_SOURCE_REVISION}" + actual_revision="$(git rev-parse HEAD)" + test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}" + test -z "$(git status --porcelain)" + test -f dockerfiles/Dockerfile.hermes-agent + case "${BUILD_NUMBER}" in + ''|0*|*[!0-9]*) + echo "BUILD_NUMBER must be a positive decimal integer" >&2 + exit 2 + ;; + esac + printf '%s\n' \ + "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \ + > build/hermes-agent.destination + printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision + ''' + } + container('kaniko') { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh '''#!/busybox/sh + set -eu + set +x + config_path=/kaniko/.docker/config.json + destination="$(cat build/hermes-agent.destination)-amd64" + source_revision="$(cat build/hermes-agent.source-revision)" + umask 077 + auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')" + /busybox/mkdir -p /kaniko/.docker + /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}" + unset HARBOR_USER HARBOR_PASSWORD auth + trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM + umask 022 + /kaniko/executor \ + --registry-mirror=harbor-core.harbor.svc.cluster.local \ + --insecure-registry=harbor-core.harbor.svc.cluster.local \ + --context="dir://${WORKSPACE}" \ + --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \ + --destination="${destination}" \ + --digest-file="${WORKSPACE}/build/hermes-agent-amd64.digest" \ + --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-amd64.image" \ + --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \ + --label="org.opencontainers.image.revision=${source_revision}" \ + --cleanup \ + --push-retry=3 + /busybox/chmod 644 build/hermes-agent-amd64.digest build/hermes-agent-amd64.image + ''' + } + } + stash( + name: 'hermes-agent-amd64-evidence', + includes: 'build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image' + ) + } + } + stage('Combine multi-arch index') { + steps { + unstash 'hermes-agent-amd64-evidence' + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-agent.destination)" + source_revision="$(cat build/hermes-agent.source-revision)" + python3 ci/scripts/hermes_multiarch_combine.py \ + --destination "${destination}" \ + --source-revision "${source_revision}" \ + --build-number "${BUILD_NUMBER}" \ + --arm64-digest-file build/hermes-agent-arm64.digest \ + --arm64-image-file build/hermes-agent-arm64.image \ + --amd64-digest-file build/hermes-agent-amd64.digest \ + --amd64-image-file build/hermes-agent-amd64.image \ + --digest-file build/hermes-agent.digest \ + --image-file build/hermes-agent.image + test -s build/hermes-agent.digest + test -s build/hermes-agent.image + ''' + } + } + } stage('Render reviewed Flux handoff') { steps { withCredentials([usernamePassword( @@ -287,6 +462,10 @@ spec: build/hermes-agent.digest \ build/hermes-agent.image \ build/hermes-agent.source-revision \ + build/hermes-agent-arm64.digest \ + build/hermes-agent-arm64.image \ + build/hermes-agent-amd64.digest \ + build/hermes-agent-amd64.image \ build/hermes-agent-release/hermes-agent-image.json \ build/hermes-agent-release/hermes-image-update.patch \ build/hermes-agent-release/hermes-kustomization.yaml \ @@ -305,7 +484,7 @@ spec: --output-dir build/hermes-agent-release ''' archiveArtifacts( - artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.source-revision,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml', + artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.source-revision,build/hermes-agent-arm64.digest,build/hermes-agent-arm64.image,build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml', allowEmptyArchive: false, fingerprint: true ) diff --git a/ci/scripts/hermes_multiarch_combine.py b/ci/scripts/hermes_multiarch_combine.py new file mode 100644 index 00000000..c5dbc300 --- /dev/null +++ b/ci/scripts/hermes_multiarch_combine.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +"""Assemble a fail-closed multi-arch manifest list from two per-arch leaves. + +Kaniko builds one native image per architecture (arm64 on an rpi5 pod, amd64 on +titan-24) and pushes each under an arch-suffixed candidate tag +``...-build--``. Kaniko cannot combine, so this step: + +1. Independently re-reads each per-arch candidate manifest from the registry and + binds it to the exact Kaniko digest evidence. +2. Proves each leaf really is the architecture it claims by reading its image + config (a swapped or cross-built leaf fails closed here). +3. Builds a Docker manifest *list* (not an OCI index) from the two verified + leaves -- Docker manifest lists are already inside the promotion allow-list, + so this keeps the security surface of ``hermes_oci_promote.py`` unchanged. +4. Refuses to overwrite an existing final tag, PUTs the list to the final + ``...-build-`` tag, and re-reads it to confirm the registry resolved the + exact index digest referencing exactly the two expected leaves. + +The output ``--digest-file``/``--image-file`` deliberately use the SAME format +the single-arch Kaniko step produced (```` and ``@`` +for the final, arch-less tag). The whole downstream evidence chain -- +``hermes_image_release.py`` render/verify-evidence and ``hermes_oci_promote.py`` +-- therefore promotes the multi-arch INDEX with no further change. +""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +import re +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any, Callable + + +REGISTRY_ORIGIN = "https://registry.bstein.dev" +# The final (arch-less) Flux-visible tag; identical contract to the promoter. +DESTINATION_PATTERN = re.compile( + r"^registry\.bstein\.dev/bstein/hermes-agent:" + r"git-(?P[0-9a-f]{40})-build-(?P[1-9][0-9]*)$" +) +DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") +DOCKER_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json" +# A per-arch leaf must be a single-image manifest, never itself a list/index. +LEAF_MANIFEST_TYPES = { + "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.oci.image.manifest.v1+json", +} +IMAGE_CONFIG_TYPES = { + "application/vnd.docker.container.image.v1+json", + "application/vnd.oci.image.config.v1+json", +} +# Deterministic architecture order -> deterministic manifest-list bytes/digest. +ARCHITECTURES = ("amd64", "arm64") +MAX_MANIFEST_BYTES = 4 * 1024 * 1024 +MAX_CONFIG_BYTES = 1024 * 1024 + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Never forward registry credentials to another origin.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +def _registry_request(request: urllib.request.Request, timeout: int) -> Any: + """Return normal and HTTP error responses without following redirects.""" + opener = urllib.request.build_opener(_NoRedirect()) + try: + return opener.open(request, timeout=timeout) + except urllib.error.HTTPError as exc: + return exc + + +def _status(response: Any) -> int: + """Normalize urllib response and HTTPError status fields.""" + return int(getattr(response, "status", getattr(response, "code", 0))) + + +def _authorization(username: str, password: str) -> str: + """Build a Basic authorization value without placing it in a URL.""" + if not username or not password: + raise RuntimeError("Harbor credentials are unavailable") + encoded = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + return f"Basic {encoded}" + + +def _manifest_url(component: str, reference: str) -> str: + """Return one same-origin, path-escaped Docker Registry manifest URL.""" + encoded = urllib.parse.quote(reference, safe="") + return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/manifests/{encoded}" + + +def _blob_url(component: str, digest: str) -> str: + """Return one same-origin, path-escaped Docker Registry blob URL.""" + encoded = urllib.parse.quote(digest, safe="") + return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/blobs/{encoded}" + + +def _read_evidence_pair( + *, digest_text: str, image_text: str, per_arch_tag_ref: str +) -> str: + """Cross-check both Kaniko output files for one arch against its tag.""" + digest_lines = digest_text.splitlines() + image_lines = image_text.splitlines() + if len(digest_lines) != 1: + raise ValueError("Kaniko digest evidence must contain exactly one line") + if len(image_lines) != 1: + raise ValueError("Kaniko image evidence must contain exactly one line") + digest = digest_lines[0].strip() + if not DIGEST_PATTERN.fullmatch(digest): + raise ValueError("invalid per-arch image digest") + if image_lines[0].strip() != f"{per_arch_tag_ref}@{digest}": + raise ValueError("per-arch image evidence does not match tag and digest") + return digest + + +def _verified_leaf( + *, + component: str, + per_arch_tag: str, + architecture: str, + expected_digest: str, + authorization: str, + opener: Callable[[urllib.request.Request, int], Any], +) -> dict[str, Any]: + """Re-read one per-arch leaf and prove its digest, type, and architecture.""" + accept = ", ".join(sorted(LEAF_MANIFEST_TYPES)) + request = urllib.request.Request( + _manifest_url(component, per_arch_tag), + headers={"Accept": accept, "Authorization": authorization}, + method="GET", + ) + with opener(request, 30) as response: + if _status(response) != 200: + raise RuntimeError( + f"{architecture} leaf manifest returned HTTP {_status(response)}" + ) + body = response.read(MAX_MANIFEST_BYTES + 1) + if len(body) > MAX_MANIFEST_BYTES: + raise RuntimeError(f"{architecture} leaf manifest exceeded the size limit") + observed_digest = response.headers.get("Docker-Content-Digest", "") + content_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip() + if observed_digest != expected_digest: + raise RuntimeError(f"{architecture} leaf digest does not match build evidence") + # Defence in depth: the digest header is registry-asserted; recompute it too. + if f"sha256:{hashlib.sha256(body).hexdigest()}" != expected_digest: + raise RuntimeError(f"{architecture} leaf bytes do not hash to its digest") + if content_type not in LEAF_MANIFEST_TYPES: + raise RuntimeError(f"{architecture} leaf is not a single-image manifest") + try: + manifest = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError(f"{architecture} leaf manifest is not valid JSON") from exc + config = manifest.get("config") + if not isinstance(config, dict): + raise RuntimeError(f"{architecture} leaf manifest omits its config descriptor") + config_digest = str(config.get("digest") or "") + config_type = str(config.get("mediaType") or "") + if not DIGEST_PATTERN.fullmatch(config_digest): + raise RuntimeError(f"{architecture} leaf config digest is invalid") + if config_type not in IMAGE_CONFIG_TYPES: + raise RuntimeError(f"{architecture} leaf config media type is unsupported") + config_request = urllib.request.Request( + _blob_url(component, config_digest), + headers={"Accept": config_type, "Authorization": authorization}, + method="GET", + ) + with opener(config_request, 30) as response: + if _status(response) != 200: + raise RuntimeError( + f"{architecture} leaf config returned HTTP {_status(response)}" + ) + config_body = response.read(MAX_CONFIG_BYTES + 1) + if len(config_body) > MAX_CONFIG_BYTES: + raise RuntimeError(f"{architecture} leaf config exceeded the size limit") + if f"sha256:{hashlib.sha256(config_body).hexdigest()}" != config_digest: + raise RuntimeError(f"{architecture} leaf config bytes do not hash to its digest") + try: + config_json = json.loads(config_body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError(f"{architecture} leaf config is not valid JSON") from exc + if config_json.get("architecture") != architecture: + raise RuntimeError( + f"{architecture} leaf config reports architecture " + f"{config_json.get('architecture')!r}" + ) + if config_json.get("os") != "linux": + raise RuntimeError(f"{architecture} leaf config reports a non-linux os") + return { + "mediaType": content_type, + "size": len(body), + "digest": expected_digest, + "platform": {"architecture": architecture, "os": "linux"}, + } + + +def _manifest_list_bytes(descriptors: list[dict[str, Any]]) -> bytes: + """Serialize the manifest list deterministically for a stable index digest.""" + document = { + "schemaVersion": 2, + "mediaType": DOCKER_MANIFEST_LIST, + "manifests": descriptors, + } + return json.dumps(document, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def combine_multiarch_index( + *, + destination: str, + arch_digests: dict[str, str], + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> dict[str, str]: + """Verify both leaves, publish, and re-verify one multi-arch index tag.""" + match = DESTINATION_PATTERN.fullmatch(destination.strip()) + if not match: + raise ValueError("invalid multi-arch destination") + if set(arch_digests) != set(ARCHITECTURES): + raise ValueError("expected exactly the arm64 and amd64 per-arch digests") + component = "hermes-agent" + index_tag = destination.rsplit(":", 1)[1] + authorization = _authorization(username, password) + + descriptors = [ + _verified_leaf( + component=component, + per_arch_tag=f"{index_tag}-{architecture}", + architecture=architecture, + expected_digest=arch_digests[architecture], + authorization=authorization, + opener=opener, + ) + for architecture in ARCHITECTURES + ] + manifest_list = _manifest_list_bytes(descriptors) + if len(manifest_list) > MAX_MANIFEST_BYTES: + raise RuntimeError("assembled manifest list exceeded the size limit") + index_digest = f"sha256:{hashlib.sha256(manifest_list).hexdigest()}" + + index_url = _manifest_url(component, index_tag) + head_request = urllib.request.Request( + index_url, + headers={"Accept": DOCKER_MANIFEST_LIST, "Authorization": authorization}, + method="HEAD", + ) + with opener(head_request, 20) as response: + head_status = _status(response) + existing_digest = response.headers.get("Docker-Content-Digest", "") + if head_status == 200: + if existing_digest != index_digest: + raise RuntimeError("final tag already exists with another index digest") + result = "already-present" + elif head_status == 404: + put_request = urllib.request.Request( + index_url, + data=manifest_list, + headers={ + "Authorization": authorization, + "Content-Type": DOCKER_MANIFEST_LIST, + }, + method="PUT", + ) + with opener(put_request, 30) as response: + put_status = _status(response) + put_digest = response.headers.get("Docker-Content-Digest", "") + if put_status not in {201, 202}: + raise RuntimeError(f"index manifest returned HTTP {put_status}") + if put_digest and put_digest != index_digest: + raise RuntimeError("index manifest digest changed during publish") + result = "published" + else: + raise RuntimeError(f"final tag preflight returned HTTP {head_status}") + + verify_request = urllib.request.Request( + index_url, + headers={"Accept": DOCKER_MANIFEST_LIST, "Authorization": authorization}, + method="GET", + ) + with opener(verify_request, 30) as response: + if _status(response) != 200: + raise RuntimeError(f"index verification returned HTTP {_status(response)}") + verify_body = response.read(MAX_MANIFEST_BYTES + 1) + if len(verify_body) > MAX_MANIFEST_BYTES: + raise RuntimeError("index verification exceeded the size limit") + verify_digest = response.headers.get("Docker-Content-Digest", "") + verify_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip() + if verify_digest != index_digest: + raise RuntimeError("registry resolved the final tag to another index digest") + if verify_type != DOCKER_MANIFEST_LIST: + raise RuntimeError("registry did not store a Docker manifest list") + try: + published = json.loads(verify_body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("registry returned invalid index JSON") from exc + published_leaves = { + ( + str((item or {}).get("platform", {}).get("architecture")), + str((item or {}).get("digest")), + ) + for item in published.get("manifests") or [] + } + expected_leaves = { + (architecture, arch_digests[architecture]) for architecture in ARCHITECTURES + } + if published_leaves != expected_leaves: + raise RuntimeError("published index does not reference the exact two leaves") + + return { + "component": component, + "index_digest": index_digest, + "index_tag": index_tag, + "result": result, + **{f"{architecture}_digest": arch_digests[architecture] for architecture in ARCHITECTURES}, + } + + +def _load_arch_digest( + *, destination: str, architecture: str, digest_file: Path, image_file: Path +) -> str: + """Bind one arch's two Kaniko evidence files to its arch-suffixed tag.""" + per_arch_tag_ref = f"{destination}-{architecture}" + return _read_evidence_pair( + digest_text=digest_file.read_text(encoding="utf-8"), + image_text=image_file.read_text(encoding="utf-8"), + per_arch_tag_ref=per_arch_tag_ref, + ) + + +def main() -> int: + """Combine two verified per-arch leaves and emit index digest evidence.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--destination", required=True) + parser.add_argument("--source-revision", required=True) + parser.add_argument("--build-number", required=True) + parser.add_argument("--arm64-digest-file", required=True, type=Path) + parser.add_argument("--arm64-image-file", required=True, type=Path) + parser.add_argument("--amd64-digest-file", required=True, type=Path) + parser.add_argument("--amd64-image-file", required=True, type=Path) + parser.add_argument("--digest-file", required=True, type=Path) + parser.add_argument("--image-file", required=True, type=Path) + args = parser.parse_args() + try: + match = DESTINATION_PATTERN.fullmatch(args.destination.strip()) + if not match: + raise ValueError("invalid multi-arch destination") + if match.group("revision") != args.source_revision.strip(): + raise ValueError("destination revision does not match evidence") + if match.group("build") != args.build_number.strip(): + raise ValueError("destination build number does not match evidence") + destination = args.destination.strip() + arch_digests = { + "arm64": _load_arch_digest( + destination=destination, + architecture="arm64", + digest_file=args.arm64_digest_file, + image_file=args.arm64_image_file, + ), + "amd64": _load_arch_digest( + destination=destination, + architecture="amd64", + digest_file=args.amd64_digest_file, + image_file=args.amd64_image_file, + ), + } + result = combine_multiarch_index( + destination=destination, + arch_digests=arch_digests, + username=os.environ.get("HARBOR_USER", ""), + password=os.environ.get("HARBOR_PASSWORD", ""), + ) + args.digest_file.write_text(result["index_digest"] + "\n", encoding="utf-8") + args.image_file.write_text( + f"{destination}@{result['index_digest']}\n", encoding="utf-8" + ) + except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc: + print(json.dumps({"error": str(exc)}, sort_keys=True)) + return 1 + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/dockerfiles/Dockerfile.hermes-agent b/dockerfiles/Dockerfile.hermes-agent index 4f41fec2..2ec4b6e5 100644 --- a/dockerfiles/Dockerfile.hermes-agent +++ b/dockerfiles/Dockerfile.hermes-agent @@ -1,6 +1,16 @@ # syntax=docker/dockerfile:1 # dockerfiles/Dockerfile.hermes-agent -FROM nousresearch/hermes-agent@sha256:47d4bd4cc420b70e40ed75efdade373e45b86b7382d4013a054208982bb6ba08 +# +# Multi-arch base: this digest is the upstream OCI image INDEX for tag +# v2026.7.7.2 (revision 9de9c25f620ff7f1ce0fd5457d596052d5159596). The index +# fans out to both native leaves of the same reviewed upstream version: +# linux/arm64 -> sha256:47d4bd4cc420b70e40ed75efdade373e45b86b7382d4013a054208982bb6ba08 +# linux/amd64 -> sha256:3db34ce19adfa080736a2a3feb0316dbcccc588faa9afe7fd8ae1c03b4f1a53a +# The arm64 leaf is byte-for-byte the previously pinned single-arch base, so the +# arm64 build is unchanged; Kaniko/containerd auto-selects the matching leaf per +# build platform (arm64 rpi5 pod vs amd64 titan-24 pod). Do NOT replace this with +# a per-arch leaf digest -- that would break the amd64 build leg. +FROM nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973 USER root diff --git a/testing/tests/test_hermes_image_builder.py b/testing/tests/test_hermes_image_builder.py index 248f1ba2..f517794f 100644 --- a/testing/tests/test_hermes_image_builder.py +++ b/testing/tests/test_hermes_image_builder.py @@ -113,7 +113,8 @@ def test_pipeline_requires_reviewed_main_and_runtime_credentials() -> None: assert "--destination=" in source assert "assert-absent" in source assert "git-${actual_revision}-build-${BUILD_NUMBER}" in source - assert source.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1 + # One reviewed heredoc-compat build-arg per native Kaniko leg (arm64 + amd64). + assert source.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 2 def test_kaniko_replays_only_the_exact_reviewed_heredoc_contract() -> None: diff --git a/testing/tests/test_hermes_image_builder_fresh_review.py b/testing/tests/test_hermes_image_builder_fresh_review.py index 5581f683..6f7e7dc4 100644 --- a/testing/tests/test_hermes_image_builder_fresh_review.py +++ b/testing/tests/test_hermes_image_builder_fresh_review.py @@ -32,6 +32,15 @@ def _pod_spec() -> dict: return yaml.safe_load(pod_yaml)["spec"] +def _amd64_pod_spec() -> dict: + """Parse the second inline pod (the native amd64 build leg on titan-24).""" + source = PIPELINE.read_text(encoding="utf-8") + blocks = source.split('yaml """') + assert len(blocks) == 3, "expected exactly the arm64 and amd64 build pods" + pod_yaml = blocks[2].split('"""', 1)[0] + return yaml.safe_load(pod_yaml)["spec"] + + def test_builder_prefers_rpi5_with_healthy_arm64_worker_fallback() -> None: """Disposable builds prefer rpi5 without excluding schedulable rpi4 workers.""" spec = _pod_spec() @@ -106,7 +115,8 @@ def test_exact_reviewed_heredocs_remain_buildkit_native_by_default() -> None: assert source.count("ARG HERMES_KANIKO_HEREDOC_COMPAT=0") == 1 assert "Kaniko v1.23.2" in RUNNER.read_text(encoding="utf-8") pipeline = PIPELINE.read_text(encoding="utf-8") - assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1 + # The reviewed heredoc replay is enabled on both native Kaniko legs. + assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 2 def test_appended_tenth_reviewed_form_rejects_before_execution( @@ -338,11 +348,91 @@ def test_pipeline_requires_and_archives_exact_release_evidence() -> None: assert "allowEmptyArchive: false" in evidence archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0] paths = archive.split(",") - assert len(paths) == 7 - assert len(set(paths)) == 7 + # Multi-arch adds both per-arch leaf digests/images plus the index digest. + assert len(paths) == 11 + assert len(set(paths)) == 11 assert all("*" not in path for path in paths) assert "find build -type f" in evidence assert "build/hermes-agent.source-revision" in paths + assert "build/hermes-agent.digest" in paths + for arch in ("arm64", "amd64"): + assert f"build/hermes-agent-{arch}.digest" in paths + assert f"build/hermes-agent-{arch}.image" in paths promotion = source.split("stage('Publish Flux release tag')", 1)[1] assert "ci/scripts/hermes_oci_promote.py" in promotion assert " post {" not in source + + +def test_amd64_leg_is_pinned_to_titan24_and_resource_capped() -> None: + """The amd64 build leg lands on titan-24, tolerates its guard taint, and is capped.""" + spec = _amd64_pod_spec() + assert spec["nodeSelector"] == { + "kubernetes.io/arch": "amd64", + "kubernetes.io/hostname": "titan-24", + "node-role.kubernetes.io/worker": "true", + } + # titan-24 co-hosts the Sui validator; the disposable build must tolerate + # whatever guard taint the node carries so the pinned pod still schedules. + assert {"operator": "Exists"} in spec["tolerations"] + assert spec["serviceAccountName"] == "hermes-image-builder" + assert spec["automountServiceAccountToken"] is False + assert spec["enableServiceLinks"] is False + + containers = {item["name"]: item for item in spec["containers"]} + kaniko = containers["kaniko"] + # Identical pinned Kaniko across both legs -- no second, unreviewed builder. + assert kaniko["image"] == ( + "gcr.io/kaniko-project/executor@sha256:" + "c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e" + ) + assert kaniko["securityContext"]["capabilities"]["add"] == [ + "CHOWN", + "FOWNER", + "DAC_OVERRIDE", + "SETGID", + "SETUID", + ] + for container in containers.values(): + security = container["securityContext"] + assert security["allowPrivilegeEscalation"] is False + assert security["capabilities"]["drop"] == ["ALL"] + assert security["seccompProfile"]["type"] == "RuntimeDefault" + assert security.get("privileged", False) is False + # Tight caps keep the disposable build from starving the co-hosted validator: + # the amd64 kaniko ceiling is strictly below the arm64 leg's 2 CPU / 4Gi. + limits = kaniko["resources"]["limits"] + assert limits["cpu"] == "1500m" + assert limits["memory"] == "3Gi" + + +def test_amd64_leg_source_is_independently_boundary_checked() -> None: + """The amd64 pod re-derives and re-verifies the reviewed revision itself.""" + source = PIPELINE.read_text(encoding="utf-8") + amd64_stage = source.split("stage('Build amd64 leg without a daemon')", 1)[1] + amd64_stage = amd64_stage.split("stage('Combine multi-arch index')", 1)[0] + # Same fail-closed boundary as the coordinating pod, re-run in the amd64 pod. + assert 'merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}"' in amd64_stage + assert 'checkout --detach "${EXPECTED_SOURCE_REVISION}"' in amd64_stage + assert 'test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"' in amd64_stage + assert 'status --porcelain' in amd64_stage + assert '$(cat build/hermes-agent.destination)-amd64' in amd64_stage + # The amd64 leaf evidence crosses pods only through an explicit stash. + assert "stash" in amd64_stage + assert "hermes-agent-amd64.digest" in amd64_stage + + +def test_combine_stage_publishes_and_reverifies_the_index() -> None: + """Kaniko cannot combine; the reviewed python combiner assembles the index.""" + source = PIPELINE.read_text(encoding="utf-8") + combine = source.split("stage('Combine multi-arch index')", 1)[1] + combine = combine.split("stage('Render reviewed Flux handoff')", 1)[0] + assert "unstash 'hermes-agent-amd64-evidence'" in combine + assert "ci/scripts/hermes_multiarch_combine.py" in combine + assert "--arm64-digest-file build/hermes-agent-arm64.digest" in combine + assert "--amd64-digest-file build/hermes-agent-amd64.digest" in combine + # The combiner emits the arch-less index evidence the existing chain promotes. + assert "--digest-file build/hermes-agent.digest" in combine + assert "--image-file build/hermes-agent.image" in combine + # Downstream render/verify/promote still consume the single index digest file. + render = source.split("stage('Render reviewed Flux handoff')", 1)[1] + assert "--digest-file build/hermes-agent.digest" in render diff --git a/testing/tests/test_hermes_multiarch_combine.py b/testing/tests/test_hermes_multiarch_combine.py new file mode 100644 index 00000000..9887b07e --- /dev/null +++ b/testing/tests/test_hermes_multiarch_combine.py @@ -0,0 +1,385 @@ +"""Safety tests for the fail-closed multi-arch manifest-list combiner.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import io +import json +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "ci/scripts/hermes_multiarch_combine.py" +REVISION = "a" * 40 +BUILD = "17" +DESTINATION = f"registry.bstein.dev/bstein/hermes-agent:git-{REVISION}-build-{BUILD}" +DOCKER_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json" +DOCKER_MANIFEST = "application/vnd.docker.distribution.manifest.v2+json" +IMAGE_CONFIG = "application/vnd.docker.container.image.v1+json" + + +def _load(): + spec = importlib.util.spec_from_file_location("hermes_multiarch_combine", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _sha(body: bytes) -> str: + return "sha256:" + hashlib.sha256(body).hexdigest() + + +class Response(io.BytesIO): + """Minimal context-managed urllib response.""" + + def __init__(self, status: int, body: bytes = b"", headers=None): + super().__init__(body) + self.status = status + self.headers = headers or {} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + +def _leaf(architecture: str, *, os_name: str = "linux"): + """Build a self-consistent config+manifest whose bytes hash to real digests.""" + config_body = json.dumps( + {"architecture": architecture, "os": os_name}, sort_keys=True + ).encode() + config_digest = _sha(config_body) + manifest_body = json.dumps( + { + "schemaVersion": 2, + "mediaType": DOCKER_MANIFEST, + "config": { + "mediaType": IMAGE_CONFIG, + "digest": config_digest, + "size": len(config_body), + }, + "layers": [], + }, + sort_keys=True, + ).encode() + return { + "arch": architecture, + "config_body": config_body, + "config_digest": config_digest, + "manifest_body": manifest_body, + "digest": _sha(manifest_body), + "content_type": DOCKER_MANIFEST, + } + + +def _descriptor(module, leaf): + return { + "mediaType": leaf["content_type"], + "size": len(leaf["manifest_body"]), + "digest": leaf["digest"], + "platform": {"architecture": leaf["arch"], "os": "linux"}, + } + + +def _index_digest(module, amd64, arm64) -> str: + body = module._manifest_list_bytes( + [_descriptor(module, amd64), _descriptor(module, arm64)] + ) + return _sha(body) + + +class Registry: + """Route the combiner's deterministic request sequence by method and path.""" + + def __init__(self, module, amd64, arm64, *, head_status=404, existing="") -> None: + self.module = module + self.leaves = {amd64["arch"]: amd64, arm64["arch"]: arm64} + self.index_digest = _index_digest(module, amd64, arm64) + self.head_status = head_status + self.existing = existing + self.put_body = None + self.calls = [] + + def __call__(self, request, timeout): + method = request.method + url = request.full_url + self.calls.append((method, url)) + for arch, leaf in self.leaves.items(): + if url.endswith(f"-{arch}"): + return Response( + 200, + leaf["manifest_body"], + { + "Docker-Content-Digest": leaf["digest"], + "Content-Type": leaf["content_type"], + }, + ) + # Blob URLs percent-encode the ``sha256:`` colon; match the raw hex. + if url.endswith(leaf["config_digest"].split(":", 1)[1]): + return Response(200, leaf["config_body"], {}) + # Final index tag (no arch suffix, ends with the build tag). + if method == "HEAD": + return Response( + self.head_status, + headers={"Docker-Content-Digest": self.existing}, + ) + if method == "PUT": + self.put_body = request.data + return Response(201, headers={"Docker-Content-Digest": self.index_digest}) + return Response( + 200, + self.module._manifest_list_bytes( + [ + _descriptor(self.module, self.leaves["amd64"]), + _descriptor(self.module, self.leaves["arm64"]), + ] + ), + { + "Docker-Content-Digest": self.index_digest, + "Content-Type": DOCKER_MANIFEST_LIST, + }, + ) + + +def _combine(module, registry, arch_digests=None): + if arch_digests is None: + arch_digests = { + "amd64": registry.leaves["amd64"]["digest"], + "arm64": registry.leaves["arm64"]["digest"], + } + return module.combine_multiarch_index( + destination=DESTINATION, + arch_digests=arch_digests, + username="robot", + password="private", + opener=registry, + ) + + +def test_combines_two_verified_leaves_into_one_index() -> None: + """Both native leaves are re-read, arch-proven, and published as a list.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + registry = Registry(module, amd64, arm64) + result = _combine(module, registry) + assert result["result"] == "published" + assert result["index_digest"] == registry.index_digest + assert result["amd64_digest"] == amd64["digest"] + assert result["arm64_digest"] == arm64["digest"] + # The published bytes are exactly what we hashed for the index digest. + assert _sha(registry.put_body) == registry.index_digest + assert "private" not in json.dumps(result) + + +def test_idempotent_when_index_already_matches() -> None: + """A replay is accepted only when the existing index digest is identical.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + index_digest = _index_digest(module, amd64, arm64) + registry = Registry( + module, amd64, arm64, head_status=200, existing=index_digest + ) + result = _combine(module, registry) + assert result["result"] == "already-present" + assert registry.put_body is None + + +def test_existing_index_with_other_digest_fails_closed() -> None: + """An occupied final tag with a different index digest never republishes.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + registry = Registry( + module, amd64, arm64, head_status=200, existing="sha256:" + "c" * 64 + ) + with pytest.raises(RuntimeError, match="another index digest"): + _combine(module, registry) + + +def test_rejects_leaf_digest_that_disagrees_with_evidence() -> None: + """A leaf whose registry digest is not the Kaniko evidence fails closed.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + registry = Registry(module, amd64, arm64) + with pytest.raises(RuntimeError, match="amd64 leaf digest"): + _combine( + module, + registry, + arch_digests={"amd64": "sha256:" + "d" * 64, "arm64": arm64["digest"]}, + ) + + +def test_rejects_leaf_whose_config_architecture_is_wrong() -> None: + """A leaf that claims the wrong architecture in its config fails closed.""" + module = _load() + # Build an "amd64" candidate tag whose config actually says arm64. + swapped = _leaf("arm64") + swapped["arch"] = "amd64" # served under the amd64 tag, but arm64 inside + arm64 = _leaf("arm64") + registry = Registry(module, swapped, arm64) + with pytest.raises(RuntimeError, match="architecture"): + _combine( + module, + registry, + arch_digests={"amd64": swapped["digest"], "arm64": arm64["digest"]}, + ) + + +def test_rejects_leaf_served_as_a_manifest_list() -> None: + """A per-arch leaf must be a single image, never itself an index/list.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + amd64["content_type"] = DOCKER_MANIFEST_LIST + registry = Registry(module, amd64, arm64) + with pytest.raises(RuntimeError, match="single-image manifest"): + _combine(module, registry) + + +def test_requires_both_architectures() -> None: + """The combiner refuses any arch set other than exactly arm64 and amd64.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + registry = Registry(module, amd64, arm64) + with pytest.raises(ValueError, match="arm64 and amd64"): + module.combine_multiarch_index( + destination=DESTINATION, + arch_digests={"amd64": amd64["digest"]}, + username="robot", + password="private", + opener=registry, + ) + + +def test_rejects_destination_with_arch_suffix() -> None: + """Only the arch-less final tag is a valid combine destination.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + with pytest.raises(ValueError, match="invalid multi-arch destination"): + module.combine_multiarch_index( + destination=f"{DESTINATION}-arm64", + arch_digests={"amd64": amd64["digest"], "arm64": arm64["digest"]}, + username="robot", + password="private", + opener=Registry(module, amd64, arm64), + ) + + +def test_cli_binds_evidence_files_and_writes_index_outputs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """The command reads both Kaniko pairs and emits arch-less index evidence.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + registry = Registry(module, amd64, arm64) + files = {} + for arch, leaf in (("amd64", amd64), ("arm64", arm64)): + digest_file = tmp_path / f"{arch}.digest" + image_file = tmp_path / f"{arch}.image" + digest_file.write_text(leaf["digest"] + "\n", encoding="utf-8") + image_file.write_text( + f"{DESTINATION}-{arch}@{leaf['digest']}\n", encoding="utf-8" + ) + files[arch] = (digest_file, image_file) + out_digest = tmp_path / "index.digest" + out_image = tmp_path / "index.image" + monkeypatch.setenv("HARBOR_USER", "robot") + monkeypatch.setenv("HARBOR_PASSWORD", "private") + captured = {} + + def fake_combine(*, destination, arch_digests, username, password): + captured["destination"] = destination + captured["arch_digests"] = dict(arch_digests) + return {"index_digest": registry.index_digest, "result": "published"} + + monkeypatch.setattr(module, "combine_multiarch_index", fake_combine) + monkeypatch.setattr( + sys, + "argv", + [ + "hermes_multiarch_combine.py", + "--destination", + DESTINATION, + "--source-revision", + REVISION, + "--build-number", + BUILD, + "--arm64-digest-file", + str(files["arm64"][0]), + "--arm64-image-file", + str(files["arm64"][1]), + "--amd64-digest-file", + str(files["amd64"][0]), + "--amd64-image-file", + str(files["amd64"][1]), + "--digest-file", + str(out_digest), + "--image-file", + str(out_image), + ], + ) + assert module.main() == 0 + assert out_digest.read_text(encoding="utf-8").strip() == registry.index_digest + assert ( + out_image.read_text(encoding="utf-8").strip() + == f"{DESTINATION}@{registry.index_digest}" + ) + assert json.loads(capsys.readouterr().out)["result"] == "published" + # main() must bind each arch's two evidence files to the correct leaf digest. + assert captured["destination"] == DESTINATION + assert captured["arch_digests"] == { + "amd64": amd64["digest"], + "arm64": arm64["digest"], + } + + +def test_cli_rejects_mismatched_per_arch_evidence_pair( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A digest/image evidence file mismatch is reported as a JSON error.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + files = {} + for arch, leaf in (("amd64", amd64), ("arm64", arm64)): + digest_file = tmp_path / f"{arch}.digest" + image_file = tmp_path / f"{arch}.image" + digest_file.write_text(leaf["digest"] + "\n", encoding="utf-8") + # amd64 image file points at the wrong digest. + recorded = "sha256:" + "e" * 64 if arch == "amd64" else leaf["digest"] + image_file.write_text( + f"{DESTINATION}-{arch}@{recorded}\n", encoding="utf-8" + ) + files[arch] = (digest_file, image_file) + monkeypatch.setenv("HARBOR_USER", "robot") + monkeypatch.setenv("HARBOR_PASSWORD", "private") + monkeypatch.setattr( + sys, + "argv", + [ + "hermes_multiarch_combine.py", + "--destination", + DESTINATION, + "--source-revision", + REVISION, + "--build-number", + BUILD, + "--arm64-digest-file", + str(files["arm64"][0]), + "--arm64-image-file", + str(files["arm64"][1]), + "--amd64-digest-file", + str(files["amd64"][0]), + "--amd64-image-file", + str(files["amd64"][1]), + "--digest-file", + str(tmp_path / "index.digest"), + "--image-file", + str(tmp_path / "index.image"), + ], + ) + assert module.main() == 1 + assert "does not match" in json.loads(capsys.readouterr().out)["error"]