diff --git a/ci/Jenkinsfile.hermes-agent-image b/ci/Jenkinsfile.hermes-agent-image index 76a32000..3f7ea910 100644 --- a/ci/Jenkinsfile.hermes-agent-image +++ b/ci/Jenkinsfile.hermes-agent-image @@ -188,6 +188,7 @@ spec: testing/tests/test_hermes_image_builder_coverage.py \ testing/tests/test_hermes_image_builder_fresh_review.py \ testing/tests/test_hermes_oci_promote.py \ + testing/tests/test_hermes_multiarch_combine.py \ testing/tests/test_hermes_image_automation.py ''' } @@ -213,7 +214,7 @@ spec: } } } - stage('Build and publish without a daemon') { + stage('Build arm64 leg without a daemon') { steps { container('kaniko') { withCredentials([usernamePassword( @@ -225,7 +226,7 @@ spec: set -eu set +x config_path=/kaniko/.docker/config.json - destination="$(cat build/hermes-agent.destination)" + destination="$(cat build/hermes-agent.destination)-arm64" source_revision="$(cat build/hermes-agent.source-revision)" umask 077 auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')" @@ -240,18 +241,192 @@ spec: --context="dir://${WORKSPACE}" \ --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \ --destination="${destination}" \ - --digest-file="${WORKSPACE}/build/hermes-agent.digest" \ - --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \ + --digest-file="${WORKSPACE}/build/hermes-agent-arm64.digest" \ + --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-arm64.image" \ --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \ --label="org.opencontainers.image.revision=${source_revision}" \ --cleanup \ --push-retry=3 - /busybox/chmod 644 build/hermes-agent.digest build/hermes-agent.image + /busybox/chmod 644 build/hermes-agent-arm64.digest build/hermes-agent-arm64.image ''' } } } } + stage('Build amd64 leg without a daemon') { + agent { + kubernetes { + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + atlas.bstein.dev/workload: hermes-agent-image-builder-amd64 +spec: + serviceAccountName: hermes-image-builder + automountServiceAccountToken: false + enableServiceLinks: false + restartPolicy: Never + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/hostname: titan-24 + node-role.kubernetes.io/worker: "true" + tolerations: + # titan-24 co-hosts the out-of-cluster Sui validator; tolerate whatever + # PreferNoSchedule/NoSchedule guard taint the node carries so the pinned + # build lands, and rely on the tight resource caps below (not scheduling + # priority) to keep the disposable build from starving the validator. + - operator: Exists + imagePullSecrets: + - name: harbor-bstein-robot + containers: + - name: jnlp + image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 25m + memory: 128Mi + limits: + cpu: 250m + memory: 384Mi + - name: kaniko + image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e + command: ["/busybox/sh", "-c"] + args: ["/busybox/sleep 99d"] + tty: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"] + privileged: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + cpu: "1500m" + memory: 3Gi +""" + } + } + steps { + container('jnlp') { + sh ''' + set -eu + mkdir -p build + test "${PUBLISH_IMAGE}" = "true" + test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT" + case "${EXPECTED_SOURCE_REVISION}" in + *[!0-9a-f]*|'') + echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2 + exit 2 + ;; + esac + test "${#EXPECTED_SOURCE_REVISION}" -eq 40 + main_revision="$(git rev-parse origin/main)" + git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}" + git checkout --detach "${EXPECTED_SOURCE_REVISION}" + actual_revision="$(git rev-parse HEAD)" + test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}" + test -z "$(git status --porcelain)" + test -f dockerfiles/Dockerfile.hermes-agent + case "${BUILD_NUMBER}" in + ''|0*|*[!0-9]*) + echo "BUILD_NUMBER must be a positive decimal integer" >&2 + exit 2 + ;; + esac + printf '%s\n' \ + "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \ + > build/hermes-agent.destination + printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision + ''' + } + container('kaniko') { + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh '''#!/busybox/sh + set -eu + set +x + config_path=/kaniko/.docker/config.json + destination="$(cat build/hermes-agent.destination)-amd64" + source_revision="$(cat build/hermes-agent.source-revision)" + umask 077 + auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')" + /busybox/mkdir -p /kaniko/.docker + /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}" + unset HARBOR_USER HARBOR_PASSWORD auth + trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM + umask 022 + /kaniko/executor \ + --registry-mirror=harbor-core.harbor.svc.cluster.local \ + --insecure-registry=harbor-core.harbor.svc.cluster.local \ + --context="dir://${WORKSPACE}" \ + --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \ + --destination="${destination}" \ + --digest-file="${WORKSPACE}/build/hermes-agent-amd64.digest" \ + --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-amd64.image" \ + --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \ + --label="org.opencontainers.image.revision=${source_revision}" \ + --cleanup \ + --push-retry=3 + /busybox/chmod 644 build/hermes-agent-amd64.digest build/hermes-agent-amd64.image + ''' + } + } + stash( + name: 'hermes-agent-amd64-evidence', + includes: 'build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image' + ) + } + } + stage('Combine multi-arch index') { + steps { + unstash 'hermes-agent-amd64-evidence' + withCredentials([usernamePassword( + credentialsId: 'harbor-robot', + usernameVariable: 'HARBOR_USER', + passwordVariable: 'HARBOR_PASSWORD' + )]) { + sh ''' + set -eu + set +x + destination="$(cat build/hermes-agent.destination)" + source_revision="$(cat build/hermes-agent.source-revision)" + python3 ci/scripts/hermes_multiarch_combine.py \ + --destination "${destination}" \ + --source-revision "${source_revision}" \ + --build-number "${BUILD_NUMBER}" \ + --arm64-digest-file build/hermes-agent-arm64.digest \ + --arm64-image-file build/hermes-agent-arm64.image \ + --amd64-digest-file build/hermes-agent-amd64.digest \ + --amd64-image-file build/hermes-agent-amd64.image \ + --digest-file build/hermes-agent.digest \ + --image-file build/hermes-agent.image + test -s build/hermes-agent.digest + test -s build/hermes-agent.image + ''' + } + } + } stage('Render reviewed Flux handoff') { steps { withCredentials([usernamePassword( @@ -287,6 +462,10 @@ spec: build/hermes-agent.digest \ build/hermes-agent.image \ build/hermes-agent.source-revision \ + build/hermes-agent-arm64.digest \ + build/hermes-agent-arm64.image \ + build/hermes-agent-amd64.digest \ + build/hermes-agent-amd64.image \ build/hermes-agent-release/hermes-agent-image.json \ build/hermes-agent-release/hermes-image-update.patch \ build/hermes-agent-release/hermes-kustomization.yaml \ @@ -305,7 +484,7 @@ spec: --output-dir build/hermes-agent-release ''' archiveArtifacts( - artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.source-revision,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml', + artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.source-revision,build/hermes-agent-arm64.digest,build/hermes-agent-arm64.image,build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml', allowEmptyArchive: false, fingerprint: true ) diff --git a/ci/scripts/hermes_multiarch_combine.py b/ci/scripts/hermes_multiarch_combine.py new file mode 100644 index 00000000..c5dbc300 --- /dev/null +++ b/ci/scripts/hermes_multiarch_combine.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +"""Assemble a fail-closed multi-arch manifest list from two per-arch leaves. + +Kaniko builds one native image per architecture (arm64 on an rpi5 pod, amd64 on +titan-24) and pushes each under an arch-suffixed candidate tag +``...-build--``. Kaniko cannot combine, so this step: + +1. Independently re-reads each per-arch candidate manifest from the registry and + binds it to the exact Kaniko digest evidence. +2. Proves each leaf really is the architecture it claims by reading its image + config (a swapped or cross-built leaf fails closed here). +3. Builds a Docker manifest *list* (not an OCI index) from the two verified + leaves -- Docker manifest lists are already inside the promotion allow-list, + so this keeps the security surface of ``hermes_oci_promote.py`` unchanged. +4. Refuses to overwrite an existing final tag, PUTs the list to the final + ``...-build-`` tag, and re-reads it to confirm the registry resolved the + exact index digest referencing exactly the two expected leaves. + +The output ``--digest-file``/``--image-file`` deliberately use the SAME format +the single-arch Kaniko step produced (```` and ``@`` +for the final, arch-less tag). The whole downstream evidence chain -- +``hermes_image_release.py`` render/verify-evidence and ``hermes_oci_promote.py`` +-- therefore promotes the multi-arch INDEX with no further change. +""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +import re +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any, Callable + + +REGISTRY_ORIGIN = "https://registry.bstein.dev" +# The final (arch-less) Flux-visible tag; identical contract to the promoter. +DESTINATION_PATTERN = re.compile( + r"^registry\.bstein\.dev/bstein/hermes-agent:" + r"git-(?P[0-9a-f]{40})-build-(?P[1-9][0-9]*)$" +) +DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") +DOCKER_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json" +# A per-arch leaf must be a single-image manifest, never itself a list/index. +LEAF_MANIFEST_TYPES = { + "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.oci.image.manifest.v1+json", +} +IMAGE_CONFIG_TYPES = { + "application/vnd.docker.container.image.v1+json", + "application/vnd.oci.image.config.v1+json", +} +# Deterministic architecture order -> deterministic manifest-list bytes/digest. +ARCHITECTURES = ("amd64", "arm64") +MAX_MANIFEST_BYTES = 4 * 1024 * 1024 +MAX_CONFIG_BYTES = 1024 * 1024 + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Never forward registry credentials to another origin.""" + + def redirect_request(self, _request, _file, _code, _message, _headers, _url): + return None + + +def _registry_request(request: urllib.request.Request, timeout: int) -> Any: + """Return normal and HTTP error responses without following redirects.""" + opener = urllib.request.build_opener(_NoRedirect()) + try: + return opener.open(request, timeout=timeout) + except urllib.error.HTTPError as exc: + return exc + + +def _status(response: Any) -> int: + """Normalize urllib response and HTTPError status fields.""" + return int(getattr(response, "status", getattr(response, "code", 0))) + + +def _authorization(username: str, password: str) -> str: + """Build a Basic authorization value without placing it in a URL.""" + if not username or not password: + raise RuntimeError("Harbor credentials are unavailable") + encoded = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + return f"Basic {encoded}" + + +def _manifest_url(component: str, reference: str) -> str: + """Return one same-origin, path-escaped Docker Registry manifest URL.""" + encoded = urllib.parse.quote(reference, safe="") + return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/manifests/{encoded}" + + +def _blob_url(component: str, digest: str) -> str: + """Return one same-origin, path-escaped Docker Registry blob URL.""" + encoded = urllib.parse.quote(digest, safe="") + return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/blobs/{encoded}" + + +def _read_evidence_pair( + *, digest_text: str, image_text: str, per_arch_tag_ref: str +) -> str: + """Cross-check both Kaniko output files for one arch against its tag.""" + digest_lines = digest_text.splitlines() + image_lines = image_text.splitlines() + if len(digest_lines) != 1: + raise ValueError("Kaniko digest evidence must contain exactly one line") + if len(image_lines) != 1: + raise ValueError("Kaniko image evidence must contain exactly one line") + digest = digest_lines[0].strip() + if not DIGEST_PATTERN.fullmatch(digest): + raise ValueError("invalid per-arch image digest") + if image_lines[0].strip() != f"{per_arch_tag_ref}@{digest}": + raise ValueError("per-arch image evidence does not match tag and digest") + return digest + + +def _verified_leaf( + *, + component: str, + per_arch_tag: str, + architecture: str, + expected_digest: str, + authorization: str, + opener: Callable[[urllib.request.Request, int], Any], +) -> dict[str, Any]: + """Re-read one per-arch leaf and prove its digest, type, and architecture.""" + accept = ", ".join(sorted(LEAF_MANIFEST_TYPES)) + request = urllib.request.Request( + _manifest_url(component, per_arch_tag), + headers={"Accept": accept, "Authorization": authorization}, + method="GET", + ) + with opener(request, 30) as response: + if _status(response) != 200: + raise RuntimeError( + f"{architecture} leaf manifest returned HTTP {_status(response)}" + ) + body = response.read(MAX_MANIFEST_BYTES + 1) + if len(body) > MAX_MANIFEST_BYTES: + raise RuntimeError(f"{architecture} leaf manifest exceeded the size limit") + observed_digest = response.headers.get("Docker-Content-Digest", "") + content_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip() + if observed_digest != expected_digest: + raise RuntimeError(f"{architecture} leaf digest does not match build evidence") + # Defence in depth: the digest header is registry-asserted; recompute it too. + if f"sha256:{hashlib.sha256(body).hexdigest()}" != expected_digest: + raise RuntimeError(f"{architecture} leaf bytes do not hash to its digest") + if content_type not in LEAF_MANIFEST_TYPES: + raise RuntimeError(f"{architecture} leaf is not a single-image manifest") + try: + manifest = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError(f"{architecture} leaf manifest is not valid JSON") from exc + config = manifest.get("config") + if not isinstance(config, dict): + raise RuntimeError(f"{architecture} leaf manifest omits its config descriptor") + config_digest = str(config.get("digest") or "") + config_type = str(config.get("mediaType") or "") + if not DIGEST_PATTERN.fullmatch(config_digest): + raise RuntimeError(f"{architecture} leaf config digest is invalid") + if config_type not in IMAGE_CONFIG_TYPES: + raise RuntimeError(f"{architecture} leaf config media type is unsupported") + config_request = urllib.request.Request( + _blob_url(component, config_digest), + headers={"Accept": config_type, "Authorization": authorization}, + method="GET", + ) + with opener(config_request, 30) as response: + if _status(response) != 200: + raise RuntimeError( + f"{architecture} leaf config returned HTTP {_status(response)}" + ) + config_body = response.read(MAX_CONFIG_BYTES + 1) + if len(config_body) > MAX_CONFIG_BYTES: + raise RuntimeError(f"{architecture} leaf config exceeded the size limit") + if f"sha256:{hashlib.sha256(config_body).hexdigest()}" != config_digest: + raise RuntimeError(f"{architecture} leaf config bytes do not hash to its digest") + try: + config_json = json.loads(config_body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError(f"{architecture} leaf config is not valid JSON") from exc + if config_json.get("architecture") != architecture: + raise RuntimeError( + f"{architecture} leaf config reports architecture " + f"{config_json.get('architecture')!r}" + ) + if config_json.get("os") != "linux": + raise RuntimeError(f"{architecture} leaf config reports a non-linux os") + return { + "mediaType": content_type, + "size": len(body), + "digest": expected_digest, + "platform": {"architecture": architecture, "os": "linux"}, + } + + +def _manifest_list_bytes(descriptors: list[dict[str, Any]]) -> bytes: + """Serialize the manifest list deterministically for a stable index digest.""" + document = { + "schemaVersion": 2, + "mediaType": DOCKER_MANIFEST_LIST, + "manifests": descriptors, + } + return json.dumps(document, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def combine_multiarch_index( + *, + destination: str, + arch_digests: dict[str, str], + username: str, + password: str, + opener: Callable[[urllib.request.Request, int], Any] = _registry_request, +) -> dict[str, str]: + """Verify both leaves, publish, and re-verify one multi-arch index tag.""" + match = DESTINATION_PATTERN.fullmatch(destination.strip()) + if not match: + raise ValueError("invalid multi-arch destination") + if set(arch_digests) != set(ARCHITECTURES): + raise ValueError("expected exactly the arm64 and amd64 per-arch digests") + component = "hermes-agent" + index_tag = destination.rsplit(":", 1)[1] + authorization = _authorization(username, password) + + descriptors = [ + _verified_leaf( + component=component, + per_arch_tag=f"{index_tag}-{architecture}", + architecture=architecture, + expected_digest=arch_digests[architecture], + authorization=authorization, + opener=opener, + ) + for architecture in ARCHITECTURES + ] + manifest_list = _manifest_list_bytes(descriptors) + if len(manifest_list) > MAX_MANIFEST_BYTES: + raise RuntimeError("assembled manifest list exceeded the size limit") + index_digest = f"sha256:{hashlib.sha256(manifest_list).hexdigest()}" + + index_url = _manifest_url(component, index_tag) + head_request = urllib.request.Request( + index_url, + headers={"Accept": DOCKER_MANIFEST_LIST, "Authorization": authorization}, + method="HEAD", + ) + with opener(head_request, 20) as response: + head_status = _status(response) + existing_digest = response.headers.get("Docker-Content-Digest", "") + if head_status == 200: + if existing_digest != index_digest: + raise RuntimeError("final tag already exists with another index digest") + result = "already-present" + elif head_status == 404: + put_request = urllib.request.Request( + index_url, + data=manifest_list, + headers={ + "Authorization": authorization, + "Content-Type": DOCKER_MANIFEST_LIST, + }, + method="PUT", + ) + with opener(put_request, 30) as response: + put_status = _status(response) + put_digest = response.headers.get("Docker-Content-Digest", "") + if put_status not in {201, 202}: + raise RuntimeError(f"index manifest returned HTTP {put_status}") + if put_digest and put_digest != index_digest: + raise RuntimeError("index manifest digest changed during publish") + result = "published" + else: + raise RuntimeError(f"final tag preflight returned HTTP {head_status}") + + verify_request = urllib.request.Request( + index_url, + headers={"Accept": DOCKER_MANIFEST_LIST, "Authorization": authorization}, + method="GET", + ) + with opener(verify_request, 30) as response: + if _status(response) != 200: + raise RuntimeError(f"index verification returned HTTP {_status(response)}") + verify_body = response.read(MAX_MANIFEST_BYTES + 1) + if len(verify_body) > MAX_MANIFEST_BYTES: + raise RuntimeError("index verification exceeded the size limit") + verify_digest = response.headers.get("Docker-Content-Digest", "") + verify_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip() + if verify_digest != index_digest: + raise RuntimeError("registry resolved the final tag to another index digest") + if verify_type != DOCKER_MANIFEST_LIST: + raise RuntimeError("registry did not store a Docker manifest list") + try: + published = json.loads(verify_body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("registry returned invalid index JSON") from exc + published_leaves = { + ( + str((item or {}).get("platform", {}).get("architecture")), + str((item or {}).get("digest")), + ) + for item in published.get("manifests") or [] + } + expected_leaves = { + (architecture, arch_digests[architecture]) for architecture in ARCHITECTURES + } + if published_leaves != expected_leaves: + raise RuntimeError("published index does not reference the exact two leaves") + + return { + "component": component, + "index_digest": index_digest, + "index_tag": index_tag, + "result": result, + **{f"{architecture}_digest": arch_digests[architecture] for architecture in ARCHITECTURES}, + } + + +def _load_arch_digest( + *, destination: str, architecture: str, digest_file: Path, image_file: Path +) -> str: + """Bind one arch's two Kaniko evidence files to its arch-suffixed tag.""" + per_arch_tag_ref = f"{destination}-{architecture}" + return _read_evidence_pair( + digest_text=digest_file.read_text(encoding="utf-8"), + image_text=image_file.read_text(encoding="utf-8"), + per_arch_tag_ref=per_arch_tag_ref, + ) + + +def main() -> int: + """Combine two verified per-arch leaves and emit index digest evidence.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--destination", required=True) + parser.add_argument("--source-revision", required=True) + parser.add_argument("--build-number", required=True) + parser.add_argument("--arm64-digest-file", required=True, type=Path) + parser.add_argument("--arm64-image-file", required=True, type=Path) + parser.add_argument("--amd64-digest-file", required=True, type=Path) + parser.add_argument("--amd64-image-file", required=True, type=Path) + parser.add_argument("--digest-file", required=True, type=Path) + parser.add_argument("--image-file", required=True, type=Path) + args = parser.parse_args() + try: + match = DESTINATION_PATTERN.fullmatch(args.destination.strip()) + if not match: + raise ValueError("invalid multi-arch destination") + if match.group("revision") != args.source_revision.strip(): + raise ValueError("destination revision does not match evidence") + if match.group("build") != args.build_number.strip(): + raise ValueError("destination build number does not match evidence") + destination = args.destination.strip() + arch_digests = { + "arm64": _load_arch_digest( + destination=destination, + architecture="arm64", + digest_file=args.arm64_digest_file, + image_file=args.arm64_image_file, + ), + "amd64": _load_arch_digest( + destination=destination, + architecture="amd64", + digest_file=args.amd64_digest_file, + image_file=args.amd64_image_file, + ), + } + result = combine_multiarch_index( + destination=destination, + arch_digests=arch_digests, + username=os.environ.get("HARBOR_USER", ""), + password=os.environ.get("HARBOR_PASSWORD", ""), + ) + args.digest_file.write_text(result["index_digest"] + "\n", encoding="utf-8") + args.image_file.write_text( + f"{destination}@{result['index_digest']}\n", encoding="utf-8" + ) + except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc: + print(json.dumps({"error": str(exc)}, sort_keys=True)) + return 1 + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/dockerfiles/Dockerfile.hermes-agent b/dockerfiles/Dockerfile.hermes-agent index 4f41fec2..2ec4b6e5 100644 --- a/dockerfiles/Dockerfile.hermes-agent +++ b/dockerfiles/Dockerfile.hermes-agent @@ -1,6 +1,16 @@ # syntax=docker/dockerfile:1 # dockerfiles/Dockerfile.hermes-agent -FROM nousresearch/hermes-agent@sha256:47d4bd4cc420b70e40ed75efdade373e45b86b7382d4013a054208982bb6ba08 +# +# Multi-arch base: this digest is the upstream OCI image INDEX for tag +# v2026.7.7.2 (revision 9de9c25f620ff7f1ce0fd5457d596052d5159596). The index +# fans out to both native leaves of the same reviewed upstream version: +# linux/arm64 -> sha256:47d4bd4cc420b70e40ed75efdade373e45b86b7382d4013a054208982bb6ba08 +# linux/amd64 -> sha256:3db34ce19adfa080736a2a3feb0316dbcccc588faa9afe7fd8ae1c03b4f1a53a +# The arm64 leaf is byte-for-byte the previously pinned single-arch base, so the +# arm64 build is unchanged; Kaniko/containerd auto-selects the matching leaf per +# build platform (arm64 rpi5 pod vs amd64 titan-24 pod). Do NOT replace this with +# a per-arch leaf digest -- that would break the amd64 build leg. +FROM nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973 USER root diff --git a/testing/tests/test_hermes_image_builder.py b/testing/tests/test_hermes_image_builder.py index 248f1ba2..f517794f 100644 --- a/testing/tests/test_hermes_image_builder.py +++ b/testing/tests/test_hermes_image_builder.py @@ -113,7 +113,8 @@ def test_pipeline_requires_reviewed_main_and_runtime_credentials() -> None: assert "--destination=" in source assert "assert-absent" in source assert "git-${actual_revision}-build-${BUILD_NUMBER}" in source - assert source.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1 + # One reviewed heredoc-compat build-arg per native Kaniko leg (arm64 + amd64). + assert source.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 2 def test_kaniko_replays_only_the_exact_reviewed_heredoc_contract() -> None: diff --git a/testing/tests/test_hermes_image_builder_fresh_review.py b/testing/tests/test_hermes_image_builder_fresh_review.py index 5581f683..6f7e7dc4 100644 --- a/testing/tests/test_hermes_image_builder_fresh_review.py +++ b/testing/tests/test_hermes_image_builder_fresh_review.py @@ -32,6 +32,15 @@ def _pod_spec() -> dict: return yaml.safe_load(pod_yaml)["spec"] +def _amd64_pod_spec() -> dict: + """Parse the second inline pod (the native amd64 build leg on titan-24).""" + source = PIPELINE.read_text(encoding="utf-8") + blocks = source.split('yaml """') + assert len(blocks) == 3, "expected exactly the arm64 and amd64 build pods" + pod_yaml = blocks[2].split('"""', 1)[0] + return yaml.safe_load(pod_yaml)["spec"] + + def test_builder_prefers_rpi5_with_healthy_arm64_worker_fallback() -> None: """Disposable builds prefer rpi5 without excluding schedulable rpi4 workers.""" spec = _pod_spec() @@ -106,7 +115,8 @@ def test_exact_reviewed_heredocs_remain_buildkit_native_by_default() -> None: assert source.count("ARG HERMES_KANIKO_HEREDOC_COMPAT=0") == 1 assert "Kaniko v1.23.2" in RUNNER.read_text(encoding="utf-8") pipeline = PIPELINE.read_text(encoding="utf-8") - assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1 + # The reviewed heredoc replay is enabled on both native Kaniko legs. + assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 2 def test_appended_tenth_reviewed_form_rejects_before_execution( @@ -338,11 +348,91 @@ def test_pipeline_requires_and_archives_exact_release_evidence() -> None: assert "allowEmptyArchive: false" in evidence archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0] paths = archive.split(",") - assert len(paths) == 7 - assert len(set(paths)) == 7 + # Multi-arch adds both per-arch leaf digests/images plus the index digest. + assert len(paths) == 11 + assert len(set(paths)) == 11 assert all("*" not in path for path in paths) assert "find build -type f" in evidence assert "build/hermes-agent.source-revision" in paths + assert "build/hermes-agent.digest" in paths + for arch in ("arm64", "amd64"): + assert f"build/hermes-agent-{arch}.digest" in paths + assert f"build/hermes-agent-{arch}.image" in paths promotion = source.split("stage('Publish Flux release tag')", 1)[1] assert "ci/scripts/hermes_oci_promote.py" in promotion assert " post {" not in source + + +def test_amd64_leg_is_pinned_to_titan24_and_resource_capped() -> None: + """The amd64 build leg lands on titan-24, tolerates its guard taint, and is capped.""" + spec = _amd64_pod_spec() + assert spec["nodeSelector"] == { + "kubernetes.io/arch": "amd64", + "kubernetes.io/hostname": "titan-24", + "node-role.kubernetes.io/worker": "true", + } + # titan-24 co-hosts the Sui validator; the disposable build must tolerate + # whatever guard taint the node carries so the pinned pod still schedules. + assert {"operator": "Exists"} in spec["tolerations"] + assert spec["serviceAccountName"] == "hermes-image-builder" + assert spec["automountServiceAccountToken"] is False + assert spec["enableServiceLinks"] is False + + containers = {item["name"]: item for item in spec["containers"]} + kaniko = containers["kaniko"] + # Identical pinned Kaniko across both legs -- no second, unreviewed builder. + assert kaniko["image"] == ( + "gcr.io/kaniko-project/executor@sha256:" + "c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e" + ) + assert kaniko["securityContext"]["capabilities"]["add"] == [ + "CHOWN", + "FOWNER", + "DAC_OVERRIDE", + "SETGID", + "SETUID", + ] + for container in containers.values(): + security = container["securityContext"] + assert security["allowPrivilegeEscalation"] is False + assert security["capabilities"]["drop"] == ["ALL"] + assert security["seccompProfile"]["type"] == "RuntimeDefault" + assert security.get("privileged", False) is False + # Tight caps keep the disposable build from starving the co-hosted validator: + # the amd64 kaniko ceiling is strictly below the arm64 leg's 2 CPU / 4Gi. + limits = kaniko["resources"]["limits"] + assert limits["cpu"] == "1500m" + assert limits["memory"] == "3Gi" + + +def test_amd64_leg_source_is_independently_boundary_checked() -> None: + """The amd64 pod re-derives and re-verifies the reviewed revision itself.""" + source = PIPELINE.read_text(encoding="utf-8") + amd64_stage = source.split("stage('Build amd64 leg without a daemon')", 1)[1] + amd64_stage = amd64_stage.split("stage('Combine multi-arch index')", 1)[0] + # Same fail-closed boundary as the coordinating pod, re-run in the amd64 pod. + assert 'merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}"' in amd64_stage + assert 'checkout --detach "${EXPECTED_SOURCE_REVISION}"' in amd64_stage + assert 'test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"' in amd64_stage + assert 'status --porcelain' in amd64_stage + assert '$(cat build/hermes-agent.destination)-amd64' in amd64_stage + # The amd64 leaf evidence crosses pods only through an explicit stash. + assert "stash" in amd64_stage + assert "hermes-agent-amd64.digest" in amd64_stage + + +def test_combine_stage_publishes_and_reverifies_the_index() -> None: + """Kaniko cannot combine; the reviewed python combiner assembles the index.""" + source = PIPELINE.read_text(encoding="utf-8") + combine = source.split("stage('Combine multi-arch index')", 1)[1] + combine = combine.split("stage('Render reviewed Flux handoff')", 1)[0] + assert "unstash 'hermes-agent-amd64-evidence'" in combine + assert "ci/scripts/hermes_multiarch_combine.py" in combine + assert "--arm64-digest-file build/hermes-agent-arm64.digest" in combine + assert "--amd64-digest-file build/hermes-agent-amd64.digest" in combine + # The combiner emits the arch-less index evidence the existing chain promotes. + assert "--digest-file build/hermes-agent.digest" in combine + assert "--image-file build/hermes-agent.image" in combine + # Downstream render/verify/promote still consume the single index digest file. + render = source.split("stage('Render reviewed Flux handoff')", 1)[1] + assert "--digest-file build/hermes-agent.digest" in render diff --git a/testing/tests/test_hermes_multiarch_combine.py b/testing/tests/test_hermes_multiarch_combine.py new file mode 100644 index 00000000..9887b07e --- /dev/null +++ b/testing/tests/test_hermes_multiarch_combine.py @@ -0,0 +1,385 @@ +"""Safety tests for the fail-closed multi-arch manifest-list combiner.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import io +import json +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "ci/scripts/hermes_multiarch_combine.py" +REVISION = "a" * 40 +BUILD = "17" +DESTINATION = f"registry.bstein.dev/bstein/hermes-agent:git-{REVISION}-build-{BUILD}" +DOCKER_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json" +DOCKER_MANIFEST = "application/vnd.docker.distribution.manifest.v2+json" +IMAGE_CONFIG = "application/vnd.docker.container.image.v1+json" + + +def _load(): + spec = importlib.util.spec_from_file_location("hermes_multiarch_combine", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _sha(body: bytes) -> str: + return "sha256:" + hashlib.sha256(body).hexdigest() + + +class Response(io.BytesIO): + """Minimal context-managed urllib response.""" + + def __init__(self, status: int, body: bytes = b"", headers=None): + super().__init__(body) + self.status = status + self.headers = headers or {} + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + +def _leaf(architecture: str, *, os_name: str = "linux"): + """Build a self-consistent config+manifest whose bytes hash to real digests.""" + config_body = json.dumps( + {"architecture": architecture, "os": os_name}, sort_keys=True + ).encode() + config_digest = _sha(config_body) + manifest_body = json.dumps( + { + "schemaVersion": 2, + "mediaType": DOCKER_MANIFEST, + "config": { + "mediaType": IMAGE_CONFIG, + "digest": config_digest, + "size": len(config_body), + }, + "layers": [], + }, + sort_keys=True, + ).encode() + return { + "arch": architecture, + "config_body": config_body, + "config_digest": config_digest, + "manifest_body": manifest_body, + "digest": _sha(manifest_body), + "content_type": DOCKER_MANIFEST, + } + + +def _descriptor(module, leaf): + return { + "mediaType": leaf["content_type"], + "size": len(leaf["manifest_body"]), + "digest": leaf["digest"], + "platform": {"architecture": leaf["arch"], "os": "linux"}, + } + + +def _index_digest(module, amd64, arm64) -> str: + body = module._manifest_list_bytes( + [_descriptor(module, amd64), _descriptor(module, arm64)] + ) + return _sha(body) + + +class Registry: + """Route the combiner's deterministic request sequence by method and path.""" + + def __init__(self, module, amd64, arm64, *, head_status=404, existing="") -> None: + self.module = module + self.leaves = {amd64["arch"]: amd64, arm64["arch"]: arm64} + self.index_digest = _index_digest(module, amd64, arm64) + self.head_status = head_status + self.existing = existing + self.put_body = None + self.calls = [] + + def __call__(self, request, timeout): + method = request.method + url = request.full_url + self.calls.append((method, url)) + for arch, leaf in self.leaves.items(): + if url.endswith(f"-{arch}"): + return Response( + 200, + leaf["manifest_body"], + { + "Docker-Content-Digest": leaf["digest"], + "Content-Type": leaf["content_type"], + }, + ) + # Blob URLs percent-encode the ``sha256:`` colon; match the raw hex. + if url.endswith(leaf["config_digest"].split(":", 1)[1]): + return Response(200, leaf["config_body"], {}) + # Final index tag (no arch suffix, ends with the build tag). + if method == "HEAD": + return Response( + self.head_status, + headers={"Docker-Content-Digest": self.existing}, + ) + if method == "PUT": + self.put_body = request.data + return Response(201, headers={"Docker-Content-Digest": self.index_digest}) + return Response( + 200, + self.module._manifest_list_bytes( + [ + _descriptor(self.module, self.leaves["amd64"]), + _descriptor(self.module, self.leaves["arm64"]), + ] + ), + { + "Docker-Content-Digest": self.index_digest, + "Content-Type": DOCKER_MANIFEST_LIST, + }, + ) + + +def _combine(module, registry, arch_digests=None): + if arch_digests is None: + arch_digests = { + "amd64": registry.leaves["amd64"]["digest"], + "arm64": registry.leaves["arm64"]["digest"], + } + return module.combine_multiarch_index( + destination=DESTINATION, + arch_digests=arch_digests, + username="robot", + password="private", + opener=registry, + ) + + +def test_combines_two_verified_leaves_into_one_index() -> None: + """Both native leaves are re-read, arch-proven, and published as a list.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + registry = Registry(module, amd64, arm64) + result = _combine(module, registry) + assert result["result"] == "published" + assert result["index_digest"] == registry.index_digest + assert result["amd64_digest"] == amd64["digest"] + assert result["arm64_digest"] == arm64["digest"] + # The published bytes are exactly what we hashed for the index digest. + assert _sha(registry.put_body) == registry.index_digest + assert "private" not in json.dumps(result) + + +def test_idempotent_when_index_already_matches() -> None: + """A replay is accepted only when the existing index digest is identical.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + index_digest = _index_digest(module, amd64, arm64) + registry = Registry( + module, amd64, arm64, head_status=200, existing=index_digest + ) + result = _combine(module, registry) + assert result["result"] == "already-present" + assert registry.put_body is None + + +def test_existing_index_with_other_digest_fails_closed() -> None: + """An occupied final tag with a different index digest never republishes.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + registry = Registry( + module, amd64, arm64, head_status=200, existing="sha256:" + "c" * 64 + ) + with pytest.raises(RuntimeError, match="another index digest"): + _combine(module, registry) + + +def test_rejects_leaf_digest_that_disagrees_with_evidence() -> None: + """A leaf whose registry digest is not the Kaniko evidence fails closed.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + registry = Registry(module, amd64, arm64) + with pytest.raises(RuntimeError, match="amd64 leaf digest"): + _combine( + module, + registry, + arch_digests={"amd64": "sha256:" + "d" * 64, "arm64": arm64["digest"]}, + ) + + +def test_rejects_leaf_whose_config_architecture_is_wrong() -> None: + """A leaf that claims the wrong architecture in its config fails closed.""" + module = _load() + # Build an "amd64" candidate tag whose config actually says arm64. + swapped = _leaf("arm64") + swapped["arch"] = "amd64" # served under the amd64 tag, but arm64 inside + arm64 = _leaf("arm64") + registry = Registry(module, swapped, arm64) + with pytest.raises(RuntimeError, match="architecture"): + _combine( + module, + registry, + arch_digests={"amd64": swapped["digest"], "arm64": arm64["digest"]}, + ) + + +def test_rejects_leaf_served_as_a_manifest_list() -> None: + """A per-arch leaf must be a single image, never itself an index/list.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + amd64["content_type"] = DOCKER_MANIFEST_LIST + registry = Registry(module, amd64, arm64) + with pytest.raises(RuntimeError, match="single-image manifest"): + _combine(module, registry) + + +def test_requires_both_architectures() -> None: + """The combiner refuses any arch set other than exactly arm64 and amd64.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + registry = Registry(module, amd64, arm64) + with pytest.raises(ValueError, match="arm64 and amd64"): + module.combine_multiarch_index( + destination=DESTINATION, + arch_digests={"amd64": amd64["digest"]}, + username="robot", + password="private", + opener=registry, + ) + + +def test_rejects_destination_with_arch_suffix() -> None: + """Only the arch-less final tag is a valid combine destination.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + with pytest.raises(ValueError, match="invalid multi-arch destination"): + module.combine_multiarch_index( + destination=f"{DESTINATION}-arm64", + arch_digests={"amd64": amd64["digest"], "arm64": arm64["digest"]}, + username="robot", + password="private", + opener=Registry(module, amd64, arm64), + ) + + +def test_cli_binds_evidence_files_and_writes_index_outputs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """The command reads both Kaniko pairs and emits arch-less index evidence.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + registry = Registry(module, amd64, arm64) + files = {} + for arch, leaf in (("amd64", amd64), ("arm64", arm64)): + digest_file = tmp_path / f"{arch}.digest" + image_file = tmp_path / f"{arch}.image" + digest_file.write_text(leaf["digest"] + "\n", encoding="utf-8") + image_file.write_text( + f"{DESTINATION}-{arch}@{leaf['digest']}\n", encoding="utf-8" + ) + files[arch] = (digest_file, image_file) + out_digest = tmp_path / "index.digest" + out_image = tmp_path / "index.image" + monkeypatch.setenv("HARBOR_USER", "robot") + monkeypatch.setenv("HARBOR_PASSWORD", "private") + captured = {} + + def fake_combine(*, destination, arch_digests, username, password): + captured["destination"] = destination + captured["arch_digests"] = dict(arch_digests) + return {"index_digest": registry.index_digest, "result": "published"} + + monkeypatch.setattr(module, "combine_multiarch_index", fake_combine) + monkeypatch.setattr( + sys, + "argv", + [ + "hermes_multiarch_combine.py", + "--destination", + DESTINATION, + "--source-revision", + REVISION, + "--build-number", + BUILD, + "--arm64-digest-file", + str(files["arm64"][0]), + "--arm64-image-file", + str(files["arm64"][1]), + "--amd64-digest-file", + str(files["amd64"][0]), + "--amd64-image-file", + str(files["amd64"][1]), + "--digest-file", + str(out_digest), + "--image-file", + str(out_image), + ], + ) + assert module.main() == 0 + assert out_digest.read_text(encoding="utf-8").strip() == registry.index_digest + assert ( + out_image.read_text(encoding="utf-8").strip() + == f"{DESTINATION}@{registry.index_digest}" + ) + assert json.loads(capsys.readouterr().out)["result"] == "published" + # main() must bind each arch's two evidence files to the correct leaf digest. + assert captured["destination"] == DESTINATION + assert captured["arch_digests"] == { + "amd64": amd64["digest"], + "arm64": arm64["digest"], + } + + +def test_cli_rejects_mismatched_per_arch_evidence_pair( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A digest/image evidence file mismatch is reported as a JSON error.""" + module = _load() + amd64, arm64 = _leaf("amd64"), _leaf("arm64") + files = {} + for arch, leaf in (("amd64", amd64), ("arm64", arm64)): + digest_file = tmp_path / f"{arch}.digest" + image_file = tmp_path / f"{arch}.image" + digest_file.write_text(leaf["digest"] + "\n", encoding="utf-8") + # amd64 image file points at the wrong digest. + recorded = "sha256:" + "e" * 64 if arch == "amd64" else leaf["digest"] + image_file.write_text( + f"{DESTINATION}-{arch}@{recorded}\n", encoding="utf-8" + ) + files[arch] = (digest_file, image_file) + monkeypatch.setenv("HARBOR_USER", "robot") + monkeypatch.setenv("HARBOR_PASSWORD", "private") + monkeypatch.setattr( + sys, + "argv", + [ + "hermes_multiarch_combine.py", + "--destination", + DESTINATION, + "--source-revision", + REVISION, + "--build-number", + BUILD, + "--arm64-digest-file", + str(files["arm64"][0]), + "--arm64-image-file", + str(files["arm64"][1]), + "--amd64-digest-file", + str(files["amd64"][0]), + "--amd64-image-file", + str(files["amd64"][1]), + "--digest-file", + str(tmp_path / "index.digest"), + "--image-file", + str(tmp_path / "index.image"), + ], + ) + assert module.main() == 1 + assert "does not match" in json.loads(capsys.readouterr().out)["error"]