ci/Jenkinsfile.hermes-agent-image

@@ -188,7 +188,6 @@ spec:
               testing/tests/test_hermes_image_builder_coverage.py \
               testing/tests/test_hermes_image_builder_fresh_review.py \
               testing/tests/test_hermes_oci_promote.py \
-              testing/tests/test_hermes_multiarch_combine.py \
               testing/tests/test_hermes_image_automation.py
           '''
         }
@@ -214,7 +213,7 @@ spec:
         }
       }
     }
-    stage('Build arm64 leg without a daemon') {
+    stage('Build and publish without a daemon') {
       steps {
         container('kaniko') {
           withCredentials([usernamePassword(
@@ -226,7 +225,7 @@ spec:
               set -eu
               set +x
               config_path=/kaniko/.docker/config.json
-              destination="$(cat build/hermes-agent.destination)-arm64"
+              destination="$(cat build/hermes-agent.destination)"
               source_revision="$(cat build/hermes-agent.source-revision)"
               umask 077
               auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
@@ -241,192 +240,18 @@ spec:
                 --context="dir://${WORKSPACE}" \
                 --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
                 --destination="${destination}" \
-                --digest-file="${WORKSPACE}/build/hermes-agent-arm64.digest" \
+                --digest-file="${WORKSPACE}/build/hermes-agent.digest" \
-                --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-arm64.image" \
+                --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \
                 --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
                 --label="org.opencontainers.image.revision=${source_revision}" \
                 --cleanup \
                 --push-retry=3
-              /busybox/chmod 644 build/hermes-agent-arm64.digest build/hermes-agent-arm64.image
+              /busybox/chmod 644 build/hermes-agent.digest build/hermes-agent.image
             '''
           }
         }
       }
     }
-    stage('Build amd64 leg without a daemon') {
-      agent {
-        kubernetes {
-          yaml """
-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    atlas.bstein.dev/workload: hermes-agent-image-builder-amd64
-spec:
-  serviceAccountName: hermes-image-builder
-  automountServiceAccountToken: false
-  enableServiceLinks: false
-  restartPolicy: Never
-  securityContext:
-    fsGroup: 1000
-    fsGroupChangePolicy: OnRootMismatch
-  nodeSelector:
-    kubernetes.io/arch: amd64
-    kubernetes.io/hostname: titan-24
-    node-role.kubernetes.io/worker: "true"
-  tolerations:
-    # titan-24 co-hosts the out-of-cluster Sui validator; tolerate whatever
-    # PreferNoSchedule/NoSchedule guard taint the node carries so the pinned
-    # build lands, and rely on the tight resource caps below (not scheduling
-    # priority) to keep the disposable build from starving the validator.
-    - operator: Exists
-  imagePullSecrets:
-    - name: harbor-bstein-robot
-  containers:
-    - name: jnlp
-      image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
-      securityContext:
-        allowPrivilegeEscalation: false
-        capabilities:
-          drop: ["ALL"]
-        runAsNonRoot: true
-        runAsUser: 1000
-        seccompProfile:
-          type: RuntimeDefault
-      resources:
-        requests:
-          cpu: 25m
-          memory: 128Mi
-        limits:
-          cpu: 250m
-          memory: 384Mi
-    - name: kaniko
-      image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
-      command: ["/busybox/sh", "-c"]
-      args: ["/busybox/sleep 99d"]
-      tty: true
-      securityContext:
-        allowPrivilegeEscalation: false
-        capabilities:
-          drop: ["ALL"]
-          add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
-        privileged: false
-        runAsUser: 0
-        seccompProfile:
-          type: RuntimeDefault
-      resources:
-        requests:
-          cpu: 100m
-          memory: 512Mi
-        limits:
-          cpu: "1500m"
-          memory: 3Gi
-"""
-        }
-      }
-      steps {
-        container('jnlp') {
-          sh '''
-            set -eu
-            mkdir -p build
-            test "${PUBLISH_IMAGE}" = "true"
-            test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
-            case "${EXPECTED_SOURCE_REVISION}" in
-              *[!0-9a-f]*|'')
-                echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
-                exit 2
-                ;;
-            esac
-            test "${#EXPECTED_SOURCE_REVISION}" -eq 40
-            main_revision="$(git rev-parse origin/main)"
-            git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
-            git checkout --detach "${EXPECTED_SOURCE_REVISION}"
-            actual_revision="$(git rev-parse HEAD)"
-            test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
-            test -z "$(git status --porcelain)"
-            test -f dockerfiles/Dockerfile.hermes-agent
-            case "${BUILD_NUMBER}" in
-              ''|0*|*[!0-9]*)
-                echo "BUILD_NUMBER must be a positive decimal integer" >&2
-                exit 2
-                ;;
-            esac
-            printf '%s\n' \
-              "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
-              > build/hermes-agent.destination
-            printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision
-          '''
-        }
-        container('kaniko') {
-          withCredentials([usernamePassword(
-            credentialsId: 'harbor-robot',
-            usernameVariable: 'HARBOR_USER',
-            passwordVariable: 'HARBOR_PASSWORD'
-          )]) {
-            sh '''#!/busybox/sh
-              set -eu
-              set +x
-              config_path=/kaniko/.docker/config.json
-              destination="$(cat build/hermes-agent.destination)-amd64"
-              source_revision="$(cat build/hermes-agent.source-revision)"
-              umask 077
-              auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
-              /busybox/mkdir -p /kaniko/.docker
-              /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
-              unset HARBOR_USER HARBOR_PASSWORD auth
-              trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
-              umask 022
-              /kaniko/executor \
-                --registry-mirror=harbor-core.harbor.svc.cluster.local \
-                --insecure-registry=harbor-core.harbor.svc.cluster.local \
-                --context="dir://${WORKSPACE}" \
-                --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
-                --destination="${destination}" \
-                --digest-file="${WORKSPACE}/build/hermes-agent-amd64.digest" \
-                --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-amd64.image" \
-                --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
-                --label="org.opencontainers.image.revision=${source_revision}" \
-                --cleanup \
-                --push-retry=3
-              /busybox/chmod 644 build/hermes-agent-amd64.digest build/hermes-agent-amd64.image
-            '''
-          }
-        }
-        stash(
-          name: 'hermes-agent-amd64-evidence',
-          includes: 'build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image'
-        )
-      }
-    }
-    stage('Combine multi-arch index') {
-      steps {
-        unstash 'hermes-agent-amd64-evidence'
-        withCredentials([usernamePassword(
-          credentialsId: 'harbor-robot',
-          usernameVariable: 'HARBOR_USER',
-          passwordVariable: 'HARBOR_PASSWORD'
-        )]) {
-          sh '''
-            set -eu
-            set +x
-            destination="$(cat build/hermes-agent.destination)"
-            source_revision="$(cat build/hermes-agent.source-revision)"
-            python3 ci/scripts/hermes_multiarch_combine.py \
-              --destination "${destination}" \
-              --source-revision "${source_revision}" \
-              --build-number "${BUILD_NUMBER}" \
-              --arm64-digest-file build/hermes-agent-arm64.digest \
-              --arm64-image-file build/hermes-agent-arm64.image \
-              --amd64-digest-file build/hermes-agent-amd64.digest \
-              --amd64-image-file build/hermes-agent-amd64.image \
-              --digest-file build/hermes-agent.digest \
-              --image-file build/hermes-agent.image
-            test -s build/hermes-agent.digest
-            test -s build/hermes-agent.image
-          '''
-        }
-      }
-    }
     stage('Render reviewed Flux handoff') {
       steps {
         withCredentials([usernamePassword(
@@ -462,10 +287,6 @@ spec:
           build/hermes-agent.digest \
           build/hermes-agent.image \
           build/hermes-agent.source-revision \
-          build/hermes-agent-arm64.digest \
-          build/hermes-agent-arm64.image \
-          build/hermes-agent-amd64.digest \
-          build/hermes-agent-amd64.image \
           build/hermes-agent-release/hermes-agent-image.json \
           build/hermes-agent-release/hermes-image-update.patch \
           build/hermes-agent-release/hermes-kustomization.yaml \
@@ -484,7 +305,7 @@ spec:
           --output-dir build/hermes-agent-release
       '''
       archiveArtifacts(
-        artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.source-revision,build/hermes-agent-arm64.digest,build/hermes-agent-arm64.image,build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
+        artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.source-revision,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
         allowEmptyArchive: false,
         fingerprint: true
       )

ci/scripts/hermes_multiarch_combine.py

@@ -1,390 +0,0 @@
-#!/usr/bin/env python3
-"""Assemble a fail-closed multi-arch manifest list from two per-arch leaves.
-
-Kaniko builds one native image per architecture (arm64 on an rpi5 pod, amd64 on
-titan-24) and pushes each under an arch-suffixed candidate tag
-``...-build-<N>-<arch>``.  Kaniko cannot combine, so this step:
-
-1. Independently re-reads each per-arch candidate manifest from the registry and
-   binds it to the exact Kaniko digest evidence.
-2. Proves each leaf really is the architecture it claims by reading its image
-   config (a swapped or cross-built leaf fails closed here).
-3. Builds a Docker manifest *list* (not an OCI index) from the two verified
-   leaves -- Docker manifest lists are already inside the promotion allow-list,
-   so this keeps the security surface of ``hermes_oci_promote.py`` unchanged.
-4. Refuses to overwrite an existing final tag, PUTs the list to the final
-   ``...-build-<N>`` tag, and re-reads it to confirm the registry resolved the
-   exact index digest referencing exactly the two expected leaves.
-
-The output ``--digest-file``/``--image-file`` deliberately use the SAME format
-the single-arch Kaniko step produced (``<digest>`` and ``<destination>@<digest>``
-for the final, arch-less tag).  The whole downstream evidence chain --
-``hermes_image_release.py`` render/verify-evidence and ``hermes_oci_promote.py``
--- therefore promotes the multi-arch INDEX with no further change.
-"""
-
-from __future__ import annotations
-
-import argparse
-import base64
-import hashlib
-import json
-import os
-import re
-import urllib.error
-import urllib.parse
-import urllib.request
-from pathlib import Path
-from typing import Any, Callable
-
-
-REGISTRY_ORIGIN = "https://registry.bstein.dev"
-# The final (arch-less) Flux-visible tag; identical contract to the promoter.
-DESTINATION_PATTERN = re.compile(
-    r"^registry\.bstein\.dev/bstein/hermes-agent:"
-    r"git-(?P<revision>[0-9a-f]{40})-build-(?P<build>[1-9][0-9]*)$"
-)
-DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
-DOCKER_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json"
-# A per-arch leaf must be a single-image manifest, never itself a list/index.
-LEAF_MANIFEST_TYPES = {
-    "application/vnd.docker.distribution.manifest.v2+json",
-    "application/vnd.oci.image.manifest.v1+json",
-}
-IMAGE_CONFIG_TYPES = {
-    "application/vnd.docker.container.image.v1+json",
-    "application/vnd.oci.image.config.v1+json",
-}
-# Deterministic architecture order -> deterministic manifest-list bytes/digest.
-ARCHITECTURES = ("amd64", "arm64")
-MAX_MANIFEST_BYTES = 4 * 1024 * 1024
-MAX_CONFIG_BYTES = 1024 * 1024
-
-
-class _NoRedirect(urllib.request.HTTPRedirectHandler):
-    """Never forward registry credentials to another origin."""
-
-    def redirect_request(self, _request, _file, _code, _message, _headers, _url):
-        return None
-
-
-def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
-    """Return normal and HTTP error responses without following redirects."""
-    opener = urllib.request.build_opener(_NoRedirect())
-    try:
-        return opener.open(request, timeout=timeout)
-    except urllib.error.HTTPError as exc:
-        return exc
-
-
-def _status(response: Any) -> int:
-    """Normalize urllib response and HTTPError status fields."""
-    return int(getattr(response, "status", getattr(response, "code", 0)))
-
-
-def _authorization(username: str, password: str) -> str:
-    """Build a Basic authorization value without placing it in a URL."""
-    if not username or not password:
-        raise RuntimeError("Harbor credentials are unavailable")
-    encoded = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
-    return f"Basic {encoded}"
-
-
-def _manifest_url(component: str, reference: str) -> str:
-    """Return one same-origin, path-escaped Docker Registry manifest URL."""
-    encoded = urllib.parse.quote(reference, safe="")
-    return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/manifests/{encoded}"
-
-
-def _blob_url(component: str, digest: str) -> str:
-    """Return one same-origin, path-escaped Docker Registry blob URL."""
-    encoded = urllib.parse.quote(digest, safe="")
-    return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/blobs/{encoded}"
-
-
-def _read_evidence_pair(
-    *, digest_text: str, image_text: str, per_arch_tag_ref: str
-) -> str:
-    """Cross-check both Kaniko output files for one arch against its tag."""
-    digest_lines = digest_text.splitlines()
-    image_lines = image_text.splitlines()
-    if len(digest_lines) != 1:
-        raise ValueError("Kaniko digest evidence must contain exactly one line")
-    if len(image_lines) != 1:
-        raise ValueError("Kaniko image evidence must contain exactly one line")
-    digest = digest_lines[0].strip()
-    if not DIGEST_PATTERN.fullmatch(digest):
-        raise ValueError("invalid per-arch image digest")
-    if image_lines[0].strip() != f"{per_arch_tag_ref}@{digest}":
-        raise ValueError("per-arch image evidence does not match tag and digest")
-    return digest
-
-
-def _verified_leaf(
-    *,
-    component: str,
-    per_arch_tag: str,
-    architecture: str,
-    expected_digest: str,
-    authorization: str,
-    opener: Callable[[urllib.request.Request, int], Any],
-) -> dict[str, Any]:
-    """Re-read one per-arch leaf and prove its digest, type, and architecture."""
-    accept = ", ".join(sorted(LEAF_MANIFEST_TYPES))
-    request = urllib.request.Request(
-        _manifest_url(component, per_arch_tag),
-        headers={"Accept": accept, "Authorization": authorization},
-        method="GET",
-    )
-    with opener(request, 30) as response:
-        if _status(response) != 200:
-            raise RuntimeError(
-                f"{architecture} leaf manifest returned HTTP {_status(response)}"
-            )
-        body = response.read(MAX_MANIFEST_BYTES + 1)
-        if len(body) > MAX_MANIFEST_BYTES:
-            raise RuntimeError(f"{architecture} leaf manifest exceeded the size limit")
-        observed_digest = response.headers.get("Docker-Content-Digest", "")
-        content_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip()
-    if observed_digest != expected_digest:
-        raise RuntimeError(f"{architecture} leaf digest does not match build evidence")
-    # Defence in depth: the digest header is registry-asserted; recompute it too.
-    if f"sha256:{hashlib.sha256(body).hexdigest()}" != expected_digest:
-        raise RuntimeError(f"{architecture} leaf bytes do not hash to its digest")
-    if content_type not in LEAF_MANIFEST_TYPES:
-        raise RuntimeError(f"{architecture} leaf is not a single-image manifest")
-    try:
-        manifest = json.loads(body.decode("utf-8"))
-    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
-        raise RuntimeError(f"{architecture} leaf manifest is not valid JSON") from exc
-    config = manifest.get("config")
-    if not isinstance(config, dict):
-        raise RuntimeError(f"{architecture} leaf manifest omits its config descriptor")
-    config_digest = str(config.get("digest") or "")
-    config_type = str(config.get("mediaType") or "")
-    if not DIGEST_PATTERN.fullmatch(config_digest):
-        raise RuntimeError(f"{architecture} leaf config digest is invalid")
-    if config_type not in IMAGE_CONFIG_TYPES:
-        raise RuntimeError(f"{architecture} leaf config media type is unsupported")
-    config_request = urllib.request.Request(
-        _blob_url(component, config_digest),
-        headers={"Accept": config_type, "Authorization": authorization},
-        method="GET",
-    )
-    with opener(config_request, 30) as response:
-        if _status(response) != 200:
-            raise RuntimeError(
-                f"{architecture} leaf config returned HTTP {_status(response)}"
-            )
-        config_body = response.read(MAX_CONFIG_BYTES + 1)
-        if len(config_body) > MAX_CONFIG_BYTES:
-            raise RuntimeError(f"{architecture} leaf config exceeded the size limit")
-    if f"sha256:{hashlib.sha256(config_body).hexdigest()}" != config_digest:
-        raise RuntimeError(f"{architecture} leaf config bytes do not hash to its digest")
-    try:
-        config_json = json.loads(config_body.decode("utf-8"))
-    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
-        raise RuntimeError(f"{architecture} leaf config is not valid JSON") from exc
-    if config_json.get("architecture") != architecture:
-        raise RuntimeError(
-            f"{architecture} leaf config reports architecture "
-            f"{config_json.get('architecture')!r}"
-        )
-    if config_json.get("os") != "linux":
-        raise RuntimeError(f"{architecture} leaf config reports a non-linux os")
-    return {
-        "mediaType": content_type,
-        "size": len(body),
-        "digest": expected_digest,
-        "platform": {"architecture": architecture, "os": "linux"},
-    }
-
-
-def _manifest_list_bytes(descriptors: list[dict[str, Any]]) -> bytes:
-    """Serialize the manifest list deterministically for a stable index digest."""
-    document = {
-        "schemaVersion": 2,
-        "mediaType": DOCKER_MANIFEST_LIST,
-        "manifests": descriptors,
-    }
-    return json.dumps(document, sort_keys=True, separators=(",", ":")).encode("utf-8")
-
-
-def combine_multiarch_index(
-    *,
-    destination: str,
-    arch_digests: dict[str, str],
-    username: str,
-    password: str,
-    opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
-) -> dict[str, str]:
-    """Verify both leaves, publish, and re-verify one multi-arch index tag."""
-    match = DESTINATION_PATTERN.fullmatch(destination.strip())
-    if not match:
-        raise ValueError("invalid multi-arch destination")
-    if set(arch_digests) != set(ARCHITECTURES):
-        raise ValueError("expected exactly the arm64 and amd64 per-arch digests")
-    component = "hermes-agent"
-    index_tag = destination.rsplit(":", 1)[1]
-    authorization = _authorization(username, password)
-
-    descriptors = [
-        _verified_leaf(
-            component=component,
-            per_arch_tag=f"{index_tag}-{architecture}",
-            architecture=architecture,
-            expected_digest=arch_digests[architecture],
-            authorization=authorization,
-            opener=opener,
-        )
-        for architecture in ARCHITECTURES
-    ]
-    manifest_list = _manifest_list_bytes(descriptors)
-    if len(manifest_list) > MAX_MANIFEST_BYTES:
-        raise RuntimeError("assembled manifest list exceeded the size limit")
-    index_digest = f"sha256:{hashlib.sha256(manifest_list).hexdigest()}"
-
-    index_url = _manifest_url(component, index_tag)
-    head_request = urllib.request.Request(
-        index_url,
-        headers={"Accept": DOCKER_MANIFEST_LIST, "Authorization": authorization},
-        method="HEAD",
-    )
-    with opener(head_request, 20) as response:
-        head_status = _status(response)
-        existing_digest = response.headers.get("Docker-Content-Digest", "")
-    if head_status == 200:
-        if existing_digest != index_digest:
-            raise RuntimeError("final tag already exists with another index digest")
-        result = "already-present"
-    elif head_status == 404:
-        put_request = urllib.request.Request(
-            index_url,
-            data=manifest_list,
-            headers={
-                "Authorization": authorization,
-                "Content-Type": DOCKER_MANIFEST_LIST,
-            },
-            method="PUT",
-        )
-        with opener(put_request, 30) as response:
-            put_status = _status(response)
-            put_digest = response.headers.get("Docker-Content-Digest", "")
-        if put_status not in {201, 202}:
-            raise RuntimeError(f"index manifest returned HTTP {put_status}")
-        if put_digest and put_digest != index_digest:
-            raise RuntimeError("index manifest digest changed during publish")
-        result = "published"
-    else:
-        raise RuntimeError(f"final tag preflight returned HTTP {head_status}")
-
-    verify_request = urllib.request.Request(
-        index_url,
-        headers={"Accept": DOCKER_MANIFEST_LIST, "Authorization": authorization},
-        method="GET",
-    )
-    with opener(verify_request, 30) as response:
-        if _status(response) != 200:
-            raise RuntimeError(f"index verification returned HTTP {_status(response)}")
-        verify_body = response.read(MAX_MANIFEST_BYTES + 1)
-        if len(verify_body) > MAX_MANIFEST_BYTES:
-            raise RuntimeError("index verification exceeded the size limit")
-        verify_digest = response.headers.get("Docker-Content-Digest", "")
-        verify_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip()
-    if verify_digest != index_digest:
-        raise RuntimeError("registry resolved the final tag to another index digest")
-    if verify_type != DOCKER_MANIFEST_LIST:
-        raise RuntimeError("registry did not store a Docker manifest list")
-    try:
-        published = json.loads(verify_body.decode("utf-8"))
-    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
-        raise RuntimeError("registry returned invalid index JSON") from exc
-    published_leaves = {
-        (
-            str((item or {}).get("platform", {}).get("architecture")),
-            str((item or {}).get("digest")),
-        )
-        for item in published.get("manifests") or []
-    }
-    expected_leaves = {
-        (architecture, arch_digests[architecture]) for architecture in ARCHITECTURES
-    }
-    if published_leaves != expected_leaves:
-        raise RuntimeError("published index does not reference the exact two leaves")
-
-    return {
-        "component": component,
-        "index_digest": index_digest,
-        "index_tag": index_tag,
-        "result": result,
-        **{f"{architecture}_digest": arch_digests[architecture] for architecture in ARCHITECTURES},
-    }
-
-
-def _load_arch_digest(
-    *, destination: str, architecture: str, digest_file: Path, image_file: Path
-) -> str:
-    """Bind one arch's two Kaniko evidence files to its arch-suffixed tag."""
-    per_arch_tag_ref = f"{destination}-{architecture}"
-    return _read_evidence_pair(
-        digest_text=digest_file.read_text(encoding="utf-8"),
-        image_text=image_file.read_text(encoding="utf-8"),
-        per_arch_tag_ref=per_arch_tag_ref,
-    )
-
-
-def main() -> int:
-    """Combine two verified per-arch leaves and emit index digest evidence."""
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--destination", required=True)
-    parser.add_argument("--source-revision", required=True)
-    parser.add_argument("--build-number", required=True)
-    parser.add_argument("--arm64-digest-file", required=True, type=Path)
-    parser.add_argument("--arm64-image-file", required=True, type=Path)
-    parser.add_argument("--amd64-digest-file", required=True, type=Path)
-    parser.add_argument("--amd64-image-file", required=True, type=Path)
-    parser.add_argument("--digest-file", required=True, type=Path)
-    parser.add_argument("--image-file", required=True, type=Path)
-    args = parser.parse_args()
-    try:
-        match = DESTINATION_PATTERN.fullmatch(args.destination.strip())
-        if not match:
-            raise ValueError("invalid multi-arch destination")
-        if match.group("revision") != args.source_revision.strip():
-            raise ValueError("destination revision does not match evidence")
-        if match.group("build") != args.build_number.strip():
-            raise ValueError("destination build number does not match evidence")
-        destination = args.destination.strip()
-        arch_digests = {
-            "arm64": _load_arch_digest(
-                destination=destination,
-                architecture="arm64",
-                digest_file=args.arm64_digest_file,
-                image_file=args.arm64_image_file,
-            ),
-            "amd64": _load_arch_digest(
-                destination=destination,
-                architecture="amd64",
-                digest_file=args.amd64_digest_file,
-                image_file=args.amd64_image_file,
-            ),
-        }
-        result = combine_multiarch_index(
-            destination=destination,
-            arch_digests=arch_digests,
-            username=os.environ.get("HARBOR_USER", ""),
-            password=os.environ.get("HARBOR_PASSWORD", ""),
-        )
-        args.digest_file.write_text(result["index_digest"] + "\n", encoding="utf-8")
-        args.image_file.write_text(
-            f"{destination}@{result['index_digest']}\n", encoding="utf-8"
-        )
-    except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc:
-        print(json.dumps({"error": str(exc)}, sort_keys=True))
-        return 1
-    print(json.dumps(result, indent=2, sort_keys=True))
-    return 0
-
-
-if __name__ == "__main__":  # pragma: no cover - exercised through main()
-    raise SystemExit(main())

dockerfiles/Dockerfile.hermes-agent

@@ -1,16 +1,6 @@
 # syntax=docker/dockerfile:1
 # dockerfiles/Dockerfile.hermes-agent
-#
+FROM nousresearch/hermes-agent@sha256:47d4bd4cc420b70e40ed75efdade373e45b86b7382d4013a054208982bb6ba08
-# Multi-arch base: this digest is the upstream OCI image INDEX for tag
-# v2026.7.7.2 (revision 9de9c25f620ff7f1ce0fd5457d596052d5159596). The index
-# fans out to both native leaves of the same reviewed upstream version:
-#   linux/arm64 -> sha256:47d4bd4cc420b70e40ed75efdade373e45b86b7382d4013a054208982bb6ba08
-#   linux/amd64 -> sha256:3db34ce19adfa080736a2a3feb0316dbcccc588faa9afe7fd8ae1c03b4f1a53a
-# The arm64 leaf is byte-for-byte the previously pinned single-arch base, so the
-# arm64 build is unchanged; Kaniko/containerd auto-selects the matching leaf per
-# build platform (arm64 rpi5 pod vs amd64 titan-24 pod). Do NOT replace this with
-# a per-arch leaf digest -- that would break the amd64 build leg.
-FROM nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973
 
 USER root
 

docs/hermes_agent_multiarch.md

@@ -1,156 +0,0 @@
-# hermes-agent multi-arch rollout runbook
-
-Goal: build the `hermes-agent` container image for **both** `linux/arm64` and
-`linux/amd64` so the `hermes-agent` workload can eventually run on the amd64
-node `titan-22`, while the arm64 pi fleet remains a valid fallback.
-
-This branch (`feature/hermes-agent-multiarch`) contains everything needed. The
-steps below are the **operator** procedure, in order. Do not reorder: the
-titan-22 runtime flip is deliberately the last step and only after a validated
-multi-arch image exists.
-
----
-
-## What changed on the branch (for reviewers)
-
-| Area | File | Change |
-| --- | --- | --- |
-| Base image | `dockerfiles/Dockerfile.hermes-agent` | `FROM` repointed from the arm64-only leaf `sha256:47d4bd4c…` to the upstream **multi-arch OCI index** `sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973` (tag `v2026.7.7.2`, revision `9de9c25f…`). Same reviewed upstream version; arm64 leaf is byte-identical to the old pin. |
-| Pipeline | `ci/Jenkinsfile.hermes-agent-image` | arm64 kaniko leg now pushes `…-build-<N>-arm64`; new amd64 kaniko leg on a titan-24-pinned, resource-capped pod pushes `…-build-<N>-amd64`; new `Combine multi-arch index` stage; evidence archive expanded. |
-| Combiner | `ci/scripts/hermes_multiarch_combine.py` | New fail-closed pure-python step that verifies both leaves and publishes/re-verifies a Docker manifest **list** at the arch-less `…-build-<N>` tag, emitting the index digest in the existing digest-file/image-file format. |
-| Promotion | `ci/scripts/hermes_oci_promote.py` | **Unchanged.** Docker manifest lists are already inside its `MANIFEST_TYPES` allow-list, so it promotes the multi-arch INDEX to `…-release` with no edit. |
-| Release scripts | `ci/scripts/hermes_image_release.py` | **Unchanged.** The combiner hands it the arch-less tag + index digest, exactly the shape it already validates/renders. |
-| Worker join | `infrastructure/core/node-prefer-noschedule-cronjob.yaml` | `clear_worker titan-24 amd64` + a soft `PreferNoSchedule` guard taint protecting the co-hosted Sui validator. |
-| Runtime flip (LAST) | `services/hermes/agent-deployment.yaml` | Adds titan-22 to the allowed set with a soft preference + media-taint toleration, keeping the arm64 fleet as fallback. **Prepared as a separate commit — apply only after validation.** |
-
-### Correction to the original assumption
-
-Worker/arch/taint labels in this cluster are **not** managed by Ansible. They are
-reconciled every minute by the Flux-managed `node-prefer-noschedule` CronJob
-(`kubectl label`/`kubectl taint`). `hosts/` Ansible is a thin skeleton and
-titan-24 is already present there. Therefore the "worker-join" change lives in
-that CronJob, and it is applied by **Flux at merge**, not by a hardware playbook
-run. The only genuine hardware/operator action is ensuring titan-24 is
-uncordoned (it is on the Ananke recovery uncordon denylist).
-
----
-
-## Step (a) — Prepare titan-24 as a schedulable amd64 worker
-
-The worker label + guard taint ship in this branch's CronJob change and land
-automatically once Flux reconciles after merge (step b). Before the first amd64
-build, the operator must make sure titan-24 can actually accept the pinned pod:
-
-```sh
-# On a control-plane host / with cluster admin kubeconfig:
-kubectl get node titan-24 -o wide            # confirm Ready, amd64
-kubectl uncordon titan-24                     # recovery keeps it cordoned by default
-# The Sui validator runs as docker-compose at /opt/sui-validator (outside k8s);
-# it is protected by the build pod's tight resource caps, not by k8s priority.
-```
-
-After merge (step b), confirm the CronJob applied the labels/taint:
-
-```sh
-kubectl get node titan-24 --show-labels | tr ',' '\n' | grep -E 'worker|hardware'
-# expect: node-role.kubernetes.io/worker=true and hardware=amd64
-kubectl describe node titan-24 | grep -i taint
-# expect: atlas.bstein.dev/sui-validator=true:PreferNoSchedule
-```
-
-If titan-24 must NOT be labeled yet, hold the merge; the labels are inert until
-the pod is scheduled and the guard taint only softly repels routine pods.
-
-## Step (b) — Merge the branch to main
-
-Merge `feature/hermes-agent-multiarch` to `main` **without** the final titan-22
-flip commit if you want to stage the runtime change separately (see step d). The
-Dockerfile, Jenkinsfile, combiner, tests, and CronJob are safe to merge: the
-arm64 build path is unchanged and the amd64 leg + combine only add new,
-fail-closed artifacts. Flux then applies the CronJob (titan-24 becomes a
-worker).
-
-## Step (c) — Trigger ONE validation build
-
-Trigger the existing manual `hermes-agent-image` job exactly as today (the same
-`PUBLISH_IMAGE=true`, `EXPECTED_SOURCE_REVISION=<full reviewed sha on main>`,
-`CONFIRM_PUBLISH=PUBLISH HERMES AGENT` gate). Nothing about the trigger changes.
-
-Success looks like:
-
-1. **arm64 leg** builds on an rpi5 pod and pushes `…-build-<N>-arm64`.
-2. **amd64 leg** builds on titan-24 and pushes `…-build-<N>-amd64`.
-3. **Combine multi-arch index** prints `"result": "published"` with an
-   `index_digest`, and the registry resolves `…-build-<N>` to a Docker manifest
-   list referencing exactly the two leaves.
-4. Render / Verify-and-archive / Publish stages behave exactly as before but on
-   the **index** digest; the `…-build-<N>-release` tag is the multi-arch index.
-5. Archived evidence now includes both per-arch digests/images plus the index
-   digest/image.
-
-Independent confirmation the release is multi-arch:
-
-```sh
-skopeo inspect --raw docker://registry.bstein.dev/bstein/hermes-agent:git-<sha>-build-<N>-release \
-  | jq -r '.mediaType, (.manifests[] | "\(.platform.os)/\(.platform.architecture) \(.digest)")'
-# expect: application/vnd.docker.distribution.manifest.list.v2+json
-#         linux/amd64 sha256:…
-#         linux/arm64 sha256:…
-```
-
-Flux will roll the `hermes-agent` deployment to the new index digest. It still
-runs on arm64 (unchanged affinity), and arm64 nodes keep pulling the arm64 leaf
-— so this step is safe even though the runtime is still arm64-only.
-
-## Step (d) — Apply the titan-22 runtime affinity flip
-
-**Only after step (c) proves the multi-arch image.** Apply the final, separately
-messaged commit `APPLY ONLY AFTER multi-arch image validated: allow hermes-agent
-on titan-22` (it edits `services/hermes/agent-deployment.yaml` and the matching
-layout test). It:
-
-- adds a second `nodeSelectorTerm` matching amd64 `titan-22` (worker) — the
-  arm64 fleet term is untouched, so the pi fleet remains a valid fallback;
-- adds a soft `preferredDuringScheduling` nudge toward titan-22;
-- adds a toleration for titan-22's `atlas.bstein.dev/media-primary=true:PreferNoSchedule`
-  taint. This does **not** change jellyfin's own scheduling or priority — a
-  toleration only lets hermes-agent *also* consider titan-22 as spare capacity.
-
-Merge that commit; Flux recreates the pod (deployment strategy is `Recreate`).
-
-## Step (e) — Verify the worker lands correctly
-
-```sh
-kubectl -n hermes get pod -l app=hermes-agent -o wide
-# It may still land on an rpi5 (fallback is intact and pis are preferred-eligible);
-# it is now ALSO permitted on titan-22. To confirm titan-22 can host it, cordon
-# the pi fleet briefly or scale/drain, or simply confirm the pod tolerates the
-# media taint and schedules when pis are unavailable.
-kubectl -n hermes describe pod -l app=hermes-agent | grep -iE 'node:|toleration'
-```
-
-To roll back the runtime flip, revert the step-(d) commit; the arm64-only
-affinity returns and the multi-arch image keeps working on arm64.
-
----
-
-## What is validated locally vs what only a real build proves
-
-**Validated locally on this branch:**
-- The base index digest resolves to both `linux/arm64` (the old pin) and
-  `linux/amd64`, same revision `9de9c25f…` (`skopeo inspect --raw`).
-- `hermes_multiarch_combine.py` end-to-end unit tests (full hash + arch-config
-  verification chain, idempotency, and every fail-closed branch).
-- The updated + new image-builder tests, `hermes_oci_promote`, and the automation
-  tests pass.
-- `kustomize build services/hermes` and `kustomize build infrastructure/core`
-  succeed.
-
-**Only a real Jenkins build can prove:**
-- Kaniko auto-selects the correct native leaf from the index on each arch, and
-  the amd64 heredoc-compat replay produces a working amd64 image.
-- The amd64 pod actually schedules on titan-24 within its resource cap without
-  starving the Sui validator.
-- Harbor accepts the manually-assembled manifest list PUT and immutability
-  applies to the index tag.
-- Cross-pod `stash`/`unstash` of the amd64 leaf evidence behaves as expected.

infrastructure/core/node-prefer-noschedule-cronjob.yaml

@@ -59,7 +59,6 @@ spec:
                   clear_worker titan-14 rpi4
                   clear_worker titan-18 rpi4
                   clear_worker titan-22 amd64
-                  clear_worker titan-24 amd64
 
                   if k get node titan-22 >/dev/null 2>&1; then
                     k label node titan-22 \
@@ -70,21 +69,6 @@ spec:
                     k taint node titan-22 atlas.bstein.dev/media-primary=true:PreferNoSchedule --overwrite=true || true
                   fi
 
-                  if k get node titan-24 >/dev/null 2>&1; then
-                    # titan-24 (amd64, RTX 3080) co-hosts the out-of-cluster Sui
-                    # validator (docker-compose at /opt/sui-validator). It joins as a
-                    # worker so the native amd64 hermes-agent image leg can build here,
-                    # but a soft guard taint keeps routine pods from crowding the
-                    # validator. GPU workloads pinned to titan-24 by hostname are
-                    # unaffected (PreferNoSchedule never blocks a pinned pod); the
-                    # amd64 build pod tolerates this taint explicitly.
-                    k label node titan-24 \
-                      atlas.bstein.dev/general-compute=last-resort \
-                      atlas.bstein.dev/sui-validator=true \
-                      --overwrite=true || true
-                    k taint node titan-24 atlas.bstein.dev/sui-validator=true:PreferNoSchedule --overwrite=true || true
-                  fi
-
                   if k get node titan-23 >/dev/null 2>&1; then
                     k label node titan-23 \
                       veles.bstein.dev/simulation=true \

testing/tests/test_hermes_image_builder.py

@@ -113,8 +113,7 @@ def test_pipeline_requires_reviewed_main_and_runtime_credentials() -> None:
     assert "--destination=" in source
     assert "assert-absent" in source
     assert "git-${actual_revision}-build-${BUILD_NUMBER}" in source
-    # One reviewed heredoc-compat build-arg per native Kaniko leg (arm64 + amd64).
+    assert source.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1
-    assert source.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 2
 
 
 def test_kaniko_replays_only_the_exact_reviewed_heredoc_contract() -> None:

testing/tests/test_hermes_image_builder_fresh_review.py

@@ -32,15 +32,6 @@ def _pod_spec() -> dict:
     return yaml.safe_load(pod_yaml)["spec"]
 
 
-def _amd64_pod_spec() -> dict:
-    """Parse the second inline pod (the native amd64 build leg on titan-24)."""
-    source = PIPELINE.read_text(encoding="utf-8")
-    blocks = source.split('yaml """')
-    assert len(blocks) == 3, "expected exactly the arm64 and amd64 build pods"
-    pod_yaml = blocks[2].split('"""', 1)[0]
-    return yaml.safe_load(pod_yaml)["spec"]
-
-
 def test_builder_prefers_rpi5_with_healthy_arm64_worker_fallback() -> None:
     """Disposable builds prefer rpi5 without excluding schedulable rpi4 workers."""
     spec = _pod_spec()
@@ -115,8 +106,7 @@ def test_exact_reviewed_heredocs_remain_buildkit_native_by_default() -> None:
     assert source.count("ARG HERMES_KANIKO_HEREDOC_COMPAT=0") == 1
     assert "Kaniko v1.23.2" in RUNNER.read_text(encoding="utf-8")
     pipeline = PIPELINE.read_text(encoding="utf-8")
-    # The reviewed heredoc replay is enabled on both native Kaniko legs.
+    assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1
-    assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 2
 
 
 def test_appended_tenth_reviewed_form_rejects_before_execution(
@@ -348,91 +338,11 @@ def test_pipeline_requires_and_archives_exact_release_evidence() -> None:
     assert "allowEmptyArchive: false" in evidence
     archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0]
     paths = archive.split(",")
-    # Multi-arch adds both per-arch leaf digests/images plus the index digest.
+    assert len(paths) == 7
-    assert len(paths) == 11
+    assert len(set(paths)) == 7
-    assert len(set(paths)) == 11
     assert all("*" not in path for path in paths)
     assert "find build -type f" in evidence
     assert "build/hermes-agent.source-revision" in paths
-    assert "build/hermes-agent.digest" in paths
-    for arch in ("arm64", "amd64"):
-        assert f"build/hermes-agent-{arch}.digest" in paths
-        assert f"build/hermes-agent-{arch}.image" in paths
     promotion = source.split("stage('Publish Flux release tag')", 1)[1]
     assert "ci/scripts/hermes_oci_promote.py" in promotion
     assert "  post {" not in source
-
-
-def test_amd64_leg_is_pinned_to_titan24_and_resource_capped() -> None:
-    """The amd64 build leg lands on titan-24, tolerates its guard taint, and is capped."""
-    spec = _amd64_pod_spec()
-    assert spec["nodeSelector"] == {
-        "kubernetes.io/arch": "amd64",
-        "kubernetes.io/hostname": "titan-24",
-        "node-role.kubernetes.io/worker": "true",
-    }
-    # titan-24 co-hosts the Sui validator; the disposable build must tolerate
-    # whatever guard taint the node carries so the pinned pod still schedules.
-    assert {"operator": "Exists"} in spec["tolerations"]
-    assert spec["serviceAccountName"] == "hermes-image-builder"
-    assert spec["automountServiceAccountToken"] is False
-    assert spec["enableServiceLinks"] is False
-
-    containers = {item["name"]: item for item in spec["containers"]}
-    kaniko = containers["kaniko"]
-    # Identical pinned Kaniko across both legs -- no second, unreviewed builder.
-    assert kaniko["image"] == (
-        "gcr.io/kaniko-project/executor@sha256:"
-        "c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e"
-    )
-    assert kaniko["securityContext"]["capabilities"]["add"] == [
-        "CHOWN",
-        "FOWNER",
-        "DAC_OVERRIDE",
-        "SETGID",
-        "SETUID",
-    ]
-    for container in containers.values():
-        security = container["securityContext"]
-        assert security["allowPrivilegeEscalation"] is False
-        assert security["capabilities"]["drop"] == ["ALL"]
-        assert security["seccompProfile"]["type"] == "RuntimeDefault"
-        assert security.get("privileged", False) is False
-    # Tight caps keep the disposable build from starving the co-hosted validator:
-    # the amd64 kaniko ceiling is strictly below the arm64 leg's 2 CPU / 4Gi.
-    limits = kaniko["resources"]["limits"]
-    assert limits["cpu"] == "1500m"
-    assert limits["memory"] == "3Gi"
-
-
-def test_amd64_leg_source_is_independently_boundary_checked() -> None:
-    """The amd64 pod re-derives and re-verifies the reviewed revision itself."""
-    source = PIPELINE.read_text(encoding="utf-8")
-    amd64_stage = source.split("stage('Build amd64 leg without a daemon')", 1)[1]
-    amd64_stage = amd64_stage.split("stage('Combine multi-arch index')", 1)[0]
-    # Same fail-closed boundary as the coordinating pod, re-run in the amd64 pod.
-    assert 'merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}"' in amd64_stage
-    assert 'checkout --detach "${EXPECTED_SOURCE_REVISION}"' in amd64_stage
-    assert 'test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"' in amd64_stage
-    assert 'status --porcelain' in amd64_stage
-    assert '$(cat build/hermes-agent.destination)-amd64' in amd64_stage
-    # The amd64 leaf evidence crosses pods only through an explicit stash.
-    assert "stash" in amd64_stage
-    assert "hermes-agent-amd64.digest" in amd64_stage
-
-
-def test_combine_stage_publishes_and_reverifies_the_index() -> None:
-    """Kaniko cannot combine; the reviewed python combiner assembles the index."""
-    source = PIPELINE.read_text(encoding="utf-8")
-    combine = source.split("stage('Combine multi-arch index')", 1)[1]
-    combine = combine.split("stage('Render reviewed Flux handoff')", 1)[0]
-    assert "unstash 'hermes-agent-amd64-evidence'" in combine
-    assert "ci/scripts/hermes_multiarch_combine.py" in combine
-    assert "--arm64-digest-file build/hermes-agent-arm64.digest" in combine
-    assert "--amd64-digest-file build/hermes-agent-amd64.digest" in combine
-    # The combiner emits the arch-less index evidence the existing chain promotes.
-    assert "--digest-file build/hermes-agent.digest" in combine
-    assert "--image-file build/hermes-agent.image" in combine
-    # Downstream render/verify/promote still consume the single index digest file.
-    render = source.split("stage('Render reviewed Flux handoff')", 1)[1]
-    assert "--digest-file build/hermes-agent.digest" in render

testing/tests/test_hermes_multiarch_combine.py

@@ -1,385 +0,0 @@
-"""Safety tests for the fail-closed multi-arch manifest-list combiner."""
-
-from __future__ import annotations
-
-import hashlib
-import importlib.util
-import io
-import json
-import sys
-from pathlib import Path
-
-import pytest
-
-
-ROOT = Path(__file__).resolve().parents[2]
-SCRIPT = ROOT / "ci/scripts/hermes_multiarch_combine.py"
-REVISION = "a" * 40
-BUILD = "17"
-DESTINATION = f"registry.bstein.dev/bstein/hermes-agent:git-{REVISION}-build-{BUILD}"
-DOCKER_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json"
-DOCKER_MANIFEST = "application/vnd.docker.distribution.manifest.v2+json"
-IMAGE_CONFIG = "application/vnd.docker.container.image.v1+json"
-
-
-def _load():
-    spec = importlib.util.spec_from_file_location("hermes_multiarch_combine", SCRIPT)
-    assert spec is not None and spec.loader is not None
-    module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)
-    return module
-
-
-def _sha(body: bytes) -> str:
-    return "sha256:" + hashlib.sha256(body).hexdigest()
-
-
-class Response(io.BytesIO):
-    """Minimal context-managed urllib response."""
-
-    def __init__(self, status: int, body: bytes = b"", headers=None):
-        super().__init__(body)
-        self.status = status
-        self.headers = headers or {}
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *_args):
-        self.close()
-
-
-def _leaf(architecture: str, *, os_name: str = "linux"):
-    """Build a self-consistent config+manifest whose bytes hash to real digests."""
-    config_body = json.dumps(
-        {"architecture": architecture, "os": os_name}, sort_keys=True
-    ).encode()
-    config_digest = _sha(config_body)
-    manifest_body = json.dumps(
-        {
-            "schemaVersion": 2,
-            "mediaType": DOCKER_MANIFEST,
-            "config": {
-                "mediaType": IMAGE_CONFIG,
-                "digest": config_digest,
-                "size": len(config_body),
-            },
-            "layers": [],
-        },
-        sort_keys=True,
-    ).encode()
-    return {
-        "arch": architecture,
-        "config_body": config_body,
-        "config_digest": config_digest,
-        "manifest_body": manifest_body,
-        "digest": _sha(manifest_body),
-        "content_type": DOCKER_MANIFEST,
-    }
-
-
-def _descriptor(module, leaf):
-    return {
-        "mediaType": leaf["content_type"],
-        "size": len(leaf["manifest_body"]),
-        "digest": leaf["digest"],
-        "platform": {"architecture": leaf["arch"], "os": "linux"},
-    }
-
-
-def _index_digest(module, amd64, arm64) -> str:
-    body = module._manifest_list_bytes(
-        [_descriptor(module, amd64), _descriptor(module, arm64)]
-    )
-    return _sha(body)
-
-
-class Registry:
-    """Route the combiner's deterministic request sequence by method and path."""
-
-    def __init__(self, module, amd64, arm64, *, head_status=404, existing="") -> None:
-        self.module = module
-        self.leaves = {amd64["arch"]: amd64, arm64["arch"]: arm64}
-        self.index_digest = _index_digest(module, amd64, arm64)
-        self.head_status = head_status
-        self.existing = existing
-        self.put_body = None
-        self.calls = []
-
-    def __call__(self, request, timeout):
-        method = request.method
-        url = request.full_url
-        self.calls.append((method, url))
-        for arch, leaf in self.leaves.items():
-            if url.endswith(f"-{arch}"):
-                return Response(
-                    200,
-                    leaf["manifest_body"],
-                    {
-                        "Docker-Content-Digest": leaf["digest"],
-                        "Content-Type": leaf["content_type"],
-                    },
-                )
-            # Blob URLs percent-encode the ``sha256:`` colon; match the raw hex.
-            if url.endswith(leaf["config_digest"].split(":", 1)[1]):
-                return Response(200, leaf["config_body"], {})
-        # Final index tag (no arch suffix, ends with the build tag).
-        if method == "HEAD":
-            return Response(
-                self.head_status,
-                headers={"Docker-Content-Digest": self.existing},
-            )
-        if method == "PUT":
-            self.put_body = request.data
-            return Response(201, headers={"Docker-Content-Digest": self.index_digest})
-        return Response(
-            200,
-            self.module._manifest_list_bytes(
-                [
-                    _descriptor(self.module, self.leaves["amd64"]),
-                    _descriptor(self.module, self.leaves["arm64"]),
-                ]
-            ),
-            {
-                "Docker-Content-Digest": self.index_digest,
-                "Content-Type": DOCKER_MANIFEST_LIST,
-            },
-        )
-
-
-def _combine(module, registry, arch_digests=None):
-    if arch_digests is None:
-        arch_digests = {
-            "amd64": registry.leaves["amd64"]["digest"],
-            "arm64": registry.leaves["arm64"]["digest"],
-        }
-    return module.combine_multiarch_index(
-        destination=DESTINATION,
-        arch_digests=arch_digests,
-        username="robot",
-        password="private",
-        opener=registry,
-    )
-
-
-def test_combines_two_verified_leaves_into_one_index() -> None:
-    """Both native leaves are re-read, arch-proven, and published as a list."""
-    module = _load()
-    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
-    registry = Registry(module, amd64, arm64)
-    result = _combine(module, registry)
-    assert result["result"] == "published"
-    assert result["index_digest"] == registry.index_digest
-    assert result["amd64_digest"] == amd64["digest"]
-    assert result["arm64_digest"] == arm64["digest"]
-    # The published bytes are exactly what we hashed for the index digest.
-    assert _sha(registry.put_body) == registry.index_digest
-    assert "private" not in json.dumps(result)
-
-
-def test_idempotent_when_index_already_matches() -> None:
-    """A replay is accepted only when the existing index digest is identical."""
-    module = _load()
-    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
-    index_digest = _index_digest(module, amd64, arm64)
-    registry = Registry(
-        module, amd64, arm64, head_status=200, existing=index_digest
-    )
-    result = _combine(module, registry)
-    assert result["result"] == "already-present"
-    assert registry.put_body is None
-
-
-def test_existing_index_with_other_digest_fails_closed() -> None:
-    """An occupied final tag with a different index digest never republishes."""
-    module = _load()
-    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
-    registry = Registry(
-        module, amd64, arm64, head_status=200, existing="sha256:" + "c" * 64
-    )
-    with pytest.raises(RuntimeError, match="another index digest"):
-        _combine(module, registry)
-
-
-def test_rejects_leaf_digest_that_disagrees_with_evidence() -> None:
-    """A leaf whose registry digest is not the Kaniko evidence fails closed."""
-    module = _load()
-    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
-    registry = Registry(module, amd64, arm64)
-    with pytest.raises(RuntimeError, match="amd64 leaf digest"):
-        _combine(
-            module,
-            registry,
-            arch_digests={"amd64": "sha256:" + "d" * 64, "arm64": arm64["digest"]},
-        )
-
-
-def test_rejects_leaf_whose_config_architecture_is_wrong() -> None:
-    """A leaf that claims the wrong architecture in its config fails closed."""
-    module = _load()
-    # Build an "amd64" candidate tag whose config actually says arm64.
-    swapped = _leaf("arm64")
-    swapped["arch"] = "amd64"  # served under the amd64 tag, but arm64 inside
-    arm64 = _leaf("arm64")
-    registry = Registry(module, swapped, arm64)
-    with pytest.raises(RuntimeError, match="architecture"):
-        _combine(
-            module,
-            registry,
-            arch_digests={"amd64": swapped["digest"], "arm64": arm64["digest"]},
-        )
-
-
-def test_rejects_leaf_served_as_a_manifest_list() -> None:
-    """A per-arch leaf must be a single image, never itself an index/list."""
-    module = _load()
-    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
-    amd64["content_type"] = DOCKER_MANIFEST_LIST
-    registry = Registry(module, amd64, arm64)
-    with pytest.raises(RuntimeError, match="single-image manifest"):
-        _combine(module, registry)
-
-
-def test_requires_both_architectures() -> None:
-    """The combiner refuses any arch set other than exactly arm64 and amd64."""
-    module = _load()
-    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
-    registry = Registry(module, amd64, arm64)
-    with pytest.raises(ValueError, match="arm64 and amd64"):
-        module.combine_multiarch_index(
-            destination=DESTINATION,
-            arch_digests={"amd64": amd64["digest"]},
-            username="robot",
-            password="private",
-            opener=registry,
-        )
-
-
-def test_rejects_destination_with_arch_suffix() -> None:
-    """Only the arch-less final tag is a valid combine destination."""
-    module = _load()
-    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
-    with pytest.raises(ValueError, match="invalid multi-arch destination"):
-        module.combine_multiarch_index(
-            destination=f"{DESTINATION}-arm64",
-            arch_digests={"amd64": amd64["digest"], "arm64": arm64["digest"]},
-            username="robot",
-            password="private",
-            opener=Registry(module, amd64, arm64),
-        )
-
-
-def test_cli_binds_evidence_files_and_writes_index_outputs(
-    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
-) -> None:
-    """The command reads both Kaniko pairs and emits arch-less index evidence."""
-    module = _load()
-    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
-    registry = Registry(module, amd64, arm64)
-    files = {}
-    for arch, leaf in (("amd64", amd64), ("arm64", arm64)):
-        digest_file = tmp_path / f"{arch}.digest"
-        image_file = tmp_path / f"{arch}.image"
-        digest_file.write_text(leaf["digest"] + "\n", encoding="utf-8")
-        image_file.write_text(
-            f"{DESTINATION}-{arch}@{leaf['digest']}\n", encoding="utf-8"
-        )
-        files[arch] = (digest_file, image_file)
-    out_digest = tmp_path / "index.digest"
-    out_image = tmp_path / "index.image"
-    monkeypatch.setenv("HARBOR_USER", "robot")
-    monkeypatch.setenv("HARBOR_PASSWORD", "private")
-    captured = {}
-
-    def fake_combine(*, destination, arch_digests, username, password):
-        captured["destination"] = destination
-        captured["arch_digests"] = dict(arch_digests)
-        return {"index_digest": registry.index_digest, "result": "published"}
-
-    monkeypatch.setattr(module, "combine_multiarch_index", fake_combine)
-    monkeypatch.setattr(
-        sys,
-        "argv",
-        [
-            "hermes_multiarch_combine.py",
-            "--destination",
-            DESTINATION,
-            "--source-revision",
-            REVISION,
-            "--build-number",
-            BUILD,
-            "--arm64-digest-file",
-            str(files["arm64"][0]),
-            "--arm64-image-file",
-            str(files["arm64"][1]),
-            "--amd64-digest-file",
-            str(files["amd64"][0]),
-            "--amd64-image-file",
-            str(files["amd64"][1]),
-            "--digest-file",
-            str(out_digest),
-            "--image-file",
-            str(out_image),
-        ],
-    )
-    assert module.main() == 0
-    assert out_digest.read_text(encoding="utf-8").strip() == registry.index_digest
-    assert (
-        out_image.read_text(encoding="utf-8").strip()
-        == f"{DESTINATION}@{registry.index_digest}"
-    )
-    assert json.loads(capsys.readouterr().out)["result"] == "published"
-    # main() must bind each arch's two evidence files to the correct leaf digest.
-    assert captured["destination"] == DESTINATION
-    assert captured["arch_digests"] == {
-        "amd64": amd64["digest"],
-        "arm64": arm64["digest"],
-    }
-
-
-def test_cli_rejects_mismatched_per_arch_evidence_pair(
-    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
-) -> None:
-    """A digest/image evidence file mismatch is reported as a JSON error."""
-    module = _load()
-    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
-    files = {}
-    for arch, leaf in (("amd64", amd64), ("arm64", arm64)):
-        digest_file = tmp_path / f"{arch}.digest"
-        image_file = tmp_path / f"{arch}.image"
-        digest_file.write_text(leaf["digest"] + "\n", encoding="utf-8")
-        # amd64 image file points at the wrong digest.
-        recorded = "sha256:" + "e" * 64 if arch == "amd64" else leaf["digest"]
-        image_file.write_text(
-            f"{DESTINATION}-{arch}@{recorded}\n", encoding="utf-8"
-        )
-        files[arch] = (digest_file, image_file)
-    monkeypatch.setenv("HARBOR_USER", "robot")
-    monkeypatch.setenv("HARBOR_PASSWORD", "private")
-    monkeypatch.setattr(
-        sys,
-        "argv",
-        [
-            "hermes_multiarch_combine.py",
-            "--destination",
-            DESTINATION,
-            "--source-revision",
-            REVISION,
-            "--build-number",
-            BUILD,
-            "--arm64-digest-file",
-            str(files["arm64"][0]),
-            "--arm64-image-file",
-            str(files["arm64"][1]),
-            "--amd64-digest-file",
-            str(files["amd64"][0]),
-            "--amd64-image-file",
-            str(files["amd64"][1]),
-            "--digest-file",
-            str(tmp_path / "index.digest"),
-            "--image-file",
-            str(tmp_path / "index.image"),
-        ],
-    )
-    assert module.main() == 1
-    assert "does not match" in json.loads(capsys.readouterr().out)["error"]