docs(hermes-agent): multi-arch rollout runbook

Operator steps in order: prepare/uncordon titan-24, merge, run one validation
build (both arch leaves + promoted index), apply the titan-22 affinity flip,
verify placement. Documents the correction that worker membership is reconciled
by the Flux node-prefer-noschedule CronJob, not Ansible, and lists what was
validated locally vs what only a real Jenkins build can prove.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf

ci/Jenkinsfile.hermes-agent-image

@@ -188,6 +188,7 @@ spec:
               testing/tests/test_hermes_image_builder_coverage.py \
               testing/tests/test_hermes_image_builder_fresh_review.py \
               testing/tests/test_hermes_oci_promote.py \
+              testing/tests/test_hermes_multiarch_combine.py \
               testing/tests/test_hermes_image_automation.py
           '''
         }
@@ -213,7 +214,7 @@ spec:
         }
       }
     }
-    stage('Build and publish without a daemon') {
+    stage('Build arm64 leg without a daemon') {
       steps {
         container('kaniko') {
           withCredentials([usernamePassword(
@@ -225,7 +226,7 @@ spec:
               set -eu
               set +x
               config_path=/kaniko/.docker/config.json
-              destination="$(cat build/hermes-agent.destination)"
+              destination="$(cat build/hermes-agent.destination)-arm64"
               source_revision="$(cat build/hermes-agent.source-revision)"
               umask 077
               auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
@@ -240,18 +241,192 @@ spec:
                 --context="dir://${WORKSPACE}" \
                 --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
                 --destination="${destination}" \
-                --digest-file="${WORKSPACE}/build/hermes-agent.digest" \
-                --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \
+                --digest-file="${WORKSPACE}/build/hermes-agent-arm64.digest" \
+                --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-arm64.image" \
                 --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
                 --label="org.opencontainers.image.revision=${source_revision}" \
                 --cleanup \
                 --push-retry=3
-              /busybox/chmod 644 build/hermes-agent.digest build/hermes-agent.image
+              /busybox/chmod 644 build/hermes-agent-arm64.digest build/hermes-agent-arm64.image
             '''
           }
         }
       }
     }
+    stage('Build amd64 leg without a daemon') {
+      agent {
+        kubernetes {
+          yaml """
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    atlas.bstein.dev/workload: hermes-agent-image-builder-amd64
+spec:
+  serviceAccountName: hermes-image-builder
+  automountServiceAccountToken: false
+  enableServiceLinks: false
+  restartPolicy: Never
+  securityContext:
+    fsGroup: 1000
+    fsGroupChangePolicy: OnRootMismatch
+  nodeSelector:
+    kubernetes.io/arch: amd64
+    kubernetes.io/hostname: titan-24
+    node-role.kubernetes.io/worker: "true"
+  tolerations:
+    # titan-24 co-hosts the out-of-cluster Sui validator; tolerate whatever
+    # PreferNoSchedule/NoSchedule guard taint the node carries so the pinned
+    # build lands, and rely on the tight resource caps below (not scheduling
+    # priority) to keep the disposable build from starving the validator.
+    - operator: Exists
+  imagePullSecrets:
+    - name: harbor-bstein-robot
+  containers:
+    - name: jnlp
+      image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop: ["ALL"]
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      resources:
+        requests:
+          cpu: 25m
+          memory: 128Mi
+        limits:
+          cpu: 250m
+          memory: 384Mi
+    - name: kaniko
+      image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
+      command: ["/busybox/sh", "-c"]
+      args: ["/busybox/sleep 99d"]
+      tty: true
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop: ["ALL"]
+          add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
+        privileged: false
+        runAsUser: 0
+        seccompProfile:
+          type: RuntimeDefault
+      resources:
+        requests:
+          cpu: 100m
+          memory: 512Mi
+        limits:
+          cpu: "1500m"
+          memory: 3Gi
+"""
+        }
+      }
+      steps {
+        container('jnlp') {
+          sh '''
+            set -eu
+            mkdir -p build
+            test "${PUBLISH_IMAGE}" = "true"
+            test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
+            case "${EXPECTED_SOURCE_REVISION}" in
+              *[!0-9a-f]*|'')
+                echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
+                exit 2
+                ;;
+            esac
+            test "${#EXPECTED_SOURCE_REVISION}" -eq 40
+            main_revision="$(git rev-parse origin/main)"
+            git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
+            git checkout --detach "${EXPECTED_SOURCE_REVISION}"
+            actual_revision="$(git rev-parse HEAD)"
+            test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
+            test -z "$(git status --porcelain)"
+            test -f dockerfiles/Dockerfile.hermes-agent
+            case "${BUILD_NUMBER}" in
+              ''|0*|*[!0-9]*)
+                echo "BUILD_NUMBER must be a positive decimal integer" >&2
+                exit 2
+                ;;
+            esac
+            printf '%s\n' \
+              "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
+              > build/hermes-agent.destination
+            printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision
+          '''
+        }
+        container('kaniko') {
+          withCredentials([usernamePassword(
+            credentialsId: 'harbor-robot',
+            usernameVariable: 'HARBOR_USER',
+            passwordVariable: 'HARBOR_PASSWORD'
+          )]) {
+            sh '''#!/busybox/sh
+              set -eu
+              set +x
+              config_path=/kaniko/.docker/config.json
+              destination="$(cat build/hermes-agent.destination)-amd64"
+              source_revision="$(cat build/hermes-agent.source-revision)"
+              umask 077
+              auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
+              /busybox/mkdir -p /kaniko/.docker
+              /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
+              unset HARBOR_USER HARBOR_PASSWORD auth
+              trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
+              umask 022
+              /kaniko/executor \
+                --registry-mirror=harbor-core.harbor.svc.cluster.local \
+                --insecure-registry=harbor-core.harbor.svc.cluster.local \
+                --context="dir://${WORKSPACE}" \
+                --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
+                --destination="${destination}" \
+                --digest-file="${WORKSPACE}/build/hermes-agent-amd64.digest" \
+                --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-amd64.image" \
+                --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
+                --label="org.opencontainers.image.revision=${source_revision}" \
+                --cleanup \
+                --push-retry=3
+              /busybox/chmod 644 build/hermes-agent-amd64.digest build/hermes-agent-amd64.image
+            '''
+          }
+        }
+        stash(
+          name: 'hermes-agent-amd64-evidence',
+          includes: 'build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image'
+        )
+      }
+    }
+    stage('Combine multi-arch index') {
+      steps {
+        unstash 'hermes-agent-amd64-evidence'
+        withCredentials([usernamePassword(
+          credentialsId: 'harbor-robot',
+          usernameVariable: 'HARBOR_USER',
+          passwordVariable: 'HARBOR_PASSWORD'
+        )]) {
+          sh '''
+            set -eu
+            set +x
+            destination="$(cat build/hermes-agent.destination)"
+            source_revision="$(cat build/hermes-agent.source-revision)"
+            python3 ci/scripts/hermes_multiarch_combine.py \
+              --destination "${destination}" \
+              --source-revision "${source_revision}" \
+              --build-number "${BUILD_NUMBER}" \
+              --arm64-digest-file build/hermes-agent-arm64.digest \
+              --arm64-image-file build/hermes-agent-arm64.image \
+              --amd64-digest-file build/hermes-agent-amd64.digest \
+              --amd64-image-file build/hermes-agent-amd64.image \
+              --digest-file build/hermes-agent.digest \
+              --image-file build/hermes-agent.image
+            test -s build/hermes-agent.digest
+            test -s build/hermes-agent.image
+          '''
+        }
+      }
+    }
     stage('Render reviewed Flux handoff') {
       steps {
         withCredentials([usernamePassword(
@@ -287,6 +462,10 @@ spec:
           build/hermes-agent.digest \
           build/hermes-agent.image \
           build/hermes-agent.source-revision \
+          build/hermes-agent-arm64.digest \
+          build/hermes-agent-arm64.image \
+          build/hermes-agent-amd64.digest \
+          build/hermes-agent-amd64.image \
           build/hermes-agent-release/hermes-agent-image.json \
           build/hermes-agent-release/hermes-image-update.patch \
           build/hermes-agent-release/hermes-kustomization.yaml \
@@ -305,7 +484,7 @@ spec:
           --output-dir build/hermes-agent-release
       '''
       archiveArtifacts(
-        artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.source-revision,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
+        artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.source-revision,build/hermes-agent-arm64.digest,build/hermes-agent-arm64.image,build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
         allowEmptyArchive: false,
         fingerprint: true
       )

ci/scripts/hermes_multiarch_combine.py

@@ -0,0 +1,390 @@
+#!/usr/bin/env python3
+"""Assemble a fail-closed multi-arch manifest list from two per-arch leaves.
+
+Kaniko builds one native image per architecture (arm64 on an rpi5 pod, amd64 on
+titan-24) and pushes each under an arch-suffixed candidate tag
+``...-build-<N>-<arch>``.  Kaniko cannot combine, so this step:
+
+1. Independently re-reads each per-arch candidate manifest from the registry and
+   binds it to the exact Kaniko digest evidence.
+2. Proves each leaf really is the architecture it claims by reading its image
+   config (a swapped or cross-built leaf fails closed here).
+3. Builds a Docker manifest *list* (not an OCI index) from the two verified
+   leaves -- Docker manifest lists are already inside the promotion allow-list,
+   so this keeps the security surface of ``hermes_oci_promote.py`` unchanged.
+4. Refuses to overwrite an existing final tag, PUTs the list to the final
+   ``...-build-<N>`` tag, and re-reads it to confirm the registry resolved the
+   exact index digest referencing exactly the two expected leaves.
+
+The output ``--digest-file``/``--image-file`` deliberately use the SAME format
+the single-arch Kaniko step produced (``<digest>`` and ``<destination>@<digest>``
+for the final, arch-less tag).  The whole downstream evidence chain --
+``hermes_image_release.py`` render/verify-evidence and ``hermes_oci_promote.py``
+-- therefore promotes the multi-arch INDEX with no further change.
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import hashlib
+import json
+import os
+import re
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import Any, Callable
+
+
+REGISTRY_ORIGIN = "https://registry.bstein.dev"
+# The final (arch-less) Flux-visible tag; identical contract to the promoter.
+DESTINATION_PATTERN = re.compile(
+    r"^registry\.bstein\.dev/bstein/hermes-agent:"
+    r"git-(?P<revision>[0-9a-f]{40})-build-(?P<build>[1-9][0-9]*)$"
+)
+DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
+DOCKER_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json"
+# A per-arch leaf must be a single-image manifest, never itself a list/index.
+LEAF_MANIFEST_TYPES = {
+    "application/vnd.docker.distribution.manifest.v2+json",
+    "application/vnd.oci.image.manifest.v1+json",
+}
+IMAGE_CONFIG_TYPES = {
+    "application/vnd.docker.container.image.v1+json",
+    "application/vnd.oci.image.config.v1+json",
+}
+# Deterministic architecture order -> deterministic manifest-list bytes/digest.
+ARCHITECTURES = ("amd64", "arm64")
+MAX_MANIFEST_BYTES = 4 * 1024 * 1024
+MAX_CONFIG_BYTES = 1024 * 1024
+
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+    """Never forward registry credentials to another origin."""
+
+    def redirect_request(self, _request, _file, _code, _message, _headers, _url):
+        return None
+
+
+def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
+    """Return normal and HTTP error responses without following redirects."""
+    opener = urllib.request.build_opener(_NoRedirect())
+    try:
+        return opener.open(request, timeout=timeout)
+    except urllib.error.HTTPError as exc:
+        return exc
+
+
+def _status(response: Any) -> int:
+    """Normalize urllib response and HTTPError status fields."""
+    return int(getattr(response, "status", getattr(response, "code", 0)))
+
+
+def _authorization(username: str, password: str) -> str:
+    """Build a Basic authorization value without placing it in a URL."""
+    if not username or not password:
+        raise RuntimeError("Harbor credentials are unavailable")
+    encoded = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
+    return f"Basic {encoded}"
+
+
+def _manifest_url(component: str, reference: str) -> str:
+    """Return one same-origin, path-escaped Docker Registry manifest URL."""
+    encoded = urllib.parse.quote(reference, safe="")
+    return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/manifests/{encoded}"
+
+
+def _blob_url(component: str, digest: str) -> str:
+    """Return one same-origin, path-escaped Docker Registry blob URL."""
+    encoded = urllib.parse.quote(digest, safe="")
+    return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/blobs/{encoded}"
+
+
+def _read_evidence_pair(
+    *, digest_text: str, image_text: str, per_arch_tag_ref: str
+) -> str:
+    """Cross-check both Kaniko output files for one arch against its tag."""
+    digest_lines = digest_text.splitlines()
+    image_lines = image_text.splitlines()
+    if len(digest_lines) != 1:
+        raise ValueError("Kaniko digest evidence must contain exactly one line")
+    if len(image_lines) != 1:
+        raise ValueError("Kaniko image evidence must contain exactly one line")
+    digest = digest_lines[0].strip()
+    if not DIGEST_PATTERN.fullmatch(digest):
+        raise ValueError("invalid per-arch image digest")
+    if image_lines[0].strip() != f"{per_arch_tag_ref}@{digest}":
+        raise ValueError("per-arch image evidence does not match tag and digest")
+    return digest
+
+
+def _verified_leaf(
+    *,
+    component: str,
+    per_arch_tag: str,
+    architecture: str,
+    expected_digest: str,
+    authorization: str,
+    opener: Callable[[urllib.request.Request, int], Any],
+) -> dict[str, Any]:
+    """Re-read one per-arch leaf and prove its digest, type, and architecture."""
+    accept = ", ".join(sorted(LEAF_MANIFEST_TYPES))
+    request = urllib.request.Request(
+        _manifest_url(component, per_arch_tag),
+        headers={"Accept": accept, "Authorization": authorization},
+        method="GET",
+    )
+    with opener(request, 30) as response:
+        if _status(response) != 200:
+            raise RuntimeError(
+                f"{architecture} leaf manifest returned HTTP {_status(response)}"
+            )
+        body = response.read(MAX_MANIFEST_BYTES + 1)
+        if len(body) > MAX_MANIFEST_BYTES:
+            raise RuntimeError(f"{architecture} leaf manifest exceeded the size limit")
+        observed_digest = response.headers.get("Docker-Content-Digest", "")
+        content_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip()
+    if observed_digest != expected_digest:
+        raise RuntimeError(f"{architecture} leaf digest does not match build evidence")
+    # Defence in depth: the digest header is registry-asserted; recompute it too.
+    if f"sha256:{hashlib.sha256(body).hexdigest()}" != expected_digest:
+        raise RuntimeError(f"{architecture} leaf bytes do not hash to its digest")
+    if content_type not in LEAF_MANIFEST_TYPES:
+        raise RuntimeError(f"{architecture} leaf is not a single-image manifest")
+    try:
+        manifest = json.loads(body.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise RuntimeError(f"{architecture} leaf manifest is not valid JSON") from exc
+    config = manifest.get("config")
+    if not isinstance(config, dict):
+        raise RuntimeError(f"{architecture} leaf manifest omits its config descriptor")
+    config_digest = str(config.get("digest") or "")
+    config_type = str(config.get("mediaType") or "")
+    if not DIGEST_PATTERN.fullmatch(config_digest):
+        raise RuntimeError(f"{architecture} leaf config digest is invalid")
+    if config_type not in IMAGE_CONFIG_TYPES:
+        raise RuntimeError(f"{architecture} leaf config media type is unsupported")
+    config_request = urllib.request.Request(
+        _blob_url(component, config_digest),
+        headers={"Accept": config_type, "Authorization": authorization},
+        method="GET",
+    )
+    with opener(config_request, 30) as response:
+        if _status(response) != 200:
+            raise RuntimeError(
+                f"{architecture} leaf config returned HTTP {_status(response)}"
+            )
+        config_body = response.read(MAX_CONFIG_BYTES + 1)
+        if len(config_body) > MAX_CONFIG_BYTES:
+            raise RuntimeError(f"{architecture} leaf config exceeded the size limit")
+    if f"sha256:{hashlib.sha256(config_body).hexdigest()}" != config_digest:
+        raise RuntimeError(f"{architecture} leaf config bytes do not hash to its digest")
+    try:
+        config_json = json.loads(config_body.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise RuntimeError(f"{architecture} leaf config is not valid JSON") from exc
+    if config_json.get("architecture") != architecture:
+        raise RuntimeError(
+            f"{architecture} leaf config reports architecture "
+            f"{config_json.get('architecture')!r}"
+        )
+    if config_json.get("os") != "linux":
+        raise RuntimeError(f"{architecture} leaf config reports a non-linux os")
+    return {
+        "mediaType": content_type,
+        "size": len(body),
+        "digest": expected_digest,
+        "platform": {"architecture": architecture, "os": "linux"},
+    }
+
+
+def _manifest_list_bytes(descriptors: list[dict[str, Any]]) -> bytes:
+    """Serialize the manifest list deterministically for a stable index digest."""
+    document = {
+        "schemaVersion": 2,
+        "mediaType": DOCKER_MANIFEST_LIST,
+        "manifests": descriptors,
+    }
+    return json.dumps(document, sort_keys=True, separators=(",", ":")).encode("utf-8")
+
+
+def combine_multiarch_index(
+    *,
+    destination: str,
+    arch_digests: dict[str, str],
+    username: str,
+    password: str,
+    opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
+) -> dict[str, str]:
+    """Verify both leaves, publish, and re-verify one multi-arch index tag."""
+    match = DESTINATION_PATTERN.fullmatch(destination.strip())
+    if not match:
+        raise ValueError("invalid multi-arch destination")
+    if set(arch_digests) != set(ARCHITECTURES):
+        raise ValueError("expected exactly the arm64 and amd64 per-arch digests")
+    component = "hermes-agent"
+    index_tag = destination.rsplit(":", 1)[1]
+    authorization = _authorization(username, password)
+
+    descriptors = [
+        _verified_leaf(
+            component=component,
+            per_arch_tag=f"{index_tag}-{architecture}",
+            architecture=architecture,
+            expected_digest=arch_digests[architecture],
+            authorization=authorization,
+            opener=opener,
+        )
+        for architecture in ARCHITECTURES
+    ]
+    manifest_list = _manifest_list_bytes(descriptors)
+    if len(manifest_list) > MAX_MANIFEST_BYTES:
+        raise RuntimeError("assembled manifest list exceeded the size limit")
+    index_digest = f"sha256:{hashlib.sha256(manifest_list).hexdigest()}"
+
+    index_url = _manifest_url(component, index_tag)
+    head_request = urllib.request.Request(
+        index_url,
+        headers={"Accept": DOCKER_MANIFEST_LIST, "Authorization": authorization},
+        method="HEAD",
+    )
+    with opener(head_request, 20) as response:
+        head_status = _status(response)
+        existing_digest = response.headers.get("Docker-Content-Digest", "")
+    if head_status == 200:
+        if existing_digest != index_digest:
+            raise RuntimeError("final tag already exists with another index digest")
+        result = "already-present"
+    elif head_status == 404:
+        put_request = urllib.request.Request(
+            index_url,
+            data=manifest_list,
+            headers={
+                "Authorization": authorization,
+                "Content-Type": DOCKER_MANIFEST_LIST,
+            },
+            method="PUT",
+        )
+        with opener(put_request, 30) as response:
+            put_status = _status(response)
+            put_digest = response.headers.get("Docker-Content-Digest", "")
+        if put_status not in {201, 202}:
+            raise RuntimeError(f"index manifest returned HTTP {put_status}")
+        if put_digest and put_digest != index_digest:
+            raise RuntimeError("index manifest digest changed during publish")
+        result = "published"
+    else:
+        raise RuntimeError(f"final tag preflight returned HTTP {head_status}")
+
+    verify_request = urllib.request.Request(
+        index_url,
+        headers={"Accept": DOCKER_MANIFEST_LIST, "Authorization": authorization},
+        method="GET",
+    )
+    with opener(verify_request, 30) as response:
+        if _status(response) != 200:
+            raise RuntimeError(f"index verification returned HTTP {_status(response)}")
+        verify_body = response.read(MAX_MANIFEST_BYTES + 1)
+        if len(verify_body) > MAX_MANIFEST_BYTES:
+            raise RuntimeError("index verification exceeded the size limit")
+        verify_digest = response.headers.get("Docker-Content-Digest", "")
+        verify_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip()
+    if verify_digest != index_digest:
+        raise RuntimeError("registry resolved the final tag to another index digest")
+    if verify_type != DOCKER_MANIFEST_LIST:
+        raise RuntimeError("registry did not store a Docker manifest list")
+    try:
+        published = json.loads(verify_body.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise RuntimeError("registry returned invalid index JSON") from exc
+    published_leaves = {
+        (
+            str((item or {}).get("platform", {}).get("architecture")),
+            str((item or {}).get("digest")),
+        )
+        for item in published.get("manifests") or []
+    }
+    expected_leaves = {
+        (architecture, arch_digests[architecture]) for architecture in ARCHITECTURES
+    }
+    if published_leaves != expected_leaves:
+        raise RuntimeError("published index does not reference the exact two leaves")
+
+    return {
+        "component": component,
+        "index_digest": index_digest,
+        "index_tag": index_tag,
+        "result": result,
+        **{f"{architecture}_digest": arch_digests[architecture] for architecture in ARCHITECTURES},
+    }
+
+
+def _load_arch_digest(
+    *, destination: str, architecture: str, digest_file: Path, image_file: Path
+) -> str:
+    """Bind one arch's two Kaniko evidence files to its arch-suffixed tag."""
+    per_arch_tag_ref = f"{destination}-{architecture}"
+    return _read_evidence_pair(
+        digest_text=digest_file.read_text(encoding="utf-8"),
+        image_text=image_file.read_text(encoding="utf-8"),
+        per_arch_tag_ref=per_arch_tag_ref,
+    )
+
+
+def main() -> int:
+    """Combine two verified per-arch leaves and emit index digest evidence."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--destination", required=True)
+    parser.add_argument("--source-revision", required=True)
+    parser.add_argument("--build-number", required=True)
+    parser.add_argument("--arm64-digest-file", required=True, type=Path)
+    parser.add_argument("--arm64-image-file", required=True, type=Path)
+    parser.add_argument("--amd64-digest-file", required=True, type=Path)
+    parser.add_argument("--amd64-image-file", required=True, type=Path)
+    parser.add_argument("--digest-file", required=True, type=Path)
+    parser.add_argument("--image-file", required=True, type=Path)
+    args = parser.parse_args()
+    try:
+        match = DESTINATION_PATTERN.fullmatch(args.destination.strip())
+        if not match:
+            raise ValueError("invalid multi-arch destination")
+        if match.group("revision") != args.source_revision.strip():
+            raise ValueError("destination revision does not match evidence")
+        if match.group("build") != args.build_number.strip():
+            raise ValueError("destination build number does not match evidence")
+        destination = args.destination.strip()
+        arch_digests = {
+            "arm64": _load_arch_digest(
+                destination=destination,
+                architecture="arm64",
+                digest_file=args.arm64_digest_file,
+                image_file=args.arm64_image_file,
+            ),
+            "amd64": _load_arch_digest(
+                destination=destination,
+                architecture="amd64",
+                digest_file=args.amd64_digest_file,
+                image_file=args.amd64_image_file,
+            ),
+        }
+        result = combine_multiarch_index(
+            destination=destination,
+            arch_digests=arch_digests,
+            username=os.environ.get("HARBOR_USER", ""),
+            password=os.environ.get("HARBOR_PASSWORD", ""),
+        )
+        args.digest_file.write_text(result["index_digest"] + "\n", encoding="utf-8")
+        args.image_file.write_text(
+            f"{destination}@{result['index_digest']}\n", encoding="utf-8"
+        )
+    except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc:
+        print(json.dumps({"error": str(exc)}, sort_keys=True))
+        return 1
+    print(json.dumps(result, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover - exercised through main()
+    raise SystemExit(main())

dockerfiles/Dockerfile.hermes-agent

@@ -1,6 +1,16 @@
 # syntax=docker/dockerfile:1
 # dockerfiles/Dockerfile.hermes-agent
-FROM nousresearch/hermes-agent@sha256:47d4bd4cc420b70e40ed75efdade373e45b86b7382d4013a054208982bb6ba08
+#
+# Multi-arch base: this digest is the upstream OCI image INDEX for tag
+# v2026.7.7.2 (revision 9de9c25f620ff7f1ce0fd5457d596052d5159596). The index
+# fans out to both native leaves of the same reviewed upstream version:
+#   linux/arm64 -> sha256:47d4bd4cc420b70e40ed75efdade373e45b86b7382d4013a054208982bb6ba08
+#   linux/amd64 -> sha256:3db34ce19adfa080736a2a3feb0316dbcccc588faa9afe7fd8ae1c03b4f1a53a
+# The arm64 leaf is byte-for-byte the previously pinned single-arch base, so the
+# arm64 build is unchanged; Kaniko/containerd auto-selects the matching leaf per
+# build platform (arm64 rpi5 pod vs amd64 titan-24 pod). Do NOT replace this with
+# a per-arch leaf digest -- that would break the amd64 build leg.
+FROM nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973
 
 USER root
 

docs/hermes_agent_multiarch.md

@@ -0,0 +1,156 @@
+# hermes-agent multi-arch rollout runbook
+
+Goal: build the `hermes-agent` container image for **both** `linux/arm64` and
+`linux/amd64` so the `hermes-agent` workload can eventually run on the amd64
+node `titan-22`, while the arm64 pi fleet remains a valid fallback.
+
+This branch (`feature/hermes-agent-multiarch`) contains everything needed. The
+steps below are the **operator** procedure, in order. Do not reorder: the
+titan-22 runtime flip is deliberately the last step and only after a validated
+multi-arch image exists.
+
+---
+
+## What changed on the branch (for reviewers)
+
+| Area | File | Change |
+| --- | --- | --- |
+| Base image | `dockerfiles/Dockerfile.hermes-agent` | `FROM` repointed from the arm64-only leaf `sha256:47d4bd4c…` to the upstream **multi-arch OCI index** `sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973` (tag `v2026.7.7.2`, revision `9de9c25f…`). Same reviewed upstream version; arm64 leaf is byte-identical to the old pin. |
+| Pipeline | `ci/Jenkinsfile.hermes-agent-image` | arm64 kaniko leg now pushes `…-build-<N>-arm64`; new amd64 kaniko leg on a titan-24-pinned, resource-capped pod pushes `…-build-<N>-amd64`; new `Combine multi-arch index` stage; evidence archive expanded. |
+| Combiner | `ci/scripts/hermes_multiarch_combine.py` | New fail-closed pure-python step that verifies both leaves and publishes/re-verifies a Docker manifest **list** at the arch-less `…-build-<N>` tag, emitting the index digest in the existing digest-file/image-file format. |
+| Promotion | `ci/scripts/hermes_oci_promote.py` | **Unchanged.** Docker manifest lists are already inside its `MANIFEST_TYPES` allow-list, so it promotes the multi-arch INDEX to `…-release` with no edit. |
+| Release scripts | `ci/scripts/hermes_image_release.py` | **Unchanged.** The combiner hands it the arch-less tag + index digest, exactly the shape it already validates/renders. |
+| Worker join | `infrastructure/core/node-prefer-noschedule-cronjob.yaml` | `clear_worker titan-24 amd64` + a soft `PreferNoSchedule` guard taint protecting the co-hosted Sui validator. |
+| Runtime flip (LAST) | `services/hermes/agent-deployment.yaml` | Adds titan-22 to the allowed set with a soft preference + media-taint toleration, keeping the arm64 fleet as fallback. **Prepared as a separate commit — apply only after validation.** |
+
+### Correction to the original assumption
+
+Worker/arch/taint labels in this cluster are **not** managed by Ansible. They are
+reconciled every minute by the Flux-managed `node-prefer-noschedule` CronJob
+(`kubectl label`/`kubectl taint`). `hosts/` Ansible is a thin skeleton and
+titan-24 is already present there. Therefore the "worker-join" change lives in
+that CronJob, and it is applied by **Flux at merge**, not by a hardware playbook
+run. The only genuine hardware/operator action is ensuring titan-24 is
+uncordoned (it is on the Ananke recovery uncordon denylist).
+
+---
+
+## Step (a) — Prepare titan-24 as a schedulable amd64 worker
+
+The worker label + guard taint ship in this branch's CronJob change and land
+automatically once Flux reconciles after merge (step b). Before the first amd64
+build, the operator must make sure titan-24 can actually accept the pinned pod:
+
+```sh
+# On a control-plane host / with cluster admin kubeconfig:
+kubectl get node titan-24 -o wide            # confirm Ready, amd64
+kubectl uncordon titan-24                     # recovery keeps it cordoned by default
+# The Sui validator runs as docker-compose at /opt/sui-validator (outside k8s);
+# it is protected by the build pod's tight resource caps, not by k8s priority.
+```
+
+After merge (step b), confirm the CronJob applied the labels/taint:
+
+```sh
+kubectl get node titan-24 --show-labels | tr ',' '\n' | grep -E 'worker|hardware'
+# expect: node-role.kubernetes.io/worker=true and hardware=amd64
+kubectl describe node titan-24 | grep -i taint
+# expect: atlas.bstein.dev/sui-validator=true:PreferNoSchedule
+```
+
+If titan-24 must NOT be labeled yet, hold the merge; the labels are inert until
+the pod is scheduled and the guard taint only softly repels routine pods.
+
+## Step (b) — Merge the branch to main
+
+Merge `feature/hermes-agent-multiarch` to `main` **without** the final titan-22
+flip commit if you want to stage the runtime change separately (see step d). The
+Dockerfile, Jenkinsfile, combiner, tests, and CronJob are safe to merge: the
+arm64 build path is unchanged and the amd64 leg + combine only add new,
+fail-closed artifacts. Flux then applies the CronJob (titan-24 becomes a
+worker).
+
+## Step (c) — Trigger ONE validation build
+
+Trigger the existing manual `hermes-agent-image` job exactly as today (the same
+`PUBLISH_IMAGE=true`, `EXPECTED_SOURCE_REVISION=<full reviewed sha on main>`,
+`CONFIRM_PUBLISH=PUBLISH HERMES AGENT` gate). Nothing about the trigger changes.
+
+Success looks like:
+
+1. **arm64 leg** builds on an rpi5 pod and pushes `…-build-<N>-arm64`.
+2. **amd64 leg** builds on titan-24 and pushes `…-build-<N>-amd64`.
+3. **Combine multi-arch index** prints `"result": "published"` with an
+   `index_digest`, and the registry resolves `…-build-<N>` to a Docker manifest
+   list referencing exactly the two leaves.
+4. Render / Verify-and-archive / Publish stages behave exactly as before but on
+   the **index** digest; the `…-build-<N>-release` tag is the multi-arch index.
+5. Archived evidence now includes both per-arch digests/images plus the index
+   digest/image.
+
+Independent confirmation the release is multi-arch:
+
+```sh
+skopeo inspect --raw docker://registry.bstein.dev/bstein/hermes-agent:git-<sha>-build-<N>-release \
+  | jq -r '.mediaType, (.manifests[] | "\(.platform.os)/\(.platform.architecture) \(.digest)")'
+# expect: application/vnd.docker.distribution.manifest.list.v2+json
+#         linux/amd64 sha256:…
+#         linux/arm64 sha256:…
+```
+
+Flux will roll the `hermes-agent` deployment to the new index digest. It still
+runs on arm64 (unchanged affinity), and arm64 nodes keep pulling the arm64 leaf
+— so this step is safe even though the runtime is still arm64-only.
+
+## Step (d) — Apply the titan-22 runtime affinity flip
+
+**Only after step (c) proves the multi-arch image.** Apply the final, separately
+messaged commit `APPLY ONLY AFTER multi-arch image validated: allow hermes-agent
+on titan-22` (it edits `services/hermes/agent-deployment.yaml` and the matching
+layout test). It:
+
+- adds a second `nodeSelectorTerm` matching amd64 `titan-22` (worker) — the
+  arm64 fleet term is untouched, so the pi fleet remains a valid fallback;
+- adds a soft `preferredDuringScheduling` nudge toward titan-22;
+- adds a toleration for titan-22's `atlas.bstein.dev/media-primary=true:PreferNoSchedule`
+  taint. This does **not** change jellyfin's own scheduling or priority — a
+  toleration only lets hermes-agent *also* consider titan-22 as spare capacity.
+
+Merge that commit; Flux recreates the pod (deployment strategy is `Recreate`).
+
+## Step (e) — Verify the worker lands correctly
+
+```sh
+kubectl -n hermes get pod -l app=hermes-agent -o wide
+# It may still land on an rpi5 (fallback is intact and pis are preferred-eligible);
+# it is now ALSO permitted on titan-22. To confirm titan-22 can host it, cordon
+# the pi fleet briefly or scale/drain, or simply confirm the pod tolerates the
+# media taint and schedules when pis are unavailable.
+kubectl -n hermes describe pod -l app=hermes-agent | grep -iE 'node:|toleration'
+```
+
+To roll back the runtime flip, revert the step-(d) commit; the arm64-only
+affinity returns and the multi-arch image keeps working on arm64.
+
+---
+
+## What is validated locally vs what only a real build proves
+
+**Validated locally on this branch:**
+- The base index digest resolves to both `linux/arm64` (the old pin) and
+  `linux/amd64`, same revision `9de9c25f…` (`skopeo inspect --raw`).
+- `hermes_multiarch_combine.py` end-to-end unit tests (full hash + arch-config
+  verification chain, idempotency, and every fail-closed branch).
+- The updated + new image-builder tests, `hermes_oci_promote`, and the automation
+  tests pass.
+- `kustomize build services/hermes` and `kustomize build infrastructure/core`
+  succeed.
+
+**Only a real Jenkins build can prove:**
+- Kaniko auto-selects the correct native leaf from the index on each arch, and
+  the amd64 heredoc-compat replay produces a working amd64 image.
+- The amd64 pod actually schedules on titan-24 within its resource cap without
+  starving the Sui validator.
+- Harbor accepts the manually-assembled manifest list PUT and immutability
+  applies to the index tag.
+- Cross-pod `stash`/`unstash` of the amd64 leaf evidence behaves as expected.

infrastructure/core/node-prefer-noschedule-cronjob.yaml

@@ -59,6 +59,7 @@ spec:
                   clear_worker titan-14 rpi4
                   clear_worker titan-18 rpi4
                   clear_worker titan-22 amd64
+                  clear_worker titan-24 amd64
 
                   if k get node titan-22 >/dev/null 2>&1; then
                     k label node titan-22 \
@@ -69,6 +70,21 @@ spec:
                     k taint node titan-22 atlas.bstein.dev/media-primary=true:PreferNoSchedule --overwrite=true || true
                   fi
 
+                  if k get node titan-24 >/dev/null 2>&1; then
+                    # titan-24 (amd64, RTX 3080) co-hosts the out-of-cluster Sui
+                    # validator (docker-compose at /opt/sui-validator). It joins as a
+                    # worker so the native amd64 hermes-agent image leg can build here,
+                    # but a soft guard taint keeps routine pods from crowding the
+                    # validator. GPU workloads pinned to titan-24 by hostname are
+                    # unaffected (PreferNoSchedule never blocks a pinned pod); the
+                    # amd64 build pod tolerates this taint explicitly.
+                    k label node titan-24 \
+                      atlas.bstein.dev/general-compute=last-resort \
+                      atlas.bstein.dev/sui-validator=true \
+                      --overwrite=true || true
+                    k taint node titan-24 atlas.bstein.dev/sui-validator=true:PreferNoSchedule --overwrite=true || true
+                  fi
+
                   if k get node titan-23 >/dev/null 2>&1; then
                     k label node titan-23 \
                       veles.bstein.dev/simulation=true \

testing/tests/test_hermes_image_builder.py

@@ -113,7 +113,8 @@ def test_pipeline_requires_reviewed_main_and_runtime_credentials() -> None:
     assert "--destination=" in source
     assert "assert-absent" in source
     assert "git-${actual_revision}-build-${BUILD_NUMBER}" in source
-    assert source.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1
+    # One reviewed heredoc-compat build-arg per native Kaniko leg (arm64 + amd64).
+    assert source.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 2
 
 
 def test_kaniko_replays_only_the_exact_reviewed_heredoc_contract() -> None:

testing/tests/test_hermes_image_builder_fresh_review.py

@@ -32,6 +32,15 @@ def _pod_spec() -> dict:
     return yaml.safe_load(pod_yaml)["spec"]
 
 
+def _amd64_pod_spec() -> dict:
+    """Parse the second inline pod (the native amd64 build leg on titan-24)."""
+    source = PIPELINE.read_text(encoding="utf-8")
+    blocks = source.split('yaml """')
+    assert len(blocks) == 3, "expected exactly the arm64 and amd64 build pods"
+    pod_yaml = blocks[2].split('"""', 1)[0]
+    return yaml.safe_load(pod_yaml)["spec"]
+
+
 def test_builder_prefers_rpi5_with_healthy_arm64_worker_fallback() -> None:
     """Disposable builds prefer rpi5 without excluding schedulable rpi4 workers."""
     spec = _pod_spec()
@@ -106,7 +115,8 @@ def test_exact_reviewed_heredocs_remain_buildkit_native_by_default() -> None:
     assert source.count("ARG HERMES_KANIKO_HEREDOC_COMPAT=0") == 1
     assert "Kaniko v1.23.2" in RUNNER.read_text(encoding="utf-8")
     pipeline = PIPELINE.read_text(encoding="utf-8")
-    assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1
+    # The reviewed heredoc replay is enabled on both native Kaniko legs.
+    assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 2
 
 
 def test_appended_tenth_reviewed_form_rejects_before_execution(
@@ -338,11 +348,91 @@ def test_pipeline_requires_and_archives_exact_release_evidence() -> None:
     assert "allowEmptyArchive: false" in evidence
     archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0]
     paths = archive.split(",")
-    assert len(paths) == 7
-    assert len(set(paths)) == 7
+    # Multi-arch adds both per-arch leaf digests/images plus the index digest.
+    assert len(paths) == 11
+    assert len(set(paths)) == 11
     assert all("*" not in path for path in paths)
     assert "find build -type f" in evidence
     assert "build/hermes-agent.source-revision" in paths
+    assert "build/hermes-agent.digest" in paths
+    for arch in ("arm64", "amd64"):
+        assert f"build/hermes-agent-{arch}.digest" in paths
+        assert f"build/hermes-agent-{arch}.image" in paths
     promotion = source.split("stage('Publish Flux release tag')", 1)[1]
     assert "ci/scripts/hermes_oci_promote.py" in promotion
     assert "  post {" not in source
+
+
+def test_amd64_leg_is_pinned_to_titan24_and_resource_capped() -> None:
+    """The amd64 build leg lands on titan-24, tolerates its guard taint, and is capped."""
+    spec = _amd64_pod_spec()
+    assert spec["nodeSelector"] == {
+        "kubernetes.io/arch": "amd64",
+        "kubernetes.io/hostname": "titan-24",
+        "node-role.kubernetes.io/worker": "true",
+    }
+    # titan-24 co-hosts the Sui validator; the disposable build must tolerate
+    # whatever guard taint the node carries so the pinned pod still schedules.
+    assert {"operator": "Exists"} in spec["tolerations"]
+    assert spec["serviceAccountName"] == "hermes-image-builder"
+    assert spec["automountServiceAccountToken"] is False
+    assert spec["enableServiceLinks"] is False
+
+    containers = {item["name"]: item for item in spec["containers"]}
+    kaniko = containers["kaniko"]
+    # Identical pinned Kaniko across both legs -- no second, unreviewed builder.
+    assert kaniko["image"] == (
+        "gcr.io/kaniko-project/executor@sha256:"
+        "c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e"
+    )
+    assert kaniko["securityContext"]["capabilities"]["add"] == [
+        "CHOWN",
+        "FOWNER",
+        "DAC_OVERRIDE",
+        "SETGID",
+        "SETUID",
+    ]
+    for container in containers.values():
+        security = container["securityContext"]
+        assert security["allowPrivilegeEscalation"] is False
+        assert security["capabilities"]["drop"] == ["ALL"]
+        assert security["seccompProfile"]["type"] == "RuntimeDefault"
+        assert security.get("privileged", False) is False
+    # Tight caps keep the disposable build from starving the co-hosted validator:
+    # the amd64 kaniko ceiling is strictly below the arm64 leg's 2 CPU / 4Gi.
+    limits = kaniko["resources"]["limits"]
+    assert limits["cpu"] == "1500m"
+    assert limits["memory"] == "3Gi"
+
+
+def test_amd64_leg_source_is_independently_boundary_checked() -> None:
+    """The amd64 pod re-derives and re-verifies the reviewed revision itself."""
+    source = PIPELINE.read_text(encoding="utf-8")
+    amd64_stage = source.split("stage('Build amd64 leg without a daemon')", 1)[1]
+    amd64_stage = amd64_stage.split("stage('Combine multi-arch index')", 1)[0]
+    # Same fail-closed boundary as the coordinating pod, re-run in the amd64 pod.
+    assert 'merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}"' in amd64_stage
+    assert 'checkout --detach "${EXPECTED_SOURCE_REVISION}"' in amd64_stage
+    assert 'test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"' in amd64_stage
+    assert 'status --porcelain' in amd64_stage
+    assert '$(cat build/hermes-agent.destination)-amd64' in amd64_stage
+    # The amd64 leaf evidence crosses pods only through an explicit stash.
+    assert "stash" in amd64_stage
+    assert "hermes-agent-amd64.digest" in amd64_stage
+
+
+def test_combine_stage_publishes_and_reverifies_the_index() -> None:
+    """Kaniko cannot combine; the reviewed python combiner assembles the index."""
+    source = PIPELINE.read_text(encoding="utf-8")
+    combine = source.split("stage('Combine multi-arch index')", 1)[1]
+    combine = combine.split("stage('Render reviewed Flux handoff')", 1)[0]
+    assert "unstash 'hermes-agent-amd64-evidence'" in combine
+    assert "ci/scripts/hermes_multiarch_combine.py" in combine
+    assert "--arm64-digest-file build/hermes-agent-arm64.digest" in combine
+    assert "--amd64-digest-file build/hermes-agent-amd64.digest" in combine
+    # The combiner emits the arch-less index evidence the existing chain promotes.
+    assert "--digest-file build/hermes-agent.digest" in combine
+    assert "--image-file build/hermes-agent.image" in combine
+    # Downstream render/verify/promote still consume the single index digest file.
+    render = source.split("stage('Render reviewed Flux handoff')", 1)[1]
+    assert "--digest-file build/hermes-agent.digest" in render

testing/tests/test_hermes_multiarch_combine.py

@@ -0,0 +1,385 @@
+"""Safety tests for the fail-closed multi-arch manifest-list combiner."""
+
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import io
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = ROOT / "ci/scripts/hermes_multiarch_combine.py"
+REVISION = "a" * 40
+BUILD = "17"
+DESTINATION = f"registry.bstein.dev/bstein/hermes-agent:git-{REVISION}-build-{BUILD}"
+DOCKER_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json"
+DOCKER_MANIFEST = "application/vnd.docker.distribution.manifest.v2+json"
+IMAGE_CONFIG = "application/vnd.docker.container.image.v1+json"
+
+
+def _load():
+    spec = importlib.util.spec_from_file_location("hermes_multiarch_combine", SCRIPT)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _sha(body: bytes) -> str:
+    return "sha256:" + hashlib.sha256(body).hexdigest()
+
+
+class Response(io.BytesIO):
+    """Minimal context-managed urllib response."""
+
+    def __init__(self, status: int, body: bytes = b"", headers=None):
+        super().__init__(body)
+        self.status = status
+        self.headers = headers or {}
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_args):
+        self.close()
+
+
+def _leaf(architecture: str, *, os_name: str = "linux"):
+    """Build a self-consistent config+manifest whose bytes hash to real digests."""
+    config_body = json.dumps(
+        {"architecture": architecture, "os": os_name}, sort_keys=True
+    ).encode()
+    config_digest = _sha(config_body)
+    manifest_body = json.dumps(
+        {
+            "schemaVersion": 2,
+            "mediaType": DOCKER_MANIFEST,
+            "config": {
+                "mediaType": IMAGE_CONFIG,
+                "digest": config_digest,
+                "size": len(config_body),
+            },
+            "layers": [],
+        },
+        sort_keys=True,
+    ).encode()
+    return {
+        "arch": architecture,
+        "config_body": config_body,
+        "config_digest": config_digest,
+        "manifest_body": manifest_body,
+        "digest": _sha(manifest_body),
+        "content_type": DOCKER_MANIFEST,
+    }
+
+
+def _descriptor(module, leaf):
+    return {
+        "mediaType": leaf["content_type"],
+        "size": len(leaf["manifest_body"]),
+        "digest": leaf["digest"],
+        "platform": {"architecture": leaf["arch"], "os": "linux"},
+    }
+
+
+def _index_digest(module, amd64, arm64) -> str:
+    body = module._manifest_list_bytes(
+        [_descriptor(module, amd64), _descriptor(module, arm64)]
+    )
+    return _sha(body)
+
+
+class Registry:
+    """Route the combiner's deterministic request sequence by method and path."""
+
+    def __init__(self, module, amd64, arm64, *, head_status=404, existing="") -> None:
+        self.module = module
+        self.leaves = {amd64["arch"]: amd64, arm64["arch"]: arm64}
+        self.index_digest = _index_digest(module, amd64, arm64)
+        self.head_status = head_status
+        self.existing = existing
+        self.put_body = None
+        self.calls = []
+
+    def __call__(self, request, timeout):
+        method = request.method
+        url = request.full_url
+        self.calls.append((method, url))
+        for arch, leaf in self.leaves.items():
+            if url.endswith(f"-{arch}"):
+                return Response(
+                    200,
+                    leaf["manifest_body"],
+                    {
+                        "Docker-Content-Digest": leaf["digest"],
+                        "Content-Type": leaf["content_type"],
+                    },
+                )
+            # Blob URLs percent-encode the ``sha256:`` colon; match the raw hex.
+            if url.endswith(leaf["config_digest"].split(":", 1)[1]):
+                return Response(200, leaf["config_body"], {})
+        # Final index tag (no arch suffix, ends with the build tag).
+        if method == "HEAD":
+            return Response(
+                self.head_status,
+                headers={"Docker-Content-Digest": self.existing},
+            )
+        if method == "PUT":
+            self.put_body = request.data
+            return Response(201, headers={"Docker-Content-Digest": self.index_digest})
+        return Response(
+            200,
+            self.module._manifest_list_bytes(
+                [
+                    _descriptor(self.module, self.leaves["amd64"]),
+                    _descriptor(self.module, self.leaves["arm64"]),
+                ]
+            ),
+            {
+                "Docker-Content-Digest": self.index_digest,
+                "Content-Type": DOCKER_MANIFEST_LIST,
+            },
+        )
+
+
+def _combine(module, registry, arch_digests=None):
+    if arch_digests is None:
+        arch_digests = {
+            "amd64": registry.leaves["amd64"]["digest"],
+            "arm64": registry.leaves["arm64"]["digest"],
+        }
+    return module.combine_multiarch_index(
+        destination=DESTINATION,
+        arch_digests=arch_digests,
+        username="robot",
+        password="private",
+        opener=registry,
+    )
+
+
+def test_combines_two_verified_leaves_into_one_index() -> None:
+    """Both native leaves are re-read, arch-proven, and published as a list."""
+    module = _load()
+    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
+    registry = Registry(module, amd64, arm64)
+    result = _combine(module, registry)
+    assert result["result"] == "published"
+    assert result["index_digest"] == registry.index_digest
+    assert result["amd64_digest"] == amd64["digest"]
+    assert result["arm64_digest"] == arm64["digest"]
+    # The published bytes are exactly what we hashed for the index digest.
+    assert _sha(registry.put_body) == registry.index_digest
+    assert "private" not in json.dumps(result)
+
+
+def test_idempotent_when_index_already_matches() -> None:
+    """A replay is accepted only when the existing index digest is identical."""
+    module = _load()
+    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
+    index_digest = _index_digest(module, amd64, arm64)
+    registry = Registry(
+        module, amd64, arm64, head_status=200, existing=index_digest
+    )
+    result = _combine(module, registry)
+    assert result["result"] == "already-present"
+    assert registry.put_body is None
+
+
+def test_existing_index_with_other_digest_fails_closed() -> None:
+    """An occupied final tag with a different index digest never republishes."""
+    module = _load()
+    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
+    registry = Registry(
+        module, amd64, arm64, head_status=200, existing="sha256:" + "c" * 64
+    )
+    with pytest.raises(RuntimeError, match="another index digest"):
+        _combine(module, registry)
+
+
+def test_rejects_leaf_digest_that_disagrees_with_evidence() -> None:
+    """A leaf whose registry digest is not the Kaniko evidence fails closed."""
+    module = _load()
+    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
+    registry = Registry(module, amd64, arm64)
+    with pytest.raises(RuntimeError, match="amd64 leaf digest"):
+        _combine(
+            module,
+            registry,
+            arch_digests={"amd64": "sha256:" + "d" * 64, "arm64": arm64["digest"]},
+        )
+
+
+def test_rejects_leaf_whose_config_architecture_is_wrong() -> None:
+    """A leaf that claims the wrong architecture in its config fails closed."""
+    module = _load()
+    # Build an "amd64" candidate tag whose config actually says arm64.
+    swapped = _leaf("arm64")
+    swapped["arch"] = "amd64"  # served under the amd64 tag, but arm64 inside
+    arm64 = _leaf("arm64")
+    registry = Registry(module, swapped, arm64)
+    with pytest.raises(RuntimeError, match="architecture"):
+        _combine(
+            module,
+            registry,
+            arch_digests={"amd64": swapped["digest"], "arm64": arm64["digest"]},
+        )
+
+
+def test_rejects_leaf_served_as_a_manifest_list() -> None:
+    """A per-arch leaf must be a single image, never itself an index/list."""
+    module = _load()
+    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
+    amd64["content_type"] = DOCKER_MANIFEST_LIST
+    registry = Registry(module, amd64, arm64)
+    with pytest.raises(RuntimeError, match="single-image manifest"):
+        _combine(module, registry)
+
+
+def test_requires_both_architectures() -> None:
+    """The combiner refuses any arch set other than exactly arm64 and amd64."""
+    module = _load()
+    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
+    registry = Registry(module, amd64, arm64)
+    with pytest.raises(ValueError, match="arm64 and amd64"):
+        module.combine_multiarch_index(
+            destination=DESTINATION,
+            arch_digests={"amd64": amd64["digest"]},
+            username="robot",
+            password="private",
+            opener=registry,
+        )
+
+
+def test_rejects_destination_with_arch_suffix() -> None:
+    """Only the arch-less final tag is a valid combine destination."""
+    module = _load()
+    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
+    with pytest.raises(ValueError, match="invalid multi-arch destination"):
+        module.combine_multiarch_index(
+            destination=f"{DESTINATION}-arm64",
+            arch_digests={"amd64": amd64["digest"], "arm64": arm64["digest"]},
+            username="robot",
+            password="private",
+            opener=Registry(module, amd64, arm64),
+        )
+
+
+def test_cli_binds_evidence_files_and_writes_index_outputs(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    """The command reads both Kaniko pairs and emits arch-less index evidence."""
+    module = _load()
+    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
+    registry = Registry(module, amd64, arm64)
+    files = {}
+    for arch, leaf in (("amd64", amd64), ("arm64", arm64)):
+        digest_file = tmp_path / f"{arch}.digest"
+        image_file = tmp_path / f"{arch}.image"
+        digest_file.write_text(leaf["digest"] + "\n", encoding="utf-8")
+        image_file.write_text(
+            f"{DESTINATION}-{arch}@{leaf['digest']}\n", encoding="utf-8"
+        )
+        files[arch] = (digest_file, image_file)
+    out_digest = tmp_path / "index.digest"
+    out_image = tmp_path / "index.image"
+    monkeypatch.setenv("HARBOR_USER", "robot")
+    monkeypatch.setenv("HARBOR_PASSWORD", "private")
+    captured = {}
+
+    def fake_combine(*, destination, arch_digests, username, password):
+        captured["destination"] = destination
+        captured["arch_digests"] = dict(arch_digests)
+        return {"index_digest": registry.index_digest, "result": "published"}
+
+    monkeypatch.setattr(module, "combine_multiarch_index", fake_combine)
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            "hermes_multiarch_combine.py",
+            "--destination",
+            DESTINATION,
+            "--source-revision",
+            REVISION,
+            "--build-number",
+            BUILD,
+            "--arm64-digest-file",
+            str(files["arm64"][0]),
+            "--arm64-image-file",
+            str(files["arm64"][1]),
+            "--amd64-digest-file",
+            str(files["amd64"][0]),
+            "--amd64-image-file",
+            str(files["amd64"][1]),
+            "--digest-file",
+            str(out_digest),
+            "--image-file",
+            str(out_image),
+        ],
+    )
+    assert module.main() == 0
+    assert out_digest.read_text(encoding="utf-8").strip() == registry.index_digest
+    assert (
+        out_image.read_text(encoding="utf-8").strip()
+        == f"{DESTINATION}@{registry.index_digest}"
+    )
+    assert json.loads(capsys.readouterr().out)["result"] == "published"
+    # main() must bind each arch's two evidence files to the correct leaf digest.
+    assert captured["destination"] == DESTINATION
+    assert captured["arch_digests"] == {
+        "amd64": amd64["digest"],
+        "arm64": arm64["digest"],
+    }
+
+
+def test_cli_rejects_mismatched_per_arch_evidence_pair(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    """A digest/image evidence file mismatch is reported as a JSON error."""
+    module = _load()
+    amd64, arm64 = _leaf("amd64"), _leaf("arm64")
+    files = {}
+    for arch, leaf in (("amd64", amd64), ("arm64", arm64)):
+        digest_file = tmp_path / f"{arch}.digest"
+        image_file = tmp_path / f"{arch}.image"
+        digest_file.write_text(leaf["digest"] + "\n", encoding="utf-8")
+        # amd64 image file points at the wrong digest.
+        recorded = "sha256:" + "e" * 64 if arch == "amd64" else leaf["digest"]
+        image_file.write_text(
+            f"{DESTINATION}-{arch}@{recorded}\n", encoding="utf-8"
+        )
+        files[arch] = (digest_file, image_file)
+    monkeypatch.setenv("HARBOR_USER", "robot")
+    monkeypatch.setenv("HARBOR_PASSWORD", "private")
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            "hermes_multiarch_combine.py",
+            "--destination",
+            DESTINATION,
+            "--source-revision",
+            REVISION,
+            "--build-number",
+            BUILD,
+            "--arm64-digest-file",
+            str(files["arm64"][0]),
+            "--arm64-image-file",
+            str(files["arm64"][1]),
+            "--amd64-digest-file",
+            str(files["amd64"][0]),
+            "--amd64-image-file",
+            str(files["amd64"][1]),
+            "--digest-file",
+            str(tmp_path / "index.digest"),
+            "--image-file",
+            str(tmp_path / "index.image"),
+        ],
+    )
+    assert module.main() == 1
+    assert "does not match" in json.loads(capsys.readouterr().out)["error"]