release(hermes): automate validated image promotion

ci/Jenkinsfile.hermes-agent-image

@@ -114,7 +114,7 @@ spec:
     string(
       name: 'EXPECTED_SOURCE_REVISION',
       defaultValue: '',
-      description: 'Exact 40-character commit on atlas/titan-iac main.'
+      description: 'Full reviewed commit that must be contained by atlas/titan-iac main.'
     )
     string(
       name: 'CONFIRM_PUBLISH',
@@ -152,9 +152,11 @@ spec:
               ;;
           esac
           test "${#EXPECTED_SOURCE_REVISION}" -eq 40
+          git fetch --no-tags origin main
+          git checkout --detach origin/main
           actual_revision="$(git rev-parse HEAD)"
-          test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
           test "${actual_revision}" = "$(git rev-parse origin/main)"
+          git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${actual_revision}"
           test -z "$(git status --porcelain)"
           test -f dockerfiles/Dockerfile.hermes-agent
           case "${BUILD_NUMBER}" in
@@ -166,10 +168,31 @@ spec:
           printf '%s\n' \
             "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
             > build/hermes-agent.destination
+          printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision
         '''
         }
       }
     }
+    stage('Validate reviewed release source') {
+      steps {
+        container('python') {
+          sh '''
+            set -eu
+            python3 -m pip install --disable-pip-version-check --no-cache-dir \
+              --target=/tmp/hermes-agent-release-test-deps \
+              pytest==8.3.4 PyYAML==6.0.2
+            PYTHONPATH=/tmp/hermes-agent-release-test-deps \
+              python3 -m pytest -q \
+              testing/tests/test_hermes_image_builder.py \
+              testing/tests/test_hermes_image_builder_adversarial.py \
+              testing/tests/test_hermes_image_builder_coverage.py \
+              testing/tests/test_hermes_image_builder_fresh_review.py \
+              testing/tests/test_hermes_oci_promote.py \
+              testing/tests/test_hermes_image_automation.py
+          '''
+        }
+      }
+    }
     stage('Reject replay before publish') {
       steps {
         withCredentials([usernamePassword(
@@ -181,8 +204,9 @@ spec:
             set -eu
             set +x
             destination="$(cat build/hermes-agent.destination)"
+            source_revision="$(cat build/hermes-agent.source-revision)"
             python3 ci/scripts/hermes_image_release.py assert-absent \
-              --source-revision "${EXPECTED_SOURCE_REVISION}" \
+              --source-revision "${source_revision}" \
               --build-number "${BUILD_NUMBER}" \
               --destination "${destination}"
           '''
@@ -202,6 +226,7 @@ spec:
               set +x
               config_path=/kaniko/.docker/config.json
               destination="$(cat build/hermes-agent.destination)"
+              source_revision="$(cat build/hermes-agent.source-revision)"
               umask 077
               auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
               /busybox/mkdir -p /kaniko/.docker
@@ -218,7 +243,7 @@ spec:
                 --digest-file="${WORKSPACE}/build/hermes-agent.digest" \
                 --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \
                 --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
-                --label="org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" \
+                --label="org.opencontainers.image.revision=${source_revision}" \
                 --cleanup \
                 --push-retry=3
               /busybox/chmod 644 build/hermes-agent.digest build/hermes-agent.image
@@ -238,10 +263,11 @@ spec:
             set -eu
             set +x
             destination="$(cat build/hermes-agent.destination)"
+            source_revision="$(cat build/hermes-agent.source-revision)"
             python3 ci/scripts/hermes_image_release.py render \
               --digest-file build/hermes-agent.digest \
               --image-file build/hermes-agent.image \
-              --source-revision "${EXPECTED_SOURCE_REVISION}" \
+              --source-revision "${source_revision}" \
               --build-number "${BUILD_NUMBER}" \
               --destination "${destination}" \
               --kustomization services/hermes/kustomization.yaml \
@@ -252,15 +278,15 @@ spec:
         }
       }
     }
-  }
+    stage('Verify and archive release evidence') {
-  post {
+      steps {
-    success {
       sh '''
         set -eu
         expected_files="$(printf '%s\n' \
           build/hermes-agent.destination \
           build/hermes-agent.digest \
           build/hermes-agent.image \
+          build/hermes-agent.source-revision \
           build/hermes-agent-release/hermes-agent-image.json \
           build/hermes-agent-release/hermes-image-update.patch \
           build/hermes-agent-release/hermes-kustomization.yaml \
@@ -268,26 +294,42 @@ spec:
         actual_files="$(find build -type f -print | LC_ALL=C sort)"
         test "${actual_files}" = "${expected_files}"
         destination="$(cat build/hermes-agent.destination)"
+        source_revision="$(cat build/hermes-agent.source-revision)"
         python3 ci/scripts/hermes_image_release.py verify-evidence \
           --digest-file build/hermes-agent.digest \
           --image-file build/hermes-agent.image \
-          --source-revision "${EXPECTED_SOURCE_REVISION}" \
+          --source-revision "${source_revision}" \
           --build-number "${BUILD_NUMBER}" \
           --destination "${destination}" \
           --kustomization services/hermes/kustomization.yaml \
           --output-dir build/hermes-agent-release
       '''
       archiveArtifacts(
-        artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
+        artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.source-revision,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
         allowEmptyArchive: false,
         fingerprint: true
       )
+      }
     }
-    cleanup {
+    stage('Publish Flux release tag') {
-      container('kaniko') {
+      steps {
-        sh '''#!/busybox/sh
+        withCredentials([usernamePassword(
-          /busybox/rm -f /kaniko/.docker/config.json
+          credentialsId: 'harbor-robot',
-        '''
+          usernameVariable: 'HARBOR_USER',
+          passwordVariable: 'HARBOR_PASSWORD'
+        )]) {
+          sh '''
+            set -eu
+            set +x
+            destination="$(cat build/hermes-agent.destination)"
+            source_revision="$(cat build/hermes-agent.source-revision)"
+            python3 ci/scripts/hermes_oci_promote.py \
+              --destination "${destination}" \
+              --digest-file build/hermes-agent.digest \
+              --source-revision "${source_revision}" \
+              --build-number "${BUILD_NUMBER}"
+          '''
+        }
       }
     }
   }

ci/Jenkinsfile.hermes-webui-image

@@ -114,7 +114,7 @@ spec:
     string(
       name: 'EXPECTED_SOURCE_REVISION',
       defaultValue: '',
-      description: 'Exact 40-character commit on atlas/titan-iac main.'
+      description: 'Full reviewed commit that must be contained by atlas/titan-iac main.'
     )
     string(
       name: 'CONFIRM_PUBLISH',
@@ -152,9 +152,11 @@ spec:
               ;;
           esac
           test "${#EXPECTED_SOURCE_REVISION}" -eq 40
+          git fetch --no-tags origin main
+          git checkout --detach origin/main
           actual_revision="$(git rev-parse HEAD)"
-          test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
           test "${actual_revision}" = "$(git rev-parse origin/main)"
+          git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${actual_revision}"
           test -z "$(git status --porcelain)"
           test -f dockerfiles/Dockerfile.hermes-webui
           case "${BUILD_NUMBER}" in
@@ -166,6 +168,7 @@ spec:
           printf '%s\n' \
             "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
             > build/hermes-webui.destination
+          printf '%s\n' "${actual_revision}" > build/hermes-webui.source-revision
         '''
         }
       }
@@ -181,7 +184,9 @@ spec:
             PYTHONPATH=/tmp/hermes-webui-release-test-deps \
               python3 -m pytest -q \
               testing/tests/test_hermes_webui_brand.py \
-              testing/tests/test_hermes_webui_release.py
+              testing/tests/test_hermes_webui_release.py \
+              testing/tests/test_hermes_oci_promote.py \
+              testing/tests/test_hermes_image_automation.py
           '''
         }
       }
@@ -197,8 +202,9 @@ spec:
             set -eu
             set +x
             destination="$(cat build/hermes-webui.destination)"
+            source_revision="$(cat build/hermes-webui.source-revision)"
             python3 ci/scripts/hermes_webui_release.py assert-absent \
-              --source-revision "${EXPECTED_SOURCE_REVISION}" \
+              --source-revision "${source_revision}" \
               --build-number "${BUILD_NUMBER}" \
               --destination "${destination}"
           '''
@@ -218,6 +224,7 @@ spec:
               set +x
               config_path=/kaniko/.docker/config.json
               destination="$(cat build/hermes-webui.destination)"
+              source_revision="$(cat build/hermes-webui.source-revision)"
               umask 077
               auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
               /busybox/mkdir -p /kaniko/.docker
@@ -233,7 +240,7 @@ spec:
                 --destination="${destination}" \
                 --digest-file="${WORKSPACE}/build/hermes-webui.digest" \
                 --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-webui.image" \
-                --label="org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" \
+                --label="org.opencontainers.image.revision=${source_revision}" \
                 --label="org.opencontainers.image.source=https://scm.bstein.dev/atlas/titan-iac" \
                 --label="org.opencontainers.image.title=hermes-webui" \
                 --cleanup \
@@ -255,10 +262,11 @@ spec:
             set -eu
             set +x
             destination="$(cat build/hermes-webui.destination)"
+            source_revision="$(cat build/hermes-webui.source-revision)"
             python3 ci/scripts/hermes_webui_release.py render \
               --digest-file build/hermes-webui.digest \
               --image-file build/hermes-webui.image \
-              --source-revision "${EXPECTED_SOURCE_REVISION}" \
+              --source-revision "${source_revision}" \
               --build-number "${BUILD_NUMBER}" \
               --destination "${destination}" \
               --chat-manifest services/hermes/chat-statefulset.yaml \
@@ -270,15 +278,15 @@ spec:
         }
       }
     }
-  }
+    stage('Verify and archive release evidence') {
-  post {
+      steps {
-    success {
       sh '''
         set -eu
         expected_files="$(printf '%s\n' \
           build/hermes-webui.destination \
           build/hermes-webui.digest \
           build/hermes-webui.image \
+          build/hermes-webui.source-revision \
           build/hermes-webui-release/hermes-chat-statefulset.yaml \
           build/hermes-webui-release/hermes-dashboard-deployment.yaml \
           build/hermes-webui-release/hermes-webui-image.json \
@@ -287,10 +295,11 @@ spec:
         actual_files="$(find build -type f -print | LC_ALL=C sort)"
         test "${actual_files}" = "${expected_files}"
         destination="$(cat build/hermes-webui.destination)"
+        source_revision="$(cat build/hermes-webui.source-revision)"
         python3 ci/scripts/hermes_webui_release.py verify-evidence \
           --digest-file build/hermes-webui.digest \
           --image-file build/hermes-webui.image \
-          --source-revision "${EXPECTED_SOURCE_REVISION}" \
+          --source-revision "${source_revision}" \
           --build-number "${BUILD_NUMBER}" \
           --destination "${destination}" \
           --chat-manifest services/hermes/chat-statefulset.yaml \
@@ -298,16 +307,31 @@ spec:
           --output-dir build/hermes-webui-release
       '''
       archiveArtifacts(
-        artifacts: 'build/hermes-webui.destination,build/hermes-webui.digest,build/hermes-webui.image,build/hermes-webui-release/hermes-chat-statefulset.yaml,build/hermes-webui-release/hermes-dashboard-deployment.yaml,build/hermes-webui-release/hermes-webui-image.json,build/hermes-webui-release/hermes-webui-image-update.patch',
+        artifacts: 'build/hermes-webui.destination,build/hermes-webui.digest,build/hermes-webui.image,build/hermes-webui.source-revision,build/hermes-webui-release/hermes-chat-statefulset.yaml,build/hermes-webui-release/hermes-dashboard-deployment.yaml,build/hermes-webui-release/hermes-webui-image.json,build/hermes-webui-release/hermes-webui-image-update.patch',
         allowEmptyArchive: false,
         fingerprint: true
       )
+      }
     }
-    cleanup {
+    stage('Publish Flux release tag') {
-      container('kaniko') {
+      steps {
-        sh '''#!/busybox/sh
+        withCredentials([usernamePassword(
-          /busybox/rm -f /kaniko/.docker/config.json
+          credentialsId: 'harbor-robot',
-        '''
+          usernameVariable: 'HARBOR_USER',
+          passwordVariable: 'HARBOR_PASSWORD'
+        )]) {
+          sh '''
+            set -eu
+            set +x
+            destination="$(cat build/hermes-webui.destination)"
+            source_revision="$(cat build/hermes-webui.source-revision)"
+            python3 ci/scripts/hermes_oci_promote.py \
+              --destination "${destination}" \
+              --digest-file build/hermes-webui.digest \
+              --source-revision "${source_revision}" \
+              --build-number "${BUILD_NUMBER}"
+          '''
+        }
       }
     }
   }

ci/scripts/hermes_image_release.py

@@ -174,9 +174,7 @@ def _normalized_immutable_rule(rule: dict[str, Any]) -> dict[str, Any]:
                     "decoration": item.get("decoration"),
                     "pattern": item.get("pattern"),
                 }
-                for item in (rule.get("scope_selectors") or {}).get(
+                for item in (rule.get("scope_selectors") or {}).get("repository", [])
-                    "repository", []
-                )
                 if isinstance(item, dict)
             ]
         },
@@ -317,7 +315,10 @@ def render_kustomization(source: str, digest: str, image: str = DEFAULT_IMAGE) -
     index = matches[0]
     newline = "\n" if lines[index].endswith("\n") else ""
     prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())]
-    lines[index] = f"{prefix}digest: {digest}{newline}"
+    value = lines[index].strip().removeprefix("digest:").strip()
+    _current_digest, separator, comment = value.partition(" #")
+    suffix = f" #{comment}" if separator else ""
+    lines[index] = f"{prefix}digest: {digest}{suffix}{newline}"
     return "".join(lines)
 
 
@@ -414,7 +415,8 @@ def validate_release_artifacts(
         "source_revision": source_revision,
     }
     expected = {
-        "hermes-agent-image.json": json.dumps(metadata, indent=2, sort_keys=True) + "\n",
+        "hermes-agent-image.json": json.dumps(metadata, indent=2, sort_keys=True)
+        + "\n",
         "hermes-image-update.patch": patch,
         "hermes-kustomization.yaml": rendered,
     }

ci/scripts/hermes_oci_promote.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+"""Publish a validated Hermes candidate manifest under its Flux release tag."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import os
+import re
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import Any, Callable
+
+
+REGISTRY_ORIGIN = "https://registry.bstein.dev"
+DESTINATION_PATTERN = re.compile(
+    r"^registry\.bstein\.dev/bstein/(?P<component>hermes-(?:agent|webui)):"
+    r"git-(?P<revision>[0-9a-f]{40})-build-(?P<build>[1-9][0-9]*)$"
+)
+DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
+MANIFEST_TYPES = {
+    "application/vnd.docker.distribution.manifest.v2+json",
+    "application/vnd.docker.distribution.manifest.list.v2+json",
+    "application/vnd.oci.image.manifest.v1+json",
+    "application/vnd.oci.image.index.v1+json",
+}
+MAX_MANIFEST_BYTES = 4 * 1024 * 1024
+
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+    """Never forward registry credentials to another origin."""
+
+    def redirect_request(self, _request, _file, _code, _message, _headers, _url):
+        return None
+
+
+def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
+    """Return normal and HTTP error responses without following redirects."""
+    opener = urllib.request.build_opener(_NoRedirect())
+    try:
+        return opener.open(request, timeout=timeout)
+    except urllib.error.HTTPError as exc:
+        return exc
+
+
+def _status(response: Any) -> int:
+    """Normalize urllib response and HTTPError status fields."""
+    return int(getattr(response, "status", getattr(response, "code", 0)))
+
+
+def _authorization(username: str, password: str) -> str:
+    """Build a Basic authorization value without placing it in a URL."""
+    if not username or not password:
+        raise RuntimeError("Harbor credentials are unavailable")
+    encoded = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
+    return f"Basic {encoded}"
+
+
+def _validated_release(
+    destination: str,
+    digest: str,
+    source_revision: str,
+    build_number: str,
+) -> tuple[str, str, str]:
+    """Bind the candidate and release tag to one reviewed source and build."""
+    match = DESTINATION_PATTERN.fullmatch(destination.strip())
+    if not match:
+        raise ValueError("invalid Hermes candidate destination")
+    if match.group("revision") != source_revision.strip():
+        raise ValueError("candidate source revision does not match evidence")
+    if match.group("build") != build_number.strip():
+        raise ValueError("candidate build number does not match evidence")
+    normalized_digest = digest.strip()
+    if not DIGEST_PATTERN.fullmatch(normalized_digest):
+        raise ValueError("invalid candidate digest")
+    candidate_tag = destination.rsplit(":", 1)[1]
+    return match.group("component"), candidate_tag, normalized_digest
+
+
+def _manifest_url(component: str, tag: str) -> str:
+    """Return one same-origin, path-escaped Docker Registry manifest URL."""
+    encoded_tag = urllib.parse.quote(tag, safe="")
+    return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/manifests/{encoded_tag}"
+
+
+def promote_candidate(
+    *,
+    destination: str,
+    digest: str,
+    source_revision: str,
+    build_number: str,
+    username: str,
+    password: str,
+    opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
+) -> dict[str, str]:
+    """Copy an exact candidate manifest to the immutable ``-release`` tag."""
+    component, candidate_tag, normalized_digest = _validated_release(
+        destination, digest, source_revision, build_number
+    )
+    release_tag = f"{candidate_tag}-release"
+    authorization = _authorization(username, password)
+    accept = ", ".join(sorted(MANIFEST_TYPES))
+    candidate_request = urllib.request.Request(
+        _manifest_url(component, candidate_tag),
+        headers={"Accept": accept, "Authorization": authorization},
+        method="GET",
+    )
+    with opener(candidate_request, 30) as response:
+        if _status(response) != 200:
+            raise RuntimeError(f"candidate manifest returned HTTP {_status(response)}")
+        manifest = response.read(MAX_MANIFEST_BYTES + 1)
+        if len(manifest) > MAX_MANIFEST_BYTES:
+            raise RuntimeError("candidate manifest exceeded the size limit")
+        observed_digest = response.headers.get("Docker-Content-Digest", "")
+        content_type = response.headers.get("Content-Type", "").split(";", 1)[0]
+    if observed_digest != normalized_digest:
+        raise RuntimeError("candidate manifest digest does not match build evidence")
+    if content_type not in MANIFEST_TYPES:
+        raise RuntimeError("candidate manifest returned an unsupported content type")
+
+    release_url = _manifest_url(component, release_tag)
+    head_request = urllib.request.Request(
+        release_url,
+        headers={"Accept": accept, "Authorization": authorization},
+        method="HEAD",
+    )
+    with opener(head_request, 20) as response:
+        head_status = _status(response)
+        existing_digest = response.headers.get("Docker-Content-Digest", "")
+    if head_status == 200:
+        if existing_digest != normalized_digest:
+            raise RuntimeError("release tag already exists with another digest")
+        result = "already-present"
+    elif head_status == 404:
+        put_request = urllib.request.Request(
+            release_url,
+            data=manifest,
+            headers={
+                "Authorization": authorization,
+                "Content-Type": content_type,
+            },
+            method="PUT",
+        )
+        with opener(put_request, 30) as response:
+            put_status = _status(response)
+            promoted_digest = response.headers.get("Docker-Content-Digest", "")
+        if put_status not in {201, 202}:
+            raise RuntimeError(f"release manifest returned HTTP {put_status}")
+        if promoted_digest and promoted_digest != normalized_digest:
+            raise RuntimeError("release manifest digest changed during promotion")
+        result = "published"
+    else:
+        raise RuntimeError(f"release tag preflight returned HTTP {head_status}")
+
+    return {
+        "component": component,
+        "digest": normalized_digest,
+        "release_tag": release_tag,
+        "result": result,
+        "source_revision": source_revision,
+    }
+
+
+def main() -> int:
+    """Promote one evidence-bound candidate and print credential-free metadata."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--destination", required=True)
+    parser.add_argument("--digest-file", required=True, type=Path)
+    parser.add_argument("--source-revision", required=True)
+    parser.add_argument("--build-number", required=True)
+    args = parser.parse_args()
+    try:
+        result = promote_candidate(
+            destination=args.destination,
+            digest=args.digest_file.read_text(encoding="utf-8"),
+            source_revision=args.source_revision,
+            build_number=args.build_number,
+            username=os.environ.get("HARBOR_USER", ""),
+            password=os.environ.get("HARBOR_PASSWORD", ""),
+        )
+    except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc:
+        print(json.dumps({"error": str(exc)}, sort_keys=True))
+        return 1
+    print(json.dumps(result, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover - exercised through main()
+    raise SystemExit(main())

ci/scripts/hermes_webui_flux_release.py

@@ -60,13 +60,16 @@ def render_workload(
         raise ValueError(f"Flux target identity changed: expected {kind}/{name}")
     lines = source.splitlines(keepends=True)
     matches: list[int] = []
+    suffixes: dict[int, str] = {}
     for index, line in enumerate(lines):
         stripped = line.strip()
         if not stripped.startswith(f"image: {image}@"):
             continue
-        current_digest = stripped.removeprefix(f"image: {image}@")
+        value = stripped.removeprefix(f"image: {image}@")
+        current_digest, separator, comment = value.partition(" #")
         validated(current_digest, DIGEST_PATTERN, "current Flux image digest")
         matches.append(index)
+        suffixes[index] = f" #{comment}" if separator else ""
     if len(matches) != 1:
         raise ValueError(
             f"expected exactly one {image!r} image in {kind}/{name}; "
@@ -75,7 +78,7 @@ def render_workload(
     index = matches[0]
     newline = "\n" if lines[index].endswith("\n") else ""
     prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())]
-    lines[index] = f"{prefix}image: {image}@{digest}{newline}"
+    lines[index] = f"{prefix}image: {image}@{digest}{suffixes[index]}{newline}"
     return "".join(lines)
 
 

clusters/atlas/flux-system/applications/hermes/image-automation.yaml

@@ -0,0 +1,26 @@
+# clusters/atlas/flux-system/applications/hermes/image-automation.yaml
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImageUpdateAutomation
+metadata:
+  name: hermes
+  namespace: hermes
+spec:
+  interval: 1m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  git:
+    checkout:
+      ref:
+        branch: main
+    commit:
+      author:
+        email: ops@bstein.dev
+        name: flux-bot
+      messageTemplate: "chore(hermes): promote validated image release"
+    push:
+      branch: main
+  update:
+    strategy: Setters
+    path: services/hermes

clusters/atlas/flux-system/applications/kustomization.yaml

@@ -31,6 +31,7 @@ resources:
   - hermes-scm-namespace/kustomization.yaml
   - hermes-scm-broker-code/kustomization.yaml
   - hermes/kustomization.yaml
+  - hermes/image-automation.yaml
   - hermes-observer-rbac/kustomization.yaml
   - hermes-observer-bindings/kustomization.yaml
   - hermes-scm-broker/kustomization.yaml

services/hermes/agent-configmap.yaml

@@ -369,6 +369,17 @@ data:
     file as user state. PR publication ends at the verified open draft; Brad
     owns review and merge authority.
 
+    For an explicitly requested Hermes runtime release, trigger only the
+    reviewed image lanes with `/opt/coordinator/jenkins_image_build_trigger.py`.
+    Use `--component agent` for backend/runtime changes and `--component webui`
+    for chat UI changes, passing a full commit already contained by `main`.
+    Jenkins builds the newest main containing that commit, publishes a final
+    immutable release tag only after its evidence passes, and Flux applies the
+    resulting digest. Candidate tags and failed builds never deploy. Follow the
+    returned queue item with `jenkins_build_evidence.py`, then verify the Flux
+    revision, pod image digest, rollout health, and public SSO redirect before
+    reporting the release complete.
+
     The terminal PATH contains the pinned operator tools. Start cluster work
     with `kubectl config current-context`, read-only status/events/logs, and the
     relevant `titan-iac` manifests. The broker namespace is deliberately

services/hermes/agent-deployment.yaml

@@ -25,7 +25,7 @@ spec:
         ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
         ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
         ai.bstein.dev/placement: titan-08 rpi5; storage-backbone nodes excluded
-        ai.bstein.dev/config-rev: "20260823-claude-auth-hysteresis"
+        ai.bstein.dev/config-rev: "20260823-image-release-automation"
         prometheus.io/scrape: "true"
         prometheus.io/path: /metrics
         prometheus.io/port: "9010"

services/hermes/chat-statefulset.yaml

@@ -320,7 +320,7 @@ spec:
             requests: {cpu: 250m, memory: 256Mi}
             limits: {cpu: "1", memory: 2Gi}
         - name: webui
-          image: registry.bstein.dev/bstein/hermes-webui@sha256:c276a9e17c9057237472f39640c9ebac9d4359d9f62a34df52a759eef43167bf
+          image: registry.bstein.dev/bstein/hermes-webui@sha256:c276a9e17c9057237472f39640c9ebac9d4359d9f62a34df52a759eef43167bf # {"$imagepolicy": "hermes:hermes-webui-release:digest"}
           imagePullPolicy: IfNotPresent
           command: [/bin/sh, -ec]
           args:

services/hermes/deployment.yaml

@@ -385,7 +385,7 @@ spec:
               cpu: "2"
               memory: 4Gi
         - name: webui
-          image: registry.bstein.dev/bstein/hermes-webui@sha256:ac6ba7bfd8a86227f31a9a96ebea41227ccf70391f7dcf34206f4d58e835e50e
+          image: registry.bstein.dev/bstein/hermes-webui@sha256:ac6ba7bfd8a86227f31a9a96ebea41227ccf70391f7dcf34206f4d58e835e50e # {"$imagepolicy": "hermes:hermes-webui-release:digest"}
           imagePullPolicy: IfNotPresent
           command: [/bin/sh, -ec]
           args:

services/hermes/image.yaml

@@ -0,0 +1,52 @@
+# services/hermes/image.yaml
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImageRepository
+metadata:
+  name: hermes-agent-release
+  namespace: hermes
+spec:
+  image: registry.bstein.dev/bstein/hermes-agent
+  interval: 1m0s
+---
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImagePolicy
+metadata:
+  name: hermes-agent-release
+  namespace: hermes
+spec:
+  imageRepositoryRef:
+    name: hermes-agent-release
+  filterTags:
+    pattern: '^git-[0-9a-f]{40}-build-(?P<build>[1-9][0-9]*)-release$'
+    extract: '$build'
+  policy:
+    numerical:
+      order: asc
+  digestReflectionPolicy: Always
+  interval: 1m0s
+---
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImageRepository
+metadata:
+  name: hermes-webui-release
+  namespace: hermes
+spec:
+  image: registry.bstein.dev/bstein/hermes-webui
+  interval: 1m0s
+---
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImagePolicy
+metadata:
+  name: hermes-webui-release
+  namespace: hermes
+spec:
+  imageRepositoryRef:
+    name: hermes-webui-release
+  filterTags:
+    pattern: '^git-[0-9a-f]{40}-build-(?P<build>[1-9][0-9]*)-release$'
+    extract: '$build'
+  policy:
+    numerical:
+      order: asc
+  digestReflectionPolicy: Always
+  interval: 1m0s

services/hermes/kustomization.yaml

@@ -4,9 +4,10 @@ kind: Kustomization
 namespace: hermes
 images:
   - name: registry.bstein.dev/bstein/hermes-agent
-    digest: sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37
+    digest: sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 # {"$imagepolicy": "hermes:hermes-agent-release:digest"}
 resources:
   - namespace.yaml
+  - image.yaml
   - scm-common
   - vault-serviceaccount.yaml
   - configmap.yaml

services/hermes/scripts/jenkins_image_build_trigger.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Trigger only the reviewed-main Hermes agent image release job."""
+"""Trigger a bounded reviewed-main Hermes image release job."""
 
 from __future__ import annotations
 
@@ -14,7 +14,17 @@ from pathlib import Path
 
 JENKINS_ORIGIN = "https://ci.bstein.dev"
 JENKINS_BUILD_URL = f"{JENKINS_ORIGIN}/buildByToken/buildWithParameters"
-JOB_NAME = "hermes-agent-image"
+JOBS = {
+    "agent": {
+        "job": "hermes-agent-image",
+        "confirmation": "PUBLISH HERMES AGENT",
+    },
+    "webui": {
+        "job": "hermes-webui-image",
+        "confirmation": "PUBLISH HERMES WEBUI",
+    },
+}
+JOB_NAME = JOBS["agent"]["job"]
 TOKEN_FILE = Path("/runtime-access/jenkins-image-build-token")
 REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
 QUEUE_PATH_PATTERN = re.compile(r"^/queue/item/[0-9]+/?$")
@@ -41,23 +51,27 @@ def _open_without_redirect(request: urllib.request.Request, timeout: int):
 def trigger_build(
     revision: str,
     *,
+    component: str = "agent",
     token_file: Path = TOKEN_FILE,
     opener=_open_without_redirect,
 ) -> dict[str, str | int]:
-    """Post the fixed job parameters using its job-scoped build token."""
+    """Post one allow-listed job's fixed parameters using the release token."""
     revision = revision.strip()
     if not REVISION_PATTERN.fullmatch(revision):
         raise ValueError("revision must be a lowercase full 40-character commit")
+    job = JOBS.get(component)
+    if job is None:
+        raise ValueError("component must be agent or webui")
     token = token_file.read_text(encoding="utf-8").strip()
     if not token:
         raise RuntimeError("Jenkins image-build token is empty")
     payload = urllib.parse.urlencode(
         {
-            "job": JOB_NAME,
+            "job": job["job"],
             "token": token,
             "PUBLISH_IMAGE": "true",
             "EXPECTED_SOURCE_REVISION": revision,
-            "CONFIRM_PUBLISH": "PUBLISH HERMES AGENT",
+            "CONFIRM_PUBLISH": job["confirmation"],
         }
     ).encode("utf-8")
     request = urllib.request.Request(
@@ -87,7 +101,8 @@ def trigger_build(
     # Never return the submitted URL: its form body contains the job token.
     queue_path = parsed_queue.path
     return {
-        "job": JOB_NAME,
+        "component": component,
+        "job": job["job"],
         "queue_path": queue_path,
         "source_revision": revision,
         "status": status,
@@ -97,10 +112,13 @@ def trigger_build(
 def main() -> int:
     """Validate one revision, trigger the bounded job, and print safe metadata."""
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("revision", help="reviewed full commit currently on main")
+    parser.add_argument(
+        "--component", choices=sorted(JOBS), default="agent", help="image to release"
+    )
+    parser.add_argument("revision", help="reviewed full commit contained by main")
     args = parser.parse_args()
     try:
-        result = trigger_build(args.revision)
+        result = trigger_build(args.revision, component=args.component)
     except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc:
         print(json.dumps({"error": str(exc)}, sort_keys=True))
         return 1

services/jenkins/configmap-jcasc.yaml

@@ -654,7 +654,7 @@ data:
           }
           pipelineJob('hermes-agent-image') {
             disabled(false)
-            description('Human-gated, daemonless Kaniko build for the reviewed atlas/titan-iac main revision. Publishes a content-addressed Hermes agent image and archives a Flux digest patch; it never mutates Git or deploys.')
+            description('Bounded daemonless Kaniko release for the latest atlas/titan-iac main containing a reviewed commit. Archives exact evidence, then publishes an immutable tag for Flux deployment.')
             authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))
             definition {
               cpsScm {
@@ -673,7 +673,8 @@ data:
           }
           pipelineJob('hermes-webui-image') {
             disabled(false)
-            description('Human-gated, daemonless Kaniko build for the reviewed atlas/titan-iac main revision. Publishes an immutable Hermes WebUI image and archives a narrow two-workload Flux digest patch; it never mutates Git or deploys.')
+            description('Bounded daemonless Kaniko release for the latest atlas/titan-iac main containing a reviewed commit. Archives exact evidence, then publishes an immutable tag for Flux deployment.')
+            authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))
             definition {
               cpsScm {
                 scm {

services/jenkins/deployment.yaml

@@ -68,7 +68,7 @@ spec:
           {{ with secret "kv/data/atlas/hermes/developer-jenkins" }}
           HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }}
           {{ end }}
-        bstein.dev/restarted-at: "2026-05-20T09:40:31Z"
+        bstein.dev/restarted-at: "2026-08-23T16:40:25Z"
     spec:
       serviceAccountName: jenkins
       priorityClassName: scavenger

testing/tests/test_hermes_image_automation.py

@@ -0,0 +1,67 @@
+"""Contracts for automatic deployment of validated Hermes image releases."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SERVICE = ROOT / "services/hermes"
+APPLICATIONS = ROOT / "clusters/atlas/flux-system/applications"
+
+
+def test_image_policies_observe_only_validated_release_tags() -> None:
+    """Candidates remain invisible until Jenkins publishes the release suffix."""
+    documents = list(
+        yaml.safe_load_all((SERVICE / "image.yaml").read_text(encoding="utf-8"))
+    )
+    repositories = {
+        item["metadata"]["name"]: item
+        for item in documents
+        if item["kind"] == "ImageRepository"
+    }
+    policies = {
+        item["metadata"]["name"]: item
+        for item in documents
+        if item["kind"] == "ImagePolicy"
+    }
+    assert set(repositories) == {"hermes-agent-release", "hermes-webui-release"}
+    assert set(policies) == set(repositories)
+    for name, policy in policies.items():
+        assert policy["metadata"]["namespace"] == "hermes"
+        assert policy["spec"]["imageRepositoryRef"]["name"] == name
+        assert policy["spec"]["filterTags"] == {
+            "pattern": ("^git-[0-9a-f]{40}-build-" "(?P<build>[1-9][0-9]*)-release$"),
+            "extract": "$build",
+        }
+        assert policy["spec"]["policy"] == {"numerical": {"order": "asc"}}
+        assert policy["spec"]["digestReflectionPolicy"] == "Always"
+
+
+def test_flux_updates_only_the_three_hermes_image_digests() -> None:
+    """Flux persists selected digests to Git and rolls all matching workloads."""
+    service_kustomization = (SERVICE / "kustomization.yaml").read_text(encoding="utf-8")
+    applications_kustomization = (APPLICATIONS / "kustomization.yaml").read_text(
+        encoding="utf-8"
+    )
+    automation = yaml.safe_load(
+        (APPLICATIONS / "hermes/image-automation.yaml").read_text(encoding="utf-8")
+    )
+    agent = (SERVICE / "kustomization.yaml").read_text(encoding="utf-8")
+    chat = (SERVICE / "chat-statefulset.yaml").read_text(encoding="utf-8")
+    dashboard = (SERVICE / "deployment.yaml").read_text(encoding="utf-8")
+
+    assert "  - image.yaml" in service_kustomization
+    assert "  - hermes/image-automation.yaml" in applications_kustomization
+    assert automation["spec"]["git"]["checkout"]["ref"]["branch"] == "main"
+    assert automation["spec"]["git"]["push"]["branch"] == "main"
+    assert automation["spec"]["update"] == {
+        "strategy": "Setters",
+        "path": "services/hermes",
+    }
+    assert agent.count('"$imagepolicy": "hermes:hermes-agent-release:digest"') == 1
+    webui_marker = '"$imagepolicy": "hermes:hermes-webui-release:digest"'
+    assert chat.count(webui_marker) == 1
+    assert dashboard.count(webui_marker) == 1

testing/tests/test_hermes_image_builder.py

@@ -103,7 +103,7 @@ def test_pipeline_requires_reviewed_main_and_runtime_credentials() -> None:
     assert "set +x" in source
     assert "umask 077" in source
     assert "unset HARBOR_USER HARBOR_PASSWORD auth" in source
-    assert "/busybox/rm -f /kaniko/.docker/config.json" in source
+    assert '/busybox/rm -f "${config_path}"' in source
     assert "--digest-file=" in source
     assert "--image-name-tag-with-digest-file=" in source
     assert "--destination=" in source
@@ -237,6 +237,41 @@ def test_agent_trigger_rejects_unsafe_revision_and_empty_token(tmp_path: Path) -
         module.trigger_build("a" * 40, token_file=token_path)
 
 
+def test_agent_trigger_can_select_only_the_bounded_webui_job(tmp_path: Path) -> None:
+    """Hermes can release WebUI without gaining a caller-controlled Jenkins job."""
+    module = _load_trigger_module()
+    token_path = tmp_path / "token"
+    token_path.write_text("private-job-token\n", encoding="utf-8")
+    captured = {}
+
+    class Response(io.BytesIO):
+        status = 201
+        headers = {"Location": "https://ci.bstein.dev/queue/item/84/"}
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *_args):
+            self.close()
+
+    def opener(request, timeout):
+        captured["request"] = request
+        captured["timeout"] = timeout
+        return Response(b"")
+
+    result = module.trigger_build(
+        "b" * 40, component="webui", token_file=token_path, opener=opener
+    )
+    fields = urllib.parse.parse_qs(captured["request"].data.decode("utf-8"))
+    assert fields["job"] == ["hermes-webui-image"]
+    assert fields["CONFIRM_PUBLISH"] == ["PUBLISH HERMES WEBUI"]
+    assert result["component"] == "webui"
+    with pytest.raises(ValueError, match="agent or webui"):
+        module.trigger_build(
+            "b" * 40, component="other", token_file=token_path, opener=opener
+        )
+
+
 def test_agent_trigger_accepts_existing_queue_redirect(tmp_path: Path) -> None:
     """HTTP 303 means the exact release is already queued, not a trigger failure."""
     module = _load_trigger_module()

testing/tests/test_hermes_image_builder_coverage.py

@@ -16,8 +16,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2]
 RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py"
 DOCKERFILE = REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent"
 HARBOR = (
-    REPO_ROOT
+    REPO_ROOT / "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py"
-    / "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py"
 )
 TRIGGER = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py"
 
@@ -94,9 +93,7 @@ def _robot(module, *, immutable: bool = False, duration=-1) -> dict:
 
 
 def _robot_list(module, *, robot_id=41, name=None):
-    return json.dumps(
+    return json.dumps([{"id": robot_id, "name": name or module.PUBLISH_ROBOT}]).encode()
-        [{"id": robot_id, "name": name or module.PUBLISH_ROBOT}]
-    ).encode()
 
 
 def test_runner_rejects_directive_and_heredoc_boundaries(tmp_path: Path) -> None:
@@ -141,7 +138,9 @@ def test_runner_main_dispatches_the_exact_path_and_index(
     module = _load(RUNNER, "runner_main_coverage")
     dockerfile = tmp_path / "Dockerfile"
     seen = []
-    monkeypatch.setattr(module, "replay", lambda path, index: seen.append((path, index)))
+    monkeypatch.setattr(
+        module, "replay", lambda path, index: seen.append((path, index))
+    )
     monkeypatch.setattr(
         sys,
         "argv",
@@ -156,7 +155,9 @@ def test_trigger_redirect_wrapper_accepts_only_plugin_303(
 ) -> None:
     """The low-level opener returns a real response or the one expected redirect."""
     module = _load(TRIGGER, "trigger_opener_coverage")
-    assert module._NoRedirect().redirect_request(None, None, 0, None, None, None) is None
+    assert (
+        module._NoRedirect().redirect_request(None, None, 0, None, None, None) is None
+    )
     request = module.urllib.request.Request(module.JENKINS_BUILD_URL)
 
     class Opener:
@@ -199,12 +200,19 @@ def test_trigger_main_reports_safe_success_and_errors(
     monkeypatch.setattr(
         module,
         "trigger_build",
-        lambda value: {"job": module.JOB_NAME, "source_revision": value, "status": 201},
+        lambda value, component="agent": {
+            "component": component,
+            "job": module.JOBS[component]["job"],
+            "source_revision": value,
+            "status": 201,
+        },
     )
     assert module.main() == 0
     assert json.loads(capsys.readouterr().out)["source_revision"] == revision
     monkeypatch.setattr(
-        module, "trigger_build", lambda _value: (_ for _ in ()).throw(OSError("closed"))
+        module,
+        "trigger_build",
+        lambda _value, component="agent": (_ for _ in ()).throw(OSError("closed")),
     )
     assert module.main() == 1
     assert json.loads(capsys.readouterr().out) == {"error": "closed"}
@@ -278,11 +286,7 @@ def test_harbor_json_lists_reject_invalid_shapes(body: bytes) -> None:
             "access",
         ),
         (
-            {
+            {"permissions": [{"kind": "project", "namespace": "bstein", "access": []}]},
-                "permissions": [
-                    {"kind": "project", "namespace": "bstein", "access": []}
-                ]
-            },
             "pull/push",
         ),
     ],
@@ -304,7 +308,11 @@ def test_publisher_scope_rejects_malformed_or_incomplete_permissions(
         ([(403, b"", {})], "403", False),
         ([(200, b"[]", _count(0))], "exactly one", False),
         ([(200, b"[]", _count(0))], "exactly one", False),
-        ([(200, b'[{\"name\":\"robot$jenkins-pipelines\",\"id\":0}]', _count(1))], "valid ID", False),
+        (
+            [(200, b'[{"name":"robot$jenkins-pipelines","id":0}]', _count(1))],
+            "valid ID",
+            False,
+        ),
     ],
 )
 def test_publisher_list_rejects_status_count_and_id(
@@ -324,7 +332,7 @@ def test_publisher_list_rejects_status_count_and_id(
         ((403, b"", {}), "403", False),
         ((200, b"not-json", {}), "invalid robot JSON", False),
         ((200, b"[]", {}), "invalid shape", False),
-        ((200, b'{}', {}), "not active and exact", False),
+        ((200, b"{}", {}), "not active and exact", False),
     ],
 )
 def test_publisher_read_rejects_status_and_identity(
@@ -405,9 +413,7 @@ def test_rule_rejects_ids_create_status_and_location_suffix() -> None:
             FakeClient([(200, json.dumps([bad_rule]).encode(), _count(1))])
         )
     with pytest.raises(module.HarborUnavailable):
-        module.ensure_rule(
+        module.ensure_rule(FakeClient([(200, b"[]", _count(0)), (503, b"", {})]))
-            FakeClient([(200, b"[]", _count(0)), (503, b"", {})])
-        )
     with pytest.raises(RuntimeError, match="invalid ID"):
         module.ensure_rule(
             FakeClient(

testing/tests/test_hermes_image_builder_fresh_review.py

@@ -90,7 +90,9 @@ def test_all_unsupported_run_heredoc_forms_reject_before_execution(
         encoding="utf-8",
     )
     calls = []
-    monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1))
+    monkeypatch.setattr(
+        module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)
+    )
     with pytest.raises(ValueError, match="unsupported RUN heredoc"):
         module.replay(dockerfile, 1)
     assert calls == []
@@ -119,7 +121,9 @@ def test_appended_tenth_reviewed_form_rejects_before_execution(
         encoding="utf-8",
     )
     calls = []
-    monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1))
+    monkeypatch.setattr(
+        module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)
+    )
     with pytest.raises(ValueError, match="contract changed"):
         module.replay(dockerfile, 1)
     assert calls == []
@@ -155,7 +159,9 @@ def test_logical_instruction_normalization_rejects_split_heredocs_before_executi
         encoding="utf-8",
     )
     calls = []
-    monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1))
+    monkeypatch.setattr(
+        module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1)
+    )
     with pytest.raises(ValueError, match="unsupported RUN heredoc"):
         module.replay(dockerfile, 1)
     assert calls == []
@@ -208,7 +214,9 @@ def test_builder_service_account_is_explicit_tokenless_and_unbound() -> None:
     assert spec["automountServiceAccountToken"] is False
     for volume in spec.get("volumes", []):
         projected = volume.get("projected", {})
-        assert all("serviceAccountToken" not in item for item in projected.get("sources", []))
+        assert all(
+            "serviceAccountToken" not in item for item in projected.get("sources", [])
+        )
 
     for manifest in (REPO_ROOT / "services/jenkins").glob("*.yaml"):
         for document in yaml.safe_load_all(manifest.read_text(encoding="utf-8")):
@@ -278,7 +286,9 @@ def test_success_evidence_revalidation_accepts_only_exact_complete_set(
 
     metadata = kwargs["output_dir"] / "hermes-agent-image.json"
     original = metadata.read_text(encoding="utf-8")
-    metadata.write_text(original.replace('"build_number": "23"', '"build_number": "24"'))
+    metadata.write_text(
+        original.replace('"build_number": "23"', '"build_number": "24"')
+    )
     with pytest.raises(ValueError, match="incomplete or mismatched"):
         module.validate_release_artifacts(**kwargs)
     metadata.write_text(original, encoding="utf-8")
@@ -319,17 +329,20 @@ def test_verify_evidence_cli_needs_no_runtime_registry_credential(
     assert module.main() == 0
 
 
-def test_pipeline_success_post_requires_and_archives_exact_six_files() -> None:
+def test_pipeline_requires_and_archives_exact_release_evidence() -> None:
-    """Missing or partial post-success evidence must change the build to failed."""
+    """Missing evidence must fail before the candidate becomes a release."""
     source = PIPELINE.read_text(encoding="utf-8")
-    post = source.split("  post {", 1)[1]
+    evidence = source.split("stage('Verify and archive release evidence')", 1)[1]
-    assert "success {" in post
+    evidence = evidence.split("stage('Publish Flux release tag')", 1)[0]
-    assert "always {" not in post
+    assert "verify-evidence" in evidence
-    assert "verify-evidence" in post
+    assert "allowEmptyArchive: false" in evidence
-    assert 'allowEmptyArchive: false' in post
+    archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0]
-    archive = post.split("artifacts: '", 1)[1].split("'", 1)[0]
     paths = archive.split(",")
-    assert len(paths) == 6
+    assert len(paths) == 7
-    assert len(set(paths)) == 6
+    assert len(set(paths)) == 7
     assert all("*" not in path for path in paths)
-    assert "find build -type f" in post
+    assert "find build -type f" in evidence
+    assert "build/hermes-agent.source-revision" in paths
+    promotion = source.split("stage('Publish Flux release tag')", 1)[1]
+    assert "ci/scripts/hermes_oci_promote.py" in promotion
+    assert "  post {" not in source

testing/tests/test_hermes_oci_promote.py

@@ -0,0 +1,205 @@
+"""Safety tests for the evidence-bound Hermes OCI release promotion."""
+
+from __future__ import annotations
+
+import importlib.util
+import io
+import json
+import sys
+import urllib.request
+from pathlib import Path
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = ROOT / "ci/scripts/hermes_oci_promote.py"
+REVISION = "a" * 40
+DIGEST = "sha256:" + "b" * 64
+DESTINATION = f"registry.bstein.dev/bstein/hermes-agent:git-{REVISION}-build-17"
+CONTENT_TYPE = "application/vnd.docker.distribution.manifest.v2+json"
+MANIFEST = b'{"schemaVersion":2}'
+
+
+def _load():
+    spec = importlib.util.spec_from_file_location("hermes_oci_promote", SCRIPT)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+class Response(io.BytesIO):
+    """Minimal context-managed urllib response."""
+
+    def __init__(self, status: int, body: bytes = b"", headers=None):
+        super().__init__(body)
+        self.status = status
+        self.headers = headers or {}
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_args):
+        self.close()
+
+
+def _candidate(headers=None, body: bytes = MANIFEST) -> Response:
+    values = {
+        "Docker-Content-Digest": DIGEST,
+        "Content-Type": CONTENT_TYPE,
+    }
+    values.update(headers or {})
+    return Response(200, body, values)
+
+
+def _promote(module, opener):
+    return module.promote_candidate(
+        destination=DESTINATION,
+        digest=DIGEST,
+        source_revision=REVISION,
+        build_number="17",
+        username="robot",
+        password="private",
+        opener=opener,
+    )
+
+
+def test_promotion_copies_exact_candidate_to_release_tag() -> None:
+    """Only the exact evidence digest is copied to the Flux-visible tag."""
+    module = _load()
+    calls: list[urllib.request.Request] = []
+    responses = iter(
+        [
+            _candidate(),
+            Response(404),
+            Response(201, headers={"Docker-Content-Digest": DIGEST}),
+        ]
+    )
+
+    def opener(request, timeout):
+        calls.append(request)
+        assert timeout in {20, 30}
+        return next(responses)
+
+    result = _promote(module, opener)
+    assert [request.method for request in calls] == ["GET", "HEAD", "PUT"]
+    assert calls[0].full_url.endswith(f"/manifests/git-{REVISION}-build-17")
+    assert calls[2].full_url.endswith(f"/manifests/git-{REVISION}-build-17-release")
+    assert calls[2].data == MANIFEST
+    assert result["result"] == "published"
+    assert result["digest"] == DIGEST
+    assert "private" not in json.dumps(result)
+
+
+def test_promotion_is_idempotent_for_the_same_digest() -> None:
+    """A replay succeeds only when the immutable release already matches."""
+    module = _load()
+    responses = iter(
+        [_candidate(), Response(200, headers={"Docker-Content-Digest": DIGEST})]
+    )
+    result = _promote(module, lambda *_args: next(responses))
+    assert result["result"] == "already-present"
+
+
+@pytest.mark.parametrize(
+    ("changes", "message"),
+    [
+        ({"destination": "registry.invalid/hermes:latest"}, "destination"),
+        ({"source_revision": "c" * 40}, "revision"),
+        ({"build_number": "18"}, "build number"),
+        ({"digest": "sha256:bad"}, "digest"),
+        ({"username": ""}, "credentials"),
+    ],
+)
+def test_promotion_rejects_unbound_inputs(changes: dict, message: str) -> None:
+    """Destination, source, build, digest, and credentials fail closed."""
+    module = _load()
+    kwargs = {
+        "destination": DESTINATION,
+        "digest": DIGEST,
+        "source_revision": REVISION,
+        "build_number": "17",
+        "username": "robot",
+        "password": "private",
+        "opener": lambda *_args: _candidate(),
+    }
+    kwargs.update(changes)
+    with pytest.raises((ValueError, RuntimeError), match=message):
+        module.promote_candidate(**kwargs)
+
+
+@pytest.mark.parametrize(
+    ("responses", "message"),
+    [
+        ([Response(401)], "candidate manifest returned HTTP 401"),
+        ([_candidate({"Docker-Content-Digest": "sha256:" + "c" * 64})], "digest"),
+        ([_candidate({"Content-Type": "text/plain"})], "content type"),
+        (
+            [
+                _candidate(),
+                Response(200, headers={"Docker-Content-Digest": "sha256:" + "c" * 64}),
+            ],
+            "another digest",
+        ),
+        ([_candidate(), Response(500)], "preflight returned HTTP 500"),
+        (
+            [_candidate(), Response(404), Response(500)],
+            "release manifest returned HTTP 500",
+        ),
+        (
+            [
+                _candidate(),
+                Response(404),
+                Response(201, headers={"Docker-Content-Digest": "sha256:" + "c" * 64}),
+            ],
+            "changed",
+        ),
+    ],
+)
+def test_registry_failures_cannot_create_a_valid_release(
+    responses: list[Response], message: str
+) -> None:
+    """Registry ambiguity or mismatch always fails the final release boundary."""
+    module = _load()
+    queued = iter(responses)
+    with pytest.raises(RuntimeError, match=message):
+        _promote(module, lambda *_args: next(queued))
+
+
+def test_cli_reports_credential_free_success_and_error(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    """The command emits bounded metadata and converts expected failures to JSON."""
+    module = _load()
+    digest_file = tmp_path / "digest"
+    digest_file.write_text(DIGEST + "\n", encoding="utf-8")
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            "promote",
+            "--destination",
+            DESTINATION,
+            "--digest-file",
+            str(digest_file),
+            "--source-revision",
+            REVISION,
+            "--build-number",
+            "17",
+        ],
+    )
+    monkeypatch.setattr(
+        module,
+        "promote_candidate",
+        lambda **_kwargs: {"result": "published", "digest": DIGEST},
+    )
+    assert module.main() == 0
+    assert json.loads(capsys.readouterr().out)["result"] == "published"
+    monkeypatch.setattr(
+        module,
+        "promote_candidate",
+        lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("denied")),
+    )
+    assert module.main() == 1
+    assert json.loads(capsys.readouterr().out) == {"error": "denied"}

testing/tests/test_hermes_webui_release.py

@@ -69,8 +69,8 @@ def _release_fixture(tmp_path: Path):
     return module, digest, kwargs
 
 
-def test_webui_job_is_independent_manual_and_main_only() -> None:
+def test_webui_job_is_independent_bounded_and_main_only() -> None:
-    """WebUI has its own job and never widens the existing agent-only lane."""
+    """WebUI has its own bounded job using the same Vault-injected release token."""
     config = yaml.safe_load(
         (ROOT / "services/jenkins/configmap-jcasc.yaml").read_text(encoding="utf-8")
     )
@@ -83,14 +83,17 @@ def test_webui_job_is_independent_manual_and_main_only() -> None:
     assert "branches('*/main')" in block
     assert "scriptPath('ci/Jenkinsfile.hermes-webui-image')" in block
     assert "pipelineTriggers" not in block
-    assert "HERMES_AGENT_IMAGE_BUILD_TOKEN" not in block
+    assert (
+        "authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))" in block
+    )
 
 
-def test_pipeline_builds_exact_reviewed_main_and_never_deploys() -> None:
+def test_pipeline_builds_latest_main_containing_reviewed_anchor() -> None:
-    """Publish is explicit, immutable, evidence-producing, and Git/Flux review only."""
+    """Publish is explicit, evidence-bound, and handed only to Flux."""
     source = PIPELINE.read_text(encoding="utf-8")
     assert 'test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES WEBUI"' in source
     assert 'test "${actual_revision}" = "$(git rev-parse origin/main)"' in source
+    assert 'git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}"' in source
     assert "dockerfiles/Dockerfile.hermes-webui" in source
     assert "ci/scripts/hermes_webui_release.py" in source
     assert "registry.bstein.dev/bstein/hermes-webui" in source
@@ -99,8 +102,9 @@ def test_pipeline_builds_exact_reviewed_main_and_never_deploys() -> None:
     assert "HERMES_KANIKO_HEREDOC_COMPAT" not in source
     assert "--digest-file=" in source
     assert "--image-name-tag-with-digest-file=" in source
-    assert "org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" in source
+    assert "org.opencontainers.image.revision=${source_revision}" in source
     assert "assert-absent" in source and "verify-evidence" in source
+    assert "ci/scripts/hermes_oci_promote.py" in source
     assert "test_hermes_webui_brand.py" in source
     assert "test_hermes_webui_release.py" in source
     for forbidden in ("kubectl ", "flux reconcile", "git push", "git commit"):
@@ -300,7 +304,9 @@ def test_release_verifies_exact_webui_harbor_artifact_and_policy() -> None:
     )
 
 
-@pytest.mark.parametrize("labels", [None, {}, {"org.opencontainers.image.revision": "c" * 40}])
+@pytest.mark.parametrize(
+    "labels", [None, {}, {"org.opencontainers.image.revision": "c" * 40}]
+)
 def test_release_rejects_missing_or_wrong_harbor_source_revision(labels) -> None:
     """A tag derived from Git cannot substitute for the persisted OCI label."""
     module = _load(RELEASE, f"webui_registry_revision_{labels!r}")
@@ -379,9 +385,7 @@ def test_flux_tracks_webui_policy_before_jenkins() -> None:
     assert container["securityContext"]["runAsNonRoot"] is True
     assert container["securityContext"]["capabilities"]["drop"] == ["ALL"]
 
-    release_docs = (ROOT / "docs/hermes_webui_release.md").read_text(
+    release_docs = (ROOT / "docs/hermes_webui_release.md").read_text(encoding="utf-8")
-        encoding="utf-8"
-    )
     assert "immutable-tag:list" in release_docs
     assert "harbor-hermes-agent-immutability-ensure-1" in release_docs
 
@@ -466,14 +470,17 @@ def test_webui_policy_rejects_disabled_duplicate_or_truncated_rules() -> None:
         policy.list_rules(_FakePolicyClient([(200, b"[]", {"X-Total-Count": "1"})]))
 
 
-def test_pipeline_archives_exact_seven_files() -> None:
+def test_pipeline_archives_exact_eight_files_before_release() -> None:
-    """Publication cannot pass with missing digest, workload, or metadata evidence."""
+    """Release cannot pass with missing digest, workload, or metadata evidence."""
     source = PIPELINE.read_text(encoding="utf-8")
-    post = source.split("  post {", 1)[1]
+    evidence = source.split("stage('Verify and archive release evidence')", 1)[1]
-    archive = post.split("artifacts: '", 1)[1].split("'", 1)[0].split(",")
+    evidence = evidence.split("stage('Publish Flux release tag')", 1)[0]
-    assert len(archive) == len(set(archive)) == 7
+    archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0].split(",")
+    assert len(archive) == len(set(archive)) == 8
     assert all("*" not in path for path in archive)
-    assert "find build -type f" in post
+    assert "find build -type f" in evidence
-    assert "allowEmptyArchive: false" in post
+    assert "allowEmptyArchive: false" in evidence
-    assert "hermes-chat-statefulset.yaml" in post
+    assert "build/hermes-webui.source-revision" in archive
-    assert "hermes-dashboard-deployment.yaml" in post
+    assert "hermes-chat-statefulset.yaml" in evidence
+    assert "hermes-dashboard-deployment.yaml" in evidence
+    assert "  post {" not in source