hermes: add daemonless agent image release lane

Reviewed-on: atlas/titan-iac#13

ci/Jenkinsfile.hermes-agent-image

@@ -0,0 +1,281 @@
+pipeline {
+  agent {
+    kubernetes {
+      defaultContainer 'python'
+      yaml """
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    atlas.bstein.dev/workload: hermes-agent-image-builder
+spec:
+  serviceAccountName: hermes-image-builder
+  automountServiceAccountToken: false
+  enableServiceLinks: false
+  restartPolicy: Never
+  securityContext:
+    fsGroup: 1000
+    fsGroupChangePolicy: OnRootMismatch
+  nodeSelector:
+    kubernetes.io/arch: arm64
+    node-role.kubernetes.io/worker: "true"
+    hardware: rpi5
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: kubernetes.io/hostname
+                operator: NotIn
+                values:
+                  - titan-04
+                  - titan-14
+                  - titan-18
+                  - titan-19
+                  - titan-22
+                  - titan-24
+  imagePullSecrets:
+    - name: harbor-bstein-robot
+  containers:
+    - name: jnlp
+      image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop: ["ALL"]
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      resources:
+        requests:
+          cpu: 25m
+          memory: 256Mi
+        limits:
+          cpu: 500m
+          memory: 512Mi
+    - name: python
+      image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
+      command: ["sleep"]
+      args: ["99d"]
+      tty: true
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop: ["ALL"]
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      resources:
+        requests:
+          cpu: 25m
+          memory: 64Mi
+        limits:
+          cpu: 250m
+          memory: 256Mi
+    - name: kaniko
+      image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
+      command: ["/busybox/sh", "-c"]
+      args: ["/busybox/sleep 99d"]
+      tty: true
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop: ["ALL"]
+          add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
+        privileged: false
+        runAsUser: 0
+        seccompProfile:
+          type: RuntimeDefault
+      resources:
+        requests:
+          cpu: 250m
+          memory: 1Gi
+        limits:
+          cpu: "2"
+          memory: 4Gi
+"""
+    }
+  }
+  parameters {
+    booleanParam(
+      name: 'PUBLISH_IMAGE',
+      defaultValue: false,
+      description: 'Publish the reviewed main revision to Harbor.'
+    )
+    string(
+      name: 'EXPECTED_SOURCE_REVISION',
+      defaultValue: '',
+      description: 'Exact 40-character commit on atlas/titan-iac main.'
+    )
+    string(
+      name: 'CONFIRM_PUBLISH',
+      defaultValue: '',
+      description: 'Enter PUBLISH HERMES AGENT to confirm the release.'
+    )
+  }
+  environment {
+    HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent'
+  }
+  options {
+    disableConcurrentBuilds()
+    buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
+    skipDefaultCheckout(true)
+    timeout(time: 90, unit: 'MINUTES')
+  }
+  stages {
+    stage('Checkout reviewed source') {
+      steps {
+        checkout scm
+      }
+    }
+    stage('Enforce release boundary') {
+      steps {
+        sh '''
+          set -eu
+          mkdir -p build
+          test "${PUBLISH_IMAGE}" = "true"
+          test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
+          case "${EXPECTED_SOURCE_REVISION}" in
+            *[!0-9a-f]*|'')
+              echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
+              exit 2
+              ;;
+          esac
+          test "${#EXPECTED_SOURCE_REVISION}" -eq 40
+          actual_revision="$(git rev-parse HEAD)"
+          test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
+          test "${actual_revision}" = "$(git rev-parse origin/main)"
+          test -z "$(git status --porcelain)"
+          test -f dockerfiles/Dockerfile.hermes-agent
+          case "${BUILD_NUMBER}" in
+            ''|0*|*[!0-9]*)
+              echo "BUILD_NUMBER must be a positive decimal integer" >&2
+              exit 2
+              ;;
+          esac
+          printf '%s\n' \
+            "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
+            > build/hermes-agent.destination
+        '''
+      }
+    }
+    stage('Reject replay before publish') {
+      steps {
+        withCredentials([usernamePassword(
+          credentialsId: 'harbor-robot',
+          usernameVariable: 'HARBOR_USER',
+          passwordVariable: 'HARBOR_PASSWORD'
+        )]) {
+          sh '''
+            set -eu
+            set +x
+            destination="$(cat build/hermes-agent.destination)"
+            python3 ci/scripts/hermes_image_release.py assert-absent \
+              --source-revision "${EXPECTED_SOURCE_REVISION}" \
+              --build-number "${BUILD_NUMBER}" \
+              --destination "${destination}"
+          '''
+        }
+      }
+    }
+    stage('Build and publish without a daemon') {
+      steps {
+        container('kaniko') {
+          withCredentials([usernamePassword(
+            credentialsId: 'harbor-robot',
+            usernameVariable: 'HARBOR_USER',
+            passwordVariable: 'HARBOR_PASSWORD'
+          )]) {
+            sh '''#!/busybox/sh
+              set -eu
+              set +x
+              config_path=/kaniko/.docker/config.json
+              destination="$(cat build/hermes-agent.destination)"
+              umask 077
+              auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
+              /busybox/mkdir -p /kaniko/.docker
+              /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
+              unset HARBOR_USER HARBOR_PASSWORD auth
+              trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
+              /kaniko/executor \
+                --context="dir://${WORKSPACE}" \
+                --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
+                --destination="${destination}" \
+                --digest-file="${WORKSPACE}/build/hermes-agent.digest" \
+                --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \
+                --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
+                --label="org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" \
+                --cleanup \
+                --push-retry=3
+            '''
+          }
+        }
+      }
+    }
+    stage('Render reviewed Flux handoff') {
+      steps {
+        withCredentials([usernamePassword(
+          credentialsId: 'harbor-robot',
+          usernameVariable: 'HARBOR_USER',
+          passwordVariable: 'HARBOR_PASSWORD'
+        )]) {
+          sh '''
+            set -eu
+            set +x
+            destination="$(cat build/hermes-agent.destination)"
+            python3 ci/scripts/hermes_image_release.py render \
+              --digest-file build/hermes-agent.digest \
+              --image-file build/hermes-agent.image \
+              --source-revision "${EXPECTED_SOURCE_REVISION}" \
+              --build-number "${BUILD_NUMBER}" \
+              --destination "${destination}" \
+              --kustomization services/hermes/kustomization.yaml \
+              --output-dir build/hermes-agent-release
+            test -s build/hermes-agent-release/hermes-image-update.patch
+            test -s build/hermes-agent-release/hermes-agent-image.json
+          '''
+        }
+      }
+    }
+  }
+  post {
+    success {
+      sh '''
+        set -eu
+        expected_files="$(printf '%s\n' \
+          build/hermes-agent.destination \
+          build/hermes-agent.digest \
+          build/hermes-agent.image \
+          build/hermes-agent-release/hermes-agent-image.json \
+          build/hermes-agent-release/hermes-image-update.patch \
+          build/hermes-agent-release/hermes-kustomization.yaml \
+          | LC_ALL=C sort)"
+        actual_files="$(find build -type f -print | LC_ALL=C sort)"
+        test "${actual_files}" = "${expected_files}"
+        destination="$(cat build/hermes-agent.destination)"
+        python3 ci/scripts/hermes_image_release.py verify-evidence \
+          --digest-file build/hermes-agent.digest \
+          --image-file build/hermes-agent.image \
+          --source-revision "${EXPECTED_SOURCE_REVISION}" \
+          --build-number "${BUILD_NUMBER}" \
+          --destination "${destination}" \
+          --kustomization services/hermes/kustomization.yaml \
+          --output-dir build/hermes-agent-release
+      '''
+      archiveArtifacts(
+        artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
+        allowEmptyArchive: false,
+        fingerprint: true
+      )
+    }
+    cleanup {
+      container('kaniko') {
+        sh '''#!/busybox/sh
+          /busybox/rm -f /kaniko/.docker/config.json
+        '''
+      }
+    }
+  }
+}

ci/scripts/hermes_image_release.py

@@ -0,0 +1,500 @@
+#!/usr/bin/env python3
+"""Verify and render a reviewable Hermes agent image release."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import difflib
+import json
+import os
+import re
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import Any, Callable
+
+
+DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-agent"
+HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0"
+HARBOR_PROJECT = "bstein"
+HARBOR_REPOSITORY = "hermes-agent"
+IMMUTABLE_REPOSITORY_PATTERN = "hermes-agent"
+IMMUTABLE_TAG_PATTERN = "git-*-build-*"
+DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
+REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
+BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$")
+DESTINATION_PATTERN = re.compile(
+    r"^registry\.bstein\.dev/bstein/hermes-agent:"
+    r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$"
+)
+
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+    """Never send registry credentials to a redirect target."""
+
+    def redirect_request(self, _request, _file, _code, _message, _headers, _url):
+        return None
+
+
+def _validated(value: str, pattern: re.Pattern[str], label: str) -> str:
+    """Return a normalized value when it matches the release contract."""
+    normalized = value.strip()
+    if not pattern.fullmatch(normalized):
+        raise ValueError(f"invalid {label}: expected {pattern.pattern}")
+    return normalized
+
+
+def validate_destination(
+    destination: str, source_revision: str, build_number: str
+) -> tuple[str, str]:
+    """Bind one unique build tag to the reviewed revision and Jenkins build."""
+    revision = _validated(source_revision, REVISION_PATTERN, "source revision")
+    build = _validated(build_number, BUILD_PATTERN, "build number")
+    match = DESTINATION_PATTERN.fullmatch(destination.strip())
+    if not match or match.groups() != (revision, build):
+        raise ValueError(
+            "destination must bind the reviewed revision and unique Jenkins build"
+        )
+    return revision, build
+
+
+def validate_kaniko_evidence(
+    *, digest_text: str, image_text: str, destination: str
+) -> str:
+    """Cross-check both independent Kaniko output files against the destination."""
+    digest_lines = digest_text.splitlines()
+    image_lines = image_text.splitlines()
+    if len(digest_lines) != 1:
+        raise ValueError("Kaniko digest evidence must contain exactly one line")
+    if len(image_lines) != 1:
+        raise ValueError("Kaniko image evidence must contain exactly one line")
+    digest = _validated(digest_lines[0], DIGEST_PATTERN, "image digest")
+    if image_lines[0].strip() != f"{destination}@{digest}":
+        raise ValueError("Kaniko image evidence does not match destination and digest")
+    return digest
+
+
+def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
+    """Make a registry request without following redirects."""
+    opener = urllib.request.build_opener(_NoRedirect())
+    try:
+        return opener.open(request, timeout=timeout)
+    except urllib.error.HTTPError as exc:
+        return exc
+
+
+def _artifact_response(
+    destination: str,
+    *,
+    username: str,
+    password: str,
+    opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
+) -> tuple[int, bytes]:
+    """Read one exact Harbor artifact by tag with bounded response size."""
+    match = DESTINATION_PATTERN.fullmatch(destination)
+    if not match:
+        raise ValueError("invalid destination")
+    if not username or not password:
+        raise RuntimeError("Harbor credentials are empty")
+    tag = destination.rsplit(":", 1)[1]
+    encoded_tag = urllib.parse.quote(tag, safe="")
+    auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
+    request = urllib.request.Request(
+        f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/"
+        f"{HARBOR_REPOSITORY}/artifacts/{encoded_tag}"
+        "?with_immutable_status=true",
+        headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
+        method="GET",
+    )
+    with opener(request, 20) as response:
+        body = response.read(1_048_577)
+        if len(body) > 1_048_576:
+            raise RuntimeError("Harbor artifact response exceeded the size limit")
+        return int(response.status), body
+
+
+def _immutable_rules_response(
+    *,
+    username: str,
+    password: str,
+    opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
+) -> tuple[int, bytes, dict[str, str]]:
+    """Read the project policy with the same least-privilege publish identity."""
+    if not username or not password:
+        raise RuntimeError("Harbor credentials are empty")
+    auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
+    request = urllib.request.Request(
+        f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules"
+        "?page=1&page_size=100",
+        headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
+        method="GET",
+    )
+    with opener(request, 20) as response:
+        body = response.read(1_048_577)
+        if len(body) > 1_048_576:
+            raise RuntimeError("Harbor immutable rule response exceeded the size limit")
+        return int(response.status), body, dict(response.headers)
+
+
+def _require_complete_rule_page(
+    rules: list[dict[str, Any]], headers: dict[str, str]
+) -> None:
+    """Require proof that the bounded first page contains every rule."""
+    raw_total = next(
+        (value for key, value in headers.items() if key.lower() == "x-total-count"),
+        None,
+    )
+    if raw_total is None or not str(raw_total).isdecimal():
+        raise RuntimeError("Harbor immutable rule list omitted a valid total count")
+    if int(raw_total) != len(rules):
+        raise RuntimeError("Harbor immutable rule list was truncated")
+
+
+def _normalized_immutable_rule(rule: dict[str, Any]) -> dict[str, Any]:
+    """Select only fields that bind the server-side build-tag policy."""
+    return {
+        "disabled": bool(rule.get("disabled", False)),
+        "action": rule.get("action"),
+        "template": rule.get("template"),
+        "tag_selectors": [
+            {
+                "kind": item.get("kind"),
+                "decoration": item.get("decoration"),
+                "pattern": item.get("pattern"),
+            }
+            for item in rule.get("tag_selectors") or []
+            if isinstance(item, dict)
+        ],
+        "scope_selectors": {
+            "repository": [
+                {
+                    "kind": item.get("kind"),
+                    "decoration": item.get("decoration"),
+                    "pattern": item.get("pattern"),
+                }
+                for item in (rule.get("scope_selectors") or {}).get(
+                    "repository", []
+                )
+                if isinstance(item, dict)
+            ]
+        },
+    }
+
+
+def verify_immutable_policy(
+    *,
+    username: str,
+    password: str,
+    opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
+) -> None:
+    """Fail closed before build unless the exact Harbor rule is active."""
+    status, body, headers = _immutable_rules_response(
+        username=username, password=password, opener=opener
+    )
+    if status != 200:
+        raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}")
+    try:
+        rules = json.loads(body.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc
+    if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules):
+        raise RuntimeError("Harbor immutable rule list has an invalid shape")
+    _require_complete_rule_page(rules, headers)
+    expected = {
+        "disabled": False,
+        "action": "immutable",
+        "template": "immutable_template",
+        "tag_selectors": [
+            {
+                "kind": "doublestar",
+                "decoration": "matches",
+                "pattern": IMMUTABLE_TAG_PATTERN,
+            }
+        ],
+        "scope_selectors": {
+            "repository": [
+                {
+                    "kind": "doublestar",
+                    "decoration": "repoMatches",
+                    "pattern": IMMUTABLE_REPOSITORY_PATTERN,
+                }
+            ]
+        },
+    }
+    matches = [
+        _normalized_immutable_rule(item)
+        for item in rules
+        if _normalized_immutable_rule(item)["tag_selectors"]
+        == expected["tag_selectors"]
+        and _normalized_immutable_rule(item)["scope_selectors"]
+        == expected["scope_selectors"]
+    ]
+    if matches != [expected]:
+        raise RuntimeError("Harbor immutable build-tag policy is absent or not exact")
+
+
+def assert_tag_absent(
+    destination: str,
+    *,
+    username: str,
+    password: str,
+    opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
+) -> None:
+    """Reject replay before Kaniko can push an already-used immutable identity."""
+    status, _body = _artifact_response(
+        destination, username=username, password=password, opener=opener
+    )
+    if status == 404:
+        return
+    if status == 200:
+        raise RuntimeError("Harbor destination tag already exists; refusing overwrite")
+    raise RuntimeError(f"Harbor destination preflight returned HTTP {status}")
+
+
+def verify_registry_digest(
+    destination: str,
+    digest: str,
+    *,
+    username: str,
+    password: str,
+    opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
+) -> None:
+    """Verify Harbor independently resolves the pushed tag to Kaniko's digest."""
+    digest = _validated(digest, DIGEST_PATTERN, "image digest")
+    status, body = _artifact_response(
+        destination, username=username, password=password, opener=opener
+    )
+    if status != 200:
+        raise RuntimeError(f"Harbor manifest verification returned HTTP {status}")
+    try:
+        artifact = json.loads(body.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise RuntimeError("Harbor returned invalid artifact JSON") from exc
+    harbor_digest = str(artifact.get("digest") or "").strip()
+    if not DIGEST_PATTERN.fullmatch(harbor_digest):
+        raise RuntimeError("Harbor response omitted a valid artifact digest")
+    if harbor_digest != digest:
+        raise RuntimeError("Harbor digest does not match Kaniko evidence")
+    expected_tag = destination.rsplit(":", 1)[1]
+    matching_tags = [
+        item
+        for item in artifact.get("tags") or []
+        if isinstance(item, dict) and item.get("name") == expected_tag
+    ]
+    if len(matching_tags) != 1:
+        raise RuntimeError("Harbor artifact does not contain the expected tag")
+    if matching_tags[0].get("immutable") is not True:
+        raise RuntimeError("Harbor did not enforce the expected tag as immutable")
+
+
+def render_kustomization(source: str, digest: str, image: str = DEFAULT_IMAGE) -> str:
+    """Replace exactly one matching Kustomize image digest without reformatting."""
+    digest = _validated(digest, DIGEST_PATTERN, "image digest")
+    lines = source.splitlines(keepends=True)
+    matches: list[int] = []
+
+    for index, line in enumerate(lines):
+        if line.strip() != f"- name: {image}":
+            continue
+        name_indent = len(line) - len(line.lstrip())
+        for candidate_index in range(index + 1, len(lines)):
+            candidate = lines[candidate_index]
+            stripped = candidate.strip()
+            candidate_indent = len(candidate) - len(candidate.lstrip())
+            if stripped.startswith("- name:") and candidate_indent == name_indent:
+                break
+            if stripped.startswith("digest:") and candidate_indent > name_indent:
+                matches.append(candidate_index)
+                break
+
+    if len(matches) != 1:
+        raise ValueError(
+            f"expected exactly one digest for image {image!r}; found {len(matches)}"
+        )
+
+    index = matches[0]
+    newline = "\n" if lines[index].endswith("\n") else ""
+    prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())]
+    lines[index] = f"{prefix}digest: {digest}{newline}"
+    return "".join(lines)
+
+
+def write_release_artifacts(
+    *,
+    digest: str,
+    source_revision: str,
+    build_number: str,
+    destination: str,
+    kustomization: Path,
+    output_dir: Path,
+) -> dict[str, str]:
+    """Write a rendered manifest, patch, and credential-free release metadata."""
+    digest = _validated(digest, DIGEST_PATTERN, "image digest")
+    source_revision, build_number = validate_destination(
+        destination, source_revision, build_number
+    )
+    source = kustomization.read_text(encoding="utf-8")
+    rendered = render_kustomization(source, digest)
+    relative_name = kustomization.name
+    patch = "".join(
+        difflib.unified_diff(
+            source.splitlines(keepends=True),
+            rendered.splitlines(keepends=True),
+            fromfile=f"a/services/hermes/{relative_name}",
+            tofile=f"b/services/hermes/{relative_name}",
+        )
+    )
+    if not patch:
+        raise ValueError("published digest already matches the Flux manifest")
+    output_dir.mkdir(parents=True, exist_ok=True)
+    (output_dir / "hermes-kustomization.yaml").write_text(rendered, encoding="utf-8")
+    (output_dir / "hermes-image-update.patch").write_text(patch, encoding="utf-8")
+    metadata = {
+        "build_number": build_number,
+        "digest": digest,
+        "flux_image": f"{DEFAULT_IMAGE}@{digest}",
+        "image": DEFAULT_IMAGE,
+        "published_tag": destination,
+        "source_revision": source_revision,
+    }
+    (output_dir / "hermes-agent-image.json").write_text(
+        json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    return metadata
+
+
+def validate_release_artifacts(
+    *,
+    digest_file: Path,
+    image_file: Path,
+    source_revision: str,
+    build_number: str,
+    destination: str,
+    kustomization: Path,
+    output_dir: Path,
+) -> None:
+    """Revalidate the exact successful-build evidence without rewriting it."""
+    digest = validate_kaniko_evidence(
+        digest_text=digest_file.read_text(encoding="utf-8"),
+        image_text=image_file.read_text(encoding="utf-8"),
+        destination=destination,
+    )
+    source_revision, build_number = validate_destination(
+        destination, source_revision, build_number
+    )
+    expected_names = {
+        "hermes-agent-image.json",
+        "hermes-image-update.patch",
+        "hermes-kustomization.yaml",
+    }
+    entries = list(output_dir.iterdir())
+    if {entry.name for entry in entries} != expected_names or not all(
+        entry.is_file() and not entry.is_symlink() for entry in entries
+    ):
+        raise ValueError("release output must contain exactly three evidence files")
+    source = kustomization.read_text(encoding="utf-8")
+    rendered = render_kustomization(source, digest)
+    relative_name = kustomization.name
+    patch = "".join(
+        difflib.unified_diff(
+            source.splitlines(keepends=True),
+            rendered.splitlines(keepends=True),
+            fromfile=f"a/services/hermes/{relative_name}",
+            tofile=f"b/services/hermes/{relative_name}",
+        )
+    )
+    metadata = {
+        "build_number": build_number,
+        "digest": digest,
+        "flux_image": f"{DEFAULT_IMAGE}@{digest}",
+        "image": DEFAULT_IMAGE,
+        "published_tag": destination,
+        "source_revision": source_revision,
+    }
+    expected = {
+        "hermes-agent-image.json": json.dumps(metadata, indent=2, sort_keys=True) + "\n",
+        "hermes-image-update.patch": patch,
+        "hermes-kustomization.yaml": rendered,
+    }
+    for name, expected_text in expected.items():
+        if (output_dir / name).read_text(encoding="utf-8") != expected_text:
+            raise ValueError(f"release evidence is incomplete or mismatched: {name}")
+
+
+def _credentials() -> tuple[str, str]:
+    """Read the masked, runtime-only Jenkins credential environment."""
+    username = os.environ.get("HARBOR_USER", "")
+    password = os.environ.get("HARBOR_PASSWORD", "")
+    if not username or not password:
+        raise RuntimeError("Harbor credentials are unavailable")
+    return username, password
+
+
+def _common_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--source-revision", required=True)
+    parser.add_argument("--build-number", required=True)
+    parser.add_argument("--destination", required=True)
+
+
+def main() -> int:
+    """Fail closed around the unique tag, then verify and render after push."""
+    parser = argparse.ArgumentParser()
+    commands = parser.add_subparsers(dest="command", required=True)
+    absent = commands.add_parser("assert-absent")
+    _common_arguments(absent)
+    render = commands.add_parser("render")
+    _common_arguments(render)
+    render.add_argument("--digest-file", required=True, type=Path)
+    render.add_argument("--image-file", required=True, type=Path)
+    render.add_argument("--kustomization", required=True, type=Path)
+    render.add_argument("--output-dir", required=True, type=Path)
+    verify = commands.add_parser("verify-evidence")
+    _common_arguments(verify)
+    verify.add_argument("--digest-file", required=True, type=Path)
+    verify.add_argument("--image-file", required=True, type=Path)
+    verify.add_argument("--kustomization", required=True, type=Path)
+    verify.add_argument("--output-dir", required=True, type=Path)
+    args = parser.parse_args()
+
+    validate_destination(args.destination, args.source_revision, args.build_number)
+    if args.command == "verify-evidence":
+        validate_release_artifacts(
+            digest_file=args.digest_file,
+            image_file=args.image_file,
+            source_revision=args.source_revision,
+            build_number=args.build_number,
+            destination=args.destination,
+            kustomization=args.kustomization,
+            output_dir=args.output_dir,
+        )
+        return 0
+
+    username, password = _credentials()
+    if args.command == "assert-absent":
+        verify_immutable_policy(username=username, password=password)
+        assert_tag_absent(args.destination, username=username, password=password)
+        return 0
+
+    digest = validate_kaniko_evidence(
+        digest_text=args.digest_file.read_text(encoding="utf-8"),
+        image_text=args.image_file.read_text(encoding="utf-8"),
+        destination=args.destination,
+    )
+    verify_registry_digest(
+        args.destination, digest, username=username, password=password
+    )
+    write_release_artifacts(
+        digest=digest,
+        source_revision=args.source_revision,
+        build_number=args.build_number,
+        destination=args.destination,
+        kustomization=args.kustomization,
+        output_dir=args.output_dir,
+    )
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover - exercised through main()
+    raise SystemExit(main())

clusters/atlas/flux-system/applications/harbor/kustomization.yaml

@@ -15,7 +15,13 @@ spec:
     kind: GitRepository
     name: flux-system
     namespace: flux-system
-  wait: false
+  wait: true
+  timeout: 10m
+  healthChecks:
+    - apiVersion: batch/v1
+      kind: Job
+      name: harbor-hermes-agent-immutability-ensure-1
+      namespace: harbor
   dependsOn:
     - name: core
     - name: longhorn

clusters/atlas/flux-system/applications/hermes/kustomization.yaml

@@ -60,3 +60,4 @@ spec:
     - name: keycloak
     - name: longhorn
     - name: vault
+    - name: jenkins

clusters/atlas/flux-system/applications/jenkins/kustomization.yaml

@@ -6,10 +6,9 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
-    atlas.bstein.dev/suspend-reason: "CI controller changes are applied only during planned maintenance."
 spec:
   interval: 10m
-  suspend: true
+  suspend: false
   path: ./services/jenkins
   prune: true
   sourceRef:
@@ -18,6 +17,8 @@ spec:
   targetNamespace: jenkins
   dependsOn:
     - name: helm
+    - name: harbor
+    - name: vault-hermes-jenkins-token-seed
   healthChecks:
     - apiVersion: apps/v1
       kind: Deployment

clusters/atlas/flux-system/applications/kustomization.yaml

@@ -4,6 +4,7 @@ kind: Kustomization
 resources:
   - gitea/kustomization.yaml
   - vault/kustomization.yaml
+  - vault-hermes-jenkins-token-seed/kustomization.yaml
   - vaultwarden/kustomization.yaml
   - comms/kustomization.yaml
   - crypto/kustomization.yaml

clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml

@@ -0,0 +1,26 @@
+# clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: vault-hermes-jenkins-token-seed
+  namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+spec:
+  interval: 10m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  path: ./services/vault-hermes-jenkins-token-seed
+  targetNamespace: vault
+  prune: true
+  wait: true
+  timeout: 5m
+  dependsOn:
+    - name: vault
+  healthChecks:
+    - apiVersion: batch/v1
+      kind: Job
+      name: vault-hermes-jenkins-build-token-seed-2
+      namespace: vault

clusters/atlas/flux-system/applications/vault/kustomization.yaml

@@ -16,6 +16,11 @@ spec:
   targetNamespace: vault
   prune: true
   wait: true
+  healthChecks:
+    - apiVersion: batch/v1
+      kind: Job
+      name: vault-k8s-auth-hermes-9
+      namespace: vault
   dependsOn:
     - name: longhorn
     - name: helm

dockerfiles/Dockerfile.hermes-agent

@@ -15,6 +15,10 @@ RUN apt-get update \
 # Paid/provider-backed search remains selectable through normal Hermes config.
 RUN uv pip install --python /opt/hermes/.venv/bin/python ddgs==9.14.4
 
+ARG HERMES_KANIKO_HEREDOC_COMPAT=0
+COPY dockerfiles/Dockerfile.hermes-agent /tmp/hermes-agent.Dockerfile
+COPY dockerfiles/hermes-kaniko-heredoc-runner.py /tmp/hermes-kaniko-heredoc-runner.py
+
 # Keep dashboard chat sockets tied to the intended React mount and conversation.
 # A resumed conversation needs a different PTY attachment key from a fresh chat;
 # reconnects to that same conversation must keep using the same key.
@@ -252,6 +256,7 @@ for (const [before, after, label] of [
 }
 fs.writeFileSync(sidebarPath, sidebar);
 NODE
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 1 ;; *) exit 2 ;; esac
 
 # The upstream OIDC gate authenticates users but deliberately treats the
 # dashboard as one shared workstation. Allow a deployment to narrow that
@@ -324,6 +329,7 @@ for before, after, label in (
     source = source.replace(before, after, 1)
 path.write_text(source)
 PY
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 2 ;; *) exit 2 ;; esac
 
 # Give trusted plugins a pre-turn routing hook. It runs after fallback runtime
 # restoration but before Hermes builds its provider-specific system prompt.
@@ -493,6 +499,7 @@ if oneshot.count(oneshot_before) != 1:
     )
 oneshot_path.write_text(oneshot.replace(oneshot_before, oneshot_after, 1))
 PY
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 3 ;; *) exit 2 ;; esac
 
 # Hermes WebUI sends its model/provider/reasoning selection on /v1/runs.
 # Upstream currently applies only statically declared model_routes there, so
@@ -628,17 +635,127 @@ runs_source = runs_source.replace(runs_agent_before, runs_agent_after, 1)
 source = source[:runs_start] + runs_source
 path.write_text(source)
 PY
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 4 ;; *) exit 2 ;; esac
+
+# ``create_task(initial_status="blocked")`` is an explicit operator park, but
+# upstream only treats later block_task() events as sticky. Recognize the
+# existing created(status=blocked) event too. This also protects tasks created
+# before this image patch without making dependency or circuit-breaker blocks
+# depend on a new persisted field.
+RUN python - <<'PY'
+from pathlib import Path
+
+db_path = Path("/opt/hermes/hermes_cli/kanban_db.py")
+db_source = db_path.read_text()
+
+# The rewritten sticky-block gate below classifies a task from the ``created``
+# event payload that ``create_task`` writes. That producer is upstream code we
+# do not own, so anchor the exact contract the consumer depends on instead of
+# trusting it: an ``initial_status="blocked"`` park must resolve ``task_status``
+# to ``"blocked"``, every other creation must resolve it to something else, and
+# the ``created`` event must carry that same variable under the ``"status"``
+# key. If upstream renames the key, drops it, hardcodes it, or stops deriving it
+# from ``initial_status``, fail the build here rather than ship a consumer that
+# silently mis-classifies every task it reads. These assert only -- they
+# deliberately do not rewrite the producer. The regression suite copied in below
+# is the second net: it proves the same contract against the real create_task,
+# for the drift these textual anchors cannot see.
+producer_park_anchor = '''                if initial_status == "blocked":
+                    task_status = "blocked"
+'''
+if db_source.count(producer_park_anchor) != 1:
+    raise SystemExit(
+        "Hermes Kanban create_task park semantics changed: expected 1, "
+        f"found {db_source.count(producer_park_anchor)}"
+    )
+# The mirror of the park branch: every non-park creation must resolve to a
+# status the gate does *not* read as a park. Without this, drift that widened
+# the ladder to park ordinary work would leave both other anchors intact.
+producer_unparked_anchor = '''                elif triage:
+                    task_status = "triage"
+                else:
+                    task_status = "ready"
+'''
+if db_source.count(producer_unparked_anchor) != 1:
+    raise SystemExit(
+        "Hermes Kanban create_task non-park status ladder changed: expected 1, "
+        f"found {db_source.count(producer_unparked_anchor)}"
+    )
+producer_event_anchor = '''                _append_event(
+                    conn,
+                    task_id,
+                    "created",
+                    {
+                        "assignee": assignee,
+                        "status": task_status,
+'''
+if db_source.count(producer_event_anchor) != 1:
+    raise SystemExit(
+        "Hermes Kanban created-event status payload changed: expected 1, "
+        f"found {db_source.count(producer_event_anchor)}"
+    )
+
+sticky_doc_before = '''    The cheapest signal that distinguishes the two is the most recent
+    ``"blocked"`` / ``"unblocked"`` event for the task.  If the most
+    recent one is ``"blocked"`` (or there is a ``"blocked"`` event and
+    no ``"unblocked"`` event has fired since), the task is sticky and
+    ``recompute_ready`` must *not* auto-promote it.
+
+    Returns ``False`` when there is no such event at all (e.g. the task
+'''
+sticky_doc_after = '''    The cheapest signal that distinguishes the two is the most recent
+    ``"created"`` / ``"blocked"`` / ``"unblocked"`` event for the task.
+    A ``"created"`` event whose payload has ``status="blocked"`` is the
+    explicit park requested by ``create_task(initial_status="blocked")``.
+    A later ``"unblocked"`` event clears either kind of sticky block.
+
+    Returns ``False`` when there is no such event at all (e.g. the task
+'''
+if db_source.count(sticky_doc_before) != 1:
+    raise SystemExit(
+        "Hermes Kanban sticky-block documentation changed: expected 1, "
+        f"found {db_source.count(sticky_doc_before)}"
+    )
+db_source = db_source.replace(sticky_doc_before, sticky_doc_after, 1)
+
+sticky_before = '''    row = conn.execute(
+        "SELECT kind FROM task_events "
+        "WHERE task_id = ? AND kind IN ('blocked', 'unblocked') "
+        "ORDER BY id DESC LIMIT 1",
+        (task_id,),
+    ).fetchone()
+    return bool(row) and row["kind"] == "blocked"
+'''
+sticky_after = '''    row = conn.execute(
+        "SELECT kind, payload FROM task_events "
+        "WHERE task_id = ? AND kind IN ('created', 'blocked', 'unblocked') "
+        "ORDER BY id DESC LIMIT 1",
+        (task_id,),
+    ).fetchone()
+    if not row or row["kind"] == "unblocked":
+        return False
+    if row["kind"] == "blocked":
+        return True
+    try:
+        payload = json.loads(row["payload"] or "{}")
+    except (TypeError, ValueError):
+        return False
+    if not isinstance(payload, dict):
+        return False
+    return payload.get("status") == "blocked"
+'''
+if db_source.count(sticky_before) != 1:
+    raise SystemExit(
+        "Hermes Kanban sticky-block gate changed: expected 1, "
+        f"found {db_source.count(sticky_before)}"
+    )
+db_source = db_source.replace(sticky_before, sticky_after, 1)
 
 # A manual evidence-based close must not expose a parked task to the dispatcher
 # between separate `unblock` and `complete` commands. Upstream already permits
 # direct completion from `blocked`; extend the same atomic path to `scheduled`
 # and make that contract visible in CLI help so agents do not create a ready
 # window that can launch redundant work.
-RUN python - <<'PY'
-from pathlib import Path
-
-db_path = Path("/opt/hermes/hermes_cli/kanban_db.py")
-db_source = db_path.read_text()
 state_before = '''                 WHERE id = ?
                    AND status IN ('running', 'ready', 'blocked')
 '''
@@ -670,6 +787,11 @@ if cli_source.count(help_before) != 1:
     )
 cli_path.write_text(cli_source.replace(help_before, help_after, 1))
 PY
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 5 ;; *) exit 2 ;; esac
+
+COPY dockerfiles/hermes-kanban-blocked-regression.py /tmp/hermes-kanban-blocked-regression.py
+RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py \
+    && rm /tmp/hermes-kanban-blocked-regression.py
 
 COPY dockerfiles/hermes-python-sandbox-tool.py /opt/hermes/tools/python_sandbox_tool.py
 COPY dockerfiles/hermes-public-extract/__init__.py /opt/hermes/plugins/web/public_extract/__init__.py
@@ -703,6 +825,7 @@ if source.count(before) != 1:
     )
 path.write_text(source.replace(before, after, 1))
 PY
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 6 ;; *) exit 2 ;; esac
 
 # Keep the dashboard's account cards aligned with the managed provider lanes.
 # Switchyard receives Claude Code subscription OAuth through the environment;
@@ -769,6 +892,7 @@ for before, after, label in replacements:
 
 path.write_text(source)
 PY
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 7 ;; *) exit 2 ;; esac
 
 # Make parent-linked API workers first-class live dashboard sessions. They stay
 # active until the API runner closes them, even during a long final model call;
@@ -862,6 +986,7 @@ source = source.replace(row_before, row_after)
 
 path.write_text(source)
 PY
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 8 ;; *) exit 2 ;; esac
 
 # Keep the browser terminal reliable across GPU context loss, make durable
 # worker lineage visible, expose an accessible API-worker transcript, and
@@ -1430,9 +1555,12 @@ function RootRedirect() {
   fs.writeFileSync(path, source);
 }
 NODE
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 9 ;; *) exit 2 ;; esac
 
 COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate
 
+RUN rm -f /tmp/hermes-agent.Dockerfile /tmp/hermes-kaniko-heredoc-runner.py
+
 RUN cd /opt/hermes/web \
     && npm run build \
     && grep -Fq 'await api.getSessions(1, 0' src/pages/ChatPage.tsx \

dockerfiles/Dockerfile.hermes-agent.dockerignore

@@ -1,6 +1,9 @@
 **
+!dockerfiles/hermes-kanban-blocked-regression.py
 !dockerfiles/hermes-python-sandbox-tool.py
 !dockerfiles/hermes-public-extract/
 !dockerfiles/hermes-public-extract/**
 !dockerfiles/hermes-session-activity-panel.tsx
 !dockerfiles/hermes-session-migrate.py
+!dockerfiles/Dockerfile.hermes-agent
+!dockerfiles/hermes-kaniko-heredoc-runner.py

dockerfiles/hermes-kanban-blocked-regression.py

@@ -0,0 +1,249 @@
+"""Regression tests for Hermes Kanban blocked-task scheduling semantics."""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+
+_HERMES_HOME = tempfile.TemporaryDirectory(prefix="hermes-kanban-test-home-")
+os.environ["HERMES_HOME"] = _HERMES_HOME.name
+
+from hermes_cli import kanban_db  # noqa: E402
+
+
+class BlockedTaskSchedulingTests(unittest.TestCase):
+    """Exercise the real upstream database API against isolated databases."""
+
+    def setUp(self) -> None:
+        self._database_dir = tempfile.TemporaryDirectory(
+            prefix="hermes-kanban-test-db-"
+        )
+        self.connection = kanban_db.connect(
+            Path(self._database_dir.name) / "kanban.db"
+        )
+
+    def tearDown(self) -> None:
+        self.connection.close()
+        self._database_dir.cleanup()
+
+    def create_task(self, title: str, **kwargs: object) -> str:
+        return kanban_db.create_task(self.connection, title=title, **kwargs)
+
+    def status(self, task_id: str) -> str:
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertIsNotNone(task)
+        return task.status
+
+    def created_payload(self, task_id: str) -> dict:
+        """Return the decoded payload of the task's ``created`` event."""
+        row = self.connection.execute(
+            "SELECT payload FROM task_events "
+            "WHERE task_id = ? AND kind = 'created' ORDER BY id LIMIT 1",
+            (task_id,),
+        ).fetchone()
+        self.assertIsNotNone(row, "create_task emitted no 'created' event")
+        return json.loads(row["payload"] or "{}")
+
+    def set_created_payload(self, task_id: str, payload: object) -> None:
+        """Overwrite a ``created`` event payload to simulate producer drift."""
+        with self.connection:
+            self.connection.execute(
+                "UPDATE task_events SET payload = ? "
+                "WHERE task_id = ? AND kind = 'created'",
+                (payload, task_id),
+            )
+
+    def block_via_circuit_breaker(self, title: str) -> str:
+        """Create + claim a normal task, then trip the breaker once on it.
+
+        The resulting task is ``blocked`` with exactly one recorded failure and
+        -- critically -- no ``blocked`` event, because the breaker emits
+        ``gave_up``. Its most recent created/blocked/unblocked event is
+        therefore its own ``created`` event, which is what makes it the right
+        probe for the created-event branch of the sticky gate.
+        """
+        task_id = self.create_task(title)
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        self.assertTrue(
+            kanban_db._record_spawn_failure(
+                self.connection,
+                task_id,
+                "worker failed",
+                failure_limit=1,
+            )
+        )
+        self.assertEqual(self.status(task_id), "blocked")
+        return task_id
+
+    def test_initial_block_without_parents_is_sticky(self) -> None:
+        task_id = self.create_task("operator parked", initial_status="blocked")
+
+        self.assertEqual(kanban_db.recompute_ready(self.connection), 0)
+        self.assertEqual(self.status(task_id), "blocked")
+
+    def test_initial_block_with_complete_parents_is_sticky(self) -> None:
+        parent_id = self.create_task("completed prerequisite")
+        self.assertTrue(kanban_db.complete_task(self.connection, parent_id))
+        task_id = self.create_task(
+            "operator parked after prerequisite",
+            initial_status="blocked",
+            parents=[parent_id],
+        )
+
+        self.assertEqual(kanban_db.recompute_ready(self.connection), 0)
+        self.assertEqual(self.status(task_id), "blocked")
+
+    def test_initial_block_remains_sticky_when_parent_later_completes(self) -> None:
+        parent_id = self.create_task("incomplete prerequisite")
+        task_id = self.create_task(
+            "operator parked before prerequisite completes",
+            initial_status="blocked",
+            parents=[parent_id],
+        )
+
+        self.assertEqual(self.status(task_id), "blocked")
+        self.assertTrue(kanban_db.complete_task(self.connection, parent_id))
+        self.assertEqual(self.status(task_id), "blocked")
+
+    def test_initial_block_is_released_by_an_explicit_unblock(self) -> None:
+        task_id = self.create_task("operator parked", initial_status="blocked")
+
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertEqual(self.status(task_id), "ready")
+
+    def test_dependency_block_promotes_after_parent_completes(self) -> None:
+        parent_id = self.create_task("incomplete prerequisite")
+        task_id = self.create_task("dependency gated", parents=[parent_id])
+        self.assertEqual(self.status(task_id), "todo")
+
+        self.assertTrue(kanban_db.complete_task(self.connection, parent_id))
+        self.assertEqual(self.status(task_id), "ready")
+
+    def test_explicit_block_and_unblock_remain_sticky_and_reversible(self) -> None:
+        task_id = self.create_task("worker handoff")
+
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="human review required",
+            )
+        )
+        self.assertEqual(kanban_db.recompute_ready(self.connection), 0)
+        self.assertEqual(self.status(task_id), "blocked")
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertEqual(self.status(task_id), "ready")
+
+    def test_circuit_breaker_block_is_not_auto_promoted(self) -> None:
+        task_id = self.create_task("repeated worker failure")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+
+        # Upstream has no public circuit-breaker entry point; exercise the
+        # dispatcher helper that owns its persisted failure-limit semantics.
+        self.assertTrue(
+            kanban_db._record_spawn_failure(
+                self.connection,
+                task_id,
+                "worker failed",
+                failure_limit=1,
+            )
+        )
+        self.assertEqual(self.status(task_id), "blocked")
+        self.assertEqual(
+            kanban_db.recompute_ready(self.connection, failure_limit=1),
+            0,
+        )
+        self.assertEqual(self.status(task_id), "blocked")
+
+    def test_circuit_breaker_block_is_promoted_below_the_failure_limit(self) -> None:
+        """Non-sticky oracle for the created-event branch of the sticky gate.
+
+        A circuit-breaker block must still auto-recover once the effective
+        failure limit rises above the recorded failure count -- that is the
+        pre-existing upstream contract the sticky-block patch must not widen.
+
+        This task's most recent created/blocked/unblocked event is its own
+        ``created`` event carrying ``status="ready"``, so it is exactly the
+        case that a mis-classifying gate strands. It fails if the gate treats
+        any ``created`` event as a park (over-broad consumer) and it fails if
+        ``create_task`` drifts to labelling every ``created`` event
+        ``"blocked"`` (over-broad producer), while the
+        circuit-breaker-at-current-limit case above still holds.
+        """
+        task_id = self.block_via_circuit_breaker("recoverable worker failure")
+
+        self.assertFalse(
+            kanban_db._has_sticky_block(self.connection, task_id),
+            "a circuit-breaker block must never be classified as a park",
+        )
+        self.assertEqual(
+            kanban_db.recompute_ready(self.connection, failure_limit=2),
+            1,
+        )
+        self.assertEqual(self.status(task_id), "ready")
+
+    def test_created_event_carries_the_status_the_sticky_gate_reads(self) -> None:
+        """Producer contract asserted directly against the real create_task.
+
+        The sticky gate reads ``created`` payload ``status``. Pin both poles of
+        that contract so producer drift is a test failure here as well as a
+        build failure at the Dockerfile source anchor.
+        """
+        parked_id = self.create_task("operator parked", initial_status="blocked")
+        normal_id = self.create_task("ordinary work")
+
+        self.assertEqual(self.created_payload(parked_id).get("status"), "blocked")
+        self.assertEqual(self.created_payload(normal_id).get("status"), "ready")
+
+    def test_non_blocked_created_payloads_are_not_treated_as_a_park(self) -> None:
+        """Only ``status="blocked"`` parks; every other value must promote."""
+        for payload_status in ("ready", "todo", "running", "triage"):
+            with self.subTest(payload_status=payload_status):
+                task_id = self.block_via_circuit_breaker(
+                    f"breaker trip with {payload_status} created payload"
+                )
+                self.set_created_payload(
+                    task_id, json.dumps({"status": payload_status})
+                )
+
+                self.assertFalse(
+                    kanban_db._has_sticky_block(self.connection, task_id)
+                )
+                self.assertEqual(
+                    kanban_db.recompute_ready(self.connection, failure_limit=2),
+                    1,
+                )
+                self.assertEqual(self.status(task_id), "ready")
+
+    def test_malformed_created_payloads_fail_closed_to_not_parked(self) -> None:
+        """Undecodable or non-object payloads must not park -- or crash.
+
+        ``recompute_ready`` walks every blocked task in one pass, so an
+        exception escaping the gate would stall promotion board-wide. Falling
+        back to "not parked" preserves upstream's pre-patch auto-recover
+        default for rows the gate cannot interpret.
+        """
+        malformed = (None, "", "   ", "{not json", "[]", "null", '"blocked"', "17")
+        for payload in malformed:
+            with self.subTest(payload=payload):
+                task_id = self.block_via_circuit_breaker(
+                    f"breaker trip with payload {payload!r}"
+                )
+                self.set_created_payload(task_id, payload)
+
+                self.assertFalse(
+                    kanban_db._has_sticky_block(self.connection, task_id)
+                )
+                self.assertEqual(
+                    kanban_db.recompute_ready(self.connection, failure_limit=2),
+                    1,
+                )
+                self.assertEqual(self.status(task_id), "ready")
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

dockerfiles/hermes-kaniko-heredoc-runner.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""Replay the reviewed Hermes Dockerfile heredocs for pinned Kaniko v1.23.2.
+
+Kaniko v1.23.2 parses shell heredoc bodies into ``RunCommand.Files`` but its RUN
+implementation executes only ``CmdLine``. The interpreters therefore receive
+empty stdin and exit successfully. Docker and BuildKit execute these blocks
+normally, so this compatibility runner is enabled only by the Kaniko pipeline.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import subprocess
+from pathlib import Path
+
+
+MAX_DOCKERFILE_BYTES = 2_000_000
+BLOCKS = (
+    ("RUN node <<'NODE'", "NODE", ("node",)),
+    ("RUN python - <<'PY'", "PY", ("python", "-")),
+)
+EXPECTED_COMMANDS = ("node", *("python",) * 7, "node")
+PARSER_DIRECTIVE = re.compile(
+    r"^\s*#\s*([a-z]+)\s*=\s*(.*?)\s*$", re.IGNORECASE
+)
+
+
+def _escape_character(lines: list[str]) -> str:
+    """Return the one Dockerfile parser escape directive, or its default."""
+    escape = "\\"
+    directives: set[str] = set()
+    for line in lines:
+        if not line.strip():
+            break
+        match = PARSER_DIRECTIVE.fullmatch(line)
+        if not match:
+            break
+        name, value = match.group(1).lower(), match.group(2)
+        if name in directives:
+            raise ValueError(f"multiple Dockerfile {name} directives")
+        directives.add(name)
+        if name == "escape":
+            if value not in {"\\", "`"}:
+                raise ValueError("unsupported Dockerfile escape directive")
+            escape = value
+    return escape
+
+
+def _continued(line: str, escape: str) -> bool:
+    """Match Docker's unescaped continuation marker at physical line end."""
+    stripped = line.rstrip()
+    count = len(stripped) - len(stripped.rstrip(escape))
+    return count % 2 == 1
+
+
+def _logical_instruction(
+    lines: list[str], index: int, escape: str
+) -> tuple[str, int]:
+    """Normalize escape-newline pairs, including split opcodes/operators."""
+    logical = lines[index]
+    while _continued(logical.split("\n")[-1], escape):
+        logical = logical.rstrip()
+        logical = logical[:-1]
+        index += 1
+        while index < len(lines) and lines[index].lstrip().startswith("#"):
+            index += 1
+        if index >= len(lines) or not lines[index].strip():
+            raise ValueError("unterminated Dockerfile line continuation")
+        logical += lines[index].lstrip()
+    return logical, index
+
+
+def extract_blocks(source: str) -> list[tuple[tuple[str, ...], str]]:
+    """Inventory every RUN heredoc, then return only the reviewed contract."""
+    markers = {start: (end, command) for start, end, command in BLOCKS}
+    lines = source.splitlines()
+    escape = _escape_character(lines)
+    blocks: list[tuple[tuple[str, ...], str]] = []
+    index = 0
+    while index < len(lines):
+        marker = markers.get(lines[index])
+        if marker is None:
+            if not lines[index].strip() or lines[index].lstrip().startswith("#"):
+                index += 1
+                continue
+            logical, index = _logical_instruction(lines, index, escape)
+            stripped = logical.lstrip()
+            if stripped[:3].upper() == "RUN" and (
+                len(stripped) == 3 or stripped[3].isspace()
+            ):
+                if "<<" in stripped:
+                    raise ValueError(
+                        "unsupported RUN heredoc outside the exact reviewed contract"
+                    )
+            index += 1
+            continue
+        end, command = marker
+        body_start = index + 1
+        index = body_start
+        while index < len(lines) and lines[index] != end:
+            index += 1
+        if index == len(lines):
+            raise ValueError(f"unterminated {command[0]} heredoc")
+        body = "\n".join(lines[body_start:index]) + "\n"
+        if not body.strip():
+            raise ValueError(f"empty {command[0]} heredoc")
+        blocks.append((command, body))
+        index += 1
+
+    commands = tuple(command[0] for command, _body in blocks)
+    if commands != EXPECTED_COMMANDS:
+        raise ValueError(
+            "Hermes Dockerfile heredoc contract changed: "
+            f"expected {EXPECTED_COMMANDS!r}, received {commands!r}"
+        )
+    return blocks
+
+
+def replay(dockerfile: Path, block_index: int) -> None:
+    """Execute one exact heredoc after inventorying the complete Dockerfile."""
+    size = dockerfile.stat().st_size
+    if size < 1 or size > MAX_DOCKERFILE_BYTES:
+        raise ValueError("Hermes Dockerfile size is outside the reviewed boundary")
+    source = dockerfile.read_text(encoding="utf-8")
+    blocks = extract_blocks(source)
+    if block_index < 1 or block_index > len(blocks):
+        raise ValueError("heredoc block index is outside the reviewed contract")
+    command, body = blocks[block_index - 1]
+    print(f"replaying reviewed Dockerfile heredoc {block_index}/{len(blocks)}")
+    subprocess.run(command, input=body, text=True, check=True)
+
+
+def main() -> int:
+    """Parse the one explicit Dockerfile path and replay its reviewed patches."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dockerfile", required=True, type=Path)
+    parser.add_argument("--block-index", required=True, type=int)
+    args = parser.parse_args()
+    replay(args.dockerfile, args.block_index)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

services/harbor/hermes-agent-immutability-job.yaml

@@ -0,0 +1,79 @@
+# services/harbor/hermes-agent-immutability-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: harbor-hermes-agent-immutability-ensure-1
+  namespace: harbor
+spec:
+  backoffLimit: 2
+  activeDeadlineSeconds: 600
+  template:
+    metadata:
+      annotations:
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/agent-run-as-user: "65532"
+        vault.hashicorp.com/agent-run-as-group: "65532"
+        vault.hashicorp.com/role: harbor-policy-bootstrap
+        vault.hashicorp.com/agent-inject-secret-harbor-admin-password: kv/data/atlas/harbor/harbor-core
+        vault.hashicorp.com/agent-inject-template-harbor-admin-password: |
+          {{- with secret "kv/data/atlas/harbor/harbor-core" -}}
+          {{ .Data.data.harbor_admin_password }}
+          {{- end -}}
+    spec:
+      serviceAccountName: harbor-policy-bootstrap
+      enableServiceLinks: false
+      restartPolicy: Never
+      nodeSelector:
+        hardware: rpi5
+        kubernetes.io/arch: arm64
+        node-role.kubernetes.io/worker: "true"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values: [titan-04, titan-14, titan-18, titan-19, titan-24]
+      securityContext:
+        fsGroup: 65532
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: ensure
+          image: docker.io/library/python@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757
+          imagePullPolicy: IfNotPresent
+          command: [python3, /scripts/harbor_hermes_agent_immutability_ensure.py]
+          env:
+            - name: HARBOR_API_ORIGIN
+              value: https://registry.bstein.dev/api/v2.0
+            - name: HARBOR_ADMIN_PASSWORD_FILE
+              value: /vault/secrets/harbor-admin-password
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+            runAsGroup: 65532
+            runAsNonRoot: true
+            runAsUser: 65532
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - name: scripts
+              mountPath: /scripts
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+          resources:
+            requests: {cpu: 25m, memory: 32Mi}
+            limits: {cpu: 250m, memory: 128Mi}
+      volumes:
+        - name: scripts
+          configMap:
+            name: harbor-hermes-agent-immutability-script
+            defaultMode: 0555
+        - name: tmp
+          emptyDir: {}

services/harbor/kustomization.yaml

@@ -12,9 +12,14 @@ resources:
   - certificate.yaml
   - helmrelease.yaml
   - vault-sync-deployment.yaml
+  - policy-bootstrap-serviceaccount.yaml
+  - hermes-agent-immutability-job.yaml
   - bootstrap-jobs/cassandra-registry-ensure-job.yaml
   - image.yaml
 configMapGenerator:
   - name: harbor-vault-entrypoint
     files:
       - scripts/vault-entrypoint.sh
+  - name: harbor-hermes-agent-immutability-script
+    files:
+      - harbor_hermes_agent_immutability_ensure.py=scripts/harbor_hermes_agent_immutability_ensure.py

services/harbor/policy-bootstrap-serviceaccount.yaml

@@ -0,0 +1,7 @@
+# services/harbor/policy-bootstrap-serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: harbor-policy-bootstrap
+  namespace: harbor
+automountServiceAccountToken: true

services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py

@@ -0,0 +1,378 @@
+#!/usr/bin/env python3
+"""Create and verify the narrowly scoped Hermes agent immutable-tag rule."""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+
+PROJECT = "bstein"
+REPOSITORY_PATTERN = "hermes-agent"
+TAG_PATTERN = "git-*-build-*"
+PUBLISH_ROBOT = "robot$jenkins-pipelines"
+EXPECTED_ORIGIN = "https://registry.bstein.dev/api/v2.0"
+MAX_RESPONSE = 1_048_576
+TRANSIENT_STATUSES = {429, 502, 503, 504}
+EXPECTED_RULE = {
+    "disabled": False,
+    "action": "immutable",
+    "template": "immutable_template",
+    "tag_selectors": [
+        {
+            "kind": "doublestar",
+            "decoration": "matches",
+            "pattern": TAG_PATTERN,
+        }
+    ],
+    "scope_selectors": {
+        "repository": [
+            {
+                "kind": "doublestar",
+                "decoration": "repoMatches",
+                "pattern": REPOSITORY_PATTERN,
+            }
+        ]
+    },
+}
+
+
+class NoRedirect(urllib.request.HTTPRedirectHandler):
+    """Prevent Basic credentials from following an unexpected redirect."""
+
+    def redirect_request(self, _request, _file, _code, _message, _headers, _url):
+        return None
+
+
+class HarborUnavailable(RuntimeError):
+    """Harbor is not ready yet, rather than returning a policy decision."""
+
+
+def normalized_rule(rule: dict[str, Any]) -> dict[str, Any]:
+    """Return only the immutable contract fields Harbor must preserve."""
+    return {
+        "disabled": bool(rule.get("disabled", False)),
+        "action": rule.get("action"),
+        "template": rule.get("template"),
+        "tag_selectors": [
+            {
+                "kind": selector.get("kind"),
+                "decoration": selector.get("decoration"),
+                "pattern": selector.get("pattern"),
+            }
+            for selector in rule.get("tag_selectors") or []
+            if isinstance(selector, dict)
+        ],
+        "scope_selectors": {
+            "repository": [
+                {
+                    "kind": selector.get("kind"),
+                    "decoration": selector.get("decoration"),
+                    "pattern": selector.get("pattern"),
+                }
+                for selector in (rule.get("scope_selectors") or {}).get(
+                    "repository", []
+                )
+                if isinstance(selector, dict)
+            ]
+        },
+    }
+
+
+def targets_hermes_builds(rule: dict[str, Any]) -> bool:
+    """Detect a rule that claims this exact repository and tag selector."""
+    normalized = normalized_rule(rule)
+    return (
+        normalized["tag_selectors"] == EXPECTED_RULE["tag_selectors"]
+        and normalized["scope_selectors"] == EXPECTED_RULE["scope_selectors"]
+    )
+
+
+class HarborClient:
+    """Bounded same-origin client for Harbor's immutable-tag API."""
+
+    def __init__(self, origin: str, username: str, password: str) -> None:
+        normalized_origin = origin.rstrip("/")
+        if normalized_origin != EXPECTED_ORIGIN:
+            raise ValueError("Harbor API origin is not the pinned production API")
+        self.origin = normalized_origin
+        token = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
+        self.headers = {"Authorization": f"Basic {token}"}
+        self.opener = urllib.request.build_opener(NoRedirect())
+
+    def request(
+        self, method: str, path: str, payload: dict[str, Any] | None = None
+    ) -> tuple[int, bytes, dict[str, str]]:
+        """Issue one request, returning even non-2xx responses for strict checks."""
+        data = None
+        headers = dict(self.headers)
+        if payload is not None:
+            data = json.dumps(payload, separators=(",", ":")).encode()
+            headers["Content-Type"] = "application/json"
+        request = urllib.request.Request(
+            f"{self.origin}{path}", data=data, headers=headers, method=method
+        )
+        try:
+            response = self.opener.open(request, timeout=20)
+        except urllib.error.HTTPError as exc:
+            response = exc
+        except (urllib.error.URLError, TimeoutError) as exc:
+            raise HarborUnavailable("Harbor policy API is unavailable") from exc
+        with response:
+            body = response.read(MAX_RESPONSE + 1)
+            if len(body) > MAX_RESPONSE:
+                raise RuntimeError("Harbor response exceeded the size limit")
+            return int(response.status), body, dict(response.headers)
+
+
+def list_rules(client: HarborClient) -> list[dict[str, Any]]:
+    """Read and validate the complete small rule set for the project."""
+    path = f"/projects/{PROJECT}/immutabletagrules?page=1&page_size=100"
+    status, body, headers = client.request("GET", path)
+    if status in TRANSIENT_STATUSES:
+        raise HarborUnavailable(f"Harbor immutable rule list returned HTTP {status}")
+    if status != 200:
+        raise RuntimeError(f"Harbor immutable rule list returned HTTP {status}")
+    values = _json_list(body, "immutable rule")
+    _require_complete_page(values, headers, "immutable rule")
+    return values
+
+
+def _json_list(body: bytes, label: str) -> list[dict[str, Any]]:
+    """Decode one bounded Harbor list without accepting a partial shape."""
+    try:
+        value = json.loads(body.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise RuntimeError(f"Harbor returned invalid {label} JSON") from exc
+    if not isinstance(value, list) or not all(isinstance(item, dict) for item in value):
+        raise RuntimeError(f"Harbor {label} list has an invalid shape")
+    return value
+
+
+def _require_complete_page(
+    values: list[dict[str, Any]], headers: dict[str, str], label: str
+) -> None:
+    """Reject a truncated first page or a proxy that strips count evidence."""
+    raw_total = next(
+        (value for key, value in headers.items() if key.lower() == "x-total-count"),
+        None,
+    )
+    if raw_total is None or not str(raw_total).isdecimal():
+        raise RuntimeError(f"Harbor {label} list omitted a valid total count")
+    if int(raw_total) != len(values):
+        raise RuntimeError(f"Harbor {label} list was truncated")
+
+
+def _allowed(access: dict[str, Any], resource: str, action: str) -> bool:
+    """Match one positive robot permission, accepting Harbor's default effect."""
+    return (
+        access.get("resource") == resource
+        and access.get("action") == action
+        and access.get("effect") in (None, "", "allow")
+    )
+
+
+def _publisher_scope(robot: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+    """Return the existing bstein scope after validating its push boundary."""
+    permissions = robot.get("permissions")
+    if not isinstance(permissions, list) or not all(
+        isinstance(item, dict) for item in permissions
+    ):
+        raise RuntimeError("Jenkins Harbor robot permissions have an invalid shape")
+    matches = [
+        item
+        for item in permissions
+        if item.get("kind") == "project" and item.get("namespace") == PROJECT
+    ]
+    if len(matches) != 1:
+        raise RuntimeError("Jenkins Harbor robot must have one existing bstein scope")
+    access = matches[0].get("access")
+    if not isinstance(access, list) or not all(isinstance(item, dict) for item in access):
+        raise RuntimeError("Jenkins Harbor robot bstein access has an invalid shape")
+    for required in (("repository", "pull"), ("repository", "push")):
+        if not any(_allowed(item, *required) for item in access):
+            raise RuntimeError("Jenkins Harbor robot lacks its existing pull/push boundary")
+    immutable_access = [
+        item for item in access if item.get("resource") == "immutable-tag"
+    ]
+    if immutable_access and not (
+        len(immutable_access) == 1
+        and _allowed(immutable_access[0], "immutable-tag", "list")
+    ):
+        raise RuntimeError("Jenkins Harbor robot has broader immutable-tag access")
+    return matches[0], access
+
+
+def _verified_publisher(robot: dict[str, Any]) -> bool:
+    """Check the exact identity and least-privilege policy after persistence."""
+    if (
+        robot.get("name") != PUBLISH_ROBOT
+        or robot.get("level") != "system"
+        or robot.get("editable") is not True
+        or robot.get("disable") is not False
+    ):
+        return False
+    try:
+        _scope, access = _publisher_scope(robot)
+    except RuntimeError:
+        return False
+    return any(_allowed(item, "immutable-tag", "list") for item in access)
+
+
+def ensure_publisher_can_read_rule(client: HarborClient) -> int:
+    """Grant only immutable-tag:list to the existing Jenkins push robot."""
+    status, body, headers = client.request("GET", "/robots?page=1&page_size=100")
+    if status in TRANSIENT_STATUSES:
+        raise HarborUnavailable(f"Harbor robot list returned HTTP {status}")
+    if status != 200:
+        raise RuntimeError(f"Harbor robot list returned HTTP {status}")
+    robots = _json_list(body, "robot")
+    _require_complete_page(robots, headers, "robot")
+    matches = [
+        item
+        for item in robots
+        if item.get("name") == PUBLISH_ROBOT
+    ]
+    if len(matches) != 1:
+        raise RuntimeError("expected exactly one Jenkins Harbor publisher robot")
+    robot_id = matches[0].get("id")
+    if not isinstance(robot_id, int) or robot_id < 1:
+        raise RuntimeError("Jenkins Harbor robot omitted a valid ID")
+    status, body, _headers = client.request("GET", f"/robots/{robot_id}")
+    if status in TRANSIENT_STATUSES:
+        raise HarborUnavailable(f"Harbor robot read returned HTTP {status}")
+    if status != 200:
+        raise RuntimeError(f"Harbor robot read returned HTTP {status}")
+    try:
+        robot = json.loads(body.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise RuntimeError("Harbor returned invalid robot JSON") from exc
+    if not isinstance(robot, dict):
+        raise RuntimeError("Harbor robot response has an invalid shape")
+    if (
+        robot.get("name") != PUBLISH_ROBOT
+        or robot.get("level") != "system"
+        or robot.get("editable") is not True
+        or robot.get("disable") is not False
+    ):
+        raise RuntimeError("Jenkins Harbor publisher identity is not active and exact")
+    scope, access = _publisher_scope(robot)
+    if any(_allowed(item, "immutable-tag", "list") for item in access):
+        return robot_id
+
+    # Harbor updates permissions through PUT /robots/{id}; its separate PATCH
+    # endpoint is the only secret-rotation operation. Preserve every current
+    # scope and field while adding the one read-only action.
+    access.append({"resource": "immutable-tag", "action": "list"})
+    payload = {
+        "name": robot["name"],
+        "description": robot.get("description") or "",
+        "level": robot["level"],
+        "disable": robot["disable"],
+        "permissions": robot["permissions"],
+    }
+    duration = robot.get("duration")
+    if duration is not None:
+        if not isinstance(duration, int):
+            raise RuntimeError("Jenkins Harbor robot duration has an invalid shape")
+        payload["duration"] = duration
+    # Keep the validated object reference live in the preserved permission list.
+    if scope.get("access") is not access:
+        raise RuntimeError("Jenkins Harbor robot permission normalization drifted")
+    status, _body, _headers = client.request(
+        "PUT", f"/robots/{robot_id}", payload
+    )
+    if status in TRANSIENT_STATUSES:
+        raise HarborUnavailable(f"Harbor robot update returned HTTP {status}")
+    if status != 200:
+        raise RuntimeError(f"Harbor robot policy update returned HTTP {status}")
+
+    for attempt in range(1, 6):
+        status, body, _headers = client.request("GET", f"/robots/{robot_id}")
+        if status == 200:
+            try:
+                persisted = json.loads(body.decode("utf-8"))
+            except (UnicodeDecodeError, json.JSONDecodeError):
+                persisted = None
+            if isinstance(persisted, dict) and _verified_publisher(persisted):
+                return robot_id
+        if attempt < 5:
+            time.sleep(attempt)
+    raise RuntimeError("Jenkins Harbor robot policy did not verify exactly")
+
+
+def ensure_rule(client: HarborClient) -> int:
+    """Create once, or validate the one exact enabled rule already present."""
+    rules = list_rules(client)
+    matches = [rule for rule in rules if targets_hermes_builds(rule)]
+    if len(matches) > 1:
+        raise RuntimeError("multiple Hermes agent immutable rules exist")
+    if matches:
+        if normalized_rule(matches[0]) != EXPECTED_RULE:
+            raise RuntimeError("Hermes agent immutable rule exists but is not enabled/exact")
+        rule_id = matches[0].get("id")
+        if not isinstance(rule_id, int) or rule_id < 1:
+            raise RuntimeError("Harbor immutable rule omitted a valid ID")
+        return rule_id
+
+    path = f"/projects/{PROJECT}/immutabletagrules"
+    status, _body, headers = client.request("POST", path, EXPECTED_RULE)
+    if status in TRANSIENT_STATUSES:
+        raise HarborUnavailable(f"Harbor immutable rule create returned HTTP {status}")
+    if status != 201:
+        raise RuntimeError(f"Harbor immutable rule create returned HTTP {status}")
+    location = headers.get("Location") or headers.get("location") or ""
+    api_path = urllib.parse.urlsplit(client.origin).path.rstrip("/")
+    expected_prefix = f"{api_path}{path}/"
+    if not location.startswith(expected_prefix):
+        raise RuntimeError("Harbor immutable rule create omitted the exact Location")
+    suffix = location[len(expected_prefix) :]
+    if not suffix.isdecimal() or int(suffix) < 1:
+        raise RuntimeError("Harbor immutable rule Location has an invalid ID")
+
+    # Harbor persists synchronously, but bounded retries distinguish a stale
+    # read from accepting an unverified policy.
+    for attempt in range(1, 6):
+        matches = [rule for rule in list_rules(client) if targets_hermes_builds(rule)]
+        if len(matches) == 1 and normalized_rule(matches[0]) == EXPECTED_RULE:
+            rule_id = matches[0].get("id")
+            if rule_id == int(suffix):
+                return rule_id
+        if attempt < 5:
+            time.sleep(attempt)
+    raise RuntimeError("created Harbor immutable rule did not verify exactly")
+
+
+def main() -> int:
+    """Load the runtime-only admin credential and enforce the tracked policy."""
+    origin = os.environ.get("HARBOR_API_ORIGIN", "")
+    password_file = Path(os.environ.get("HARBOR_ADMIN_PASSWORD_FILE", ""))
+    password = password_file.read_text(encoding="utf-8").strip()
+    if not password:
+        raise RuntimeError("Harbor admin password is empty")
+    client = HarborClient(origin, "admin", password)
+    for attempt in range(1, 13):
+        try:
+            rule_id = ensure_rule(client)
+            robot_id = ensure_publisher_can_read_rule(client)
+            break
+        except HarborUnavailable:
+            if attempt == 12:
+                raise
+            time.sleep(min(attempt * 2, 15))
+    print(
+        "Hermes agent immutable build-tag rule is active "
+        f"(id={rule_id}); publisher preflight access is active (robot={robot_id})"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

services/hermes/NOTES.md

@@ -102,6 +102,105 @@ console tails and named artifact contents in its deterministic bundle. A report
 must say `retained Ariadne evidence` when that fallback is used; it must not
 pretend direct Jenkins access succeeded.
 
+## Publishing an agent image after review
+
+The `hermes-agent-image` Jenkins job is the only supported agent image builder.
+It runs daemonless Kaniko v1.23.2 under the unbound `hermes-image-builder`
+ServiceAccount without a service-account token, host socket,
+privileged container, or writable Git credential. It accepts only an exact
+40-character revision that is both the checked-out commit and current
+`atlas/titan-iac` `main`, so a human must merge the source PR first.
+
+Kaniko runs as UID 0 because it must unpack an image root filesystem and enter
+that filesystem for Dockerfile `RUN` instructions. Its capability set is the
+minimum proven by an exact ARM64 no-push build of this Dockerfile:
+`CHOWN`, `FOWNER`, `DAC_OVERRIDE`, `SETGID`, and `SETUID`. `SETGID` is required
+because pinned Kaniko applies the base image's supplementary group list before
+each Dockerfile `RUN`; `SETUID` lets apt drop privileges to its `_apt` account
+while downloading package indexes. It still has no privilege
+escalation, service-account token, host path, daemon socket, or Docker/BuildKit
+TCP endpoint and uses the runtime-default seccomp profile. This is residual
+root-in-the-build-pod risk, bounded to disposable `emptyDir` storage and a
+human-reviewed `main` revision; Dockerfile changes require the same scrutiny as
+executable cluster code.
+
+Pinned Kaniko v1.23.2 parses Dockerfile `RUN` heredocs but does not materialize
+their inline files when executing a command. The Jenkins lane therefore enables
+a bounded compatibility replay after copying the reviewed Dockerfile and runner
+from the same checked-out commit. Before every replay, the runner inventories
+the entire Dockerfile as logical instructions, including split opcodes,
+operators, and either Dockerfile escape character. It rejects every heredoc
+form outside the exact nine Node/Python blocks before launching an interpreter.
+Each accepted body is then replayed immediately after its corresponding Kaniko
+`RUN`, preserving source order before any dependent regression. Docker and
+BuildKit keep their native behavior because the compatibility argument defaults
+off. The final TypeScript build, Python compile, and source assertions remain
+mandatory, so an omitted or drifted replay fails before any image can pass
+release verification.
+
+From agent.hermes, trigger that one fixed job with:
+
+```sh
+jenkins_image_build_trigger.py '<reviewed full main commit>'
+```
+
+The helper has no general Jenkins credential or caller-selectable job name. Its
+Vault-projected token is bound by Jenkins only to `hermes-agent-image`. The job
+also requires its fixed publish confirmation. Each run claims the unique tag
+`git-<reviewed-sha>-build-<jenkins-build-number>` and refuses to overwrite an
+existing tag. It cross-checks Kaniko's digest and tagged-image evidence, then
+independently reads the pushed tag from Harbor before archiving a JSON record
+and Flux digest patch. A successful post-condition then revalidates the exact
+six-file evidence set and archives those six explicit paths with empty archives
+forbidden. Apply that patch on a new branch and submit it for human review; the
+build never changes Git, reconciles Flux, or deploys by itself.
+The helper posts only to the fixed HTTPS Build Token Root endpoint and accepts
+only its real queue responses: HTTP 201 or a non-followed HTTP 303 with an exact
+same-origin `/queue/item/<number>/` location. An unauthenticated request is
+denied, and the token cannot select, configure, read, or administer another
+Jenkins job. The pipeline independently rejects any commit that is not the
+current `origin/main`, preserving the human merge/review boundary.
+
+Harbor independently enforces an enabled immutable-tag rule scoped to only the
+`bstein/hermes-agent` repository and `git-*-build-*` tags. A second manifest PUT,
+retag, or deletion is rejected by Harbor even if a caller bypasses the Jenkins
+preflight. A revisioned, Flux-tracked policy Job creates or verifies that exact
+rule with a runtime-only Vault credential; it refuses to alter a conflicting
+rule and retries only explicit transport/readiness failures while Harbor
+starts. The same Job grants the existing Jenkins publisher only
+`immutable-tag:list` on `bstein`, preserving its other project scopes and never
+calling Harbor's separate secret-rotation endpoint. The pipeline uses that
+read-only permission to require the exact enabled rule before Kaniko starts.
+That check also makes an already-running Jenkins controller fail closed during
+rollout; Jenkins cannot become Ready on the new revision before the policy Job
+succeeds.
+
+The tracked rollout order is deliberate: the revisioned Vault role Job must
+complete before the Vault Kustomization becomes Ready. A separate Flux
+Kustomization then runs the seed Job under the dedicated
+`hermes-jenkins-token-seed` identity, which can only create/read the exact
+`kv/data/atlas/hermes/developer-jenkins` path and request random bytes. Jenkins
+depends on both that seed and the Harbor policy Job; Hermes depends on Jenkins.
+The seeder uses KV-v2 CAS create-only semantics. It never changes an existing
+token, never fills a missing field in an existing secret, and fails closed when
+it cannot distinguish absence from a read error. The recurring Vault
+configuration job reconciles the same narrow role and policy.
+
+Rotate this fixed-job token only as a coordinated operator action:
+
+1. Stop new `hermes-agent-image` triggers and wait for its queue and executor to
+   drain.
+2. Read the current KV-v2 metadata version for
+   `kv/atlas/hermes/developer-jenkins`.
+3. Generate a fresh value from Vault and patch only `build_token` with
+   `vault kv patch -cas=<current-version>`. Keep the value in a mode-0600
+   temporary file or standard input, never a command argument or log.
+4. Roll Jenkins first and wait until it is Ready, then roll `hermes-agent` and
+   wait until it is Ready. Do not resume triggers between those two consumers.
+5. Run one reviewed-main canary. If rollback is required, repeat the CAS patch
+   with the previous value as another coordinated rotation; never delete the
+   secret to make the seeder recreate it.
+
 ## The actual supervised triage algorithm
 
 1. Classify the request as test/build triage, service health, or alert tuning.

services/hermes/agent-deployment.yaml

@@ -56,6 +56,11 @@ spec:
           {{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
           {{ .Data.data.username }}
           {{- end }}
+        vault.hashicorp.com/agent-inject-secret-jenkins-image-build-token: kv/data/atlas/hermes/developer-jenkins
+        vault.hashicorp.com/agent-inject-template-jenkins-image-build-token: |
+          {{- with secret "kv/data/atlas/hermes/developer-jenkins" -}}
+          {{ .Data.data.build_token }}
+          {{- end }}
         vault.hashicorp.com/agent-inject-secret-node-ssh-private-key: kv/data/atlas/hermes/developer-ssh
         vault.hashicorp.com/agent-inject-template-node-ssh-private-key: |
           {{- with secret "kv/data/atlas/hermes/developer-ssh" -}}
@@ -91,6 +96,7 @@ spec:
         vault.hashicorp.com/agent-limits-mem: 128Mi
     spec:
       serviceAccountName: hermes-agent
+      enableServiceLinks: false
       automountServiceAccountToken: true
       securityContext:
         fsGroup: 10000
@@ -110,7 +116,7 @@ spec:
                     values: ["true"]
                   - key: kubernetes.io/hostname
                     operator: NotIn
-                    values: [titan-08, titan-13, titan-14, titan-17, titan-18]
+                    values: [titan-04, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19]
           preferredDuringSchedulingIgnoredDuringExecution:
             - weight: 100
               preference:
@@ -689,7 +695,7 @@ spec:
             seccompProfile:
               type: RuntimeDefault
           resources:
-            requests: {cpu: 350m, memory: 768Mi}
+            requests: {cpu: 300m, memory: 768Mi}
             limits: {cpu: "3", memory: 6Gi}
         - name: oauth2-proxy
           image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561
@@ -873,7 +879,7 @@ spec:
             - {name: CLAUDE_CONFIG_DIR, value: /runtime-access/claude}
             - {name: KUBECONFIG, value: /opt/data/home/.kube/config}
             - {name: PYTHONPATH, value: /opt/hermes}
-            - {name: HERMES_CLI_LANE_CONCURRENCY, value: "4"}
+            - {name: HERMES_CLI_LANE_CONCURRENCY, value: "2"}
             - {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
             - {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
           securityContext:
@@ -896,7 +902,7 @@ spec:
             - {name: tmp, mountPath: /tmp}
           resources:
             requests: {cpu: 100m, memory: 256Mi}
-            limits: {cpu: "3", memory: 6Gi}
+            limits: {cpu: "2", memory: 6Gi}
         - name: model-steward
           image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
           imagePullPolicy: IfNotPresent

services/hermes/kustomization.yaml

@@ -77,6 +77,7 @@ configMapGenerator:
       - image_broker.py=scripts/image_broker.py
       - install_agent_tools.sh=scripts/install_agent_tools.sh
       - jenkins_build_evidence.py=scripts/jenkins_build_evidence.py
+      - jenkins_image_build_trigger.py=scripts/jenkins_image_build_trigger.py
       - kanban_status_recovery.py=scripts/kanban_status_recovery.py
       - migrate_herdr_state.py=scripts/migrate_herdr_state.py
       - migrate_api_session_lineage.py=scripts/migrate_api_session_lineage.py

services/hermes/scripts/jenkins_image_build_trigger.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""Trigger only the reviewed-main Hermes agent image release job."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+
+JENKINS_ORIGIN = "https://ci.bstein.dev"
+JENKINS_BUILD_URL = f"{JENKINS_ORIGIN}/buildByToken/buildWithParameters"
+JOB_NAME = "hermes-agent-image"
+TOKEN_FILE = Path("/runtime-access/jenkins-image-build-token")
+REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
+QUEUE_PATH_PATTERN = re.compile(r"^/queue/item/[0-9]+/?$")
+
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+    """Keep a queued-build redirect from becoming an unauthorized job read."""
+
+    def redirect_request(self, _request, _file, _code, _message, _headers, _url):
+        return None
+
+
+def _open_without_redirect(request: urllib.request.Request, timeout: int):
+    """Return the Build Token Root response, including its expected HTTP 303."""
+    opener = urllib.request.build_opener(_NoRedirect())
+    try:
+        return opener.open(request, timeout=timeout)
+    except urllib.error.HTTPError as exc:
+        if exc.code == 303:
+            return exc
+        raise
+
+
+def trigger_build(
+    revision: str,
+    *,
+    token_file: Path = TOKEN_FILE,
+    opener=_open_without_redirect,
+) -> dict[str, str | int]:
+    """Post the fixed job parameters using its job-scoped build token."""
+    revision = revision.strip()
+    if not REVISION_PATTERN.fullmatch(revision):
+        raise ValueError("revision must be a lowercase full 40-character commit")
+    token = token_file.read_text(encoding="utf-8").strip()
+    if not token:
+        raise RuntimeError("Jenkins image-build token is empty")
+    payload = urllib.parse.urlencode(
+        {
+            "job": JOB_NAME,
+            "token": token,
+            "PUBLISH_IMAGE": "true",
+            "EXPECTED_SOURCE_REVISION": revision,
+            "CONFIRM_PUBLISH": "PUBLISH HERMES AGENT",
+        }
+    ).encode("utf-8")
+    request = urllib.request.Request(
+        JENKINS_BUILD_URL,
+        data=payload,
+        headers={"Content-Type": "application/x-www-form-urlencoded"},
+        method="POST",
+    )
+    with opener(request, timeout=20) as response:
+        status = int(response.status)
+        location = response.headers.get("Location", "")
+    if status not in {201, 303}:
+        raise RuntimeError(f"Jenkins trigger returned HTTP {status}")
+    if not location:
+        raise RuntimeError("Jenkins trigger omitted the queue Location")
+    queue_url = urllib.parse.urljoin(f"{JENKINS_ORIGIN}/", location)
+    parsed_queue = urllib.parse.urlsplit(queue_url)
+    expected_origin = urllib.parse.urlsplit(JENKINS_ORIGIN)
+    if (
+        parsed_queue.scheme != expected_origin.scheme
+        or parsed_queue.netloc != expected_origin.netloc
+        or parsed_queue.query
+        or parsed_queue.fragment
+        or not QUEUE_PATH_PATTERN.fullmatch(parsed_queue.path)
+    ):
+        raise RuntimeError("Jenkins returned an invalid queue Location")
+    # Never return the submitted URL: its form body contains the job token.
+    queue_path = parsed_queue.path
+    return {
+        "job": JOB_NAME,
+        "queue_path": queue_path,
+        "source_revision": revision,
+        "status": status,
+    }
+
+
+def main() -> int:
+    """Validate one revision, trigger the bounded job, and print safe metadata."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("revision", help="reviewed full commit currently on main")
+    args = parser.parse_args()
+    try:
+        result = trigger_build(args.revision)
+    except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc:
+        print(json.dumps({"error": str(exc)}, sort_keys=True))
+        return 1
+    print(json.dumps(result, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover - exercised through main()
+    raise SystemExit(main())

services/hermes/scripts/stage_runtime_access.py

@@ -95,6 +95,7 @@ def stage_agent() -> None:
         "chat-relay-key",
         "gitea-token",
         "gitea-username",
+        "jenkins-image-build-token",
         "node-ssh-private-key",
         "node-ssh-config",
         "node-ssh-known-hosts",

services/hermes/switchyard-deployment.yaml

@@ -27,7 +27,7 @@ spec:
         prometheus.io/port: "9005"
         prometheus.io/path: /metrics
         vault.hashicorp.com/agent-inject: "true"
-        vault.hashicorp.com/role: hermes-agent
+        vault.hashicorp.com/role: hermes-switchyard
         vault.hashicorp.com/agent-inject-secret-relay-key: kv/data/atlas/hermes/chat-telegram
         vault.hashicorp.com/agent-inject-template-relay-key: |
           {{- with secret "kv/data/atlas/hermes/chat-telegram" -}}

services/jenkins/configmap-jcasc.yaml

@@ -652,6 +652,25 @@ data:
               }
             }
           }
+          pipelineJob('hermes-agent-image') {
+            disabled(false)
+            description('Human-gated, daemonless Kaniko build for the reviewed atlas/titan-iac main revision. Publishes a content-addressed Hermes agent image and archives a Flux digest patch; it never mutates Git or deploys.')
+            authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))
+            definition {
+              cpsScm {
+                scm {
+                  git {
+                    remote {
+                      url('https://scm.bstein.dev/atlas/titan-iac.git')
+                      credentials('gitea-pat')
+                    }
+                    branches('*/main')
+                  }
+                }
+                scriptPath('ci/Jenkinsfile.hermes-agent-image')
+              }
+            }
+          }
           multibranchPipelineJob('titan-iac-quality-gate') {
             branchSources {
               branchSource {

services/jenkins/configmap-plugins.yaml

@@ -20,6 +20,7 @@ data:
     gitea:268.v75e47974c01d
     gitea-checks:603.621.vc708da_fb_371d
     multibranch-scan-webhook-trigger:1.0.11
+    build-token-root:365.v717f8685a_09e
     # Structured test evidence. Without junit the `junit` step throws
     # NoSuchMethodError, jenkins.failed_tests is always empty, and triage
     # has only raw console text to work from. Pinned to the newest release

services/jenkins/deployment.yaml

@@ -65,6 +65,9 @@ spec:
           ARIADNE_JENKINS_API_USER={{ .Data.data.username }}
           ARIADNE_JENKINS_API_TOKEN={{ .Data.data.token }}
           {{ end }}
+          {{ with secret "kv/data/atlas/hermes/developer-jenkins" }}
+          HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }}
+          {{ end }}
         bstein.dev/restarted-at: "2026-05-20T09:40:31Z"
     spec:
       serviceAccountName: jenkins

services/jenkins/hermes-image-builder-serviceaccount.yaml

@@ -0,0 +1,7 @@
+# services/jenkins/hermes-image-builder-serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: hermes-image-builder
+  namespace: jenkins
+automountServiceAccountToken: false

services/jenkins/kustomization.yaml

@@ -5,6 +5,7 @@ namespace: jenkins
 resources:
   - namespace.yaml
   - serviceaccount.yaml
+  - hermes-image-builder-serviceaccount.yaml
   - vault-serviceaccount.yaml
   - pvc.yaml
   - cache-pvc.yaml

services/jenkins/secretproviderclass.yaml

@@ -8,7 +8,7 @@ spec:
   provider: vault
   parameters:
     vaultAddress: "http://vault.vault.svc.cluster.local:8200"
-    roleName: "jenkins"
+    roleName: "jenkins-vault-sync"
     objects: |
       - objectName: "harbor-pull__dockerconfigjson"
         secretPath: "kv/data/atlas/shared/harbor-pull"

services/maintenance/apps/metis-configmap.yaml

@@ -15,7 +15,7 @@ data:
   METIS_MAX_DEVICE_BYTES: "1000000000000"
   METIS_NAMESPACE: maintenance
   METIS_REMOTE_POD_TIMEOUT_SEC: "14400"
-  METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-295-amd64 # {"$imagepolicy": "maintenance:metis-amd64"}
+  METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-296-amd64 # {"$imagepolicy": "maintenance:metis-amd64"}
   METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-295-arm64 # {"$imagepolicy": "maintenance:metis-arm64"}
   METIS_HARBOR_REGISTRY: registry.bstein.dev
   METIS_HARBOR_PROJECT: metis

services/maintenance/kustomization.yaml

@@ -11,7 +11,7 @@ resources:
   - networking
 images:
   - name: registry.bstein.dev/bstein/ariadne
-    newTag: 0.1.0-460 # {"$imagepolicy": "maintenance:ariadne:tag"}
+    newTag: 0.1.0-461 # {"$imagepolicy": "maintenance:ariadne:tag"}
   - name: registry.bstein.dev/bstein/metis
     newTag: 0.1.0-295-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"}
   - name: registry.bstein.dev/bstein/soteria

services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml

@@ -32,7 +32,7 @@ spec:
         kubernetes.io/arch: amd64
       containers:
         - name: metis-sentinel
-          image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-295-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"}
+          image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-296-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"}
           imagePullPolicy: Always
           envFrom:
             - configMapRef:

services/vault-hermes-jenkins-token-seed/job.yaml

@@ -0,0 +1,68 @@
+# services/vault-hermes-jenkins-token-seed/job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: vault-hermes-jenkins-build-token-seed-2
+  namespace: vault
+spec:
+  backoffLimit: 2
+  template:
+    spec:
+      serviceAccountName: hermes-jenkins-token-seed
+      enableServiceLinks: false
+      restartPolicy: Never
+      nodeSelector:
+        hardware: rpi5
+        kubernetes.io/arch: arm64
+        node-role.kubernetes.io/worker: "true"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values: [titan-04, titan-14, titan-18, titan-19, titan-24]
+      securityContext:
+        fsGroup: 1000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: seed
+          image: docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569
+          imagePullPolicy: IfNotPresent
+          command: [sh, /scripts/vault_hermes_jenkins_build_token_ensure.sh]
+          env:
+            - name: HOME
+              value: /tmp
+            - name: VAULT_ADDR
+              value: http://vault.vault.svc.cluster.local:8200
+            - name: VAULT_K8S_ROLE
+              value: hermes-jenkins-token-seed
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+            runAsGroup: 1000
+            runAsNonRoot: true
+            runAsUser: 100
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - name: scripts
+              mountPath: /scripts
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+          resources:
+            requests: {cpu: 25m, memory: 32Mi}
+            limits: {cpu: 250m, memory: 128Mi}
+      volumes:
+        - name: scripts
+          configMap:
+            name: vault-hermes-jenkins-token-seed-script
+            defaultMode: 0555
+        - name: tmp
+          emptyDir: {}

services/vault-hermes-jenkins-token-seed/kustomization.yaml

@@ -0,0 +1,13 @@
+# services/vault-hermes-jenkins-token-seed/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: vault
+resources:
+  - serviceaccount.yaml
+  - job.yaml
+generatorOptions:
+  disableNameSuffixHash: true
+configMapGenerator:
+  - name: vault-hermes-jenkins-token-seed-script
+    files:
+      - vault_hermes_jenkins_build_token_ensure.sh=scripts/vault_hermes_jenkins_build_token_ensure.sh

services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh

@@ -0,0 +1,167 @@
+#!/usr/bin/env sh
+set -eu
+
+secret_path="kv/atlas/hermes/developer-jenkins"
+secret_api_path="kv/data/atlas/hermes/developer-jenkins"
+jwt_file="${VAULT_K8S_JWT_FILE:-/var/run/secrets/kubernetes.io/serviceaccount/token}"
+vault_role="${VAULT_K8S_ROLE:-hermes-jenkins-token-seed}"
+payload_file="${TMPDIR:-/tmp}/hermes-jenkins-token.json"
+
+log() { printf '[hermes-jenkins-token] %s\n' "$*" >&2; }
+cleanup() { rm -f "${payload_file}"; }
+trap cleanup EXIT HUP INT TERM
+
+retry_command() {
+  attempt=1
+  while [ "${attempt}" -le 5 ]; do
+    set +e
+    command_output="$("$@" 2>&1)"
+    command_status=$?
+    set -e
+    if [ "${command_status}" -eq 0 ]; then
+      printf '%s' "${command_output}"
+      return 0
+    fi
+    if [ "${attempt}" -lt 5 ]; then
+      sleep "${attempt}"
+    fi
+    attempt=$((attempt + 1))
+  done
+  return "${command_status}"
+}
+
+ensure_token() {
+  if [ -n "${VAULT_TOKEN:-}" ]; then
+    return
+  fi
+  jwt="$(cat "${jwt_file}")"
+  VAULT_TOKEN="$(retry_command vault write -field=token \
+    auth/kubernetes/login role="${vault_role}" jwt="${jwt}")" || {
+      log "Vault Kubernetes login failed after retries"
+      exit 1
+    }
+  unset jwt
+  if [ -z "${VAULT_TOKEN}" ]; then
+    log "Vault Kubernetes login returned an empty token"
+    exit 1
+  fi
+  export VAULT_TOKEN
+}
+
+# Return 0 for the complete field, 10 when absent, 11 when present but
+# incomplete, and 12 for a persistent authorization/transport failure.
+read_build_token() {
+  attempt=1
+  while [ "${attempt}" -le 5 ]; do
+    set +e
+    secret_json="$(vault read -format=json "${secret_api_path}" 2>&1)"
+    secret_status=$?
+    set -e
+    if [ "${secret_status}" -eq 0 ]; then
+      unset secret_json
+      set +e
+      build_token="$(vault kv get -field=build_token "${secret_path}" 2>&1)"
+      field_status=$?
+      set -e
+      if [ "${field_status}" -eq 0 ]; then
+        case "${build_token}" in
+          ''|*[!0-9a-f]*) unset build_token; return 11 ;;
+        esac
+        if [ "${#build_token}" -eq 64 ]; then
+          unset build_token
+          return 0
+        fi
+        unset build_token
+        return 11
+      fi
+      if printf '%s' "${build_token}" | grep -q 'No value found'; then
+        unset build_token
+        return 11
+      fi
+      unset build_token
+    elif printf '%s' "${secret_json}" | grep -Eq 'Code: 404|No value found at'; then
+      unset secret_json
+      return 10
+    fi
+    if [ "${attempt}" -lt 5 ]; then
+      sleep "${attempt}"
+    fi
+    attempt=$((attempt + 1))
+  done
+  unset secret_json
+  return 12
+}
+
+ensure_token
+
+if read_build_token; then
+  read_status=0
+else
+  read_status=$?
+fi
+case "${read_status}" in
+  0)
+    log "job-scoped token already present; no write performed"
+    exit 0
+    ;;
+  11)
+    log "secret exists without a valid build_token; refusing to overwrite existing fields"
+    exit 1
+    ;;
+  12)
+    log "secret read failed after retries; refusing to seed"
+    exit 1
+    ;;
+  10) ;;
+  *)
+    log "unexpected secret read state ${read_status}"
+    exit 1
+    ;;
+esac
+
+new_token="$(retry_command vault write -field=random_bytes \
+  sys/tools/random/32 format=hex)" || {
+    log "Vault random token generation failed after retries"
+    exit 1
+  }
+case "${new_token}" in
+  ''|*[!0-9a-f]*)
+    unset new_token
+    log "Vault returned an invalid random token"
+    exit 1
+    ;;
+esac
+if [ "${#new_token}" -ne 64 ]; then
+  unset new_token
+  log "Vault returned an invalid random token"
+  exit 1
+fi
+
+umask 077
+printf '{"options":{"cas":0},"data":{"build_token":"%s"}}\n' \
+  "${new_token}" > "${payload_file}"
+unset new_token
+
+set +e
+create_error="$(vault write "${secret_api_path}" @"${payload_file}" 2>&1 >/dev/null)"
+create_status=$?
+set -e
+cleanup
+if [ "${create_status}" -eq 0 ]; then
+  log "job-scoped token created with KV-v2 CAS create-only semantics"
+  exit 0
+fi
+
+if printf '%s' "${create_error}" | grep -qi 'check-and-set'; then
+  unset create_error
+  if read_build_token; then
+    log "another seeder won CAS; existing token preserved"
+    exit 0
+  fi
+  log "CAS race did not produce a readable build_token; refusing to continue"
+  exit 1
+fi
+
+unset create_error
+log "KV-v2 CAS create failed; refusing to retry a write"
+exit 1

services/vault-hermes-jenkins-token-seed/serviceaccount.yaml

@@ -0,0 +1,7 @@
+# services/vault-hermes-jenkins-token-seed/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: hermes-jenkins-token-seed
+  namespace: vault
+automountServiceAccountToken: true

services/vault/hermes-auth-role-bootstrap-job.yaml

@@ -3,7 +3,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: vault-k8s-auth-hermes-8
+  name: vault-k8s-auth-hermes-9
   namespace: vault
 spec:
   backoffLimit: 2
@@ -15,16 +15,26 @@ spec:
         hardware: rpi5
         kubernetes.io/arch: arm64
         node-role.kubernetes.io/worker: "true"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values: [titan-04, titan-14, titan-18, titan-19, titan-24]
       containers:
         - name: configure-k8s-auth
-          image: hashicorp/vault:1.21.4
+          image: docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569
           imagePullPolicy: IfNotPresent
           command:
             - sh
             - /scripts/vault_k8s_auth_configure.sh
           env:
+            - name: HOME
+              value: /tmp
             - name: VAULT_ADDR
-              value: http://10.43.57.249:8200
+              value: http://vault.vault.svc.cluster.local:8200
             - name: VAULT_K8S_ROLE
               value: vault-admin
             - name: VAULT_K8S_TOKEN_REVIEWER_JWT_FILE
@@ -38,6 +48,18 @@ spec:
             - name: token-reviewer
               mountPath: /var/run/secrets/vault-token-reviewer
               readOnly: true
+            - name: tmp
+              mountPath: /tmp
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+            runAsGroup: 1000
+            runAsNonRoot: true
+            runAsUser: 100
+            seccompProfile:
+              type: RuntimeDefault
           resources:
             requests:
               cpu: 25m
@@ -53,3 +75,5 @@ spec:
         - name: token-reviewer
           secret:
             secretName: vault-admin-token-reviewer
+        - name: tmp
+          emptyDir: {}

services/vault/scripts/vault_k8s_auth_configure.sh

@@ -235,12 +235,26 @@ write_policy_and_role "mailu-mailserver" "mailu-mailserver" "mailu-vault-sync" \
   "mailu/* shared/postmark-relay shared/harbor-pull" ""
 write_policy_and_role "harbor" "harbor" "harbor-vault-sync" \
   "harbor/* shared/harbor-pull" "hermes/developer-harbor"
+harbor_policy_bootstrap_policy='
+path "kv/data/atlas/harbor/harbor-core" {
+  capabilities = ["read"]
+}
+'
+write_raw_policy "harbor-policy-bootstrap" "${harbor_policy_bootstrap_policy}"
+log "writing role harbor-policy-bootstrap"
+vault_cmd write "auth/kubernetes/role/harbor-policy-bootstrap" \
+  bound_service_account_names="harbor-policy-bootstrap" \
+  bound_service_account_namespaces="harbor" \
+  policies="harbor-policy-bootstrap" \
+  ttl="${role_ttl}"
 write_policy_and_role "nextcloud" "nextcloud" "nextcloud-vault" \
   "nextcloud/* shared/keycloak-admin shared/postmark-relay" ""
 write_policy_and_role "comms" "comms" "comms-vault,atlasbot" \
   "comms/* shared/chat-ai-keys-runtime shared/harbor-pull" ""
-write_policy_and_role "jenkins" "jenkins" "jenkins,jenkins-vault-sync" \
-  "jenkins/* shared/harbor-pull quality/sonarqube-oidc" "hermes/developer-jenkins"
+write_policy_and_role "jenkins" "jenkins" "jenkins" \
+  "jenkins/* shared/harbor-pull quality/sonarqube-oidc hermes/developer-jenkins" ""
+write_policy_and_role "jenkins-vault-sync" "jenkins" "jenkins-vault-sync" \
+  "shared/harbor-pull" ""
 write_policy_and_role "monitoring" "monitoring" "monitoring-vault-sync" \
   "monitoring/* shared/postmark-relay shared/harbor-pull" ""
 write_policy_and_role "logging" "logging" "logging-vault-sync" \
@@ -255,8 +269,25 @@ write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \
   "game-stream/*" ""
 write_policy_and_role "hermes" "hermes" "hermes-vault,hermes-triage" \
   "hermes/triage-oidc hermes/agent-tokens hermes/triage-api" ""
-write_policy_and_role "hermes-agent" "hermes" "hermes-agent,hermes-switchyard" \
+write_policy_and_role "hermes-agent" "hermes" "hermes-agent" \
   "hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-gitea hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" ""
+write_policy_and_role "hermes-switchyard" "hermes" "hermes-switchyard" \
+  "hermes/chat-telegram" ""
+hermes_jenkins_token_seed_policy='
+path "kv/data/atlas/hermes/developer-jenkins" {
+  capabilities = ["create", "read"]
+}
+path "sys/tools/random/32" {
+  capabilities = ["update"]
+}
+'
+write_raw_policy "hermes-jenkins-token-seed" "${hermes_jenkins_token_seed_policy}"
+log "writing role hermes-jenkins-token-seed"
+vault_cmd write "auth/kubernetes/role/hermes-jenkins-token-seed" \
+  bound_service_account_names="hermes-jenkins-token-seed" \
+  bound_service_account_namespaces="vault" \
+  policies="hermes-jenkins-token-seed" \
+  ttl="${role_ttl}"
 write_policy_and_role "hermes-credential-sync" "hermes" "hermes-agent" \
   "" "hermes/agent-tokens"
 write_policy_and_role "hermes-node-ssh" "hermes" "hermes-node-ssh-access" \

testing/quality_contract.json

@@ -14,6 +14,7 @@
     }
   ],
   "managed_modules": [
+    "ci/scripts/hermes_image_release.py",
     "ci/scripts/publish_test_metrics.py",
     "ci/scripts/publish_test_metrics_quality.py",
     "ci/scripts/semgrep_report.py",
@@ -35,6 +36,7 @@
     "testing/tests/test_quality_gate.py"
   ],
   "lint_paths": [
+    "ci/scripts/hermes_image_release.py",
     "ci/scripts/publish_test_metrics.py",
     "ci/scripts/publish_test_metrics_quality.py",
     "ci/scripts/semgrep_report.py",
@@ -168,6 +170,7 @@
   "coverage": {
     "minimum_percent": 95.0,
     "tracked_files": [
+      "ci/scripts/hermes_image_release.py",
       "ci/scripts/publish_test_metrics.py",
       "ci/scripts/publish_test_metrics_quality.py",
       "ci/scripts/semgrep_report.py",

testing/tests/test_hermes_chat_quality.py

@@ -120,6 +120,9 @@ def test_agent_image_completes_parked_kanban_tasks_atomically():
         "Atomically mark running, ready, blocked, or scheduled tasks done"
         in dockerfile
     )
+    assert "kind IN ('created', 'blocked', 'unblocked')" in dockerfile
+    assert "payload.get(\"status\") == \"blocked\"" in dockerfile
+    assert "hermes-kanban-blocked-regression.py" in dockerfile
 
 
 def test_sandbox_shares_only_the_tenant_workspace_without_credentials():

testing/tests/test_hermes_cli_lanes.py

@@ -1074,6 +1074,7 @@ def test_agent_uses_one_native_kanban_control_plane():
 
     deployment = _agent_deployment()
     pod = deployment["spec"]["template"]["spec"]
+    assert pod["enableServiceLinks"] is False
     names = {item["name"] for item in pod["containers"]}
     assert "cli-lane-runner" in names
     assert "terminal" in names
@@ -1083,6 +1084,49 @@ def test_agent_uses_one_native_kanban_control_plane():
     assert "herdr-dispatch" not in rendered
 
 
+def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth():
+    deployment = _agent_deployment()
+    containers = {
+        item["name"]: item
+        for item in deployment["spec"]["template"]["spec"]["containers"]
+    }
+    lane = containers["cli-lane-runner"]
+    environment = {item["name"]: item["value"] for item in lane["env"]}
+
+    assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2"
+    assert lane["resources"] == {
+        "requests": {"cpu": "100m", "memory": "256Mi"},
+        "limits": {"cpu": "2", "memory": "6Gi"},
+    }
+
+
+def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity():
+    """Placement correction: keep the agent off nodes that cannot hold it.
+
+    titan-04 is cordoned after repeated kernel undervoltage and kubelet
+    failure, and titan-19 was probe/Longhorn unstable under worker load, so
+    both must join the existing hard exclusions. That leaves titan-05 as the
+    healthy candidate, which is tight enough on requested CPU that the main
+    container has to give back 50m to schedule there.
+    """
+    pod = _agent_deployment()["spec"]["template"]["spec"]
+    hostnames = next(
+        item
+        for item in pod["affinity"]["nodeAffinity"][
+            "requiredDuringSchedulingIgnoredDuringExecution"
+        ]["nodeSelectorTerms"][0]["matchExpressions"]
+        if item["key"] == "kubernetes.io/hostname"
+    )
+
+    assert hostnames["operator"] == "NotIn"
+    assert set(hostnames["values"]) >= {"titan-04", "titan-19"}
+
+    hermes = next(
+        item for item in pod["containers"] if item["name"] == "hermes"
+    )
+    assert hermes["resources"]["requests"]["cpu"] == "300m"
+
+
 def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path():
     deployment = _agent_deployment()
     pod = deployment["spec"]["template"]["spec"]

testing/tests/test_hermes_image_builder.py

@@ -0,0 +1,494 @@
+"""Safety and artifact contracts for the Hermes agent image release lane."""
+
+from __future__ import annotations
+
+import importlib.util
+import io
+import json
+import re
+import urllib.parse
+from pathlib import Path
+
+import pytest
+import yaml
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+PIPELINE_PATH = REPO_ROOT / "ci/Jenkinsfile.hermes-agent-image"
+RELEASE_SCRIPT = REPO_ROOT / "ci/scripts/hermes_image_release.py"
+TRIGGER_SCRIPT = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py"
+HEREDOC_RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py"
+
+
+def _load_release_module():
+    spec = importlib.util.spec_from_file_location(
+        "hermes_image_release", RELEASE_SCRIPT
+    )
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _load_trigger_module():
+    spec = importlib.util.spec_from_file_location(
+        "jenkins_image_build_trigger", TRIGGER_SCRIPT
+    )
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _load_heredoc_runner():
+    spec = importlib.util.spec_from_file_location(
+        "hermes_kaniko_heredoc_runner", HEREDOC_RUNNER
+    )
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _pod_spec() -> dict:
+    source = PIPELINE_PATH.read_text(encoding="utf-8")
+    pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0]
+    return yaml.safe_load(pod_yaml)["spec"]
+
+
+def test_builder_pod_is_daemonless_and_kernel_bounded() -> None:
+    """The builder gets only proven build caps, never host, daemon, or K8s access."""
+    source = PIPELINE_PATH.read_text(encoding="utf-8")
+    spec = _pod_spec()
+    assert spec["serviceAccountName"] == "hermes-image-builder"
+    assert spec["automountServiceAccountToken"] is False
+    assert spec["enableServiceLinks"] is False
+    assert "hostPath" not in source
+    assert "docker.sock" not in source
+    assert "tcp://" not in source
+    assert "buildkitd" not in source.lower()
+    assert "dind" not in source.lower()
+    assert "serviceAccountToken" not in source
+
+    containers = {item["name"]: item for item in spec["containers"]}
+    kaniko = containers["kaniko"]
+    assert kaniko["image"] == (
+        "gcr.io/kaniko-project/executor@sha256:"
+        "c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e"
+    )
+    assert kaniko["securityContext"]["capabilities"]["add"] == [
+        "CHOWN",
+        "FOWNER",
+        "DAC_OVERRIDE",
+        "SETGID",
+        "SETUID",
+    ]
+    for container in containers.values():
+        security = container["securityContext"]
+        assert security["allowPrivilegeEscalation"] is False
+        assert security["capabilities"]["drop"] == ["ALL"]
+        assert security["seccompProfile"]["type"] == "RuntimeDefault"
+        assert security.get("privileged", False) is False
+    assert "add" not in containers["jnlp"]["securityContext"]["capabilities"]
+    assert "add" not in containers["python"]["securityContext"]["capabilities"]
+
+
+def test_builder_is_restricted_to_healthy_rpi5_capacity() -> None:
+    """Disposable builds must stay off unhealthy and reserved Atlas nodes."""
+    spec = _pod_spec()
+    assert spec["nodeSelector"]["hardware"] == "rpi5"
+    expressions = spec["affinity"]["nodeAffinity"][
+        "requiredDuringSchedulingIgnoredDuringExecution"
+    ]["nodeSelectorTerms"][0]["matchExpressions"]
+    host_rule = next(
+        rule for rule in expressions if rule["key"] == "kubernetes.io/hostname"
+    )
+    assert host_rule["operator"] == "NotIn"
+    assert set(host_rule["values"]) >= {
+        "titan-04",
+        "titan-14",
+        "titan-18",
+        "titan-19",
+        "titan-22",
+        "titan-24",
+    }
+
+
+def test_pipeline_requires_reviewed_main_and_runtime_credentials() -> None:
+    """Publishing requires explicit confirmation and a reviewed main commit."""
+    source = PIPELINE_PATH.read_text(encoding="utf-8")
+    assert 'test "${PUBLISH_IMAGE}" = "true"' in source
+    assert 'test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"' in source
+    assert "git rev-parse origin/main" in source
+    assert "credentialsId: 'harbor-robot'" in source
+    assert "set +x" in source
+    assert "umask 077" in source
+    assert "unset HARBOR_USER HARBOR_PASSWORD auth" in source
+    assert "/busybox/rm -f /kaniko/.docker/config.json" in source
+    assert "--digest-file=" in source
+    assert "--image-name-tag-with-digest-file=" in source
+    assert "--destination=" in source
+    assert "assert-absent" in source
+    assert "git-${actual_revision}-build-${BUILD_NUMBER}" in source
+    assert source.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1
+
+
+def test_kaniko_replays_only_the_exact_reviewed_heredoc_contract() -> None:
+    """Pinned Kaniko's ignored inline files are replayed in exact source order."""
+    module = _load_heredoc_runner()
+    dockerfile = (REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent").read_text()
+    blocks = module.extract_blocks(dockerfile)
+    assert tuple(command[0] for command, _body in blocks) == (
+        "node",
+        "python",
+        "python",
+        "python",
+        "python",
+        "python",
+        "python",
+        "python",
+        "node",
+    )
+    assert "session message total" in blocks[-1][1]
+    compat = dockerfile.split("ARG HERMES_KANIKO_HEREDOC_COMPAT=0", 1)[1]
+    assert compat.count("python /tmp/hermes-kaniko-heredoc-runner.py") == 9
+    assert [int(value) for value in re.findall(r"--block-index (\d+)", compat)] == list(
+        range(1, 10)
+    )
+    ignored = (
+        REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent.dockerignore"
+    ).read_text()
+    assert "!dockerfiles/Dockerfile.hermes-agent" in ignored
+    assert "!dockerfiles/hermes-kaniko-heredoc-runner.py" in ignored
+
+
+def test_kaniko_heredoc_runner_rejects_drift_and_executes_separately(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    """Missing blocks fail closed and each reviewed body gets a fresh process."""
+    module = _load_heredoc_runner()
+    source = (REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent").read_text()
+    with pytest.raises(ValueError, match="contract changed"):
+        module.extract_blocks(source.replace("RUN node <<'NODE'", "RUN node", 1))
+
+    calls = []
+
+    def run(command, **kwargs):
+        calls.append((command, kwargs))
+
+    monkeypatch.setattr(module.subprocess, "run", run)
+    dockerfile = tmp_path / "Dockerfile"
+    dockerfile.write_text(source, encoding="utf-8")
+    for block_index in range(1, 10):
+        module.replay(dockerfile, block_index)
+    assert len(calls) == 9
+    assert all(call[1]["check"] is True for call in calls)
+    assert all(call[1]["text"] is True for call in calls)
+    assert all(call[1]["input"].endswith("\n") for call in calls)
+
+
+def test_jenkins_job_is_manual_and_reads_pipeline_from_main() -> None:
+    """JCasC must not publish unreviewed branch contents or poll automatically."""
+    config = yaml.safe_load(
+        (REPO_ROOT / "services/jenkins/configmap-jcasc.yaml").read_text(
+            encoding="utf-8"
+        )
+    )
+    jobs = config["data"]["jobs.yaml"]
+    block = jobs.split("pipelineJob('hermes-agent-image')", 1)[1].split(
+        "pipelineJob(", 1
+    )[0]
+    assert "branches('*/main')" in block
+    assert "scriptPath('ci/Jenkinsfile.hermes-agent-image')" in block
+    assert (
+        "authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))" in block
+    )
+    assert "pipelineTriggers" not in block
+    assert "scmTrigger" not in block
+
+
+def test_agent_trigger_is_limited_to_the_image_job(tmp_path: Path) -> None:
+    """Agent Hermes gets one job token, fixed parameters, and no Jenkins admin API."""
+    module = _load_trigger_module()
+    revision = "a" * 40
+    token_path = tmp_path / "token"
+    token_path.write_text("private-job-token\n", encoding="utf-8")
+    captured = {}
+
+    class Response(io.BytesIO):
+        status = 201
+        headers = {"Location": "https://ci.bstein.dev/queue/item/42/"}
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *_args):
+            self.close()
+
+    def opener(request, timeout):
+        captured["request"] = request
+        captured["timeout"] = timeout
+        return Response(b"")
+
+    result = module.trigger_build(revision, token_file=token_path, opener=opener)
+    request = captured["request"]
+    fields = urllib.parse.parse_qs(request.data.decode("utf-8"))
+    assert request.full_url == module.JENKINS_BUILD_URL
+    assert fields == {
+        "CONFIRM_PUBLISH": ["PUBLISH HERMES AGENT"],
+        "EXPECTED_SOURCE_REVISION": [revision],
+        "PUBLISH_IMAGE": ["true"],
+        "job": ["hermes-agent-image"],
+        "token": ["private-job-token"],
+    }
+    assert captured["timeout"] == 20
+    assert "private-job-token" not in json.dumps(result)
+    assert result["source_revision"] == revision
+
+
+def test_agent_trigger_rejects_unsafe_revision_and_empty_token(tmp_path: Path) -> None:
+    """No user-controlled job, URL, or abbreviated revision reaches Jenkins."""
+    module = _load_trigger_module()
+    token_path = tmp_path / "token"
+    token_path.write_text("token\n", encoding="utf-8")
+    with pytest.raises(ValueError):
+        module.trigger_build("main", token_file=token_path)
+    token_path.write_text("\n", encoding="utf-8")
+    with pytest.raises(RuntimeError, match="empty"):
+        module.trigger_build("a" * 40, token_file=token_path)
+
+
+def test_agent_trigger_accepts_existing_queue_redirect(tmp_path: Path) -> None:
+    """HTTP 303 means the exact release is already queued, not a trigger failure."""
+    module = _load_trigger_module()
+    token_path = tmp_path / "token"
+    token_path.write_text("token\n", encoding="utf-8")
+
+    class Response(io.BytesIO):
+        status = 303
+        headers = {"Location": "https://ci.bstein.dev/queue/item/7/"}
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *_args):
+            self.close()
+
+    result = module.trigger_build(
+        "e" * 40, token_file=token_path, opener=lambda *_args, **_kwargs: Response()
+    )
+    assert result["status"] == 303
+    assert result["queue_path"] == "/queue/item/7/"
+
+
+def test_job_token_is_generated_and_injected_only_at_runtime() -> None:
+    """The fixed-job credential stays in Vault and pod-lifetime memory."""
+    plugins = (REPO_ROOT / "services/jenkins/configmap-plugins.yaml").read_text(
+        encoding="utf-8"
+    )
+    vault = (
+        REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh"
+    ).read_text(encoding="utf-8")
+    seeder = (
+        REPO_ROOT
+        / "services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh"
+    ).read_text(encoding="utf-8")
+    jenkins = (REPO_ROOT / "services/jenkins/deployment.yaml").read_text(
+        encoding="utf-8"
+    )
+    agent = (REPO_ROOT / "services/hermes/agent-deployment.yaml").read_text(
+        encoding="utf-8"
+    )
+    stage = (REPO_ROOT / "services/hermes/scripts/stage_runtime_access.py").read_text(
+        encoding="utf-8"
+    )
+    assert "build-token-root:365.v717f8685a_09e" in plugins
+    assert 'write_raw_policy "hermes-jenkins-token-seed"' in vault
+    assert "sys/tools/random/32 format=hex" in seeder
+    assert '"options":{"cas":0}' in seeder
+    assert "kv/data/atlas/hermes/developer-jenkins" in seeder
+    assert "HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }}" in jenkins
+    assert "agent-inject-secret-jenkins-image-build-token" in agent
+    assert '"jenkins-image-build-token"' in stage
+
+
+def test_release_renderer_preserves_manifest_and_emits_safe_artifacts(
+    tmp_path: Path,
+) -> None:
+    """The release artifact is exact, reviewable, and contains no credentials."""
+    module = _load_release_module()
+    old_digest = "sha256:" + "1" * 64
+    new_digest = "sha256:" + "2" * 64
+    revision = "a" * 40
+    build_number = "17"
+    manifest = tmp_path / "kustomization.yaml"
+    manifest.write_text(
+        "apiVersion: kustomize.config.k8s.io/v1beta1\n"
+        "kind: Kustomization\n"
+        "images:\n"
+        f"  - name: {module.DEFAULT_IMAGE}\n"
+        f"    digest: {old_digest}\n",
+        encoding="utf-8",
+    )
+    output = tmp_path / "out"
+    metadata = module.write_release_artifacts(
+        digest=f"{new_digest}\n",
+        source_revision=revision,
+        build_number=build_number,
+        destination=f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build_number}",
+        kustomization=manifest,
+        output_dir=output,
+    )
+
+    assert manifest.read_text(encoding="utf-8").endswith(f"{old_digest}\n")
+    assert (
+        (output / "hermes-kustomization.yaml")
+        .read_text(encoding="utf-8")
+        .endswith(f"{new_digest}\n")
+    )
+    patch = (output / "hermes-image-update.patch").read_text(encoding="utf-8")
+    assert f"-    digest: {old_digest}" in patch
+    assert f"+    digest: {new_digest}" in patch
+    assert (
+        json.loads((output / "hermes-agent-image.json").read_text(encoding="utf-8"))
+        == metadata
+    )
+    assert set(metadata) == {
+        "digest",
+        "build_number",
+        "flux_image",
+        "image",
+        "published_tag",
+        "source_revision",
+    }
+
+
+@pytest.mark.parametrize(
+    ("digest", "revision", "destination"),
+    [
+        (
+            "latest",
+            "a" * 40,
+            "registry.bstein.dev/bstein/hermes-agent:git-" + "a" * 40 + "-build-1",
+        ),
+        (
+            "sha256:" + "1" * 64,
+            "short",
+            "registry.bstein.dev/bstein/hermes-agent:git-short-build-1",
+        ),
+        (
+            "sha256:" + "1" * 64,
+            "a" * 40,
+            "registry.bstein.dev/bstein/hermes-agent:latest",
+        ),
+    ],
+)
+def test_release_renderer_rejects_unpinned_inputs(
+    tmp_path: Path, digest: str, revision: str, destination: str
+) -> None:
+    """Only exact digests and immutable source-derived tags are accepted."""
+    module = _load_release_module()
+    manifest = tmp_path / "kustomization.yaml"
+    manifest.write_text(
+        "images:\n"
+        f"  - name: {module.DEFAULT_IMAGE}\n"
+        "    digest: sha256:" + "0" * 64 + "\n",
+        encoding="utf-8",
+    )
+    with pytest.raises(ValueError):
+        module.write_release_artifacts(
+            digest=digest,
+            source_revision=revision,
+            build_number="1",
+            destination=destination,
+            kustomization=manifest,
+            output_dir=tmp_path / "out",
+        )
+
+
+def test_release_renderer_fails_closed_on_manifest_drift(tmp_path: Path) -> None:
+    """Missing, duplicate, or already-current image entries require human review."""
+    module = _load_release_module()
+    digest = "sha256:" + "f" * 64
+    missing = "images:\n  - name: example.invalid/other\n    digest: " + digest + "\n"
+    with pytest.raises(ValueError, match="found 0"):
+        module.render_kustomization(missing, digest)
+
+    duplicate = (
+        "images:\n"
+        f"  - name: {module.DEFAULT_IMAGE}\n    digest: {digest}\n"
+        f"  - name: {module.DEFAULT_IMAGE}\n    digest: {digest}\n"
+    )
+    with pytest.raises(ValueError, match="found 2"):
+        module.render_kustomization(duplicate, digest)
+
+    manifest = tmp_path / "kustomization.yaml"
+    manifest.write_text(
+        f"images:\n  - name: {module.DEFAULT_IMAGE}\n    digest: {digest}\n",
+        encoding="utf-8",
+    )
+    revision = "b" * 40
+    with pytest.raises(ValueError, match="already matches"):
+        module.write_release_artifacts(
+            digest=digest,
+            source_revision=revision,
+            build_number="1",
+            destination=f"{module.DEFAULT_IMAGE}:git-{revision}-build-1",
+            kustomization=manifest,
+            output_dir=tmp_path / "out",
+        )
+
+
+def test_release_cli_reads_digest_file(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """The pipeline CLI uses the Kaniko digest file as its sole digest input."""
+    module = _load_release_module()
+    digest = "sha256:" + "c" * 64
+    revision = "d" * 40
+    manifest = tmp_path / "kustomization.yaml"
+    manifest.write_text(
+        f"images:\n  - name: {module.DEFAULT_IMAGE}\n    digest: sha256:"
+        + "0" * 64
+        + "\n",
+        encoding="utf-8",
+    )
+    digest_file = tmp_path / "digest"
+    digest_file.write_text(digest + "\n", encoding="utf-8")
+    destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-9"
+    image_file = tmp_path / "image"
+    image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8")
+    output = tmp_path / "out"
+    monkeypatch.setenv("HARBOR_USER", "robot")
+    monkeypatch.setenv("HARBOR_PASSWORD", "secret")
+    monkeypatch.setattr(
+        module, "verify_registry_digest", lambda *_args, **_kwargs: None
+    )
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "hermes_image_release.py",
+            "render",
+            "--digest-file",
+            str(digest_file),
+            "--image-file",
+            str(image_file),
+            "--source-revision",
+            revision,
+            "--build-number",
+            "9",
+            "--destination",
+            destination,
+            "--kustomization",
+            str(manifest),
+            "--output-dir",
+            str(output),
+        ],
+    )
+    assert module.main() == 0
+    assert (
+        json.loads((output / "hermes-agent-image.json").read_text(encoding="utf-8"))[
+            "digest"
+        ]
+        == digest
+    )

testing/tests/test_hermes_image_builder_adversarial.py

@@ -0,0 +1,469 @@
+"""Adversarial boundaries for the Hermes image publisher and trigger."""
+
+from __future__ import annotations
+
+import importlib.util
+import io
+import json
+from pathlib import Path
+
+import pytest
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+RELEASE_SCRIPT = REPO_ROOT / "ci/scripts/hermes_image_release.py"
+TRIGGER_SCRIPT = REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py"
+
+
+def _load(path: Path, name: str):
+    spec = importlib.util.spec_from_file_location(name, path)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+class Response(io.BytesIO):
+    """Small context-managed HTTP response fixture."""
+
+    def __init__(
+        self,
+        status: int,
+        headers: dict[str, str] | None = None,
+        body: bytes = b"",
+    ):
+        super().__init__(body)
+        self.status = status
+        self.headers = headers or {}
+
+
+def test_kaniko_evidence_binds_digest_destination_and_unique_build() -> None:
+    """Both Kaniko artifacts must describe one exact non-replayable tag."""
+    module = _load(RELEASE_SCRIPT, "hermes_image_release_evidence")
+    revision = "a" * 40
+    digest = "sha256:" + "b" * 64
+    destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-42"
+    assert module.validate_destination(destination, revision, "42") == (
+        revision,
+        "42",
+    )
+    assert (
+        module.validate_kaniko_evidence(
+            digest_text=f"{digest}\n",
+            image_text=f"{destination}@{digest}\n",
+            destination=destination,
+        )
+        == digest
+    )
+
+
+@pytest.mark.parametrize(
+    ("digest_text", "image_template"),
+    [
+        ("sha256:" + "a" * 64 + "\nextra\n", "{destination}@{digest}"),
+        ("sha256:" + "a" * 64, "{destination}@sha256:" + "b" * 64),
+        ("sha256:" + "a" * 64, "registry.invalid/x:y@{digest}"),
+        ("sha256:" + "a" * 64, "{destination}@{digest}\nextra"),
+    ],
+)
+def test_kaniko_evidence_rejects_cross_artifact_mismatch(
+    digest_text: str, image_template: str
+) -> None:
+    """A digest, tag, repository, or cardinality mismatch stops rendering."""
+    module = _load(RELEASE_SCRIPT, "hermes_image_release_mismatch")
+    revision = "c" * 40
+    digest = "sha256:" + "a" * 64
+    destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-8"
+    image_text = image_template.format(destination=destination, digest=digest)
+    with pytest.raises(ValueError):
+        module.validate_kaniko_evidence(
+            digest_text=digest_text,
+            image_text=image_text,
+            destination=destination,
+        )
+
+
+def test_registry_preflight_rejects_existing_tag_and_auth_failures() -> None:
+    """Only an authenticated 404 permits Kaniko to claim the unique tag."""
+    module = _load(RELEASE_SCRIPT, "hermes_image_release_preflight")
+    destination = f"{module.DEFAULT_IMAGE}:git-{'d' * 40}-build-3"
+    captured = {}
+
+    def missing(request, timeout):
+        captured["request"] = request
+        captured["timeout"] = timeout
+        return Response(404)
+
+    module.assert_tag_absent(
+        destination, username="robot", password="private", opener=missing
+    )
+    request = captured["request"]
+    assert request.method == "GET"
+    assert request.full_url.startswith(
+        "https://registry.bstein.dev/api/v2.0/projects/bstein/repositories/"
+        "hermes-agent/artifacts/"
+    )
+    assert "private" not in request.full_url
+    assert captured["timeout"] == 20
+
+    for status, message in (
+        (200, "already exists"),
+        (401, "HTTP 401"),
+        (503, "HTTP 503"),
+    ):
+        with pytest.raises(RuntimeError, match=message):
+            module.assert_tag_absent(
+                destination,
+                username="robot",
+                password="private",
+                opener=lambda *_args, code=status: Response(code),
+            )
+
+
+def test_registry_preflight_requires_exact_server_immutable_policy() -> None:
+    """The already-running controller cannot publish before Flux installs the rule."""
+    module = _load(RELEASE_SCRIPT, "hermes_image_release_policy_preflight")
+    expected = {
+        "disabled": False,
+        "action": "immutable",
+        "template": "immutable_template",
+        "tag_selectors": [
+            {
+                "kind": "doublestar",
+                "decoration": "matches",
+                "pattern": "git-*-build-*",
+            }
+        ],
+        "scope_selectors": {
+            "repository": [
+                {
+                    "kind": "doublestar",
+                    "decoration": "repoMatches",
+                    "pattern": "hermes-agent",
+                }
+            ]
+        },
+    }
+    captured = {}
+
+    def exact(request, timeout):
+        captured["url"] = request.full_url
+        captured["timeout"] = timeout
+        return Response(
+            200,
+            {"X-Total-Count": "1"},
+            body=json.dumps([{"id": 9, **expected}]).encode(),
+        )
+
+    module.verify_immutable_policy(username="robot", password="private", opener=exact)
+    assert captured["url"].endswith(
+        "/projects/bstein/immutabletagrules?page=1&page_size=100"
+    )
+    assert captured["timeout"] == 20
+
+    for status, body, headers, message in (
+        (403, b"", {}, "HTTP 403"),
+        (200, b"[]", {"X-Total-Count": "0"}, "absent"),
+        (
+            200,
+            json.dumps([{**expected, "disabled": True}]).encode(),
+            {"X-Total-Count": "1"},
+            "not exact",
+        ),
+        (200, b"not-json", {}, "invalid"),
+        (
+            200,
+            json.dumps([expected]).encode(),
+            {"X-Total-Count": "2"},
+            "truncated",
+        ),
+    ):
+        with pytest.raises(RuntimeError, match=message):
+            module.verify_immutable_policy(
+                username="robot",
+                password="private",
+                opener=lambda *_args, code=status, value=body, evidence=headers: Response(
+                    code, evidence, body=value
+                ),
+            )
+
+
+def test_harbor_client_fails_closed_on_input_size_auth_and_json(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Malformed inputs, oversized bodies, missing auth, and bad JSON all fail."""
+    module = _load(RELEASE_SCRIPT, "hermes_image_release_harbor_errors")
+    destination = f"{module.DEFAULT_IMAGE}:git-{'1' * 40}-build-6"
+    with pytest.raises(ValueError, match="destination"):
+        module.assert_tag_absent(
+            "registry.invalid/x:tag", username="robot", password="private"
+        )
+    with pytest.raises(RuntimeError, match="credentials"):
+        module.assert_tag_absent(destination, username="", password="private")
+    with pytest.raises(RuntimeError, match="size limit"):
+        module.assert_tag_absent(
+            destination,
+            username="robot",
+            password="private",
+            opener=lambda *_args: Response(200, body=b"x" * 1_048_577),
+        )
+    with pytest.raises(RuntimeError, match="invalid artifact JSON"):
+        module.verify_registry_digest(
+            destination,
+            "sha256:" + "2" * 64,
+            username="robot",
+            password="private",
+            opener=lambda *_args: Response(200, body=b"\xff"),
+        )
+    monkeypatch.delenv("HARBOR_USER", raising=False)
+    monkeypatch.delenv("HARBOR_PASSWORD", raising=False)
+    with pytest.raises(RuntimeError, match="unavailable"):
+        module._credentials()
+
+
+def test_default_harbor_opener_returns_http_responses(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """The real wrapper returns both normal and non-redirect HTTP responses."""
+    module = _load(RELEASE_SCRIPT, "hermes_image_release_opener")
+    request = module.urllib.request.Request("https://registry.bstein.dev/test")
+
+    class SuccessOpener:
+        def open(self, _request, timeout):
+            assert timeout == 7
+            return Response(204)
+
+    monkeypatch.setattr(
+        module.urllib.request, "build_opener", lambda *_handlers: SuccessOpener()
+    )
+    assert module._registry_request(request, 7).status == 204
+
+    class ErrorOpener:
+        def open(self, _request, timeout):
+            assert timeout == 8
+            raise module.urllib.error.HTTPError(
+                _request.full_url, 404, "missing", {}, None
+            )
+
+    monkeypatch.setattr(
+        module.urllib.request, "build_opener", lambda *_handlers: ErrorOpener()
+    )
+    assert module._registry_request(request, 8).code == 404
+
+
+@pytest.mark.parametrize(
+    ("status", "artifact", "message"),
+    [
+        (404, {}, "HTTP 404"),
+        (200, {}, "omitted"),
+        (200, {"digest": "sha256:" + "2" * 64}, "does not match"),
+        (200, {"digest": "latest"}, "omitted"),
+    ],
+)
+def test_registry_verification_rejects_missing_or_mismatched_digest(
+    status: int, artifact: dict[str, str], message: str
+) -> None:
+    """Rendering requires an independent exact Harbor digest response."""
+    module = _load(RELEASE_SCRIPT, "hermes_image_release_registry")
+    digest = "sha256:" + "1" * 64
+    destination = f"{module.DEFAULT_IMAGE}:git-{'e' * 40}-build-4"
+    body = json.dumps(artifact).encode("utf-8")
+    with pytest.raises(RuntimeError, match=message):
+        module.verify_registry_digest(
+            destination,
+            digest,
+            username="robot",
+            password="private",
+            opener=lambda *_args: Response(status, body=body),
+        )
+
+
+def test_registry_verification_accepts_exact_pushed_digest() -> None:
+    """An exact authenticated Harbor digest allows artifact rendering."""
+    module = _load(RELEASE_SCRIPT, "hermes_image_release_registry_ok")
+    digest = "sha256:" + "4" * 64
+    destination = f"{module.DEFAULT_IMAGE}:git-{'f' * 40}-build-5"
+    with pytest.raises(RuntimeError, match="expected tag"):
+        module.verify_registry_digest(
+            destination,
+            digest,
+            username="robot",
+            password="private",
+            opener=lambda *_args: Response(
+                200,
+                body=json.dumps(
+                    {"digest": digest, "tags": [{"name": "different-tag"}]}
+                ).encode("utf-8"),
+            ),
+        )
+    module.verify_registry_digest(
+        destination,
+        digest,
+        username="robot",
+        password="private",
+        opener=lambda *_args: Response(
+            200,
+            body=json.dumps(
+                {
+                    "digest": digest,
+                    "tags": [
+                        {
+                            "name": destination.rsplit(":", 1)[1],
+                            "immutable": True,
+                        }
+                    ],
+                }
+            ).encode("utf-8"),
+        ),
+    )
+
+
+def test_registry_verification_rejects_tag_without_server_immutability() -> None:
+    """Digest evidence is insufficient unless Harbor reports the build tag immutable."""
+    module = _load(RELEASE_SCRIPT, "hermes_image_release_registry_mutable")
+    digest = "sha256:" + "7" * 64
+    destination = f"{module.DEFAULT_IMAGE}:git-{'8' * 40}-build-19"
+    captured = {}
+
+    def opener(request, _timeout):
+        captured["url"] = request.full_url
+        return Response(
+            200,
+            body=json.dumps(
+                {
+                    "digest": digest,
+                    "tags": [
+                        {
+                            "name": destination.rsplit(":", 1)[1],
+                            "immutable": False,
+                        }
+                    ],
+                }
+            ).encode(),
+        )
+
+    with pytest.raises(RuntimeError, match="immutable"):
+        module.verify_registry_digest(
+            destination,
+            digest,
+            username="robot",
+            password="private",
+            opener=opener,
+        )
+    assert captured["url"].endswith("?with_immutable_status=true")
+
+
+def test_release_main_preflight_uses_fixed_credentials_and_destination(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """The preflight CLI validates and checks exactly the requested build tag."""
+    module = _load(RELEASE_SCRIPT, "hermes_image_release_main_absent")
+    revision = "3" * 40
+    destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-12"
+    captured = {}
+    monkeypatch.setenv("HARBOR_USER", "robot")
+    monkeypatch.setenv("HARBOR_PASSWORD", "private")
+
+    def assert_absent(value, **credentials):
+        captured["destination"] = value
+        captured["credentials"] = credentials
+
+    def verify_policy(**credentials):
+        captured["policy_credentials"] = credentials
+
+    monkeypatch.setattr(module, "assert_tag_absent", assert_absent)
+    monkeypatch.setattr(module, "verify_immutable_policy", verify_policy)
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "hermes_image_release.py",
+            "assert-absent",
+            "--source-revision",
+            revision,
+            "--build-number",
+            "12",
+            "--destination",
+            destination,
+        ],
+    )
+    assert module.main() == 0
+    assert captured == {
+        "destination": destination,
+        "credentials": {"username": "robot", "password": "private"},
+        "policy_credentials": {"username": "robot", "password": "private"},
+    }
+
+
+def test_renderer_skips_a_matching_entry_without_a_digest() -> None:
+    """Only the one complete image entry is changed when an earlier entry drifts."""
+    module = _load(RELEASE_SCRIPT, "hermes_image_release_manifest_scan")
+    digest = "sha256:" + "5" * 64
+    source = (
+        "images:\n"
+        f"  - name: {module.DEFAULT_IMAGE}\n"
+        "    newTag: ignored\n"
+        "  - name: example.invalid/other\n"
+        "    newTag: stable\n"
+        f"  - name: {module.DEFAULT_IMAGE}\n"
+        "    digest: sha256:" + "0" * 64 + "\n"
+    )
+    assert module.render_kustomization(source, digest).endswith(f"{digest}\n")
+
+
+@pytest.mark.parametrize("status", [200, 202, 204, 301, 302, 307, 308])
+def test_trigger_rejects_non_plugin_success_status(tmp_path: Path, status: int) -> None:
+    """Generic proxy successes and redirects are not proof of a queued build."""
+    module = _load(TRIGGER_SCRIPT, f"jenkins_trigger_status_{status}")
+    token = tmp_path / "token"
+    token.write_text("private\n", encoding="utf-8")
+    response = lambda *_args, **_kwargs: Response(  # noqa: E731
+        status, {"Location": "https://ci.bstein.dev/queue/item/9/"}
+    )
+    with pytest.raises(RuntimeError, match=f"HTTP {status}"):
+        module.trigger_build("a" * 40, token_file=token, opener=response)
+
+
+@pytest.mark.parametrize(
+    "location",
+    [
+        "",
+        "https://evil.invalid/queue/item/9/",
+        "http://ci.bstein.dev/queue/item/9/",
+        "https://ci.bstein.dev/job/hermes-agent-image/9/",
+        "https://ci.bstein.dev/queue/item/9/?token=leak",
+        "https://ci.bstein.dev/queue/item/9/#fragment",
+        "https://ci.bstein.dev/queue/item/not-a-number/",
+    ],
+)
+def test_trigger_rejects_missing_malformed_or_cross_origin_queue_location(
+    tmp_path: Path, location: str
+) -> None:
+    """A real accepted status still needs the exact same-origin queue resource."""
+    module = _load(TRIGGER_SCRIPT, "jenkins_trigger_location")
+    token = tmp_path / "token"
+    token.write_text("private\n", encoding="utf-8")
+    response = lambda *_args, **_kwargs: Response(  # noqa: E731
+        201, {"Location": location}
+    )
+    with pytest.raises(RuntimeError, match="Location"):
+        module.trigger_build("b" * 40, token_file=token, opener=response)
+
+
+def test_trigger_uses_https_and_accepts_exact_relative_queue_path(
+    tmp_path: Path,
+) -> None:
+    """A relative plugin Location is resolved only against the fixed HTTPS origin."""
+    module = _load(TRIGGER_SCRIPT, "jenkins_trigger_origin")
+    token = tmp_path / "token"
+    token.write_text("private\n", encoding="utf-8")
+    captured = {}
+
+    def opener(request, timeout):
+        captured["url"] = request.full_url
+        captured["timeout"] = timeout
+        return Response(201, {"Location": "/queue/item/11/"})
+
+    result = module.trigger_build("c" * 40, token_file=token, opener=opener)
+    assert captured["url"].startswith("https://ci.bstein.dev/")
+    assert captured["timeout"] == 20
+    assert result["queue_path"] == "/queue/item/11/"

testing/tests/test_hermes_image_builder_fresh_review.py

@@ -0,0 +1,300 @@
+"""Fresh fail-closed review cases for the Hermes image builder."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+import pytest
+import yaml
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+PIPELINE = REPO_ROOT / "ci/Jenkinsfile.hermes-agent-image"
+DOCKERFILE = REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent"
+RUNNER = REPO_ROOT / "dockerfiles/hermes-kaniko-heredoc-runner.py"
+RELEASE = REPO_ROOT / "ci/scripts/hermes_image_release.py"
+BUILDER_SA = REPO_ROOT / "services/jenkins/hermes-image-builder-serviceaccount.yaml"
+
+
+def _load(path: Path, name: str):
+    spec = importlib.util.spec_from_file_location(name, path)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _pod_spec() -> dict:
+    source = PIPELINE.read_text(encoding="utf-8")
+    pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0]
+    return yaml.safe_load(pod_yaml)["spec"]
+
+
+@pytest.mark.parametrize(
+    "unsupported",
+    [
+        "RUN cat <<EOF\nbody\nEOF",
+        "run cat <<'EOF'\nbody\nEOF",
+        '  RUN cat <<"EOF"\nbody\nEOF',
+        "RUN <<-EOF\n\tbody\n\tEOF",
+        'RUN ["/bin/sh", "-c"] <<EOF\nbody\nEOF',
+        "RUN echo before <<EOF after\nbody\nEOF",
+        "RUN set -eu \\\n    && cat <<'EOF'\nbody\nEOF",
+    ],
+)
+def test_all_unsupported_run_heredoc_forms_reject_before_execution(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path, unsupported: str
+) -> None:
+    """Any RUN heredoc outside the reviewed nine is rejected before replay."""
+    module = _load(RUNNER, f"heredoc_reject_{abs(hash(unsupported))}")
+    dockerfile = tmp_path / "Dockerfile"
+    dockerfile.write_text(
+        DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n",
+        encoding="utf-8",
+    )
+    calls = []
+    monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1))
+    with pytest.raises(ValueError, match="unsupported RUN heredoc"):
+        module.replay(dockerfile, 1)
+    assert calls == []
+
+
+def test_exact_reviewed_heredocs_remain_buildkit_native_by_default() -> None:
+    """Current source inventories cleanly and enables replay only for Kaniko."""
+    module = _load(RUNNER, "heredoc_positive_contract")
+    source = DOCKERFILE.read_text(encoding="utf-8")
+    assert len(module.extract_blocks(source)) == 9
+    assert source.count("ARG HERMES_KANIKO_HEREDOC_COMPAT=0") == 1
+    assert "Kaniko v1.23.2" in RUNNER.read_text(encoding="utf-8")
+    pipeline = PIPELINE.read_text(encoding="utf-8")
+    assert pipeline.count("--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1") == 1
+
+
+def test_appended_tenth_reviewed_form_rejects_before_execution(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    """Even an otherwise supported marker cannot expand the nine-block set."""
+    module = _load(RUNNER, "heredoc_tenth_block")
+    dockerfile = tmp_path / "Dockerfile"
+    dockerfile.write_text(
+        DOCKERFILE.read_text(encoding="utf-8")
+        + "\nRUN node <<'NODE'\nconsole.log('tenth');\nNODE\n",
+        encoding="utf-8",
+    )
+    calls = []
+    monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1))
+    with pytest.raises(ValueError, match="contract changed"):
+        module.replay(dockerfile, 1)
+    assert calls == []
+
+
+@pytest.mark.parametrize(
+    ("prefix", "unsupported"),
+    [
+        ("", "R\\\nUN cat <<EOF\nbody\nEOF"),
+        ("", "RUN cat <\\\n<EOF\nbody\nEOF"),
+        ("", "R\\\nUN cat <\\\n<EOF\nbody\nEOF"),
+        ("# escape=`\n", "R`\nUN cat <<EOF\nbody\nEOF"),
+        ("# escape=`\n", "RUN cat <`\n<EOF\nbody\nEOF"),
+        ("# escape=`\n", "R`\nUN cat <`\n<EOF\nbody\nEOF"),
+        ("", "R\\\n# removed comment\nUN cat <<EOF\nbody\nEOF"),
+        (
+            "# ordinary comment\n# escape=`\n",
+            "R\\\nUN cat <<EOF\nbody\nEOF",
+        ),
+    ],
+)
+def test_logical_instruction_normalization_rejects_split_heredocs_before_execution(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+    prefix: str,
+    unsupported: str,
+) -> None:
+    """Split RUN/opcode operators and backtick escapes cannot evade inventory."""
+    module = _load(RUNNER, f"heredoc_logical_{abs(hash((prefix, unsupported)))}")
+    dockerfile = tmp_path / "Dockerfile"
+    dockerfile.write_text(
+        prefix + DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n",
+        encoding="utf-8",
+    )
+    calls = []
+    monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1))
+    with pytest.raises(ValueError, match="unsupported RUN heredoc"):
+        module.replay(dockerfile, 1)
+    assert calls == []
+
+
+def test_merged_main_replays_each_block_before_dependent_work() -> None:
+    """PR13's blocked-task regression runs only after its source patch replay."""
+    source = DOCKERFILE.read_text(encoding="utf-8")
+    replay_positions = [
+        source.index(f"--block-index {index}") for index in range(1, 10)
+    ]
+    # Locate repeated markers rather than trusting one synthetic occurrence.
+    block_positions = []
+    for marker in ("RUN node <<'NODE'", "RUN python - <<'PY'"):
+        start = 0
+        while True:
+            position = source.find(marker, start)
+            if position < 0:
+                break
+            block_positions.append(position)
+            start = position + len(marker)
+    block_positions.sort()
+    assert len(block_positions) == len(replay_positions) == 9
+    for index, (block, replay) in enumerate(zip(block_positions, replay_positions)):
+        assert block < replay
+        if index + 1 < len(block_positions):
+            assert replay < block_positions[index + 1]
+    regression = source.index(
+        "RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py"
+    )
+    assert replay_positions[4] < regression < replay_positions[5]
+
+
+def test_builder_service_account_is_explicit_tokenless_and_unbound() -> None:
+    """The build Pod selects one tokenless SA that no tracked RBAC grants bind."""
+    account = yaml.safe_load(BUILDER_SA.read_text(encoding="utf-8"))
+    assert account == {
+        "apiVersion": "v1",
+        "kind": "ServiceAccount",
+        "metadata": {"name": "hermes-image-builder", "namespace": "jenkins"},
+        "automountServiceAccountToken": False,
+    }
+    kustomization = yaml.safe_load(
+        (REPO_ROOT / "services/jenkins/kustomization.yaml").read_text(encoding="utf-8")
+    )
+    assert "hermes-image-builder-serviceaccount.yaml" in kustomization["resources"]
+
+    spec = _pod_spec()
+    assert spec["serviceAccountName"] == "hermes-image-builder"
+    assert spec["automountServiceAccountToken"] is False
+    for volume in spec.get("volumes", []):
+        projected = volume.get("projected", {})
+        assert all("serviceAccountToken" not in item for item in projected.get("sources", []))
+
+    for manifest in (REPO_ROOT / "services/jenkins").glob("*.yaml"):
+        for document in yaml.safe_load_all(manifest.read_text(encoding="utf-8")):
+            if not isinstance(document, dict) or document.get("kind") not in {
+                "RoleBinding",
+                "ClusterRoleBinding",
+            }:
+                continue
+            subjects = document.get("subjects") or []
+            assert not any(
+                item.get("kind") == "ServiceAccount"
+                and item.get("name") == "hermes-image-builder"
+                for item in subjects
+            )
+
+
+def _release_fixture(tmp_path: Path):
+    module = _load(RELEASE, f"release_evidence_{tmp_path.name}")
+    digest = "sha256:" + "7" * 64
+    revision = "8" * 40
+    build = "23"
+    destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build}"
+    digest_file = tmp_path / "hermes-agent.digest"
+    image_file = tmp_path / "hermes-agent.image"
+    manifest = tmp_path / "kustomization.yaml"
+    output = tmp_path / "release"
+    digest_file.write_text(digest + "\n", encoding="utf-8")
+    image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8")
+    manifest.write_text(
+        "images:\n"
+        f"  - name: {module.DEFAULT_IMAGE}\n"
+        "    digest: sha256:" + "0" * 64 + "\n",
+        encoding="utf-8",
+    )
+    module.write_release_artifacts(
+        digest=digest,
+        source_revision=revision,
+        build_number=build,
+        destination=destination,
+        kustomization=manifest,
+        output_dir=output,
+    )
+    kwargs = {
+        "digest_file": digest_file,
+        "image_file": image_file,
+        "source_revision": revision,
+        "build_number": build,
+        "destination": destination,
+        "kustomization": manifest,
+        "output_dir": output,
+    }
+    return module, kwargs
+
+
+def test_success_evidence_revalidation_accepts_only_exact_complete_set(
+    tmp_path: Path,
+) -> None:
+    """Exact evidence passes; an extra, missing, or altered file fails closed."""
+    module, kwargs = _release_fixture(tmp_path)
+    module.validate_release_artifacts(**kwargs)
+
+    extra = kwargs["output_dir"] / "unexpected"
+    extra.write_text("surprise\n", encoding="utf-8")
+    with pytest.raises(ValueError, match="exactly three"):
+        module.validate_release_artifacts(**kwargs)
+    extra.unlink()
+
+    metadata = kwargs["output_dir"] / "hermes-agent-image.json"
+    original = metadata.read_text(encoding="utf-8")
+    metadata.write_text(original.replace('"build_number": "23"', '"build_number": "24"'))
+    with pytest.raises(ValueError, match="incomplete or mismatched"):
+        module.validate_release_artifacts(**kwargs)
+    metadata.write_text(original, encoding="utf-8")
+    (kwargs["output_dir"] / "hermes-image-update.patch").unlink()
+    with pytest.raises(ValueError, match="exactly three"):
+        module.validate_release_artifacts(**kwargs)
+
+
+def test_verify_evidence_cli_needs_no_runtime_registry_credential(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Post-success validation is deterministic and cannot mask missing creds."""
+    module, kwargs = _release_fixture(tmp_path)
+    monkeypatch.delenv("HARBOR_USER", raising=False)
+    monkeypatch.delenv("HARBOR_PASSWORD", raising=False)
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            "hermes_image_release.py",
+            "verify-evidence",
+            "--digest-file",
+            str(kwargs["digest_file"]),
+            "--image-file",
+            str(kwargs["image_file"]),
+            "--source-revision",
+            kwargs["source_revision"],
+            "--build-number",
+            kwargs["build_number"],
+            "--destination",
+            kwargs["destination"],
+            "--kustomization",
+            str(kwargs["kustomization"]),
+            "--output-dir",
+            str(kwargs["output_dir"]),
+        ],
+    )
+    assert module.main() == 0
+
+
+def test_pipeline_success_post_requires_and_archives_exact_six_files() -> None:
+    """Missing or partial post-success evidence must change the build to failed."""
+    source = PIPELINE.read_text(encoding="utf-8")
+    post = source.split("  post {", 1)[1]
+    assert "success {" in post
+    assert "always {" not in post
+    assert "verify-evidence" in post
+    assert 'allowEmptyArchive: false' in post
+    archive = post.split("artifacts: '", 1)[1].split("'", 1)[0]
+    paths = archive.split(",")
+    assert len(paths) == 6
+    assert len(set(paths)) == 6
+    assert all("*" not in path for path in paths)
+    assert "find build -type f" in post

testing/tests/test_hermes_image_builder_harbor.py

@@ -0,0 +1,325 @@
+"""Server-side Harbor immutability contracts for Hermes agent releases."""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+
+import pytest
+import yaml
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = (
+    REPO_ROOT
+    / "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py"
+)
+
+
+def _load_module():
+    spec = importlib.util.spec_from_file_location("harbor_immutability", SCRIPT)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _rule(module, *, rule_id: int = 17, disabled: bool = False) -> dict:
+    return {"id": rule_id, **module.EXPECTED_RULE, "disabled": disabled}
+
+
+def _robot(module, *, immutable: bool = False, extra_immutable: bool = False) -> dict:
+    access = [
+        {"resource": "repository", "action": "pull", "effect": "allow"},
+        {"resource": "repository", "action": "push", "effect": "allow"},
+    ]
+    if immutable:
+        access.append({"resource": "immutable-tag", "action": "list"})
+    if extra_immutable:
+        access.append({"resource": "immutable-tag", "action": "delete"})
+    return {
+        "id": 41,
+        "name": module.PUBLISH_ROBOT,
+        "description": "Jenkins publisher",
+        "level": "system",
+        "duration": -1,
+        "editable": True,
+        "disable": False,
+        "permissions": [
+            {
+                "kind": "project",
+                "namespace": "other",
+                "access": [{"resource": "repository", "action": "pull"}],
+            },
+            {"kind": "project", "namespace": "bstein", "access": access},
+        ],
+    }
+
+
+def _count(value: int) -> dict[str, str]:
+    return {"X-Total-Count": str(value)}
+
+
+class FakeClient:
+    """Small deterministic Harbor API fake."""
+
+    def __init__(self, responses):
+        self.origin = "https://registry.bstein.dev/api/v2.0"
+        self.responses = list(responses)
+        self.calls = []
+
+    def request(self, method, path, payload=None):
+        self.calls.append((method, path, payload))
+        return self.responses.pop(0)
+
+
+def test_flux_tracks_exact_immutable_rule_before_jenkins() -> None:
+    """The Harbor rule is a reviewed prerequisite, not a Jenkins preflight only."""
+    harbor = yaml.safe_load(
+        (
+            REPO_ROOT
+            / "clusters/atlas/flux-system/applications/harbor/kustomization.yaml"
+        ).read_text()
+    )
+    jenkins = yaml.safe_load(
+        (
+            REPO_ROOT
+            / "clusters/atlas/flux-system/applications/jenkins/kustomization.yaml"
+        ).read_text()
+    )
+    check = {
+        "apiVersion": "batch/v1",
+        "kind": "Job",
+        "name": "harbor-hermes-agent-immutability-ensure-1",
+        "namespace": "harbor",
+    }
+    assert check in harbor["spec"]["healthChecks"]
+    assert "harbor" in {item["name"] for item in jenkins["spec"]["dependsOn"]}
+
+
+def test_policy_job_uses_runtime_vault_secret_and_hardened_pinned_image() -> None:
+    """No Harbor credential is committed or retained in a mutable workload."""
+    job = yaml.safe_load(
+        (REPO_ROOT / "services/harbor/hermes-agent-immutability-job.yaml").read_text()
+    )
+    template = job["spec"]["template"]
+    annotations = template["metadata"]["annotations"]
+    assert annotations["vault.hashicorp.com/role"] == "harbor-policy-bootstrap"
+    assert (
+        annotations["vault.hashicorp.com/agent-inject-secret-harbor-admin-password"]
+        == "kv/data/atlas/harbor/harbor-core"
+    )
+    pod = template["spec"]
+    assert pod["serviceAccountName"] == "harbor-policy-bootstrap"
+    assert pod["enableServiceLinks"] is False
+    container = pod["containers"][0]
+    assert "@sha256:" in container["image"]
+    security = container["securityContext"]
+    assert security["runAsNonRoot"] is True
+    assert security["readOnlyRootFilesystem"] is True
+    assert security["allowPrivilegeEscalation"] is False
+    assert security["capabilities"]["drop"] == ["ALL"]
+
+    vault = (
+        REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh"
+    ).read_text()
+    policy = vault.split("harbor_policy_bootstrap_policy='", 1)[1].split("'", 1)[0]
+    assert 'path "kv/data/atlas/harbor/harbor-core"' in policy
+    assert 'capabilities = ["read"]' in policy
+    assert "*" not in policy
+    assert 'bound_service_account_names="harbor-policy-bootstrap"' in vault
+    script = SCRIPT.read_text()
+    assert 'PUBLISH_ROBOT = "robot$jenkins-pipelines"' in script
+    assert '{"resource": "immutable-tag", "action": "list"}' in script
+    assert 'client.request(\n        "PUT", f"/robots/{robot_id}", payload' in script
+    assert '"secret"' not in script.split("payload = {", 1)[1].split("}", 1)[0]
+
+
+def test_rule_contract_is_exact_repository_and_unique_build_tags() -> None:
+    """Unrelated bstein repositories and ordinary Hermes tags stay mutable."""
+    module = _load_module()
+    assert module.EXPECTED_RULE == {
+        "disabled": False,
+        "action": "immutable",
+        "template": "immutable_template",
+        "tag_selectors": [
+            {
+                "kind": "doublestar",
+                "decoration": "matches",
+                "pattern": "git-*-build-*",
+            }
+        ],
+        "scope_selectors": {
+            "repository": [
+                {
+                    "kind": "doublestar",
+                    "decoration": "repoMatches",
+                    "pattern": "hermes-agent",
+                }
+            ]
+        },
+    }
+
+
+def test_existing_exact_rule_is_idempotent_without_mutation() -> None:
+    """A rerun only validates the exact enabled rule."""
+    module = _load_module()
+    body = json.dumps([_rule(module)]).encode()
+    client = FakeClient([(200, body, _count(1))])
+    assert module.ensure_rule(client) == 17
+    assert [call[0] for call in client.calls] == ["GET"]
+
+
+def test_create_requires_201_exact_location_and_verified_reread() -> None:
+    """Policy bootstrap fails closed until Harbor returns the exact persisted rule."""
+    module = _load_module()
+    body = json.dumps([_rule(module, rule_id=23)]).encode()
+    client = FakeClient(
+        [
+            (200, b"[]", _count(0)),
+            (
+                201,
+                b"",
+                {"Location": "/api/v2.0/projects/bstein/immutabletagrules/23"},
+            ),
+            (200, body, _count(1)),
+        ]
+    )
+    assert module.ensure_rule(client) == 23
+    assert client.calls[1] == (
+        "POST",
+        "/projects/bstein/immutabletagrules",
+        module.EXPECTED_RULE,
+    )
+
+
+@pytest.mark.parametrize(
+    "responses,match",
+    [
+        ([(200, b"[]", _count(0)), (200, b"", {})], "HTTP 200"),
+        (
+            [
+                (200, b"[]", _count(0)),
+                (201, b"", {"Location": "https://evil.invalid/1"}),
+            ],
+            "Location",
+        ),
+    ],
+)
+def test_create_rejects_noncanonical_responses(responses, match: str) -> None:
+    """Proxy success pages and foreign locations can never count as enforcement."""
+    module = _load_module()
+    with pytest.raises(RuntimeError, match=match):
+        module.ensure_rule(FakeClient(responses))
+
+
+def test_readiness_failures_are_distinct_from_policy_rejections() -> None:
+    """The tracked Job may wait for Harbor without retrying an auth denial."""
+    module = _load_module()
+    with pytest.raises(module.HarborUnavailable):
+        module.ensure_rule(FakeClient([(503, b"", {})]))
+    with pytest.raises(RuntimeError, match="HTTP 403") as exc:
+        module.ensure_rule(FakeClient([(403, b"", {})]))
+    assert not isinstance(exc.value, module.HarborUnavailable)
+
+
+def test_disabled_or_duplicate_exact_scope_fails_closed() -> None:
+    """Bootstrap never silently edits a conflicting security policy."""
+    module = _load_module()
+    disabled = json.dumps([_rule(module, disabled=True)]).encode()
+    with pytest.raises(RuntimeError, match="not enabled"):
+        module.ensure_rule(FakeClient([(200, disabled, _count(1))]))
+
+    duplicate = json.dumps([_rule(module), _rule(module, rule_id=18)]).encode()
+    with pytest.raises(RuntimeError, match="multiple"):
+        module.ensure_rule(FakeClient([(200, duplicate, _count(2))]))
+
+
+def test_policy_lists_reject_missing_or_truncated_count_evidence() -> None:
+    """A hidden second page can never produce a duplicate rule or robot update."""
+    module = _load_module()
+    body = json.dumps([_rule(module)]).encode()
+    for headers in ({}, _count(2)):
+        with pytest.raises(RuntimeError, match="count|truncated"):
+            module.ensure_rule(FakeClient([(200, body, headers)]))
+    robot_body = json.dumps(
+        [{"id": 41, "name": module.PUBLISH_ROBOT}]
+    ).encode()
+    with pytest.raises(RuntimeError, match="truncated"):
+        module.ensure_publisher_can_read_rule(
+            FakeClient([(200, robot_body, _count(2))])
+        )
+
+
+def test_publisher_policy_adds_only_read_access_and_preserves_robot() -> None:
+    """Bootstrap preserves every existing scope and never touches robot secret state."""
+    module = _load_module()
+    original = _robot(module)
+    persisted = _robot(module, immutable=True)
+    client = FakeClient(
+        [
+            (
+                200,
+                json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(),
+                _count(1),
+            ),
+            (200, json.dumps(original).encode(), {}),
+            (200, b"", {}),
+            (200, json.dumps(persisted).encode(), {}),
+        ]
+    )
+    assert module.ensure_publisher_can_read_rule(client) == 41
+    method, path, payload = client.calls[2]
+    assert (method, path) == ("PUT", "/robots/41")
+    assert "secret" not in payload
+    assert payload["permissions"][0] == original["permissions"][0]
+    assert payload["permissions"][1]["access"][-1] == {
+        "resource": "immutable-tag",
+        "action": "list",
+    }
+
+
+def test_publisher_policy_is_idempotent_and_rejects_broad_access() -> None:
+    """An exact read policy is stable; mutation-capable immutable access is blocked."""
+    module = _load_module()
+    exact = _robot(module, immutable=True)
+    client = FakeClient(
+        [
+            (
+                200,
+                json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(),
+                _count(1),
+            ),
+            (200, json.dumps(exact).encode(), {}),
+        ]
+    )
+    assert module.ensure_publisher_can_read_rule(client) == 41
+    assert [call[0] for call in client.calls] == ["GET", "GET"]
+
+    broad = _robot(module, immutable=True, extra_immutable=True)
+    client = FakeClient(
+        [
+            (
+                200,
+                json.dumps([{"id": 41, "name": module.PUBLISH_ROBOT}]).encode(),
+                _count(1),
+            ),
+            (200, json.dumps(broad).encode(), {}),
+        ]
+    )
+    with pytest.raises(RuntimeError, match="broader"):
+        module.ensure_publisher_can_read_rule(client)
+
+
+def test_harbor_client_rejects_plaintext_or_ambiguous_origins() -> None:
+    """Runtime admin credentials are never sent over HTTP or a URL with query state."""
+    module = _load_module()
+    for origin in (
+        "http://registry.bstein.dev/api/v2.0",
+        "https://registry.bstein.dev/api/v2.0?next=evil",
+        "https://other.invalid/api/v2.0",
+        "",
+    ):
+        with pytest.raises(ValueError):
+            module.HarborClient(origin, "admin", "secret")

testing/tests/test_hermes_image_builder_vault.py

@@ -0,0 +1,329 @@
+"""Vault least-privilege and fail-closed seeding tests for image releases."""
+
+from __future__ import annotations
+
+import os
+import stat
+import subprocess
+from pathlib import Path
+
+import pytest
+import yaml
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+VAULT_CONFIG = REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh"
+SEEDER = (
+    REPO_ROOT
+    / "services/vault-hermes-jenkins-token-seed/scripts/vault_hermes_jenkins_build_token_ensure.sh"
+)
+
+
+def test_developer_jenkins_access_is_read_only_and_bound_to_two_consumers() -> None:
+    """Only the Jenkins controller and Hermes agent can read the fixed-job token."""
+    source = VAULT_CONFIG.read_text(encoding="utf-8")
+    assert 'write_policy_and_role "jenkins" "jenkins" "jenkins"' in source
+    assert (
+        '"jenkins/* shared/harbor-pull quality/sonarqube-oidc '
+        'hermes/developer-jenkins" ""'
+    ) in source
+    assert 'write_policy_and_role "hermes-agent" "hermes" "hermes-agent"' in source
+    assert 'hermes/developer-jenkins hermes/developer-ssh" ""' in source
+    assert 'write_policy_and_role "hermes-switchyard" "hermes"' in source
+    assert '"hermes/chat-telegram" ""' in source
+    assert '"hermes/developer-jenkins"' not in source
+
+    jenkins_spc = yaml.safe_load(
+        (REPO_ROOT / "services/jenkins/secretproviderclass.yaml").read_text()
+    )
+    assert jenkins_spc["spec"]["parameters"]["roleName"] == "jenkins-vault-sync"
+    switchyard = yaml.safe_load(
+        (REPO_ROOT / "services/hermes/switchyard-deployment.yaml").read_text()
+    )
+    annotations = switchyard["spec"]["template"]["metadata"]["annotations"]
+    assert annotations["vault.hashicorp.com/role"] == "hermes-switchyard"
+
+
+def test_flux_orders_seed_then_jenkins_then_hermes() -> None:
+    """The fixed token must be Ready before either consumer is rolled."""
+    app_root = REPO_ROOT / "clusters/atlas/flux-system/applications"
+    vault = yaml.safe_load((app_root / "vault/kustomization.yaml").read_text())
+    seed = yaml.safe_load(
+        (app_root / "vault-hermes-jenkins-token-seed/kustomization.yaml").read_text()
+    )
+    jenkins = yaml.safe_load((app_root / "jenkins/kustomization.yaml").read_text())
+    hermes = yaml.safe_load((app_root / "hermes/kustomization.yaml").read_text())
+    role_check = {
+        "apiVersion": "batch/v1",
+        "kind": "Job",
+        "name": "vault-k8s-auth-hermes-9",
+        "namespace": "vault",
+    }
+    seed_check = {
+        "apiVersion": "batch/v1",
+        "kind": "Job",
+        "name": "vault-hermes-jenkins-build-token-seed-2",
+        "namespace": "vault",
+    }
+    assert role_check in vault["spec"]["healthChecks"]
+    assert {item["name"] for item in seed["spec"]["dependsOn"]} == {"vault"}
+    assert seed_check in seed["spec"]["healthChecks"]
+    assert jenkins["spec"]["suspend"] is False
+    assert {item["name"] for item in jenkins["spec"]["dependsOn"]} >= {
+        "helm",
+        "harbor",
+        "vault-hermes-jenkins-token-seed",
+    }
+    assert "jenkins" in {item["name"] for item in hermes["spec"]["dependsOn"]}
+
+
+def test_seed_job_is_revisioned_bounded_and_tracks_fail_closed_script() -> None:
+    """The prerequisite is a tracked one-shot on healthy non-reserved capacity."""
+    job = yaml.safe_load(
+        (
+            REPO_ROOT / "services/vault-hermes-jenkins-token-seed/job.yaml"
+        ).read_text()
+    )
+    assert job["metadata"]["name"] == "vault-hermes-jenkins-build-token-seed-2"
+    pod = job["spec"]["template"]["spec"]
+    assert pod["serviceAccountName"] == "hermes-jenkins-token-seed"
+    assert pod["enableServiceLinks"] is False
+    assert pod["restartPolicy"] == "Never"
+    assert pod["nodeSelector"]["hardware"] == "rpi5"
+    expression = pod["affinity"]["nodeAffinity"][
+        "requiredDuringSchedulingIgnoredDuringExecution"
+    ]["nodeSelectorTerms"][0]["matchExpressions"][0]
+    assert set(expression["values"]) >= {
+        "titan-04",
+        "titan-14",
+        "titan-18",
+        "titan-19",
+        "titan-24",
+    }
+    container = pod["containers"][0]
+    assert "@sha256:" in container["image"]
+    security = container["securityContext"]
+    assert security["runAsNonRoot"] is True
+    assert security["readOnlyRootFilesystem"] is True
+    assert security["allowPrivilegeEscalation"] is False
+    assert security["capabilities"]["drop"] == ["ALL"]
+    source = SEEDER.read_text(encoding="utf-8")
+    assert '"options":{"cas":0}' in source
+    assert 'vault kv get -field=build_token "${secret_path}"' in source
+    assert "kv/data/atlas/hermes/developer-jenkins" in source
+    assert '"build_token"[[:space:]]' not in source
+    assert "kv patch" not in source
+    assert "kv delete" not in source
+    assert SEEDER.name not in VAULT_CONFIG.read_text(encoding="utf-8")
+
+
+def test_seed_vault_policy_is_exact_create_read_and_rng_only() -> None:
+    """The seed identity cannot rotate, delete, list, or touch unrelated KV."""
+    source = VAULT_CONFIG.read_text(encoding="utf-8")
+    policy = source.split("hermes_jenkins_token_seed_policy='", 1)[1].split("'", 1)[0]
+    assert 'path "kv/data/atlas/hermes/developer-jenkins"' in policy
+    assert 'capabilities = ["create", "read"]' in policy
+    assert 'path "sys/tools/random/32"' in policy
+    assert 'capabilities = ["update"]' in policy
+    for forbidden in ("delete", "patch", "list", 'kv/data/atlas/hermes/*'):
+        assert forbidden not in policy
+    assert 'bound_service_account_names="hermes-jenkins-token-seed"' in source
+    assert 'bound_service_account_namespaces="vault"' in source
+
+
+def _fake_vault_tools(tmp_path: Path) -> tuple[Path, Path, Path]:
+    """Create deterministic Vault/sleep fakes and return their evidence paths."""
+    binary_dir = tmp_path / "bin"
+    binary_dir.mkdir(parents=True)
+    calls = tmp_path / "calls"
+    stdin_capture = tmp_path / "stdin"
+    vault = binary_dir / "vault"
+    vault.write_text(
+        """#!/bin/sh
+set -eu
+printf '%s\n' "$*" >> "$FAKE_CALL_LOG"
+scenario="$FAKE_SCENARIO"
+if [ "$1" = "write" ] && [ "$2" = "-field=random_bytes" ] && [ "$3" = "sys/tools/random/32" ]; then
+  printf '%064d\n' 0
+  exit 0
+fi
+if [ "$1" = "kv" ] && [ "$2" = "get" ] && [ "$3" = "-field=build_token" ]; then
+  case "$scenario" in
+    existing)
+      printf '%064d\n' 1
+      exit 0
+      ;;
+    cas-race)
+      printf '%064d\n' 2
+      exit 0
+      ;;
+    field-prefix)
+      printf 'prefix%064d\n' 3
+      exit 0
+      ;;
+    field-suffix)
+      printf '%064dsuffix\n' 4
+      exit 0
+      ;;
+    field-multiline)
+      printf '%064d\nextra\n' 5
+      exit 0
+      ;;
+    missing-field|metadata-only|custom-metadata)
+      echo 'No value found at build_token' >&2
+      exit 2
+      ;;
+  esac
+fi
+if [ "$1" = "write" ] && [ "$2" = "kv/data/atlas/hermes/developer-jenkins" ]; then
+  payload="${3#@}"
+  cp "$payload" "$FAKE_STDIN_CAPTURE"
+  if [ "$scenario" = "cas-race" ]; then
+    : > "$FAKE_WINNER"
+    echo 'check-and-set parameter did not match' >&2
+    exit 2
+  fi
+  exit 0
+fi
+if [ "$1" = "read" ] && [ "$2" = "-format=json" ]; then
+  case "$scenario" in
+    existing)
+      printf '{"data":{"data":{"build_token":"%064d"},"metadata":{"build_token":"decoy"}}}\n' 1
+      exit 0
+      ;;
+    missing-field)
+      printf '{"data":{"data":{"other":"preserve-me"}}}\n'
+      exit 0
+      ;;
+    metadata-only)
+      printf '{"data":{"data":{"other":"preserve-me"},"metadata":{"build_token":"%064d"}}}\n' 6
+      exit 0
+      ;;
+    custom-metadata)
+      printf '{"data":{"data":{"other":"preserve-me"},"metadata":{"custom_metadata":{"build_token":"%064d"}}}}\n' 7
+      exit 0
+      ;;
+    field-prefix|field-suffix|field-multiline)
+      printf '{"data":{"data":{"build_token":"present"}}}\n'
+      exit 0
+      ;;
+    absent)
+      echo 'Code: 404' >&2
+      exit 2
+      ;;
+    cas-race)
+      if [ -f "$FAKE_WINNER" ]; then
+        printf '{"data":{"data":{"build_token":"%064d"}}}\n' 2
+        exit 0
+      fi
+      echo 'Code: 404' >&2
+      exit 2
+      ;;
+    transient)
+      count=0
+      if [ -f "$FAKE_COUNT" ]; then count="$(cat "$FAKE_COUNT")"; fi
+      count=$((count + 1))
+      printf '%s' "$count" > "$FAKE_COUNT"
+      if [ "$count" -lt 3 ]; then echo 'temporary upstream error' >&2; exit 2; fi
+      echo 'Code: 404' >&2
+      exit 2
+      ;;
+    read-error)
+      echo 'permission denied' >&2
+      exit 2
+      ;;
+  esac
+fi
+echo "unexpected fake Vault call: $*" >&2
+exit 99
+""",
+        encoding="utf-8",
+    )
+    sleep = binary_dir / "sleep"
+    sleep.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+    for executable in (vault, sleep):
+        executable.chmod(executable.stat().st_mode | stat.S_IXUSR)
+    return binary_dir, calls, stdin_capture
+
+
+def _run_seeder(
+    tmp_path: Path, scenario: str
+) -> tuple[subprocess.CompletedProcess, str]:
+    binary_dir, calls, stdin_capture = _fake_vault_tools(tmp_path)
+    env = {
+        **os.environ,
+        "PATH": f"{binary_dir}:{os.environ['PATH']}",
+        "VAULT_TOKEN": "test-token",
+        "FAKE_SCENARIO": scenario,
+        "FAKE_CALL_LOG": str(calls),
+        "FAKE_STDIN_CAPTURE": str(stdin_capture),
+        "FAKE_WINNER": str(tmp_path / "winner"),
+        "FAKE_COUNT": str(tmp_path / "count"),
+        "TMPDIR": str(tmp_path),
+    }
+    result = subprocess.run(
+        ["sh", str(SEEDER)],
+        env=env,
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    call_text = calls.read_text(encoding="utf-8") if calls.exists() else ""
+    return result, call_text
+
+
+@pytest.mark.parametrize(
+    "scenario",
+    ["existing", "missing-field", "metadata-only", "custom-metadata", "read-error"],
+)
+def test_seeder_never_overwrites_existing_or_ambiguous_state(
+    tmp_path: Path, scenario: str
+) -> None:
+    """Existing fields and persistent read failures can never become a put."""
+    result, calls = _run_seeder(tmp_path, scenario)
+    assert (result.returncode == 0) is (scenario == "existing")
+    assert "write kv/data/atlas/hermes/developer-jenkins" not in calls
+    assert "sys/tools/random" not in calls
+
+
+@pytest.mark.parametrize("scenario", ["field-prefix", "field-suffix", "field-multiline"])
+def test_seeder_requires_the_entire_exact_data_field(
+    tmp_path: Path, scenario: str
+) -> None:
+    """Prefix, suffix, or multiline values cannot satisfy readiness."""
+    result, calls = _run_seeder(tmp_path, scenario)
+    assert result.returncode != 0
+    assert "kv get -field=build_token kv/atlas/hermes/developer-jenkins" in calls
+    assert "write kv/data/atlas/hermes/developer-jenkins" not in calls
+    assert "sys/tools/random" not in calls
+
+
+def test_seeder_uses_single_cas_create_and_stdin_for_absent_secret(
+    tmp_path: Path,
+) -> None:
+    """A confirmed 404 produces one create-only write without a token argument."""
+    result, calls = _run_seeder(tmp_path, "absent")
+    assert result.returncode == 0, result.stderr
+    assert calls.count("write kv/data/atlas/hermes/developer-jenkins") == 1
+    assert "@" in calls
+    assert "0000000000000000000000000000000000000000000000000000000000000000" not in calls
+    payload = (tmp_path / "stdin").read_text(encoding="utf-8")
+    assert '"options":{"cas":0}' in payload
+    assert '"build_token":"' in payload
+    assert not (tmp_path / "hermes-jenkins-token.json").exists()
+
+
+def test_seeder_retries_reads_and_accepts_only_a_verified_cas_winner(
+    tmp_path: Path,
+) -> None:
+    """Transient reads retry; a competing create is accepted only after reread."""
+    transient, transient_calls = _run_seeder(tmp_path / "transient", "transient")
+    assert transient.returncode == 0, transient.stderr
+    assert transient_calls.count("read -format=json kv/data/atlas/hermes/developer-jenkins") == 3
+    assert transient_calls.count("write kv/data/atlas/hermes/developer-jenkins") == 1
+
+    race, race_calls = _run_seeder(tmp_path / "race", "cas-race")
+    assert race.returncode == 0, race.stderr
+    assert race_calls.count("write kv/data/atlas/hermes/developer-jenkins") == 1
+    assert race_calls.count("read -format=json kv/data/atlas/hermes/developer-jenkins") == 2
+    assert "another seeder won CAS" in race.stderr

testing/tests/test_hermes_runtime_access.py

@@ -177,6 +177,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp
         "chat-relay-key": "relay-key",
         "gitea-token": "gitea-key",
         "gitea-username": "hermes-automation",
+        "jenkins-image-build-token": "job-scoped-token",
         "node-ssh-private-key": "private-key",
         "node-ssh-config": "host-config",
         "node-ssh-known-hosts": "known-hosts",
@@ -196,6 +197,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp
 
     assert (runtime / "claude/.credentials.json").stat().st_mode & 0o777 == 0o600
     assert (runtime / "codex/auth.json").stat().st_mode & 0o777 == 0o600
+    assert (runtime / "jenkins-image-build-token").stat().st_mode & 0o777 == 0o600
     assert (runtime / "claude/settings.json").is_symlink()
     assert (runtime / "codex/skills").is_symlink()
     assert not (runtime / "claude/backups").exists()