Merge remote-tracking branch 'origin/feature/hermes-distributed-worker-pool' into feature/hermes-full-handoff-acceptance

# Conflicts:
#	ci/scripts/semgrep_report.py
#	testing/quality_contract.json
#	testing/quality_coverage.py
#	testing/tests/conftest.py
#	testing/tests/test_hermes_auto_router.py
#	testing/tests/test_quality_contract.py
#	testing/tests/test_quality_coverage_helpers.py
#	testing/tests/test_semgrep_report.py

ci/Jenkinsfile.hermes-agent-image

@@ -0,0 +1,288 @@
+pipeline {
+  agent {
+    kubernetes {
+      defaultContainer 'python'
+      yaml """
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    atlas.bstein.dev/workload: hermes-agent-image-builder
+spec:
+  serviceAccountName: hermes-image-builder
+  automountServiceAccountToken: false
+  enableServiceLinks: false
+  restartPolicy: Never
+  securityContext:
+    fsGroup: 1000
+    fsGroupChangePolicy: OnRootMismatch
+  nodeSelector:
+    kubernetes.io/arch: arm64
+    node-role.kubernetes.io/worker: "true"
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: kubernetes.io/hostname
+                operator: NotIn
+                values:
+                  - titan-04
+                  - titan-14
+                  - titan-18
+                  - titan-19
+                  - titan-22
+                  - titan-24
+      preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 100
+          preference:
+            matchExpressions:
+              - key: hardware
+                operator: In
+                values:
+                  - rpi5
+  imagePullSecrets:
+    - name: harbor-bstein-robot
+  containers:
+    - name: jnlp
+      image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop: ["ALL"]
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      resources:
+        requests:
+          cpu: 25m
+          memory: 256Mi
+        limits:
+          cpu: 500m
+          memory: 512Mi
+    - name: python
+      image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
+      command: ["sleep"]
+      args: ["99d"]
+      tty: true
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop: ["ALL"]
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      resources:
+        requests:
+          cpu: 25m
+          memory: 64Mi
+        limits:
+          cpu: 250m
+          memory: 256Mi
+    - name: kaniko
+      image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
+      command: ["/busybox/sh", "-c"]
+      args: ["/busybox/sleep 99d"]
+      tty: true
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop: ["ALL"]
+          add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
+        privileged: false
+        runAsUser: 0
+        seccompProfile:
+          type: RuntimeDefault
+      resources:
+        requests:
+          cpu: 250m
+          memory: 1Gi
+        limits:
+          cpu: "2"
+          memory: 4Gi
+"""
+    }
+  }
+  parameters {
+    booleanParam(
+      name: 'PUBLISH_IMAGE',
+      defaultValue: false,
+      description: 'Publish the reviewed main revision to Harbor.'
+    )
+    string(
+      name: 'EXPECTED_SOURCE_REVISION',
+      defaultValue: '',
+      description: 'Exact 40-character commit on atlas/titan-iac main.'
+    )
+    string(
+      name: 'CONFIRM_PUBLISH',
+      defaultValue: '',
+      description: 'Enter PUBLISH HERMES AGENT to confirm the release.'
+    )
+  }
+  environment {
+    HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent'
+  }
+  options {
+    disableConcurrentBuilds()
+    buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
+    skipDefaultCheckout(true)
+    timeout(time: 90, unit: 'MINUTES')
+  }
+  stages {
+    stage('Checkout reviewed source') {
+      steps {
+        checkout scm
+      }
+    }
+    stage('Enforce release boundary') {
+      steps {
+        sh '''
+          set -eu
+          mkdir -p build
+          test "${PUBLISH_IMAGE}" = "true"
+          test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
+          case "${EXPECTED_SOURCE_REVISION}" in
+            *[!0-9a-f]*|'')
+              echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
+              exit 2
+              ;;
+          esac
+          test "${#EXPECTED_SOURCE_REVISION}" -eq 40
+          actual_revision="$(git rev-parse HEAD)"
+          test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
+          test "${actual_revision}" = "$(git rev-parse origin/main)"
+          test -z "$(git status --porcelain)"
+          test -f dockerfiles/Dockerfile.hermes-agent
+          case "${BUILD_NUMBER}" in
+            ''|0*|*[!0-9]*)
+              echo "BUILD_NUMBER must be a positive decimal integer" >&2
+              exit 2
+              ;;
+          esac
+          printf '%s\n' \
+            "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
+            > build/hermes-agent.destination
+        '''
+      }
+    }
+    stage('Reject replay before publish') {
+      steps {
+        withCredentials([usernamePassword(
+          credentialsId: 'harbor-robot',
+          usernameVariable: 'HARBOR_USER',
+          passwordVariable: 'HARBOR_PASSWORD'
+        )]) {
+          sh '''
+            set -eu
+            set +x
+            destination="$(cat build/hermes-agent.destination)"
+            python3 ci/scripts/hermes_image_release.py assert-absent \
+              --source-revision "${EXPECTED_SOURCE_REVISION}" \
+              --build-number "${BUILD_NUMBER}" \
+              --destination "${destination}"
+          '''
+        }
+      }
+    }
+    stage('Build and publish without a daemon') {
+      steps {
+        container('kaniko') {
+          withCredentials([usernamePassword(
+            credentialsId: 'harbor-robot',
+            usernameVariable: 'HARBOR_USER',
+            passwordVariable: 'HARBOR_PASSWORD'
+          )]) {
+            sh '''#!/busybox/sh
+              set -eu
+              set +x
+              config_path=/kaniko/.docker/config.json
+              destination="$(cat build/hermes-agent.destination)"
+              umask 077
+              auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
+              /busybox/mkdir -p /kaniko/.docker
+              /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
+              unset HARBOR_USER HARBOR_PASSWORD auth
+              trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
+              /kaniko/executor \
+                --context="dir://${WORKSPACE}" \
+                --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
+                --destination="${destination}" \
+                --digest-file="${WORKSPACE}/build/hermes-agent.digest" \
+                --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \
+                --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
+                --label="org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" \
+                --cleanup \
+                --push-retry=3
+            '''
+          }
+        }
+      }
+    }
+    stage('Render reviewed Flux handoff') {
+      steps {
+        withCredentials([usernamePassword(
+          credentialsId: 'harbor-robot',
+          usernameVariable: 'HARBOR_USER',
+          passwordVariable: 'HARBOR_PASSWORD'
+        )]) {
+          sh '''
+            set -eu
+            set +x
+            destination="$(cat build/hermes-agent.destination)"
+            python3 ci/scripts/hermes_image_release.py render \
+              --digest-file build/hermes-agent.digest \
+              --image-file build/hermes-agent.image \
+              --source-revision "${EXPECTED_SOURCE_REVISION}" \
+              --build-number "${BUILD_NUMBER}" \
+              --destination "${destination}" \
+              --kustomization services/hermes/kustomization.yaml \
+              --output-dir build/hermes-agent-release
+            test -s build/hermes-agent-release/hermes-image-update.patch
+            test -s build/hermes-agent-release/hermes-agent-image.json
+          '''
+        }
+      }
+    }
+  }
+  post {
+    success {
+      sh '''
+        set -eu
+        expected_files="$(printf '%s\n' \
+          build/hermes-agent.destination \
+          build/hermes-agent.digest \
+          build/hermes-agent.image \
+          build/hermes-agent-release/hermes-agent-image.json \
+          build/hermes-agent-release/hermes-image-update.patch \
+          build/hermes-agent-release/hermes-kustomization.yaml \
+          | LC_ALL=C sort)"
+        actual_files="$(find build -type f -print | LC_ALL=C sort)"
+        test "${actual_files}" = "${expected_files}"
+        destination="$(cat build/hermes-agent.destination)"
+        python3 ci/scripts/hermes_image_release.py verify-evidence \
+          --digest-file build/hermes-agent.digest \
+          --image-file build/hermes-agent.image \
+          --source-revision "${EXPECTED_SOURCE_REVISION}" \
+          --build-number "${BUILD_NUMBER}" \
+          --destination "${destination}" \
+          --kustomization services/hermes/kustomization.yaml \
+          --output-dir build/hermes-agent-release
+      '''
+      archiveArtifacts(
+        artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
+        allowEmptyArchive: false,
+        fingerprint: true
+      )
+    }
+    cleanup {
+      container('kaniko') {
+        sh '''#!/busybox/sh
+          /busybox/rm -f /kaniko/.docker/config.json
+        '''
+      }
+    }
+  }
+}

ci/scripts/hermes_image_release.py

@@ -0,0 +1,500 @@
+#!/usr/bin/env python3
+"""Verify and render a reviewable Hermes agent image release."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import difflib
+import json
+import os
+import re
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import Any, Callable
+
+
+DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-agent"
+HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0"
+HARBOR_PROJECT = "bstein"
+HARBOR_REPOSITORY = "hermes-agent"
+IMMUTABLE_REPOSITORY_PATTERN = "hermes-agent"
+IMMUTABLE_TAG_PATTERN = "git-*-build-*"
+DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
+REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
+BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$")
+DESTINATION_PATTERN = re.compile(
+    r"^registry\.bstein\.dev/bstein/hermes-agent:"
+    r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$"
+)
+
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+    """Never send registry credentials to a redirect target."""
+
+    def redirect_request(self, _request, _file, _code, _message, _headers, _url):
+        return None
+
+
+def _validated(value: str, pattern: re.Pattern[str], label: str) -> str:
+    """Return a normalized value when it matches the release contract."""
+    normalized = value.strip()
+    if not pattern.fullmatch(normalized):
+        raise ValueError(f"invalid {label}: expected {pattern.pattern}")
+    return normalized
+
+
+def validate_destination(
+    destination: str, source_revision: str, build_number: str
+) -> tuple[str, str]:
+    """Bind one unique build tag to the reviewed revision and Jenkins build."""
+    revision = _validated(source_revision, REVISION_PATTERN, "source revision")
+    build = _validated(build_number, BUILD_PATTERN, "build number")
+    match = DESTINATION_PATTERN.fullmatch(destination.strip())
+    if not match or match.groups() != (revision, build):
+        raise ValueError(
+            "destination must bind the reviewed revision and unique Jenkins build"
+        )
+    return revision, build
+
+
+def validate_kaniko_evidence(
+    *, digest_text: str, image_text: str, destination: str
+) -> str:
+    """Cross-check both independent Kaniko output files against the destination."""
+    digest_lines = digest_text.splitlines()
+    image_lines = image_text.splitlines()
+    if len(digest_lines) != 1:
+        raise ValueError("Kaniko digest evidence must contain exactly one line")
+    if len(image_lines) != 1:
+        raise ValueError("Kaniko image evidence must contain exactly one line")
+    digest = _validated(digest_lines[0], DIGEST_PATTERN, "image digest")
+    if image_lines[0].strip() != f"{destination}@{digest}":
+        raise ValueError("Kaniko image evidence does not match destination and digest")
+    return digest
+
+
+def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
+    """Make a registry request without following redirects."""
+    opener = urllib.request.build_opener(_NoRedirect())
+    try:
+        return opener.open(request, timeout=timeout)
+    except urllib.error.HTTPError as exc:
+        return exc
+
+
+def _artifact_response(
+    destination: str,
+    *,
+    username: str,
+    password: str,
+    opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
+) -> tuple[int, bytes]:
+    """Read one exact Harbor artifact by tag with bounded response size."""
+    match = DESTINATION_PATTERN.fullmatch(destination)
+    if not match:
+        raise ValueError("invalid destination")
+    if not username or not password:
+        raise RuntimeError("Harbor credentials are empty")
+    tag = destination.rsplit(":", 1)[1]
+    encoded_tag = urllib.parse.quote(tag, safe="")
+    auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
+    request = urllib.request.Request(
+        f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/"
+        f"{HARBOR_REPOSITORY}/artifacts/{encoded_tag}"
+        "?with_immutable_status=true",
+        headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
+        method="GET",
+    )
+    with opener(request, 20) as response:
+        body = response.read(1_048_577)
+        if len(body) > 1_048_576:
+            raise RuntimeError("Harbor artifact response exceeded the size limit")
+        return int(response.status), body
+
+
+def _immutable_rules_response(
+    *,
+    username: str,
+    password: str,
+    opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
+) -> tuple[int, bytes, dict[str, str]]:
+    """Read the project policy with the same least-privilege publish identity."""
+    if not username or not password:
+        raise RuntimeError("Harbor credentials are empty")
+    auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
+    request = urllib.request.Request(
+        f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules"
+        "?page=1&page_size=100",
+        headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
+        method="GET",
+    )
+    with opener(request, 20) as response:
+        body = response.read(1_048_577)
+        if len(body) > 1_048_576:
+            raise RuntimeError("Harbor immutable rule response exceeded the size limit")
+        return int(response.status), body, dict(response.headers)
+
+
+def _require_complete_rule_page(
+    rules: list[dict[str, Any]], headers: dict[str, str]
+) -> None:
+    """Require proof that the bounded first page contains every rule."""
+    raw_total = next(
+        (value for key, value in headers.items() if key.lower() == "x-total-count"),
+        None,
+    )
+    if raw_total is None or not str(raw_total).isdecimal():
+        raise RuntimeError("Harbor immutable rule list omitted a valid total count")
+    if int(raw_total) != len(rules):
+        raise RuntimeError("Harbor immutable rule list was truncated")
+
+
+def _normalized_immutable_rule(rule: dict[str, Any]) -> dict[str, Any]:
+    """Select only fields that bind the server-side build-tag policy."""
+    return {
+        "disabled": bool(rule.get("disabled", False)),
+        "action": rule.get("action"),
+        "template": rule.get("template"),
+        "tag_selectors": [
+            {
+                "kind": item.get("kind"),
+                "decoration": item.get("decoration"),
+                "pattern": item.get("pattern"),
+            }
+            for item in rule.get("tag_selectors") or []
+            if isinstance(item, dict)
+        ],
+        "scope_selectors": {
+            "repository": [
+                {
+                    "kind": item.get("kind"),
+                    "decoration": item.get("decoration"),
+                    "pattern": item.get("pattern"),
+                }
+                for item in (rule.get("scope_selectors") or {}).get(
+                    "repository", []
+                )
+                if isinstance(item, dict)
+            ]
+        },
+    }
+
+
+def verify_immutable_policy(
+    *,
+    username: str,
+    password: str,
+    opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
+) -> None:
+    """Fail closed before build unless the exact Harbor rule is active."""
+    status, body, headers = _immutable_rules_response(
+        username=username, password=password, opener=opener
+    )
+    if status != 200:
+        raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}")
+    try:
+        rules = json.loads(body.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc
+    if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules):
+        raise RuntimeError("Harbor immutable rule list has an invalid shape")
+    _require_complete_rule_page(rules, headers)
+    expected = {
+        "disabled": False,
+        "action": "immutable",
+        "template": "immutable_template",
+        "tag_selectors": [
+            {
+                "kind": "doublestar",
+                "decoration": "matches",
+                "pattern": IMMUTABLE_TAG_PATTERN,
+            }
+        ],
+        "scope_selectors": {
+            "repository": [
+                {
+                    "kind": "doublestar",
+                    "decoration": "repoMatches",
+                    "pattern": IMMUTABLE_REPOSITORY_PATTERN,
+                }
+            ]
+        },
+    }
+    matches = [
+        _normalized_immutable_rule(item)
+        for item in rules
+        if _normalized_immutable_rule(item)["tag_selectors"]
+        == expected["tag_selectors"]
+        and _normalized_immutable_rule(item)["scope_selectors"]
+        == expected["scope_selectors"]
+    ]
+    if matches != [expected]:
+        raise RuntimeError("Harbor immutable build-tag policy is absent or not exact")
+
+
+def assert_tag_absent(
+    destination: str,
+    *,
+    username: str,
+    password: str,
+    opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
+) -> None:
+    """Reject replay before Kaniko can push an already-used immutable identity."""
+    status, _body = _artifact_response(
+        destination, username=username, password=password, opener=opener
+    )
+    if status == 404:
+        return
+    if status == 200:
+        raise RuntimeError("Harbor destination tag already exists; refusing overwrite")
+    raise RuntimeError(f"Harbor destination preflight returned HTTP {status}")
+
+
+def verify_registry_digest(
+    destination: str,
+    digest: str,
+    *,
+    username: str,
+    password: str,
+    opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
+) -> None:
+    """Verify Harbor independently resolves the pushed tag to Kaniko's digest."""
+    digest = _validated(digest, DIGEST_PATTERN, "image digest")
+    status, body = _artifact_response(
+        destination, username=username, password=password, opener=opener
+    )
+    if status != 200:
+        raise RuntimeError(f"Harbor manifest verification returned HTTP {status}")
+    try:
+        artifact = json.loads(body.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise RuntimeError("Harbor returned invalid artifact JSON") from exc
+    harbor_digest = str(artifact.get("digest") or "").strip()
+    if not DIGEST_PATTERN.fullmatch(harbor_digest):
+        raise RuntimeError("Harbor response omitted a valid artifact digest")
+    if harbor_digest != digest:
+        raise RuntimeError("Harbor digest does not match Kaniko evidence")
+    expected_tag = destination.rsplit(":", 1)[1]
+    matching_tags = [
+        item
+        for item in artifact.get("tags") or []
+        if isinstance(item, dict) and item.get("name") == expected_tag
+    ]
+    if len(matching_tags) != 1:
+        raise RuntimeError("Harbor artifact does not contain the expected tag")
+    if matching_tags[0].get("immutable") is not True:
+        raise RuntimeError("Harbor did not enforce the expected tag as immutable")
+
+
+def render_kustomization(source: str, digest: str, image: str = DEFAULT_IMAGE) -> str:
+    """Replace exactly one matching Kustomize image digest without reformatting."""
+    digest = _validated(digest, DIGEST_PATTERN, "image digest")
+    lines = source.splitlines(keepends=True)
+    matches: list[int] = []
+
+    for index, line in enumerate(lines):
+        if line.strip() != f"- name: {image}":
+            continue
+        name_indent = len(line) - len(line.lstrip())
+        for candidate_index in range(index + 1, len(lines)):
+            candidate = lines[candidate_index]
+            stripped = candidate.strip()
+            candidate_indent = len(candidate) - len(candidate.lstrip())
+            if stripped.startswith("- name:") and candidate_indent == name_indent:
+                break
+            if stripped.startswith("digest:") and candidate_indent > name_indent:
+                matches.append(candidate_index)
+                break
+
+    if len(matches) != 1:
+        raise ValueError(
+            f"expected exactly one digest for image {image!r}; found {len(matches)}"
+        )
+
+    index = matches[0]
+    newline = "\n" if lines[index].endswith("\n") else ""
+    prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())]
+    lines[index] = f"{prefix}digest: {digest}{newline}"
+    return "".join(lines)
+
+
+def write_release_artifacts(
+    *,
+    digest: str,
+    source_revision: str,
+    build_number: str,
+    destination: str,
+    kustomization: Path,
+    output_dir: Path,
+) -> dict[str, str]:
+    """Write a rendered manifest, patch, and credential-free release metadata."""
+    digest = _validated(digest, DIGEST_PATTERN, "image digest")
+    source_revision, build_number = validate_destination(
+        destination, source_revision, build_number
+    )
+    source = kustomization.read_text(encoding="utf-8")
+    rendered = render_kustomization(source, digest)
+    relative_name = kustomization.name
+    patch = "".join(
+        difflib.unified_diff(
+            source.splitlines(keepends=True),
+            rendered.splitlines(keepends=True),
+            fromfile=f"a/services/hermes/{relative_name}",
+            tofile=f"b/services/hermes/{relative_name}",
+        )
+    )
+    if not patch:
+        raise ValueError("published digest already matches the Flux manifest")
+    output_dir.mkdir(parents=True, exist_ok=True)
+    (output_dir / "hermes-kustomization.yaml").write_text(rendered, encoding="utf-8")
+    (output_dir / "hermes-image-update.patch").write_text(patch, encoding="utf-8")
+    metadata = {
+        "build_number": build_number,
+        "digest": digest,
+        "flux_image": f"{DEFAULT_IMAGE}@{digest}",
+        "image": DEFAULT_IMAGE,
+        "published_tag": destination,
+        "source_revision": source_revision,
+    }
+    (output_dir / "hermes-agent-image.json").write_text(
+        json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    return metadata
+
+
+def validate_release_artifacts(
+    *,
+    digest_file: Path,
+    image_file: Path,
+    source_revision: str,
+    build_number: str,
+    destination: str,
+    kustomization: Path,
+    output_dir: Path,
+) -> None:
+    """Revalidate the exact successful-build evidence without rewriting it."""
+    digest = validate_kaniko_evidence(
+        digest_text=digest_file.read_text(encoding="utf-8"),
+        image_text=image_file.read_text(encoding="utf-8"),
+        destination=destination,
+    )
+    source_revision, build_number = validate_destination(
+        destination, source_revision, build_number
+    )
+    expected_names = {
+        "hermes-agent-image.json",
+        "hermes-image-update.patch",
+        "hermes-kustomization.yaml",
+    }
+    entries = list(output_dir.iterdir())
+    if {entry.name for entry in entries} != expected_names or not all(
+        entry.is_file() and not entry.is_symlink() for entry in entries
+    ):
+        raise ValueError("release output must contain exactly three evidence files")
+    source = kustomization.read_text(encoding="utf-8")
+    rendered = render_kustomization(source, digest)
+    relative_name = kustomization.name
+    patch = "".join(
+        difflib.unified_diff(
+            source.splitlines(keepends=True),
+            rendered.splitlines(keepends=True),
+            fromfile=f"a/services/hermes/{relative_name}",
+            tofile=f"b/services/hermes/{relative_name}",
+        )
+    )
+    metadata = {
+        "build_number": build_number,
+        "digest": digest,
+        "flux_image": f"{DEFAULT_IMAGE}@{digest}",
+        "image": DEFAULT_IMAGE,
+        "published_tag": destination,
+        "source_revision": source_revision,
+    }
+    expected = {
+        "hermes-agent-image.json": json.dumps(metadata, indent=2, sort_keys=True) + "\n",
+        "hermes-image-update.patch": patch,
+        "hermes-kustomization.yaml": rendered,
+    }
+    for name, expected_text in expected.items():
+        if (output_dir / name).read_text(encoding="utf-8") != expected_text:
+            raise ValueError(f"release evidence is incomplete or mismatched: {name}")
+
+
+def _credentials() -> tuple[str, str]:
+    """Read the masked, runtime-only Jenkins credential environment."""
+    username = os.environ.get("HARBOR_USER", "")
+    password = os.environ.get("HARBOR_PASSWORD", "")
+    if not username or not password:
+        raise RuntimeError("Harbor credentials are unavailable")
+    return username, password
+
+
+def _common_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--source-revision", required=True)
+    parser.add_argument("--build-number", required=True)
+    parser.add_argument("--destination", required=True)
+
+
+def main() -> int:
+    """Fail closed around the unique tag, then verify and render after push."""
+    parser = argparse.ArgumentParser()
+    commands = parser.add_subparsers(dest="command", required=True)
+    absent = commands.add_parser("assert-absent")
+    _common_arguments(absent)
+    render = commands.add_parser("render")
+    _common_arguments(render)
+    render.add_argument("--digest-file", required=True, type=Path)
+    render.add_argument("--image-file", required=True, type=Path)
+    render.add_argument("--kustomization", required=True, type=Path)
+    render.add_argument("--output-dir", required=True, type=Path)
+    verify = commands.add_parser("verify-evidence")
+    _common_arguments(verify)
+    verify.add_argument("--digest-file", required=True, type=Path)
+    verify.add_argument("--image-file", required=True, type=Path)
+    verify.add_argument("--kustomization", required=True, type=Path)
+    verify.add_argument("--output-dir", required=True, type=Path)
+    args = parser.parse_args()
+
+    validate_destination(args.destination, args.source_revision, args.build_number)
+    if args.command == "verify-evidence":
+        validate_release_artifacts(
+            digest_file=args.digest_file,
+            image_file=args.image_file,
+            source_revision=args.source_revision,
+            build_number=args.build_number,
+            destination=args.destination,
+            kustomization=args.kustomization,
+            output_dir=args.output_dir,
+        )
+        return 0
+
+    username, password = _credentials()
+    if args.command == "assert-absent":
+        verify_immutable_policy(username=username, password=password)
+        assert_tag_absent(args.destination, username=username, password=password)
+        return 0
+
+    digest = validate_kaniko_evidence(
+        digest_text=args.digest_file.read_text(encoding="utf-8"),
+        image_text=args.image_file.read_text(encoding="utf-8"),
+        destination=args.destination,
+    )
+    verify_registry_digest(
+        args.destination, digest, username=username, password=password
+    )
+    write_release_artifacts(
+        digest=digest,
+        source_revision=args.source_revision,
+        build_number=args.build_number,
+        destination=args.destination,
+        kustomization=args.kustomization,
+        output_dir=args.output_dir,
+    )
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover - exercised through main()
+    raise SystemExit(main())

ci/scripts/semgrep_report.py

@@ -27,10 +27,7 @@ def _read_json(path: Path) -> dict[str, Any]:
     except json.JSONDecodeError as exc:
         return {"errors": [{"message": f"invalid JSON: {exc}"}], "results": []}
     if not isinstance(payload, dict):
-        return {
-            "errors": [{"message": "report payload is not an object"}],
-            "results": [],
-        }
+        return {"errors": [{"message": "report payload is not an object"}], "results": []}
     return payload
 
 
@@ -52,16 +49,8 @@ def build_report(
     blocking = blocking_severities or BLOCKING_SEVERITIES
     raw_results = semgrep_payload.get("results", [])
     raw_errors = semgrep_payload.get("errors", [])
-    results = (
-        [item for item in raw_results if isinstance(item, dict)]
-        if isinstance(raw_results, list)
-        else []
-    )
-    errors = (
-        [item for item in raw_errors if isinstance(item, dict)]
-        if isinstance(raw_errors, list)
-        else []
-    )
+    results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
+    errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else []
 
     severity_counts = dict.fromkeys(SEVERITIES, 0)
     blocking_findings = 0
@@ -104,18 +93,15 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None:
     start = finding.get("start") if isinstance(finding.get("start"), dict) else {}
     end = finding.get("end") if isinstance(finding.get("end"), dict) else {}
     start_line = _line_number(start.get("line") if isinstance(start, dict) else None)
-    end_line = _line_number(
-        end.get("line") if isinstance(end, dict) else None, start_line
-    )
+    # _line_number floors the end line at start_line, so the range never inverts.
+    end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line)
     return {
         "engineId": "semgrep",
         "ruleId": str(finding.get("check_id") or "semgrep.unknown"),
         "type": "VULNERABILITY" if severity == "ERROR" else "CODE_SMELL",
         "severity": SONAR_SEVERITY_BY_SEMGREP[severity],
         "primaryLocation": {
-            "message": str(
-                extra.get("message") or finding.get("check_id") or "Semgrep finding"
-            ),
+            "message": str(extra.get("message") or finding.get("check_id") or "Semgrep finding"),
             "filePath": path,
             "textRange": {
                 "startLine": start_line,
@@ -125,19 +111,11 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None:
     }
 
 
-def build_sonar_issues(
-    semgrep_payload: dict[str, Any],
-) -> dict[str, list[dict[str, Any]]]:
+def build_sonar_issues(semgrep_payload: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
     """Build SonarQube generic external issues from Semgrep JSON output."""
     raw_results = semgrep_payload.get("results", [])
-    results = (
-        [item for item in raw_results if isinstance(item, dict)]
-        if isinstance(raw_results, list)
-        else []
-    )
-    issues = [
-        issue for finding in results if (issue := _sonar_issue_from_finding(finding))
-    ]
+    results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
+    issues = [issue for finding in results if (issue := _sonar_issue_from_finding(finding))]
     return {"issues": issues}
 
 
@@ -165,9 +143,7 @@ def main(argv: list[str] | None = None) -> int:
     )
     output = Path(args.output)
     output.parent.mkdir(parents=True, exist_ok=True)
-    output.write_text(
-        json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8"
-    )
+    output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
     if args.sonar_issues_output:
         sonar_output = Path(args.sonar_issues_output)
         sonar_output.parent.mkdir(parents=True, exist_ok=True)

clusters/atlas/flux-system/applications/harbor/kustomization.yaml

@@ -15,7 +15,13 @@ spec:
     kind: GitRepository
     name: flux-system
     namespace: flux-system
-  wait: false
+  wait: true
+  timeout: 10m
+  healthChecks:
+    - apiVersion: batch/v1
+      kind: Job
+      name: harbor-hermes-agent-immutability-ensure-1
+      namespace: harbor
   dependsOn:
     - name: core
     - name: longhorn

clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml

@@ -0,0 +1,19 @@
+# clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: hermes-observer-bindings
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/hermes-observer-bindings
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: true
+  timeout: 5m
+  dependsOn:
+    - name: hermes
+    - name: hermes-observer-rbac

clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml

@@ -0,0 +1,16 @@
+# clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: hermes-observer-rbac
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/hermes-observer-rbac
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: true
+  timeout: 5m

clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml

@@ -0,0 +1,19 @@
+# clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: hermes-scm-broker-code
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/hermes/scm-common
+  targetNamespace: hermes-scm
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: true
+  timeout: 5m
+  dependsOn:
+    - name: hermes-scm-namespace

clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml

@@ -0,0 +1,27 @@
+# clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: hermes-scm-broker
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/hermes-scm-broker
+  targetNamespace: hermes-scm
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: true
+  timeout: 10m
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: hermes-scm-broker
+      namespace: hermes-scm
+  dependsOn:
+    - name: vault
+    - name: gitea
+    - name: hermes-scm-namespace
+    - name: hermes-scm-broker-code

clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml

@@ -0,0 +1,16 @@
+# clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: hermes-scm-namespace
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/hermes-scm-namespace
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: true
+  timeout: 5m

clusters/atlas/flux-system/applications/hermes/kustomization.yaml

@@ -30,6 +30,12 @@ spec:
       kind: Deployment
       name: hermes-agent
       namespace: hermes
+    # hermes-execution-worker and its mediators are deliberately absent. They run
+    # at scavenger priority on a best-effort pool whose first start installs the
+    # provider CLIs, so gating this Kustomization's 10m health window on them
+    # would stall hermes-chat and hermes-observer-bindings, which dependsOn
+    # hermes. The pool reports its own health through the worker readiness probe,
+    # the per-ordinal mediator /ready endpoint, and the coordinator on :9007.
     - apiVersion: apps/v1
       kind: DaemonSet
       name: hermes-node-ssh-access
@@ -60,3 +66,6 @@ spec:
     - name: keycloak
     - name: longhorn
     - name: vault
+    - name: jenkins
+    - name: hermes-scm-broker
+    - name: hermes-observer-rbac

clusters/atlas/flux-system/applications/jenkins/kustomization.yaml

@@ -6,10 +6,9 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
-    atlas.bstein.dev/suspend-reason: "CI controller changes are applied only during planned maintenance."
 spec:
   interval: 10m
-  suspend: true
+  suspend: false
   path: ./services/jenkins
   prune: true
   sourceRef:
@@ -18,6 +17,8 @@ spec:
   targetNamespace: jenkins
   dependsOn:
     - name: helm
+    - name: harbor
+    - name: vault-hermes-jenkins-token-seed
   healthChecks:
     - apiVersion: apps/v1
       kind: Deployment

clusters/atlas/flux-system/applications/kustomization.yaml

@@ -4,6 +4,7 @@ kind: Kustomization
 resources:
   - gitea/kustomization.yaml
   - vault/kustomization.yaml
+  - vault-hermes-jenkins-token-seed/kustomization.yaml
   - vaultwarden/kustomization.yaml
   - comms/kustomization.yaml
   - crypto/kustomization.yaml
@@ -27,7 +28,12 @@ resources:
   - jenkins/kustomization.yaml
   - ai-llm/kustomization.yaml
   - openclaw/kustomization.yaml
+  - hermes-scm-namespace/kustomization.yaml
+  - hermes-scm-broker-code/kustomization.yaml
   - hermes/kustomization.yaml
+  - hermes-observer-rbac/kustomization.yaml
+  - hermes-observer-bindings/kustomization.yaml
+  - hermes-scm-broker/kustomization.yaml
   - hermes-chat/kustomization.yaml
   - hermes-triage-demo/kustomization.yaml
   - game-stream/kustomization.yaml

clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml

@@ -0,0 +1,26 @@
+# clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: vault-hermes-jenkins-token-seed
+  namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+spec:
+  interval: 10m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  path: ./services/vault-hermes-jenkins-token-seed
+  targetNamespace: vault
+  prune: true
+  wait: true
+  timeout: 5m
+  dependsOn:
+    - name: vault
+  healthChecks:
+    - apiVersion: batch/v1
+      kind: Job
+      name: vault-hermes-jenkins-build-token-seed-2
+      namespace: vault

clusters/atlas/flux-system/applications/vault/kustomization.yaml

@@ -16,6 +16,11 @@ spec:
   targetNamespace: vault
   prune: true
   wait: true
+  healthChecks:
+    - apiVersion: batch/v1
+      kind: Job
+      name: vault-k8s-auth-hermes-10
+      namespace: vault
   dependsOn:
     - name: longhorn
     - name: helm

dockerfiles/Dockerfile.hermes-agent

@@ -15,6 +15,10 @@ RUN apt-get update \
 # Paid/provider-backed search remains selectable through normal Hermes config.
 RUN uv pip install --python /opt/hermes/.venv/bin/python ddgs==9.14.4
 
+ARG HERMES_KANIKO_HEREDOC_COMPAT=0
+COPY dockerfiles/Dockerfile.hermes-agent /tmp/hermes-agent.Dockerfile
+COPY dockerfiles/hermes-kaniko-heredoc-runner.py /tmp/hermes-kaniko-heredoc-runner.py
+
 # Keep dashboard chat sockets tied to the intended React mount and conversation.
 # A resumed conversation needs a different PTY attachment key from a fresh chat;
 # reconnects to that same conversation must keep using the same key.
@@ -252,6 +256,7 @@ for (const [before, after, label] of [
 }
 fs.writeFileSync(sidebarPath, sidebar);
 NODE
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 1 ;; *) exit 2 ;; esac
 
 # The upstream OIDC gate authenticates users but deliberately treats the
 # dashboard as one shared workstation. Allow a deployment to narrow that
@@ -324,6 +329,7 @@ for before, after, label in (
     source = source.replace(before, after, 1)
 path.write_text(source)
 PY
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 2 ;; *) exit 2 ;; esac
 
 # Give trusted plugins a pre-turn routing hook. It runs after fallback runtime
 # restoration but before Hermes builds its provider-specific system prompt.
@@ -493,6 +499,7 @@ if oneshot.count(oneshot_before) != 1:
     )
 oneshot_path.write_text(oneshot.replace(oneshot_before, oneshot_after, 1))
 PY
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 3 ;; *) exit 2 ;; esac
 
 # Hermes WebUI sends its model/provider/reasoning selection on /v1/runs.
 # Upstream currently applies only statically declared model_routes there, so
@@ -628,6 +635,7 @@ runs_source = runs_source.replace(runs_agent_before, runs_agent_after, 1)
 source = source[:runs_start] + runs_source
 path.write_text(source)
 PY
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 4 ;; *) exit 2 ;; esac
 
 # ``create_task(initial_status="blocked")`` is an explicit operator park, but
 # upstream only treats later block_task() events as sticky. Recognize the
@@ -779,6 +787,7 @@ if cli_source.count(help_before) != 1:
     )
 cli_path.write_text(cli_source.replace(help_before, help_after, 1))
 PY
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 5 ;; *) exit 2 ;; esac
 
 COPY dockerfiles/hermes-kanban-blocked-regression.py /tmp/hermes-kanban-blocked-regression.py
 RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py \
@@ -816,6 +825,7 @@ if source.count(before) != 1:
     )
 path.write_text(source.replace(before, after, 1))
 PY
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 6 ;; *) exit 2 ;; esac
 
 # Keep the dashboard's account cards aligned with the managed provider lanes.
 # Switchyard receives Claude Code subscription OAuth through the environment;
@@ -882,6 +892,7 @@ for before, after, label in replacements:
 
 path.write_text(source)
 PY
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 7 ;; *) exit 2 ;; esac
 
 # Make parent-linked API workers first-class live dashboard sessions. They stay
 # active until the API runner closes them, even during a long final model call;
@@ -975,6 +986,7 @@ source = source.replace(row_before, row_after)
 
 path.write_text(source)
 PY
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 8 ;; *) exit 2 ;; esac
 
 # Keep the browser terminal reliable across GPU context loss, make durable
 # worker lineage visible, expose an accessible API-worker transcript, and
@@ -1543,9 +1555,42 @@ function RootRedirect() {
   fs.writeFileSync(path, source);
 }
 NODE
+RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 9 ;; *) exit 2 ;; esac
+
+COPY dockerfiles/patch-hermes-execution-safety.py /tmp/patch-hermes-execution-safety.py
+COPY dockerfiles/hermes-execution-safety-regression.py /tmp/hermes-execution-safety-regression.py
+COPY dockerfiles/hermes_execution_patch_support.py /tmp/hermes_execution_patch_support.py
+COPY dockerfiles/patch_hermes_run_safety.py /tmp/patch_hermes_run_safety.py
+COPY dockerfiles/patch_hermes_decomposition_safety.py /tmp/patch_hermes_decomposition_safety.py
+COPY dockerfiles/hermes_execution_regression_support.py /tmp/hermes_execution_regression_support.py
+COPY dockerfiles/hermes_run_safety_regression.py /tmp/hermes_run_safety_regression.py
+COPY dockerfiles/hermes_decomposition_safety_regression.py /tmp/hermes_decomposition_safety_regression.py
+COPY dockerfiles/hermes_lane_compatibility_regression.py /tmp/hermes_lane_compatibility_regression.py
+COPY services/hermes/scripts/cli_lane_*.py /tmp/hermes-lane-regression/
+RUN HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
+        HERMES_COMPATIBILITY_MODE=legacy \
+        /opt/hermes/.venv/bin/python /tmp/hermes_lane_compatibility_regression.py \
+    && /opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py \
+    && HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
+        HERMES_COMPATIBILITY_MODE=patched \
+        /opt/hermes/.venv/bin/python /tmp/hermes_lane_compatibility_regression.py \
+    && HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
+        /opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py \
+    && rm /tmp/patch-hermes-execution-safety.py \
+        /tmp/hermes-execution-safety-regression.py \
+        /tmp/hermes_execution_patch_support.py \
+        /tmp/patch_hermes_run_safety.py \
+        /tmp/patch_hermes_decomposition_safety.py \
+        /tmp/hermes_execution_regression_support.py \
+        /tmp/hermes_run_safety_regression.py \
+        /tmp/hermes_decomposition_safety_regression.py \
+        /tmp/hermes_lane_compatibility_regression.py \
+    && find /tmp/hermes-lane-regression -depth -delete
 
 COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate
 
+RUN rm -f /tmp/hermes-agent.Dockerfile /tmp/hermes-kaniko-heredoc-runner.py
+
 RUN cd /opt/hermes/web \
     && npm run build \
     && grep -Fq 'await api.getSessions(1, 0' src/pages/ChatPage.tsx \
@@ -1602,6 +1647,8 @@ RUN cd /opt/hermes/web \
         /opt/hermes/agent/turn_context.py \
         /opt/hermes/agent/conversation_loop.py \
         /opt/hermes/hermes_cli/oneshot.py \
+        /opt/hermes/hermes_cli/kanban_decompose.py \
+        /opt/hermes/gateway/kanban_watchers.py \
         /opt/hermes/tools/delegate_tool.py \
         /opt/hermes/tools/web_tools.py \
         /opt/hermes/tools/python_sandbox_tool.py \

dockerfiles/Dockerfile.hermes-agent.dockerignore

@@ -5,3 +5,18 @@
 !dockerfiles/hermes-public-extract/**
 !dockerfiles/hermes-session-activity-panel.tsx
 !dockerfiles/hermes-session-migrate.py
+!dockerfiles/Dockerfile.hermes-agent
+!dockerfiles/hermes-kaniko-heredoc-runner.py
+!dockerfiles/patch-hermes-execution-safety.py
+!dockerfiles/hermes-execution-safety-regression.py
+!dockerfiles/hermes_execution_patch_support.py
+!dockerfiles/patch_hermes_run_safety.py
+!dockerfiles/patch_hermes_decomposition_safety.py
+!dockerfiles/hermes_execution_regression_support.py
+!dockerfiles/hermes_run_safety_regression.py
+!dockerfiles/hermes_decomposition_safety_regression.py
+!dockerfiles/hermes_lane_compatibility_regression.py
+!services/
+!services/hermes/
+!services/hermes/scripts/
+!services/hermes/scripts/cli_lane_*.py

dockerfiles/hermes-execution-safety-regression.py

@@ -0,0 +1,23 @@
+"""Run the composable Hermes execution-safety build regressions."""
+
+import unittest
+
+from hermes_decomposition_safety_regression import DecompositionSafetyTests
+from hermes_run_safety_regression import ExactRunSafetyTests
+
+
+def main() -> int:
+    """Run both transaction domains and return a build-friendly status."""
+    loader = unittest.defaultTestLoader
+    suite = unittest.TestSuite(
+        (
+            loader.loadTestsFromTestCase(ExactRunSafetyTests),
+            loader.loadTestsFromTestCase(DecompositionSafetyTests),
+        )
+    )
+    result = unittest.TextTestRunner(verbosity=2).run(suite)
+    return 0 if result.wasSuccessful() else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dockerfiles/hermes-kaniko-heredoc-runner.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""Replay the reviewed Hermes Dockerfile heredocs for pinned Kaniko v1.23.2.
+
+Kaniko v1.23.2 parses shell heredoc bodies into ``RunCommand.Files`` but its RUN
+implementation executes only ``CmdLine``. The interpreters therefore receive
+empty stdin and exit successfully. Docker and BuildKit execute these blocks
+normally, so this compatibility runner is enabled only by the Kaniko pipeline.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import subprocess
+from pathlib import Path
+
+
+MAX_DOCKERFILE_BYTES = 2_000_000
+BLOCKS = (
+    ("RUN node <<'NODE'", "NODE", ("node",)),
+    ("RUN python - <<'PY'", "PY", ("python", "-")),
+)
+EXPECTED_COMMANDS = ("node", *("python",) * 7, "node")
+PARSER_DIRECTIVE = re.compile(
+    r"^\s*#\s*([a-z]+)\s*=\s*(.*?)\s*$", re.IGNORECASE
+)
+
+
+def _escape_character(lines: list[str]) -> str:
+    """Return the one Dockerfile parser escape directive, or its default."""
+    escape = "\\"
+    directives: set[str] = set()
+    for line in lines:
+        if not line.strip():
+            break
+        match = PARSER_DIRECTIVE.fullmatch(line)
+        if not match:
+            break
+        name, value = match.group(1).lower(), match.group(2)
+        if name in directives:
+            raise ValueError(f"multiple Dockerfile {name} directives")
+        directives.add(name)
+        if name == "escape":
+            if value not in {"\\", "`"}:
+                raise ValueError("unsupported Dockerfile escape directive")
+            escape = value
+    return escape
+
+
+def _continued(line: str, escape: str) -> bool:
+    """Match Docker's unescaped continuation marker at physical line end."""
+    stripped = line.rstrip()
+    count = len(stripped) - len(stripped.rstrip(escape))
+    return count % 2 == 1
+
+
+def _logical_instruction(
+    lines: list[str], index: int, escape: str
+) -> tuple[str, int]:
+    """Normalize escape-newline pairs, including split opcodes/operators."""
+    logical = lines[index]
+    while _continued(logical.split("\n")[-1], escape):
+        logical = logical.rstrip()
+        logical = logical[:-1]
+        index += 1
+        while index < len(lines) and lines[index].lstrip().startswith("#"):
+            index += 1
+        if index >= len(lines) or not lines[index].strip():
+            raise ValueError("unterminated Dockerfile line continuation")
+        logical += lines[index].lstrip()
+    return logical, index
+
+
+def extract_blocks(source: str) -> list[tuple[tuple[str, ...], str]]:
+    """Inventory every RUN heredoc, then return only the reviewed contract."""
+    markers = {start: (end, command) for start, end, command in BLOCKS}
+    lines = source.splitlines()
+    escape = _escape_character(lines)
+    blocks: list[tuple[tuple[str, ...], str]] = []
+    index = 0
+    while index < len(lines):
+        marker = markers.get(lines[index])
+        if marker is None:
+            if not lines[index].strip() or lines[index].lstrip().startswith("#"):
+                index += 1
+                continue
+            logical, index = _logical_instruction(lines, index, escape)
+            stripped = logical.lstrip()
+            if (
+                stripped[:3].upper() == "RUN"
+                and (len(stripped) == 3 or stripped[3].isspace())
+                and "<<" in stripped
+            ):
+                raise ValueError(
+                    "unsupported RUN heredoc outside the exact reviewed contract"
+                )
+            index += 1
+            continue
+        end, command = marker
+        body_start = index + 1
+        index = body_start
+        while index < len(lines) and lines[index] != end:
+            index += 1
+        if index == len(lines):
+            raise ValueError(f"unterminated {command[0]} heredoc")
+        body = "\n".join(lines[body_start:index]) + "\n"
+        if not body.strip():
+            raise ValueError(f"empty {command[0]} heredoc")
+        blocks.append((command, body))
+        index += 1
+
+    commands = tuple(command[0] for command, _body in blocks)
+    if commands != EXPECTED_COMMANDS:
+        raise ValueError(
+            "Hermes Dockerfile heredoc contract changed: "
+            f"expected {EXPECTED_COMMANDS!r}, received {commands!r}"
+        )
+    return blocks
+
+
+def replay(dockerfile: Path, block_index: int) -> None:
+    """Execute one exact heredoc after inventorying the complete Dockerfile."""
+    size = dockerfile.stat().st_size
+    if size < 1 or size > MAX_DOCKERFILE_BYTES:
+        raise ValueError("Hermes Dockerfile size is outside the reviewed boundary")
+    source = dockerfile.read_text(encoding="utf-8")
+    blocks = extract_blocks(source)
+    if block_index < 1 or block_index > len(blocks):
+        raise ValueError("heredoc block index is outside the reviewed contract")
+    command, body = blocks[block_index - 1]
+    print(f"replaying reviewed Dockerfile heredoc {block_index}/{len(blocks)}")
+    subprocess.run(command, input=body, text=True, check=True)
+
+
+def main() -> int:
+    """Parse the one explicit Dockerfile path and replay its reviewed patches."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dockerfile", required=True, type=Path)
+    parser.add_argument("--block-index", required=True, type=int)
+    args = parser.parse_args()
+    replay(args.dockerfile, args.block_index)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dockerfiles/hermes_decomposition_safety_regression.py

@@ -0,0 +1,120 @@
+"""Regress automatic decomposition against existing execution history."""
+
+from __future__ import annotations
+
+
+from hermes_cli import kanban_db
+from hermes_cli import kanban_decompose
+from hermes_execution_regression_support import ExecutionSafetyTestCase
+
+
+class DecompositionSafetyTests(ExecutionSafetyTestCase):
+    """Verify only unexecuted triage roots may decompose automatically."""
+
+    def test_automatic_decomposer_skips_executed_triage_task(self) -> None:
+        task_id = self._route_executed_task_to_triage()
+        completions = self._client(
+            {"fanout": False, "title": "must not run", "body": "must not run"}
+        )
+        before = len(kanban_db.list_tasks(self.connection))
+
+        outcome = kanban_decompose.decompose_task(task_id, automatic=True)
+
+        self.assertFalse(outcome.ok)
+        self.assertIn("execution history", outcome.reason)
+        self.assertEqual(completions.calls, 0)
+        self.assertEqual(self._status(task_id), "triage")
+        self.assertEqual(len(kanban_db.list_tasks(self.connection)), before)
+
+    def test_manual_decomposition_remains_available_after_execution(self) -> None:
+        task_id = self._route_executed_task_to_triage()
+        completions = self._client(
+            {
+                "fanout": False,
+                "title": "operator-approved retry",
+                "body": "Retain one bounded task.",
+                "assignee": "default",
+            }
+        )
+
+        outcome = kanban_decompose.decompose_task(task_id, author="operator")
+
+        self.assertTrue(outcome.ok)
+        self.assertFalse(outcome.fanout)
+        self.assertEqual(completions.calls, 1)
+        self.assertEqual(self._status(task_id), "ready")
+
+    def test_fresh_triage_task_still_auto_promotes(self) -> None:
+        task_id = kanban_db.create_task(
+            self.connection,
+            title="fresh objective",
+            triage=True,
+        )
+        completions = self._client(
+            {
+                "fanout": False,
+                "title": "specified objective",
+                "body": "One bounded task.",
+                "assignee": "default",
+            }
+        )
+
+        outcome = kanban_decompose.decompose_task(task_id, automatic=True)
+
+        self.assertTrue(outcome.ok)
+        self.assertEqual(completions.calls, 1)
+        self.assertEqual(self._status(task_id), "ready")
+
+    def test_execution_history_gained_during_single_llm_call_blocks_commit(self) -> None:
+        task_id = kanban_db.create_task(
+            self.connection,
+            title="concurrent single objective",
+            triage=True,
+        )
+        completions = self._client(
+            {
+                "fanout": False,
+                "title": "redundant rewrite",
+                "body": "must not be committed",
+                "assignee": "default",
+            },
+            before_response=lambda: self._insert_run(task_id),
+        )
+
+        outcome = kanban_decompose.decompose_task(task_id, automatic=True)
+
+        self.assertFalse(outcome.ok)
+        self.assertIn("gained execution history", outcome.reason)
+        self.assertEqual(completions.calls, 1)
+        self.assertEqual(self._status(task_id), "triage")
+
+    def test_execution_history_gained_during_fanout_llm_call_blocks_commit(self) -> None:
+        task_id = kanban_db.create_task(
+            self.connection,
+            title="concurrent triage objective",
+            triage=True,
+        )
+
+        completions = self._client(
+            {
+                "fanout": True,
+                "tasks": [
+                    {
+                        "title": "redundant child",
+                        "body": "must not be created",
+                        "assignee": "default",
+                        "parents": [],
+                    }
+                ],
+            },
+            before_response=lambda: self._insert_run(task_id),
+        )
+        before = len(kanban_db.list_tasks(self.connection))
+
+        outcome = kanban_decompose.decompose_task(task_id, automatic=True)
+
+        self.assertFalse(outcome.ok)
+        self.assertIn("gained execution history", outcome.reason)
+        self.assertEqual(completions.calls, 1)
+        self.assertEqual(self._status(task_id), "triage")
+        self.assertEqual(len(kanban_db.list_tasks(self.connection)), before)

dockerfiles/hermes_execution_patch_support.py

@@ -0,0 +1,12 @@
+"""Fail-closed helpers shared by Hermes source patches."""
+
+
+
+def replace_once(source: str, before: str, after: str, label: str) -> str:
+    """Replace one anchored upstream fragment and fail closed on source drift."""
+    count = source.count(before)
+    if count != 1:
+        raise SystemExit(
+            f"Hermes {label} patch context changed: expected 1, found {count}"
+        )
+    return source.replace(before, after, 1)

dockerfiles/hermes_execution_regression_support.py

@@ -0,0 +1,132 @@
+"""Shared isolated database fixture for Hermes execution-safety regressions."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest import mock
+
+_HERMES_HOME = tempfile.TemporaryDirectory(prefix="hermes-execution-test-home-")
+os.environ["HERMES_HOME"] = _HERMES_HOME.name
+_LANE_SOURCE = Path(
+    os.environ.get("HERMES_CLI_LANE_SOURCE", "/tmp/hermes-lane-regression")
+)
+sys.path.insert(0, str(_LANE_SOURCE))
+
+from agent import auxiliary_client  # noqa: E402
+from cli_lane_config import TerminalIdentity  # noqa: E402
+from cli_lane_finalization import _finalize_document_db  # noqa: E402
+from hermes_cli import kanban_db  # noqa: E402
+from hermes_cli import kanban_decompose  # noqa: E402
+
+__all__ = ("ExecutionSafetyTestCase", "TerminalIdentity", "_finalize_document_db")
+
+
+class _FakeCompletions:
+    def __init__(self, payload: dict, before_response=None) -> None:
+        self.payload = payload
+        self.before_response = before_response
+        self.calls = 0
+
+    def create(self, **_kwargs):
+        self.calls += 1
+        if self.before_response is not None:
+            self.before_response()
+        return SimpleNamespace(
+            choices=[
+                SimpleNamespace(
+                    message=SimpleNamespace(content=json.dumps(self.payload))
+                )
+            ]
+        )
+
+
+class _FakeClient:
+    def __init__(self, completions: _FakeCompletions) -> None:
+        self.chat = SimpleNamespace(completions=completions)
+
+
+class ExecutionSafetyTestCase(unittest.TestCase):
+    """Exercise patched upstream APIs against an isolated real database."""
+
+    def setUp(self) -> None:
+        self.connection = kanban_db.connect()
+        self.patches = [
+            mock.patch.object(
+                kanban_decompose,
+                "_build_roster",
+                return_value=([], {"default"}),
+            ),
+            mock.patch.object(
+                kanban_decompose,
+                "_load_config",
+                return_value={"kanban": {"auto_promote_children": True}},
+            ),
+            mock.patch.object(
+                auxiliary_client,
+                "get_auxiliary_extra_body",
+                return_value={},
+            ),
+        ]
+        for patch in self.patches:
+            patch.start()
+
+    def tearDown(self) -> None:
+        for patch in reversed(self.patches):
+            patch.stop()
+        self.connection.close()
+
+    def _client(self, payload: dict, before_response=None) -> _FakeCompletions:
+        completions = _FakeCompletions(payload, before_response)
+        client = _FakeClient(completions)
+        patch = mock.patch.object(
+            auxiliary_client,
+            "get_text_auxiliary_client",
+            return_value=(client, "test-decomposer"),
+        )
+        patch.start()
+        self.addCleanup(patch.stop)
+        return completions
+
+    def _status(self, task_id: str) -> str:
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertIsNotNone(task)
+        return task.status
+
+    def _route_executed_task_to_triage(self) -> str:
+        task_id = kanban_db.create_task(
+            self.connection,
+            title="already implemented parent",
+        )
+        for attempt in range(kanban_db.BLOCK_RECURRENCE_LIMIT):
+            if attempt:
+                self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+            self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+            self.assertTrue(
+                kanban_db.block_task(
+                    self.connection,
+                    task_id,
+                    reason="same unavailable capability",
+                    kind="capability",
+                )
+            )
+        self.assertEqual(self._status(task_id), "triage")
+        self.assertEqual(
+            len(kanban_db.list_runs(self.connection, task_id)),
+            kanban_db.BLOCK_RECURRENCE_LIMIT,
+        )
+        return task_id
+
+    def _insert_run(self, task_id: str) -> None:
+        with kanban_db.connect_closing() as connection:
+            kanban_db._synthesize_ended_run(
+                connection,
+                task_id,
+                outcome="reclaimed",
+                summary="concurrent execution evidence",
+            )

dockerfiles/hermes_lane_compatibility_regression.py

@@ -0,0 +1,120 @@
+"""Exercise mounted CLI scripts against the exact image Kanban API."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+
+_HOME = tempfile.TemporaryDirectory(prefix="hermes-lane-compatibility-")
+os.environ["HERMES_HOME"] = _HOME.name
+os.environ["HERMES_CLI_LANE_CONCURRENCY"] = "1"
+lane_source = Path(os.environ["HERMES_CLI_LANE_SOURCE"])
+sys.path.insert(0, str(lane_source))
+
+from cli_lane_capabilities import (  # noqa: E402
+    detect_kanban_capabilities,
+    initialize_kanban_capabilities,
+    runtime_health,
+)
+from cli_lane_config import TerminalIdentity  # noqa: E402
+from cli_lane_dispatch import recover_orphans  # noqa: E402
+from cli_lane_finalization import _finalize_document_db  # noqa: E402
+from hermes_cli import kanban_db  # noqa: E402
+
+
+class MountedLaneCompatibilityTests(unittest.TestCase):
+    """Verify safe behavior before and after the image API patch."""
+
+    def setUp(self) -> None:
+        self.connection = kanban_db.connect()
+
+    def tearDown(self) -> None:
+        self.connection.close()
+
+    @staticmethod
+    def _document(summary: str) -> dict:
+        return {
+            "result": json.dumps(
+                {
+                    "status": "completed",
+                    "summary": summary,
+                    "changed_files": [],
+                    "tests_run": ["mixed source compatibility"],
+                    "artifacts": [],
+                    "findings": [],
+                    "blockers": [],
+                },
+                sort_keys=True,
+            ),
+            "summary": summary,
+            "metadata": {"matrix": os.environ["HERMES_COMPATIBILITY_MODE"]},
+        }
+
+    def test_exact_image_api_matrix(self) -> None:
+        mode = os.environ["HERMES_COMPATIBILITY_MODE"]
+        capabilities = initialize_kanban_capabilities(kanban_db)
+        self.assertTrue(capabilities.exact_run_completion)
+        self.assertEqual(capabilities.ready, mode == "patched")
+        self.assertEqual(runtime_health()["state"], "ready" if mode == "patched" else "deferred")
+        self.assertLess(Path(_HOME.name, "cli-lanes/runtime-health.json").stat().st_size, 2048)
+
+    def test_current_active_run_completion_stays_safe(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="active accepted result")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        run_id = kanban_db.get_task(self.connection, task_id).current_run_id
+        outcome = _finalize_document_db(
+            kanban_db,
+            TerminalIdentity("default", task_id, run_id, "pending"),
+            self._document("active result"),
+        )
+        self.assertEqual(outcome, "committed")
+        self.assertEqual(kanban_db.get_task(self.connection, task_id).status, "done")
+
+    def test_ended_run_replay_is_enabled_only_after_patch(self) -> None:
+        mode = os.environ["HERMES_COMPATIBILITY_MODE"]
+        task_id = kanban_db.create_task(self.connection, title="ended accepted result")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        run_id = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="simulate post-journal error",
+                expected_run_id=run_id,
+            )
+        )
+        outcome = _finalize_document_db(
+            kanban_db,
+            TerminalIdentity("default", task_id, run_id, "pending"),
+            self._document("ended result"),
+        )
+        self.assertEqual(outcome, "committed" if mode == "patched" else "deferred")
+
+    def test_old_reclaim_api_is_never_called_unguarded(self) -> None:
+        capabilities = detect_kanban_capabilities(kanban_db)
+        if capabilities.exact_run_reclaim:
+            self.skipTest("patched API exercises guarded reclaim in the exact-run suite")
+        with mock.patch.object(
+            kanban_db,
+            "reclaim_task",
+            side_effect=AssertionError("legacy unguarded reclaim called"),
+        ), mock.patch("cli_lane_dispatch.recover_pending_finalizations", return_value=0):
+            recover_orphans()
+
+
+def main() -> int:
+    """Run the compatibility matrix for the selected exact image source."""
+    result = unittest.TextTestRunner(verbosity=2).run(
+        unittest.defaultTestLoader.loadTestsFromTestCase(MountedLaneCompatibilityTests)
+    )
+    return 0 if result.wasSuccessful() else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dockerfiles/hermes_run_safety_regression.py

@@ -0,0 +1,467 @@
+"""Regress exact-run replay, provenance, and reclaim safety."""
+
+from __future__ import annotations
+
+import threading
+from unittest import mock
+
+from hermes_cli import kanban_db
+from hermes_execution_regression_support import (
+    ExecutionSafetyTestCase,
+    TerminalIdentity,
+    _finalize_document_db,
+)
+from cli_lane_files import state_path
+from cli_lane_records import _write_terminal_record
+from cli_lane_recovery import recover_pending_finalizations
+
+
+class ExactRunSafetyTests(ExecutionSafetyTestCase):
+    """Verify exact-run transactions reject every stale concurrent actor."""
+
+    def _lane_document(self, summary: str) -> dict:
+        """Return the DB-bearing portion of one accepted lane journal."""
+        return {
+            "result": summary,
+            "summary": summary,
+            "metadata": {"source": "exact-run-regression"},
+        }
+
+    def test_lane_marks_replaced_ended_run_stale_without_mutation(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="replaced journal")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        first_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="first run ended after persistence",
+                expected_run_id=first_run,
+            )
+        )
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        second_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="replacement run also ended",
+                expected_run_id=second_run,
+            )
+        )
+        before = kanban_db.get_task(self.connection, task_id)
+
+        outcome = _finalize_document_db(
+            kanban_db,
+            TerminalIdentity("default", task_id, first_run, "pending"),
+            self._lane_document("stale first result"),
+        )
+
+        after = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(outcome, "stale")
+        self.assertEqual(after.status, before.status)
+        self.assertEqual(after.current_run_id, before.current_run_id)
+        self.assertEqual(after.result, before.result)
+        self.assertEqual(after.completed_run_id, before.completed_run_id)
+
+    def test_lane_marks_nonexistent_ended_run_stale_without_mutation(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="no such run")
+        before = kanban_db.get_task(self.connection, task_id)
+
+        outcome = _finalize_document_db(
+            kanban_db,
+            TerminalIdentity("default", task_id, 42, "pending"),
+            self._lane_document("orphan result"),
+        )
+
+        after = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(outcome, "stale")
+        self.assertEqual(after.status, before.status)
+        self.assertEqual(after.result, before.result)
+        self.assertEqual(kanban_db.list_runs(self.connection, task_id), [])
+
+    def test_full_lane_recovery_conflicts_a_replaced_ended_run(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="recovery conflict")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        first_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="first accepted run ended",
+                expected_run_id=first_run,
+            )
+        )
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        second_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="replacement ended",
+                expected_run_id=second_run,
+            )
+        )
+        before = kanban_db.get_task(self.connection, task_id)
+        state_file = state_path("default", task_id)
+        pending, _record = _write_terminal_record(
+            state_file,
+            board="default",
+            task_id=task_id,
+            run_id=first_run,
+            structured={
+                "status": "completed",
+                "summary": "accepted stale result",
+                "changed_files": [],
+                "tests_run": ["real SQLite recovery regression"],
+                "artifacts": [],
+                "findings": [],
+                "blockers": [],
+            },
+            summary="accepted stale result",
+            metadata={"source": "exact-run-regression"},
+        )
+
+        self.assertEqual(recover_pending_finalizations(), 0)
+
+        after = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(after.status, before.status)
+        self.assertEqual(after.current_run_id, before.current_run_id)
+        self.assertEqual(after.result, before.result)
+        self.assertFalse(pending.exists())
+        conflicts = list(
+            state_file.parent.glob(
+                f"{state_file.stem}.run-{first_run}.terminal.conflict-*.json"
+            )
+        )
+        self.assertEqual(len(conflicts), 1)
+
+    def test_latest_ended_run_can_replay_a_durable_completion(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="journaled result")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        run_id = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="legacy post-journal failure",
+                kind="capability",
+                expected_run_id=run_id,
+            )
+        )
+
+        with mock.patch.object(
+            kanban_db, "_fire_kanban_lifecycle_hook"
+        ) as lifecycle:
+            self.assertTrue(
+                kanban_db.complete_task(
+                    self.connection,
+                    task_id,
+                    result="durable terminal result",
+                    summary="durable terminal result",
+                    replay_ended_run_id=run_id,
+                )
+            )
+        lifecycle.assert_called_once()
+        self.assertEqual(lifecycle.call_args.args[:2], ("kanban_task_completed", task_id))
+        self.assertEqual(lifecycle.call_args.kwargs["run_id"], run_id)
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(task.status, "done")
+        self.assertEqual(task.completed_run_id, run_id)
+        runs = kanban_db.list_runs(self.connection, task_id)
+        self.assertEqual(len(runs), 1)
+        self.assertEqual(runs[0].id, run_id)
+        self.assertEqual(runs[0].status, "done")
+        self.assertEqual(runs[0].outcome, "completed")
+        self.assertEqual(runs[0].summary, "durable terminal result")
+        completed_events = self.connection.execute(
+            "SELECT run_id FROM task_events "
+            "WHERE task_id = ? AND kind = 'completed' ORDER BY id",
+            (task_id,),
+        ).fetchall()
+        self.assertEqual([row["run_id"] for row in completed_events], [run_id])
+
+    def test_unchanged_triage_can_replay_its_exact_latest_ended_run(self) -> None:
+        task_id = self._route_executed_task_to_triage()
+        runs = kanban_db.list_runs(self.connection, task_id)
+        run_id = runs[-1].id
+
+        self.assertTrue(
+            kanban_db.complete_task(
+                self.connection,
+                task_id,
+                result="accepted before the loop breaker routed triage",
+                summary="accepted before the loop breaker routed triage",
+                replay_ended_run_id=run_id,
+            )
+        )
+
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(task.status, "done")
+        self.assertEqual(task.completed_run_id, run_id)
+        updated_runs = kanban_db.list_runs(self.connection, task_id)
+        self.assertEqual(len(updated_runs), kanban_db.BLOCK_RECURRENCE_LIMIT)
+        self.assertEqual(updated_runs[-1].id, run_id)
+        self.assertEqual(updated_runs[-1].outcome, "completed")
+
+    def test_triage_status_transition_prevents_stale_exact_run_replay(self) -> None:
+        task_id = self._route_executed_task_to_triage()
+        run_id = kanban_db.list_runs(self.connection, task_id)[-1].id
+        self.assertTrue(
+            kanban_db.specify_triage_task(
+                self.connection,
+                task_id,
+                title="operator changed the objective",
+                author="operator",
+            )
+        )
+
+        self.assertFalse(
+            kanban_db.complete_task(
+                self.connection,
+                task_id,
+                result="stale accepted result",
+                replay_ended_run_id=run_id,
+            )
+        )
+        self.assertNotEqual(self._status(task_id), "done")
+
+    def test_decomposition_lineage_prevents_replay_even_if_status_returns(self) -> None:
+        task_id = self._route_executed_task_to_triage()
+        run_id = kanban_db.list_runs(self.connection, task_id)[-1].id
+        children = kanban_db.decompose_triage_task(
+            self.connection,
+            task_id,
+            root_assignee="default",
+            children=[{"title": "new child objective"}],
+            author="operator",
+            auto_promote=False,
+        )
+        self.assertEqual(len(children or []), 1)
+        with kanban_db.write_txn(self.connection):
+            self.connection.execute(
+                "UPDATE tasks SET status = 'ready' WHERE id = ?",
+                (task_id,),
+            )
+
+        self.assertFalse(
+            kanban_db.complete_task(
+                self.connection,
+                task_id,
+                result="pre-decomposition accepted result",
+                replay_ended_run_id=run_id,
+            )
+        )
+        self.assertEqual(self._status(task_id), "ready")
+
+    def test_older_ended_run_cannot_complete_over_a_replacement(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="replacement guard")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        old_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="first run",
+                kind="capability",
+                expected_run_id=old_run,
+            )
+        )
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        replacement_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertNotEqual(old_run, replacement_run)
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="replacement run",
+                kind="capability",
+                expected_run_id=replacement_run,
+            )
+        )
+
+        self.assertFalse(
+            kanban_db.complete_task(
+                self.connection,
+                task_id,
+                result="stale result",
+                replay_ended_run_id=old_run,
+            )
+        )
+        self.assertIn(self._status(task_id), {"blocked", "triage"})
+
+    def test_equal_result_bytes_remain_bound_to_the_completing_run(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="same result retries")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        first_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="first attempt ended",
+                expected_run_id=first_run,
+            )
+        )
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        second_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.complete_task(
+                self.connection,
+                task_id,
+                result="identical result bytes",
+                summary="second run wins",
+                expected_run_id=second_run,
+            )
+        )
+
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(task.completed_run_id, second_run)
+        self.assertNotEqual(task.completed_run_id, first_run)
+        runs = {run.id: run for run in kanban_db.list_runs(self.connection, task_id)}
+        self.assertEqual(runs[first_run].outcome, "blocked")
+        self.assertEqual(runs[second_run].outcome, "completed")
+
+    def test_exact_run_reclaim_succeeds_for_the_authoritative_run(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="recover exact run")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        run_id = kanban_db.get_task(self.connection, task_id).current_run_id
+
+        self.assertTrue(
+            kanban_db.reclaim_task(
+                self.connection,
+                task_id,
+                reason="invalid exact journal",
+                expected_run_id=run_id,
+            )
+        )
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(task.status, "ready")
+        self.assertIsNone(task.current_run_id)
+
+    def test_stale_reclaim_before_transaction_preserves_replacement_run(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="replacement before txn")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        old_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="old recovery run",
+                expected_run_id=old_run,
+            )
+        )
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        replacement = kanban_db.get_task(self.connection, task_id)
+        replacement_run = replacement.current_run_id
+        self.assertNotEqual(old_run, replacement_run)
+        signals = []
+
+        self.assertFalse(
+            kanban_db.reclaim_task(
+                self.connection,
+                task_id,
+                reason="stale journal",
+                expected_run_id=old_run,
+                signal_fn=lambda *args: signals.append(args),
+            )
+        )
+        latest = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(latest.status, "running")
+        self.assertEqual(latest.current_run_id, replacement_run)
+        self.assertEqual(latest.claim_lock, replacement.claim_lock)
+        self.assertEqual(signals, [])
+
+    def test_reclaim_update_guard_preserves_run_changed_inside_transaction(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="replacement in txn")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        old_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        replacement = {}
+
+        def install_replacement(*_args, **_kwargs):
+            cursor = self.connection.execute(
+                "INSERT INTO task_runs (task_id, status, claim_lock, started_at) "
+                "VALUES (?, 'running', ?, strftime('%s','now'))",
+                (task_id, "replacement-lock"),
+            )
+            replacement["run_id"] = int(cursor.lastrowid)
+            self.connection.execute(
+                "UPDATE tasks SET current_run_id = ?, claim_lock = ? WHERE id = ?",
+                (replacement["run_id"], "replacement-lock", task_id),
+            )
+            return {}
+
+        with mock.patch.object(
+            kanban_db,
+            "_terminate_reclaimed_worker",
+            side_effect=install_replacement,
+        ):
+            self.assertFalse(
+                kanban_db.reclaim_task(
+                    self.connection,
+                    task_id,
+                    reason="journal for old run",
+                    expected_run_id=old_run,
+                )
+            )
+
+        latest = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(latest.status, "running")
+        self.assertEqual(latest.current_run_id, replacement["run_id"])
+        self.assertEqual(latest.claim_lock, "replacement-lock")
+
+    def test_concurrent_exact_reclaim_and_completion_have_one_winner(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="concurrent finalizer")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        run_id = kanban_db.get_task(self.connection, task_id).current_run_id
+        barrier = threading.Barrier(2)
+        outcomes = {}
+        errors = []
+
+        def reclaim() -> None:
+            try:
+                with kanban_db.connect_closing() as connection:
+                    barrier.wait()
+                    outcomes["reclaim"] = kanban_db.reclaim_task(
+                        connection,
+                        task_id,
+                        reason="concurrent invalid journal",
+                        expected_run_id=run_id,
+                    )
+            except BaseException as error:  # pragma: no cover - assertion relay
+                errors.append(error)
+
+        def finalize() -> None:
+            try:
+                with kanban_db.connect_closing() as connection:
+                    barrier.wait()
+                    outcomes["complete"] = kanban_db.complete_task(
+                        connection,
+                        task_id,
+                        result="durable winner",
+                        summary="durable winner",
+                        expected_run_id=run_id,
+                    )
+            except BaseException as error:  # pragma: no cover - assertion relay
+                errors.append(error)
+
+        threads = [threading.Thread(target=reclaim), threading.Thread(target=finalize)]
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join(timeout=10)
+            self.assertFalse(thread.is_alive())
+
+        self.assertEqual(errors, [])
+        self.assertEqual(set(outcomes), {"reclaim", "complete"})
+        self.assertEqual(sum(bool(value) for value in outcomes.values()), 1)
+        latest = kanban_db.get_task(self.connection, task_id)
+        self.assertIn(latest.status, {"done", "ready"})
+        self.assertIsNone(latest.current_run_id)
+        runs = kanban_db.list_runs(self.connection, task_id)
+        self.assertEqual(len(runs), 1)
+        self.assertIsNotNone(runs[0].ended_at)

dockerfiles/patch-hermes-execution-safety.py

@@ -0,0 +1,6 @@
+"""Apply the composable Hermes execution-safety source patches."""
+
+# Import order is intentional: the decomposition DB contract builds on the
+# exact-run schema and transaction changes. Each module fails closed on drift.
+import patch_hermes_run_safety  # noqa: F401
+import patch_hermes_decomposition_safety  # noqa: F401

dockerfiles/patch_hermes_decomposition_safety.py

@@ -0,0 +1,262 @@
+"""Patch transactional automatic-decomposition execution-history guards."""
+
+import os
+from pathlib import Path
+
+from hermes_execution_patch_support import replace_once
+
+source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes"))
+db_path = source_root / "hermes_cli/kanban_db.py"
+db = db_path.read_text(encoding="utf-8")
+
+specify_signature_before = '''def specify_triage_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    title: Optional[str] = None,
+    body: Optional[str] = None,
+    assignee: Optional[str] = None,
+    author: Optional[str] = None,
+) -> bool:
+'''
+specify_signature_after = '''def specify_triage_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    title: Optional[str] = None,
+    body: Optional[str] = None,
+    assignee: Optional[str] = None,
+    author: Optional[str] = None,
+    require_no_runs: bool = False,
+) -> bool:
+'''
+db = replace_once(
+    db,
+    specify_signature_before,
+    specify_signature_after,
+    "transactional specify signature",
+)
+
+specify_guard_before = '''        if existing is None:
+            return False
+        sets: list[str] = ["status = 'todo'"]
+'''
+specify_guard_after = '''        if existing is None:
+            return False
+        if require_no_runs and conn.execute(
+            "SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1",
+            (task_id,),
+        ).fetchone() is not None:
+            return False
+        sets: list[str] = ["status = 'todo'"]
+'''
+db = replace_once(
+    db,
+    specify_guard_before,
+    specify_guard_after,
+    "transactional specify execution-history guard",
+)
+
+decompose_signature_before = '''def decompose_triage_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    root_assignee: Optional[str],
+    children: list[dict],
+    author: Optional[str] = None,
+    auto_promote: bool = True,
+) -> Optional[list[str]]:
+'''
+decompose_signature_after = '''def decompose_triage_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    root_assignee: Optional[str],
+    children: list[dict],
+    author: Optional[str] = None,
+    auto_promote: bool = True,
+    require_no_runs: bool = False,
+) -> Optional[list[str]]:
+'''
+db = replace_once(
+    db,
+    decompose_signature_before,
+    decompose_signature_after,
+    "transactional decomposition signature",
+)
+
+decompose_guard_before = '''        if root_row["status"] != "triage":
+            return None
+        tenant = root_row["tenant"]
+'''
+decompose_guard_after = '''        if root_row["status"] != "triage":
+            return None
+        if require_no_runs and conn.execute(
+            "SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1",
+            (task_id,),
+        ).fetchone() is not None:
+            return None
+        tenant = root_row["tenant"]
+'''
+db = replace_once(
+    db,
+    decompose_guard_before,
+    decompose_guard_after,
+    "transactional decomposition execution-history guard",
+)
+db_path.write_text(db, encoding="utf-8")
+
+decompose_path = source_root / "hermes_cli/kanban_decompose.py"
+decompose = decompose_path.read_text(encoding="utf-8")
+
+helper_anchor = '''    if chosen not in valid_names:
+        return default_assignee
+    return chosen
+
+
+'''
+helper_replacement = helper_anchor + '''def _has_execution_history(task_id: str) -> bool:
+    """Return whether a task has ever entered a worker run."""
+    with kb.connect_closing() as conn:
+        return bool(kb.list_runs(conn, task_id))
+
+
+'''
+decompose = replace_once(
+    decompose,
+    helper_anchor,
+    helper_replacement,
+    "auto-decompose execution-history helper",
+)
+
+signature_before = '''def decompose_task(
+    task_id: str,
+    *,
+    author: Optional[str] = None,
+    timeout: Optional[int] = None,
+) -> DecomposeOutcome:
+'''
+signature_after = '''def decompose_task(
+    task_id: str,
+    *,
+    author: Optional[str] = None,
+    timeout: Optional[int] = None,
+    automatic: bool = False,
+) -> DecomposeOutcome:
+'''
+decompose = replace_once(
+    decompose,
+    signature_before,
+    signature_after,
+    "auto-decompose function signature",
+)
+
+status_before = '''    if task.status != "triage":
+        return DecomposeOutcome(
+            task_id, False, f"task is not in triage (status={task.status!r})"
+        )
+
+    cfg = _load_config()
+'''
+status_after = '''    if task.status != "triage":
+        return DecomposeOutcome(
+            task_id, False, f"task is not in triage (status={task.status!r})"
+        )
+    if automatic and _has_execution_history(task_id):
+        return DecomposeOutcome(
+            task_id,
+            False,
+            "task has execution history and requires deliberate manual triage",
+        )
+
+    cfg = _load_config()
+'''
+decompose = replace_once(
+    decompose,
+    status_before,
+    status_after,
+    "auto-decompose preflight guard",
+)
+
+single_before = '''                author=audit_author,
+            )
+        if not ok:
+'''
+single_after = '''                author=audit_author,
+                require_no_runs=automatic,
+            )
+        if not ok:
+            if automatic and _has_execution_history(task_id):
+                return DecomposeOutcome(
+                    task_id,
+                    False,
+                    "task gained execution history and requires deliberate manual triage",
+                )
+'''
+decompose = replace_once(
+    decompose,
+    single_before,
+    single_after,
+    "auto-decompose single-task commit guard",
+)
+
+fanout_before = '''                author=audit_author,
+                auto_promote=auto_promote,
+            )
+'''
+fanout_after = '''                author=audit_author,
+                auto_promote=auto_promote,
+                require_no_runs=automatic,
+            )
+'''
+decompose = replace_once(
+    decompose,
+    fanout_before,
+    fanout_after,
+    "auto-decompose transactional fanout guard",
+)
+
+fanout_outcome_before = '''    if child_ids is None:
+        return DecomposeOutcome(
+            task_id, False, "task moved out of triage before decomposition",
+        )
+'''
+fanout_outcome_after = '''    if child_ids is None:
+        if automatic and _has_execution_history(task_id):
+            return DecomposeOutcome(
+                task_id,
+                False,
+                "task gained execution history and requires deliberate manual triage",
+            )
+        return DecomposeOutcome(
+            task_id, False, "task moved out of triage before decomposition",
+        )
+'''
+decompose = replace_once(
+    decompose,
+    fanout_outcome_before,
+    fanout_outcome_after,
+    "auto-decompose fanout rejection reason",
+)
+decompose_path.write_text(decompose, encoding="utf-8")
+
+
+watcher_path = source_root / "gateway/kanban_watchers.py"
+watcher = watcher_path.read_text(encoding="utf-8")
+call_before = '''                            outcome = _decomp.decompose_task(
+                                tid, author="auto-decomposer",
+                            )
+'''
+call_after = '''                            outcome = _decomp.decompose_task(
+                                tid,
+                                author="auto-decomposer",
+                                automatic=True,
+                            )
+'''
+watcher = replace_once(
+    watcher,
+    call_before,
+    call_after,
+    "gateway automatic-decomposition call",
+)
+watcher_path.write_text(watcher, encoding="utf-8")

dockerfiles/patch_hermes_run_safety.py

@@ -0,0 +1,480 @@
+"""Patch exact-run completion, provenance, and reclaim transactions."""
+
+import os
+from pathlib import Path
+
+from hermes_execution_patch_support import replace_once
+
+
+
+source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes"))
+db_path = source_root / "hermes_cli/kanban_db.py"
+db = db_path.read_text(encoding="utf-8")
+
+complete_signature_before = '''def complete_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    result: Optional[str] = None,
+    summary: Optional[str] = None,
+    metadata: Optional[dict] = None,
+    created_cards: Optional[Iterable[str]] = None,
+    expected_run_id: Optional[int] = None,
+) -> bool:
+'''
+complete_signature_after = '''def complete_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    result: Optional[str] = None,
+    summary: Optional[str] = None,
+    metadata: Optional[dict] = None,
+    created_cards: Optional[Iterable[str]] = None,
+    expected_run_id: Optional[int] = None,
+    replay_ended_run_id: Optional[int] = None,
+) -> bool:
+'''
+db = replace_once(
+    db,
+    complete_signature_before,
+    complete_signature_after,
+    "exact ended-run completion signature",
+)
+
+complete_branch_before = '''    with write_txn(conn):
+        if expected_run_id is None:
+            cur = conn.execute(
+                """
+                UPDATE tasks
+                   SET status       = 'done',
+                       result       = ?,
+                       completed_at = ?,
+                       claim_lock   = NULL,
+                       claim_expires= NULL,
+                       worker_pid   = NULL,
+                       block_kind   = NULL,
+                       block_recurrences = 0
+                 WHERE id = ?
+                   AND status IN ('running', 'ready', 'blocked', 'scheduled')
+                """,
+                (result, now, task_id),
+            )
+        else:
+'''
+complete_branch_after = '''    # Journal recovery may arrive after a legacy error path ended the
+    # accepted run. Verify that it is still the latest run in this same
+    # transaction; a replacement attempt must make the replay fail closed.
+    with write_txn(conn):
+        replayed_run_id = None
+        if expected_run_id is not None and replay_ended_run_id is not None:
+            raise ValueError("expected_run_id and replay_ended_run_id are mutually exclusive")
+        if replay_ended_run_id is not None:
+            replayed_run_id = int(replay_ended_run_id)
+            # Replay is permitted only while the exact run-ending transition
+            # remains the latest status mutation. Comments and diagnostics do
+            # not invalidate durable accepted work, but an unblock, explicit
+            # triage specification, decomposition, reclaim, or newer attempt
+            # does. BEGIN IMMEDIATE keeps this lineage check and the guarded
+            # task/run updates in one transaction.
+            last_transition = conn.execute(
+                """
+                SELECT kind, run_id
+                  FROM task_events
+                 WHERE task_id = ?
+                   AND kind IN (
+                       'archived', 'block_loop_detected', 'blocked', 'claimed',
+                       'completed', 'decomposed', 'dependency_wait', 'gave_up',
+                       'promoted', 'promoted_manual', 'reclaimed', 'scheduled',
+                       'specified', 'stale', 'timed_out', 'unblocked'
+                   )
+                 ORDER BY id DESC
+                 LIMIT 1
+                """,
+                (task_id,),
+            ).fetchone()
+            if (
+                last_transition is None
+                or last_transition["run_id"] != replayed_run_id
+                or last_transition["kind"] not in {
+                    "block_loop_detected",
+                    "blocked",
+                    "reclaimed",
+                    "stale",
+                    "timed_out",
+                    "gave_up",
+                }
+            ):
+                return False
+            cur = conn.execute(
+                """
+                UPDATE tasks
+                   SET status       = 'done',
+                       result       = ?,
+                       completed_at = ?,
+                       claim_lock   = NULL,
+                       claim_expires= NULL,
+                       worker_pid   = NULL,
+                       block_kind   = NULL,
+                       block_recurrences = 0
+                 WHERE id = ?
+                   AND status IN ('ready', 'blocked', 'triage')
+                   AND current_run_id IS NULL
+                   AND ? = (
+                       SELECT id FROM task_runs
+                        WHERE task_id = ?
+                        ORDER BY id DESC
+                        LIMIT 1
+                   )
+                   AND EXISTS (
+                       SELECT 1 FROM task_runs
+                        WHERE id = ?
+                          AND task_id = ?
+                          AND ended_at IS NOT NULL
+                   )
+                """,
+                (
+                    result,
+                    now,
+                    task_id,
+                    replayed_run_id,
+                    task_id,
+                    replayed_run_id,
+                    task_id,
+                ),
+            )
+        elif expected_run_id is None:
+            cur = conn.execute(
+                """
+                UPDATE tasks
+                   SET status       = 'done',
+                       result       = ?,
+                       completed_at = ?,
+                       claim_lock   = NULL,
+                       claim_expires= NULL,
+                       worker_pid   = NULL,
+                       block_kind   = NULL,
+                       block_recurrences = 0
+                 WHERE id = ?
+                   AND status IN ('running', 'ready', 'blocked', 'scheduled')
+                """,
+                (result, now, task_id),
+            )
+        else:
+'''
+db = replace_once(
+    db,
+    complete_branch_before,
+    complete_branch_after,
+    "transactional exact ended-run completion",
+)
+
+run_completion_before = '''        run_id = _end_run(
+            conn, task_id,
+            outcome="completed", status="done",
+            summary=summary if summary is not None else result,
+            metadata=metadata,
+        )
+        # If complete_task was called on a never-claimed task (ready or
+        # blocked → done with no run in flight), synthesize a
+        # zero-duration run so the handoff fields are persisted in
+        # attempt history instead of silently lost.
+        if run_id is None and (summary or metadata or result):
+            run_id = _synthesize_ended_run(
+                conn, task_id,
+                outcome="completed",
+                summary=summary if summary is not None else result,
+                metadata=metadata,
+            )
+'''
+run_completion_after = '''        if replayed_run_id is None:
+            run_id = _end_run(
+                conn, task_id,
+                outcome="completed", status="done",
+                summary=summary if summary is not None else result,
+                metadata=metadata,
+            )
+            # If complete_task was called on a never-claimed task (ready or
+            # blocked → done with no run in flight), synthesize a
+            # zero-duration run so the handoff fields are persisted in
+            # attempt history instead of silently lost.
+            if run_id is None and (summary or metadata or result):
+                run_id = _synthesize_ended_run(
+                    conn, task_id,
+                    outcome="completed",
+                    summary=summary if summary is not None else result,
+                    metadata=metadata,
+                )
+        else:
+            run_id = replayed_run_id
+            replayed = conn.execute(
+                """
+                UPDATE task_runs
+                   SET status = 'done',
+                       outcome = 'completed',
+                       summary = ?,
+                       error = NULL,
+                       metadata = ?,
+                       claim_lock = NULL,
+                       claim_expires = NULL,
+                       worker_pid = NULL
+                 WHERE id = ?
+                   AND task_id = ?
+                   AND ended_at IS NOT NULL
+                """,
+                (
+                    summary if summary is not None else result,
+                    json.dumps(metadata, ensure_ascii=False) if metadata else None,
+                    run_id,
+                    task_id,
+                ),
+            )
+            if replayed.rowcount != 1:
+                raise RuntimeError("exact ended-run completion lost its run identity")
+        conn.execute(
+            "UPDATE tasks SET completed_run_id = ? WHERE id = ?",
+            (run_id, task_id),
+        )
+'''
+db = replace_once(
+    db,
+    run_completion_before,
+    run_completion_after,
+    "exact completion run provenance",
+)
+
+reclaim_before = '''def reclaim_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    reason: Optional[str] = None,
+    signal_fn=None,
+) -> bool:
+    """Operator-driven reclaim: release the claim and reset to ``ready``.
+
+    Unlike :func:`release_stale_claims` which only acts on tasks whose
+    ``claim_expires`` has passed, this function reclaims immediately
+    regardless of TTL. Intended for the dashboard/CLI recovery flow
+    when an operator wants to abort a running worker without waiting
+    for the TTL to expire (e.g. after seeing a hallucination warning).
+
+    Returns True if a reclaim happened, False if the task isn't in a
+    reclaimable state (not running, or doesn't exist).
+    """
+    row = conn.execute(
+        "SELECT status, claim_lock, worker_pid FROM tasks WHERE id = ?",
+        (task_id,),
+    ).fetchone()
+    if not row:
+        return False
+    if row["status"] != "running" and row["claim_lock"] is None:
+        # Nothing to reclaim — already ready / blocked / done.
+        return False
+    prev_lock = row["claim_lock"]
+    termination = _terminate_reclaimed_worker(
+        row["worker_pid"], prev_lock, signal_fn=signal_fn,
+    )
+    with write_txn(conn):
+        cur = conn.execute(
+            "UPDATE tasks SET status = 'ready', claim_lock = NULL, "
+            "claim_expires = NULL, worker_pid = NULL "
+            "WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
+            "AND claim_lock IS ?",
+            (task_id, prev_lock),
+        )
+        if cur.rowcount != 1:
+            return False
+        run_id = _end_run(
+            conn, task_id,
+            outcome="reclaimed", status="reclaimed",
+            error=(
+                f"manual_reclaim: {reason}" if reason
+                else f"manual_reclaim lock={prev_lock}"
+            ),
+            metadata=termination,
+        )
+        payload = {
+            "manual": True,
+            "reason": reason,
+            "prev_lock": prev_lock,
+        }
+        payload.update(termination)
+        _append_event(
+            conn, task_id, "reclaimed",
+            payload,
+            run_id=run_id,
+        )
+    # Operator intervention — they've looked at the task, so the
+    # consecutive-failures counter is now stale. Give the next retry
+    # a fresh budget. (_clear_failure_counter opens its own write_txn,
+    # so it runs after the enclosing one commits.)
+    _clear_failure_counter(conn, task_id)
+    return True
+'''
+reclaim_after = '''def reclaim_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    reason: Optional[str] = None,
+    signal_fn=None,
+    expected_run_id: Optional[int] = None,
+) -> bool:
+    """Operator-driven reclaim: release the claim and reset to ``ready``.
+
+    Unlike :func:`release_stale_claims` which only acts on tasks whose
+    ``claim_expires`` has passed, this function reclaims immediately
+    regardless of TTL. Intended for the dashboard/CLI recovery flow
+    when an operator wants to abort a running worker without waiting
+    for the TTL to expire (e.g. after seeing a hallucination warning).
+
+    When ``expected_run_id`` is supplied, the run identity is checked after
+    ``BEGIN IMMEDIATE`` and included in the guarded update. A replacement run
+    is therefore neither signalled nor reclaimed by stale recovery evidence.
+
+    Returns True if a reclaim happened, False if the task isn't in a
+    reclaimable state (not running, or doesn't exist), or its run changed.
+    """
+    with write_txn(conn):
+        row = conn.execute(
+            "SELECT status, claim_lock, worker_pid, current_run_id "
+            "FROM tasks WHERE id = ?",
+            (task_id,),
+        ).fetchone()
+        if not row:
+            return False
+        if expected_run_id is not None and row["current_run_id"] != expected_run_id:
+            return False
+        if row["status"] != "running" and row["claim_lock"] is None:
+            # Nothing to reclaim — already ready / blocked / done.
+            return False
+        prev_lock = row["claim_lock"]
+        termination = _terminate_reclaimed_worker(
+            row["worker_pid"], prev_lock, signal_fn=signal_fn,
+        )
+        if expected_run_id is None:
+            cur = conn.execute(
+                "UPDATE tasks SET status = 'ready', claim_lock = NULL, "
+                "claim_expires = NULL, worker_pid = NULL "
+                "WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
+                "AND claim_lock IS ?",
+                (task_id, prev_lock),
+            )
+        else:
+            cur = conn.execute(
+                "UPDATE tasks SET status = 'ready', claim_lock = NULL, "
+                "claim_expires = NULL, worker_pid = NULL "
+                "WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
+                "AND claim_lock IS ? AND current_run_id = ?",
+                (task_id, prev_lock, int(expected_run_id)),
+            )
+        if cur.rowcount != 1:
+            return False
+        run_id = _end_run(
+            conn, task_id,
+            outcome="reclaimed", status="reclaimed",
+            error=(
+                f"manual_reclaim: {reason}" if reason
+                else f"manual_reclaim lock={prev_lock}"
+            ),
+            metadata=termination,
+        )
+        payload = {
+            "manual": True,
+            "reason": reason,
+            "prev_lock": prev_lock,
+        }
+        payload.update(termination)
+        _append_event(
+            conn, task_id, "reclaimed",
+            payload,
+            run_id=run_id,
+        )
+    # Operator intervention — they've looked at the task, so the
+    # consecutive-failures counter is now stale. Give the next retry
+    # a fresh budget. (_clear_failure_counter opens its own write_txn,
+    # so it runs after the enclosing one commits.)
+    _clear_failure_counter(conn, task_id)
+    return True
+'''
+db = replace_once(
+    db,
+    reclaim_before,
+    reclaim_after,
+    "transactional exact-run reclaim",
+)
+
+task_field_before = '''    current_run_id: Optional[int] = None
+    workflow_template_id: Optional[str] = None
+'''
+task_field_after = '''    current_run_id: Optional[int] = None
+    # Durable provenance for the run that authoritatively completed this task.
+    completed_run_id: Optional[int] = None
+    workflow_template_id: Optional[str] = None
+'''
+db = replace_once(
+    db,
+    task_field_before,
+    task_field_after,
+    "completed-run task field",
+)
+
+task_row_before = '''            current_run_id=(
+                row["current_run_id"] if "current_run_id" in keys else None
+            ),
+            workflow_template_id=(
+'''
+task_row_after = '''            current_run_id=(
+                row["current_run_id"] if "current_run_id" in keys else None
+            ),
+            completed_run_id=(
+                row["completed_run_id"] if "completed_run_id" in keys else None
+            ),
+            workflow_template_id=(
+'''
+db = replace_once(
+    db,
+    task_row_before,
+    task_row_after,
+    "completed-run row mapping",
+)
+
+task_schema_before = '''    -- run is in-flight). Denormalised for cheap reads.
+    current_run_id       INTEGER,
+    -- Forward-compat for v2 workflow routing. In v1 the kernel writes
+'''
+task_schema_after = '''    -- run is in-flight). Denormalised for cheap reads.
+    current_run_id       INTEGER,
+    -- Immutable provenance for the task result's authoritative run.
+    completed_run_id     INTEGER,
+    -- Forward-compat for v2 workflow routing. In v1 the kernel writes
+'''
+db = replace_once(
+    db,
+    task_schema_before,
+    task_schema_after,
+    "completed-run schema",
+)
+
+task_migration_before = '''    if "current_run_id" not in cols:
+        _add_column_if_missing(
+            conn, "tasks", "current_run_id", "current_run_id INTEGER"
+        )
+    if "workflow_template_id" not in cols:
+'''
+task_migration_after = '''    if "current_run_id" not in cols:
+        _add_column_if_missing(
+            conn, "tasks", "current_run_id", "current_run_id INTEGER"
+        )
+    if "completed_run_id" not in cols:
+        _add_column_if_missing(
+            conn, "tasks", "completed_run_id", "completed_run_id INTEGER"
+        )
+    if "workflow_template_id" not in cols:
+'''
+db = replace_once(
+    db,
+    task_migration_before,
+    task_migration_after,
+    "completed-run migration",
+)
+
+db_path.write_text(db, encoding="utf-8")

scripts/tests/test_dashboard_render_support.py

@@ -0,0 +1,25 @@
+import importlib.util
+
+import pathlib
+
+def load_module():
+    path = pathlib.Path(__file__).resolve().parents[1] / "render" / "dashboards_render_atlas.py"
+    spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+def flatten_panels(panels):
+    flat = []
+    for panel in panels:
+        flat.append(panel)
+        flat.extend(panel.get("panels", []))
+    return flat
+
+__all__ = (
+    "flatten_panels",
+    "importlib",
+    "load_module",
+    "pathlib",
+)

scripts/tests/test_dashboards_render_atlas_drilldowns.py

@@ -0,0 +1,127 @@
+"""Detailed Atlas Jobs dashboard collapse and drilldown contracts."""
+
+from scripts.tests.test_dashboard_render_support import load_module
+
+
+def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
+    mod = load_module()
+    dashboard = mod.build_jobs_dashboard()
+    panels = dashboard["panels"]
+    rows = [panel for panel in panels if panel["type"] == "row"]
+    visible_query_panels = [panel for panel in panels if panel["type"] != "row"]
+    nested_panels_by_title = {
+        child["title"]: child for row in rows for child in row.get("panels", [])
+    }
+
+    assert len(panels) == 18
+    assert len(visible_query_panels) == 12
+    assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12
+    assert all(
+        panel["title"] != "Coverage Gap to 95% by Suite"
+        for panel in visible_query_panels
+    )
+    assert [row["title"] for row in rows] == [
+        "CI Runs And Test Result History",
+        "Check Failure Rates By Suite",
+        "Check Healthy Rates By Suite",
+        "Test Drilldowns And Problem Tests",
+        "Telemetry Completeness And Branches",
+        "SonarQube Project Health",
+    ]
+    assert all(row["collapsed"] for row in rows)
+
+    assert "Coverage Failure Rate" in nested_panels_by_title
+    assert "Supply Chain Healthy Rate" in nested_panels_by_title
+    assert "Test Category Health History" in nested_panels_by_title
+    assert "Selected Test Pass Rate History" in nested_panels_by_title
+    assert "Coverage Metrics Present by Suite" in nested_panels_by_title
+    assert "SonarQube API Up" in nested_panels_by_title
+
+    failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"]
+    assert failure_rate_panel["type"] == "state-timeline"
+    assert failure_rate_panel["fieldConfig"]["defaults"]["unit"] == "percent"
+    assert failure_rate_panel["fieldConfig"]["defaults"]["max"] == 100
+    assert (
+        failure_rate_panel["fieldConfig"]["defaults"]["thresholds"]["steps"][0]["color"]
+        == "dark-blue"
+    )
+    assert "increase(" not in failure_rate_panel["targets"][0]["expr"]
+    assert (
+        "platform_quality:check_failed_flag:present_1h"
+        in failure_rate_panel["targets"][0]["expr"]
+    )
+    assert (
+        "platform_quality:check_seen_flag:present_1h"
+        in failure_rate_panel["targets"][0]["expr"]
+    )
+    assert (
+        "platform_quality:check_status:present_1h"
+        not in failure_rate_panel["targets"][0]["expr"]
+    )
+    assert (
+        ".*_quality_gate_checks_total" not in failure_rate_panel["targets"][0]["expr"]
+    )
+    assert "last_over_time" not in failure_rate_panel["targets"][0]["expr"]
+    assert "label_replace" not in failure_rate_panel["targets"][0]["expr"]
+    assert "0 *" in failure_rate_panel["targets"][0]["expr"]
+    assert "and on(suite)" not in failure_rate_panel["targets"][0]["expr"]
+
+    pass_rate_panel = nested_panels_by_title["Selected Test Pass Rate History"]
+    assert pass_rate_panel["type"] == "state-timeline"
+    assert (
+        "platform_quality:test_case_pass_rate:percent_1h"
+        in pass_rate_panel["targets"][0]["expr"]
+    )
+    assert (
+        "platform_quality_gate_test_case_result"
+        not in pass_rate_panel["targets"][0]["expr"]
+    )
+
+    pass_fail_panel = nested_panels_by_title["Selected Test Pass/Fail History"]
+    assert pass_fail_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
+    assert all(
+        "platform_quality:test_case_status:count_1h" in target["expr"]
+        for target in pass_fail_panel["targets"]
+    )
+
+    problematic_panel = nested_panels_by_title[
+        "Problematic Tests Over Time (Top failures)"
+    ]
+    assert problematic_panel["type"] == "state-timeline"
+    assert problematic_panel["gridPos"]["w"] == 24
+    assert 'test!=""' in problematic_panel["targets"][0]["expr"]
+    assert "vector(0)" not in problematic_panel["targets"][0]["expr"]
+    assert problematic_panel["fieldConfig"]["defaults"]["thresholds"]["steps"] == [
+        {"color": "dark-blue", "value": None},
+        {"color": "dark-green", "value": 2},
+        {"color": "dark-yellow", "value": 3},
+        {"color": "dark-orange", "value": 5},
+        {"color": "dark-red", "value": 8},
+    ]
+    assert "rolling 24h failure count" in problematic_panel["description"]
+    assert "at least two recent failures" in problematic_panel["description"]
+
+    sonar_mix_panel = nested_panels_by_title["Sonar Gate Status Mix (Selected)"]
+    sonar_health_panel = nested_panels_by_title["Sonar Gate Health by Project"]
+    assert sonar_mix_panel["gridPos"]["w"] == 4
+    assert sonar_health_panel["gridPos"]["w"] == 8
+    assert sonar_health_panel["type"] == "state-timeline"
+    assert (
+        "platform_quality:sonar_gate_health_percent:latest_1h"
+        in sonar_health_panel["targets"][0]["expr"]
+    )
+    assert (
+        "sonarqube_project_quality_gate_pass"
+        not in sonar_health_panel["targets"][0]["expr"]
+    )
+
+    branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"]
+    recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"]
+    assert branch_panel["gridPos"]["x"] == 12
+    assert recent_branch_panel["gridPos"]["x"] == 18
+    assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"]
+    assert "[7d:1h]" in branch_panel["targets"][0]["expr"]
+    assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent"
+    assert "unless on(suite)" in branch_panel["targets"][0]["expr"]
+    assert "> bool 0" in branch_panel["targets"][0]["expr"]
+    assert branch_panel["targets"][0]["expr"].startswith("sort(")

scripts/tests/test_dashboards_render_jobs.py

@@ -1,241 +1,11 @@
-import importlib.util
-import pathlib
+"""Atlas testing and jobs dashboard rendering contracts."""
 
+from __future__ import annotations
 
-def load_module():
-    path = pathlib.Path(__file__).resolve().parents[1] / "render" / "dashboards_render_atlas.py"
-    spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path)
-    module = importlib.util.module_from_spec(spec)
-    assert spec.loader is not None
-    spec.loader.exec_module(module)
-    return module
-
-
-def flatten_panels(panels):
-    flat = []
-    for panel in panels:
-        flat.append(panel)
-        flat.extend(panel.get("panels", []))
-    return flat
-
-
-def test_table_panel_options_and_filterable():
-    mod = load_module()
-    panel = mod.table_panel(
-        1,
-        "test",
-        "metric",
-        {"h": 1, "w": 1, "x": 0, "y": 0},
-        unit="percent",
-        transformations=[{"id": "labelsToFields", "options": {}}],
-        instant=True,
-        options={"showColumnFilters": False},
-        filterable=False,
-        footer={"show": False, "fields": "", "calcs": []},
-        format="table",
-    )
-    assert panel["fieldConfig"]["defaults"]["unit"] == "percent"
-    assert panel["fieldConfig"]["defaults"]["custom"]["filterable"] is False
-    assert panel["options"]["showHeader"] is True
-    assert panel["targets"][0]["format"] == "table"
-
-
-def test_node_filter_and_expr_helpers():
-    mod = load_module()
-    expr = mod.node_filter("titan-.*")
-    assert "label_replace" in expr
-    cpu_expr = mod.node_cpu_expr("titan-.*")
-    mem_expr = mod.node_mem_expr("titan-.*")
-    assert "node_cpu_seconds_total" in cpu_expr
-    assert "clamp_max(clamp_min(" in cpu_expr
-    assert "* 100, 0), 100)" in cpu_expr
-    assert "node_memory_MemAvailable_bytes" in mem_expr
-
-
-def test_overview_availability_panel_uses_recorded_365d_rollup():
-    mod = load_module()
-    dashboard = mod.build_overview()
-    panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27)
-
-    assert panel["title"] == "Atlas Availability (365d)"
-    availability_expr = panel["targets"][0]["expr"]
-    assert (
-        'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])'
-        in availability_expr
-    )
-    assert 'code=~"5.."' in availability_expr
-    assert 'code=~"[1-5].."' in availability_expr
-    assert "atlas:availability:failures_1d" not in availability_expr
-    assert "atlas:availability:requests_1d" not in availability_expr
-    assert "sum_over_time" not in availability_expr
-    assert "kube_node_status_condition" not in availability_expr
-    assert "kube_deployment_status_replicas_available" not in availability_expr
-    assert panel["targets"][0]["instant"] is True
-    assert "Every server-side 5xx" in panel["description"]
-    assert "Replica counts, Grafana health" in panel["description"]
-    assert "daily rollup job publishes one annual sample" in panel["description"]
-    assert "keeps it for up to 48 hours" in panel["description"]
-
-
-def test_overview_uses_readable_quality_power_and_gitops_panels():
-    mod = load_module()
-    dashboard = mod.build_overview()
-    panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
-
-    assert dashboard["links"] == [
-        {"title": "Atlas Testing", "url": "/d/atlas-testing", "targetBlank": True}
-    ]
-    assert "atlas-jobs" not in repr(dashboard)
-    assert "Platform Test Success Rate" not in panels_by_title
-    assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
-    assert panels_by_title["Test Category Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 13}
-    assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
-    assert panels_by_title["Test Category Health"]["targets"][0]["format"] == "time_series"
-    assert panels_by_title["Test Category Health"]["targets"][0]["range"] is True
-    assert "${overview_suite:regex}" not in panels_by_title["Test Category Health"]["targets"][0]["expr"]
-    assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in panels_by_title["Test Category Health"]["targets"][0]["expr"]
-    assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Test Category Health"]["targets"][0]["expr"]
-    assert panels_by_title["Test Category Health"]["timeFrom"] == "24h"
-    assert f'category=~"{mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
-    assert "manual" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
-    assert "unit" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
-    assert panels_by_title["UPS History (Power Draw)"]["gridPos"] == {"h": 6, "w": 6, "x": 3, "y": 7}
-    assert panels_by_title["Ariadne Run Volume"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 7}
-    assert panels_by_title["Pyrphoros UPS Current"]["gridPos"]["w"] == 3
-    assert panels_by_title["Current Enclosure Climate"]["gridPos"]["w"] == 3
-    assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["placement"] == "bottom"
-    assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["displayMode"] == "list"
-    assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "line"
-    assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["fillOpacity"] == 22
-    assert all(target["expr"].startswith("max(") for target in panels_by_title["UPS History (Power Draw)"]["targets"])
-    ups_overrides = panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["overrides"]
-    ups_override_by_name = {override["matcher"]["options"]: override for override in ups_overrides}
-    assert ups_override_by_name["Pyrphoros"]["properties"] == [
-        {"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-blue"}},
-    ]
-    assert ups_override_by_name["Statera"]["properties"] == [
-        {"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-yellow"}},
-    ]
-    assert panels_by_title["Ariadne Run Volume"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
-    assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["placement"] == "bottom"
-    assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["displayMode"] == "list"
-    assert "Fan History (0-10)" not in panels_by_title
-    assert panels_by_title["Fan Intensity History"]["type"] == "state-timeline"
-    assert panels_by_title["Fan Intensity History"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 13}
-    assert panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["max"] == 10
-    assert panels_by_title["Fan Intensity History"]["targets"][0]["legendFormat"] == "{{fan}}"
-    fan_steps = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["thresholds"]["steps"]
-    assert [step["value"] for step in fan_steps] == [None, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-    assert fan_steps[0]["color"] == "#8f1d1d"
-    assert fan_steps[5]["color"] == "#d4b106"
-    assert fan_steps[-1]["color"] == "#1f60c4"
-    fan_mappings = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["mappings"][0]["options"]
-    assert fan_mappings["0"]["text"] == "Off"
-    assert fan_mappings["10"]["text"] == "10"
-    assert "Tent Interior" in panels_by_title["Fan Intensity History"]["targets"][0]["expr"]
-    assert panels_by_title["Fan Intensity History"]["options"]["legend"]["displayMode"] == "list"
-    assert panels_by_title["Fan Intensity History"]["options"]["legend"]["showLegend"] is False
-    assert panels_by_title["Fan Intensity History"]["options"]["mergeValues"] is False
-    assert panels_by_title["Fan Intensity History"]["options"]["showValue"] == "auto"
-
-    assert panels_by_title["Flux Source"]["type"] == "stat"
-    assert panels_by_title["Flux Source"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 7}
-    assert panels_by_title["Flux Source"]["targets"][0]["legendFormat"] == "{{branch}}"
-    assert panels_by_title["Current Gate Health"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 9}
-    assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Current Gate Health"]["targets"][0]["expr"]
-    assert panels_by_title["CI Run Success (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 11}
-    assert panels_by_title["Suites With Runs (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 15}
-    suites_reporting_expr = panels_by_title["Suites With Runs (24h)"]["targets"][0]["expr"]
-    assert "> bool 0" in suites_reporting_expr
-    assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in suites_reporting_expr
-    assert "bstein-home" not in suites_reporting_expr
-    assert "published quality-gate run" in panels_by_title["Suites With Runs (24h)"]["description"]
-    assert panels_by_title["Avg Coverage"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 17}
-    assert "LOC Clean Suites" not in panels_by_title
-    assert panels_by_title["GitOps Health"]["type"] == "state-timeline"
-    assert panels_by_title["GitOps Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 7}
-    gitops_expr = panels_by_title["GitOps Health"]["targets"][0]["expr"]
-    assert "Kustomizations Not Suspended" in gitops_expr
-    assert "HelmReleases Not Suspended" in gitops_expr
-    assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
-    assert panels_by_title["Test Category Health"]["options"]["legend"]["showLegend"] is False
-    assert panels_by_title["Test Category Health"]["options"]["mergeValues"] is False
-    assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto"
-    assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9
-    assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
-    assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
-    assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"])
-
-    pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"]
-    assert "backup-telemetry-missing" in pvc_backup_expr
-    assert 'pvc_backup_(count|last_success_timestamp_seconds|health_reason)' in pvc_backup_expr
-
-    gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
-    assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
-    assert "nvidia_gpu_device_utilization_percent" in gpu_expr
-    assert "jetson_gr3d_active_seconds_total" in gpu_expr
-    assert "increase" in gpu_expr
-    assert "sum_over_time" not in gpu_expr
-    assert "count_over_time" not in gpu_expr
-    assert "avg_over_time" in gpu_expr
-    assert "$__range" in gpu_expr
-    assert "$__range_s" in gpu_expr
-    assert "sum by (namespace)" in gpu_expr
-    assert 'namespace", "shared"' not in gpu_expr
-    assert "kube_pod_container_resource_requests" in gpu_expr
-    assert mod.GPU_RESOURCE_REGEX in gpu_expr
-    assert '"gpu_source", "nvidia"' in gpu_expr
-    assert '"gpu_source", "jetson"' in gpu_expr
-    assert "100 *" in gpu_expr
-    assert "100 -" not in gpu_expr
-    assert 'namespace", "unattributed"' in gpu_expr
-    assert 'namespace", "idle"' in gpu_expr
-    assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
-    assert "selected time range" in panels_by_title["Namespace GPU Utilization"]["description"]
-    assert "continuously sampled" in panels_by_title["Namespace GPU Utilization"]["description"]
-    assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
-
-
-def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
-    mod = load_module()
-    dashboard = mod.build_gpu_dashboard()
-    panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
-
-    node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"]
-    assert "nvidia_gpu_device_utilization_percent" in node_expr
-    assert "DCGM_FI_DEV_GPU_UTIL" in node_expr
-    assert "jetson_gr3d_freq_percent" in node_expr
-    assert "unless on(node)" in node_expr
-    assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
-
-    process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"]
-    assert "nvidia_process_gpu_sm_util_percent" in process_expr
-    assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
-
-
-def test_overview_and_testing_panels_all_have_concise_descriptions():
-    mod = load_module()
-
-    for dashboard in [mod.build_overview(), mod.build_jobs_dashboard(), mod.build_testing_dashboard()]:
-        panels = flatten_panels(dashboard["panels"])
-        assert all(panel.get("description") for panel in panels if panel["type"] != "row")
-        assert "85.7" not in repr(dashboard)
-
-
-def test_render_configmap_writes(tmp_path):
-    mod = load_module()
-    mod.DASHBOARD_DIR = tmp_path / "dash"
-    mod.ROOT = tmp_path
-    uid = "atlas-test"
-    info = {"configmap": tmp_path / "cm.yaml"}
-    data = {"title": "Atlas Test"}
-    mod.write_json(uid, data)
-    mod.render_configmap(uid, info)
-    json_path = mod.DASHBOARD_DIR / f"{uid}.json"
-    assert json_path.exists()
-    content = (tmp_path / "cm.yaml").read_text()
-    assert "kind: ConfigMap" in content
-    assert f"{uid}.json" in content
+from test_dashboard_render_support import (
+    flatten_panels,
+    load_module,
+)
 
 
 def test_testing_suite_variable_uses_configured_suite_values_only():
@@ -257,7 +27,6 @@ def test_testing_suite_variable_uses_configured_suite_values_only():
         option["value"] in out_of_scope_names for option in variable["options"]
     )
 
-
 def test_testing_dashboard_is_public_but_jobs_dashboard_remains_internal():
     mod = load_module()
     jobs = mod.build_jobs_dashboard()
@@ -269,7 +38,6 @@ def test_testing_dashboard_is_public_but_jobs_dashboard_remains_internal():
     assert testing["folderUid"] == mod.PUBLIC_DASHBOARD_FOLDER
     assert testing["editable"] is False
 
-
 def test_jobs_dashboard_separates_current_gate_health_from_reliability():
     mod = load_module()
     dashboard = mod.build_jobs_dashboard()
@@ -395,7 +163,6 @@ def test_jobs_dashboard_separates_current_gate_health_from_reliability():
     assert run_volume_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
     assert "[$__interval]" not in run_volume_panel["targets"][0]["expr"]
 
-
 def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors():
     mod = load_module()
     dashboard = mod.build_jobs_dashboard()
@@ -416,8 +183,6 @@ def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors():
 
     assert {"color": "dark-yellow", "value": 93} in threshold_steps
     assert {"color": "dark-blue", "value": 100} in threshold_steps
-
-
 def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
     mod = load_module()
     dashboard = mod.build_jobs_dashboard()

scripts/tests/test_dashboards_render_overview.py

@@ -0,0 +1,221 @@
+"""Atlas overview dashboard rendering contracts."""
+
+from __future__ import annotations
+
+from test_dashboard_render_support import (
+    flatten_panels,
+    load_module,
+)
+
+
+def test_table_panel_options_and_filterable():
+    mod = load_module()
+    panel = mod.table_panel(
+        1,
+        "test",
+        "metric",
+        {"h": 1, "w": 1, "x": 0, "y": 0},
+        unit="percent",
+        transformations=[{"id": "labelsToFields", "options": {}}],
+        instant=True,
+        options={"showColumnFilters": False},
+        filterable=False,
+        footer={"show": False, "fields": "", "calcs": []},
+        format="table",
+    )
+    assert panel["fieldConfig"]["defaults"]["unit"] == "percent"
+    assert panel["fieldConfig"]["defaults"]["custom"]["filterable"] is False
+    assert panel["options"]["showHeader"] is True
+    assert panel["targets"][0]["format"] == "table"
+
+def test_node_filter_and_expr_helpers():
+    mod = load_module()
+    expr = mod.node_filter("titan-.*")
+    assert "label_replace" in expr
+    cpu_expr = mod.node_cpu_expr("titan-.*")
+    mem_expr = mod.node_mem_expr("titan-.*")
+    assert "node_cpu_seconds_total" in cpu_expr
+    assert "clamp_max(clamp_min(" in cpu_expr
+    assert "* 100, 0), 100)" in cpu_expr
+    assert "node_memory_MemAvailable_bytes" in mem_expr
+
+def test_overview_availability_panel_uses_recorded_365d_rollup():
+    mod = load_module()
+    dashboard = mod.build_overview()
+    panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27)
+
+    assert panel["title"] == "Atlas Availability (365d)"
+    availability_expr = panel["targets"][0]["expr"]
+    assert (
+        'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])'
+        in availability_expr
+    )
+    assert 'code=~"5.."' in availability_expr
+    assert 'code=~"[1-5].."' in availability_expr
+    assert "atlas:availability:failures_1d" not in availability_expr
+    assert "atlas:availability:requests_1d" not in availability_expr
+    assert "sum_over_time" not in availability_expr
+    assert "kube_node_status_condition" not in availability_expr
+    assert "kube_deployment_status_replicas_available" not in availability_expr
+    assert panel["targets"][0]["instant"] is True
+    assert "Every server-side 5xx" in panel["description"]
+    assert "Replica counts, Grafana health" in panel["description"]
+    assert "daily rollup job publishes one annual sample" in panel["description"]
+    assert "keeps it for up to 48 hours" in panel["description"]
+
+def test_overview_uses_readable_quality_power_and_gitops_panels():
+    mod = load_module()
+    dashboard = mod.build_overview()
+    panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
+
+    assert dashboard["links"] == [
+        {"title": "Atlas Testing", "url": "/d/atlas-testing", "targetBlank": True}
+    ]
+    assert "atlas-jobs" not in repr(dashboard)
+    assert "Platform Test Success Rate" not in panels_by_title
+    assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
+    assert panels_by_title["Test Category Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 13}
+    assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
+    assert panels_by_title["Test Category Health"]["targets"][0]["format"] == "time_series"
+    assert panels_by_title["Test Category Health"]["targets"][0]["range"] is True
+    assert "${overview_suite:regex}" not in panels_by_title["Test Category Health"]["targets"][0]["expr"]
+    assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in panels_by_title["Test Category Health"]["targets"][0]["expr"]
+    assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Test Category Health"]["targets"][0]["expr"]
+    assert panels_by_title["Test Category Health"]["timeFrom"] == "24h"
+    assert f'category=~"{mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
+    assert "manual" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
+    assert "unit" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
+    assert panels_by_title["UPS History (Power Draw)"]["gridPos"] == {"h": 6, "w": 6, "x": 3, "y": 7}
+    assert panels_by_title["Ariadne Run Volume"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 7}
+    assert panels_by_title["Pyrphoros UPS Current"]["gridPos"]["w"] == 3
+    assert panels_by_title["Current Enclosure Climate"]["gridPos"]["w"] == 3
+    assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["placement"] == "bottom"
+    assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["displayMode"] == "list"
+    assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "line"
+    assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["fillOpacity"] == 22
+    assert all(target["expr"].startswith("max(") for target in panels_by_title["UPS History (Power Draw)"]["targets"])
+    ups_overrides = panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["overrides"]
+    ups_override_by_name = {override["matcher"]["options"]: override for override in ups_overrides}
+    assert ups_override_by_name["Pyrphoros"]["properties"] == [
+        {"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-blue"}},
+    ]
+    assert ups_override_by_name["Statera"]["properties"] == [
+        {"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-yellow"}},
+    ]
+    assert panels_by_title["Ariadne Run Volume"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
+    assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["placement"] == "bottom"
+    assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["displayMode"] == "list"
+    assert "Fan History (0-10)" not in panels_by_title
+    assert panels_by_title["Fan Intensity History"]["type"] == "state-timeline"
+    assert panels_by_title["Fan Intensity History"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 13}
+    assert panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["max"] == 10
+    assert panels_by_title["Fan Intensity History"]["targets"][0]["legendFormat"] == "{{fan}}"
+    fan_steps = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["thresholds"]["steps"]
+    assert [step["value"] for step in fan_steps] == [None, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+    assert fan_steps[0]["color"] == "#8f1d1d"
+    assert fan_steps[5]["color"] == "#d4b106"
+    assert fan_steps[-1]["color"] == "#1f60c4"
+    fan_mappings = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["mappings"][0]["options"]
+    assert fan_mappings["0"]["text"] == "Off"
+    assert fan_mappings["10"]["text"] == "10"
+    assert "Tent Interior" in panels_by_title["Fan Intensity History"]["targets"][0]["expr"]
+    assert panels_by_title["Fan Intensity History"]["options"]["legend"]["displayMode"] == "list"
+    assert panels_by_title["Fan Intensity History"]["options"]["legend"]["showLegend"] is False
+    assert panels_by_title["Fan Intensity History"]["options"]["mergeValues"] is False
+    assert panels_by_title["Fan Intensity History"]["options"]["showValue"] == "auto"
+
+    assert panels_by_title["Flux Source"]["type"] == "stat"
+    assert panels_by_title["Flux Source"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 7}
+    assert panels_by_title["Flux Source"]["targets"][0]["legendFormat"] == "{{branch}}"
+    assert panels_by_title["Current Gate Health"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 9}
+    assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Current Gate Health"]["targets"][0]["expr"]
+    assert panels_by_title["CI Run Success (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 11}
+    assert panels_by_title["Suites With Runs (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 15}
+    suites_reporting_expr = panels_by_title["Suites With Runs (24h)"]["targets"][0]["expr"]
+    assert "> bool 0" in suites_reporting_expr
+    assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in suites_reporting_expr
+    assert "bstein-home" not in suites_reporting_expr
+    assert "published quality-gate run" in panels_by_title["Suites With Runs (24h)"]["description"]
+    assert panels_by_title["Avg Coverage"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 17}
+    assert "LOC Clean Suites" not in panels_by_title
+    assert panels_by_title["GitOps Health"]["type"] == "state-timeline"
+    assert panels_by_title["GitOps Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 7}
+    gitops_expr = panels_by_title["GitOps Health"]["targets"][0]["expr"]
+    assert "Kustomizations Not Suspended" in gitops_expr
+    assert "HelmReleases Not Suspended" in gitops_expr
+    assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
+    assert panels_by_title["Test Category Health"]["options"]["legend"]["showLegend"] is False
+    assert panels_by_title["Test Category Health"]["options"]["mergeValues"] is False
+    assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto"
+    assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9
+    assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
+    assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
+    assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"])
+
+    pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"]
+    assert "backup-telemetry-missing" in pvc_backup_expr
+    assert 'pvc_backup_(count|last_success_timestamp_seconds|health_reason)' in pvc_backup_expr
+
+    gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
+    assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
+    assert "nvidia_gpu_device_utilization_percent" in gpu_expr
+    assert "jetson_gr3d_active_seconds_total" in gpu_expr
+    assert "increase" in gpu_expr
+    assert "sum_over_time" not in gpu_expr
+    assert "count_over_time" not in gpu_expr
+    assert "avg_over_time" in gpu_expr
+    assert "$__range" in gpu_expr
+    assert "$__range_s" in gpu_expr
+    assert "sum by (namespace)" in gpu_expr
+    assert 'namespace", "shared"' not in gpu_expr
+    assert "kube_pod_container_resource_requests" in gpu_expr
+    assert mod.GPU_RESOURCE_REGEX in gpu_expr
+    assert '"gpu_source", "nvidia"' in gpu_expr
+    assert '"gpu_source", "jetson"' in gpu_expr
+    assert "100 *" in gpu_expr
+    assert "100 -" not in gpu_expr
+    assert 'namespace", "unattributed"' in gpu_expr
+    assert 'namespace", "idle"' in gpu_expr
+    assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
+    assert "selected time range" in panels_by_title["Namespace GPU Utilization"]["description"]
+    assert "continuously sampled" in panels_by_title["Namespace GPU Utilization"]["description"]
+    assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
+
+def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
+    mod = load_module()
+    dashboard = mod.build_gpu_dashboard()
+    panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
+
+    node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"]
+    assert "nvidia_gpu_device_utilization_percent" in node_expr
+    assert "DCGM_FI_DEV_GPU_UTIL" in node_expr
+    assert "jetson_gr3d_freq_percent" in node_expr
+    assert "unless on(node)" in node_expr
+    assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
+
+    process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"]
+    assert "nvidia_process_gpu_sm_util_percent" in process_expr
+    assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
+
+def test_overview_and_testing_panels_all_have_concise_descriptions():
+    mod = load_module()
+
+    for dashboard in [mod.build_overview(), mod.build_jobs_dashboard(), mod.build_testing_dashboard()]:
+        panels = flatten_panels(dashboard["panels"])
+        assert all(panel.get("description") for panel in panels if panel["type"] != "row")
+        assert "85.7" not in repr(dashboard)
+
+def test_render_configmap_writes(tmp_path):
+    mod = load_module()
+    mod.DASHBOARD_DIR = tmp_path / "dash"
+    mod.ROOT = tmp_path
+    uid = "atlas-test"
+    info = {"configmap": tmp_path / "cm.yaml"}
+    data = {"title": "Atlas Test"}
+    mod.write_json(uid, data)
+    mod.render_configmap(uid, info)
+    json_path = mod.DASHBOARD_DIR / f"{uid}.json"
+    assert json_path.exists()
+    content = (tmp_path / "cm.yaml").read_text()
+    assert "kind: ConfigMap" in content
+    assert f"{uid}.json" in content

scripts/tests/test_mailu_sync.py

@@ -232,6 +232,24 @@ def test_retry_db_connect_reraises_final_error(monkeypatch):
         sync.retry_db_connect(attempts=1)
 
 
+def test_retry_helpers_return_none_without_attempts(monkeypatch):
+    sync = load_sync_module(monkeypatch)
+
+    assert sync.retry_request("request", lambda: "never", attempts=0) is None
+    assert sync.retry_db_connect(attempts=0) is None
+
+
+def test_kc_update_attributes_replaces_non_dict_attributes(monkeypatch):
+    sync = load_sync_module(monkeypatch)
+    current_resp = _FakeResponse({"attributes": "not-a-dict"})
+    ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}})
+    sync.SESSION = _FakeSession(_FakeResponse({}), [current_resp, ok_resp])
+
+    sync.kc_update_attributes("token", {"id": "u1", "username": "u1"}, {"mailu_app_password": "abc"})
+
+    assert sync.SESSION.put_called
+
+
 def test_ensure_mailu_user_skips_foreign_domain(monkeypatch):
     sync = load_sync_module(monkeypatch)
     executed = []
@@ -405,3 +423,62 @@ def test_main_generates_password_and_upserts(monkeypatch):
     # Only mail-enabled users (or legacy users with a mailbox) are synced and backfilled.
     assert len(updated) == 3
     assert conns and len(conns[0]._cursor.executions) == 3
+
+
+def test_main_skips_disabled_users_and_provisioned_users_without_update(monkeypatch):
+    sync = load_sync_module(monkeypatch)
+    monkeypatch.setattr(sync.bcrypt_sha256, "hash", lambda password: f"hash:{password}")
+    users = [
+        {
+            "id": "u1",
+            "username": "disabled",
+            "email": "disabled@example.com",
+            "enabled": False,
+            "attributes": {"mailu_enabled": ["true"]},
+        },
+        {
+            "id": "u2",
+            "username": "settled",
+            "email": "settled@example.com",
+            "attributes": {
+                "mailu_enabled": ["true"],
+                "mailu_email": ["settled@example.com"],
+                "mailu_app_password": ["existing"],
+            },
+        },
+    ]
+    updated = []
+
+    class _Cursor:
+        def __init__(self):
+            self.executions = []
+
+        def execute(self, sql, params):
+            self.executions.append(params)
+
+        def close(self):
+            return None
+
+    class _Conn:
+        def __init__(self):
+            self.autocommit = False
+            self._cursor = _Cursor()
+
+        def cursor(self, cursor_factory=None):
+            return self._cursor
+
+        def close(self):
+            return None
+
+    conn = _Conn()
+    monkeypatch.setattr(sync, "get_kc_token", lambda: "tok")
+    monkeypatch.setattr(sync, "kc_get_users", lambda token: users)
+    monkeypatch.setattr(sync, "kc_update_attributes", lambda token, user, attrs: updated.append(user["id"]))
+    monkeypatch.setattr(sync.psycopg2, "connect", lambda **kwargs: conn)
+
+    sync.main()
+
+    # The already-provisioned user is synced without a Keycloak write; the
+    # disabled user never reaches the mailbox upsert.
+    assert updated == []
+    assert len(conn._cursor.executions) == 1

scripts/tests/test_mailu_sync_edges.py

@@ -0,0 +1,96 @@
+"""Retry, attribute-corruption, and skip-path edges for the Mailu sync job."""
+
+from test_mailu_sync import _FakeResponse, _FakeSession, load_sync_module
+
+def test_retry_helpers_with_zero_attempts_do_nothing(monkeypatch):
+    sync = load_sync_module(monkeypatch)
+    monkeypatch.setattr(
+        sync.psycopg2,
+        "connect",
+        lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not connect")),
+    )
+
+    assert sync.retry_request(
+        "request",
+        lambda: (_ for _ in ()).throw(AssertionError("must not run")),
+        attempts=0,
+    ) is None
+    assert sync.retry_db_connect(attempts=0) is None
+
+
+def test_kc_update_attributes_replaces_a_corrupt_attribute_map(monkeypatch):
+    sync = load_sync_module(monkeypatch)
+    current_resp = _FakeResponse({"username": "u1", "attributes": "corrupt"})
+    ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}})
+    sync.SESSION = _FakeSession(_FakeResponse({}), [current_resp, ok_resp])
+
+    sync.kc_update_attributes(
+        "token",
+        {"id": "u1", "username": "u1"},
+        {"mailu_app_password": "abc"},
+    )
+
+    assert sync.SESSION.put_called and sync.SESSION.get_calls == 2
+
+
+def test_main_skips_disabled_users_and_settled_attribute_sets(monkeypatch):
+    sync = load_sync_module(monkeypatch)
+    monkeypatch.setattr(sync.bcrypt_sha256, "hash", lambda password: f"hash:{password}")
+    users = [
+        {
+            "id": "u_disabled",
+            "username": "disabled",
+            "email": "disabled@example.com",
+            "enabled": False,
+            "attributes": {"mailu_enabled": ["true"]},
+        },
+        {
+            "id": "u_settled",
+            "username": "settled",
+            "email": "settled@example.com",
+            "attributes": {
+                "mailu_enabled": ["true"],
+                "mailu_email": ["settled@example.com"],
+                "mailu_app_password": ["keepme"],
+            },
+        },
+    ]
+    updated = []
+
+    class _Cursor:
+        def __init__(self):
+            self.executions = []
+
+        def execute(self, sql, params):
+            self.executions.append(params)
+
+        def close(self):
+            return None
+
+    class _Conn:
+        def __init__(self):
+            self.autocommit = False
+            self._cursor = _Cursor()
+
+        def cursor(self, cursor_factory=None):
+            return self._cursor
+
+        def close(self):
+            return None
+
+    conn = _Conn()
+    monkeypatch.setattr(sync, "get_kc_token", lambda: "tok")
+    monkeypatch.setattr(sync, "kc_get_users", lambda token: users)
+    monkeypatch.setattr(
+        sync,
+        "kc_update_attributes",
+        lambda token, user, attrs: updated.append(user["id"]),
+    )
+    monkeypatch.setattr(sync.psycopg2, "connect", lambda **kwargs: conn)
+
+    sync.main()
+
+    assert updated == []
+    assert [params["email"] for params in conn._cursor.executions] == [
+        "settled@example.com"
+    ]

scripts/tests/test_mailu_sync_listener.py

@@ -127,8 +127,64 @@ def test_listener_post_wait_keeps_running_request_successful(monkeypatch):
     assert handler.responses == [200]
 
 
+def test_listener_post_treats_non_object_json_as_plain_trigger(monkeypatch):
+    listener = load_listener_module(monkeypatch)
+    called = []
+    monkeypatch.setattr(listener, "_trigger_sync_async", lambda force=False: called.append(force) or True)
+    handler = _handler_for(listener, "[1, 2]")
+
+    handler.do_POST()
+
+    assert called == [False]
+    assert handler.responses == [202]
+
+
 def test_listener_log_message_is_quiet(monkeypatch):
     listener = load_listener_module(monkeypatch)
     handler = listener.Handler.__new__(listener.Handler)
 
     assert handler.log_message("ignored %s", "value") is None
+
+
+def test_listener_post_treats_non_object_payloads_as_plain_triggers(monkeypatch):
+    listener = load_listener_module(monkeypatch)
+    called = []
+    monkeypatch.setattr(
+        listener,
+        "_trigger_sync_async",
+        lambda force=False: called.append(force) or True,
+    )
+    handler = _handler_for(listener, "[1, 2, 3]")
+
+    handler.do_POST()
+
+    assert called == [False]
+    assert handler.responses == [202]
+
+
+def test_listener_module_serves_forever_when_run_directly(monkeypatch):
+    import http.server
+    import runpy
+
+    served = []
+
+    class _FakeServer:
+        def __init__(self, address, handler):
+            served.append((address, handler.__name__))
+
+        def serve_forever(self):
+            served.append("serving")
+
+    monkeypatch.setenv("MAILU_SYNC_WAIT_TIMEOUT_SEC", "0")
+    monkeypatch.setattr(http.server, "ThreadingHTTPServer", _FakeServer)
+    module_path = (
+        pathlib.Path(__file__).resolve().parents[2]
+        / "services"
+        / "mailu"
+        / "scripts"
+        / "mailu_sync_listener.py"
+    )
+
+    runpy.run_path(str(module_path), run_name="__main__")
+
+    assert served == [(("", 8080), "Handler"), "serving"]

services/bstein-dev-home/kustomization.yaml

@@ -22,7 +22,7 @@ images:
   - name: registry.bstein.dev/bstein/bstein-dev-home-frontend
     newTag: 0.1.1-471 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"}
   - name: registry.bstein.dev/bstein/bstein-dev-home-backend
-    newTag: 0.1.1-470 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"}
+    newTag: 0.1.1-471 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"}
 configMapGenerator:
   - name: chat-ai-gateway
     namespace: bstein-dev-home

services/gitea/atlas-identity-bootstrap-job.yaml

@@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: gitea-atlas-identity-bootstrap-3
+  name: gitea-atlas-identity-bootstrap-5
   namespace: gitea
   labels:
     app.kubernetes.io/name: gitea
@@ -34,6 +34,20 @@ spec:
       nodeSelector:
         node-role.kubernetes.io/worker: "true"
         hardware: rpi5
+      initContainers:
+        - name: python-runtime
+          image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df
+          command: [/bin/sh, -ec]
+          args: ["cp -R /usr/local/. /python/"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 1000
+            runAsGroup: 1000
+          volumeMounts:
+            - {name: python-runtime, mountPath: /python}
       containers:
         - name: bootstrap
           image: gitea/gitea:1.23
@@ -80,6 +94,7 @@ spec:
             - name: bootstrap
               mountPath: /opt/bootstrap
               readOnly: true
+            - {name: python-runtime, mountPath: /opt/python, readOnly: true}
       volumes:
         - name: data
           persistentVolumeClaim:
@@ -88,3 +103,6 @@ spec:
           configMap:
             name: gitea-atlas-identity-bootstrap
             defaultMode: 0555
+        - name: python-runtime
+          emptyDir:
+            sizeLimit: 128Mi

services/gitea/kustomization.yaml

@@ -15,6 +15,7 @@ configMapGenerator:
   - name: gitea-atlas-identity-bootstrap
     files:
       - gitea_atlas_identity_ensure.sh=scripts/gitea_atlas_identity_ensure.sh
+      - gitea_branch_protection_check.py=scripts/gitea_branch_protection_check.py
 
 generatorOptions:
   disableNameSuffixHash: true

services/gitea/scripts/gitea_atlas_identity_ensure.sh

@@ -19,6 +19,7 @@ source_owner="${GITEA_SOURCE_OWNER:-bstein}"
 managed_repositories="${GITEA_MANAGED_REPOSITORIES:-hermes-code-demo cassandra soteria pegasus metis ananke ariadne typhon titan-iac}"
 contributor_team=Contributors
 contributor_units='["repo.actions","repo.packages","repo.code","repo.issues","repo.wiki","repo.pulls","repo.releases","repo.projects"]'
+protected_reviewer=bstein
 
 die() {
   echo "Gitea Atlas identity bootstrap failed: $*" >&2
@@ -272,6 +273,46 @@ transfer_managed_repositories() {
   trap - EXIT HUP INT TERM
 }
 
+ensure_default_branch_protection() {
+  repository=$1
+  repo_file=$(mktemp)
+  protections_file=$(mktemp)
+  response_file=$(mktemp)
+
+  status=$(api_request GET "/repos/${organization}/${repository}" '' "${repo_file}")
+  expect_status "${status}" 200 "read ${repository} metadata for branch protection"
+  default_branch=$(sed -n 's/.*"default_branch":"\([^"]*\)".*/\1/p' "${repo_file}")
+  case "${default_branch}" in
+    main|master) ;;
+    *) die "${repository} has unsupported default branch ${default_branch:-missing}" ;;
+  esac
+
+  status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}")
+  expect_status "${status}" 200 "list ${repository} branch protections"
+  if ! protection_state=$(
+    /opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \
+      "${protections_file}" "${default_branch}" "${protected_reviewer}"
+  ); then
+    die "${repository} ${default_branch} protection differs from the human-review policy"
+  fi
+
+  if [ "${protection_state}" = ABSENT ]; then
+    payload="{\"rule_name\":\"${default_branch}\",\"enable_push\":true,\"enable_push_whitelist\":true,\"push_whitelist_usernames\":[\"${protected_reviewer}\"],\"push_whitelist_teams\":[],\"push_whitelist_deploy_keys\":false,\"enable_force_push\":false,\"enable_force_push_allowlist\":false,\"force_push_allowlist_usernames\":[],\"force_push_allowlist_teams\":[],\"force_push_allowlist_deploy_keys\":false,\"enable_merge_whitelist\":true,\"merge_whitelist_usernames\":[\"${protected_reviewer}\"],\"merge_whitelist_teams\":[],\"enable_approvals_whitelist\":true,\"approvals_whitelist_username\":[\"${protected_reviewer}\"],\"approvals_whitelist_teams\":[],\"required_approvals\":1,\"block_on_rejected_reviews\":true,\"block_on_outdated_branch\":true,\"dismiss_stale_approvals\":true,\"block_admin_merge_override\":true}"
+    status=$(api_request POST "/repos/${organization}/${repository}/branch_protections" "${payload}" "${response_file}")
+    expect_status "${status}" 201 "protect ${repository} ${default_branch}"
+    status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}")
+    expect_status "${status}" 200 "read back ${repository} branch protections"
+    protection_state=$(
+      /opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \
+        "${protections_file}" "${default_branch}" "${protected_reviewer}"
+    ) || die "${repository} ${default_branch} protection differs from the human-review policy"
+  fi
+  [ "${protection_state}" = PRESENT ] || \
+    die "${repository} ${default_branch} protection did not become effective"
+
+  rm -f "${repo_file}" "${protections_file}" "${response_file}"
+}
+
 wait_for_gitea
 ensure_user "${reconciler_user}" "${reconciler_email}" true
 ensure_user "${hermes_user}" "${hermes_email}" false
@@ -318,6 +359,10 @@ expect_status "${status}" 204 "add Hermes to Atlas contributors"
 
 transfer_managed_repositories
 
+for repository in ${managed_repositories}; do
+  ensure_default_branch_protection "${repository}"
+done
+
 vault_write gitea/atlas-reconciler "{\"data\":{\"username\":\"${reconciler_user}\",\"token\":\"${reconciler_token}\",\"base_url\":\"${public_url}\"}}"
 vault_write hermes/developer-gitea "{\"data\":{\"username\":\"${hermes_user}\",\"email\":\"${hermes_email}\",\"token\":\"${hermes_token}\",\"base_url\":\"${public_url}\"}}"
 

services/gitea/scripts/gitea_branch_protection_check.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""Validate the first effective Gitea protection for an Atlas primary branch."""
+
+from __future__ import annotations
+
+import argparse
+import datetime
+import json
+import os
+import stat
+from pathlib import Path
+
+MAX_INPUT = 1024 * 1024
+MAX_RULES = 100
+MAX_RULE_NAME = 255
+SPECIAL = frozenset("*?\\[]{}")
+
+
+class PolicyError(RuntimeError):
+    """The protection response is ambiguous or violates human review policy."""
+
+
+def _read_bounded(path: Path) -> bytes:
+    descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+    try:
+        metadata = os.fstat(descriptor)
+        if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > MAX_INPUT:
+            raise PolicyError("branch protection response exceeds the safe limit")
+        value = os.read(descriptor, MAX_INPUT + 1)
+    finally:
+        os.close(descriptor)
+    if len(value) != metadata.st_size:
+        raise PolicyError("branch protection response changed while reading")
+    return value
+
+
+def _is_plain(pattern: str) -> bool:
+    return not any(character in SPECIAL for character in pattern)
+
+
+def _literal_rule(pattern: str) -> str:
+    """Return one exact rule name or reject Gitea glob interpretation.
+
+    Gitea 1.23.8 delegates matching to gobwas/glob.  Reimplementing that
+    grammar and its error quoting rules here would create a second policy
+    engine.  The reconciler instead accepts only plain ASCII literals.  This
+    deliberately rejects braces, ranges, escapes, malformed globs, and even
+    special rules that appear unrelated: an operator must remove ambiguity
+    before this job may attest the primary-branch boundary.
+    """
+    if not pattern.isascii() or not 1 <= len(pattern) <= MAX_RULE_NAME:
+        raise PolicyError("branch protection rule name is invalid")
+    if not _is_plain(pattern):
+        raise PolicyError("glob branch protection rules require human review")
+    return pattern
+
+
+def _created(value: object) -> datetime.datetime:
+    if not isinstance(value, str) or len(value) > 64:
+        raise PolicyError("branch protection creation time is invalid")
+    try:
+        result = datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError as exc:
+        raise PolicyError("branch protection creation time is invalid") from exc
+    if result.tzinfo is None:
+        raise PolicyError("branch protection creation time is invalid")
+    return result
+
+
+def _required(reviewer: str) -> dict[str, object]:
+    return {
+        "enable_push": True,
+        "enable_push_whitelist": True,
+        "push_whitelist_usernames": [reviewer],
+        "push_whitelist_deploy_keys": False,
+        "enable_force_push": False,
+        "enable_merge_whitelist": True,
+        "merge_whitelist_usernames": [reviewer],
+        "enable_approvals_whitelist": True,
+        "approvals_whitelist_username": [reviewer],
+        "required_approvals": 1,
+        "block_on_rejected_reviews": True,
+        "block_on_outdated_branch": True,
+        "dismiss_stale_approvals": True,
+        "block_admin_merge_override": True,
+    }
+
+
+def evaluate(value: bytes, branch: str, reviewer: str) -> str:
+    """Return PRESENT/ABSENT or fail on ambiguous/effective policy drift."""
+    if len(value) > MAX_INPUT or branch not in {"main", "master"}:
+        raise PolicyError("branch protection input is invalid")
+    data = json.loads(value)
+    if not isinstance(data, list) or len(data) > MAX_RULES:
+        raise PolicyError("branch protection response is invalid")
+    ordered: list[tuple[int, datetime.datetime, int, dict[str, object]]] = []
+    for index, raw in enumerate(data):
+        if not isinstance(raw, dict):
+            raise PolicyError("branch protection entry is invalid")
+        priority = raw.get("priority")
+        rule_name = raw.get("rule_name")
+        if (
+            not isinstance(priority, int)
+            or isinstance(priority, bool)
+            or not 0 <= priority <= 1_000_000
+            or not isinstance(rule_name, str)
+        ):
+            raise PolicyError("branch protection priority is ambiguous")
+        created = _created(raw.get("created_at"))
+        literal = _literal_rule(rule_name)
+        if literal == branch:
+            # With every rule constrained to a plain literal, Gitea's glob
+            # comparator cannot affect this result.  Preserve API ordering:
+            # priority first (including legacy zero), creation time, then the
+            # stable response order for an otherwise exact tie.
+            ordered.append((priority, created, index, raw))
+    if not ordered:
+        return "ABSENT"
+    if len(ordered) != 1:
+        raise PolicyError("duplicate primary-branch protections are ambiguous")
+    effective = ordered[0][3]
+    for key, expected in _required(reviewer).items():
+        if effective.get(key) != expected:
+            raise PolicyError(
+                f"effective {branch} protection differs from human-review policy"
+            )
+    return "PRESENT"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("response", type=Path)
+    parser.add_argument("branch", choices=("main", "master"))
+    parser.add_argument("reviewer")
+    args = parser.parse_args()
+    try:
+        value = _read_bounded(args.response)
+        print(evaluate(value, args.branch, args.reviewer))
+    except (OSError, json.JSONDecodeError, PolicyError) as exc:
+        print(f"branch protection check failed: {exc}", file=__import__("sys").stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

services/harbor/hermes-agent-immutability-job.yaml

@@ -0,0 +1,79 @@
+# services/harbor/hermes-agent-immutability-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: harbor-hermes-agent-immutability-ensure-1
+  namespace: harbor
+spec:
+  backoffLimit: 2
+  activeDeadlineSeconds: 600
+  template:
+    metadata:
+      annotations:
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/agent-run-as-user: "65532"
+        vault.hashicorp.com/agent-run-as-group: "65532"
+        vault.hashicorp.com/role: harbor-policy-bootstrap
+        vault.hashicorp.com/agent-inject-secret-harbor-admin-password: kv/data/atlas/harbor/harbor-core
+        vault.hashicorp.com/agent-inject-template-harbor-admin-password: |
+          {{- with secret "kv/data/atlas/harbor/harbor-core" -}}
+          {{ .Data.data.harbor_admin_password }}
+          {{- end -}}
+    spec:
+      serviceAccountName: harbor-policy-bootstrap
+      enableServiceLinks: false
+      restartPolicy: Never
+      nodeSelector:
+        hardware: rpi5
+        kubernetes.io/arch: arm64
+        node-role.kubernetes.io/worker: "true"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values: [titan-04, titan-14, titan-18, titan-19, titan-24]
+      securityContext:
+        fsGroup: 65532
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: ensure
+          image: docker.io/library/python@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757
+          imagePullPolicy: IfNotPresent
+          command: [python3, /scripts/harbor_hermes_agent_immutability_ensure.py]
+          env:
+            - name: HARBOR_API_ORIGIN
+              value: https://registry.bstein.dev/api/v2.0
+            - name: HARBOR_ADMIN_PASSWORD_FILE
+              value: /vault/secrets/harbor-admin-password
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+            runAsGroup: 65532
+            runAsNonRoot: true
+            runAsUser: 65532
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - name: scripts
+              mountPath: /scripts
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+          resources:
+            requests: {cpu: 25m, memory: 32Mi}
+            limits: {cpu: 250m, memory: 128Mi}
+      volumes:
+        - name: scripts
+          configMap:
+            name: harbor-hermes-agent-immutability-script
+            defaultMode: 0555
+        - name: tmp
+          emptyDir: {}

services/harbor/kustomization.yaml

@@ -12,9 +12,14 @@ resources:
   - certificate.yaml
   - helmrelease.yaml
   - vault-sync-deployment.yaml
+  - policy-bootstrap-serviceaccount.yaml
+  - hermes-agent-immutability-job.yaml
   - bootstrap-jobs/cassandra-registry-ensure-job.yaml
   - image.yaml
 configMapGenerator:
   - name: harbor-vault-entrypoint
     files:
       - scripts/vault-entrypoint.sh
+  - name: harbor-hermes-agent-immutability-script
+    files:
+      - harbor_hermes_agent_immutability_ensure.py=scripts/harbor_hermes_agent_immutability_ensure.py

services/harbor/policy-bootstrap-serviceaccount.yaml

@@ -0,0 +1,7 @@
+# services/harbor/policy-bootstrap-serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: harbor-policy-bootstrap
+  namespace: harbor
+automountServiceAccountToken: true

services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py

@@ -0,0 +1,378 @@
+#!/usr/bin/env python3
+"""Create and verify the narrowly scoped Hermes agent immutable-tag rule."""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+
+PROJECT = "bstein"
+REPOSITORY_PATTERN = "hermes-agent"
+TAG_PATTERN = "git-*-build-*"
+PUBLISH_ROBOT = "robot$jenkins-pipelines"
+EXPECTED_ORIGIN = "https://registry.bstein.dev/api/v2.0"
+MAX_RESPONSE = 1_048_576
+TRANSIENT_STATUSES = {429, 502, 503, 504}
+EXPECTED_RULE = {
+    "disabled": False,
+    "action": "immutable",
+    "template": "immutable_template",
+    "tag_selectors": [
+        {
+            "kind": "doublestar",
+            "decoration": "matches",
+            "pattern": TAG_PATTERN,
+        }
+    ],
+    "scope_selectors": {
+        "repository": [
+            {
+                "kind": "doublestar",
+                "decoration": "repoMatches",
+                "pattern": REPOSITORY_PATTERN,
+            }
+        ]
+    },
+}
+
+
+class NoRedirect(urllib.request.HTTPRedirectHandler):
+    """Prevent Basic credentials from following an unexpected redirect."""
+
+    def redirect_request(self, _request, _file, _code, _message, _headers, _url):
+        return None
+
+
+class HarborUnavailable(RuntimeError):
+    """Harbor is not ready yet, rather than returning a policy decision."""
+
+
+def normalized_rule(rule: dict[str, Any]) -> dict[str, Any]:
+    """Return only the immutable contract fields Harbor must preserve."""
+    return {
+        "disabled": bool(rule.get("disabled", False)),
+        "action": rule.get("action"),
+        "template": rule.get("template"),
+        "tag_selectors": [
+            {
+                "kind": selector.get("kind"),
+                "decoration": selector.get("decoration"),
+                "pattern": selector.get("pattern"),
+            }
+            for selector in rule.get("tag_selectors") or []
+            if isinstance(selector, dict)
+        ],
+        "scope_selectors": {
+            "repository": [
+                {
+                    "kind": selector.get("kind"),
+                    "decoration": selector.get("decoration"),
+                    "pattern": selector.get("pattern"),
+                }
+                for selector in (rule.get("scope_selectors") or {}).get(
+                    "repository", []
+                )
+                if isinstance(selector, dict)
+            ]
+        },
+    }
+
+
+def targets_hermes_builds(rule: dict[str, Any]) -> bool:
+    """Detect a rule that claims this exact repository and tag selector."""
+    normalized = normalized_rule(rule)
+    return (
+        normalized["tag_selectors"] == EXPECTED_RULE["tag_selectors"]
+        and normalized["scope_selectors"] == EXPECTED_RULE["scope_selectors"]
+    )
+
+
+class HarborClient:
+    """Bounded same-origin client for Harbor's immutable-tag API."""
+
+    def __init__(self, origin: str, username: str, password: str) -> None:
+        normalized_origin = origin.rstrip("/")
+        if normalized_origin != EXPECTED_ORIGIN:
+            raise ValueError("Harbor API origin is not the pinned production API")
+        self.origin = normalized_origin
+        token = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
+        self.headers = {"Authorization": f"Basic {token}"}
+        self.opener = urllib.request.build_opener(NoRedirect())
+
+    def request(
+        self, method: str, path: str, payload: dict[str, Any] | None = None
+    ) -> tuple[int, bytes, dict[str, str]]:
+        """Issue one request, returning even non-2xx responses for strict checks."""
+        data = None
+        headers = dict(self.headers)
+        if payload is not None:
+            data = json.dumps(payload, separators=(",", ":")).encode()
+            headers["Content-Type"] = "application/json"
+        request = urllib.request.Request(
+            f"{self.origin}{path}", data=data, headers=headers, method=method
+        )
+        try:
+            response = self.opener.open(request, timeout=20)
+        except urllib.error.HTTPError as exc:
+            response = exc
+        except (urllib.error.URLError, TimeoutError) as exc:
+            raise HarborUnavailable("Harbor policy API is unavailable") from exc
+        with response:
+            body = response.read(MAX_RESPONSE + 1)
+            if len(body) > MAX_RESPONSE:
+                raise RuntimeError("Harbor response exceeded the size limit")
+            return int(response.status), body, dict(response.headers)
+
+
+def list_rules(client: HarborClient) -> list[dict[str, Any]]:
+    """Read and validate the complete small rule set for the project."""
+    path = f"/projects/{PROJECT}/immutabletagrules?page=1&page_size=100"
+    status, body, headers = client.request("GET", path)
+    if status in TRANSIENT_STATUSES:
+        raise HarborUnavailable(f"Harbor immutable rule list returned HTTP {status}")
+    if status != 200:
+        raise RuntimeError(f"Harbor immutable rule list returned HTTP {status}")
+    values = _json_list(body, "immutable rule")
+    _require_complete_page(values, headers, "immutable rule")
+    return values
+
+
+def _json_list(body: bytes, label: str) -> list[dict[str, Any]]:
+    """Decode one bounded Harbor list without accepting a partial shape."""
+    try:
+        value = json.loads(body.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise RuntimeError(f"Harbor returned invalid {label} JSON") from exc
+    if not isinstance(value, list) or not all(isinstance(item, dict) for item in value):
+        raise RuntimeError(f"Harbor {label} list has an invalid shape")
+    return value
+
+
+def _require_complete_page(
+    values: list[dict[str, Any]], headers: dict[str, str], label: str
+) -> None:
+    """Reject a truncated first page or a proxy that strips count evidence."""
+    raw_total = next(
+        (value for key, value in headers.items() if key.lower() == "x-total-count"),
+        None,
+    )
+    if raw_total is None or not str(raw_total).isdecimal():
+        raise RuntimeError(f"Harbor {label} list omitted a valid total count")
+    if int(raw_total) != len(values):
+        raise RuntimeError(f"Harbor {label} list was truncated")
+
+
+def _allowed(access: dict[str, Any], resource: str, action: str) -> bool:
+    """Match one positive robot permission, accepting Harbor's default effect."""
+    return (
+        access.get("resource") == resource
+        and access.get("action") == action
+        and access.get("effect") in (None, "", "allow")
+    )
+
+
+def _publisher_scope(robot: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+    """Return the existing bstein scope after validating its push boundary."""
+    permissions = robot.get("permissions")
+    if not isinstance(permissions, list) or not all(
+        isinstance(item, dict) for item in permissions
+    ):
+        raise RuntimeError("Jenkins Harbor robot permissions have an invalid shape")
+    matches = [
+        item
+        for item in permissions
+        if item.get("kind") == "project" and item.get("namespace") == PROJECT
+    ]
+    if len(matches) != 1:
+        raise RuntimeError("Jenkins Harbor robot must have one existing bstein scope")
+    access = matches[0].get("access")
+    if not isinstance(access, list) or not all(isinstance(item, dict) for item in access):
+        raise RuntimeError("Jenkins Harbor robot bstein access has an invalid shape")
+    for required in (("repository", "pull"), ("repository", "push")):
+        if not any(_allowed(item, *required) for item in access):
+            raise RuntimeError("Jenkins Harbor robot lacks its existing pull/push boundary")
+    immutable_access = [
+        item for item in access if item.get("resource") == "immutable-tag"
+    ]
+    if immutable_access and not (
+        len(immutable_access) == 1
+        and _allowed(immutable_access[0], "immutable-tag", "list")
+    ):
+        raise RuntimeError("Jenkins Harbor robot has broader immutable-tag access")
+    return matches[0], access
+
+
+def _verified_publisher(robot: dict[str, Any]) -> bool:
+    """Check the exact identity and least-privilege policy after persistence."""
+    if (
+        robot.get("name") != PUBLISH_ROBOT
+        or robot.get("level") != "system"
+        or robot.get("editable") is not True
+        or robot.get("disable") is not False
+    ):
+        return False
+    try:
+        _scope, access = _publisher_scope(robot)
+    except RuntimeError:
+        return False
+    return any(_allowed(item, "immutable-tag", "list") for item in access)
+
+
+def ensure_publisher_can_read_rule(client: HarborClient) -> int:
+    """Grant only immutable-tag:list to the existing Jenkins push robot."""
+    status, body, headers = client.request("GET", "/robots?page=1&page_size=100")
+    if status in TRANSIENT_STATUSES:
+        raise HarborUnavailable(f"Harbor robot list returned HTTP {status}")
+    if status != 200:
+        raise RuntimeError(f"Harbor robot list returned HTTP {status}")
+    robots = _json_list(body, "robot")
+    _require_complete_page(robots, headers, "robot")
+    matches = [
+        item
+        for item in robots
+        if item.get("name") == PUBLISH_ROBOT
+    ]
+    if len(matches) != 1:
+        raise RuntimeError("expected exactly one Jenkins Harbor publisher robot")
+    robot_id = matches[0].get("id")
+    if not isinstance(robot_id, int) or robot_id < 1:
+        raise RuntimeError("Jenkins Harbor robot omitted a valid ID")
+    status, body, _headers = client.request("GET", f"/robots/{robot_id}")
+    if status in TRANSIENT_STATUSES:
+        raise HarborUnavailable(f"Harbor robot read returned HTTP {status}")
+    if status != 200:
+        raise RuntimeError(f"Harbor robot read returned HTTP {status}")
+    try:
+        robot = json.loads(body.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise RuntimeError("Harbor returned invalid robot JSON") from exc
+    if not isinstance(robot, dict):
+        raise RuntimeError("Harbor robot response has an invalid shape")
+    if (
+        robot.get("name") != PUBLISH_ROBOT
+        or robot.get("level") != "system"
+        or robot.get("editable") is not True
+        or robot.get("disable") is not False
+    ):
+        raise RuntimeError("Jenkins Harbor publisher identity is not active and exact")
+    scope, access = _publisher_scope(robot)
+    if any(_allowed(item, "immutable-tag", "list") for item in access):
+        return robot_id
+
+    # Harbor updates permissions through PUT /robots/{id}; its separate PATCH
+    # endpoint is the only secret-rotation operation. Preserve every current
+    # scope and field while adding the one read-only action.
+    access.append({"resource": "immutable-tag", "action": "list"})
+    payload = {
+        "name": robot["name"],
+        "description": robot.get("description") or "",
+        "level": robot["level"],
+        "disable": robot["disable"],
+        "permissions": robot["permissions"],
+    }
+    duration = robot.get("duration")
+    if duration is not None:
+        if not isinstance(duration, int):
+            raise RuntimeError("Jenkins Harbor robot duration has an invalid shape")
+        payload["duration"] = duration
+    # Keep the validated object reference live in the preserved permission list.
+    if scope.get("access") is not access:
+        raise RuntimeError("Jenkins Harbor robot permission normalization drifted")
+    status, _body, _headers = client.request(
+        "PUT", f"/robots/{robot_id}", payload
+    )
+    if status in TRANSIENT_STATUSES:
+        raise HarborUnavailable(f"Harbor robot update returned HTTP {status}")
+    if status != 200:
+        raise RuntimeError(f"Harbor robot policy update returned HTTP {status}")
+
+    for attempt in range(1, 6):
+        status, body, _headers = client.request("GET", f"/robots/{robot_id}")
+        if status == 200:
+            try:
+                persisted = json.loads(body.decode("utf-8"))
+            except (UnicodeDecodeError, json.JSONDecodeError):
+                persisted = None
+            if isinstance(persisted, dict) and _verified_publisher(persisted):
+                return robot_id
+        if attempt < 5:
+            time.sleep(attempt)
+    raise RuntimeError("Jenkins Harbor robot policy did not verify exactly")
+
+
+def ensure_rule(client: HarborClient) -> int:
+    """Create once, or validate the one exact enabled rule already present."""
+    rules = list_rules(client)
+    matches = [rule for rule in rules if targets_hermes_builds(rule)]
+    if len(matches) > 1:
+        raise RuntimeError("multiple Hermes agent immutable rules exist")
+    if matches:
+        if normalized_rule(matches[0]) != EXPECTED_RULE:
+            raise RuntimeError("Hermes agent immutable rule exists but is not enabled/exact")
+        rule_id = matches[0].get("id")
+        if not isinstance(rule_id, int) or rule_id < 1:
+            raise RuntimeError("Harbor immutable rule omitted a valid ID")
+        return rule_id
+
+    path = f"/projects/{PROJECT}/immutabletagrules"
+    status, _body, headers = client.request("POST", path, EXPECTED_RULE)
+    if status in TRANSIENT_STATUSES:
+        raise HarborUnavailable(f"Harbor immutable rule create returned HTTP {status}")
+    if status != 201:
+        raise RuntimeError(f"Harbor immutable rule create returned HTTP {status}")
+    location = headers.get("Location") or headers.get("location") or ""
+    api_path = urllib.parse.urlsplit(client.origin).path.rstrip("/")
+    expected_prefix = f"{api_path}{path}/"
+    if not location.startswith(expected_prefix):
+        raise RuntimeError("Harbor immutable rule create omitted the exact Location")
+    suffix = location[len(expected_prefix) :]
+    if not suffix.isdecimal() or int(suffix) < 1:
+        raise RuntimeError("Harbor immutable rule Location has an invalid ID")
+
+    # Harbor persists synchronously, but bounded retries distinguish a stale
+    # read from accepting an unverified policy.
+    for attempt in range(1, 6):
+        matches = [rule for rule in list_rules(client) if targets_hermes_builds(rule)]
+        if len(matches) == 1 and normalized_rule(matches[0]) == EXPECTED_RULE:
+            rule_id = matches[0].get("id")
+            if rule_id == int(suffix):
+                return rule_id
+        if attempt < 5:
+            time.sleep(attempt)
+    raise RuntimeError("created Harbor immutable rule did not verify exactly")
+
+
+def main() -> int:
+    """Load the runtime-only admin credential and enforce the tracked policy."""
+    origin = os.environ.get("HARBOR_API_ORIGIN", "")
+    password_file = Path(os.environ.get("HARBOR_ADMIN_PASSWORD_FILE", ""))
+    password = password_file.read_text(encoding="utf-8").strip()
+    if not password:
+        raise RuntimeError("Harbor admin password is empty")
+    client = HarborClient(origin, "admin", password)
+    for attempt in range(1, 13):
+        try:
+            rule_id = ensure_rule(client)
+            robot_id = ensure_publisher_can_read_rule(client)
+            break
+        except HarborUnavailable:
+            if attempt == 12:
+                raise
+            time.sleep(min(attempt * 2, 15))
+    print(
+        "Hermes agent immutable build-tag rule is active "
+        f"(id={rule_id}); publisher preflight access is active (robot={robot_id})"
+    )
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover - exercised through main()
+    raise SystemExit(main())

services/hermes-observer-bindings/kustomization.yaml

@@ -0,0 +1,5 @@
+# services/hermes-observer-bindings/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - rolebindings.yaml

services/hermes-observer-bindings/rolebindings.yaml

@@ -0,0 +1,204 @@
+# services/hermes-observer-bindings/rolebindings.yaml
+apiVersion: v1
+kind: List
+items:
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: ai}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: bstein-dev-home}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: cassandra}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: cert-manager}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: climate}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: comms}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: crypto}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: default}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: finance}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: flux-system}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: game-stream}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: gitea}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: harbor}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: health}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: hermes}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: hermes-chat}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: hermes-triage-demo}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: jellyfin}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: jenkins}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: kube-node-lease}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: kube-public}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: kube-system}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: logging}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: longhorn-system}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: mailu-mailserver}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: maintenance}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: metallb-system}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: monitoring}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: nextcloud}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: openclaw}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: outline}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: planka}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: postgres}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: quality}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: sso}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: sui-metrics}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: traefik}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: vault}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: vaultwarden}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata: {name: hermes-agent-observer, namespace: veles}
+    subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
+    roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}

services/hermes-observer-rbac/kustomization.yaml

@@ -0,0 +1,5 @@
+# services/hermes-observer-rbac/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - rbac.yaml

services/hermes-observer-rbac/rbac.yaml

@@ -0,0 +1,112 @@
+# services/hermes-observer-rbac/rbac.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: hermes-agent-cluster-observer-v2
+  labels:
+    app.kubernetes.io/name: hermes-agent
+    app.kubernetes.io/part-of: hermes
+rules:
+  - apiGroups: [""]
+    resources:
+      - namespaces
+      - nodes
+      - persistentvolumes
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["ingressclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources:
+      - csidrivers
+      - csinodes
+      - storageclasses
+      - volumeattachments
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["metrics.k8s.io"]
+    resources: ["nodes"]
+    verbs: ["get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: hermes-agent-namespaced-observer-v2
+  labels:
+    app.kubernetes.io/name: hermes-agent
+    app.kubernetes.io/part-of: hermes
+rules:
+  - apiGroups: [""]
+    resources:
+      - configmaps
+      - endpoints
+      - events
+      - persistentvolumeclaims
+      - pods
+      - pods/log
+      - replicationcontrollers
+      - resourcequotas
+      - services
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["apps"]
+    resources:
+      - controllerrevisions
+      - daemonsets
+      - deployments
+      - replicasets
+      - statefulsets
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["batch"]
+    resources: ["cronjobs", "jobs"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["autoscaling"]
+    resources: ["horizontalpodautoscalers"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["ingresses", "networkpolicies"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["discovery.k8s.io"]
+    resources: ["endpointslices"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["policy"]
+    resources: ["poddisruptionbudgets"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csistoragecapacities"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["metrics.k8s.io"]
+    resources: ["pods"]
+    verbs: ["get", "list"]
+  - apiGroups:
+      - helm.toolkit.fluxcd.io
+      - image.toolkit.fluxcd.io
+      - kustomize.toolkit.fluxcd.io
+      - notification.toolkit.fluxcd.io
+      - source.toolkit.fluxcd.io
+    resources: ["*"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["longhorn.io"]
+    resources:
+      - backingimages
+      - engines
+      - instancemanagers
+      - nodes
+      - replicas
+      - settings
+      - volumes
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: hermes-agent-cluster-observer-v2
+  labels:
+    app.kubernetes.io/name: hermes-agent
+    app.kubernetes.io/part-of: hermes
+subjects:
+  - kind: ServiceAccount
+    name: hermes-agent
+    namespace: hermes
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: hermes-agent-cluster-observer-v2

services/hermes-scm-broker/deployment.yaml

@@ -0,0 +1,99 @@
+# services/hermes-scm-broker/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hermes-scm-broker
+  namespace: hermes-scm
+  labels:
+    app: hermes-scm-broker
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: hermes-scm-broker
+  template:
+    metadata:
+      labels:
+        app: hermes-scm-broker
+      annotations:
+        ai.bstein.dev/config-rev: scm-boundary-v2
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: hermes-scm-broker
+        vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea
+        vault.hashicorp.com/agent-inject-template-gitea-token: |
+          {{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
+          {{ .Data.data.token }}
+          {{- end }}
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/agent-init-first: "true"
+        vault.hashicorp.com/agent-requests-cpu: 10m
+        vault.hashicorp.com/agent-requests-mem: 32Mi
+        vault.hashicorp.com/agent-limits-cpu: 50m
+        vault.hashicorp.com/agent-limits-mem: 64Mi
+    spec:
+      serviceAccountName: hermes-scm-broker
+      automountServiceAccountToken: true
+      securityContext:
+        fsGroup: 10000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: [arm64]
+                  - key: node-role.kubernetes.io/worker
+                    operator: In
+                    values: ["true"]
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: [rpi5]
+      containers:
+        - name: broker
+          image: registry.bstein.dev/bstein/hermes-agent
+          imagePullPolicy: IfNotPresent
+          command:
+            - /opt/hermes/.venv/bin/python
+            - /opt/broker/scm_broker.py
+          ports:
+            - {name: http, containerPort: 9081, protocol: TCP}
+          readinessProbe:
+            httpGet: {path: /healthz, port: http}
+            initialDelaySeconds: 3
+            periodSeconds: 10
+          livenessProbe:
+            httpGet: {path: /healthz, port: http}
+            initialDelaySeconds: 15
+            periodSeconds: 30
+            failureThreshold: 10
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 10000
+            runAsGroup: 10000
+          volumeMounts:
+            - {name: broker-code, mountPath: /opt/broker, readOnly: true}
+            - {name: tmp, mountPath: /tmp}
+          resources:
+            requests: {cpu: 50m, memory: 128Mi}
+            limits: {cpu: "1", memory: 768Mi}
+      volumes:
+        - name: broker-code
+          configMap:
+            name: hermes-scm-boundary-v2
+            defaultMode: 0555
+        - name: tmp
+          emptyDir:
+            sizeLimit: 1Gi

services/hermes-scm-broker/kustomization.yaml

@@ -0,0 +1,11 @@
+# services/hermes-scm-broker/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: hermes-scm
+resources:
+  - service.yaml
+  - deployment.yaml
+  - networkpolicy.yaml
+images:
+  - name: registry.bstein.dev/bstein/hermes-agent
+    digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7

services/hermes-scm-broker/networkpolicy.yaml

@@ -0,0 +1,52 @@
+# services/hermes-scm-broker/networkpolicy.yaml
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-scm-broker-isolation
+  namespace: hermes-scm
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-scm-broker
+  policyTypes: [Ingress, Egress]
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: hermes
+          podSelector:
+            matchLabels:
+              app: hermes-agent
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: hermes
+          podSelector:
+            matchLabels:
+              app: hermes-execution-mediator
+      ports:
+        - {protocol: TCP, port: 9081}
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      ports:
+        - {protocol: UDP, port: 53}
+        - {protocol: TCP, port: 53}
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: vault
+          podSelector:
+            matchLabels:
+              app: vault
+      ports:
+        - {protocol: TCP, port: 8200}
+    - to:
+        - ipBlock:
+            cidr: 192.168.22.9/32
+      ports:
+        - {protocol: TCP, port: 443}

services/hermes-scm-broker/service.yaml

@@ -0,0 +1,13 @@
+# services/hermes-scm-broker/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: hermes-scm-broker
+  namespace: hermes-scm
+spec:
+  selector:
+    app: hermes-scm-broker
+  ports:
+    - name: http
+      port: 9081
+      targetPort: http

services/hermes-scm-namespace/kustomization.yaml

@@ -0,0 +1,5 @@
+# services/hermes-scm-namespace/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml

services/hermes-scm-namespace/namespace.yaml

@@ -0,0 +1,14 @@
+# services/hermes-scm-namespace/namespace.yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: hermes-scm
+  labels:
+    app.kubernetes.io/name: hermes-scm-broker
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: hermes-scm-broker
+  namespace: hermes-scm
+automountServiceAccountToken: true

services/hermes/NOTES.md

@@ -102,6 +102,110 @@ console tails and named artifact contents in its deterministic bundle. A report
 must say `retained Ariadne evidence` when that fallback is used; it must not
 pretend direct Jenkins access succeeded.
 
+## Publishing an agent image after review
+
+The `hermes-agent-image` Jenkins job is the only supported agent image builder.
+It runs daemonless Kaniko v1.23.2 under the unbound `hermes-image-builder`
+ServiceAccount without a service-account token, host socket,
+privileged container, or writable Git credential. It accepts only an exact
+40-character revision that is both the checked-out commit and current
+`atlas/titan-iac` `main`, so a human must merge the source PR first.
+
+The build pod requires an ARM64 worker and gives `hardware=rpi5` the maximum
+scheduler preference. Healthy rpi4 workers remain valid fallback capacity when
+the rpi5 pool cannot fit the full request. Cordoned, unhealthy, amd64, and
+reserved last-resort nodes remain excluded by the required placement boundary.
+
+Kaniko runs as UID 0 because it must unpack an image root filesystem and enter
+that filesystem for Dockerfile `RUN` instructions. Its capability set is the
+minimum proven by an exact ARM64 no-push build of this Dockerfile:
+`CHOWN`, `FOWNER`, `DAC_OVERRIDE`, `SETGID`, and `SETUID`. `SETGID` is required
+because pinned Kaniko applies the base image's supplementary group list before
+each Dockerfile `RUN`; `SETUID` lets apt drop privileges to its `_apt` account
+while downloading package indexes. It still has no privilege
+escalation, service-account token, host path, daemon socket, or Docker/BuildKit
+TCP endpoint and uses the runtime-default seccomp profile. This is residual
+root-in-the-build-pod risk, bounded to disposable `emptyDir` storage and a
+human-reviewed `main` revision; Dockerfile changes require the same scrutiny as
+executable cluster code.
+
+Pinned Kaniko v1.23.2 parses Dockerfile `RUN` heredocs but does not materialize
+their inline files when executing a command. The Jenkins lane therefore enables
+a bounded compatibility replay after copying the reviewed Dockerfile and runner
+from the same checked-out commit. Before every replay, the runner inventories
+the entire Dockerfile as logical instructions, including split opcodes,
+operators, and either Dockerfile escape character. It rejects every heredoc
+form outside the exact nine Node/Python blocks before launching an interpreter.
+Each accepted body is then replayed immediately after its corresponding Kaniko
+`RUN`, preserving source order before any dependent regression. Docker and
+BuildKit keep their native behavior because the compatibility argument defaults
+off. The final TypeScript build, Python compile, and source assertions remain
+mandatory, so an omitted or drifted replay fails before any image can pass
+release verification.
+
+From agent.hermes, trigger that one fixed job with:
+
+```sh
+jenkins_image_build_trigger.py '<reviewed full main commit>'
+```
+
+The helper has no general Jenkins credential or caller-selectable job name. Its
+Vault-projected token is bound by Jenkins only to `hermes-agent-image`. The job
+also requires its fixed publish confirmation. Each run claims the unique tag
+`git-<reviewed-sha>-build-<jenkins-build-number>` and refuses to overwrite an
+existing tag. It cross-checks Kaniko's digest and tagged-image evidence, then
+independently reads the pushed tag from Harbor before archiving a JSON record
+and Flux digest patch. A successful post-condition then revalidates the exact
+six-file evidence set and archives those six explicit paths with empty archives
+forbidden. Apply that patch on a new branch and submit it for human review; the
+build never changes Git, reconciles Flux, or deploys by itself.
+The helper posts only to the fixed HTTPS Build Token Root endpoint and accepts
+only its real queue responses: HTTP 201 or a non-followed HTTP 303 with an exact
+same-origin `/queue/item/<number>/` location. An unauthenticated request is
+denied, and the token cannot select, configure, read, or administer another
+Jenkins job. The pipeline independently rejects any commit that is not the
+current `origin/main`, preserving the human merge/review boundary.
+
+Harbor independently enforces an enabled immutable-tag rule scoped to only the
+`bstein/hermes-agent` repository and `git-*-build-*` tags. A second manifest PUT,
+retag, or deletion is rejected by Harbor even if a caller bypasses the Jenkins
+preflight. A revisioned, Flux-tracked policy Job creates or verifies that exact
+rule with a runtime-only Vault credential; it refuses to alter a conflicting
+rule and retries only explicit transport/readiness failures while Harbor
+starts. The same Job grants the existing Jenkins publisher only
+`immutable-tag:list` on `bstein`, preserving its other project scopes and never
+calling Harbor's separate secret-rotation endpoint. The pipeline uses that
+read-only permission to require the exact enabled rule before Kaniko starts.
+That check also makes an already-running Jenkins controller fail closed during
+rollout; Jenkins cannot become Ready on the new revision before the policy Job
+succeeds.
+
+The tracked rollout order is deliberate: the revisioned Vault role Job must
+complete before the Vault Kustomization becomes Ready. A separate Flux
+Kustomization then runs the seed Job under the dedicated
+`hermes-jenkins-token-seed` identity, which can only create/read the exact
+`kv/data/atlas/hermes/developer-jenkins` path and request random bytes. Jenkins
+depends on both that seed and the Harbor policy Job; Hermes depends on Jenkins.
+The seeder uses KV-v2 CAS create-only semantics. It never changes an existing
+token, never fills a missing field in an existing secret, and fails closed when
+it cannot distinguish absence from a read error. The recurring Vault
+configuration job reconciles the same narrow role and policy.
+
+Rotate this fixed-job token only as a coordinated operator action:
+
+1. Stop new `hermes-agent-image` triggers and wait for its queue and executor to
+   drain.
+2. Read the current KV-v2 metadata version for
+   `kv/atlas/hermes/developer-jenkins`.
+3. Generate a fresh value from Vault and patch only `build_token` with
+   `vault kv patch -cas=<current-version>`. Keep the value in a mode-0600
+   temporary file or standard input, never a command argument or log.
+4. Roll Jenkins first and wait until it is Ready, then roll `hermes-agent` and
+   wait until it is Ready. Do not resume triggers between those two consumers.
+5. Run one reviewed-main canary. If rollback is required, repeat the CAS patch
+   with the previous value as another coordinated rotation; never delete the
+   secret to make the seeder recreate it.
+
 ## The actual supervised triage algorithm
 
 1. Classify the request as test/build triage, service health, or alert tuning.
@@ -257,6 +361,60 @@ Use this short explanation:
   leading different real incidents, improving the skill after failures, and
   teaching the architecture without prompts.
 
+## Distributed execution-pool rollout boundary
+
+- Do not reconcile the pool until the approved credential owner has provisioned
+  six distinct Vault fields: `execution_worker_{0,1,2}_claude_credentials_json`
+  and `execution_worker_{0,1,2}_codex_auth_json`. Each ordinal needs an
+  independent account or refresh-token lineage; copying one rotating refresh
+  token into multiple fields recreates the lost-update failure this design
+  prevents.
+- Worker credential files live on separate ordinal-owned RWO `provider-access`
+  claims. Provider refresh updates these durable private copies across Pod
+  restarts; they are never synchronized back to Vault. Rotate one bootstrap
+  credential at a time through the reviewed Vault workflow and reinitialize
+  only that ordinal after human approval.
+- Existing task worktrees remain on the single local owner lane. Only tasks
+  without `workspace_path` enter the distributed pool, where each ordinal owns
+  one RWO checkout. Repository and base-branch identity comes from the canonical
+  board registry rather than task-supplied metadata.
+- Model Pods have no pool key, broker mount, broker egress, or Kubernetes token.
+  Ordinal mediator Deployments are separate network identities that co-mount the
+  matching workspace PVC; they alone authenticate exact-run results and reach the
+  PR14 SCM broker.
+- The ordinal workspace claim is `ReadWriteMany` because the worker and its
+  mediator both mount it. With `ReadWriteOnce` a drain or preemption that moved
+  only one of them left the other holding the attachment on the old node, and
+  the ordinal deadlocked on Multi-Attach until an operator deleted a Pod.
+  Colocation is now a scheduling preference and both Pods run at `scavenger`
+  priority, so either can be evicted and rescheduled on its own. There is
+  deliberately no PodDisruptionBudget: this is a best-effort pool and a budget
+  would impede node drains. Claims only one Pod mounts stay `ReadWriteOnce`.
+- `lease_failed` is a *retryable* durable state, not a terminal one. It means the
+  ordinal was released but Kanban has not confirmed the outcome, so every
+  maintenance pass retries it. Only `finalized` and `stale` are terminal, and
+  only terminal rows are garbage-collected, so no run is dropped before the pool
+  knows what happened to it. If the log keeps reporting a deferred lease park for
+  the same run, Kanban is refusing an exact-run park on a task it still reports
+  as running: inspect that task rather than deleting the row.
+- The broker accepts branch creation only. A pool retry that adds commits
+  therefore publishes a new `<branch>-attempt-<n>` (or, failing that,
+  `<branch>-<head12>`) ref in the same reviewed namespace instead of updating the
+  ref a previous attempt published, and adopts an existing ref that already
+  points at the exact head rather than pushing again. Expect one draft per
+  published ref; each is still human-reviewed. A submission that cannot land
+  blocks the run with the reason and leaves the commits on the ordinal's
+  workspace, so prior work is recoverable.
+- Provider CLIs install once per pinned version onto a durable per-ordinal
+  `tools` claim, re-verified against the real binaries so a pruned cache
+  reinstalls. The pool is intentionally absent from the `hermes` Kustomization's
+  `healthChecks`: gating that 10m window on a best-effort pool would stall
+  `hermes-chat` and `hermes-observer-bindings`, which `dependsOn: hermes`.
+- A hashed execution-pool ConfigMap, protocol-version readiness checks, and
+  versioned SCM boundary name make code/config changes controlled rollouts. A
+  rollout is still a human-reviewed operation; this repository change does not
+  reconcile or deploy it.
+
 ## Your shortest path to fluency
 
 1. Explain the request diagram without looking.

services/hermes/agent-configmap.yaml

@@ -88,6 +88,14 @@ data:
       auto_decompose: true
       auto_decompose_per_tick: 2
       dispatch_stale_timeout_seconds: 14400
+      # Steady-state quota-aware routing for the direct CLI lane: below this
+      # remaining-percent a provider stops receiving NEW cli-auto work while
+      # active work finishes; lane-observed capacity/auth failures exclude a
+      # provider until the cooldown elapses, its quota window resets, or a
+      # fresh success is recorded.
+      provider_quota_min_remaining_percent: 15
+      provider_capacity_cooldown_seconds: 300
+      provider_auth_cooldown_seconds: 3600
 
     model_catalog:
       enabled: true
@@ -189,9 +197,9 @@ data:
     not replace these coordinator-wide rules. Never call `kanban_show` without
     a known, non-empty task ID. Ad-hoc inspection and acceptance checks do not
     need a synthetic Kanban lookup, and must load a skill only when its workflow
-    materially applies. Atlas HTTPS Git authentication is already supplied by
-    the runtime-only `GIT_ASKPASS`; use it without reading or exposing the
-    credential. Coordinator guidance lives at
+    materially applies. Atlas Git access is supplied by the isolated SCM
+    broker; no repository token is present in this pod. Use only configured
+    broker remotes and clients. Coordinator guidance lives at
     `/opt/data/workspace/AGENTS.md` when more detail is needed.
 
     The Jetson classifier is mandatory for AUTO selection. Switchyard may use
@@ -295,70 +303,75 @@ data:
     objective's difficulty warrants it; otherwise synthesize at the original
     effort.
 
-    This owner-only pod has cluster-admin access across Atlas, including logs,
-    Secrets, exec, port-forwarding, rollout operations, and Flux reconciliation.
-    Prefer the titan-iac Git/Flux workflow for every durable cluster change;
-    direct operations are available for explicit operator requests, incident
-    recovery, and verification, and must be followed by a matching source-of-
-    truth change when they alter desired state. Never expose credentials in
-    chat or logs. Triage belongs at triage.hermes.bstein.dev.
+    This owner-only pod has scoped read-only Kubernetes diagnostics across
+    Atlas, including resource status, events, logs, and Flux/Helm evidence. It
+    cannot read Secrets, exec or attach to pods, create service-account tokens,
+    mutate workloads or RBAC, or reconcile Flux. Put every durable cluster
+    change on a reviewed titan-iac branch. Never expose credentials in chat or
+    logs. Triage belongs at triage.hermes.bstein.dev.
 
     ## Atlas engineering access
 
     The Atlas organization has private visibility. Repository visibility is
     preserved per project and may be public or private; do not infer a
     repository's visibility from the organization setting. Repositories are
-    canonical at `https://scm.bstein.dev/atlas/<repo>.git`. HTTPS Git
-    authentication is already supplied through `GIT_ASKPASS`. Verify the remote
+    canonical at `https://scm.bstein.dev/atlas/<repo>.git`; worker Git traffic
+    uses `http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/git/atlas/<repo>.git`.
+    Verify the broker remote
     and cleanly separate pre-existing changes, create a task branch, run the
     repository's tests, use `git push --dry-run` when proving access, and push a
     real branch only when the requested implementation is review-ready. Never
     force-push.
 
-    `scm.bstein.dev` is Gitea, not GitHub. Never load or follow a GitHub/`gh`
-    skill for an Atlas remote, and do not interpret an unauthenticated Gitea
-    HTTP 404 as a missing private repository. Use authenticated Git for clone,
-    fetch, and push. For pull-request metadata, comments, and merges, call
-    `/opt/coordinator/gitea_api.py METHOD /api/v1/...`; it injects the runtime
-    Vault token without exposing it to the command line, environment, output,
-    or transcript. Before a consequential merge, independently verify the
-    exact base/head ancestry and diff, run the relevant tests, and confirm the
-    candidate Jenkins result. Prefer the internal endpoint in
-    `JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use the
-    existing cluster access to inspect the controller's job/build files and
-    logs rather than guessing a public hostname. The preferred read-only path
-    is `/opt/coordinator/jenkins_build_evidence.py JOB [--branch BRANCH]
-    [--commit SHA] --wait`; it distinguishes a genuinely terminal build from
-    nested Jenkins execution metadata and returns bounded JSON/log evidence.
-    With a branch, the helper also tries the conventional `JOB-branches`
-    multibranch name. For JSON Gitea mutations, prefer repeatable shell-safe
-    `--field KEY=VALUE` arguments over hand-quoted JSON.
-    After merging, observe the default-branch build to a terminal result and
-    report exact commit, build, and test evidence.
+    `scm.bstein.dev` is Forgejo/Gitea, not GitHub. Never load or follow a
+    GitHub/`gh` skill for an Atlas remote, and do not interpret an
+    unauthenticated Gitea HTTP 404 as a missing private repository. Use
+    brokered Git for clone, fetch, and creation of a new namespaced feature
+    branch. Existing-ref updates, protected refs, deletion, and force-push are
+    rejected. The broker inflates and scans every pushed object, so thin
+    packs are rejected; always push with `git push --no-thin` so the pack
+    is self-contained. For bounded
+    repository/pull-request evidence or to create a review-ready draft PR, load
+    `$manage-atlas-pull-requests` and use `/opt/scm/gitea_api.py`. The
+    client carries no repository credential; a separate least-authority
+    workload performs the allowed operation. It permits only Atlas-scoped
+    metadata reads and verified same-repository draft creation. Updates, merge,
+    approve, close, delete, comments, repository administration, and force-push
+    are unavailable; never bypass the client with raw HTTP. Leave every PR
+    unmerged for Brad's review. Independently
+    verify the exact base/head ancestry and diff, run the relevant tests, and
+    confirm the candidate Jenkins result before presenting a handoff. Prefer
+    the internal endpoint in
+    `JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use
+    Kubernetes pod logs and status evidence without exec or secret access.
+    Do not bypass the boundary to inspect controller files. If Brad separately
+    reports that he merged the PR, observe
+    the default-branch build to a terminal result and report exact commit,
+    build, and test evidence.
 
     The workspace already has a non-secret Hermes Git author identity. Do not
     use `git reset --hard`, even in a fresh clone; use a detached worktree or a
     clean branch switch when comparing revisions, and preserve any unexpected
-    file as user state. Use the Gitea pull-request merge endpoint for an open
-    PR so both Git history and PR state remain auditable. If an authorized
-    manual merge already reached the base branch, reconcile the open PR with
-    `Do=manually-merged` and its exact merge commit instead of leaving stale
-    review state.
+    file as user state. PR publication ends at the verified open draft; Brad
+    owns review and merge authority.
 
     The terminal PATH contains the pinned operator tools. Start cluster work
     with `kubectl config current-context`, read-only status/events/logs, and the
-    relevant `titan-iac` manifests. Put durable desired-state changes on a
-    reviewable `titan-iac` branch, validate Kustomize and client dry-run, then
-    use Flux reconciliation after the tracked change is published. Direct
-    `kubectl` mutations are for explicit incident recovery or ephemeral
-    verification, not ordinary delivery.
+    relevant `titan-iac` manifests. The broker namespace is deliberately
+    excluded from agent RBAC. Kubernetes cannot express a deny on one namespace
+    inside an all-namespace list, so enumerate namespaces and run namespaced
+    diagnostics instead of relying on `kubectl ... --all-namespaces`. Put
+    durable desired-state changes on a
+    reviewable `titan-iac` branch and validate Kustomize and client dry-run.
+    Brad owns merge and Flux reconciliation. Direct `kubectl` mutation is not
+    available to the agent; hand an explicit incident mutation back to Brad.
 
-    Node SSH uses a dedicated audited Hermes identity: `ssh titan-04` (or any
-    current Kubernetes node name). Host keys are pinned and password fallback
-    is disabled. Use SSH only for host-level evidence or repairs that cannot be
-    performed through Kubernetes. Read first, identify the exact node and
-    impact, avoid fleet-wide destructive commands, and reflect persistent host
-    configuration in the appropriate tracked provisioning source.
+    Node SSH uses the dedicated locked-password `hermes-agent` OS account:
+    `ssh titan-04` (or any current Kubernetes node name). Host keys are pinned,
+    password fallback is disabled, and the account has no sudo, disk, runtime,
+    or Kubernetes-storage authority. Use it for unprivileged host evidence;
+    hand privileged host repair back to Brad and reflect persistent changes in
+    the tracked provisioning source.
 
     Treat every credential, SSH identity, host-trust record, and
     credential-bearing access-client state as runtime-only Vault data. Keep
@@ -395,7 +408,7 @@ data:
     <low|medium|high|xhigh> [model]` for a persistent override. The first native
     Codex worker requires one device-code login; refreshed provider credentials
     persist through Vault while client caches remain disposable. The owner
-    workspace includes cluster-admin Kubernetes
-    access plus `kubectl`, `flux`, `helm`, `kustomize`, `vault`, `sops`, `age`,
+    workspace includes scoped read-only Kubernetes diagnostics plus `kubectl`,
+    `flux`, `helm`, `kustomize`, `vault`, `sops`, `age`,
     `terraform`, `k9s`, `jq`, `yq`, `gh`, Git, SSH, Python, Node, the
     browser/computer tools, and the native provider CLIs.

services/hermes/agent-deployment.yaml

@@ -25,7 +25,7 @@ spec:
         ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
         ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
         ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
-        ai.bstein.dev/config-rev: "20260816-auth-health-quarantine-v6"
+        ai.bstein.dev/config-rev: "20260817-cli-auto-quota-routing-v1"
         prometheus.io/scrape: "true"
         prometheus.io/path: /metrics
         prometheus.io/port: "9010"
@@ -46,15 +46,10 @@ spec:
           {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
           {{ .Data.data.codex_auth_json }}
           {{- end }}
-        vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea
-        vault.hashicorp.com/agent-inject-template-gitea-token: |
-          {{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
-          {{ .Data.data.token }}
-          {{- end }}
-        vault.hashicorp.com/agent-inject-secret-gitea-username: kv/data/atlas/hermes/developer-gitea
-        vault.hashicorp.com/agent-inject-template-gitea-username: |
-          {{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
-          {{ .Data.data.username }}
+        vault.hashicorp.com/agent-inject-secret-jenkins-image-build-token: kv/data/atlas/hermes/developer-jenkins
+        vault.hashicorp.com/agent-inject-template-jenkins-image-build-token: |
+          {{- with secret "kv/data/atlas/hermes/developer-jenkins" -}}
+          {{ .Data.data.build_token }}
           {{- end }}
         vault.hashicorp.com/agent-inject-secret-node-ssh-private-key: kv/data/atlas/hermes/developer-ssh
         vault.hashicorp.com/agent-inject-template-node-ssh-private-key: |
@@ -186,9 +181,8 @@ spec:
                 chmod 0600 "${profile_env}"
                 chown 10000:10000 "${profile_env}"
               done
-              upsert_env GIT_ASKPASS /opt/coordinator/gitea_askpass.sh
               upsert_env GIT_TERMINAL_PROMPT 0
-              upsert_env GITEA_BASE_URL https://scm.bstein.dev
+              upsert_env HERMES_SCM_BROKER_URL http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081
               upsert_env JENKINS_BASE_URL http://jenkins.jenkins.svc.cluster.local:8080
               upsert_env ARIADNE_BASE_URL http://ariadne.maintenance.svc.cluster.local
               upsert_env VICTORIA_METRICS_URL http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428
@@ -663,6 +657,8 @@ spec:
             - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py}
             - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true}
             - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}
+            - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true}
+            - {name: scm-boundary, mountPath: /opt/scm, readOnly: true}
             - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true}
             - {name: tmp, mountPath: /tmp}
           startupProbe:
@@ -826,6 +822,8 @@ spec:
             - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py}
             - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true}
             - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}
+            - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true}
+            - {name: scm-boundary, mountPath: /opt/scm, readOnly: true}
             - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true}
             - {name: tmp, mountPath: /tmp}
             - {name: ttyd-index, mountPath: /ttyd-index, readOnly: true}
@@ -866,6 +864,8 @@ spec:
               . /opt/data/.env
               set +a
               exec /opt/hermes/.venv/bin/python /opt/coordinator/cli_lane_runner.py
+          ports:
+            - {name: lane-metrics, containerPort: 9011, protocol: TCP}
           env:
             - {name: HERMES_HOME, value: /opt/data}
             - {name: HERMES_AUTH_FILE, value: /runtime-access/hermes-auth.json}
@@ -875,6 +875,9 @@ spec:
             - {name: KUBECONFIG, value: /opt/data/home/.kube/config}
             - {name: PYTHONPATH, value: /opt/hermes}
             - {name: HERMES_CLI_LANE_CONCURRENCY, value: "2"}
+            - {name: HERMES_CLI_HEALTH_MAX_AGE_SECONDS, value: "60"}
+            - {name: HERMES_CLI_LANE_METRICS_PORT, value: "9011"}
+            - {name: HERMES_CLI_QUOTA_METRICS_URL, value: "http://127.0.0.1:9010/metrics"}
             - {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
             - {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
           securityContext:
@@ -894,7 +897,18 @@ spec:
             - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
             - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true}
             - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}
+            - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true}
+            - {name: scm-boundary, mountPath: /opt/scm, readOnly: true}
             - {name: tmp, mountPath: /tmp}
+          readinessProbe:
+            exec:
+              command:
+                - /opt/hermes/.venv/bin/python
+                - /opt/coordinator/cli_lane_capabilities.py
+            initialDelaySeconds: 2
+            periodSeconds: 5
+            timeoutSeconds: 5
+            failureThreshold: 3
           resources:
             requests: {cpu: 100m, memory: 256Mi}
             limits: {cpu: "2", memory: 6Gi}
@@ -1188,6 +1202,16 @@ spec:
               - {key: dashboard-api.py, path: dashboard/plugin_api.py}
               - {key: dashboard-index.js, path: dashboard/dist/index.js}
               - {key: dashboard-style.css, path: dashboard/dist/style.css}
+        - name: atlas-pr-skill
+          configMap:
+            name: hermes-atlas-pr-skill
+            items:
+              - {key: SKILL.md, path: SKILL.md}
+              - {key: openai.yaml, path: agents/openai.yaml}
+        - name: scm-boundary
+          configMap:
+            name: hermes-scm-boundary-v2
+            defaultMode: 0555
         - name: image-policy
           configMap:
             name: hermes-image-policy

services/hermes/agent-kubeconfig.yaml

@@ -11,9 +11,9 @@ users:
     user:
       tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
 contexts:
-  - name: atlas-owner
+  - name: atlas-observer
     context:
       cluster: atlas
       user: hermes-agent
       namespace: default
-current-context: atlas-owner
+current-context: atlas-observer

services/hermes/agent-rbac.yaml

@@ -1,16 +0,0 @@
-# services/hermes/agent-rbac.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: hermes-agent-cluster-admin
-  labels:
-    app.kubernetes.io/name: hermes-agent
-    app.kubernetes.io/part-of: hermes
-subjects:
-  - kind: ServiceAccount
-    name: hermes-agent
-    namespace: hermes
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: cluster-admin

services/hermes/execution-coordinator-patch.yaml

@@ -0,0 +1,66 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hermes-agent
+  namespace: hermes
+spec:
+  template:
+    metadata:
+      annotations:
+        vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
+        vault.hashicorp.com/agent-inject-template-execution-pool-key: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
+          {{- end }}
+    spec:
+      containers:
+        - name: cli-lane-runner
+          env:
+            - {name: HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY, value: "true"}
+            - {name: HERMES_CLI_LANE_CONCURRENCY, value: "1"}
+        - name: execution-pool-coordinator
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
+          imagePullPolicy: IfNotPresent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_coordinator.py]
+          env:
+            - {name: HERMES_HOME, value: /opt/data}
+            - {name: HOME, value: /opt/data/home}
+            - {name: PYTHONPATH, value: /opt/hermes}
+            - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
+            - {name: PATH, value: /opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: home, mountPath: /opt/data}
+            - {name: runtime-access, mountPath: /pool-access/execution-pool-key, subPath: execution-pool-key, readOnly: true}
+            - {name: execution-pool-code, mountPath: /opt/coordinator, readOnly: true}
+            - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
+            - {name: tmp, mountPath: /tmp}
+          ports:
+            - {name: execution-pool, containerPort: 9007, protocol: TCP}
+          startupProbe:
+            httpGet: {path: /ready, port: execution-pool}
+            periodSeconds: 5
+            failureThreshold: 60
+          readinessProbe:
+            httpGet: {path: /ready, port: execution-pool}
+            periodSeconds: 10
+          livenessProbe:
+            httpGet: {path: /ready, port: execution-pool}
+            initialDelaySeconds: 30
+            periodSeconds: 30
+          resources:
+            requests: {cpu: 50m, memory: 128Mi}
+            limits: {cpu: 500m, memory: 512Mi}
+      volumes:
+        - name: execution-pool-code
+          configMap:
+            name: hermes-execution-pool
+            defaultMode: 0555

services/hermes/execution-mediator.yaml

@@ -0,0 +1,476 @@
+# Privileged HMAC and SCM mediation runs outside the model Pods. Each mediator is
+# colocated with exactly one ordinal's RWO workspace but has a distinct network
+# identity, private HMAC derivation, and durable integrity state.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata: {name: hermes-execution-mediator-state-0, namespace: hermes}
+spec:
+  accessModes: [ReadWriteOnce]
+  storageClassName: astreae
+  resources: {requests: {storage: 1Gi}}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata: {name: hermes-execution-mediator-state-1, namespace: hermes}
+spec:
+  accessModes: [ReadWriteOnce]
+  storageClassName: astreae
+  resources: {requests: {storage: 1Gi}}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata: {name: hermes-execution-mediator-state-2, namespace: hermes}
+spec:
+  accessModes: [ReadWriteOnce]
+  storageClassName: astreae
+  resources: {requests: {storage: 1Gi}}
+---
+apiVersion: v1
+kind: Service
+metadata: {name: hermes-execution-mediator-0, namespace: hermes}
+spec:
+  selector: {app: hermes-execution-mediator, pool-ordinal: "0"}
+  ports: [{name: mediator, port: 9009, targetPort: mediator}]
+---
+apiVersion: v1
+kind: Service
+metadata: {name: hermes-execution-mediator-1, namespace: hermes}
+spec:
+  selector: {app: hermes-execution-mediator, pool-ordinal: "1"}
+  ports: [{name: mediator, port: 9009, targetPort: mediator}]
+---
+apiVersion: v1
+kind: Service
+metadata: {name: hermes-execution-mediator-2, namespace: hermes}
+spec:
+  selector: {app: hermes-execution-mediator, pool-ordinal: "2"}
+  ports: [{name: mediator, port: 9009, targetPort: mediator}]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hermes-execution-mediator-0
+  namespace: hermes
+  labels: {app: hermes-execution-mediator, pool-ordinal: "0"}
+spec:
+  replicas: 1
+  strategy: {type: Recreate}
+  selector:
+    matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
+  template:
+    metadata:
+      labels: {app: hermes-execution-mediator, pool-ordinal: "0"}
+      annotations:
+        ai.bstein.dev/config-rev: execution-pool-v2-mediated
+        ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: hermes-execution-worker
+        vault.hashicorp.com/agent-inject-containers: stage-mediator-access
+        vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
+        vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
+        vault.hashicorp.com/agent-inject-template-execution-pool-key: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
+          {{- end }}
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/agent-init-first: "true"
+        vault.hashicorp.com/agent-requests-cpu: 2m
+        vault.hashicorp.com/agent-requests-mem: 16Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
+    spec:
+      serviceAccountName: hermes-execution-worker
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      # Priority parity with the worker: a pressured node evicts the pair
+      # symmetrically instead of moving only the lower-priority half, and the
+      # pool never preempts anything else in the cluster.
+      priorityClassName: scavenger
+      securityContext:
+        fsGroup: 10000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile: {type: RuntimeDefault}
+      affinity:
+        # Colocation with the worker is a throughput preference, not a
+        # requirement. The shared workspace claim is ReadWriteMany, so this
+        # mediator still schedules and serves when its worker is Pending or
+        # placed elsewhere -- which is what makes a drain self-healing.
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - {key: kubernetes.io/arch, operator: In, values: [arm64]}
+                  - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
+                  - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    statefulset.kubernetes.io/pod-name: hermes-execution-worker-0
+                topologyKey: kubernetes.io/hostname
+      initContainers:
+        - name: stage-mediator-access
+          image: registry.bstein.dev/bstein/hermes-agent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
+          env:
+            - {name: HERMES_WORKER_ORDINAL, value: "0"}
+            - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsUser: 0
+            runAsGroup: 0
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: pool-access, mountPath: /pool-access}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+          resources:
+            requests: {cpu: 2m, memory: 16Mi}
+            limits: {cpu: 100m, memory: 64Mi}
+      containers:
+        - name: execution-mediator
+          image: registry.bstein.dev/bstein/hermes-agent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
+          env:
+            - {name: HERMES_WORKER_ORDINAL, value: "0"}
+            - {name: HERMES_WORKER_ROOT, value: /workspace}
+            - {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
+            - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
+            - {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
+          ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
+          startupProbe:
+            httpGet: {path: /ready, port: mediator}
+            periodSeconds: 5
+            failureThreshold: 60
+          readinessProbe:
+            httpGet: {path: /ready, port: mediator}
+            periodSeconds: 10
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: workspace, mountPath: /workspace}
+            - {name: scm-state, mountPath: /scm-state}
+            - {name: pool-access, mountPath: /pool-access, readOnly: true}
+            - {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+            - {name: tmp, mountPath: /tmp}
+          resources:
+            requests: {cpu: 2m, memory: 64Mi}
+            limits: {cpu: 250m, memory: 256Mi}
+      volumes:
+        - name: workspace
+          persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-0}
+        - name: scm-state
+          persistentVolumeClaim: {claimName: hermes-execution-mediator-state-0}
+        - name: pool-access
+          emptyDir: {medium: Memory, sizeLimit: 1Mi}
+        - name: scm-broker-client
+          configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
+        - name: coordinator
+          configMap: {name: hermes-execution-pool, defaultMode: 0555}
+        - name: tmp
+          emptyDir: {sizeLimit: 64Mi}
+        - name: vault-auth-token
+          projected:
+            defaultMode: 0600
+            sources:
+              - serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
+              - configMap:
+                  name: kube-root-ca.crt
+                  items: [{key: ca.crt, path: ca.crt}]
+              - downwardAPI:
+                  items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hermes-execution-mediator-1
+  namespace: hermes
+  labels: {app: hermes-execution-mediator, pool-ordinal: "1"}
+spec:
+  replicas: 1
+  strategy: {type: Recreate}
+  selector:
+    matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
+  template:
+    metadata:
+      labels: {app: hermes-execution-mediator, pool-ordinal: "1"}
+      annotations:
+        ai.bstein.dev/config-rev: execution-pool-v2-mediated
+        ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: hermes-execution-worker
+        vault.hashicorp.com/agent-inject-containers: stage-mediator-access
+        vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
+        vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
+        vault.hashicorp.com/agent-inject-template-execution-pool-key: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
+          {{- end }}
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/agent-init-first: "true"
+        vault.hashicorp.com/agent-requests-cpu: 2m
+        vault.hashicorp.com/agent-requests-mem: 16Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
+    spec:
+      serviceAccountName: hermes-execution-worker
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      # Priority parity with the worker: a pressured node evicts the pair
+      # symmetrically instead of moving only the lower-priority half, and the
+      # pool never preempts anything else in the cluster.
+      priorityClassName: scavenger
+      securityContext:
+        fsGroup: 10000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile: {type: RuntimeDefault}
+      affinity:
+        # Colocation with the worker is a throughput preference, not a
+        # requirement. The shared workspace claim is ReadWriteMany, so this
+        # mediator still schedules and serves when its worker is Pending or
+        # placed elsewhere -- which is what makes a drain self-healing.
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - {key: kubernetes.io/arch, operator: In, values: [arm64]}
+                  - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
+                  - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    statefulset.kubernetes.io/pod-name: hermes-execution-worker-1
+                topologyKey: kubernetes.io/hostname
+      initContainers:
+        - name: stage-mediator-access
+          image: registry.bstein.dev/bstein/hermes-agent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
+          env:
+            - {name: HERMES_WORKER_ORDINAL, value: "1"}
+            - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsUser: 0
+            runAsGroup: 0
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: pool-access, mountPath: /pool-access}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+          resources:
+            requests: {cpu: 2m, memory: 16Mi}
+            limits: {cpu: 100m, memory: 64Mi}
+      containers:
+        - name: execution-mediator
+          image: registry.bstein.dev/bstein/hermes-agent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
+          env:
+            - {name: HERMES_WORKER_ORDINAL, value: "1"}
+            - {name: HERMES_WORKER_ROOT, value: /workspace}
+            - {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
+            - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
+            - {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
+          ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
+          startupProbe:
+            httpGet: {path: /ready, port: mediator}
+            periodSeconds: 5
+            failureThreshold: 60
+          readinessProbe:
+            httpGet: {path: /ready, port: mediator}
+            periodSeconds: 10
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: workspace, mountPath: /workspace}
+            - {name: scm-state, mountPath: /scm-state}
+            - {name: pool-access, mountPath: /pool-access, readOnly: true}
+            - {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+            - {name: tmp, mountPath: /tmp}
+          resources:
+            requests: {cpu: 2m, memory: 64Mi}
+            limits: {cpu: 250m, memory: 256Mi}
+      volumes:
+        - name: workspace
+          persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-1}
+        - name: scm-state
+          persistentVolumeClaim: {claimName: hermes-execution-mediator-state-1}
+        - name: pool-access
+          emptyDir: {medium: Memory, sizeLimit: 1Mi}
+        - name: scm-broker-client
+          configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
+        - name: coordinator
+          configMap: {name: hermes-execution-pool, defaultMode: 0555}
+        - name: tmp
+          emptyDir: {sizeLimit: 64Mi}
+        - name: vault-auth-token
+          projected:
+            defaultMode: 0600
+            sources:
+              - serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
+              - configMap:
+                  name: kube-root-ca.crt
+                  items: [{key: ca.crt, path: ca.crt}]
+              - downwardAPI:
+                  items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hermes-execution-mediator-2
+  namespace: hermes
+  labels: {app: hermes-execution-mediator, pool-ordinal: "2"}
+spec:
+  replicas: 1
+  strategy: {type: Recreate}
+  selector:
+    matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
+  template:
+    metadata:
+      labels: {app: hermes-execution-mediator, pool-ordinal: "2"}
+      annotations:
+        ai.bstein.dev/config-rev: execution-pool-v2-mediated
+        ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: hermes-execution-worker
+        vault.hashicorp.com/agent-inject-containers: stage-mediator-access
+        vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
+        vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
+        vault.hashicorp.com/agent-inject-template-execution-pool-key: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
+          {{- end }}
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/agent-init-first: "true"
+        vault.hashicorp.com/agent-requests-cpu: 2m
+        vault.hashicorp.com/agent-requests-mem: 16Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
+    spec:
+      serviceAccountName: hermes-execution-worker
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      # Priority parity with the worker: a pressured node evicts the pair
+      # symmetrically instead of moving only the lower-priority half, and the
+      # pool never preempts anything else in the cluster.
+      priorityClassName: scavenger
+      securityContext:
+        fsGroup: 10000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile: {type: RuntimeDefault}
+      affinity:
+        # Colocation with the worker is a throughput preference, not a
+        # requirement. The shared workspace claim is ReadWriteMany, so this
+        # mediator still schedules and serves when its worker is Pending or
+        # placed elsewhere -- which is what makes a drain self-healing.
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - {key: kubernetes.io/arch, operator: In, values: [arm64]}
+                  - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
+                  - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    statefulset.kubernetes.io/pod-name: hermes-execution-worker-2
+                topologyKey: kubernetes.io/hostname
+      initContainers:
+        - name: stage-mediator-access
+          image: registry.bstein.dev/bstein/hermes-agent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
+          env:
+            - {name: HERMES_WORKER_ORDINAL, value: "2"}
+            - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsUser: 0
+            runAsGroup: 0
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: pool-access, mountPath: /pool-access}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+          resources:
+            requests: {cpu: 2m, memory: 16Mi}
+            limits: {cpu: 100m, memory: 64Mi}
+      containers:
+        - name: execution-mediator
+          image: registry.bstein.dev/bstein/hermes-agent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
+          env:
+            - {name: HERMES_WORKER_ORDINAL, value: "2"}
+            - {name: HERMES_WORKER_ROOT, value: /workspace}
+            - {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
+            - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
+            - {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
+          ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
+          startupProbe:
+            httpGet: {path: /ready, port: mediator}
+            periodSeconds: 5
+            failureThreshold: 60
+          readinessProbe:
+            httpGet: {path: /ready, port: mediator}
+            periodSeconds: 10
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: workspace, mountPath: /workspace}
+            - {name: scm-state, mountPath: /scm-state}
+            - {name: pool-access, mountPath: /pool-access, readOnly: true}
+            - {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+            - {name: tmp, mountPath: /tmp}
+          resources:
+            requests: {cpu: 2m, memory: 64Mi}
+            limits: {cpu: 250m, memory: 256Mi}
+      volumes:
+        - name: workspace
+          persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-2}
+        - name: scm-state
+          persistentVolumeClaim: {claimName: hermes-execution-mediator-state-2}
+        - name: pool-access
+          emptyDir: {medium: Memory, sizeLimit: 1Mi}
+        - name: scm-broker-client
+          configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
+        - name: coordinator
+          configMap: {name: hermes-execution-pool, defaultMode: 0555}
+        - name: tmp
+          emptyDir: {sizeLimit: 64Mi}
+        - name: vault-auth-token
+          projected:
+            defaultMode: 0600
+            sources:
+              - serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
+              - configMap:
+                  name: kube-root-ca.crt
+                  items: [{key: ca.crt, path: ca.crt}]
+              - downwardAPI:
+                  items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]

services/hermes/execution-worker-networkpolicy.yaml

@@ -0,0 +1,206 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-worker-isolation
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-execution-worker
+  policyTypes: [Ingress, Egress]
+  ingress: []
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels: {kubernetes.io/metadata.name: kube-system}
+          podSelector:
+            matchLabels: {k8s-app: kube-dns}
+      ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}]
+    - to:
+        - namespaceSelector:
+            matchLabels: {kubernetes.io/metadata.name: vault}
+          podSelector:
+            matchLabels: {app: vault}
+      ports: [{protocol: TCP, port: 8200}]
+    - to:
+        - podSelector:
+            matchLabels: {app: hermes-switchyard}
+      ports: [{protocol: TCP, port: 9005}]
+    - to:
+        - ipBlock:
+            cidr: 0.0.0.0/0
+            except:
+              - 10.0.0.0/8
+              - 100.64.0.0/10
+              - 127.0.0.0/8
+              - 169.254.0.0/16
+              - 172.16.0.0/12
+              - 192.168.0.0/16
+              - 224.0.0.0/4
+      ports: [{protocol: TCP, port: 443}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-worker-mediator-0
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-execution-worker
+      apps.kubernetes.io/pod-index: "0"
+  policyTypes: [Egress]
+  egress:
+    - to:
+        - podSelector:
+            matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
+      ports: [{protocol: TCP, port: 9009}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-worker-mediator-1
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-execution-worker
+      apps.kubernetes.io/pod-index: "1"
+  policyTypes: [Egress]
+  egress:
+    - to:
+        - podSelector:
+            matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
+      ports: [{protocol: TCP, port: 9009}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-worker-mediator-2
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-execution-worker
+      apps.kubernetes.io/pod-index: "2"
+  policyTypes: [Egress]
+  egress:
+    - to:
+        - podSelector:
+            matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
+      ports: [{protocol: TCP, port: 9009}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-pool-ingress
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels: {app: hermes-agent}
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels: {app: hermes-execution-mediator}
+      ports: [{protocol: TCP, port: 9007}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-mediator-isolation
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels: {app: hermes-execution-mediator}
+  policyTypes: [Ingress, Egress]
+  ingress: []
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels: {kubernetes.io/metadata.name: kube-system}
+          podSelector:
+            matchLabels: {k8s-app: kube-dns}
+      ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}]
+    - to:
+        - namespaceSelector:
+            matchLabels: {kubernetes.io/metadata.name: vault}
+          podSelector:
+            matchLabels: {app: vault}
+      ports: [{protocol: TCP, port: 8200}]
+    - to:
+        - podSelector:
+            matchLabels: {app: hermes-agent}
+      ports: [{protocol: TCP, port: 9007}]
+    - to:
+        - namespaceSelector:
+            matchLabels: {kubernetes.io/metadata.name: hermes-scm}
+          podSelector:
+            matchLabels: {app: hermes-scm-broker}
+      ports: [{protocol: TCP, port: 9081}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-mediator-worker-0
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app: hermes-execution-worker
+              apps.kubernetes.io/pod-index: "0"
+      ports: [{protocol: TCP, port: 9009}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-mediator-worker-1
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app: hermes-execution-worker
+              apps.kubernetes.io/pod-index: "1"
+      ports: [{protocol: TCP, port: 9009}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-mediator-worker-2
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app: hermes-execution-worker
+              apps.kubernetes.io/pod-index: "2"
+      ports: [{protocol: TCP, port: 9009}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-switchyard-ingress
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels: {app: hermes-switchyard}
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels: {app: hermes-execution-worker}
+      ports: [{protocol: TCP, port: 9005}]

services/hermes/execution-worker-rbac.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: hermes-execution-worker
+  namespace: hermes
+  labels:
+    app.kubernetes.io/name: hermes-execution-worker
+    app.kubernetes.io/part-of: hermes
+automountServiceAccountToken: false
+# Intentionally no RoleBinding or ClusterRoleBinding. The projected, bounded
+# token is mounted only into Vault-facing containers and grants no Kubernetes
+# API verbs to the model-facing execution worker.

services/hermes/execution-worker-statefulset.yaml

@@ -0,0 +1,291 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: hermes-execution-worker
+  namespace: hermes
+  labels:
+    app: hermes-execution-worker
+spec:
+  serviceName: hermes-execution-worker
+  replicas: 3
+  podManagementPolicy: Parallel
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: hermes-execution-worker
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      partition: 0
+  template:
+    metadata:
+      labels:
+        app: hermes-execution-worker
+        app.kubernetes.io/name: hermes-execution-worker
+        app.kubernetes.io/part-of: hermes
+      annotations:
+        ai.bstein.dev/role: fenced-execution-only
+        ai.bstein.dev/scm-boundary: mediated-pr14-broker-with-completion-gates
+        ai.bstein.dev/model-policy: Switchyard AUTO with cross-provider fallback
+        ai.bstein.dev/storage: separate durable RWO workspace and OAuth refresh ownership per ordinal
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: hermes-execution-worker
+        vault.hashicorp.com/agent-inject-containers: stage-worker-access
+        vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
+        vault.hashicorp.com/agent-inject-secret-claude-credentials-0: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-claude-credentials-0: "0600"
+        vault.hashicorp.com/agent-inject-template-claude-credentials-0: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ .Data.data.execution_worker_0_claude_credentials_json }}
+          {{- end }}
+        vault.hashicorp.com/agent-inject-secret-claude-credentials-1: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-claude-credentials-1: "0600"
+        vault.hashicorp.com/agent-inject-template-claude-credentials-1: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ .Data.data.execution_worker_1_claude_credentials_json }}
+          {{- end }}
+        vault.hashicorp.com/agent-inject-secret-claude-credentials-2: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-claude-credentials-2: "0600"
+        vault.hashicorp.com/agent-inject-template-claude-credentials-2: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ .Data.data.execution_worker_2_claude_credentials_json }}
+          {{- end }}
+        vault.hashicorp.com/agent-inject-secret-codex-auth-0: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-codex-auth-0: "0600"
+        vault.hashicorp.com/agent-inject-template-codex-auth-0: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ .Data.data.execution_worker_0_codex_auth_json }}
+          {{- end }}
+        vault.hashicorp.com/agent-inject-secret-codex-auth-1: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-codex-auth-1: "0600"
+        vault.hashicorp.com/agent-inject-template-codex-auth-1: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ .Data.data.execution_worker_1_codex_auth_json }}
+          {{- end }}
+        vault.hashicorp.com/agent-inject-secret-codex-auth-2: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-codex-auth-2: "0600"
+        vault.hashicorp.com/agent-inject-template-codex-auth-2: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ .Data.data.execution_worker_2_codex_auth_json }}
+          {{- end }}
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/agent-init-first: "true"
+        vault.hashicorp.com/agent-requests-cpu: 2m
+        vault.hashicorp.com/agent-requests-mem: 16Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
+    spec:
+      serviceAccountName: hermes-execution-worker
+      priorityClassName: scavenger
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      terminationGracePeriodSeconds: 30
+      securityContext:
+        fsGroup: 10000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - {key: kubernetes.io/arch, operator: In, values: [arm64]}
+                  - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
+                  - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - {key: node-role.kubernetes.io/accelerator, operator: Exists}
+            - weight: 50
+              preference:
+                matchExpressions:
+                  - {key: hardware, operator: In, values: [rpi5]}
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: hermes-execution-worker
+              topologyKey: kubernetes.io/hostname
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app: hermes-execution-worker
+      initContainers:
+        - name: stage-worker-access
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
+          imagePullPolicy: IfNotPresent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-worker]
+          env:
+            - {name: HERMES_WORKER_ROOT, value: /workspace}
+            - {name: HERMES_PROVIDER_ACCESS_ROOT, value: /provider-access}
+            - name: HERMES_WORKER_ORDINAL
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsUser: 0
+            runAsGroup: 0
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: workspace, mountPath: /workspace}
+            - {name: provider-access, mountPath: /provider-access}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+          resources:
+            requests: {cpu: 2m, memory: 32Mi}
+            limits: {cpu: 100m, memory: 64Mi}
+        - name: install-provider-clis
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
+          imagePullPolicy: IfNotPresent
+          command: [/bin/sh, -ec]
+          args:
+            # The tools volume is durable, so this install runs once per pinned
+            # version rather than on every Pod start. The version marker is only
+            # trusted when both binaries are actually present and executable, so
+            # a partial or pruned cache reinstalls instead of failing closed
+            # forever. The install itself is time-bounded so a stalled registry
+            # surfaces as a fast init failure with backoff instead of an
+            # open-ended wait that a Flux health timeout would have to absorb.
+            - |
+              tools=/worker-data/tools
+              marker="${tools}/.cli-versions-0.147.0-2.1.226"
+              mkdir -p "${tools}/bin"
+              if [ ! -f "${marker}" ] || [ ! -x "${tools}/bin/codex" ] \
+                  || [ ! -x "${tools}/bin/claude" ]; then
+                rm -f "${marker}"
+                timeout 900 npm install --global --omit=dev --no-audit --no-fund \
+                  --fetch-timeout=120000 --fetch-retries=2 --prefix "${tools}" \
+                  @openai/codex@0.147.0 @anthropic-ai/claude-code@2.1.226
+                touch "${marker}"
+              fi
+              test -x "${tools}/bin/codex"
+              test -x "${tools}/bin/claude"
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            runAsNonRoot: true
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: tools, mountPath: /worker-data/tools}
+          resources:
+            requests: {cpu: 5m, memory: 64Mi}
+            limits: {cpu: "1", memory: 1Gi}
+      containers:
+        - name: execution-worker
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
+          imagePullPolicy: IfNotPresent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_worker.py]
+          env:
+            - {name: HERMES_HOME, value: /worker-data}
+            - {name: HERMES_WORKER_ROOT, value: /workspace}
+            - {name: HOME, value: /worker-data/home}
+            - {name: CODEX_HOME, value: /provider-access/codex}
+            - {name: CLAUDE_CONFIG_DIR, value: /provider-access/claude}
+            - {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
+            - {name: PYTHONPATH, value: /opt/hermes}
+            - {name: PATH, value: /worker-data/tools/bin:/opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
+            - name: HERMES_WORKER_ORDINAL
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
+            - name: HERMES_WORKER_NODE
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          startupProbe:
+            exec:
+              command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json && test -w /provider-access/claude/.credentials.json"]
+            periodSeconds: 5
+            failureThreshold: 60
+          readinessProbe:
+            exec:
+              command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json"]
+            periodSeconds: 10
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: workspace, mountPath: /workspace}
+            - {name: worker-data, mountPath: /worker-data}
+            - {name: tools, mountPath: /worker-data/tools, readOnly: true}
+            - {name: provider-access, mountPath: /provider-access}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+            - {name: tmp, mountPath: /tmp}
+          resources:
+            requests: {cpu: 5m, memory: 128Mi, ephemeral-storage: 1Gi}
+            limits: {cpu: "2", memory: 4Gi, ephemeral-storage: 8Gi}
+      volumes:
+        - name: worker-data
+          emptyDir: {sizeLimit: 128Mi}
+        - name: coordinator
+          configMap:
+            name: hermes-execution-pool
+            defaultMode: 0555
+        - name: tmp
+          emptyDir: {sizeLimit: 2Gi}
+        - name: vault-auth-token
+          projected:
+            defaultMode: 0600
+            sources:
+              - serviceAccountToken:
+                  audience: vault
+                  expirationSeconds: 3600
+                  path: token
+              - configMap:
+                  name: kube-root-ca.crt
+                  items:
+                    - {key: ca.crt, path: ca.crt}
+              - downwardAPI:
+                  items:
+                    - {path: namespace, fieldRef: {fieldPath: metadata.namespace}}
+  volumeClaimTemplates:
+    # The workspace is shared with this ordinal's mediator, which performs every
+    # SCM operation on it. ReadWriteMany removes the cross-node Multi-Attach
+    # deadlock that a drain or preemption used to create between the two Pods,
+    # so each can be evicted and rescheduled independently. Longhorn already
+    # serves the hermes-chat tenant workspaces this way on the same class.
+    - metadata:
+        name: workspace
+        labels:
+          app: hermes-execution-worker
+      spec:
+        accessModes: [ReadWriteMany]
+        storageClassName: astreae
+        resources:
+          requests:
+            storage: 30Gi
+    # Provider CLIs are installed once per pinned version onto a durable volume
+    # instead of being re-downloaded into an emptyDir on every Pod start.
+    - metadata:
+        name: tools
+        labels:
+          app: hermes-execution-worker
+      spec:
+        accessModes: [ReadWriteOnce]
+        storageClassName: astreae
+        resources:
+          requests:
+            storage: 2Gi
+    # OAuth refresh ownership is single-mounter by design: keep it ReadWriteOnce.
+    - metadata:
+        name: provider-access
+        labels:
+          app: hermes-execution-worker
+      spec:
+        accessModes: [ReadWriteOnce]
+        storageClassName: astreae
+        resources:
+          requests:
+            storage: 1Gi

services/hermes/kustomization.yaml

@@ -7,13 +7,13 @@ images:
     digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7
 resources:
   - namespace.yaml
+  - scm-common
   - vault-serviceaccount.yaml
   - configmap.yaml
   - agent-configmap.yaml
   - chat-configmap.yaml
   - switchyard-configmap.yaml
   - rbac.yaml
-  - agent-rbac.yaml
   - node-ssh-access.yaml
   - pvc.yaml
   - switchyard-pvc.yaml
@@ -41,6 +41,13 @@ resources:
   - oauth2-proxy.yaml
   - agent-certificate.yaml
   - agent-ingress.yaml
+  - execution-worker-rbac.yaml
+  - execution-worker-statefulset.yaml
+  - execution-mediator.yaml
+  - execution-worker-networkpolicy.yaml
+
+patches:
+  - path: execution-coordinator-patch.yaml
 
 configMapGenerator:
   - name: hermes-chat-oauth-templates
@@ -55,13 +62,60 @@ configMapGenerator:
       - OPERATOR-RUNBOOK.md=NOTES.md
     options:
       disableNameSuffixHash: true
+  - name: hermes-execution-pool
+    namespace: hermes
+    files:
+      - cli_lane_board.py=scripts/cli_lane_board.py
+      - cli_lane_capabilities.py=scripts/cli_lane_capabilities.py
+      - cli_lane_config.py=scripts/cli_lane_config.py
+      - cli_lane_dispatch.py=scripts/cli_lane_dispatch.py
+      - cli_lane_evidence.py=scripts/cli_lane_evidence.py
+      - cli_lane_execution.py=scripts/cli_lane_execution.py
+      - cli_lane_files.py=scripts/cli_lane_files.py
+      - cli_lane_finalization.py=scripts/cli_lane_finalization.py
+      - cli_lane_goal.py=scripts/cli_lane_goal.py
+      - cli_lane_prompt.py=scripts/cli_lane_prompt.py
+      - cli_lane_provider.py=scripts/cli_lane_provider.py
+      - cli_lane_quarantine.py=scripts/cli_lane_quarantine.py
+      - cli_lane_records.py=scripts/cli_lane_records.py
+      - cli_lane_recovery.py=scripts/cli_lane_recovery.py
+      - cli_lane_retention.py=scripts/cli_lane_retention.py
+      - cli_lane_routing.py=scripts/cli_lane_routing.py
+      - cli_lane_runner.py=scripts/cli_lane_runner.py
+      - execution_pool_protocol.py=scripts/execution_pool_protocol.py
+      - execution_pool_store.py=scripts/execution_pool_store.py
+      - execution_pool_project.py=scripts/execution_pool_project.py
+      - execution_pool_coordinator.py=scripts/execution_pool_coordinator.py
+      - execution_pool_maintenance.py=scripts/execution_pool_maintenance.py
+      - execution_pool_server.py=scripts/execution_pool_server.py
+      - execution_pool_client.py=scripts/execution_pool_client.py
+      - execution_pool_worker.py=scripts/execution_pool_worker.py
+      - execution_pool_scm.py=scripts/execution_pool_scm.py
+      - gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py
+      - scm_broker_client.py=scm-common/scripts/scm_broker_client.py
+      - stage_runtime_access.py=scripts/stage_runtime_access.py
   - name: hermes-coordinator
     namespace: hermes
     files:
       - ai_usage_exporter.py=scripts/ai_usage_exporter.py
       - claude=scripts/claude
       - claude_command_policy.py=scripts/claude_command_policy.py
+      - cli_lane_board.py=scripts/cli_lane_board.py
+      - cli_lane_capabilities.py=scripts/cli_lane_capabilities.py
+      - cli_lane_config.py=scripts/cli_lane_config.py
+      - cli_lane_dispatch.py=scripts/cli_lane_dispatch.py
+      - cli_lane_evidence.py=scripts/cli_lane_evidence.py
+      - cli_lane_execution.py=scripts/cli_lane_execution.py
+      - cli_lane_files.py=scripts/cli_lane_files.py
+      - cli_lane_finalization.py=scripts/cli_lane_finalization.py
       - cli_lane_goal.py=scripts/cli_lane_goal.py
+      - cli_lane_prompt.py=scripts/cli_lane_prompt.py
+      - cli_lane_provider.py=scripts/cli_lane_provider.py
+      - cli_lane_quarantine.py=scripts/cli_lane_quarantine.py
+      - cli_lane_records.py=scripts/cli_lane_records.py
+      - cli_lane_recovery.py=scripts/cli_lane_recovery.py
+      - cli_lane_retention.py=scripts/cli_lane_retention.py
+      - cli_lane_routing.py=scripts/cli_lane_routing.py
       - cli_lane_runner.py=scripts/cli_lane_runner.py
       - codex=scripts/codex
       - configure_agent_clients.py=scripts/configure_agent_clients.py
@@ -69,14 +123,13 @@ configMapGenerator:
       - classifier_broker.py=scripts/classifier_broker.py
       - claude_oauth_broker.py=scripts/claude_oauth_broker.py
       - worker_route_broker.py=scripts/worker_route_broker.py
-      - gitea_api.py=scripts/gitea_api.py
-      - gitea_askpass.sh=scripts/gitea_askpass.sh
       - hermes_coordinator.py=scripts/hermes_coordinator.py
       - hermes_model_routing.py=scripts/hermes_model_routing.py
       - hermes_stt_client.py=scripts/hermes_stt_client.py
       - image_broker.py=scripts/image_broker.py
       - install_agent_tools.sh=scripts/install_agent_tools.sh
       - jenkins_build_evidence.py=scripts/jenkins_build_evidence.py
+      - jenkins_image_build_trigger.py=scripts/jenkins_image_build_trigger.py
       - kanban_status_recovery.py=scripts/kanban_status_recovery.py
       - migrate_herdr_state.py=scripts/migrate_herdr_state.py
       - migrate_api_session_lineage.py=scripts/migrate_api_session_lineage.py
@@ -104,6 +157,15 @@ configMapGenerator:
       - config=agent-kubeconfig.yaml
     options:
       disableNameSuffixHash: true
+  - name: hermes-node-account-hardener
+    namespace: hermes
+    files:
+      - node_account_hardening.py=scripts/node_account_hardening.py
+      - node_account_audit.py=scripts/node_account_audit.py
+      - node_account_io.py=scripts/node_account_io.py
+      - node_polkit_audit.py=scripts/node_polkit_audit.py
+    options:
+      disableNameSuffixHash: true
   - name: hermes-auto-router-plugin
     namespace: hermes
     files:
@@ -158,3 +220,10 @@ configMapGenerator:
       - alert-review.md=skills/tune-atlas-alerts/references/alert-review.md
     options:
       disableNameSuffixHash: true
+  - name: hermes-atlas-pr-skill
+    namespace: hermes
+    files:
+      - SKILL.md=skills/manage-atlas-pull-requests/SKILL.md
+      - openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml
+    options:
+      disableNameSuffixHash: true

services/hermes/networkpolicy.yaml

@@ -119,11 +119,66 @@ spec:
               app: server
       ports:
         - {protocol: TCP, port: 9010}
-  # agent.hermes.bstein.dev is an owner-only engineering workstation. The
-  # browser boundary remains OAuth-protected, while its workers need to reach
-  # every cluster namespace, Atlas LAN service, and hosted provider endpoint.
+        - {protocol: TCP, port: 9011}
   egress:
-    - {}
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      ports:
+        - {protocol: UDP, port: 53}
+        - {protocol: TCP, port: 53}
+    - to:
+        - namespaceSelector:
+            matchExpressions:
+              - key: kubernetes.io/metadata.name
+                operator: NotIn
+                values: [gitea, hermes-scm]
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: hermes-scm
+          podSelector:
+            matchLabels:
+              app: hermes-scm-broker
+      ports:
+        - {protocol: TCP, port: 9081}
+    - to:
+        - ipBlock:
+            cidr: 10.43.0.1/32
+      ports:
+        - {protocol: TCP, port: 443}
+    - to:
+        - ipBlock:
+            cidr: 192.168.0.0/16
+            except:
+              - 192.168.22.9/32
+    - to:
+        - ipBlock:
+            cidr: 0.0.0.0/0
+            except:
+              - 10.0.0.0/8
+              - 100.64.0.0/10
+              - 127.0.0.0/8
+              - 169.254.0.0/16
+              - 172.16.0.0/12
+              - 192.168.0.0/16
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-node-ssh-access-isolation
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-node-ssh-access
+  policyTypes: [Ingress, Egress]
+  ingress: []
+  egress: []
 ---
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy

services/hermes/node-ssh-access.yaml

@@ -45,32 +45,16 @@ spec:
         - operator: Exists
       containers:
         - name: key-reconciler
-          image: busybox:1.37
+          # The pinned index supports the cluster's amd64 and arm64 nodes.
+          image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df
           imagePullPolicy: IfNotPresent
           command: [/bin/sh, -ec]
           args:
             - |
-              reconcile() {
-                key="$(cat /vault/secrets/node-ssh-public-key)"
-                found=0
-                for user in atlas oceanus; do
-                  home="/host-home/${user}"
-                  [ -d "${home}" ] || continue
-                  found=1
-                  identity="$(awk -F: -v name="${user}" '$1 == name {print $3 ":" $4; exit}' /host-etc/passwd)"
-                  [ -n "${identity}" ] || identity="$(stat -c %u:%g "${home}")"
-                  uid="${identity%%:*}"
-                  gid="${identity##*:}"
-                  install -d -m 0700 -o "${uid}" -g "${gid}" "${home}/.ssh"
-                  touch "${home}/.ssh/authorized_keys"
-                  grep -qxF "${key}" "${home}/.ssh/authorized_keys" || printf '%s\n' "${key}" >> "${home}/.ssh/authorized_keys"
-                  chown "${uid}:${gid}" "${home}/.ssh/authorized_keys"
-                  chmod 0600 "${home}/.ssh/authorized_keys"
-                done
-                [ "${found}" = 1 ] || { echo "no supported node SSH account found" >&2; return 1; }
-              }
               while true; do
-                reconcile
+                /usr/local/bin/python \
+                  /opt/node-hardener/node_account_hardening.py \
+                  --public-key-file /vault/secrets/node-ssh-public-key
                 sleep 300
               done
           securityContext:
@@ -83,8 +67,21 @@ spec:
           volumeMounts:
             - name: host-home
               mountPath: /host-home
-            - name: host-passwd
-              mountPath: /host-etc/passwd
+            - name: host-etc
+              mountPath: /host-etc
+            - name: host-polkit-share
+              mountPath: /host-polkit-share
+              readOnly: true
+            - name: host-k3s
+              mountPath: /host-k3s
+            - name: host-kubelet
+              mountPath: /host-kubelet
+            - name: host-run-k3s
+              mountPath: /host-run-k3s
+            - name: host-run-containerd
+              mountPath: /host-run-containerd
+            - name: coordinator
+              mountPath: /opt/node-hardener
               readOnly: true
             - name: vault-secrets
               mountPath: /vault/secrets
@@ -94,19 +91,43 @@ spec:
           resources:
             requests:
               cpu: 5m
-              memory: 8Mi
+              memory: 32Mi
             limits:
               cpu: 50m
-              memory: 32Mi
+              memory: 128Mi
       volumes:
         - name: host-home
           hostPath:
             path: /home
             type: Directory
-        - name: host-passwd
+        - name: host-etc
           hostPath:
-            path: /etc/passwd
-            type: File
+            path: /etc
+            type: Directory
+        - name: host-polkit-share
+          hostPath:
+            path: /usr/share/polkit-1
+            type: Directory
+        - name: host-k3s
+          hostPath:
+            path: /var/lib/rancher/k3s
+            type: Directory
+        - name: host-kubelet
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: host-run-k3s
+          hostPath:
+            path: /run/k3s
+            type: Directory
+        - name: host-run-containerd
+          hostPath:
+            path: /run/containerd
+            type: Directory
+        - name: coordinator
+          configMap:
+            name: hermes-node-account-hardener
+            defaultMode: 0555
         - name: vault-secrets
           csi:
             driver: secrets-store.csi.k8s.io

services/hermes/oauth2-proxy.yaml

@@ -16,6 +16,9 @@ metadata:
 spec:
   selector:
     app: hermes-agent
+  # The agent dashboard and terminal must stay reachable while the
+  # cli-lane-runner readiness probe holds the shared pod NotReady.
+  publishNotReadyAddresses: true
   ports:
     - {name: http, port: 80, targetPort: auth-http}
 ---

services/hermes/scm-common/kustomization.yaml

@@ -0,0 +1,17 @@
+# services/hermes/scm-common/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+configMapGenerator:
+  - name: hermes-scm-boundary-v2
+    files:
+      - deadline_http.py=scripts/deadline_http.py
+      - git_pack_objects.py=scripts/git_pack_objects.py
+      - gitea_api.py=scripts/gitea_api.py
+      - gitea_api_policy.py=scripts/gitea_api_policy.py
+      - receive_pack_scan.py=scripts/receive_pack_scan.py
+      - scm_broker.py=scripts/scm_broker.py
+      - scm_broker_io.py=scripts/scm_broker_io.py
+      - scm_broker_server.py=scripts/scm_broker_server.py
+      - scm_broker_client.py=scripts/scm_broker_client.py
+    options:
+      disableNameSuffixHash: true

services/hermes/scm-common/scripts/deadline_http.py

@@ -0,0 +1,344 @@
+#!/usr/bin/env python3
+"""Absolute wall-clock deadlines for every outbound Hermes HTTP exchange.
+
+Small control exchanges run inside a killable helper process, so connect,
+send, read, and DNS all share one hard bound that no blocking socket state
+can outlive.  Large streaming Git exchanges cannot cross a process boundary;
+they instead register their connection with a watchdog that force-closes the
+socket when the same kind of absolute deadline expires.
+"""
+
+from __future__ import annotations
+
+import base64
+import contextlib
+import email.message
+import http.client
+import io
+import json
+import os
+import signal
+import socket
+import subprocess
+import sys
+import threading
+import time
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+
+from gitea_api_policy import PolicyError
+
+MAX_CONTROL_BYTES = 128 * 1024
+MAX_PROTOCOL_BYTES = 4 * 1024 * 1024
+MAX_STREAM_SECONDS = 900.0
+READ_CHUNK = 64 * 1024
+
+
+class _RejectRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        raise urllib.error.URLError("redirect rejected")
+
+
+@dataclass(frozen=True)
+class Result:
+    """Minimal HTTP evidence returned across the process boundary."""
+
+    status: int
+    content_type: str
+    body: bytes
+
+
+def _read_response(response, maximum: int, deadline: float) -> bytes:
+    chunks: list[bytes] = []
+    total = 0
+    while True:
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            raise TimeoutError
+        raw = getattr(getattr(response, "fp", None), "raw", None)
+        sock = getattr(raw, "_sock", raw)
+        if hasattr(sock, "settimeout"):
+            sock.settimeout(remaining)
+        chunk = response.read(min(READ_CHUNK, maximum + 1 - total))
+        if not chunk:
+            break
+        chunks.append(chunk)
+        total += len(chunk)
+        if total > maximum:
+            raise ValueError("response too large")
+    return b"".join(chunks)
+
+
+def _child_exchange(control: dict[str, object]) -> Result:
+    url = control.get("url")
+    method = control.get("method")
+    headers = control.get("headers")
+    encoded = control.get("body")
+    maximum = control.get("maximum")
+    timeout = control.get("timeout")
+    if (
+        not isinstance(url, str)
+        or not isinstance(method, str)
+        or not isinstance(headers, dict)
+        or not all(isinstance(key, str) and isinstance(value, str) for key, value in headers.items())
+        or not isinstance(encoded, str)
+        or not isinstance(maximum, int)
+        or not 0 <= maximum <= 2 * 1024 * 1024
+        or not isinstance(timeout, (int, float))
+        or not 0 < timeout <= 120
+    ):
+        raise ValueError("invalid control")
+    body = base64.b64decode(encoded, validate=True) if encoded else None
+    if body is not None and len(body) > MAX_CONTROL_BYTES:
+        raise ValueError("request too large")
+    deadline = time.monotonic() + float(timeout)
+    request = urllib.request.Request(
+        url, data=body, method=method, headers=headers  # type: ignore[arg-type]
+    )
+    opener = urllib.request.build_opener(_RejectRedirect())
+    try:
+        response = opener.open(request, timeout=max(deadline - time.monotonic(), 0.001))
+    except urllib.error.HTTPError as error:
+        response = error
+    with response:
+        status = getattr(response, "status", response.getcode())
+        content_type = response.headers.get_content_type()
+        value = _read_response(response, maximum, deadline)
+    return Result(int(status), str(content_type), value)
+
+
+def _child_main() -> int:
+    try:
+        raw = sys.stdin.buffer.read(MAX_CONTROL_BYTES + 1)
+        if len(raw) > MAX_CONTROL_BYTES:
+            raise ValueError("control too large")
+        control = json.loads(raw)
+        if not isinstance(control, dict):
+            raise ValueError("invalid control")
+        result = _child_exchange(control)
+        output = {
+            "ok": True,
+            "status": result.status,
+            "content_type": result.content_type,
+            "body": base64.b64encode(result.body).decode("ascii"),
+        }
+    except Exception:
+        output = {"ok": False}
+    sys.stdout.write(json.dumps(output, separators=(",", ":")))
+    return 0 if output["ok"] else 1
+
+
+def _terminate(process: subprocess.Popen[bytes]) -> None:
+    with contextlib.suppress(ProcessLookupError):
+        os.killpg(process.pid, signal.SIGKILL)
+    process.communicate()
+
+
+def exchange(
+    request: urllib.request.Request,
+    *,
+    maximum: int,
+    timeout: float,
+    popen=subprocess.Popen,
+) -> Result:
+    """Execute a whole request under one monotonic deadline and size limit."""
+    if not 0 < timeout <= 120 or not 0 <= maximum <= 2 * 1024 * 1024:
+        raise PolicyError("HTTP control request bounds are invalid")
+    body = request.data or b""
+    if not isinstance(body, bytes) or len(body) > MAX_CONTROL_BYTES:
+        raise PolicyError("HTTP control request exceeds the safe size limit")
+    control = json.dumps(
+        {
+            "url": request.full_url,
+            "method": request.get_method(),
+            "headers": dict(request.header_items()),
+            "body": base64.b64encode(body).decode("ascii"),
+            "maximum": maximum,
+            "timeout": timeout,
+        },
+        separators=(",", ":"),
+    ).encode()
+    deadline = time.monotonic() + timeout
+    process = popen(
+        [sys.executable, os.path.realpath(__file__), "--child"],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        start_new_session=True,
+        env={"LANG": "C.UTF-8", "PATH": "/usr/bin:/bin"},
+    )
+    try:
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            raise subprocess.TimeoutExpired(process.args, timeout)
+        output, _error = process.communicate(control, timeout=remaining)
+    except subprocess.TimeoutExpired as exc:
+        _terminate(process)
+        raise PolicyError("HTTP control request deadline exceeded") from exc
+    if process.returncode != 0 or len(output) > MAX_PROTOCOL_BYTES:
+        raise PolicyError("HTTP control request failed")
+    try:
+        result = json.loads(output)
+        if not isinstance(result, dict) or result.get("ok") is not True:
+            raise ValueError
+        status = result["status"]
+        content_type = result["content_type"]
+        encoded = result["body"]
+        if not isinstance(status, int) or not isinstance(content_type, str) or not isinstance(encoded, str):
+            raise ValueError
+        value = base64.b64decode(encoded, validate=True)
+    except (KeyError, ValueError, TypeError, json.JSONDecodeError) as exc:
+        raise PolicyError("HTTP control helper returned invalid evidence") from exc
+    if len(value) > maximum:
+        raise PolicyError("HTTP control response exceeds the safe size limit")
+    return Result(status, content_type, value)
+
+
+class _AdaptedResponse:
+    """Present one finished exchange through the urlopen reader interface."""
+
+    def __init__(self, result: Result):
+        self.status = result.status
+        self.headers = email.message.Message()
+        self.headers["Content-Type"] = result.content_type
+        self._stream = io.BytesIO(result.body)
+
+    def read(self, limit: int = -1) -> bytes:
+        return self._stream.read(limit)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_excinfo: object) -> bool:
+        return False
+
+
+def open_bounded(
+    request: urllib.request.Request,
+    *,
+    maximum: int,
+    timeout: float,
+    popen=subprocess.Popen,
+) -> _AdaptedResponse:
+    """Open one control exchange whose whole lifetime shares one deadline."""
+    return _AdaptedResponse(
+        exchange(request, maximum=maximum, timeout=timeout, popen=popen)
+    )
+
+
+def _resolve_within(host: str, port: int, timeout: float) -> None:
+    """Resolve one host under a hard timeout the socket layer never covers.
+
+    ``getaddrinfo`` ignores socket timeouts and runs before any socket
+    exists, so a slow or hostile resolver would otherwise outlive the whole
+    stream deadline.  Running it in a joinable worker bounds resolution by the
+    same wall clock; a stuck lookup leaks only one daemon thread.
+    """
+    outcome: dict[str, object] = {}
+
+    def run() -> None:
+        try:
+            outcome["value"] = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
+        except OSError as exc:
+            outcome["error"] = exc
+
+    worker = threading.Thread(target=run, daemon=True)
+    worker.start()
+    worker.join(timeout)
+    if worker.is_alive():
+        raise PolicyError("HTTP stream DNS resolution deadline exceeded")
+    if "error" in outcome:
+        raise outcome["error"]  # type: ignore[misc]
+
+
+class StreamDeadline:
+    """Force-close tracked connections once an absolute deadline passes."""
+
+    def __init__(self, timeout: float, *, timer=threading.Timer):
+        if not 0 < timeout <= MAX_STREAM_SECONDS:
+            raise PolicyError("HTTP stream deadline bounds are invalid")
+        self._deadline = time.monotonic() + timeout
+        self._lock = threading.Lock()
+        self._connections: list[http.client.HTTPConnection] = []
+        self.expired = False
+        self._timer = timer(timeout, self._expire)
+        self._timer.daemon = True
+        self._timer.start()
+
+    def remaining(self) -> float:
+        value = self._deadline - time.monotonic()
+        if value <= 0 or self.expired:
+            raise PolicyError("HTTP stream deadline exceeded")
+        return value
+
+    def _expire(self) -> None:
+        with self._lock:
+            self.expired = True
+            connections = list(self._connections)
+        for connection in connections:
+            # Shut the raw socket down first so a blocked send or recv wakes
+            # immediately; close alone leaves peers waiting on the old fd.
+            sock = getattr(connection, "sock", None)
+            if sock is not None:
+                with contextlib.suppress(OSError):
+                    sock.shutdown(socket.SHUT_RDWR)
+            connection.close()
+
+    def _track(self, connection: http.client.HTTPConnection):
+        with self._lock:
+            expired = self.expired
+            if not expired:
+                self._connections.append(connection)
+        if expired:
+            connection.close()
+            raise PolicyError("HTTP stream deadline exceeded")
+        return connection
+
+    def _bounded_connection(self, connection: http.client.HTTPConnection):
+        """Register one connection and bound its DNS phase by the deadline.
+
+        The watchdog can only close a live socket, but ``connect`` resolves
+        the host before any socket exists.  Wrapping ``connect`` runs that
+        resolution under the same absolute deadline first, then hands off to
+        the real connect for the socket phases the watchdog already covers.
+        """
+        original_connect = connection.connect
+
+        def connect() -> None:
+            _resolve_within(connection.host, connection.port, self.remaining())
+            original_connect()
+
+        connection.connect = connect  # type: ignore[method-assign]
+        return self._track(connection)
+
+    def cancel(self) -> None:
+        self._timer.cancel()
+
+    def handlers(self) -> tuple[urllib.request.BaseHandler, ...]:
+        """Build urllib handlers whose connections obey this deadline."""
+        deadline = self
+
+        class GuardedHTTPHandler(urllib.request.HTTPHandler):
+            def http_open(self, req):
+                return self.do_open(
+                    lambda host, **kwargs: deadline._bounded_connection(
+                        http.client.HTTPConnection(host, **kwargs)
+                    ),
+                    req,
+                )
+
+        class GuardedHTTPSHandler(urllib.request.HTTPSHandler):
+            def https_open(self, req):
+                return self.do_open(
+                    lambda host, **kwargs: deadline._bounded_connection(
+                        http.client.HTTPSConnection(host, **kwargs)
+                    ),
+                    req,
+                )
+
+        return (GuardedHTTPHandler(), GuardedHTTPSHandler())
+
+
+if __name__ == "__main__":
+    raise SystemExit(_child_main() if sys.argv[1:] == ["--child"] else 2)

services/hermes/scm-common/scripts/git_pack_objects.py

@@ -0,0 +1,308 @@
+#!/usr/bin/env python3
+"""Bounded Git pack inflation and delta resolution for quarantine scanning.
+
+Pushed objects arrive zlib-compressed inside a packfile, so any content
+screening must inflate the real payloads first.  This module unpacks one
+self-contained pack under explicit object, size, and checksum bounds and
+resolves every delta against bases inside the same pack.  Thin packs are
+rejected outright: a delta against a base the broker cannot see would leave
+part of the pushed content unscanned.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import tempfile
+import zlib
+from typing import BinaryIO
+
+from gitea_api_policy import PolicyError
+
+OBJECT_TYPE_NAMES = {1: b"commit", 2: b"tree", 3: b"blob", 4: b"tag"}
+OFS_DELTA = 6
+REF_DELTA = 7
+MAX_PACK_OBJECTS = 50_000
+MAX_OBJECT_BYTES = 32 * 1024 * 1024
+MAX_TOTAL_BYTES = 256 * 1024 * 1024
+MAX_RESOLVE_PASSES = 64
+SPOOL_MEMORY_LIMIT = 256 * 1024
+READ_CHUNK = 64 * 1024
+
+
+def _read_exact(stream: BinaryIO, size: int) -> bytes:
+    value = stream.read(size)
+    if len(value) != size:
+        raise PolicyError("Git pack data ended early")
+    return value
+
+
+def _object_header(stream: BinaryIO) -> tuple[int, int]:
+    """Parse one object's type and declared inflated size."""
+    byte = _read_exact(stream, 1)[0]
+    type_code = (byte >> 4) & 0x7
+    size = byte & 0x0F
+    shift = 4
+    while byte & 0x80:
+        if shift > 60:
+            raise PolicyError("Git object header is invalid")
+        byte = _read_exact(stream, 1)[0]
+        size |= (byte & 0x7F) << shift
+        shift += 7
+    return type_code, size
+
+
+def _delta_distance(stream: BinaryIO) -> int:
+    """Parse one ofs-delta base distance."""
+    byte = _read_exact(stream, 1)[0]
+    value = byte & 0x7F
+    while byte & 0x80:
+        byte = _read_exact(stream, 1)[0]
+        value = ((value + 1) << 7) | (byte & 0x7F)
+        if value > 1 << 48:
+            raise PolicyError("Git delta base distance is invalid")
+    return value
+
+
+def _inflate(stream: BinaryIO, declared: int) -> BinaryIO:
+    """Inflate one object into a quarantine spool, bounded by its header."""
+    spool = tempfile.SpooledTemporaryFile(  # noqa: SIM115 - caller owns returned spool
+        max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
+    )
+    decompressor = zlib.decompressobj()
+    produced = 0
+    try:
+        while not decompressor.eof:
+            data = stream.read(READ_CHUNK)
+            if not data:
+                raise PolicyError("Git pack object data ended early")
+            while True:
+                value = decompressor.decompress(data, READ_CHUNK)
+                produced += len(value)
+                if produced > declared:
+                    raise PolicyError("Git object size does not match its header")
+                spool.write(value)
+                data = decompressor.unconsumed_tail
+                if decompressor.eof or not data:
+                    break
+        if decompressor.unused_data:
+            stream.seek(-len(decompressor.unused_data), 1)
+        if produced != declared:
+            raise PolicyError("Git object size does not match its header")
+        spool.seek(0)
+        return spool
+    except zlib.error as exc:
+        spool.close()
+        raise PolicyError("Git pack object data is corrupt") from exc
+    except Exception:
+        spool.close()
+        raise
+
+
+def _delta_size(delta: bytes, position: int) -> tuple[int, int]:
+    size = 0
+    shift = 0
+    while True:
+        if position >= len(delta) or shift > 60:
+            raise PolicyError("Git delta header is invalid")
+        byte = delta[position]
+        position += 1
+        size |= (byte & 0x7F) << shift
+        shift += 7
+        if not byte & 0x80:
+            return size, position
+
+
+def _apply_delta(base: bytes, delta: bytes) -> bytes:
+    """Apply one Git delta and require exact declared sizes throughout."""
+    source_size, position = _delta_size(delta, 0)
+    target_size, position = _delta_size(delta, position)
+    if source_size != len(base) or target_size > MAX_OBJECT_BYTES:
+        raise PolicyError("Git delta sizes are invalid")
+    parts: list[bytes] = []
+    produced = 0
+    while position < len(delta):
+        opcode = delta[position]
+        position += 1
+        if opcode & 0x80:
+            operands = [bit for bit in (1, 2, 4, 8, 16, 32, 64) if opcode & bit]
+            if position + len(operands) > len(delta):
+                raise PolicyError("Git delta copy is truncated")
+            offset = 0
+            size = 0
+            for shift, bit in enumerate((1, 2, 4, 8)):
+                if opcode & bit:
+                    offset |= delta[position] << (8 * shift)
+                    position += 1
+            for shift, bit in enumerate((16, 32, 64)):
+                if opcode & bit:
+                    size |= delta[position] << (8 * shift)
+                    position += 1
+            size = size or 0x10000
+            if offset + size > len(base):
+                raise PolicyError("Git delta copy is out of bounds")
+            parts.append(base[offset : offset + size])
+            produced += size
+        elif opcode:
+            if position + opcode > len(delta):
+                raise PolicyError("Git delta insert is truncated")
+            parts.append(delta[position : position + opcode])
+            position += opcode
+            produced += opcode
+        else:
+            raise PolicyError("Git delta instruction is invalid")
+        if produced > target_size:
+            raise PolicyError("Git delta result exceeds its declared size")
+    if produced != target_size:
+        raise PolicyError("Git delta result size is invalid")
+    return b"".join(parts)
+
+
+def _payload_sha(type_code: int, payload: BinaryIO, size: int) -> bytes:
+    digest = hashlib.sha1(OBJECT_TYPE_NAMES[type_code] + b" %d\x00" % size)
+    payload.seek(0)
+    while True:
+        chunk = payload.read(READ_CHUNK)
+        if not chunk:
+            break
+        digest.update(chunk)
+    payload.seek(0)
+    return digest.digest()
+
+
+def _verify_checksum(stream: BinaryIO, start: int, end: int) -> None:
+    trailer = _read_exact(stream, 20)
+    stream.seek(start)
+    digest = hashlib.sha1()
+    remaining = end - start
+    while remaining:
+        chunk = stream.read(min(READ_CHUNK, remaining))
+        digest.update(chunk)
+        remaining -= len(chunk)
+    if digest.digest() != trailer:
+        raise PolicyError("Git pack checksum is invalid")
+    stream.seek(end + 20)
+
+
+class _Entry:
+    """One pack entry: either resolved content or an unresolved delta."""
+
+    def __init__(self, type_code: int, payload: BinaryIO, size: int, base):
+        self.type_code = type_code
+        self.payload = payload
+        self.size = size
+        self.base = base
+
+
+def _resolve(entries: dict[int, _Entry], total: list[int]) -> None:
+    """Resolve every delta against in-pack bases only, to a fixed point."""
+    by_sha: dict[bytes, int] = {}
+    for offset, entry in entries.items():
+        if entry.base is None:
+            by_sha.setdefault(
+                _payload_sha(entry.type_code, entry.payload, entry.size), offset
+            )
+    for _ in range(MAX_RESOLVE_PASSES):
+        pending = [item for item in entries.items() if item[1].base is not None]
+        if not pending:
+            return
+        progressed = False
+        for offset, entry in pending:
+            base_key = entry.base
+            base_offset = base_key if isinstance(base_key, int) else by_sha.get(base_key)
+            base = entries.get(base_offset) if base_offset is not None else None
+            if base is None or base.base is not None:
+                if isinstance(base_key, int) and base is None:
+                    raise PolicyError("Git delta base offset is invalid")
+                continue
+            base.payload.seek(0)
+            entry.payload.seek(0)
+            value = _apply_delta(base.payload.read(), entry.payload.read())
+            total[0] += len(value)
+            if total[0] > MAX_TOTAL_BYTES:
+                raise PolicyError("Git pack contents exceed the safe size limit")
+            entry.payload.close()
+            spool = tempfile.SpooledTemporaryFile(  # noqa: SIM115 - entry owns spool
+                max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
+            )
+            spool.write(value)
+            spool.seek(0)
+            entry.type_code = base.type_code
+            entry.payload = spool
+            entry.size = len(value)
+            entry.base = None
+            by_sha.setdefault(_payload_sha(entry.type_code, spool, entry.size), offset)
+            progressed = True
+        if not progressed:
+            raise PolicyError("Git delta base is outside the pack; push full packs")
+    raise PolicyError("Git delta chains exceed the safe resolution limit")
+
+
+def unpack_objects(stream: BinaryIO) -> list[tuple[int, BinaryIO]]:
+    """Inflate one self-contained pack and return every resolved payload."""
+    start = stream.tell()
+    header = _read_exact(stream, 12)
+    version = int.from_bytes(header[4:8], "big")
+    count = int.from_bytes(header[8:12], "big")
+    if header[:4] != b"PACK" or version not in {2, 3}:
+        raise PolicyError("Git pack header is invalid")
+    if count > MAX_PACK_OBJECTS:
+        raise PolicyError("Git pack contains too many objects")
+    entries: dict[int, _Entry] = {}
+    total = [0]
+    try:
+        for _ in range(count):
+            offset = stream.tell() - start
+            type_code, declared = _object_header(stream)
+            if declared > MAX_OBJECT_BYTES:
+                raise PolicyError("Git object exceeds the safe size limit")
+            total[0] += declared
+            if total[0] > MAX_TOTAL_BYTES:
+                raise PolicyError("Git pack contents exceed the safe size limit")
+            base: int | bytes | None = None
+            if type_code == OFS_DELTA:
+                distance = _delta_distance(stream)
+                base = offset - distance
+                if base < 0:
+                    raise PolicyError("Git delta base offset is invalid")
+            elif type_code == REF_DELTA:
+                base = _read_exact(stream, 20)
+            elif type_code not in OBJECT_TYPE_NAMES:
+                raise PolicyError("Git pack object type is invalid")
+            entries[offset] = _Entry(type_code, _inflate(stream, declared), declared, base)
+        _verify_checksum(stream, start, stream.tell())
+        _resolve(entries, total)
+    except Exception:
+        for entry in entries.values():
+            entry.payload.close()
+        raise
+    for entry in entries.values():
+        entry.payload.seek(0)
+    return [
+        (entries[offset].type_code, entries[offset].payload)
+        for offset in sorted(entries)
+    ]
+
+
+def parse_loose_object(value: bytes) -> tuple[int, bytes]:
+    """Inflate one quarantined loose object and return its typed payload."""
+    if len(value) > MAX_OBJECT_BYTES:
+        raise PolicyError("Git object exceeds the safe size limit")
+    decompressor = zlib.decompressobj()
+    try:
+        inflated = decompressor.decompress(value, MAX_OBJECT_BYTES + 1)
+    except zlib.error as exc:
+        raise PolicyError("Git loose object data is corrupt") from exc
+    if not decompressor.eof or decompressor.unused_data or len(inflated) > MAX_OBJECT_BYTES:
+        raise PolicyError("Git loose object framing is invalid")
+    kind, separator, remainder = inflated.partition(b"\x00")
+    name, space, size = kind.partition(b" ")
+    types = {label: code for code, label in OBJECT_TYPE_NAMES.items()}
+    if (
+        not separator
+        or not space
+        or name not in types
+        or not size.isdigit()
+        or int(size) != len(remainder)
+    ):
+        raise PolicyError("Git loose object header is invalid")
+    return types[name], remainder

services/hermes/scm-common/scripts/gitea_api.py

@@ -0,0 +1,492 @@
+#!/usr/bin/env python3
+"""Use the private Atlas Forgejo API through a least-authority PR boundary."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import os
+import re
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+from collections.abc import Callable
+from pathlib import Path
+
+import deadline_http
+from gitea_api_policy import (
+    PolicyError,
+    _draft_title,
+    _reject_forbidden,
+    _validate_body,
+    _validate_pr_number,
+    _validate_pr_number_segment,
+    _validate_query,
+    _validate_ref,
+    _validate_ref_bounds,
+    _validate_repo,
+    _validate_sha,
+)
+
+CANONICAL_BASE_URL = "https://scm.bstein.dev"
+ALLOWED_OWNER = "atlas"
+DEFAULT_TOKEN_FILE = Path("/vault/secrets/gitea-token")
+CREATE_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls\Z")
+MAX_ERROR_BYTES = 8192
+MAX_RESPONSE_BYTES = 2 * 1024 * 1024
+MAX_API_TARGET_LENGTH = 768
+MAX_API_PATH_LENGTH = 512
+RAW_API_TARGET_RE = re.compile(r"[A-Za-z0-9/_.?&=-]+\Z")
+
+
+def _safe_urlopen(request: urllib.request.Request, timeout: int):
+    """Open one exchange under a killable absolute wall-clock deadline.
+
+    The helper process rejects redirects before urllib can copy
+    authentication headers, and its whole connect/send/read lifetime
+    shares one hard deadline.
+    """
+    return deadline_http.open_bounded(
+        request, maximum=MAX_RESPONSE_BYTES, timeout=timeout
+    )
+
+
+def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str:
+    """Read the token from the pod-lifetime Vault projection."""
+    token = path.read_text(encoding="utf-8").strip()
+    if not token:
+        raise ValueError(f"runtime credential is empty: {path}")
+    return token
+
+
+def configured_base_url() -> str:
+    """Reject attempts to redirect credentials away from the canonical origin."""
+    configured = os.environ.get("GITEA_BASE_URL", CANONICAL_BASE_URL)
+    if configured != CANONICAL_BASE_URL:
+        raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service")
+    return configured
+
+
+def _split_api_path(path: str) -> urllib.parse.SplitResult:
+    if not isinstance(path, str) or len(path) > MAX_API_TARGET_LENGTH:
+        raise PolicyError("API target exceeds the safe size limit")
+    if (
+        not path.isascii()
+        or any(ord(character) < 32 or ord(character) == 127 for character in path)
+        or "\\" in path
+        or not RAW_API_TARGET_RE.fullmatch(path)
+    ):
+        raise PolicyError("API target contains non-canonical raw characters")
+    target = urllib.parse.urlsplit(path)
+    canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, ""))
+    if canonical != path:
+        raise PolicyError("API target is not in exact canonical form")
+    if target.scheme or target.netloc or target.fragment:
+        raise PolicyError("API path must be relative to the Atlas SCM origin")
+    if not target.path.startswith("/api/v1/"):
+        raise PolicyError("API path must start with /api/v1/")
+    if len(target.path) > MAX_API_PATH_LENGTH:
+        raise PolicyError("API path exceeds the safe size limit")
+    segments = target.path.split("/")
+    if "//" in target.path or any(segment in {".", ".."} for segment in segments):
+        raise PolicyError("encoded or non-canonical API paths are not allowed")
+    return target
+
+
+def _authorize_read(
+    target: urllib.parse.SplitResult, *, forbidden: tuple[str, ...] = ()
+) -> str:
+    """Allow only repository, PR, branch, commit, and status metadata reads."""
+    prefix = "/api/v1/repos/atlas/"
+    remainder = target.path.removeprefix(prefix)
+    if remainder == target.path:
+        raise PolicyError("reads are limited to explicit Atlas repository metadata")
+    repo, separator, suffix = remainder.partition("/")
+    _reject_forbidden(repo, "repository", forbidden)
+    _validate_repo(repo)
+
+    if not separator:
+        _validate_query(target, set())
+        return "repository"
+    if suffix == "pulls":
+        _validate_query(target, {"page", "limit", "state"})
+        return "pull-list"
+    content_match = re.fullmatch(r"pulls/([^/]+)\.(?:patch|diff)", suffix)
+    if content_match:
+        _validate_pr_number_segment(content_match.group(1))
+        raise PolicyError("repository API route is outside the metadata read allowlist")
+    pull_match = re.fullmatch(r"pulls/([^/]+)", suffix)
+    if pull_match:
+        _validate_pr_number_segment(pull_match.group(1))
+        _validate_query(target, set())
+        return "pull"
+    evidence_match = re.fullmatch(r"pulls/([^/]+)/(?:commits|files)", suffix)
+    if evidence_match:
+        _validate_pr_number_segment(evidence_match.group(1))
+        _validate_query(target, {"page", "limit"})
+        return "pull-evidence"
+    if suffix == "branches":
+        _validate_query(target, {"page", "limit"})
+        return "branch-list"
+    branch_match = re.fullmatch(r"branches/([^/]+)", suffix)
+    if branch_match:
+        _validate_ref(branch_match.group(1), "branch")
+        _validate_query(target, set())
+        return "branch"
+    if suffix == "commits":
+        _validate_query(target, {"page", "limit"})
+        return "commit-list"
+    if re.fullmatch(r"git/commits/[0-9a-fA-F]{40}", suffix):
+        _validate_query(target, set())
+        return "commit"
+    if re.fullmatch(
+        r"(?:commits/[0-9a-fA-F]{40}/status|commits/[0-9a-fA-F]{40}/statuses|statuses/[0-9a-fA-F]{40})",
+        suffix,
+    ):
+        _validate_query(target, {"page", "limit"})
+        return "status"
+    raise PolicyError("repository API route is outside the metadata read allowlist")
+
+
+def api_url(base_url: str, path: str) -> str:
+    """Return a canonical same-origin URL without forwarding credentials."""
+    if base_url != CANONICAL_BASE_URL:
+        raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service")
+    target = _split_api_path(path)
+    return urllib.parse.urlunsplit(
+        ("https", "scm.bstein.dev", target.path, target.query, "")
+    )
+
+
+def authorize_request(
+    method: str,
+    path: str,
+    data: object | None,
+    *,
+    forbidden: tuple[str, ...] = (),
+) -> str:
+    """Validate one request and return its bounded operation name."""
+    normalized_method = method.upper()
+    target = _split_api_path(path)
+    if normalized_method == "GET":
+        if data is not None:
+            raise PolicyError("read operations cannot include a request body")
+        return _authorize_read(target, forbidden=forbidden)
+    if target.query:
+        raise PolicyError("mutating operations cannot include query parameters")
+    if normalized_method != "POST":
+        raise PolicyError("only read and create-draft operations are available")
+    match = CREATE_PATH_RE.fullmatch(target.path)
+    if not match:
+        raise PolicyError("POST is limited to creating an Atlas draft pull request")
+    _reject_forbidden(match.group(1), "repository", forbidden)
+    _validate_repo(match.group(1))
+    if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}:
+        raise PolicyError("draft creation accepts only base, body, head, and title")
+    if data["title"] != _draft_title(data["title"], forbidden=forbidden):
+        raise PolicyError("new pull requests must use the Gitea draft-title prefix")
+    _validate_body(data["body"], forbidden=forbidden)
+    _validate_ref_bounds(data["base"], "base", forbidden=forbidden)
+    _validate_ref_bounds(data["head"], "head", forbidden=forbidden)
+    _validate_ref(data["base"], "base", forbidden=forbidden)
+    _validate_ref(data["head"], "head", forbidden=forbidden)
+    return "create-draft"
+
+
+def build_request(
+    method: str,
+    path: str,
+    *,
+    base_url: str,
+    token: str,
+    data: object | None = None,
+) -> urllib.request.Request:
+    """Build an authorized request without putting the token in its URL or body."""
+    authorize_request(method, path, data, forbidden=(token,))
+    return _build_request(method, path, base_url=base_url, token=token, data=data)
+
+
+def _build_request(
+    method: str,
+    path: str,
+    *,
+    base_url: str,
+    token: str,
+    data: object | None = None,
+) -> urllib.request.Request:
+    """Build one already-authorized upstream request."""
+    payload = None
+    if data is not None:
+        payload = json.dumps(data, separators=(",", ":")).encode("utf-8")
+    return urllib.request.Request(
+        api_url(base_url, path),
+        data=payload,
+        method=method.upper(),
+        headers={
+            "Accept": "application/json",
+            "Authorization": f"token {token}",
+            "Content-Type": "application/json",
+            "User-Agent": "hermes-atlas-pr-client/2",
+        },
+    )
+
+
+def redact_bytes(value: bytes, token: str) -> bytes:
+    """Remove exact runtime-token and authorization-header values from output."""
+    redacted = value.replace(token.encode("utf-8"), b"[REDACTED]")
+    return re.sub(
+        rb"(?i)(authorization\s*[:=]\s*)(?:token|bearer)?\s*[^\s,;\"}]+",
+        rb"\1[REDACTED]",
+        redacted,
+    )
+
+
+def _reject_response_credential(value: bytes, token: str) -> None:
+    """Fail closed if an upstream body reflects any ordinary token encoding."""
+    raw = token.encode("utf-8")
+    forms = {
+        raw,
+        base64.b64encode(raw),
+        base64.b64encode(b"hermes-automation:" + raw),
+        urllib.parse.quote(token, safe="").encode("ascii"),
+    }
+    if any(form and form in value for form in forms):
+        raise PolicyError("Forgejo response contains runtime credential material")
+
+
+def _request(
+    method: str,
+    path: str,
+    data: object | None,
+    *,
+    token: str,
+    opener: Callable[..., object] = _safe_urlopen,
+    authorized: bool = False,
+    expected_status: int,
+) -> bytes:
+    builder = _build_request if authorized else build_request
+    request = builder(method, path, base_url=configured_base_url(), token=token, data=data)
+    with opener(request, timeout=30) as response:  # type: ignore[attr-defined]
+        status = getattr(response, "status", None)
+        if status is None and hasattr(response, "getcode"):
+            status = response.getcode()  # type: ignore[attr-defined]
+        if status != expected_status:
+            raise PolicyError("Forgejo returned an unexpected HTTP status")
+        if response.headers.get_content_type() != "application/json":  # type: ignore[attr-defined]
+            raise PolicyError("Forgejo returned an unexpected response type")
+        body = response.read(MAX_RESPONSE_BYTES + 1)  # type: ignore[attr-defined]
+    if len(body) > MAX_RESPONSE_BYTES:
+        raise PolicyError("Forgejo response exceeds the safe size limit")
+    _reject_response_credential(body, token)
+    return redact_bytes(body, token)
+
+
+def read(
+    path: str, *, token: str, opener: Callable[..., object] = _safe_urlopen
+) -> bytes:
+    """Read one explicitly allowed Atlas repository metadata resource."""
+    return _request(
+        "GET", path, None, token=token, opener=opener, expected_status=200
+    )
+
+
+def _nested(document: dict[str, object], *keys: str) -> object:
+    value: object = document
+    for key in keys:
+        if not isinstance(value, dict) or key not in value:
+            raise PolicyError("Forgejo omitted required pull-request evidence")
+        value = value[key]
+    return value
+
+
+def _require_create_response(
+    value: bytes,
+    *,
+    repo: str,
+    base: str,
+    head: str,
+    head_sha: str,
+    title: str,
+    body: str,
+) -> dict[str, object]:
+    """Require exact server evidence for the intended open draft handoff."""
+    try:
+        document = json.loads(value)
+    except json.JSONDecodeError as exc:
+        raise PolicyError("Forgejo returned invalid pull-request metadata") from exc
+    if not isinstance(document, dict):
+        raise PolicyError("Forgejo returned invalid pull-request metadata")
+    number = _validate_pr_number(document.get("number"))
+    full_name = f"{ALLOWED_OWNER}/{repo}"
+    expected_html = f"{CANONICAL_BASE_URL}/{full_name}/pulls/{number}"
+    checks = (
+        document.get("state") == "open",
+        document.get("draft") is True,
+        document.get("merged") is False,
+        document.get("html_url") == expected_html,
+        document.get("url") == expected_html,
+        document.get("title") == title,
+        document.get("body") == body,
+        _nested(document, "base", "ref") == base,
+        _nested(document, "base", "repo", "full_name") == full_name,
+        _nested(document, "head", "ref") == head,
+        _nested(document, "head", "sha") == head_sha,
+        _nested(document, "head", "repo", "full_name") == full_name,
+    )
+    if not all(checks):
+        raise PolicyError("Forgejo did not confirm the exact draft pull-request state")
+    return document
+
+
+def create_draft(
+    repo: str,
+    *,
+    base: str,
+    head: str,
+    head_sha: str,
+    title: str,
+    body: str,
+    token: str,
+    opener: Callable[..., object] = _safe_urlopen,
+) -> bytes:
+    """Create and verify a same-repository draft pull request for human review."""
+    _reject_forbidden(repo, "repository", (token,))
+    repo = _validate_repo(repo)
+    title = _draft_title(title, forbidden=(token,))
+    body = _validate_body(body, forbidden=(token,))
+    head_sha = _validate_sha(head_sha)
+    _validate_ref_bounds(base, "base", forbidden=(token,))
+    _validate_ref_bounds(head, "head", forbidden=(token,))
+    base = _validate_ref(base, "base", forbidden=(token,))
+    head = _validate_ref(head, "head", forbidden=(token,))
+    data = {"base": base, "body": body, "head": head, "title": title}
+    result = _request(
+        "POST",
+        f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls",
+        data,
+        token=token,
+        opener=opener,
+        authorized=True,
+        expected_status=201,
+    )
+    _require_create_response(
+        result,
+        repo=repo,
+        base=base,
+        head=head,
+        head_sha=head_sha,
+        title=title,
+        body=body,
+    )
+    return result
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    """Parse the intentionally narrow read and create-draft operations."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="validate and describe the request without reading a token or using the network",
+    )
+    subparsers = parser.add_subparsers(dest="operation", required=True)
+    read_parser = subparsers.add_parser("read", help="read allowed Atlas metadata")
+    read_parser.add_argument("path")
+    create_parser = subparsers.add_parser(
+        "create-draft", help="create a same-repository draft pull request"
+    )
+    create_parser.add_argument("repo")
+    create_parser.add_argument("--base", required=True)
+    create_parser.add_argument("--head", required=True)
+    create_parser.add_argument("--head-sha", required=True)
+    create_parser.add_argument("--title", required=True)
+    create_parser.add_argument("--body", default="")
+    return parser.parse_args(argv)
+
+
+def _operation(
+    args: argparse.Namespace, *, validate_refs: bool = True
+) -> tuple[str, str, object | None]:
+    if args.operation == "read":
+        return "GET", args.path, None
+    repo = _validate_repo(args.repo)
+    _validate_sha(args.head_sha)
+    body = _validate_body(args.body)
+    title = _draft_title(args.title)
+    _validate_ref_bounds(args.base, "base")
+    _validate_ref_bounds(args.head, "head")
+    base = _validate_ref(args.base, "base") if validate_refs else args.base
+    head = _validate_ref(args.head, "head") if validate_refs else args.head
+    data = {
+        "base": base,
+        "body": body,
+        "head": head,
+        "title": title,
+    }
+    return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data
+
+
+def _write_body(body: bytes) -> None:
+    if body:
+        sys.stdout.buffer.write(body)
+        if not body.endswith(b"\n"):
+            sys.stdout.buffer.write(b"\n")
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Validate, optionally describe, and execute one safe Forgejo operation."""
+    try:
+        args = parse_args(argv)
+        if args.dry_run:
+            method, path, data = _operation(args, validate_refs=False)
+            operation = authorize_request(method, path, data)
+            print(
+                json.dumps(
+                    {
+                        "dry_run": True,
+                        "host": "scm.bstein.dev",
+                        "operation": operation,
+                        "owner": ALLOWED_OWNER,
+                        "path": path,
+                        "request_fields": sorted(data)
+                        if isinstance(data, dict)
+                        else [],
+                    },
+                    sort_keys=True,
+                )
+            )
+            return 0
+        method, path, data = _operation(args, validate_refs=False)
+        from scm_broker_client import create_draft as broker_create_draft
+        from scm_broker_client import read as broker_read
+
+        if args.operation == "read":
+            body = broker_read(path)
+        else:
+            assert isinstance(data, dict)
+            body = broker_create_draft(
+                args.repo,
+                base=str(data["base"]),
+                head=str(data["head"]),
+                head_sha=args.head_sha,
+                title=str(data["title"]),
+                body=str(data["body"]),
+            )
+        _write_body(body)
+        return 0
+    except urllib.error.HTTPError as exc:
+        exc.read(MAX_ERROR_BYTES)
+        print(f"SCM broker request failed with HTTP {exc.code}", file=sys.stderr)
+        return 1
+    except (OSError, PolicyError, ValueError, json.JSONDecodeError):
+        print(
+            "Forgejo request rejected or unavailable; no credential was disclosed",
+            file=sys.stderr,
+        )
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

services/hermes/scm-common/scripts/gitea_api_policy.py

@@ -0,0 +1,490 @@
+"""Validation policy for Hermes' least-authority Atlas Forgejo client.
+
+The screening catches structured credential assignments, known token formats,
+and long high-entropy values. It is a fail-closed accident barrier, not proof
+that text is secret-free: a short or multiword secret under an innocuous key is
+not reliably distinguishable from prose and must never be supplied by callers.
+JSON depth/nodes/documents and multiline gaps are deliberately bounded; large
+or unusual configuration blobs are not a supported pull-request field format.
+"""
+
+# ruff: noqa: SIM905
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+import urllib.parse
+from collections import Counter
+from math import log2
+
+REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z")
+SHA_RE = re.compile(r"[0-9a-fA-F]{40}\Z")
+DRAFT_TITLE_PREFIX = "WIP: "
+GIT_BIN = "/usr/bin/git"
+MAX_QUERY_LENGTH = 128
+MAX_QUERY_ITEMS = 3
+MAX_QUERY_KEY_LENGTH = 16
+MAX_QUERY_VALUE_LENGTH = 16
+MAX_PAGE = 10_000
+MAX_LIMIT = 50
+MAX_PR_NUMBER = 2_147_483_647
+MAX_PR_NUMBER_DIGITS = 10
+MAX_REF_CHARACTERS = 200
+MAX_REF_UTF8_BYTES = 255
+MAX_TITLE_UTF8_BYTES = 512
+MAX_BODY_UTF8_BYTES = 32_768
+MAX_JSON_DOCUMENTS = 32
+MAX_JSON_NODES = 2_048
+MAX_JSON_DEPTH = 32
+CANONICAL_QUERY_RE = re.compile(r"[A-Za-z0-9_=&-]*\Z")
+ASSIGNMENT_RE = re.compile(
+    r"""(?mx)
+    (?<![A-Za-z0-9_.-])
+    (?P<key_quote>[\"']?)
+    (?P<key>\.?[A-Za-z][A-Za-z0-9_.-]{0,127})
+    (?P=key_quote)
+    [ \t]*(?P<separator>[:=])[ \t]*
+    (?P<value>
+      \"(?:\\.|[^\"\\\r\n])*\" |
+      '(?:\\.|[^'\\\r\n])*' |
+      [^\r\n,;}&]{0,2048}
+    )
+    """
+)
+JSON_KEY_RE = re.compile(r'"(?P<key>[^"]{1,512})"(?P<spacing>[\x00-\x20\x7f]{0,256}):')
+MULTILINE_ASSIGNMENT_RE = re.compile(
+    r"""(?mx)
+    (?<![A-Za-z0-9_.-])
+    (?P<key_quote>["']?)
+    (?P<key>\.?[A-Za-z][A-Za-z0-9_.-]{0,127})
+    (?P=key_quote)
+    [ \t]{0,64}(?:\r?\n[ \t]{0,64}){0,4}
+    (?P<separator>[:=])
+    [ \t]{0,64}(?:\r?\n[ \t]{0,64}){1,4}
+    (?P<value>[^\r\n,;}&]{1,2048})
+    """
+)
+CAMEL_BOUNDARY_RE = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])")
+CREDENTIAL_SCHEME_RE = re.compile(
+    r"(?i)\b(?:bearer|basic|token)\s+"
+    r"(?=[A-Za-z0-9+/_=.:-]{8,}\b)(?=[A-Za-z0-9+/_=.:-]*[0-9+/_=.-])"
+    r"[A-Za-z0-9+/_=.:-]{8,}"
+)
+STANDALONE_SECRET_PATTERNS = tuple(
+    re.compile(pattern, re.IGNORECASE)
+    for pattern in (
+        r"(?<![A-Za-z0-9])gh[pousr]_[A-Za-z0-9]{20,}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])glpat-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])xox[baprs]-[A-Za-z0-9-]{20,}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])sk-(?:ant-|proj-)?[A-Za-z0-9_-]{20,}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])[rs]k_(?:test|live)_[A-Za-z0-9]{20,}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])whsec_[A-Za-z0-9]{20,}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])npm_[A-Za-z0-9]{20,}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])pypi-[A-Za-z0-9_-]{30,}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])hf_[A-Za-z0-9]{20,}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])(?:gta|gto|gitea|forgejo)_[A-Za-z0-9_-]{20,}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])ya29\.[A-Za-z0-9_-]{20,}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])oy2[A-Za-z0-9]{40,}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{20,}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])SK[0-9a-f]{32}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])(?:AKIA|ASIA)[0-9A-Z]{16}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])AIza[0-9A-Za-z_-]{35}(?![A-Za-z0-9])",
+        r"(?<![A-Za-z0-9])eyJ[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{8,}(?![A-Za-z0-9])",
+        r"-----BEGIN\s+[A-Z0-9 ][A-Z0-9 -]{1,62}-----",
+        r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}",
+        r"https://hooks\.slack\.com/services/[A-Za-z0-9/_-]{20,}",
+        r"https://(?:discord(?:app)?\.com)/api/webhooks/[0-9]+/[A-Za-z0-9._-]{20,}",
+        r"https://[^\s/]*webhook\.office\.com/[^\s]{20,}",
+    )
+)
+HIGH_ENTROPY_TOKEN_RE = re.compile(
+    r"(?<![A-Za-z0-9+/_=-])[A-Za-z0-9+/_=-]{48,16384}(?![A-Za-z0-9+/_=-])"
+)
+PROSE_LEAD_WORDS = frozenset(
+    "add allow change check describe document ensure explain fix handle keep preserve "
+    "prevent reject remove review support test validate".split()
+)
+SENSITIVE_KEY_WORDS = frozenset(
+    "auth authentication authorization auths basic bearer credential credentials "
+    "passwd password pat private sas secret secrets sig signature signing token tokens "
+    "webhook".split()
+)
+KEY_MODIFIER_WORDS = frozenset(
+    "access account api auth client encryption identity private registry secret service "
+    "session signing ssh".split()
+)
+SENSITIVE_COMPACT_KEYS = frozenset(
+    "accessid accesskeyid accountkey clientemail clientid connectionstring "
+    "dockerconfigjson privatekeyid serviceaccountkey sharedaccesssignature".split()
+)
+SENSITIVE_COMPACT_SUFFIXES = tuple(
+    "accesskey accesskeyid accountkey apikey authkey clientemail clientid clientkey "
+    "connectionstring credential credentials encryptionkey identitykey password passwd "
+    "privatekeyid privatekey secret secretkey servicekey sessionkey signature signingkey "
+    "sshkey token webhook".split()
+)
+
+
+class PolicyError(ValueError):
+    """Raised when a requested Forgejo operation exceeds the safe boundary."""
+
+
+def _validate_repo(repo: str) -> str:
+    if not REPO_RE.fullmatch(repo) or repo in {".", ".."}:
+        raise PolicyError("repository name is outside the Atlas allowlist")
+    return repo
+
+
+def _reject_forbidden(value: object, name: str, forbidden: tuple[str, ...]) -> None:
+    """Reject exact runtime credentials before invoking helpers or the network."""
+    if not isinstance(value, str) or any(secret and secret in value for secret in forbidden):
+        raise PolicyError(f"{name} contains runtime credential material")
+
+
+def _validate_ref_bounds(
+    value: object, name: str, *, forbidden: tuple[str, ...] = ()
+) -> str:
+    """Reject oversized or non-encodable refs before invoking Git."""
+    _reject_forbidden(value, name, forbidden)
+    if not value or len(value) > MAX_REF_CHARACTERS:
+        raise PolicyError(f"{name} exceeds the safe branch-name limit")
+    try:
+        encoded = value.encode("utf-8", errors="strict")
+    except UnicodeEncodeError as exc:
+        raise PolicyError(f"{name} is not valid UTF-8 text") from exc
+    if len(encoded) > MAX_REF_UTF8_BYTES:
+        raise PolicyError(f"{name} exceeds the safe UTF-8 branch-name limit")
+    return value
+
+
+def _validate_ref(
+    value: object, name: str, *, forbidden: tuple[str, ...] = ()
+) -> str:
+    """Validate the complete Git ref grammar using Git itself."""
+    value = _validate_ref_bounds(value, name, forbidden=forbidden)
+    if value.startswith("-"):
+        raise PolicyError(f"{name} must be a same-repository branch name")
+    if any(ord(character) < 32 or ord(character) == 127 for character in value):
+        raise PolicyError(f"{name} must be a safe same-repository branch name")
+    result = subprocess.run(
+        [GIT_BIN, "check-ref-format", f"refs/heads/{value}"],
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        check=False,
+    )
+    if result.returncode != 0:
+        raise PolicyError(f"{name} must be a safe same-repository branch name")
+    return value
+
+
+def _validate_sha(value: object, name: str = "head SHA") -> str:
+    if not isinstance(value, str) or not SHA_RE.fullmatch(value):
+        raise PolicyError(f"{name} must be a full Git SHA-1")
+    return value.lower()
+
+
+def _validate_pr_number_segment(value: object) -> int:
+    """Validate a canonical bounded pull-request number from an API path."""
+    if (
+        not isinstance(value, str)
+        or len(value) > MAX_PR_NUMBER_DIGITS
+        or not re.fullmatch(r"[1-9][0-9]*", value)
+    ):
+        raise PolicyError("pull-request number must use bounded canonical ASCII digits")
+    number = int(value)
+    if number > MAX_PR_NUMBER:
+        raise PolicyError("pull-request number is outside its allowed range")
+    return number
+
+
+def _validate_pr_number(value: object) -> int:
+    """Validate a bounded pull-request number returned by Forgejo."""
+    if (
+        not isinstance(value, int)
+        or isinstance(value, bool)
+        or not 1 <= value <= MAX_PR_NUMBER
+    ):
+        raise PolicyError("Forgejo omitted a valid pull-request number")
+    return value
+
+
+def _validate_text(
+    value: object,
+    name: str,
+    maximum: int,
+    maximum_bytes: int,
+    *,
+    required: bool,
+) -> str:
+    if not isinstance(value, str):
+        raise PolicyError(f"{name} must be text")
+    if required and not value.strip():
+        raise PolicyError(f"{name} must not be empty")
+    try:
+        encoded = value.encode("utf-8", errors="strict")
+    except UnicodeEncodeError as exc:
+        raise PolicyError(f"{name} is not valid UTF-8 text") from exc
+    if len(value) > maximum or len(encoded) > maximum_bytes or "\x00" in value:
+        raise PolicyError(f"{name} exceeds the safe request limit")
+    return value
+
+
+def _normalized_key(value: str) -> tuple[tuple[str, ...], str]:
+    """Split camelCase and separator-based assignment keys into semantics."""
+    separated = CAMEL_BOUNDARY_RE.sub(" ", value.strip(".\"'"))
+    words = tuple(re.findall(r"[A-Za-z0-9]+", separated.lower()))
+    return words, "".join(words)
+
+
+def _is_sensitive_key(value: str) -> bool:
+    words, compact = _normalized_key(value)
+    word_set = set(words)
+    if word_set & SENSITIVE_KEY_WORDS:
+        return True
+    if compact in SENSITIVE_COMPACT_KEYS:
+        return True
+    if compact.endswith(SENSITIVE_COMPACT_SUFFIXES):
+        return True
+    if "key" in word_set and word_set & KEY_MODIFIER_WORDS:
+        return True
+    if {"client", "email"} <= word_set or {"client", "id"} <= word_set:
+        return True
+    if {"access", "id"} <= word_set or {"connection", "string"} <= word_set:
+        return True
+    return compact.endswith("configjson")
+
+
+def _strip_assignment_value(value: str) -> tuple[str, bool]:
+    stripped = value.strip()
+    quoted = (
+        len(stripped) >= 2 and stripped[0] in {'"', "'"} and stripped[-1] == stripped[0]
+    )
+    if quoted:
+        stripped = stripped[1:-1].strip()
+    return stripped, quoted
+
+
+def _looks_like_prose(value: str) -> bool:
+    words = re.findall(r"[A-Za-z]+(?:[-'][A-Za-z]+)?", value)
+    if len(words) < 2:
+        return False
+    if re.search(r"[$`{}\[\]\\@/:=+_]", value):
+        return False
+    return words[0].lower() in PROSE_LEAD_WORDS
+
+
+def _looks_sensitive_assignment_value(value: str) -> bool:
+    stripped, quoted = _strip_assignment_value(value)
+    if not stripped:
+        return False
+    if quoted or stripped.startswith(("{", "[", "|", ">", "!!", "&", "*")):
+        return True
+    if CREDENTIAL_SCHEME_RE.search(stripped):
+        return True
+    if any(pattern.search(stripped) for pattern in STANDALONE_SECRET_PATTERNS):
+        return True
+    if re.search(r"(?:https?://|[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+)", stripped):
+        return True
+    if re.search(r"(?:\$\{|\$\(|\\[nrt]|[A-Za-z0-9+/]{16,}={0,3}\Z)", stripped):
+        return True
+    return not _looks_like_prose(stripped)
+
+
+def _is_structural_credential_assignment(key: str, value: str) -> bool:
+    words, compact_key = _normalized_key(key)
+    stripped, _quoted = _strip_assignment_value(value)
+    _value_words, compact_value = _normalized_key(stripped)
+    return (
+        compact_key == "type" and compact_value in {"serviceaccount", "credential"}
+    ) or ("service" in words and "account" in words and bool(stripped))
+
+
+def _decode_json_key(raw: str) -> tuple[str, bool]:
+    """Decode one bounded JSON key without silently accepting bad escapes."""
+    try:
+        decoded = json.loads(f'"{raw}"')
+    except (json.JSONDecodeError, UnicodeError):
+        approximate = re.sub(r"[^A-Za-z0-9]+", "_", raw.replace("\\", ""))
+        return approximate, False
+    return decoded if isinstance(decoded, str) else "", True
+
+
+def _json_value_has_sensitive_assignment(
+    value: object, *, depth: int = 0, nodes: list[int] | None = None
+) -> bool:
+    """Walk one decoded JSON value with explicit depth and node ceilings."""
+    if nodes is None:
+        nodes = [0]
+    nodes[0] += 1
+    if nodes[0] > MAX_JSON_NODES or depth > MAX_JSON_DEPTH:
+        raise PolicyError("structured pull-request text exceeds the scan limit")
+    if isinstance(value, dict):
+        for key, assigned in value.items():
+            if not isinstance(key, str):
+                continue
+            if _is_sensitive_key(key):
+                return True
+            assigned_text = assigned if isinstance(assigned, str) else ""
+            if _is_structural_credential_assignment(key, assigned_text):
+                return True
+            if _json_value_has_sensitive_assignment(
+                assigned, depth=depth + 1, nodes=nodes
+            ):
+                return True
+    elif isinstance(value, list):
+        return any(
+            _json_value_has_sensitive_assignment(item, depth=depth + 1, nodes=nodes)
+            for item in value
+        )
+    return False
+
+
+def _decoded_json_documents(value: str):
+    """Yield bounded embedded JSON object or array fragments."""
+    decoder = json.JSONDecoder()
+    index = 0
+    attempts = 0
+    while attempts < MAX_JSON_DOCUMENTS:
+        positions = [
+            position for token in "{[" if (position := value.find(token, index)) >= 0
+        ]
+        if not positions:
+            return
+        start = min(positions)
+        attempts += 1
+        try:
+            document, end = decoder.raw_decode(value, start)
+        except (json.JSONDecodeError, RecursionError, ValueError):
+            index = start + 1
+            continue
+        yield document
+        index = max(end, start + 1)
+    if "{" in value[index:] or "[" in value[index:]:
+        raise PolicyError("structured pull-request text exceeds the scan limit")
+
+
+def _has_structured_sensitive_assignment(value: str) -> bool:
+    """Detect bounded JSON and multiline YAML/env credential assignments."""
+    for candidate in JSON_KEY_RE.finditer(value):
+        decoded, valid = _decode_json_key(candidate.group("key"))
+        if _is_sensitive_key(decoded):
+            return True
+        if not valid and _is_sensitive_key(candidate.group("key").replace("\\", "")):
+            return True
+    for candidate in MULTILINE_ASSIGNMENT_RE.finditer(value):
+        key = candidate.group("key")
+        assigned = candidate.group("value")
+        if _is_structural_credential_assignment(key, assigned) or (
+            _is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned)
+        ):
+            return True
+    return any(
+        _json_value_has_sensitive_assignment(item)
+        for item in _decoded_json_documents(value)
+    )
+
+
+def _has_high_entropy_token(value: str) -> bool:
+    for match in HIGH_ENTROPY_TOKEN_RE.finditer(value):
+        token = match.group(0)
+        if len(token) > 256:
+            return True
+        groups = sum(
+            bool(re.search(pattern, token))
+            for pattern in (r"[a-z]", r"[A-Z]", r"[0-9]", r"[+/_=-]")
+        )
+        counts = Counter(token)
+        entropy = -sum(
+            (count / len(token)) * log2(count / len(token)) for count in counts.values()
+        )
+        if groups >= 3 and entropy >= 4.0:
+            return True
+    return False
+
+
+def _reject_sensitive(value: str, name: str, forbidden: tuple[str, ...]) -> None:
+    if any(secret and secret in value for secret in forbidden):
+        raise PolicyError(f"pull-request {name} contains runtime credential material")
+    if CREDENTIAL_SCHEME_RE.search(value):
+        raise PolicyError(f"pull-request {name} resembles credential material")
+    if any(pattern.search(value) for pattern in STANDALONE_SECRET_PATTERNS):
+        raise PolicyError(f"pull-request {name} resembles credential material")
+    if _has_structured_sensitive_assignment(value):
+        raise PolicyError(f"pull-request {name} resembles credential material")
+    for candidate in ASSIGNMENT_RE.finditer(value):
+        key = candidate.group("key")
+        assigned = candidate.group("value")
+        if _is_structural_credential_assignment(key, assigned) or (
+            _is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned)
+        ):
+            raise PolicyError(f"pull-request {name} resembles credential material")
+    if _has_high_entropy_token(value):
+        raise PolicyError(f"pull-request {name} resembles credential material")
+
+
+def _validate_body(value: object, *, forbidden: tuple[str, ...] = ()) -> str:
+    body = _validate_text(value, "body", 16384, MAX_BODY_UTF8_BYTES, required=False)
+    _reject_sensitive(body, "body", forbidden)
+    return body
+
+
+def _draft_title(value: object, *, forbidden: tuple[str, ...] = ()) -> str:
+    """Return a bounded, secret-screened Gitea draft title."""
+    title = _validate_text(
+        value, "title", 251, MAX_TITLE_UTF8_BYTES, required=True
+    ).strip()
+    _reject_sensitive(title, "title", forbidden)
+    for prefix in ("WIP:", "[WIP]"):
+        if title.upper().startswith(prefix):
+            title = title[len(prefix) :].lstrip()
+            break
+    if not title:
+        raise PolicyError("title must contain text after the draft prefix")
+    return DRAFT_TITLE_PREFIX + title
+
+
+def _validate_query(target: urllib.parse.SplitResult, allowed: set[str]) -> None:
+    """Accept only short canonical ASCII query strings with bounded pagination."""
+    raw = target.query
+    if (
+        len(raw) > MAX_QUERY_LENGTH
+        or not raw.isascii()
+        or not CANONICAL_QUERY_RE.fullmatch(raw)
+        or "%" in raw
+    ):
+        raise PolicyError("API query must use short canonical ASCII form")
+    try:
+        pairs = urllib.parse.parse_qsl(raw, keep_blank_values=True, strict_parsing=True)
+    except ValueError as exc:
+        raise PolicyError("invalid API query") from exc
+    if len(pairs) > MAX_QUERY_ITEMS:
+        raise PolicyError("too many API query parameters")
+    if len({key for key, _ in pairs}) != len(pairs):
+        raise PolicyError("duplicate API query parameters are not allowed")
+    if any(
+        not key
+        or len(key) > MAX_QUERY_KEY_LENGTH
+        or len(value) > MAX_QUERY_VALUE_LENGTH
+        for key, value in pairs
+    ):
+        raise PolicyError("API query key or value exceeds its safe limit")
+    if any(key not in allowed for key, _ in pairs):
+        raise PolicyError("API query parameter is outside the read allowlist")
+    values = dict(pairs)
+    for name in ("page", "limit"):
+        if name not in values:
+            continue
+        if not re.fullmatch(r"[0-9]+", values[name]):
+            raise PolicyError(f"{name} must use ASCII decimal digits")
+        number = int(values[name])
+        if values[name] != str(number):
+            raise PolicyError(f"{name} must use canonical ASCII decimal form")
+        ceiling = MAX_PAGE if name == "page" else MAX_LIMIT
+        if not 1 <= number <= ceiling:
+            raise PolicyError(f"{name} is outside its allowed range")
+    if "state" in values and values["state"] not in {"open", "closed", "all"}:
+        raise PolicyError("pull-request state is invalid")

services/hermes/scm-common/scripts/receive_pack_scan.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""Validate Git receive-pack requests and scan pushed object content.
+
+Ref commands may only create new namespaced feature branches.  Every pushed
+object is then inflated in quarantine and its decompressed payload scanned,
+so credentials cannot ride through the broker inside compressed pack data.
+The screening is a fail-closed accident barrier for credential-shaped
+content: exact runtime-token forms, private keys, SSH key material, and
+known provider token formats.  Entropy heuristics are deliberately excluded
+so SOPS ciphertext, pinned digests, and lock files remain pushable; a short
+secret disguised as prose is not reliably detectable and must never be
+supplied by callers.
+"""
+
+from __future__ import annotations
+
+import io
+import re
+from typing import BinaryIO
+
+import git_pack_objects
+from gitea_api_policy import (
+    STANDALONE_SECRET_PATTERNS,
+    PolicyError,
+    _validate_ref,
+)
+
+ZERO_SHA = b"0" * 40
+MAX_PUSH_COMMANDS = 16
+SCAN_CHUNK = 256 * 1024
+SCAN_CARRY = 8 * 1024
+FEATURE_REF_RE = re.compile(
+    r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z"
+)
+PKT_HEADER_RE = re.compile(rb"[0-9a-f]{4}\Z")
+CONTENT_SECRET_PATTERNS = tuple(
+    re.compile(pattern.pattern.encode("ascii"), re.IGNORECASE)
+    for pattern in STANDALONE_SECRET_PATTERNS
+) + (
+    re.compile(rb"(?i)(?<![A-Za-z0-9])AGE-SECRET-KEY-1[A-Z0-9]{20,}"),
+    re.compile(rb"(?i)PuTTY-User-Key-File"),
+    re.compile(
+        rb"(?im)^[ \t]*authorization[ \t]*:[ \t]*(?:basic|bearer|token)[ \t]+"
+        rb"(?=[A-Za-z0-9+/_=.:-]*[0-9+/=.-])[A-Za-z0-9+/_=.:-]{8,}"
+    ),
+)
+
+
+def _read_pkt_line(stream: BinaryIO) -> bytes | None:
+    """Read one pkt-line; None marks the flush packet before the pack."""
+    header = stream.read(4)
+    if not header:
+        raise PolicyError("Git receive-pack request omitted its command terminator")
+    if not PKT_HEADER_RE.fullmatch(header):
+        raise PolicyError("Git receive-pack command framing is invalid")
+    size = int(header, 16)
+    if size == 0:
+        return None
+    if size < 4:
+        raise PolicyError("Git receive-pack command framing is invalid")
+    value = stream.read(size - 4)
+    if len(value) != size - 4:
+        raise PolicyError("Git receive-pack command framing is invalid")
+    return value
+
+
+def _validate_command(line: bytes, token: str, forbidden: tuple[bytes, ...]) -> None:
+    """Permit only creation of one new, namespaced feature branch."""
+    if any(form in line for form in forbidden):
+        raise PolicyError("Git request contains runtime credential material")
+    command = line.rstrip(b"\n").split(b"\x00", 1)[0]
+    fields = command.split(b" ")
+    if len(fields) != 3 or not all(
+        re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2]
+    ):
+        raise PolicyError("Git receive-pack ref command is invalid")
+    old_sha, new_sha, raw_ref = fields
+    if old_sha != ZERO_SHA or new_sha == ZERO_SHA:
+        raise PolicyError("Git broker permits only new feature-branch creation")
+    try:
+        ref = raw_ref.decode("ascii")
+    except UnicodeDecodeError as exc:
+        raise PolicyError("Git ref must be canonical ASCII") from exc
+    if not FEATURE_REF_RE.fullmatch(ref):
+        raise PolicyError("Git push is limited to namespaced feature branches")
+    _validate_ref(ref.removeprefix("refs/heads/"), "head", forbidden=(token,))
+
+
+def _scan_payload(payload: BinaryIO, forbidden: tuple[bytes, ...]) -> None:
+    """Scan one decompressed object payload across chunk boundaries."""
+    carry = b""
+    while True:
+        chunk = payload.read(SCAN_CHUNK)
+        if not chunk:
+            return
+        window = carry + chunk
+        if any(form in window for form in forbidden):
+            raise PolicyError("Git push contains runtime credential material")
+        if any(pattern.search(window) for pattern in CONTENT_SECRET_PATTERNS):
+            raise PolicyError("Git push contains credential-shaped content")
+        carry = window[-SCAN_CARRY:]
+
+
+def _scan_pack(stream: BinaryIO, forbidden: tuple[bytes, ...]) -> None:
+    objects = git_pack_objects.unpack_objects(stream)
+    try:
+        if stream.read(1):
+            raise PolicyError("Git request has bytes after its pack")
+        for _type_code, payload in objects:
+            _scan_payload(payload, forbidden)
+    finally:
+        for _type_code, payload in objects:
+            payload.close()
+
+
+def validate_receive_pack(
+    body: bytes | BinaryIO, token: str, forbidden: tuple[bytes, ...]
+) -> None:
+    """Validate one whole receive-pack request: commands, pack, and content."""
+    stream = io.BytesIO(body) if isinstance(body, bytes) else body
+    stream.seek(0)
+    try:
+        commands = 0
+        while True:
+            line = _read_pkt_line(stream)
+            if line is None:
+                break
+            _validate_command(line, token, forbidden)
+            commands += 1
+            if commands > MAX_PUSH_COMMANDS:
+                raise PolicyError("Git push contains too many ref commands")
+        if commands == 0:
+            raise PolicyError("Git receive-pack request has no ref command")
+        _scan_pack(stream, forbidden)
+    finally:
+        stream.seek(0)
+
+
+def scan_loose_object(value: bytes, forbidden: tuple[bytes, ...]) -> int:
+    """Scan one quarantined loose object and return its parsed type code."""
+    type_code, payload = git_pack_objects.parse_loose_object(value)
+    _scan_payload(io.BytesIO(payload), forbidden)
+    return type_code

services/hermes/scm-common/scripts/scm_broker.py

@@ -0,0 +1,455 @@
+#!/usr/bin/env python3
+"""Serve the credential-isolated Atlas API and Git smart-HTTP boundary."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import re
+import socket
+import tempfile
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from http.server import BaseHTTPRequestHandler
+from typing import BinaryIO
+
+import deadline_http
+from gitea_api import (
+    CANONICAL_BASE_URL,
+    PolicyError,
+    create_draft,
+    read,
+    read_token,
+)
+from gitea_api_policy import _reject_forbidden, _validate_repo
+from receive_pack_scan import validate_receive_pack
+from scm_broker_io import RejectRedirect, response_status as _status, spool_response
+from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer
+
+BROKER_PORT = 9081
+GIT_USER = "hermes-automation"
+MAX_CONTROL_BODY = 64 * 1024
+MAX_GIT_REQUEST = 128 * 1024 * 1024
+MAX_GIT_RESPONSE = 128 * 1024 * 1024
+MAX_HEADERS = 32
+MAX_HEADER_BYTES = 16 * 1024
+SPOOL_MEMORY_LIMIT = 1024 * 1024
+STREAM_CHUNK = 64 * 1024
+INBOUND_HEADER_TIMEOUT = 10.0
+INBOUND_BODY_TIMEOUT = 120.0
+UPSTREAM_DEADLINE_SECONDS = 300.0
+GIT_PATH_RE = re.compile(
+    r"/git/atlas/(?P<repo>[A-Za-z0-9][A-Za-z0-9._-]{0,99})\.git/"
+    r"(?P<operation>info/refs|git-upload-pack|git-receive-pack)\Z"
+)
+
+
+def _read_bounded(
+    stream,
+    maximum: int,
+    length: int | None = None,
+    *,
+    deadline: float | None = None,
+    set_timeout=None,
+) -> bytes:
+    if length is not None and not 0 <= length <= maximum:
+        raise PolicyError("SCM request exceeds the safe size limit")
+    wanted = maximum + 1 if length is None else length
+    chunks: list[bytes] = []
+    total = 0
+    while total < wanted:
+        if deadline is not None:
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                raise PolicyError("SCM request body deadline exceeded")
+            if set_timeout is not None:
+                set_timeout(remaining)
+        chunk = stream.read(min(STREAM_CHUNK, wanted - total))
+        if not chunk:
+            break
+        chunks.append(chunk)
+        total += len(chunk)
+    value = b"".join(chunks)
+    if len(value) > maximum or (length is not None and len(value) != length):
+        raise PolicyError("SCM request exceeds the safe size limit")
+    return value
+
+
+def _spool_bounded(
+    stream,
+    maximum: int,
+    length: int,
+    *,
+    token: str,
+    context: str,
+    deadline: float,
+    set_timeout=None,
+) -> tuple[BinaryIO, int]:
+    """Copy a fixed-length exchange through a bounded-memory disk spool."""
+    if not 0 <= length <= maximum:
+        raise PolicyError("SCM request exceeds the safe size limit")
+    spool = tempfile.SpooledTemporaryFile(  # noqa: SIM115 - caller owns returned spool
+        max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
+    )
+    remaining = length
+    forms = _credential_forms(token)
+    carry = b""
+    try:
+        while remaining:
+            remaining_time = deadline - time.monotonic()
+            if remaining_time <= 0:
+                raise PolicyError("SCM request body deadline exceeded")
+            if set_timeout is not None:
+                set_timeout(remaining_time)
+            chunk = stream.read(min(STREAM_CHUNK, remaining))
+            if not chunk:
+                raise PolicyError("SCM request body ended early")
+            candidate = carry + chunk
+            if any(form in candidate for form in forms):
+                raise PolicyError(f"{context} contains runtime credential material")
+            width = max(len(form) for form in forms) - 1
+            carry = candidate[-width:] if width else b""
+            spool.write(chunk)
+            remaining -= len(chunk)
+        spool.seek(0)
+        return spool, length
+    except Exception:
+        spool.close()
+        raise
+
+
+def _spool_response(
+    response, maximum: int, token: str, deadline_seconds: float
+) -> tuple[BinaryIO, int]:
+    return spool_response(
+        response,
+        maximum,
+        _credential_forms(token),
+        memory_limit=SPOOL_MEMORY_LIMIT,
+        chunk_size=STREAM_CHUNK,
+        deadline_seconds=deadline_seconds,
+    )
+
+
+def _load_json(handler: BaseHTTPRequestHandler) -> dict[str, object]:
+    if handler.headers.get("Transfer-Encoding"):
+        raise PolicyError("chunked broker control requests are not allowed")
+    if handler.headers.get_content_type() != "application/json":
+        raise PolicyError("broker control request must be JSON")
+    length = _content_length(handler.headers, MAX_CONTROL_BODY)
+    deadline = time.monotonic() + INBOUND_BODY_TIMEOUT
+    value = json.loads(
+        _read_bounded(
+            handler.rfile,
+            MAX_CONTROL_BODY,
+            length,
+            deadline=deadline,
+            set_timeout=handler.connection.settimeout,
+        )
+    )
+    if not isinstance(value, dict):
+        raise PolicyError("broker control request must be an object")
+    return value
+
+
+def _git_target(raw: str) -> tuple[str, str, str]:
+    if (
+        not raw.isascii()
+        or any(ord(character) < 32 or ord(character) == 127 for character in raw)
+        or "%" in raw
+        or "\\" in raw
+        or len(raw) > 512
+    ):
+        raise PolicyError("Git target is not canonical ASCII")
+    target = urllib.parse.urlsplit(raw)
+    canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, ""))
+    if canonical != raw:
+        raise PolicyError("Git target is not in exact canonical form")
+    if target.scheme or target.netloc or target.fragment:
+        raise PolicyError("Git target must be relative")
+    match = GIT_PATH_RE.fullmatch(target.path)
+    if not match:
+        raise PolicyError("Git target is outside the Atlas allowlist")
+    repo = _validate_repo(match.group("repo"))
+    operation = match.group("operation")
+    query = target.query
+    if operation == "info/refs":
+        if query not in {"service=git-upload-pack", "service=git-receive-pack"}:
+            raise PolicyError("Git discovery service is outside the allowlist")
+        service = query.removeprefix("service=")
+    elif query:
+        raise PolicyError("Git RPC query parameters are not allowed")
+    else:
+        service = operation
+    return repo, operation, service
+
+
+def _content_length(headers: object, maximum: int) -> int:
+    """Parse one short canonical bounded Content-Length header."""
+    raw = headers.get("Content-Length", "")  # type: ignore[attr-defined]
+    if (
+        not isinstance(raw, str)
+        or not raw.isascii()
+        or len(raw) > 10
+        or not raw.isdigit()
+    ):
+        raise PolicyError("SCM request length is invalid")
+    length = int(raw)
+    if raw != str(length) or not 0 <= length <= maximum:
+        raise PolicyError("SCM request length is invalid")
+    return length
+
+
+def _credential_forms(token: str) -> tuple[bytes, ...]:
+    basic = base64.b64encode(f"{GIT_USER}:{token}".encode())
+    return token.encode("utf-8"), basic, b"Basic " + basic
+
+
+def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None:
+    """Permit only vetted new-feature-branch pushes with scanned contents."""
+    validate_receive_pack(body, token, _credential_forms(token))
+
+
+def _guarded_opener(guard: deadline_http.StreamDeadline):
+    """Build an opener whose connection obeys one absolute stream deadline."""
+    return urllib.request.build_opener(RejectRedirect(), *guard.handlers()).open
+
+
+def _upstream_git_request(
+    target: str,
+    *,
+    method: str,
+    body: bytes | BinaryIO | None,
+    content_type: str | None,
+    expected_type: str,
+    token: str,
+    body_length: int | None = None,
+    stream_result: bool = False,
+    opener=None,
+) -> bytes | tuple[BinaryIO, int]:
+    credentials = base64.b64encode(f"{GIT_USER}:{token}".encode()).decode("ascii")
+    headers = {"Accept": expected_type, "Authorization": f"Basic {credentials}"}
+    if content_type:
+        headers["Content-Type"] = content_type
+    if body is not None:
+        if body_length is None:
+            if not isinstance(body, bytes):
+                raise PolicyError("Git upstream request length is missing")
+            body_length = len(body)
+        headers["Content-Length"] = str(body_length)
+    request = urllib.request.Request(
+        CANONICAL_BASE_URL + target,
+        data=body,
+        method=method,
+        headers=headers,
+    )
+    # One wall-clock budget covers connect, send, headers, and body read; the
+    # watchdog closes the upstream socket if any phase overstays it.
+    guard = deadline_http.StreamDeadline(UPSTREAM_DEADLINE_SECONDS)
+    try:
+        open_upstream = opener if opener is not None else _guarded_opener(guard)
+        with open_upstream(request, timeout=guard.remaining()) as response:
+            if _status(response) != 200:
+                raise PolicyError("Git upstream returned an unexpected HTTP status")
+            if response.headers.get_content_type() != expected_type:
+                raise PolicyError("Git upstream returned an unexpected response type")
+            result, result_length = _spool_response(
+                response, MAX_GIT_RESPONSE, token, guard.remaining()
+            )
+    finally:
+        guard.cancel()
+    if stream_result:
+        return result, result_length
+    try:
+        return result.read()
+    finally:
+        result.close()
+
+
+class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
+    """Expose only bounded metadata, draft creation, and smart-HTTP Git."""
+
+    server_version = "HermesSCMBroker/1"
+    sys_version = ""
+    header_deadline_seconds = INBOUND_HEADER_TIMEOUT
+
+    def _validate_headers(self) -> None:
+        items = list(self.headers.items())
+        if len(items) > MAX_HEADERS:
+            raise PolicyError("SCM request has too many headers")
+        total = 0
+        for name, value in items:
+            if not name.isascii() or not value.isascii():
+                raise PolicyError("SCM request headers must be ASCII")
+            total += len(name) + len(value) + 4
+            if any(ord(character) < 32 and character != "\t" for character in value):
+                raise PolicyError("SCM request header contains controls")
+        if total > MAX_HEADER_BYTES:
+            raise PolicyError("SCM request headers exceed the safe limit")
+        if len(self.headers.get_all("Content-Length", [])) > 1:
+            raise PolicyError("SCM request has duplicate Content-Length")
+
+    def _stream(
+        self, status: int, content_type: str, body: BinaryIO, length: int
+    ) -> None:
+        self.send_response(status)
+        self.send_header("Content-Type", content_type)
+        self.send_header("Content-Length", str(length))
+        self.end_headers()
+        deadline = time.monotonic() + INBOUND_BODY_TIMEOUT
+        while True:
+            chunk = body.read(STREAM_CHUNK)
+            if not chunk:
+                break
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                raise PolicyError("SCM response write deadline exceeded")
+            self.connection.settimeout(remaining)
+            self.wfile.write(chunk)
+
+    def log_message(self, _format: str, *_args: object) -> None:
+        return
+
+    def _json(self, status: int, body: bytes) -> None:
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.send_header("Cache-Control", "no-store")
+        self.end_headers()
+        self.wfile.write(body)
+
+    def _reject(self, status: int = 400) -> None:
+        self._json(status, b'{"error":"request rejected"}')
+
+    def do_GET(self) -> None:
+        try:
+            self._validate_headers()
+            if self.path == "/healthz":
+                self._json(200, b'{"status":"ok"}')
+                return
+            repo, operation, service = _git_target(self.path)
+            if operation != "info/refs":
+                raise PolicyError("Git RPC requires POST")
+            token = read_token()
+            _reject_forbidden(repo, "repository", (token,))
+            expected = f"application/x-{service}-advertisement"
+            streamed = _upstream_git_request(
+                f"/atlas/{repo}.git/info/refs?service={service}",
+                method="GET",
+                body=None,
+                content_type=None,
+                expected_type=expected,
+                token=token,
+                stream_result=True,
+            )
+            body, length = streamed
+            try:
+                self._stream(200, expected, body, length)
+            finally:
+                body.close()
+        except (
+            OSError,
+            PolicyError,
+            socket.timeout,
+            urllib.error.URLError,
+            ValueError,
+        ):
+            self._reject()
+
+    def do_POST(self) -> None:
+        try:
+            self._validate_headers()
+            self.connection.settimeout(INBOUND_BODY_TIMEOUT)
+            if self.path in {"/v1/metadata", "/v1/drafts"}:
+                self._control()
+            else:
+                self._git_rpc()
+        except (
+            OSError,
+            PolicyError,
+            socket.timeout,
+            urllib.error.URLError,
+            ValueError,
+            json.JSONDecodeError,
+        ):
+            self._reject()
+
+    def _control(self) -> None:
+        data = _load_json(self)
+        token = read_token()
+        if self.path == "/v1/metadata":
+            if set(data) != {"path"} or not isinstance(data["path"], str):
+                raise PolicyError("metadata request fields are invalid")
+            result = read(data["path"], token=token)
+        else:
+            expected = {"base", "body", "head", "head_sha", "repo", "title"}
+            if set(data) != expected or not all(
+                isinstance(data[key], str) for key in expected
+            ):
+                raise PolicyError("draft request fields are invalid")
+            result = create_draft(token=token, **data)  # type: ignore[arg-type]
+        if token.encode("utf-8") in result:
+            raise PolicyError("SCM upstream reflected credential material")
+        self._json(200, result)
+
+    def _git_rpc(self) -> None:
+        repo, operation, service = _git_target(self.path)
+        if (
+            operation not in {"git-upload-pack", "git-receive-pack"}
+            or service != operation
+        ):
+            raise PolicyError("Git RPC operation is outside the allowlist")
+        expected_request = f"application/x-{service}-request"
+        if self.headers.get_content_type() != expected_request or self.headers.get(
+            "Transfer-Encoding"
+        ):
+            raise PolicyError("Git RPC request type is invalid")
+        length = _content_length(self.headers, MAX_GIT_REQUEST)
+        token = read_token()
+        _reject_forbidden(repo, "repository", (token,))
+        body, body_length = _spool_bounded(
+            self.rfile,
+            MAX_GIT_REQUEST,
+            length,
+            token=token,
+            context="Git request",
+            deadline=time.monotonic() + INBOUND_BODY_TIMEOUT,
+            set_timeout=self.connection.settimeout,
+        )
+        try:
+            if service == "git-receive-pack":
+                _validate_receive_pack(body, token)
+            expected = f"application/x-{service}-result"
+            streamed = _upstream_git_request(
+                f"/atlas/{repo}.git/{service}",
+                method="POST",
+                body=body,
+                body_length=body_length,
+                content_type=expected_request,
+                expected_type=expected,
+                token=token,
+                stream_result=True,
+            )
+            result, result_length = streamed
+            try:
+                self._stream(200, expected, result, result_length)
+            finally:
+                result.close()
+        finally:
+            body.close()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--listen", default="0.0.0.0")
+    parser.add_argument("--port", type=int, default=BROKER_PORT)
+    args = parser.parse_args()
+    BoundedThreadingHTTPServer((args.listen, args.port), BrokerHandler).serve_forever()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

services/hermes/scm-common/scripts/scm_broker_client.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Call the credential-isolated Hermes SCM broker over its fixed cluster origin."""
+
+from __future__ import annotations
+
+import json
+import urllib.request
+from collections.abc import Callable
+
+import deadline_http
+from gitea_api_policy import PolicyError
+
+BROKER_ORIGIN = "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081"
+MAX_RESPONSE_BYTES = 2 * 1024 * 1024
+
+
+def _open(request: urllib.request.Request, timeout: int):
+    """Open one broker exchange under a killable absolute deadline.
+
+    The helper process keeps every request on its fixed in-cluster origin by
+    rejecting redirects before they are followed.
+    """
+    return deadline_http.open_bounded(
+        request, maximum=MAX_RESPONSE_BYTES, timeout=timeout
+    )
+
+
+def request(
+    endpoint: str,
+    payload: dict[str, object],
+    *,
+    opener: Callable[..., object] = _open,
+) -> bytes:
+    """Send one bounded broker operation without any repository credential."""
+    if endpoint not in {"/v1/metadata", "/v1/drafts"}:
+        raise PolicyError("SCM broker operation is outside the client allowlist")
+    body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
+    if len(body) > 64 * 1024:
+        raise PolicyError("SCM broker request exceeds the safe size limit")
+    outgoing = urllib.request.Request(
+        BROKER_ORIGIN + endpoint,
+        data=body,
+        method="POST",
+        headers={
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+            "User-Agent": "hermes-scm-broker-client/1",
+        },
+    )
+    with opener(outgoing, timeout=30) as response:  # type: ignore[attr-defined]
+        status = getattr(response, "status", None)
+        if status is None and hasattr(response, "getcode"):
+            status = response.getcode()  # type: ignore[attr-defined]
+        if status != 200:
+            raise PolicyError("SCM broker returned an unexpected HTTP status")
+        content_type = response.headers.get_content_type()  # type: ignore[attr-defined]
+        if content_type != "application/json":
+            raise PolicyError("SCM broker returned an unexpected response type")
+        result = response.read(MAX_RESPONSE_BYTES + 1)  # type: ignore[attr-defined]
+    if len(result) > MAX_RESPONSE_BYTES:
+        raise PolicyError("SCM broker response exceeds the safe size limit")
+    json.loads(result)
+    return result
+
+
+def read(path: str, *, opener: Callable[..., object] = _open) -> bytes:
+    """Read explicitly allowed Atlas metadata through the broker."""
+    return request("/v1/metadata", {"path": path}, opener=opener)
+
+
+def create_draft(
+    repo: str,
+    *,
+    base: str,
+    head: str,
+    head_sha: str,
+    title: str,
+    body: str,
+    opener: Callable[..., object] = _open,
+) -> bytes:
+    """Create one verified draft through the broker for human review."""
+    return request(
+        "/v1/drafts",
+        {
+            "base": base,
+            "body": body,
+            "head": head,
+            "head_sha": head_sha,
+            "repo": repo,
+            "title": title,
+        },
+        opener=opener,
+    )

services/hermes/scm-common/scripts/scm_broker_io.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Bounded-memory stream helpers for the Hermes SCM broker."""
+
+from __future__ import annotations
+
+import tempfile
+import time
+import urllib.request
+from typing import BinaryIO
+
+from gitea_api_policy import PolicyError
+
+
+class RejectRedirect(urllib.request.HTTPRedirectHandler):
+    """Reject every upstream redirect before credentials can be forwarded."""
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        raise PolicyError("SCM upstream redirects are not allowed")
+
+
+def response_status(response: object) -> int | None:
+    """Read an HTTP status from either urllib response interface."""
+    value = getattr(response, "status", None)
+    if value is None and hasattr(response, "getcode"):
+        value = response.getcode()  # type: ignore[attr-defined]
+    return value
+
+
+def spool_response(
+    response,
+    maximum: int,
+    forbidden: tuple[bytes, ...],
+    *,
+    memory_limit: int,
+    chunk_size: int,
+    deadline_seconds: float,
+) -> tuple[BinaryIO, int]:
+    """Spool an upstream response to disk while scanning chunk boundaries."""
+    spool = tempfile.SpooledTemporaryFile(  # noqa: SIM115 - caller owns returned spool
+        max_size=memory_limit, dir="/tmp"
+    )
+    carry = b""
+    total = 0
+    deadline = time.monotonic() + deadline_seconds
+    try:
+        while True:
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                raise PolicyError("SCM upstream response deadline exceeded")
+            raw = getattr(getattr(response, "fp", None), "raw", None)
+            sock = getattr(raw, "_sock", raw)
+            if hasattr(sock, "settimeout"):
+                sock.settimeout(remaining)
+            chunk = response.read(chunk_size)
+            if not chunk:
+                break
+            total += len(chunk)
+            if total > maximum:
+                raise PolicyError("SCM upstream response exceeds the safe size limit")
+            candidate = carry + chunk
+            if any(value in candidate for value in forbidden):
+                raise PolicyError(
+                    "Git upstream response contains runtime credential material"
+                )
+            width = max(len(value) for value in forbidden) - 1
+            carry = candidate[-width:] if width else b""
+            spool.write(chunk)
+        spool.seek(0)
+        return spool, total
+    except Exception:
+        spool.close()
+        raise

services/hermes/scm-common/scripts/scm_broker_server.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""Concurrency-bounded threaded HTTP server for the Hermes SCM broker."""
+
+from __future__ import annotations
+
+import threading
+import time
+from http.server import ThreadingHTTPServer
+
+MAX_CONCURRENT_REQUESTS = 3
+
+
+class _AbsoluteDeadlineReader:
+    """Read header lines against one wall-clock deadline, not idle timeouts."""
+
+    def __init__(self, stream, connection):
+        self._stream = stream
+        self._connection = connection
+        self._deadline: float | None = None
+
+    def begin(self, timeout: float) -> None:
+        self._deadline = time.monotonic() + timeout
+
+    def end(self) -> None:
+        self._deadline = None
+
+    def readline(self, limit: int = -1) -> bytes:
+        if self._deadline is None:
+            return self._stream.readline(limit)
+        value = bytearray()
+        while limit < 0 or len(value) < limit:
+            remaining = self._deadline - time.monotonic()
+            if remaining <= 0:
+                raise TimeoutError("absolute SCM header deadline exceeded")
+            self._connection.settimeout(remaining)
+            character = self._stream.read(1)
+            if not character:
+                break
+            value.extend(character)
+            if character == b"\n":
+                break
+        return bytes(value)
+
+    def __getattr__(self, name: str):
+        return getattr(self._stream, name)
+
+
+class AbsoluteHeaderDeadlineMixin:
+    """Apply an absolute request-line plus header deadline to HTTP handlers."""
+
+    header_deadline_seconds = 10.0
+
+    def setup(self) -> None:
+        super().setup()
+        self._header_reader = _AbsoluteDeadlineReader(self.rfile, self.connection)
+        self.rfile = self._header_reader
+
+    def handle_one_request(self) -> None:
+        try:
+            self._header_reader.begin(self.header_deadline_seconds)
+            try:
+                self.raw_requestline = self.rfile.readline(65537)
+                if len(self.raw_requestline) > 65536:
+                    self.requestline = ""
+                    self.request_version = ""
+                    self.command = ""
+                    self.send_error(414)
+                    return
+                if not self.raw_requestline:
+                    self.close_connection = True
+                    return
+                if not self.parse_request():
+                    return
+            finally:
+                self._header_reader.end()
+            method_name = "do_" + self.command
+            if not hasattr(self, method_name):
+                self.send_error(501, "Unsupported method")
+                return
+            getattr(self, method_name)()
+            self.wfile.flush()
+        except TimeoutError:
+            self.close_connection = True
+
+
+class BoundedThreadingHTTPServer(ThreadingHTTPServer):
+    """Cap active handlers so large Git exchanges cannot exhaust memory."""
+
+    daemon_threads = True
+    block_on_close = True
+    request_queue_size = 16
+
+    def __init__(self, *args, **kwargs):
+        self._slots = threading.BoundedSemaphore(MAX_CONCURRENT_REQUESTS)
+        super().__init__(*args, **kwargs)
+
+    def process_request(self, request, client_address) -> None:
+        if not self._slots.acquire(blocking=False):
+            try:
+                request.sendall(
+                    b"HTTP/1.1 503 Service Unavailable\r\n"
+                    b"Connection: close\r\nContent-Length: 0\r\n\r\n"
+                )
+            finally:
+                self.shutdown_request(request)
+            return
+        try:
+            super().process_request(request, client_address)
+        except Exception:
+            self._slots.release()
+            self.shutdown_request(request)
+            raise
+
+    def process_request_thread(self, request, client_address) -> None:
+        try:
+            super().process_request_thread(request, client_address)
+        finally:
+            self._slots.release()

services/hermes/scripts/cli_lane_board.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""Bounded Kanban access and workspace resolution for Hermes CLI workers."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import sys
+import time
+from pathlib import Path
+from typing import Any, Callable
+
+from cli_lane_config import (
+    BOARD_CORRUPTION_ERRORS,
+    EXTERNAL_PREFIX,
+    KANBAN_STORAGE_ATTEMPTS,
+    WORKTREE_LOCK,
+)
+
+
+def _task_value(task: Any, name: str, default: Any = None) -> Any:
+    return getattr(task, name, default)
+
+def _task_context(kanban_db: Any, conn: Any, task_id: str) -> str:
+    value = kanban_db.build_worker_context(conn, task_id)
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, indent=2, default=str)
+
+def _resolve_workspace(kanban_db: Any, conn: Any, task: Any, board: str) -> Path:
+    # External coding lanes always receive their own linked worktree. A task
+    # that does not resolve to a Git repository is blocked rather than sharing
+    # a mutable checkout with another unattended worker.
+    # Git worktree creation updates common-repository metadata. Serialize that
+    # short materialization step while allowing the provider workers themselves
+    # to run concurrently in independent worktrees.
+    with WORKTREE_LOCK:
+        value, branch_name = kanban_db._resolve_worktree_workspace(task, board=board)
+    kanban_db.set_branch_name(conn, str(_task_value(task, "id")), branch_name)
+    return Path(value).resolve()
+
+def _record_board_access_error(board: str, error: Exception) -> None:
+    """Report one board access failure once while allowing other lanes to run."""
+    failure_kind = "storage" if isinstance(error, (OSError, sqlite3.Error)) else "access"
+    detail = f"{failure_kind} {type(error).__name__}: {error}"
+    if BOARD_CORRUPTION_ERRORS.get(board) == detail:
+        return
+    print(
+        f"temporarily skipping Kanban board {board!r}: {detail}",
+        file=sys.stderr,
+        flush=True,
+    )
+    BOARD_CORRUPTION_ERRORS[board] = detail
+
+def _board_call(
+    kanban_db: Any,
+    board: str,
+    operation: Callable[[Any], Any],
+) -> Any:
+    """Run one bounded Kanban operation on a fresh, promptly closed connection."""
+    last_storage_error: Exception | None = None
+    for attempt in range(KANBAN_STORAGE_ATTEMPTS):
+        conn = None
+        try:
+            with kanban_db.scoped_current_board(board):
+                conn = kanban_db.connect(board=board)
+                result = operation(conn)
+            BOARD_CORRUPTION_ERRORS.pop(board, None)
+            return result
+        except (OSError, sqlite3.Error) as error:
+            last_storage_error = error
+            _record_board_access_error(board, error)
+            if attempt + 1 < KANBAN_STORAGE_ATTEMPTS:
+                time.sleep(0.2 * (attempt + 1))
+        finally:
+            if conn is not None:
+                conn.close()
+    assert last_storage_error is not None
+    raise last_storage_error
+
+def _external(task: Any) -> bool:
+    return str(_task_value(task, "assignee", "") or "").startswith(EXTERNAL_PREFIX)

services/hermes/scripts/cli_lane_capabilities.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+"""Detect and publish the Kanban API boundary required by CLI lanes.
+
+During a rollout the coordinator container and the cli-lane-runner worker can
+run different hermes-agent image generations against one shared Kanban DB.
+The image APIs that differ on the decomposition/finalization path:
+
+* ``complete_task``: both generations accept ``expected_run_id`` (active
+  exact-run completion); only the patched image accepts
+  ``replay_ended_run_id`` (ended-run replay).
+* ``reclaim_task``: only the patched image accepts ``expected_run_id``
+  (guarded orphan reclaim after a runner restart).
+* ``specify_triage_task``/``decompose_triage_task``: only the patched image
+  accepts ``require_no_runs`` (refuses redefining tasks with run history).
+
+Worker-side handling per combination (proven in
+testing/tests/test_hermes_cli_capabilities.py):
+
+* legacy worker: health/readiness stay deferred, no new claims, no unguarded
+  reclaim, replay-needing results stay journaled for a patched successor.
+* patched worker: ready health, exact completion, guarded replay/reclaim; a
+  run voided by a legacy coordinator's unguarded mid-run decomposition
+  converges to stale/conflict evidence instead of completing.
+* patched coordinator: mid-run decomposition is refused entirely, so the
+  active exact run completes on either worker generation.
+"""
+
+from __future__ import annotations
+
+import inspect
+import json
+import os
+import sys
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Callable
+
+from cli_lane_config import STATE_ROOT, utc_now
+from cli_lane_files import atomic_json, load_json
+
+
+_CAPABILITIES: KanbanCapabilities | None = None
+_CAPABILITY_SOURCE: tuple[int, int] | None = None
+
+
+@dataclass(frozen=True)
+class KanbanCapabilities:
+    """Safety-relevant keyword support exposed by the image's Kanban DB."""
+
+    exact_run_completion: bool
+    ended_run_replay: bool
+    exact_run_reclaim: bool
+
+    @property
+    def ready(self) -> bool:
+        """Return whether dispatch and every recovery transition are safe."""
+        return (
+            self.exact_run_completion
+            and self.ended_run_replay
+            and self.exact_run_reclaim
+        )
+
+    @property
+    def deferred_features(self) -> tuple[str, ...]:
+        """List fixed, bounded capability names absent from the image API."""
+        missing = []
+        if not self.exact_run_completion:
+            missing.append("exact-run-completion")
+        if not self.ended_run_replay:
+            missing.append("ended-run-replay")
+        if not self.exact_run_reclaim:
+            missing.append("exact-run-reclaim")
+        return tuple(missing)
+
+
+def _explicit_keyword(function: Callable[..., Any], keyword: str) -> bool:
+    """Require an explicit keyword parameter rather than trusting ``**kwargs``."""
+    try:
+        parameter = inspect.signature(function).parameters.get(keyword)
+    except (TypeError, ValueError):
+        return False
+    return parameter is not None and parameter.kind in {
+        inspect.Parameter.POSITIONAL_OR_KEYWORD,
+        inspect.Parameter.KEYWORD_ONLY,
+    }
+
+
+def detect_kanban_capabilities(kanban_db: Any) -> KanbanCapabilities:
+    """Inspect the loaded image API without invoking a mutating DB operation."""
+    complete_task = getattr(kanban_db, "complete_task", None)
+    reclaim_task = getattr(kanban_db, "reclaim_task", None)
+    return KanbanCapabilities(
+        exact_run_completion=callable(complete_task)
+        and _explicit_keyword(complete_task, "expected_run_id"),
+        ended_run_replay=callable(complete_task)
+        and _explicit_keyword(complete_task, "replay_ended_run_id"),
+        exact_run_reclaim=callable(reclaim_task)
+        and _explicit_keyword(reclaim_task, "expected_run_id"),
+    )
+
+
+def _source_identity(kanban_db: Any) -> tuple[int, int]:
+    """Bind cached capability evidence to the two inspected callables."""
+    return (
+        id(getattr(kanban_db, "complete_task", None)),
+        id(getattr(kanban_db, "reclaim_task", None)),
+    )
+
+
+def _health_document(capabilities: KanbanCapabilities) -> dict[str, Any]:
+    """Return a bounded operator-readable compatibility health document."""
+    return {
+        "component": "cli-lane-runner",
+        "schema_version": 1,
+        "state": "ready" if capabilities.ready else "deferred",
+        "ready": capabilities.ready,
+        "active_run_completion_safe": capabilities.exact_run_completion,
+        "deferred_features": list(capabilities.deferred_features),
+        "capabilities": asdict(capabilities),
+        "observed_at": utc_now(),
+    }
+
+
+def initialize_kanban_capabilities(
+    kanban_db: Any,
+    *,
+    health_path: Path | None = None,
+) -> KanbanCapabilities:
+    """Detect startup compatibility and durably expose ready/deferred health."""
+    global _CAPABILITIES, _CAPABILITY_SOURCE
+    capabilities = detect_kanban_capabilities(kanban_db)
+    _CAPABILITIES = capabilities
+    _CAPABILITY_SOURCE = _source_identity(kanban_db)
+    destination = health_path or STATE_ROOT / "runtime-health.json"
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    atomic_json(destination, _health_document(capabilities), 0o600)
+    if not capabilities.ready:
+        print(
+            "CLI lane safety compatibility deferred; missing image APIs: "
+            + ", ".join(capabilities.deferred_features),
+            file=sys.stderr,
+            flush=True,
+        )
+    return capabilities
+
+
+def refresh_kanban_capabilities(
+    kanban_db: Any,
+    *,
+    health_path: Path | None = None,
+) -> KanbanCapabilities:
+    """Reinspect the live callables and refresh bounded readiness evidence.
+
+    ConfigMap script updates and image changes are deliberately not imported or
+    reloaded here.  A running process observes only the already-loaded module,
+    which prevents a half-old/half-new Python module graph.  Tests may replace
+    the two callables to model an image API transition at a loop boundary.
+    """
+    return initialize_kanban_capabilities(kanban_db, health_path=health_path)
+
+
+def kanban_capabilities(kanban_db: Any) -> KanbanCapabilities:
+    """Return startup evidence, redetecting only when the loaded API changes."""
+    if _CAPABILITIES is None or _source_identity(kanban_db) != _CAPABILITY_SOURCE:
+        return initialize_kanban_capabilities(kanban_db)
+    return _CAPABILITIES
+
+
+def runtime_health(path: Path | None = None) -> dict[str, Any]:
+    """Read the bounded health document for probes and operator diagnostics."""
+    document = load_json(path or STATE_ROOT / "runtime-health.json")
+    return document if isinstance(document, dict) else {}
+
+
+def readiness_issue(
+    path: Path | None = None,
+    *,
+    now: datetime | None = None,
+    maximum_age_seconds: float = 60.0,
+) -> str | None:
+    """Return a bounded readiness reason without exposing health contents."""
+    destination = path or STATE_ROOT / "runtime-health.json"
+    try:
+        document = json.loads(destination.read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        return "runtime health is missing"
+    except (OSError, UnicodeError, json.JSONDecodeError):
+        return "runtime health is malformed"
+    if not isinstance(document, dict) or document.get("schema_version") != 1:
+        return "runtime health is malformed"
+    if document.get("state") == "deferred" and document.get("ready") is False:
+        return "runtime compatibility is deferred"
+    capabilities = document.get("capabilities")
+    if (
+        document.get("state") != "ready"
+        or document.get("ready") is not True
+        or not isinstance(capabilities, dict)
+        or any(
+            capabilities.get(name) is not True
+            for name in (
+                "exact_run_completion",
+                "ended_run_replay",
+                "exact_run_reclaim",
+            )
+        )
+    ):
+        return "runtime health is malformed"
+    observed_at = document.get("observed_at")
+    if not isinstance(observed_at, str):
+        return "runtime health is malformed"
+    try:
+        observed = datetime.fromisoformat(observed_at.replace("Z", "+00:00"))
+    except ValueError:
+        return "runtime health is malformed"
+    if observed.tzinfo is None:
+        return "runtime health is malformed"
+    current = now or datetime.now(timezone.utc)
+    age = (
+        current.astimezone(timezone.utc) - observed.astimezone(timezone.utc)
+    ).total_seconds()
+    if age < -5.0 or age > maximum_age_seconds:
+        return "runtime health is stale"
+    return None
+
+
+def main() -> int:
+    """Provide the non-mutating Kubernetes readiness command."""
+    maximum_age = float(os.environ.get("HERMES_CLI_HEALTH_MAX_AGE_SECONDS", "60"))
+    issue = readiness_issue(maximum_age_seconds=maximum_age)
+    if issue:
+        print(issue, file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

services/hermes/scripts/cli_lane_config.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+"""Shared immutable configuration and result types for Hermes CLI lanes."""
+
+from __future__ import annotations
+
+import os
+import re
+import threading
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+import cli_lane_goal
+
+
+DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data"))
+SWITCHYARD_URL = os.environ.get(
+    "HERMES_SWITCHYARD_URL",
+    "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1/chat/completions",
+)
+STATE_ROOT = DATA_ROOT / "cli-lanes"
+CODEX_BIN = DATA_ROOT / "tools/bin/codex"
+CLAUDE_BIN = DATA_ROOT / "tools/bin/claude"
+CLAUDE_SETTINGS = DATA_ROOT / "home/.claude/settings.json"
+RESULT_SCHEMA_PATH = STATE_ROOT / "worker-result.schema.json"
+EFFORTS = ("low", "medium", "high", "xhigh")
+EXTERNAL_PREFIX = "cli-"
+DEFAULT_CLAIM_TTL = 7 * 24 * 60 * 60
+DEFAULT_MAX_RUNTIME = 12 * 60 * 60
+HEARTBEAT_SECONDS = 20
+PROVIDER_HEALTH_MAX_AGE_SECONDS = 5 * 60
+PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS = 12 * 60 * 60
+QUOTA_MIN_REMAINING_PERCENT_DEFAULT = 15.0
+CAPACITY_COOLDOWN_SECONDS_DEFAULT = 300.0
+AUTH_COOLDOWN_SECONDS_DEFAULT = 3600.0
+QUOTA_METRICS_URL = os.environ.get(
+    "HERMES_CLI_QUOTA_METRICS_URL", "http://127.0.0.1:9010/metrics"
+)
+KANBAN_STORAGE_ATTEMPTS = 5
+ARTIFACT_GC_INTERVAL_SECONDS = 5 * 60
+ARTIFACT_RETENTION_AGE_SECONDS = 30 * 24 * 60 * 60
+ARTIFACT_RETENTION_COUNT = 256
+ARTIFACT_RETENTION_BYTES = 128 * 1024 * 1024
+MAX_TERMINAL_RECORD_BYTES = 1024 * 1024
+QUARANTINE_HASH_BYTES = 64 * 1024
+SQLITE_RUN_ID_MAX = 2**63 - 1
+PROVIDER_HEALTH_PATHS = {
+    "codex": DATA_ROOT / "provider-health/codex.json",
+    "claude": DATA_ROOT / "provider-health/claude.json",
+}
+WORKTREE_LOCK = threading.Lock()
+BOARD_CORRUPTION_ERRORS: dict[str, str] = {}
+LAST_ARTIFACT_GC = 0.0
+CAPACITY_PATTERN = re.compile(
+    r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529"
+    r"|authentication|unauthorized|forbidden|oauth|token.*expired|401|403)",
+    re.I,
+)
+NO_CLAUDE_SESSION = "No conversation found with session ID:"
+CLAUDE_SESSION_COLLISION = "Session ID already in use"
+NO_CODEX_THREAD = "no rollout found for thread id"
+
+
+RESULT_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "additionalProperties": False,
+    "required": [
+        "status",
+        "summary",
+        "changed_files",
+        "tests_run",
+        "artifacts",
+        "findings",
+        "blockers",
+    ],
+    "properties": {
+        "status": {
+            "type": "string",
+            "enum": sorted(cli_lane_goal.RESULT_STATUSES),
+        },
+        "summary": {"type": "string"},
+        "changed_files": {"type": "array", "items": {"type": "string"}},
+        "tests_run": {"type": "array", "items": {"type": "string"}},
+        "artifacts": {"type": "array", "items": {"type": "string"}},
+        "findings": {
+            "type": "array",
+            "items": {"type": "string"},
+            "description": (
+                "Defects, risks, or observations discovered by a review or diagnosis. "
+                "Findings do not prevent the assigned review or diagnosis from completing."
+            ),
+        },
+        "blockers": {
+            "type": "array",
+            "items": {"type": "string"},
+            "description": (
+                "Concrete obstacles that prevent completion of the assigned task itself. "
+                "This must be empty when status is completed; review findings belong in findings."
+            ),
+        },
+    },
+}
+
+
+@dataclass(frozen=True)
+class Route:
+    """One provider/model/effort decision for an observable worker boundary."""
+
+    provider: str
+    model: str
+    effort: str
+    profile: str
+    classifier: str
+    reason: str
+    latency_ms: int
+    fallback_chain: tuple[str, ...]
+
+
+@dataclass
+class ProcessResult:
+    """Captured outcome from one provider CLI invocation."""
+
+    returncode: int
+    output: str
+    structured: dict[str, Any] | None
+    capacity_failure: bool
+
+
+class TerminalFinalizationPending(RuntimeError):
+    """An accepted worker result is durable but not committed to Kanban yet."""
+
+
+def canonical_run_id(value: object) -> int | None:
+    """Return one positive SQLite-safe run ID from an int or canonical decimal."""
+    if type(value) is int:
+        candidate = value
+    elif type(value) is str:
+        # Bound conversion before ``int`` so an attacker cannot hand Python an
+        # arbitrarily large decimal from an otherwise lexically valid name.
+        if not re.fullmatch(r"[1-9][0-9]{0,18}", value):
+            return None
+        candidate = int(value)
+    else:
+        return None
+    if not 1 <= candidate <= SQLITE_RUN_ID_MAX:
+        return None
+    return candidate
+
+
+@dataclass(frozen=True)
+class TerminalIdentity:
+    """Replay authority derived only from a journal's directory and filename."""
+
+    board: str
+    task_id: str
+    run_id: int
+    state: str
+
+    def __post_init__(self) -> None:
+        """Reject noncanonical identities before they can reach SQLite."""
+        if type(self.run_id) is not int or canonical_run_id(self.run_id) is None:
+            raise ValueError("terminal run identity is not a positive SQLite int64")
+
+
+@dataclass
+class TerminalSnapshot:
+    """One inode-bound, bounded journal read held open across finalization."""
+
+    document: dict[str, Any]
+    file_stat: os.stat_result
+    descriptor: int
+    directory_descriptor: int
+
+    def close(self) -> None:
+        """Release the pinned file and directory descriptors."""
+        os.close(self.descriptor)
+        os.close(self.directory_descriptor)
+
+
+@dataclass
+class TerminalRecoverySnapshot:
+    """One raw directory entry pinned before recovery classifies its payload."""
+
+    document: dict[str, Any] | None
+    file_stat: os.stat_result
+    descriptor: int | None
+    directory_descriptor: int
+    prefix: bytes
+    invalid_reason: str | None
+
+    def close(self) -> None:
+        """Release the descriptors retained across classification/quarantine."""
+        if self.descriptor is not None:
+            os.close(self.descriptor)
+            self.descriptor = None
+        if self.directory_descriptor >= 0:
+            os.close(self.directory_descriptor)
+            self.directory_descriptor = -1
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def kanban_setting(name: str, default: float) -> float:
+    """Read one numeric routing setting from the deployed kanban config block."""
+    try:
+        document = yaml.safe_load(
+            (DATA_ROOT / "config.yaml").read_text(encoding="utf-8")
+        )
+    except (OSError, yaml.YAMLError):
+        return default
+    kanban = document.get("kanban") if isinstance(document, dict) else None
+    value = kanban.get(name) if isinstance(kanban, dict) else None
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return default
+    return float(value)

services/hermes/scripts/cli_lane_dispatch.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""Concurrent board recovery, claim, and dispatch loop for CLI workers."""
+
+from __future__ import annotations
+
+import concurrent.futures
+import os
+import sys
+import time
+from collections.abc import Callable
+from typing import Any
+
+from cli_lane_board import _external, _record_board_access_error, _task_value
+from cli_lane_capabilities import (
+    initialize_kanban_capabilities,
+    kanban_capabilities,
+    refresh_kanban_capabilities,
+)
+from cli_lane_config import (
+    BOARD_CORRUPTION_ERRORS,
+    DEFAULT_CLAIM_TTL,
+    EXTERNAL_PREFIX,
+    RESULT_SCHEMA,
+    RESULT_SCHEMA_PATH,
+)
+from cli_lane_execution import execute_claim
+from cli_lane_files import atomic_json
+from cli_lane_metrics import start_metrics_server
+from cli_lane_recovery import _has_pending_finalization, recover_pending_finalizations
+from cli_lane_retention import maybe_gc_lane_artifacts
+
+
+def _board_slug(board: Any) -> str:
+    if isinstance(board, dict):
+        return str(board.get("slug") or board.get("id") or "")
+    return str(getattr(board, "slug", None) or getattr(board, "id", None) or board)
+
+
+def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None:
+    """Open one board without letting localized storage faults stop other lanes."""
+    try:
+        return kanban_db.connect(board=board)
+    except Exception as error:
+        _record_board_access_error(board, error)
+        return None
+
+
+def recover_orphans() -> None:
+    """Return external running tasks to ready after a runner/pod restart."""
+    from hermes_cli import kanban_db
+
+    recover_pending_finalizations()
+    if not kanban_capabilities(kanban_db).exact_run_reclaim:
+        return
+    try:
+        boards = kanban_db.list_boards(include_archived=False)
+    except Exception as error:
+        _record_board_access_error("board-registry", error)
+        return
+    for raw_board in boards:
+        board = _board_slug(raw_board)
+        if not board:
+            continue
+        with kanban_db.scoped_current_board(board):
+            conn = _connect_healthy_board(kanban_db, board)
+            if conn is None:
+                continue
+            try:
+                for task in kanban_db.list_tasks(conn):
+                    if (
+                        _external(task)
+                        and str(_task_value(task, "status", "")) == "running"
+                    ):
+                        task_id = str(_task_value(task, "id"))
+                        run_id = _task_value(task, "current_run_id", None)
+                        if not isinstance(run_id, int):
+                            continue
+                        if _has_pending_finalization(board, task_id, run_id):
+                            continue
+                        kanban_db.reclaim_task(
+                            conn,
+                            task_id,
+                            reason="direct CLI lane restarted; provider session will resume",
+                            expected_run_id=run_id,
+                        )
+                BOARD_CORRUPTION_ERRORS.pop(board, None)
+            except Exception as error:
+                _record_board_access_error(board, error)
+            finally:
+                conn.close()
+
+
+def claim_ready(
+    active: set[tuple[str, str]],
+    limit: int,
+    eligible: Callable[[str, Any], bool] | None = None,
+) -> list[tuple[str, str]]:
+    """Atomically claim external ready tasks across all non-archived boards."""
+    from hermes_cli import kanban_db
+
+    claimed: list[tuple[str, str]] = []
+    if limit <= 0:
+        return claimed
+    for raw_board in kanban_db.list_boards(include_archived=False):
+        board = _board_slug(raw_board)
+        if not board:
+            continue
+        with kanban_db.scoped_current_board(board):
+            conn = _connect_healthy_board(kanban_db, board)
+            if conn is None:
+                continue
+            try:
+                kanban_db.recompute_ready(conn)
+                tasks = kanban_db.list_tasks(conn)
+                BOARD_CORRUPTION_ERRORS.pop(board, None)
+                for task in tasks:
+                    task_id = str(_task_value(task, "id", ""))
+                    assignee = str(_task_value(task, "assignee", "") or "")
+                    if (
+                        task_id
+                        and not assignee
+                        and str(_task_value(task, "status", "")) == "ready"
+                        and kanban_db.assign_task(conn, task_id, "cli-auto")
+                    ):
+                        task = kanban_db.get_task(conn, task_id)
+                        assignee = "cli-auto"
+                    if (
+                        not task_id
+                        or (board, task_id) in active
+                        or not assignee.startswith(EXTERNAL_PREFIX)
+                        or str(_task_value(task, "status", "")) != "ready"
+                        or (eligible is not None and not eligible(board, task))
+                    ):
+                        continue
+                    try:
+                        result = kanban_db.claim_task(
+                            conn,
+                            task_id,
+                            ttl_seconds=DEFAULT_CLAIM_TTL,
+                            claimer="direct-cli-lane",
+                        )
+                    except Exception:
+                        continue
+                    if result is not None:
+                        claimed.append((board, task_id))
+                    if len(claimed) >= limit:
+                        return claimed
+            except Exception as error:
+                _record_board_access_error(board, error)
+            finally:
+                conn.close()
+    return claimed
+
+
+def main() -> int:
+    """Continuously bridge external Kanban lanes to provider CLIs.
+
+    The loop is exited only by process signals or unrecoverable exceptions;
+    every per-board and per-claim fault is bounded inside one pass.
+    """
+    from hermes_cli import kanban_db
+
+    RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
+    atomic_json(RESULT_SCHEMA_PATH, RESULT_SCHEMA, 0o644)
+    start_metrics_server()
+    capabilities = initialize_kanban_capabilities(kanban_db)
+    recover_orphans()
+    workers = max(1, min(int(os.environ.get("HERMES_CLI_LANE_CONCURRENCY", "4")), 8))
+    futures: dict[concurrent.futures.Future[None], tuple[str, str]] = {}
+    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
+        while True:
+            previous_capabilities = capabilities
+            capabilities = refresh_kanban_capabilities(kanban_db)
+            if capabilities.ready and not previous_capabilities.ready:
+                recover_orphans()
+            for future in list(futures):
+                if future.done():
+                    try:
+                        future.result()
+                    except Exception as error:
+                        print(
+                            f"worker future failed: {error}",
+                            file=sys.stderr,
+                            flush=True,
+                        )
+                    del futures[future]
+            recover_pending_finalizations()
+            maybe_gc_lane_artifacts()
+            active = set(futures.values())
+            try:
+                newly_claimed = (
+                    claim_ready(active, workers - len(futures))
+                    if capabilities.ready
+                    else []
+                )
+                BOARD_CORRUPTION_ERRORS.pop("board-registry", None)
+            except Exception as error:
+                _record_board_access_error("board-registry", error)
+                newly_claimed = []
+            for board, task_id in newly_claimed:
+                future = pool.submit(execute_claim, board, task_id)
+                futures[future] = (board, task_id)
+            time.sleep(5)

services/hermes/scripts/cli_lane_evidence.py

@@ -0,0 +1,294 @@
+#!/usr/bin/env python3
+"""Immutable terminal evidence creation and identity-safe retirement."""
+
+from __future__ import annotations
+
+import ctypes
+import contextlib
+import errno
+import hashlib
+import json
+import os
+import stat
+import uuid
+from pathlib import Path
+from typing import Any
+
+from cli_lane_config import MAX_TERMINAL_RECORD_BYTES, TerminalIdentity, utc_now
+from cli_lane_files import _terminal_evidence_identity, state_path
+from cli_lane_records import _load_small_json, _terminal_record_valid
+
+
+def _rename_noreplace(
+    source: str,
+    destination: str,
+    *,
+    source_dir: int,
+    destination_dir: int,
+) -> None:
+    """Atomically rename a directory entry without replacing a collision."""
+    renameat2 = getattr(ctypes.CDLL(None, use_errno=True), "renameat2", None)
+    if renameat2 is None:
+        raise OSError(errno.ENOSYS, "renameat2 is unavailable")
+    renameat2.argtypes = (
+        ctypes.c_int,
+        ctypes.c_char_p,
+        ctypes.c_int,
+        ctypes.c_char_p,
+        ctypes.c_uint,
+    )
+    renameat2.restype = ctypes.c_int
+    result = renameat2(
+        source_dir,
+        ctypes.c_char_p(os.fsencode(source)),
+        destination_dir,
+        ctypes.c_char_p(os.fsencode(destination)),
+        1,  # RENAME_NOREPLACE
+    )
+    if result != 0:
+        error = ctypes.get_errno()
+        raise OSError(error, os.strerror(error), source, destination)
+
+def _retire_terminal_entry(
+    path: Path,
+    source_stat: os.stat_result,
+    *,
+    board_descriptor: int,
+    quarantine_descriptor: int,
+    authority_name: str | None = None,
+    source_descriptor: int | None = None,
+) -> str:
+    """Retire only the inode previously inspected; preserve any replacement."""
+    pinned = os.fstat(source_descriptor) if source_descriptor is not None else source_stat
+    if pinned.st_dev != source_stat.st_dev or pinned.st_ino != source_stat.st_ino:
+        return "replacement"
+    try:
+        current = os.stat(
+            path.name,
+            dir_fd=board_descriptor,
+            follow_symlinks=False,
+        )
+    except FileNotFoundError:
+        return "missing"
+    if current.st_dev != pinned.st_dev or current.st_ino != pinned.st_ino:
+        return "replacement"
+    # A replacement can itself be staged repeatedly while recovery races a
+    # writer. Keep every generation bound to the original canonical pending
+    # name instead of hashing an intermediate .retire name and hiding it from
+    # the next recovery pass.
+    path_digest = hashlib.sha256(
+        (authority_name or path.name).encode("utf-8")
+    ).hexdigest()[:16]
+    source_digest = hashlib.sha256(
+        f"{pinned.st_dev:x}\0{pinned.st_ino:x}".encode()
+    ).hexdigest()[:16]
+    for sequence in range(32):
+        staging = f".retire.{path_digest}.{source_digest}.{sequence}"
+        try:
+            _rename_noreplace(
+                path.name,
+                staging,
+                source_dir=board_descriptor,
+                destination_dir=quarantine_descriptor,
+            )
+        except FileExistsError:
+            continue
+        break
+    else:
+        return "collision"
+    staged = os.stat(
+        staging,
+        dir_fd=quarantine_descriptor,
+        follow_symlinks=False,
+    )
+    if staged.st_dev == pinned.st_dev and staged.st_ino == pinned.st_ino:
+        os.unlink(staging, dir_fd=quarantine_descriptor)
+        os.fsync(quarantine_descriptor)
+        os.fsync(board_descriptor)
+        return "retired"
+    # The path changed after the pre-check but before the rename. Put the
+    # replacement back without overwriting an even newer entry.
+    try:
+        _rename_noreplace(
+            staging,
+            path.name,
+            source_dir=quarantine_descriptor,
+            destination_dir=board_descriptor,
+        )
+    except FileExistsError:
+        return "replacement-staged"
+    os.fsync(quarantine_descriptor)
+    os.fsync(board_descriptor)
+    return "replacement"
+
+def _write_json_noreplace(path: Path, value: dict[str, Any]) -> bool:
+    """Durably create one immutable JSON artifact without replacing a peer."""
+    payload = (json.dumps(value, indent=2, sort_keys=True) + "\n").encode("utf-8")
+    if len(payload) > MAX_TERMINAL_RECORD_BYTES + 4096:
+        raise ValueError("terminal evidence exceeds the bounded artifact limit")
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    os.chmod(path.parent, 0o700, follow_symlinks=False)
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
+    create_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    create_flags |= getattr(os, "O_NOFOLLOW", 0)
+    directory = os.open(path.parent, directory_flags)
+    temporary = f".{path.name}.{uuid.uuid4().hex}.tmp"
+    temporary_stat = None
+    descriptor = None
+    cleanup_descriptor = None
+    try:
+        descriptor = os.open(temporary, create_flags, 0o600, dir_fd=directory)
+        temporary_stat = os.fstat(descriptor)
+        with os.fdopen(descriptor, "wb") as stream:
+            descriptor = None
+            stream.write(payload)
+            stream.flush()
+            os.fsync(stream.fileno())
+            written = os.fstat(stream.fileno())
+            if stat.S_IMODE(written.st_mode) != 0o600 or written.st_nlink != 1:
+                raise OSError("terminal evidence is not private and singly linked")
+            cleanup_descriptor = os.dup(stream.fileno())
+        try:
+            _rename_noreplace(
+                temporary,
+                path.name,
+                source_dir=directory,
+                destination_dir=directory,
+            )
+        except FileExistsError:
+            return False
+        os.fsync(directory)
+        return True
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        if temporary_stat is not None:
+            with contextlib.suppress(OSError):
+                _retire_terminal_entry(
+                    Path(temporary),
+                    temporary_stat,
+                    board_descriptor=directory,
+                    quarantine_descriptor=directory,
+                    source_descriptor=cleanup_descriptor,
+                )
+        if cleanup_descriptor is not None:
+            os.close(cleanup_descriptor)
+        os.close(directory)
+
+def _terminal_document_digest(document: dict[str, Any]) -> str:
+    """Return the stable identity for one exact accepted result document."""
+    immutable = {
+        key: document.get(key)
+        for key in (
+            "board",
+            "task_id",
+            "expected_run_id",
+            "result",
+            "summary",
+            "metadata",
+        )
+    }
+    canonical = json.dumps(immutable, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:32]
+
+def _terminal_evidence_path(
+    identity: TerminalIdentity,
+    state: str,
+    document: dict[str, Any],
+) -> Path:
+    """Return a deterministic immutable prepared or conflict evidence path."""
+    if state not in {"prepared", "conflict"}:
+        raise ValueError(f"invalid terminal evidence state: {state}")
+    base = state_path(identity.board, identity.task_id)
+    digest = _terminal_document_digest(document)
+    return base.with_name(
+        f"{base.stem}.run-{identity.run_id}.terminal.{state}-{digest}.json"
+    )
+
+def _same_terminal_document(left: dict[str, Any], right: dict[str, Any]) -> bool:
+    """Compare the immutable result-bearing fields of terminal evidence."""
+    keys = ("board", "task_id", "expected_run_id", "result", "summary", "metadata")
+    return all(left.get(key) == right.get(key) for key in keys)
+
+def _terminal_evidence_valid(
+    document: dict[str, Any],
+    identity: TerminalIdentity,
+    state: str,
+) -> bool:
+    """Validate semantic terminal content plus its immutable evidence state."""
+    if document.get("kanban_state") != state:
+        return False
+    semantic = dict(document)
+    semantic_identity = identity
+    if state == "conflict":
+        semantic["kanban_state"] = "prepared"
+        semantic_identity = TerminalIdentity(
+            identity.board,
+            identity.task_id,
+            identity.run_id,
+            "prepared",
+        )
+    return _terminal_record_valid(semantic, semantic_identity)
+
+def _persist_prepared_evidence(
+    identity: TerminalIdentity,
+    document: dict[str, Any],
+) -> Path:
+    """Fsync an immutable accepted result before entering the DB boundary."""
+    prepared = dict(document)
+    prepared["kanban_state"] = "prepared"
+    prepared["prepared_at"] = utc_now()
+    path = _terminal_evidence_path(identity, "prepared", document)
+    if _write_json_noreplace(path, prepared):
+        return path
+    if _terminal_evidence_identity(path) != TerminalIdentity(
+        identity.board, identity.task_id, identity.run_id, "prepared"
+    ):
+        raise OSError("prepared terminal evidence identity is invalid")
+    existing = _load_small_json(path)
+    prepared_identity = TerminalIdentity(
+        identity.board,
+        identity.task_id,
+        identity.run_id,
+        "prepared",
+    )
+    if not _terminal_evidence_valid(
+        existing,
+        prepared_identity,
+        "prepared",
+    ) or not _same_terminal_document(existing, prepared):
+        raise OSError("prepared terminal evidence path contains a conflicting result")
+    return path
+
+def _persist_conflict_evidence(
+    identity: TerminalIdentity,
+    document: dict[str, Any],
+    reason: str,
+) -> Path:
+    """Preserve a losing valid result in full under deterministic retention."""
+    conflict = dict(document)
+    conflict["kanban_state"] = "conflict"
+    conflict["conflict_reason"] = reason
+    conflict["conflicted_at"] = utc_now()
+    path = _terminal_evidence_path(identity, "conflict", document)
+    if _write_json_noreplace(path, conflict):
+        return path
+    if _terminal_evidence_identity(path) != TerminalIdentity(
+        identity.board, identity.task_id, identity.run_id, "conflict"
+    ):
+        raise OSError("terminal conflict evidence identity is invalid")
+    existing = _load_small_json(path)
+    conflict_identity = TerminalIdentity(
+        identity.board,
+        identity.task_id,
+        identity.run_id,
+        "conflict",
+    )
+    if not _terminal_evidence_valid(
+        existing,
+        conflict_identity,
+        "conflict",
+    ) or not _same_terminal_document(existing, conflict):
+        raise OSError("terminal conflict evidence path contains a different result")
+    return path

services/hermes/scripts/cli_lane_execution.py

@@ -0,0 +1,462 @@
+#!/usr/bin/env python3
+"""Single-claim provider execution and durable Kanban result handoff."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import sqlite3
+import time
+from pathlib import Path
+
+import cli_lane_goal
+from cli_lane_board import _board_call, _resolve_workspace, _task_context, _task_value
+from cli_lane_config import (
+    DEFAULT_MAX_RUNTIME,
+    TerminalFinalizationPending,
+)
+from cli_lane_failover import _routed_or_blocked, capacity_failover
+from cli_lane_files import (
+    _terminal_identity,
+    _terminal_path,
+    atomic_json,
+    load_json,
+    state_path,
+)
+from cli_lane_finalization import _finalize_terminal_record, _recover_exact_run
+from cli_lane_health import record_provider_success
+from cli_lane_metrics import record_route_decision
+from cli_lane_prompt import build_prompt, git_handoff, workspace_artifacts
+from cli_lane_provider import run_provider
+from cli_lane_quota import selection_constraint
+from cli_lane_records import _persist_candidate, _write_terminal_record
+from cli_lane_recovery import _has_pending_finalization
+from cli_lane_routing import fresh_unavailable_provider, select_route
+
+
+def execute_claim(board: str, task_id: str) -> None:
+    """Execute one already-claimed task and commit its outcome to Kanban."""
+    from hermes_cli import kanban_db
+
+    run_id = None
+    with kanban_db.scoped_current_board(board):
+        conn = kanban_db.connect(board=board)
+        try:
+            task = kanban_db.get_task(conn, task_id)
+            if task is None:
+                return
+            run_id = _task_value(task, "current_run_id", None)
+            assignee = str(_task_value(task, "assignee", "cli-auto") or "cli-auto")
+            state_file = state_path(board, task_id)
+            state = load_json(state_file)
+            state.update(
+                {
+                    "board": board,
+                    "task_id": task_id,
+                    "run_id": run_id,
+                    "assignee": assignee,
+                }
+            )
+            atomic_json(state_file, state)
+            log_path = Path(kanban_db.worker_log_path(task_id, board=board))
+            workspace = _resolve_workspace(kanban_db, conn, task, board)
+            kanban_db.set_workspace_path(conn, task_id, str(workspace))
+            context = _task_context(kanban_db, conn, task_id)
+        except Exception as error:
+            failure_reason = (
+                f"Direct CLI lane preparation failed: {type(error).__name__}: {error}"
+            )
+            conn.close()
+            _board_call(
+                kanban_db,
+                board,
+                lambda fresh: kanban_db.block_task(
+                    fresh,
+                    task_id,
+                    reason=failure_reason,
+                    kind="capability",
+                    expected_run_id=run_id,
+                ),
+            )
+            return
+        finally:
+            conn.close()
+
+        def heartbeat(note: str) -> bool:
+            try:
+                return bool(
+                    _board_call(
+                        kanban_db,
+                        board,
+                        lambda fresh: kanban_db.heartbeat_worker(
+                            fresh,
+                            task_id,
+                            note=note,
+                            expected_run_id=run_id,
+                        ),
+                    )
+                )
+            except (OSError, sqlite3.Error):
+                # The claim TTL is deliberately long. Keep the expensive
+                # provider process alive during a transient volume stall and
+                # retry on the next heartbeat instead of losing its work.
+                return True
+
+        def comment(body: str) -> None:
+            with contextlib.suppress(OSError, sqlite3.Error):
+                _board_call(
+                    kanban_db,
+                    board,
+                    lambda fresh: kanban_db.add_comment(
+                        fresh,
+                        task_id,
+                        "cli-lane-runner",
+                        body,
+                    ),
+                )
+            # Route state is also written to the durable lane-state file;
+            # a later heartbeat or terminal result remains authoritative.
+
+        try:
+            previous_route = state.get("current_route")
+            constraint = selection_constraint(assignee)
+            unavailable_provider = (
+                constraint.exclude_provider if constraint.source == "health" else None
+            )
+            route = _routed_or_blocked(
+                kanban_db,
+                board,
+                task_id,
+                run_id,
+                lambda: select_route(
+                    context,
+                    assignee,
+                    exclude_provider=constraint.exclude_provider,
+                    exclude_reason=constraint.exclude_reason,
+                ),
+            )
+            if route is None:
+                return
+            for note in constraint.notes:
+                comment(note)
+            if constraint.exclude_provider:
+                comment(
+                    f"Provider routing guard ({constraint.source}) excluded "
+                    f"{constraint.exclude_provider} before automatic routing.",
+                )
+            comment(
+                f"CLI route: {route.provider}/{route.model} at {route.effort}; classifier={route.classifier}; {route.reason}",
+            )
+            resume_handoff = ""
+            if (
+                isinstance(previous_route, dict)
+                and previous_route.get("provider")
+                and previous_route.get("provider") != route.provider
+            ):
+                try:
+                    previous_output = log_path.read_text(encoding="utf-8")[-10000:]
+                except OSError:
+                    previous_output = "Previous provider log was unavailable after restart."
+                resume_handoff = git_handoff(workspace, previous_output)
+                comment(
+                    f"Restart-time provider change: {previous_route.get('provider')} -> {route.provider}; explicit workspace handoff attached.",
+                )
+            prompt = build_prompt(context, workspace, resume_handoff)
+            max_runtime = int(
+                _task_value(task, "max_runtime_seconds", 0) or DEFAULT_MAX_RUNTIME
+            )
+            goal_mode = bool(_task_value(task, "goal_mode", False))
+            goal_max_turns = max(1, int(_task_value(task, "goal_max_turns", 1) or 1))
+            goal_turn = max(1, int(state.get("goal_turn", 0) or 0) + 1)
+            deadline = time.monotonic() + max_runtime
+            while True:
+                state["goal_turn"] = goal_turn
+                atomic_json(state_file, state)
+                remaining = max(1, int(deadline - time.monotonic()))
+                result = run_provider(
+                    route,
+                    prompt,
+                    workspace,
+                    state,
+                    state_file,
+                    log_path,
+                    heartbeat,
+                    remaining,
+                )
+                candidate_file = None
+                if result.structured:
+                    candidate_file = _persist_candidate(
+                        state,
+                        state_file,
+                        dict(result.structured),
+                        route=route,
+                        returncode=result.returncode,
+                        goal_turn=goal_turn,
+                    )
+                if result.capacity_failure:
+                    boundary = capacity_failover(
+                        kanban_db,
+                        board=board,
+                        task_id=task_id,
+                        run_id=run_id,
+                        assignee=assignee,
+                        comment=comment,
+                        context=context,
+                        workspace=workspace,
+                        state=state,
+                        state_file=state_file,
+                        log_path=log_path,
+                        heartbeat=heartbeat,
+                        deadline=deadline,
+                        route=route,
+                        result=result,
+                        candidate_file=candidate_file,
+                        goal_turn=goal_turn,
+                    )
+                    route = boundary.route
+                    result = boundary.result
+                    candidate_file = boundary.candidate_file
+                    if boundary.unavailable_provider is not None:
+                        unavailable_provider = boundary.unavailable_provider
+                    if boundary.blocked:
+                        break
+                if result.returncode == 0:
+                    record_provider_success(route.provider)
+                structured = result.structured
+                if structured:
+                    structured = dict(structured)
+                    structured["artifacts"] = workspace_artifacts(
+                        workspace,
+                        structured.get("artifacts"),
+                    )
+                metadata = {
+                    "executor": "direct-cli-lane",
+                    "provider": route.provider,
+                    "model": route.model,
+                    "effort": route.effort,
+                    "classifier": route.classifier,
+                    "state_file": str(state_file),
+                    "codex_thread_id": state.get("codex_thread_id"),
+                    "claude_session_id": state.get("claude_session_id"),
+                    "goal_mode": goal_mode,
+                    "goal_turn": goal_turn,
+                }
+                if candidate_file is not None:
+                    metadata["candidate_file"] = str(candidate_file)
+                if structured:
+                    for key in (
+                        "changed_files",
+                        "tests_run",
+                        "artifacts",
+                        "findings",
+                        "blockers",
+                    ):
+                        value = structured.get(key)
+                        metadata[key] = value if isinstance(value, list) else []
+
+                completion_problem = None
+                if structured and result.returncode == 0:
+                    # Resolve the role from the card itself, before any
+                    # controller evidence is appended, and hand it to the gate
+                    # so a review is judged on its verdict rather than on prose
+                    # about the artifact it reviewed.
+                    task_role, role_source = cli_lane_goal.task_role(
+                        context,
+                        structured,
+                    )
+                    metadata["task_role"] = task_role
+                    metadata["task_role_source"] = role_source
+                    completion_problem = cli_lane_goal.unfinished_result_reason(
+                        structured,
+                        role=task_role,
+                    )
+                    if (
+                        structured.get("status") == "completed"
+                        and completion_problem is None
+                        and goal_mode
+                    ):
+                        heartbeat("local goal-completion judge active")
+                        rejection_history = state.get("goal_rejections", [])
+                        if not isinstance(rejection_history, list):
+                            rejection_history = []
+                        judge_context = context
+                        if goal_turn > 1 or rejection_history:
+                            judge_context += (
+                                f"\n\n{cli_lane_goal.CONTROLLER_EVIDENCE_HEADING}\n"
+                                f"current turn {goal_turn}/{goal_max_turns}; prior rejected "
+                                f"reports: {json.dumps(rejection_history[-5:])}."
+                            )
+                        accepted, judge_reason = cli_lane_goal.judge_goal_completion(
+                            judge_context,
+                            structured,
+                        )
+                        metadata["goal_judge_reason"] = judge_reason
+                        if not accepted:
+                            completion_problem = f"local goal judge requested continuation: {judge_reason}"
+
+                if (
+                    structured
+                    and structured.get("status") == "completed"
+                    and result.returncode == 0
+                    and completion_problem is None
+                ):
+                    terminal_file = _terminal_path(state_file, run_id, "pending")
+                    metadata["terminal_record"] = str(
+                        _terminal_path(state_file, run_id, "committed")
+                    )
+                    try:
+                        terminal_file, terminal_record = _write_terminal_record(
+                            state_file,
+                            board=board,
+                            task_id=task_id,
+                            run_id=run_id,
+                            structured=structured,
+                            summary=str(structured.get("summary") or "Completed"),
+                            metadata=metadata,
+                        )
+                        outcome = _finalize_terminal_record(
+                            kanban_db,
+                            terminal_file,
+                            terminal_record,
+                        )
+                    except Exception as error:
+                        identity = _terminal_identity(terminal_file)
+                        replayable = bool(
+                            identity is not None
+                            and _has_pending_finalization(
+                                identity.board,
+                                identity.task_id,
+                                identity.run_id,
+                            )
+                        )
+                        if not replayable:
+                            _recover_exact_run(
+                                kanban_db,
+                                identity,
+                                f"persistence raised {type(error).__name__}",
+                            )
+                        raise TerminalFinalizationPending(
+                            "accepted worker result remains outside the generic block path; "
+                            f"terminal replayable={replayable}; persistence/finalization "
+                            f"raised {type(error).__name__}"
+                        ) from error
+                    if outcome != "committed":
+                        raise TerminalFinalizationPending(
+                            "accepted worker result is durably journaled but "
+                            f"Kanban finalization is {outcome}"
+                        )
+                    record_route_decision(
+                        route.provider, route.effort, route.classifier, "completed"
+                    )
+                    break
+
+                can_continue = (
+                    goal_mode
+                    and completion_problem is not None
+                    and goal_turn < goal_max_turns
+                    and deadline - time.monotonic() > 30
+                )
+                if can_continue:
+                    rejection_history = state.get("goal_rejections", [])
+                    if not isinstance(rejection_history, list):
+                        rejection_history = []
+                    state["goal_rejections"] = [
+                        *rejection_history[-4:],
+                        completion_problem,
+                    ]
+                    goal_turn += 1
+                    comment(
+                        f"Goal completion rejected; continuing turn {goal_turn}/{goal_max_turns}: {completion_problem}",
+                    )
+                    escalation_context = (
+                        context
+                        + "\n\nThe previous worker turn missed its completion quality mark: "
+                        + completion_problem
+                        + "\nSelect a route that can finish and verify the remaining work."
+                    )
+                    excluded = unavailable_provider
+                    if excluded is None and assignee == "cli-auto":
+                        excluded = fresh_unavailable_provider()
+                    record_route_decision(
+                        route.provider, route.effort, route.classifier, "goal-continued"
+                    )
+                    next_route = _routed_or_blocked(
+                        kanban_db,
+                        board,
+                        task_id,
+                        run_id,
+                        lambda boundary=escalation_context, blocked=excluded: select_route(
+                            boundary,
+                            assignee,
+                            exclude_provider=blocked,
+                            exclude_reason="is unavailable according to fresh native health"
+                            if blocked
+                            else None,
+                        ),
+                    )
+                    if next_route is None:
+                        break
+                    comment(
+                        f"Goal route {goal_turn}/{goal_max_turns}: "
+                        f"{next_route.provider}/{next_route.model} at {next_route.effort}; "
+                        f"classifier={next_route.classifier}; {next_route.reason}",
+                    )
+                    handoff = (
+                        git_handoff(workspace, result.output)
+                        if next_route.provider != route.provider
+                        else ""
+                    )
+                    route = next_route
+                    prompt = build_prompt(
+                        context,
+                        workspace,
+                        handoff
+                        + "\nGoal-loop continuation: the previous final report was rejected because "
+                        + completion_problem
+                        + ". Reinspect live state, finish the outstanding work, and return new final evidence.",
+                    )
+                    continue
+
+                reason = completion_problem
+                if reason is None and structured:
+                    blockers = structured.get("blockers", [])
+                    reason = "; ".join(str(item) for item in blockers)
+                    reason = reason or str(structured.get("summary") or "")
+                if reason is None:
+                    reason = result.output[-4000:]
+                failure_reason = reason or (
+                    f"{route.provider} worker failed with exit {result.returncode}"
+                )
+                failure_kind = (
+                    "transient" if result.capacity_failure else "capability"
+                )
+                record_route_decision(
+                    route.provider, route.effort, route.classifier, f"blocked-{failure_kind}"
+                )
+                _board_call(
+                    kanban_db,
+                    board,
+                    lambda fresh, failure_reason=failure_reason, failure_kind=failure_kind: kanban_db.block_task(
+                        fresh,
+                        task_id,
+                        reason=failure_reason,
+                        kind=failure_kind,
+                        expected_run_id=run_id,
+                    ),
+                )
+                break
+        except TerminalFinalizationPending as error:
+            comment(str(error))
+        except Exception as error:
+            failure_reason = f"Direct CLI lane failed: {type(error).__name__}: {error}"
+            _board_call(
+                kanban_db,
+                board,
+                lambda fresh: kanban_db.block_task(
+                    fresh,
+                    task_id,
+                    reason=failure_reason,
+                    kind="capability",
+                    expected_run_id=run_id,
+                ),
+            )

services/hermes/scripts/cli_lane_failover.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""Bounded, fail-closed cross-provider failover for one capacity boundary."""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable
+
+from cli_lane_board import _board_call
+from cli_lane_config import ProcessResult, Route
+from cli_lane_health import classify_capacity_failure, record_provider_failure
+from cli_lane_metrics import (
+    record_provider_fallback,
+    record_route_decision,
+    record_router_selection_failure,
+)
+from cli_lane_prompt import build_prompt, git_handoff
+from cli_lane_provider import run_provider
+from cli_lane_records import _persist_candidate
+from cli_lane_routing import select_route
+
+
+@dataclass
+class FailoverOutcome:
+    """Where one capacity boundary left the claim's route and result."""
+
+    route: Route
+    result: ProcessResult
+    candidate_file: Path | None
+    unavailable_provider: str | None
+    blocked: bool
+
+
+def _block_transient(
+    kanban_db: Any, board: str, task_id: str, run_id: Any, reason: str
+) -> None:
+    """Park one card as retryable so it re-enters when providers recover."""
+    _board_call(
+        kanban_db,
+        board,
+        lambda fresh: kanban_db.block_task(
+            fresh,
+            task_id,
+            reason=reason,
+            kind="transient",
+            expected_run_id=run_id,
+        ),
+    )
+
+
+def _routed_or_blocked(
+    kanban_db: Any,
+    board: str,
+    task_id: str,
+    run_id: Any,
+    select: Callable[[], Route],
+) -> Route | None:
+    """Turn a Switchyard outage into a transient block, never a capability one."""
+    try:
+        return select()
+    except RuntimeError as error:
+        record_router_selection_failure()
+        _block_transient(
+            kanban_db,
+            board,
+            task_id,
+            run_id,
+            "Switchyard route selection is unavailable; the card retries "
+            f"when the router recovers: {error}",
+        )
+        return None
+
+
+def capacity_failover(
+    kanban_db: Any,
+    *,
+    board: str,
+    task_id: str,
+    run_id: Any,
+    assignee: str,
+    comment: Callable[[str], None],
+    context: str,
+    workspace: Path,
+    state: dict[str, Any],
+    state_file: Path,
+    log_path: Path,
+    heartbeat: Callable[[str], bool],
+    deadline: float,
+    route: Route,
+    result: ProcessResult,
+    candidate_file: Path | None,
+    goal_turn: int,
+) -> FailoverOutcome:
+    """Apply the capacity policy once: record, fail closed, or fail over."""
+    failure_class = classify_capacity_failure(result.output)
+    record_provider_failure(route.provider, failure_class)
+    record_route_decision(
+        route.provider, route.effort, route.classifier, "capacity-failure"
+    )
+    if assignee != "cli-auto":
+        # Fail closed: an operator pinned this provider, so a capacity or
+        # auth failure must wait for that provider, never switch away.
+        _block_transient(
+            kanban_db,
+            board,
+            task_id,
+            run_id,
+            f"Manually pinned provider {route.provider} hit a "
+            f"{failure_class} failure; the explicit {assignee} lane never "
+            "switches providers. Retry after recovery or reassign to cli-auto.",
+        )
+        return FailoverOutcome(route, result, candidate_file, None, True)
+    retry_context = (
+        context
+        + "\n\nRouting boundary: the first provider failed from capacity/authentication. "
+        + "Select the alternate hosted provider at an appropriate effort."
+    )
+    alternate = "claude" if route.provider == "codex" else "codex"
+    fallback = _routed_or_blocked(
+        kanban_db,
+        board,
+        task_id,
+        run_id,
+        lambda: select_route(retry_context, f"cli-{alternate}-{route.effort}"),
+    )
+    if fallback is None:
+        return FailoverOutcome(route, result, candidate_file, route.provider, True)
+    record_provider_fallback(route.provider, fallback.provider, failure_class)
+    comment(
+        f"Provider fallback: {route.provider} -> {fallback.provider} "
+        f"after a {failure_class} failure; Jetson reclassified the retry boundary.",
+    )
+    fallback_result = run_provider(
+        fallback,
+        build_prompt(
+            context,
+            workspace,
+            git_handoff(workspace, result.output),
+        ),
+        workspace,
+        state,
+        state_file,
+        log_path,
+        heartbeat,
+        max(1, int(deadline - time.monotonic())),
+    )
+    if fallback_result.capacity_failure:
+        # Bounded failover depth: two hosted providers, never a third
+        # metered or local path.
+        fallback_class = classify_capacity_failure(fallback_result.output)
+        record_provider_failure(fallback.provider, fallback_class)
+        record_route_decision(
+            fallback.provider, fallback.effort, fallback.classifier, "capacity-failure"
+        )
+        _block_transient(
+            kanban_db,
+            board,
+            task_id,
+            run_id,
+            "Both hosted providers failed at this boundary "
+            f"({route.provider}: {failure_class}; {fallback.provider}: "
+            f"{fallback_class}); the lane never falls to a metered or "
+            "local path. Waiting for provider recovery.",
+        )
+        return FailoverOutcome(
+            fallback, fallback_result, candidate_file, route.provider, True
+        )
+    if fallback_result.structured:
+        candidate_file = _persist_candidate(
+            state,
+            state_file,
+            dict(fallback_result.structured),
+            route=fallback,
+            returncode=fallback_result.returncode,
+            goal_turn=goal_turn,
+        )
+    return FailoverOutcome(
+        fallback, fallback_result, candidate_file, route.provider, False
+    )

services/hermes/scripts/cli_lane_files.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""Durable path and filesystem primitives for Hermes CLI lane journals."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import re
+import stat
+import uuid
+from pathlib import Path
+from typing import Any
+
+from cli_lane_config import STATE_ROOT, TerminalIdentity, canonical_run_id
+
+
+def atomic_json(path: Path, value: dict[str, Any], mode: int = 0o600) -> None:
+    """Durably replace a small state document without following temp symlinks."""
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    os.chmod(path.parent, 0o700, follow_symlinks=False)
+    temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    flags |= getattr(os, "O_NOFOLLOW", 0)
+    descriptor = None
+    try:
+        descriptor = os.open(temporary, flags, mode)
+        with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
+            descriptor = None
+            json.dump(value, stream, indent=2, sort_keys=True)
+            stream.write("\n")
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.chmod(temporary, mode, follow_symlinks=False)
+        os.replace(temporary, path)
+        _fsync_directory(path.parent)
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        with contextlib.suppress(FileNotFoundError):
+            temporary.unlink()
+
+def _fsync_directory(directory: Path) -> None:
+    """Persist directory-entry changes after an atomic rename or quarantine."""
+    flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    flags |= getattr(os, "O_NOFOLLOW", 0)
+    descriptor = os.open(directory, flags)
+    try:
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+def load_json(path: Path) -> dict[str, Any]:
+    try:
+        flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+        descriptor = os.open(path, flags)
+        with os.fdopen(descriptor, "r", encoding="utf-8") as stream:
+            value = json.load(stream)
+    except (OSError, UnicodeError, json.JSONDecodeError):
+        return {}
+    return value if isinstance(value, dict) else {}
+
+def state_path(board: str, task_id: str) -> Path:
+    safe_board = re.sub(r"[^a-zA-Z0-9_.-]+", "-", board)
+    safe_task = re.sub(r"[^a-zA-Z0-9_.-]+", "-", task_id)
+    return STATE_ROOT / safe_board / f"{safe_task}.json"
+
+def _result_path(state_file: Path, run_id: Any, sequence: int) -> Path:
+    """Return a unique provider-result path; completed turns are never reused."""
+    safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown"))
+    return state_file.with_name(
+        f"{state_file.stem}.run-{safe_run}.provider-{sequence}.result.json"
+    )
+
+def _candidate_path(state_file: Path, run_id: Any, sequence: int) -> Path:
+    """Return the durable path for one exact structured worker response."""
+    safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown"))
+    return state_file.with_name(
+        f"{state_file.stem}.run-{safe_run}.candidate-{sequence}.json"
+    )
+
+def _terminal_path(
+    state_file: Path,
+    run_id: Any,
+    state: str = "pending",
+) -> Path:
+    """Return the identity-bound replay path for an accepted terminal response."""
+    if state not in {"pending", "committed"}:
+        raise ValueError(f"invalid terminal journal state: {state}")
+    canonical = canonical_run_id(run_id)
+    if type(run_id) is not int or canonical is None:
+        raise ValueError("terminal run ID must be a positive SQLite int64")
+    return state_file.with_name(
+        f"{state_file.stem}.run-{canonical}.terminal.{state}.json"
+    )
+
+_TERMINAL_NAME = re.compile(
+    r"^(?P<task>[a-zA-Z0-9_.-]+)\.run-(?P<run>[0-9]+)\."
+    r"terminal\.(?P<state>pending|committed)\.json$"
+)
+_TERMINAL_EVIDENCE_NAME = re.compile(
+    r"^(?P<task>[a-zA-Z0-9_.-]+)\.run-(?P<run>[0-9]+)\."
+    r"terminal\.(?P<state>prepared|conflict)-(?P<digest>[a-f0-9]{32})\.json$"
+)
+_RETIRE_NAME = re.compile(
+    r"^\.retire\.(?P<path_digest>[a-f0-9]{16})\."
+    r"(?P<source_digest>[a-f0-9]{16})\.(?P<sequence>[0-9]+)$"
+)
+_LEGACY_RETIRE_NAME = re.compile(r"^\.retire\.[a-f0-9]{32}\.[0-9]+$")
+_SAFE_BOARD = re.compile(r"^[a-zA-Z0-9_.-]+$")
+
+
+def _terminal_identity(path: Path) -> TerminalIdentity | None:
+    """Parse replay authority lexically before opening a journal or board."""
+    try:
+        relative = path.relative_to(STATE_ROOT)
+    except ValueError:
+        return None
+    if len(relative.parts) != 2:
+        return None
+    board, filename = relative.parts
+    match = _TERMINAL_NAME.fullmatch(filename)
+    run_id = canonical_run_id(match.group("run")) if match else None
+    try:
+        board_stat = path.parent.stat(follow_symlinks=False)
+    except OSError:
+        return None
+    if (
+        not match
+        or run_id is None
+        or not stat.S_ISDIR(board_stat.st_mode)
+        or not _SAFE_BOARD.fullmatch(board)
+        or board in {".", ".."}
+        or match.group("task") in {".", ".."}
+    ):
+        return None
+    return TerminalIdentity(
+        board=board,
+        task_id=match.group("task"),
+        run_id=run_id,
+        state=match.group("state"),
+    )
+
+def _terminal_evidence_identity(path: Path) -> TerminalIdentity | None:
+    """Parse immutable prepared/conflict evidence authority from its path."""
+    try:
+        relative = path.relative_to(STATE_ROOT)
+    except ValueError:
+        return None
+    if len(relative.parts) != 2:
+        return None
+    board, filename = relative.parts
+    match = _TERMINAL_EVIDENCE_NAME.fullmatch(filename)
+    run_id = canonical_run_id(match.group("run")) if match else None
+    try:
+        board_stat = path.parent.stat(follow_symlinks=False)
+    except OSError:
+        return None
+    if (
+        not match
+        or run_id is None
+        or not stat.S_ISDIR(board_stat.st_mode)
+        or not _SAFE_BOARD.fullmatch(board)
+        or board in {".", ".."}
+        or match.group("task") in {".", ".."}
+    ):
+        return None
+    return TerminalIdentity(
+        board=board,
+        task_id=match.group("task"),
+        run_id=run_id,
+        state=match.group("state"),
+    )

services/hermes/scripts/cli_lane_finalization.py

@@ -0,0 +1,331 @@
+#!/usr/bin/env python3
+"""Exact-run database completion and terminal evidence publication."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Any
+
+from cli_lane_board import _board_call, _external, _record_board_access_error, _task_value
+from cli_lane_capabilities import kanban_capabilities
+from cli_lane_config import (
+    TerminalIdentity,
+    TerminalRecoverySnapshot,
+    TerminalSnapshot,
+    canonical_run_id,
+    utc_now,
+)
+from cli_lane_evidence import (
+    _persist_conflict_evidence,
+    _persist_prepared_evidence,
+    _retire_terminal_entry,
+    _same_terminal_document,
+    _write_json_noreplace,
+)
+from cli_lane_files import _terminal_identity, _terminal_path, state_path
+from cli_lane_records import (
+    _load_terminal_json,
+    _open_small_json_snapshot,
+    _open_terminal_snapshot,
+    _terminal_record_valid,
+)
+
+
+def _recover_exact_run(
+    kanban_db: Any,
+    identity: TerminalIdentity | None,
+    reason: str,
+) -> bool:
+    """Make an exact external run retryable after journal recovery fails."""
+    if identity is None or canonical_run_id(identity.run_id) is None:
+        return False
+    if not kanban_capabilities(kanban_db).exact_run_reclaim:
+        return False
+
+    def operation(conn: Any) -> bool:
+        task = kanban_db.get_task(conn, identity.task_id)
+        if task is None:
+            return False
+        if (
+            str(_task_value(task, "status", "")) != "running"
+            or _task_value(task, "current_run_id", None) != identity.run_id
+            or not _external(task)
+        ):
+            return False
+        return bool(
+            kanban_db.reclaim_task(
+                conn,
+                identity.task_id,
+                reason=f"terminal journal recovery failed ({reason}); exact run may retry",
+                expected_run_id=identity.run_id,
+            )
+        )
+
+    try:
+        recovered = bool(_board_call(kanban_db, identity.board, operation))
+    except Exception as error:
+        _record_board_access_error(identity.board, error)
+        return False
+    if recovered:
+        print(
+            f"reclaimed {identity.board}/{identity.task_id} run {identity.run_id} "
+            f"after terminal journal recovery: {reason}",
+            file=sys.stderr,
+            flush=True,
+        )
+    return recovered
+
+def _retire_snapshot(
+    path: Path,
+    snapshot: TerminalSnapshot,
+    *,
+    authority_name: str | None = None,
+) -> str:
+    """Retire only the directory entry still naming an opened snapshot inode."""
+    return _retire_terminal_entry(
+        path,
+        snapshot.file_stat,
+        board_descriptor=snapshot.directory_descriptor,
+        quarantine_descriptor=snapshot.directory_descriptor,
+        authority_name=authority_name,
+        source_descriptor=snapshot.descriptor,
+    )
+
+def _retire_snapshot_after_db(path: Path, snapshot: TerminalSnapshot) -> str:
+    """Keep a committed DB result recoverable across retirement fsync errors."""
+    try:
+        return _retire_snapshot(path, snapshot)
+    except OSError as error:
+        print(
+            f"terminal journal retirement deferred after DB commit: "
+            f"{type(error).__name__}: {error}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return "deferred"
+
+def _discard_evidence(path: Path) -> str:
+    """Identity-safely remove one immutable prepared evidence artifact."""
+    snapshot = _open_small_json_snapshot(path)
+    if snapshot is None:
+        return "missing"
+    try:
+        return _retire_snapshot(path, snapshot)
+    finally:
+        snapshot.close()
+
+def _promote_prepared_evidence(
+    identity: TerminalIdentity,
+    document: dict[str, Any],
+    prepared: Path,
+    *,
+    discard_prepared: bool = True,
+) -> tuple[Path, str]:
+    """Publish the DB-winning result without overwriting first-writer evidence."""
+    committed = _terminal_path(
+        state_path(identity.board, identity.task_id),
+        identity.run_id,
+        "committed",
+    )
+    committed_document = dict(document)
+    committed_document["kanban_state"] = "committed"
+    committed_document["committed_at"] = utc_now()
+    state = "committed"
+    if not _write_json_noreplace(committed, committed_document):
+        committed_identity = TerminalIdentity(
+            identity.board,
+            identity.task_id,
+            identity.run_id,
+            "committed",
+        )
+        existing = _load_terminal_json(committed, committed_identity)
+        if (
+            not _terminal_record_valid(existing, committed_identity)
+            or existing.get("kanban_state") != "committed"
+            or not _same_terminal_document(existing, committed_document)
+        ):
+            _persist_conflict_evidence(
+                identity,
+                document,
+                "canonical committed evidence already contains a different result",
+            )
+            state = "conflict"
+    if discard_prepared:
+        _discard_evidence(prepared)
+    return committed, state
+
+def _finalize_document_db(
+    kanban_db: Any,
+    identity: TerminalIdentity,
+    document: dict[str, Any],
+) -> str:
+    """Apply one validated exact-run result to Kanban under DB run guards."""
+    if canonical_run_id(identity.run_id) is None:
+        return "stale"
+
+    def operation(conn: Any) -> str:
+        task = kanban_db.get_task(conn, identity.task_id)
+        if task is None:
+            return "stale"
+        status = str(_task_value(task, "status", ""))
+        if status == "done":
+            return (
+                "committed"
+                if (
+                    _task_value(task, "completed_run_id", None) == identity.run_id
+                    and str(_task_value(task, "result", "") or "")
+                    == document["result"]
+                )
+                else "stale"
+            )
+        current_run_id = _task_value(task, "current_run_id", None)
+        completion_guard: dict[str, int]
+        if status == "running" and current_run_id == identity.run_id:
+            if not kanban_capabilities(kanban_db).exact_run_completion:
+                return "deferred"
+            completion_guard = {"expected_run_id": identity.run_id}
+        elif status in {"ready", "blocked", "triage"} and current_run_id is None:
+            # The journal may have survived an older post-persistence error
+            # path that ended its run. The patched DB verifies atomically that
+            # this is still the latest ended run before allowing completion.
+            if not kanban_capabilities(kanban_db).ended_run_replay:
+                return "deferred"
+            completion_guard = {"replay_ended_run_id": identity.run_id}
+        else:
+            return "stale"
+        completed = kanban_db.complete_task(
+            conn,
+            identity.task_id,
+            result=document["result"],
+            summary=document["summary"],
+            metadata=document["metadata"],
+            **completion_guard,
+        )
+        if completed:
+            return "committed"
+        # A duplicate finalizer can lose the guarded UPDATE to an identical
+        # first writer. Re-read rather than reporting a false pending state.
+        latest = kanban_db.get_task(conn, identity.task_id)
+        latest_status = (
+            str(_task_value(latest, "status", "")) if latest is not None else ""
+        )
+        if latest_status == "done":
+            return (
+                "committed"
+                if (
+                    _task_value(latest, "completed_run_id", None) == identity.run_id
+                    and str(_task_value(latest, "result", "") or "")
+                    == document["result"]
+                )
+                else "stale"
+            )
+        # A guarded completion can be retried only while this exact live run
+        # still owns the task. An ended, replaced, or nonexistent run cannot
+        # become authoritative later and must converge to conflict evidence.
+        if (
+            latest_status == "running"
+            and _task_value(latest, "current_run_id", None) == identity.run_id
+        ):
+            return "pending"
+        return "stale"
+
+    return str(_board_call(kanban_db, identity.board, operation))
+
+def _resolve_pending_after_winner(
+    path: Path,
+    identity: TerminalIdentity,
+    winning_document: dict[str, Any],
+) -> bool:
+    """Retire duplicates and preserve differing replacements as conflicts."""
+    for _attempt in range(8):
+        snapshot = _open_terminal_snapshot(path, identity)
+        if snapshot is None:
+            return False
+        document = snapshot.document
+        if not _terminal_record_valid(document, identity):
+            snapshot.close()
+            return False
+        differing = not _same_terminal_document(document, winning_document)
+        try:
+            if differing:
+                _persist_conflict_evidence(
+                    identity,
+                    document,
+                    "different valid result lost the exact-run first-writer race",
+                )
+            retirement = _retire_snapshot_after_db(path, snapshot)
+        finally:
+            snapshot.close()
+        if retirement in {"retired", "missing"}:
+            if differing:
+                print(
+                    f"preserved terminal result conflict for {identity.board}/"
+                    f"{identity.task_id} run {identity.run_id}",
+                    file=sys.stderr,
+                    flush=True,
+                )
+            return differing
+        if retirement not in {"replacement", "replacement-staged", "collision"}:
+            return differing
+    return False
+
+def _finalize_terminal_record(
+    kanban_db: Any,
+    path: Path,
+    _record: dict[str, Any] | None = None,
+    *,
+    snapshot: TerminalRecoverySnapshot | None = None,
+) -> str:
+    """Commit one inode-pinned exact-run journal, or leave it safely pending."""
+    identity = _terminal_identity(path)
+    if identity is None or identity.state != "pending":
+        if snapshot is not None:
+            snapshot.close()
+        return "invalid"
+    pinned: TerminalSnapshot | TerminalRecoverySnapshot | None = snapshot
+    if pinned is None:
+        pinned = _open_terminal_snapshot(path, identity)
+    if pinned is None or pinned.document is None:
+        if pinned is not None:
+            pinned.close()
+        return "invalid"
+    document = pinned.document
+    try:
+        if not _terminal_record_valid(document):
+            return "invalid"
+        if not _terminal_record_valid(document, identity):
+            return "foreign"
+        prepared = _persist_prepared_evidence(identity, document)
+        outcome = _finalize_document_db(kanban_db, identity, document)
+        if outcome == "committed":
+            _committed, evidence_state = _promote_prepared_evidence(
+                identity,
+                document,
+                prepared,
+            )
+            retirement = _retire_snapshot_after_db(path, pinned)
+        elif outcome == "stale":
+            _persist_conflict_evidence(
+                identity,
+                document,
+                "valid terminal result no longer matches the authoritative task run",
+            )
+            _discard_evidence(prepared)
+            retirement = _retire_snapshot_after_db(path, pinned)
+            outcome = "conflict"
+            evidence_state = "conflict"
+        else:
+            return outcome
+    finally:
+        pinned.close()
+    if outcome == "committed" and retirement not in {"retired", "missing"}:
+        _resolve_pending_after_winner(path, identity, document)
+    if evidence_state == "conflict":
+        print(
+            f"terminal first-writer conflict recorded for {identity.board}/"
+            f"{identity.task_id} run {identity.run_id}",
+            file=sys.stderr,
+            flush=True,
+        )
+    return outcome

services/hermes/scripts/cli_lane_goal.py

@@ -1,5 +1,49 @@
 #!/usr/bin/env python3
-"""Fail-closed completion checks for durable external Kanban workers."""
+"""Role-aware completion checks for durable external Kanban workers.
+
+Completion is judged against the action the card assigned, not against the
+health of whatever the card asked the worker to look at.
+
+* Implementation cards keep the fail-closed local goal judge: a report is
+  accepted only when the model agrees every explicit requirement finished.
+* Review, audit, and diagnostic cards deliver a *verdict*. A completed report
+  carrying a truthful SHIP or BLOCK verdict with evidence finalizes the card
+  even when the verdict says the reviewed artifact must not ship. Findings are
+  the deliverable, not a task blocker, so the card is never resumed with an
+  instruction to repair an implementation the reviewer may not touch.
+
+Three deliberate boundaries keep that regime honest. First, ``role`` belongs to
+the caller that holds the card: ``unfinished_result_reason`` cannot classify a
+card it was never handed, so every caller holding the objective MUST pass
+``role``; the role-blind default is the replay contract below, never a weaker
+classifier that could short-circuit the role-aware gate.
+
+Second, the verdict contract binds in single-shot mode exactly as in the goal
+loop: a review card's deliverable *is* its verdict, so the lane fails closed
+rather than record a verdictless review as done. Two single-shot outcomes change
+against main d8f2d818, deliberately - a completed review whose prose calls the
+*reviewed* artifact pending now completes instead of being discarded, and one
+carrying no usable verdict now blocks instead of completing.
+
+Third, a journalled terminal record stays valid across an upgrade. PR15 re-runs
+this gate on replay without a card, so ``role=None`` keeps the unfinished-work
+heuristic - that integrity check still works - and skips it only for a report
+that satisfies the whole review contract and changed no files. That is precisely
+the set the role-aware lane accepts as a review, so a record accepted under one
+version re-validates under every other instead of being quarantined into a
+re-dispatch of an already-accepted task.
+
+The task role comes from explicit card metadata (``Hermes-Task-Role: review``)
+whenever the card author supplies it, read from real newlines and from the
+literal ``\\n`` escapes the board stores in single-line bodies. Pre-contract
+cards fall back to a narrow inference needing a read-only scope, a requested
+SHIP/BLOCK verdict, no requested mutation deliverable, and a report that changed
+no files. A card whose deliverable is a findings list rather than a verdict
+deliberately stays on the implementation regime: the verdict *is* the review
+contract's gate, so inferring a review role for a card that never asked for one
+could only fail closed. A report that changed files never resolves to the review
+role, even when the card declares it.
+"""
 
 from __future__ import annotations
 
@@ -10,6 +54,11 @@ import urllib.error
 import urllib.request
 from typing import Any, Callable
 
+try:  # The lane runs inside the agent image, which owns the canonical redactor.
+    from agent.redact import redact_sensitive_text as _canonical_redact
+except Exception:  # pragma: no cover - only when the agent runtime is absent
+    _canonical_redact = None
+
 
 GOAL_JUDGE_URL = os.environ.get(
     "HERMES_GOAL_JUDGE_URL",
@@ -24,6 +73,46 @@ GOAL_JUDGE_TIMEOUT_SECONDS = max(
     min(180, int(os.environ.get("HERMES_GOAL_JUDGE_TIMEOUT_SECONDS", "120"))),
 )
 RESULT_STATUSES = frozenset({"completed", "blocked", "incomplete"})
+REVIEW_ROLE = "review"
+IMPLEMENTATION_ROLE = "implementation"
+TASK_ROLES = frozenset({REVIEW_ROLE, IMPLEMENTATION_ROLE})
+ROLE_ALIASES = {
+    "audit": REVIEW_ROLE,
+    "assessment": REVIEW_ROLE,
+    "diagnosis": REVIEW_ROLE,
+    "diagnostic": REVIEW_ROLE,
+    "inspection": REVIEW_ROLE,
+    "review": REVIEW_ROLE,
+    "reviewer": REVIEW_ROLE,
+    "verdict": REVIEW_ROLE,
+    "build": IMPLEMENTATION_ROLE,
+    "change": IMPLEMENTATION_ROLE,
+    "implement": IMPLEMENTATION_ROLE,
+    "implementation": IMPLEMENTATION_ROLE,
+    "repair": IMPLEMENTATION_ROLE,
+}
+JUDGE_REASON_LIMIT = 600
+MIN_REVIEW_SUMMARY_CHARS = 80
+READ_ONLY_GUARD = (
+    "the assigned action is a read-only review: report the verdict and evidence, "
+    "and do not modify the reviewed implementation"
+)
+# The runner appends its own turn/rejection evidence to the judge objective.
+# It is emitted under this heading so ``card_scope`` drops it with every other
+# non-card section instead of letting a quoted worker sentence reassign a role.
+CONTROLLER_EVIDENCE_HEADING = "## Hermes goal-controller evidence"
+# ``hermes_cli.kanban_db.build_worker_context`` (Hermes runtime 0.18.2) renders
+# the worker context as H2 sections and emits exactly CARD_SECTIONS followed by
+# HISTORY_SECTIONS; ``test_hermes_cli_review_context.py`` pins that contract
+# against the installed runtime. The cut below allow-lists the two card
+# sections, so an upstream rename fails closed - the card just ends early.
+CARD_SECTIONS = ("Body", "Attachments")
+HISTORY_SECTIONS = (
+    "Prior attempts on this task",
+    "Parent task results",
+    "Recent work by",
+    "Comment thread",
+)
 GOAL_JUDGE_SCHEMA: dict[str, Any] = {
     "type": "object",
     "additionalProperties": False,
@@ -43,6 +132,73 @@ UNFINISHED_EVIDENCE = re.compile(
     r"\b(?:tests?|checks?|build|validation|verification|commit|push)\b)",
     re.IGNORECASE | re.DOTALL,
 )
+HISTORY_SECTION = re.compile(
+    r"^##\s+(?!(?:Body|Attachments)\b)"
+    r"|^Authoritative Hermes goal-controller evidence\b",
+    re.IGNORECASE | re.MULTILINE,
+)
+# Explicit, machine-readable card metadata. This is the supported contract;
+# every pattern below it only keeps pre-contract cards working.
+ROLE_DIRECTIVE = re.compile(
+    r"^[\s>#*\-]*(?:hermes[\s_-]*)?task[\s_-]*role\s*[:=]\s*([A-Za-z_-]+)\s*$",
+    re.IGNORECASE | re.MULTILINE,
+)
+OUTPUT_DIRECTIVE = re.compile(
+    r"^[\s>#*\-]*(?:hermes[\s_-]*)?expected[\s_-]*output\s*[:=]\s*([A-Za-z_-]+)\s*$",
+    re.IGNORECASE | re.MULTILINE,
+)
+READ_ONLY_SCOPE = re.compile(
+    r"(?:\b(?:do|does|must|should|shall|may|will)\s+not\s+(?:\w+\s+){0,3}?"
+    r"(?:edit|modify|change|alter|patch|rewrite|implement|repair|author)\b"
+    r"|\bdon'?t\s+(?:\w+\s+){0,3}?(?:edit|modify|change|alter|patch)\b"
+    r"|\bwithout\s+(?:editing|modifying|changing|altering|patching)\b"
+    r"|\bno\s+(?:\w+\s+){0,2}?(?:edits?|changes?|modifications?)\b"
+    r"|\bmake\s+no\s+(?:code|file|source)?\s*changes\b"
+    r"|\bpristine\b|\bread[\s-]?only\b)",
+    re.IGNORECASE,
+)
+# A SHIP/BLOCK-shaped deliverable, not merely a mention of the word "verdict":
+# cards that audit some other component's verdict machinery must not be read as
+# owing one themselves.
+VERDICT_DELIVERABLE = re.compile(
+    r"(?:\bship\b[^.\n]{0,120}?\bblock\b|\bblock\b[^.\n]{0,120}?\bship\b"
+    r"|\bship\s*/\s*not-?ship\b|\bverdict\s+of\b"
+    r"|\b(?:return|record|report|give|provide|deliver|state|produce|emit)\b"
+    r"(?:\s+[-\w']+){0,6}?\s+verdict\b)",
+    re.IGNORECASE,
+)
+MUTATION_DELIVERABLE = re.compile(
+    r"(?:\bpushed\s+(?:sha|commit|head|branch|revision)\b"
+    r"|\bpush\s+(?:\w+\s+){0,3}?(?:branch|commit|change\w*|fix\w*)\b"
+    r"|\bcommit\s+(?:\w+\s+){0,4}?and\s+push\b|\bamend\b"
+    r"|\bopen\s+(?:a|an|one|the)\s+(?:new\s+)?(?:draft\s+)?(?:pr\b|pull\s+request)"
+    r"|\bdraft\s+pr\s+only\b)",
+    re.IGNORECASE,
+)
+VERDICT_DIRECTIVE = re.compile(
+    r"\b(?:final\s+|strict\s+|overall\s+)?(?:verdict|recommendation|decision)\s*"
+    r"(?:is\s*)?[:=—-]\s*[*_`\s]*(ship|block)\b",
+    re.IGNORECASE,
+)
+VERDICT_TOKEN = re.compile(r"(?<![A-Za-z])(SHIP|BLOCK)(?![A-Za-z])")
+CONTROL_CHARACTERS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
+# Fallback for environments without the agent runtime (repo-only checkouts).
+SECRET_LIKE = re.compile(
+    r"(?:gh[pousr]_[A-Za-z0-9]{16,}|xox[abprs]-[A-Za-z0-9-]{10,}"
+    r"|sk-[A-Za-z0-9_-]{16,}|AKIA[0-9A-Z]{16}"
+    r"|eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})"
+)
+# Two shapes the canonical redactor deliberately passes through but this lane
+# handles: ``scheme://user:secret@host`` clone URLs (upstream leaves web
+# userinfo alone so round-trip workflow URLs keep working) and an opaque
+# credential named in prose. Both are anchored on surrounding syntax so a 40-hex
+# git SHA - indistinguishable from a Gitea PAT alone - survives as evidence.
+URL_USERINFO = re.compile(r"(?<=://)([^/\s:@]{1,64}):([^/\s@]{1,256})@")
+CREDENTIAL_CONTEXT = re.compile(
+    r"((?:pat|token|secret|password|passwd|credential|api[_-]?key|authorization)"
+    r"[\"'\s:=]+(?:bearer\s+|token\s+)?)([A-Za-z0-9_.\-]{20,})",
+    re.IGNORECASE,
+)
 
 
 def _bounded(value: str, limit: int) -> str:
@@ -55,19 +211,155 @@ def _bounded(value: str, limit: int) -> str:
     return f"{value[:head]}{marker}{value[-(remaining - head):]}"
 
 
-def unfinished_result_reason(result: dict[str, Any]) -> str | None:
-    """Reject self-contradictory completion reports without model inference."""
-    status = str(result.get("status") or "")
+def redact(value: str) -> str:
+    """Redact with the runtime's canonical helper plus this lane's own gaps."""
+    text = SECRET_LIKE.sub("[redacted]", value)
+    text = URL_USERINFO.sub(r"\1:[redacted]@", text)
+    text = CREDENTIAL_CONTEXT.sub(r"\1[redacted]", text)
+    return _canonical_redact(text, force=True) if _canonical_redact else text
+
+
+def sanitize_reason(value: Any, limit: int = JUDGE_REASON_LIMIT) -> str:
+    """Return one bounded, single-line, secret-free judge reason."""
+    text = redact(str(value or ""))
+    text = " ".join(CONTROL_CHARACTERS.sub(" ", text).split())
+    if not text:
+        return "local judge supplied no reason"
+    return text if len(text) <= limit else f"{text[: limit - 3].rstrip()}..."
+
+
+def _strings(value: Any) -> list[str]:
+    """Return the non-empty string entries of a reported evidence list."""
+    if not isinstance(value, list):
+        return []
+    return [str(item).strip() for item in value if str(item).strip()]
+
+
+def card_scope(objective: str) -> str:
+    """Return the card the worker was assigned, without appended run history.
+
+    Literal ``\\n`` escapes are expanded first: the board stores most bodies on
+    a single physical line, and every pattern here is line-anchored or depends
+    on word boundaries the escape sequence would otherwise destroy. Expanding
+    before the cut is monotone - it can only move the first history heading
+    earlier - so it cannot smuggle appended history into the card. The
+    normalization is for analysis only; the judge objective is never rewritten.
+    """
+    text = str(objective or "").replace("\\r\\n", "\n").replace("\\n", "\n")
+    match = HISTORY_SECTION.search(text)
+    return text[: match.start()] if match else text
+
+
+def _declared_role(objective: str) -> str | None:
+    """Return the role a card declared through explicit worker metadata."""
+    for pattern in (ROLE_DIRECTIVE, OUTPUT_DIRECTIVE):
+        roles = {
+            ROLE_ALIASES[value]
+            for value in (
+                match.group(1).strip().lower() for match in pattern.finditer(objective)
+            )
+            if value in ROLE_ALIASES
+        }
+        if len(roles) == 1:
+            return roles.pop()
+        if roles:
+            # A card that declares two different roles has no usable contract.
+            # Hold it to the stricter implementation regime rather than letting
+            # the more permissive review path win an ambiguous declaration.
+            return IMPLEMENTATION_ROLE
+    return None
+
+
+def task_role(objective: str, result: dict[str, Any] | None = None) -> tuple[str, str]:
+    """Resolve the assigned task role and how that resolution was reached."""
+    text = card_scope(objective)
+    changed = _strings((result or {}).get("changed_files"))
+    declared = _declared_role(text)
+    if declared == REVIEW_ROLE and changed:
+        # Mutation evidence outranks a declaration: a card cannot label itself
+        # read-only and then self-certify a report that edited the tree.
+        return IMPLEMENTATION_ROLE, "conflict"
+    if declared is not None:
+        return declared, "directive"
+    if (
+        not changed
+        and READ_ONLY_SCOPE.search(text)
+        and VERDICT_DELIVERABLE.search(text)
+        and not MUTATION_DELIVERABLE.search(text)
+    ):
+        return REVIEW_ROLE, "inferred"
+    return IMPLEMENTATION_ROLE, "default"
+
+
+def review_verdict(result: dict[str, Any]) -> tuple[str | None, str | None]:
+    """Return the declared SHIP/BLOCK verdict, or why it cannot be trusted."""
+    declared: set[str] = set()
+    field = result.get("verdict")
+    if field is not None:
+        value = str(field).strip().upper()
+        if value not in {"SHIP", "BLOCK"}:
+            return None, f"review verdict field is not SHIP or BLOCK: {value[:40]!r}"
+        declared.add(value)
+    summary = str(result.get("summary") or "")
+    declared.update(match.upper() for match in VERDICT_DIRECTIVE.findall(summary))
+    if len(declared) > 1:
+        return None, "review report declares both SHIP and BLOCK verdicts"
+    if declared:
+        return declared.pop(), None
+    tokens = {match.upper() for match in VERDICT_TOKEN.findall(summary)}
+    if len(tokens) > 1:
+        return None, "review summary names both SHIP and BLOCK without a verdict line"
+    if tokens:
+        return tokens.pop(), None
+    return None, "review report declares no explicit SHIP or BLOCK verdict"
+
+
+def review_completion_problem(result: dict[str, Any]) -> str | None:
+    """Fail closed on a review report that does not carry a usable verdict."""
+    verdict, problem = review_verdict(result)
+    if problem or verdict is None:
+        return problem or "review report declares no explicit SHIP or BLOCK verdict"
     summary = str(result.get("summary") or "").strip()
-    blockers = result.get("blockers")
+    if len(summary) < MIN_REVIEW_SUMMARY_CHARS:
+        return f"{verdict} verdict carries no reviewable rationale in its summary"
+    findings = _strings(result.get("findings"))
+    if verdict == "BLOCK" and not findings:
+        return "BLOCK verdict reports no findings to justify it"
+    if not findings and not _strings(result.get("tests_run")) and not _strings(
+        result.get("artifacts")
+    ):
+        return f"{verdict} verdict reports no findings, tests, or artifacts as evidence"
+    return None
+
+
+def _completion_problem(result: dict[str, Any], role: str | None) -> str | None:
+    """Return the unguarded reason this report is not a finished deliverable."""
+    status = str(result.get("status") or "")
     if status == "incomplete":
+        summary = str(result.get("summary") or "").strip()
         return summary or "worker explicitly reported incomplete work"
     if status != "completed":
         return None
+    blockers = result.get("blockers")
     if isinstance(blockers, list) and any(str(item).strip() for item in blockers):
         return "worker reported blockers while claiming completion"
+    if role == REVIEW_ROLE:
+        # A review's prose describes the reviewed artifact, so the unfinished
+        # work heuristic below would read the artifact's state as the review's
+        # own. The verdict contract is the deterministic gate instead.
+        return review_completion_problem(result)
+    if (
+        role is None
+        and not _strings(result.get("changed_files"))
+        and review_completion_problem(result) is None
+    ):
+        # A replay caller holds no card. This is exactly the shape the
+        # role-aware lane accepts as a review, so honouring it here keeps a
+        # journalled record valid without weakening the heuristic for anything
+        # that is not already a complete review deliverable.
+        return None
     tests = result.get("tests_run")
-    evidence = [summary]
+    evidence = [str(result.get("summary") or "").strip()]
     if isinstance(tests, list):
         evidence.extend(str(item) for item in tests)
     match = UNFINISHED_EVIDENCE.search("\n".join(evidence))
@@ -76,18 +368,34 @@ def unfinished_result_reason(result: dict[str, Any]) -> str | None:
     return None
 
 
-def judge_goal_completion(
-    objective: str,
+def unfinished_result_reason(
     result: dict[str, Any],
     *,
-    url: str = GOAL_JUDGE_URL,
-    open_request: Callable[..., Any] = urllib.request.urlopen,
-) -> tuple[bool, str]:
-    """Use the local model as a fail-closed judge for explicit goal cards."""
-    deterministic_reason = unfinished_result_reason(result)
-    if deterministic_reason:
-        return False, deterministic_reason
-    payload = {
+    role: str | None = None,
+) -> str | None:
+    """Reject self-contradictory completion reports without model inference.
+
+    ``role`` must be supplied by any caller holding the card; see the module
+    docstring for the role-blind replay contract a caller without one gets.
+    """
+    problem = _completion_problem(result, role)
+    if problem is None:
+        return None
+    if role == REVIEW_ROLE:
+        # Every reason a reviewer may ever read carries the guard, so a resumed
+        # review is never told to repair the implementation it must not touch.
+        problem = f"{READ_ONLY_GUARD}; {problem}"
+    return sanitize_reason(problem)
+
+
+def _completion_claimed(result: dict[str, Any]) -> bool:
+    """Report whether the worker actually claimed a finished task."""
+    return str(result.get("status") or "") == "completed"
+
+
+def _judge_payload(objective: str, result: dict[str, Any], role: str) -> dict[str, Any]:
+    """Build the fail-closed local judge request for an implementation card."""
+    return {
         "model": GOAL_JUDGE_MODEL,
         "messages": [
             {
@@ -98,13 +406,16 @@ def judge_goal_completion(
                     "Return complete only when the report gives concrete evidence that every "
                     "explicit requirement finished. Return continue when any work, test, command, "
                     "commit, push, review, or verification is pending, in progress, omitted, or "
-                    "inconclusive. Do not trust the report status field."
+                    "inconclusive. Defects the worker reports about some other artifact are "
+                    "findings, not unfinished work on this task. Do not trust the report status "
+                    "field."
                 ),
             },
             {
                 "role": "user",
                 "content": json.dumps(
                     {
+                        "task_role": role,
                         "objective": _bounded(objective, 6000),
                         "worker_report": _bounded(
                             json.dumps(result, default=str), 4000
@@ -126,9 +437,40 @@ def judge_goal_completion(
             },
         },
     }
+
+
+def judge_goal_completion(
+    objective: str,
+    result: dict[str, Any],
+    *,
+    url: str = GOAL_JUDGE_URL,
+    open_request: Callable[..., Any] = urllib.request.urlopen,
+) -> tuple[bool, str]:
+    """Judge one worker report against the action its card actually assigned."""
+    role, source = task_role(objective, result)
+    reason = unfinished_result_reason(result, role=role)
+    if reason is None and not _completion_claimed(result):
+        reason = (
+            f"worker reported status {str(result.get('status') or 'unknown')!r} "
+            "rather than a finished task"
+        )
+        if role == REVIEW_ROLE:
+            reason = f"{READ_ONLY_GUARD}; {reason}"
+    if reason:
+        return False, sanitize_reason(reason)
+    if role == REVIEW_ROLE:
+        # The requested deliverable is present and self-consistent. A verdict
+        # that the reviewed artifact must not ship is a finished review, so the
+        # card finalizes instead of resuming a worker that may not repair it.
+        verdict, _ = review_verdict(result)
+        findings = len(_strings(result.get("findings")))
+        return True, sanitize_reason(
+            f"review deliverable complete: {verdict} verdict with {findings} finding(s); "
+            f"task role {role} resolved by {source}"
+        )
     request = urllib.request.Request(
         url,
-        data=json.dumps(payload).encode("utf-8"),
+        data=json.dumps(_judge_payload(objective, result, role)).encode("utf-8"),
         headers={"Content-Type": "application/json"},
         method="POST",
     )
@@ -139,7 +481,7 @@ def judge_goal_completion(
         verdict = json.loads(content)
         if verdict.get("verdict") not in {"complete", "continue"}:
             raise ValueError("local judge returned an invalid verdict")
-        reason = str(verdict.get("reason") or "local judge supplied no reason")
+        reason = sanitize_reason(verdict.get("reason"))
         return verdict.get("verdict") == "complete", reason
     except (
         AttributeError,
@@ -151,4 +493,6 @@ def judge_goal_completion(
         json.JSONDecodeError,
         urllib.error.URLError,
     ) as error:
-        return False, f"local completion judge unavailable: {type(error).__name__}: {error}"
+        return False, sanitize_reason(
+            f"local completion judge unavailable: {type(error).__name__}: {error}"
+        )

services/hermes/scripts/cli_lane_health.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""Lane-observed provider health with failure classification and cooldowns."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import time
+from typing import Any
+
+from cli_lane_config import (
+    AUTH_COOLDOWN_SECONDS_DEFAULT,
+    CAPACITY_COOLDOWN_SECONDS_DEFAULT,
+    PROVIDER_HEALTH_PATHS,
+    kanban_setting,
+    utc_now,
+)
+from cli_lane_files import load_json
+
+
+# Ordered from most to least specific so one output maps to one actionable
+# reason instead of the single conflated capacity signal.
+FAILURE_CLASSIFICATIONS = (
+    ("auth", re.compile(r"authentication|unauthorized|forbidden|oauth|token.*expired|401|403", re.I)),
+    ("rate-limit", re.compile(r"rate.?limit|429|529|overload", re.I)),
+    ("quota", re.compile(r"usage.?limit|quota|credit|exhaust|capacity", re.I)),
+)
+
+
+def classify_capacity_failure(output: str) -> str:
+    """Split the broad capacity regex into quota/auth/rate-limit/transport."""
+    for reason, pattern in FAILURE_CLASSIFICATIONS:
+        if pattern.search(output):
+            return reason
+    return "transport"
+
+
+def _write_provider_health(
+    provider: str, updates: dict[str, Any], clear: tuple[str, ...] = ()
+) -> None:
+    """Merge one lane observation into the shared provider health snapshot."""
+    path = PROVIDER_HEALTH_PATHS[provider]
+    value = load_json(path)
+    for key in clear:
+        value.pop(key, None)
+    value.update(updates)
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
+        temporary.write_text(
+            json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+        )
+        os.replace(temporary, path)
+    except OSError:
+        # Health snapshots are advisory; never fail the lane over one write.
+        pass
+
+
+def record_provider_success(provider: str) -> None:
+    """Publish fresh evidence of success so the provider re-enters routing."""
+    _write_provider_health(
+        provider,
+        {
+            "source": "cli-lane-runner",
+            "state": "available",
+            "authenticated": True,
+            "checked_at": utc_now(),
+        },
+        clear=("failure_reason", "failed_at", "cooldown_until"),
+    )
+
+
+def record_provider_failure(
+    provider: str, reason: str, now: float | None = None
+) -> float:
+    """Persist one lane-observed failure with its re-admission cooldown."""
+    current = time.time() if now is None else now
+    if reason == "auth":
+        cooldown = kanban_setting(
+            "provider_auth_cooldown_seconds", AUTH_COOLDOWN_SECONDS_DEFAULT
+        )
+        state, authenticated = "unavailable", False
+    else:
+        cooldown = kanban_setting(
+            "provider_capacity_cooldown_seconds", CAPACITY_COOLDOWN_SECONDS_DEFAULT
+        )
+        state, authenticated = "capacity-limited", True
+    cooldown_until = current + max(0.0, cooldown)
+    _write_provider_health(
+        provider,
+        {
+            "source": "cli-lane-runner",
+            "state": state,
+            "authenticated": authenticated,
+            "checked_at": utc_now(),
+            "failure_reason": reason,
+            "failed_at": current,
+            "cooldown_until": cooldown_until,
+        },
+    )
+    return cooldown_until

services/hermes/scripts/cli_lane_metrics.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+"""In-process Prometheus metrics for the direct CLI lane's routing decisions."""
+
+from __future__ import annotations
+
+import os
+import sys
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+
+METRIC_HELP = {
+    "hermes_cli_route_decisions_total": (
+        "Terminal outcome of each executed CLI lane route decision."
+    ),
+    "hermes_cli_provider_fallbacks_total": (
+        "Cross-provider failovers taken by the CLI lane, by failure reason."
+    ),
+    "hermes_cli_router_selection_failures_total": (
+        "Switchyard route selections that failed and blocked a card as transient."
+    ),
+    "hermes_cli_quota_fetch_failures_total": (
+        "Quota snapshot reads that failed; routing then fails open."
+    ),
+    "hermes_cli_quota_remaining_percent": (
+        "Binding remaining-quota percent per provider as seen by the CLI lane."
+    ),
+    "hermes_cli_quota_reset_timestamp_seconds": (
+        "Unix reset time of the binding quota window as seen by the CLI lane."
+    ),
+    "hermes_cli_provider_soft_excluded": (
+        "Whether new automatic work is currently routed away from a provider."
+    ),
+}
+COUNTER_NAMES = frozenset(name for name in METRIC_HELP if name.endswith("_total"))
+
+
+def _escape_label(value: str) -> str:
+    """Escape one Prometheus label value."""
+    return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
+
+
+class LaneMetricsRegistry:
+    """Thread-safe counter and gauge store rendered in Prometheus text format."""
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._values: dict[tuple[str, tuple[tuple[str, str], ...]], float] = {}
+
+    def _key(
+        self, name: str, labels: dict[str, str] | None
+    ) -> tuple[str, tuple[tuple[str, str], ...]]:
+        return name, tuple(sorted((labels or {}).items()))
+
+    def increment(
+        self, name: str, labels: dict[str, str] | None = None, amount: float = 1.0
+    ) -> None:
+        key = self._key(name, labels)
+        with self._lock:
+            self._values[key] = self._values.get(key, 0.0) + amount
+
+    def set_value(
+        self, name: str, labels: dict[str, str] | None, value: float
+    ) -> None:
+        with self._lock:
+            self._values[self._key(name, labels)] = value
+
+    def render(self) -> bytes:
+        """Render every recorded sample with stable ordering."""
+        with self._lock:
+            values = dict(self._values)
+        lines: list[str] = []
+        for name in sorted({name for name, _ in values}):
+            kind = "counter" if name in COUNTER_NAMES else "gauge"
+            lines.extend((f"# HELP {name} {METRIC_HELP[name]}", f"# TYPE {name} {kind}"))
+            for (sample_name, labels), value in sorted(values.items()):
+                if sample_name != name:
+                    continue
+                rendered = ",".join(
+                    f'{key}="{_escape_label(text)}"' for key, text in labels
+                )
+                body = f"{{{rendered}}}" if rendered else ""
+                lines.append(f"{name}{body} {value:.12g}")
+        return ("\n".join(lines) + "\n").encode("utf-8")
+
+
+METRICS = LaneMetricsRegistry()
+
+
+def record_route_decision(
+    provider: str, effort: str, classifier: str, outcome: str
+) -> None:
+    """Count one executed route with its terminal outcome."""
+    METRICS.increment(
+        "hermes_cli_route_decisions_total",
+        {
+            "provider": provider,
+            "effort": effort,
+            "classifier": classifier,
+            "outcome": outcome,
+        },
+    )
+
+
+def record_provider_fallback(
+    from_provider: str, to_provider: str, reason: str
+) -> None:
+    """Count one cross-provider failover with its classified reason."""
+    METRICS.increment(
+        "hermes_cli_provider_fallbacks_total",
+        {
+            "from_provider": from_provider,
+            "to_provider": to_provider,
+            "reason": reason,
+        },
+    )
+
+
+def record_router_selection_failure() -> None:
+    """Count one Switchyard outage observed at a selection boundary."""
+    METRICS.increment("hermes_cli_router_selection_failures_total")
+
+
+def record_quota_fetch_failure() -> None:
+    """Count one unavailable quota snapshot; routing continues fail-open."""
+    METRICS.increment("hermes_cli_quota_fetch_failures_total")
+
+
+def record_provider_quota(
+    provider: str, remaining_percent: float, reset_timestamp: float | None
+) -> None:
+    """Publish the binding quota window the lane based its routing on."""
+    labels = {"provider": provider}
+    METRICS.set_value(
+        "hermes_cli_quota_remaining_percent", labels, remaining_percent
+    )
+    if reset_timestamp is not None:
+        METRICS.set_value(
+            "hermes_cli_quota_reset_timestamp_seconds", labels, reset_timestamp
+        )
+
+
+def record_soft_exclusion(provider: str, excluded: bool) -> None:
+    """Publish whether new automatic work avoids this provider right now."""
+    METRICS.set_value(
+        "hermes_cli_provider_soft_excluded",
+        {"provider": provider},
+        1.0 if excluded else 0.0,
+    )
+
+
+def start_metrics_server(port: int | None = None) -> ThreadingHTTPServer | None:
+    """Serve lane metrics on the pod-local scrape port without blocking dispatch."""
+    selected_port = (
+        int(os.environ.get("HERMES_CLI_LANE_METRICS_PORT", "9011"))
+        if port is None
+        else port
+    )
+
+    class Handler(BaseHTTPRequestHandler):
+        def do_GET(self) -> None:  # noqa: N802
+            if self.path == "/metrics":
+                payload = METRICS.render()
+                self.send_response(200)
+                self.send_header("Content-Type", "text/plain; version=0.0.4")
+                self.send_header("Content-Length", str(len(payload)))
+                self.end_headers()
+                self.wfile.write(payload)
+            elif self.path == "/healthz":
+                self.send_response(200)
+                self.end_headers()
+            else:
+                self.send_error(404)
+
+        def log_message(self, _format: str, *_args: object) -> None:
+            return
+
+    try:
+        server = ThreadingHTTPServer(("0.0.0.0", selected_port), Handler)
+    except OSError as error:
+        print(
+            f"cli lane metrics server unavailable: {error}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return None
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    return server

services/hermes/scripts/cli_lane_prompt.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""Prompt, artifact, and structured-event helpers for CLI workers."""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+from pathlib import Path
+from typing import Any
+
+import cli_lane_goal
+from cli_lane_files import atomic_json
+
+
+def build_prompt(context: str, workspace: Path, handoff: str = "") -> str:
+    """Create a bounded worker contract with an explicit machine-readable result."""
+    return f"""You are a durable coding worker managed by Hermes Kanban.
+
+Work only on this objective and its acceptance criteria:
+
+{context}
+
+Workspace: {workspace}
+{handoff}
+
+Operate autonomously inside the workspace. Inspect before editing, preserve unrelated user changes, run proportionate tests, and do not claim completion without evidence. You have owner-level Kubernetes access in every namespace. Prefer Flux-tracked manifests for durable changes, but use kubectl, Flux, exec, port-forwarding, rollout operations, and existing Vault workflows when the objective or incident requires them. Persist any desired-state mutation back to Git. Do not force-push, hard-reset, clean untracked files, or expose credentials.
+
+Return a final JSON object matching the supplied schema. Use status=incomplete when required work, tests, commands, commits, pushes, or verification are still running or remain to be done. Use status=blocked only when an obstacle prevents completion of the assigned task itself. Never use status=completed for a progress report. For review or diagnostic tasks, put defects and risks in findings; those findings can make the reviewed change unfit to ship without blocking completion of the review. The blockers array must be empty whenever status is completed. List changed files, tests run, durable artifact paths, findings, and task blockers explicitly.
+"""
+
+def git_handoff(workspace: Path, prior_output: str) -> str:
+    """Build an explicit cross-provider handoff without transferring hidden state."""
+    def read_git(*args: str) -> str:
+        completed = subprocess.run(
+            ["git", "-C", str(workspace), *args],
+            text=True,
+            capture_output=True,
+            timeout=30,
+            check=False,
+        )
+        return (completed.stdout or completed.stderr).strip()[-12000:]
+
+    return (
+        "\nCross-provider handoff from a failed or exhausted worker:\n"
+        f"Git status:\n{read_git('status', '--short', '--branch')}\n\n"
+        f"Current diff summary:\n{read_git('diff', '--stat')}\n\n"
+        f"Prior worker tail:\n{prior_output[-10000:]}\n"
+        "Reinspect the workspace and verify all inherited claims before continuing."
+    )
+
+def workspace_artifacts(workspace: Path, values: Any) -> list[str]:
+    """Return existing regular artifacts contained by the task worktree."""
+    if not isinstance(values, list):
+        return []
+    root = workspace.resolve()
+    artifacts: list[str] = []
+    for value in values:
+        if not isinstance(value, str) or not value.strip():
+            continue
+        candidate = Path(value.strip()).expanduser()
+        if not candidate.is_absolute():
+            candidate = root / candidate
+        try:
+            resolved = candidate.resolve(strict=True)
+            resolved.relative_to(root)
+        except (OSError, RuntimeError, ValueError):
+            continue
+        if resolved.is_file():
+            artifacts.append(str(resolved))
+    return list(dict.fromkeys(artifacts))
+
+def _extract_json(value: Any) -> dict[str, Any] | None:
+    if isinstance(value, dict) and value.get("status") in cli_lane_goal.RESULT_STATUSES:
+        return value
+    if not isinstance(value, str):
+        return None
+    candidates = [value]
+    match = re.search(r"\{.*\}", value, re.DOTALL)
+    if match:
+        candidates.append(match.group(0))
+    for candidate in candidates:
+        try:
+            parsed = json.loads(candidate)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(parsed, dict) and parsed.get("status") in cli_lane_goal.RESULT_STATUSES:
+            return parsed
+    return None
+
+def _event_payload(provider: str, line: str, state: dict[str, Any], state_file: Path) -> dict[str, Any] | None:
+    """Persist provider session identifiers before interpreting final output."""
+    try:
+        event = json.loads(line)
+    except json.JSONDecodeError:
+        return None
+    if not isinstance(event, dict):
+        return None
+    if provider == "codex" and event.get("type") == "thread.started":
+        thread_id = event.get("thread_id") or event.get("thread", {}).get("id")
+        if thread_id:
+            state["codex_thread_id"] = str(thread_id)
+            atomic_json(state_file, state)
+    if provider == "claude" and event.get("session_id"):
+        state["claude_session_id"] = str(event["session_id"])
+        atomic_json(state_file, state)
+    for key in ("structured_output", "result", "output", "text"):
+        parsed = _extract_json(event.get(key))
+        if parsed:
+            return parsed
+    item = event.get("item")
+    if isinstance(item, dict):
+        for key in ("text", "content"):
+            parsed = _extract_json(item.get(key))
+            if parsed:
+                return parsed
+    return None

services/hermes/scripts/cli_lane_provider.py

@@ -0,0 +1,403 @@
+#!/usr/bin/env python3
+"""Native Codex and Claude process lifecycle for Hermes CLI workers."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import selectors
+import signal
+import subprocess
+import time
+import uuid
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any, Callable
+
+import cli_lane_goal
+from cli_lane_config import (
+    CAPACITY_PATTERN,
+    CLAUDE_BIN,
+    CLAUDE_SESSION_COLLISION,
+    CLAUDE_SETTINGS,
+    CODEX_BIN,
+    DATA_ROOT,
+    HEARTBEAT_SECONDS,
+    NO_CLAUDE_SESSION,
+    NO_CODEX_THREAD,
+    ProcessResult,
+    RESULT_SCHEMA,
+    RESULT_SCHEMA_PATH,
+    Route,
+    utc_now,
+)
+from cli_lane_files import _result_path, atomic_json, load_json
+from cli_lane_prompt import _event_payload
+
+
+def stream_process(
+    command: list[str],
+    *,
+    provider: str,
+    cwd: Path,
+    env: dict[str, str],
+    log_path: Path,
+    state: dict[str, Any],
+    state_file: Path,
+    heartbeat: Callable[[str], bool],
+    max_runtime: int,
+) -> ProcessResult:
+    """Stream JSONL to Kanban logs while maintaining the authoritative lease."""
+    process = subprocess.Popen(
+        command,
+        cwd=cwd,
+        env=env,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        bufsize=1,
+        start_new_session=True,
+    )
+    assert process.stdout is not None
+    selector = selectors.DefaultSelector()
+    selector.register(process.stdout, selectors.EVENT_READ)
+    started = time.monotonic()
+    last_heartbeat = 0.0
+    known_descendants: dict[int, tuple[int, int]] = {}
+    lines: list[str] = []
+    structured: dict[str, Any] | None = None
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    forced_failure = ""
+    with log_path.open("a", encoding="utf-8") as log:
+        log.write(f"\n[{utc_now()}] starting {provider} worker\n")
+        log.flush()
+        while process.poll() is None:
+            known_descendants.update(_descendant_processes(process.pid))
+            now = time.monotonic()
+            if now - started > max_runtime:
+                _terminate_worker_process(process, known_descendants)
+                forced_failure = "worker exceeded its maximum runtime"
+                lines.append(forced_failure + "\n")
+                break
+            if now - last_heartbeat >= HEARTBEAT_SECONDS:
+                if not heartbeat(f"{provider} worker active for {round(now - started)}s"):
+                    _terminate_worker_process(process, known_descendants)
+                    forced_failure = "Kanban lease was lost; provider process terminated"
+                    lines.append(forced_failure + "\n")
+                    break
+                last_heartbeat = now
+            for key, _ in selector.select(timeout=1.0):
+                line = key.fileobj.readline()
+                if not line:
+                    continue
+                lines.append(line)
+                if len(lines) > 4000:
+                    lines = lines[-4000:]
+                log.write(line)
+                log.flush()
+                parsed = _event_payload(provider, line, state, state_file)
+                structured = parsed or structured
+        _terminate_worker_process(process, known_descendants)
+        remainder = process.stdout.read()
+        if remainder:
+            lines.append(remainder)
+            log.write(remainder)
+            for line in remainder.splitlines():
+                parsed = _event_payload(provider, line, state, state_file)
+                structured = parsed or structured
+        if forced_failure:
+            log.write(forced_failure + "\n")
+        log.write(f"\n[{utc_now()}] {provider} exit={process.returncode}\n")
+    selector.close()
+    output = "".join(lines)[-100000:]
+    returncode = int(process.returncode if process.returncode is not None else 1)
+    return ProcessResult(
+        returncode=returncode,
+        output=output,
+        structured=structured,
+        capacity_failure=returncode != 0 and bool(CAPACITY_PATTERN.search(output)),
+    )
+
+def _process_record(pid: int) -> tuple[int, int, int] | None:
+    """Return one Linux process's parent, group, and start-time identity."""
+    try:
+        raw = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8")
+        fields = raw[raw.rfind(")") + 2 :].split()
+        return int(fields[1]), int(fields[2]), int(fields[19])
+    except (IndexError, OSError, ValueError):
+        return None
+
+def _descendant_processes(root_pid: int) -> dict[int, tuple[int, int]]:
+    """Snapshot descendants as pid -> (process group, start-time identity)."""
+    records: dict[int, tuple[int, int, int]] = {}
+    for entry in Path("/proc").iterdir():
+        if not entry.name.isdigit():
+            continue
+        pid = int(entry.name)
+        record = _process_record(pid)
+        if record is not None:
+            records[pid] = record
+    family = {root_pid}
+    changed = True
+    while changed:
+        changed = False
+        for pid, (parent, _, _) in records.items():
+            if pid not in family and parent in family:
+                family.add(pid)
+                changed = True
+    return {
+        pid: (records[pid][1], records[pid][2])
+        for pid in family
+        if pid != root_pid and pid in records
+    }
+
+def _process_identity_matches(pid: int, start_time: int) -> bool:
+    record = _process_record(pid)
+    return record is not None and record[2] == start_time
+
+def _signal_worker_tree(
+    root_pid: int,
+    descendants: dict[int, tuple[int, int]],
+    sig: signal.Signals,
+) -> None:
+    """Signal captured descendants across terminal-created process groups."""
+    groups = {root_pid}
+    for pid, (process_group, start_time) in descendants.items():
+        if _process_identity_matches(pid, start_time):
+            groups.add(process_group)
+    for process_group in groups:
+        with contextlib.suppress(ProcessLookupError):
+            os.killpg(process_group, sig)
+    for pid, (_, start_time) in descendants.items():
+        if not _process_identity_matches(pid, start_time):
+            continue
+        with contextlib.suppress(ProcessLookupError):
+            os.kill(pid, sig)
+
+def _terminate_worker_process(
+    process: subprocess.Popen[str],
+    known_descendants: dict[int, tuple[int, int]] | None = None,
+) -> None:
+    """Reap a worker and terminal descendants that created their own groups."""
+    descendants = dict(known_descendants or {})
+    descendants.update(_descendant_processes(process.pid))
+    if process.poll() is None:
+        _signal_worker_tree(process.pid, descendants, signal.SIGTERM)
+        with contextlib.suppress(subprocess.TimeoutExpired):
+            process.wait(timeout=10)
+
+    # Provider terminal tools create their own sessions, so killing only the
+    # native CLI's process group leaves test/build subprocesses orphaned. The
+    # identity check prevents a reused PID from targeting an unrelated worker.
+    _signal_worker_tree(process.pid, descendants, signal.SIGKILL)
+    if process.poll() is None:
+        process.wait(timeout=10)
+
+def _base_env() -> dict[str, str]:
+    """Build a worker environment while preserving Vault-backed CLI homes."""
+    env = os.environ.copy()
+    # Subscription CLIs only: a vendor API key in the environment could
+    # silently flip a worker onto a metered path, so drop every key here.
+    for secret in ("ANTHROPIC_API_KEY", "CLAUDE_API_KEY", "OPENAI_API_KEY", "API_SERVER_KEY"):
+        env.pop(secret, None)
+    env.update(
+        {
+            "HOME": str(DATA_ROOT / "home"),
+            "CODEX_HOME": os.environ.get("CODEX_HOME", "/runtime-access/codex"),
+            "CLAUDE_CONFIG_DIR": os.environ.get(
+                "CLAUDE_CONFIG_DIR", "/runtime-access/claude"
+            ),
+            "GIT_TERMINAL_PROMPT": "0",
+        }
+    )
+    env["PATH"] = f"{DATA_ROOT / 'tools/bin'}:/opt/coordinator:" + env.get("PATH", "")
+    return env
+
+def _codex_command(route: Route, prompt: str, workspace: Path, state: dict[str, Any], result_file: Path) -> list[str]:
+    base = [
+        str(CODEX_BIN),
+        "--dangerously-bypass-approvals-and-sandbox",
+        "--dangerously-bypass-hook-trust",
+        "exec",
+    ]
+    thread_id = str(state.get("codex_thread_id") or "")
+    common = [
+        "--json",
+        "-m",
+        route.model,
+        "-c",
+        f'model_reasoning_effort="{route.effort}"',
+        "--output-schema",
+        str(RESULT_SCHEMA_PATH),
+        "-o",
+        str(result_file),
+    ]
+    if thread_id:
+        return [*base, "resume", *common, thread_id, prompt]
+    return [
+        *base,
+        *common,
+        "-C",
+        str(workspace),
+        prompt,
+    ]
+
+def _claude_command(route: Route, prompt: str, state: dict[str, Any], resume: bool) -> list[str]:
+    session_id = str(state["claude_session_id"])
+    session = ["--resume", session_id] if resume else ["--session-id", session_id]
+    denied = [
+        "Bash(git push --force *)",
+        "Bash(git reset --hard *)",
+        "Bash(git clean -f *)",
+    ]
+    return [
+        str(CLAUDE_BIN),
+        "--dangerously-skip-permissions",
+        "--autocompact",
+        "auto",
+        "--settings",
+        str(CLAUDE_SETTINGS),
+        "--disallowedTools",
+        *denied,
+        "--model",
+        route.model,
+        "--effort",
+        route.effort,
+        "--output-format",
+        "stream-json",
+        "--verbose",
+        "--json-schema",
+        json.dumps(RESULT_SCHEMA, separators=(",", ":")),
+        *session,
+        "-p",
+        prompt,
+    ]
+
+def run_provider(
+    route: Route,
+    prompt: str,
+    workspace: Path,
+    state: dict[str, Any],
+    state_file: Path,
+    log_path: Path,
+    heartbeat: Callable[[str], bool],
+    max_runtime: int,
+) -> ProcessResult:
+    """Run or resume one provider session using its pinned structured CLI."""
+    state.setdefault("attempts", []).append({"route": asdict(route), "started_at": utc_now()})
+    if route.provider == "claude" and not state.get("claude_session_id"):
+        # The reservation is durable before Claude starts, closing the crash gap.
+        state["claude_session_id"] = str(uuid.uuid4())
+    state["current_route"] = asdict(route)
+    state["updated_at"] = utc_now()
+    env = _base_env()
+    if route.provider == "codex":
+        result_sequence = max(0, int(state.get("result_sequence", 0) or 0)) + 1
+        result_file = _result_path(state_file, state.get("run_id"), result_sequence)
+        while result_file.exists():
+            result_sequence += 1
+            result_file = _result_path(
+                state_file,
+                state.get("run_id"),
+                result_sequence,
+            )
+        result_file.parent.mkdir(parents=True, exist_ok=True)
+        result_file.touch(mode=0o600, exist_ok=False)
+        state["result_sequence"] = result_sequence
+        state["current_result_file"] = str(result_file)
+        atomic_json(state_file, state)
+        command = _codex_command(route, prompt, workspace, state, result_file)
+        result = stream_process(
+            command,
+            provider="codex",
+            cwd=workspace,
+            env=env,
+            log_path=log_path,
+            state=state,
+            state_file=state_file,
+            heartbeat=heartbeat,
+            max_runtime=max_runtime,
+        )
+        if (
+            state.get("codex_thread_id")
+            and result.returncode != 0
+            and NO_CODEX_THREAD in result.output.lower()
+        ):
+            state.pop("codex_thread_id", None)
+            result_sequence += 1
+            result_file = _result_path(
+                state_file,
+                state.get("run_id"),
+                result_sequence,
+            )
+            while result_file.exists():
+                result_sequence += 1
+                result_file = _result_path(
+                    state_file,
+                    state.get("run_id"),
+                    result_sequence,
+                )
+            result_file.touch(mode=0o600, exist_ok=False)
+            state["result_sequence"] = result_sequence
+            state["current_result_file"] = str(result_file)
+            atomic_json(state_file, state)
+            result = stream_process(
+                _codex_command(route, prompt, workspace, state, result_file),
+                provider="codex",
+                cwd=workspace,
+                env=env,
+                log_path=log_path,
+                state=state,
+                state_file=state_file,
+                heartbeat=heartbeat,
+                max_runtime=max_runtime,
+            )
+        file_result = load_json(result_file)
+        if file_result.get("status") in cli_lane_goal.RESULT_STATUSES:
+            result.structured = file_result
+            with contextlib.suppress(OSError):
+                result_file.chmod(0o600)
+        return result
+
+    atomic_json(state_file, state)
+    resume = bool(state.get("claude_started"))
+    result = stream_process(
+        _claude_command(route, prompt, state, resume),
+        provider="claude",
+        cwd=workspace,
+        env=env,
+        log_path=log_path,
+        state=state,
+        state_file=state_file,
+        heartbeat=heartbeat,
+        max_runtime=max_runtime,
+    )
+    if resume and result.returncode == 1 and NO_CLAUDE_SESSION in result.output:
+        result = stream_process(
+            _claude_command(route, prompt, state, False),
+            provider="claude",
+            cwd=workspace,
+            env=env,
+            log_path=log_path,
+            state=state,
+            state_file=state_file,
+            heartbeat=heartbeat,
+            max_runtime=max_runtime,
+        )
+    elif not resume and result.returncode == 1 and CLAUDE_SESSION_COLLISION in result.output:
+        result = stream_process(
+            _claude_command(route, prompt, state, True),
+            provider="claude",
+            cwd=workspace,
+            env=env,
+            log_path=log_path,
+            state=state,
+            state_file=state_file,
+            heartbeat=heartbeat,
+            max_runtime=max_runtime,
+        )
+    if result.returncode == 0 or NO_CLAUDE_SESSION not in result.output:
+        state["claude_started"] = True
+        atomic_json(state_file, state)
+    return result

services/hermes/scripts/cli_lane_quarantine.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+"""Bounded diagnostic quarantine for malformed terminal journals."""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import json
+import os
+import re
+import stat
+import sys
+from pathlib import Path
+
+from cli_lane_config import QUARANTINE_HASH_BYTES, STATE_ROOT, TerminalIdentity, TerminalRecoverySnapshot
+from cli_lane_evidence import _retire_terminal_entry
+from cli_lane_records import _read_bounded
+
+
+def _quarantine_terminal(
+    path: Path,
+    identity: TerminalIdentity | None,
+    reason: str,
+    *,
+    snapshot: TerminalRecoverySnapshot | None = None,
+) -> Path:
+    """Record bounded metadata, then retire only the inspected journal inode."""
+    board_dir = path.parent
+    try:
+        board_stat = board_dir.stat(follow_symlinks=False)
+    except OSError:
+        board_stat = None
+    if board_stat is None or not stat.S_ISDIR(board_stat.st_mode):
+        print(
+            f"refused terminal journal path through unsafe board directory: {path}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return path
+    board_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    board_flags |= getattr(os, "O_NOFOLLOW", 0)
+    file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    create_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    create_flags |= getattr(os, "O_NOFOLLOW", 0)
+    board_descriptor = None
+    quarantine_descriptor = None
+    source_descriptor = None
+    source_stat = None
+    destination = path
+    retirement = "not-attempted"
+    owns_source = snapshot is None
+    try:
+        relative = path.relative_to(STATE_ROOT)
+    except ValueError:
+        relative = Path()
+    already_quarantined = (
+        len(relative.parts) == 3 and relative.parts[1] == "quarantine"
+    )
+    try:
+        if snapshot is None:
+            board_descriptor = os.open(board_dir, board_flags)
+            source_stat = os.stat(
+                path.name,
+                dir_fd=board_descriptor,
+                follow_symlinks=False,
+            )
+        else:
+            board_descriptor = snapshot.directory_descriptor
+            source_descriptor = snapshot.descriptor
+            source_stat = snapshot.file_stat
+            pinned_board = os.fstat(board_descriptor)
+            if (
+                pinned_board.st_dev != board_stat.st_dev
+                or pinned_board.st_ino != board_stat.st_ino
+            ):
+                raise OSError("terminal recovery board identity changed")
+        safe_source = stat.S_ISREG(source_stat.st_mode) and source_stat.st_nlink == 1
+        digest = None
+        hashed_bytes = 0
+        hash_complete = False
+        if safe_source and snapshot is None:
+            source_descriptor = os.open(
+                path.name,
+                file_flags,
+                dir_fd=board_descriptor,
+            )
+            opened_stat = os.fstat(source_descriptor)
+            if (
+                opened_stat.st_dev != source_stat.st_dev
+                or opened_stat.st_ino != source_stat.st_ino
+                or opened_stat.st_nlink != 1
+            ):
+                raise OSError("terminal journal identity changed while opening")
+            # Keep this descriptor open through retirement. The staging entry
+            # must match the exact fstat-verified inode, not merely a path stat
+            # captured before a concurrent replacement.
+            source_stat = opened_stat
+            prefix = _read_bounded(source_descriptor, QUARANTINE_HASH_BYTES)
+            hashed_bytes = len(prefix)
+            hash_complete = opened_stat.st_size <= QUARANTINE_HASH_BYTES
+            digest = hashlib.sha256(prefix).hexdigest()
+        elif safe_source and source_descriptor is not None:
+            prefix = snapshot.prefix
+            hashed_bytes = len(prefix)
+            hash_complete = (
+                source_stat.st_size <= QUARANTINE_HASH_BYTES
+                and hashed_bytes == source_stat.st_size
+            )
+            digest = hashlib.sha256(prefix).hexdigest()
+        source_kind = (
+            "regular"
+            if safe_source
+            else "hardlink"
+            if stat.S_ISREG(source_stat.st_mode) and source_stat.st_nlink != 1
+            else "non-regular"
+        )
+        diagnostic = {
+            "hash_complete": hash_complete,
+            "hashed_bytes": hashed_bytes,
+            "original_name": path.name,
+            "reason": reason,
+            "sha256": digest,
+            "size": source_stat.st_size,
+            "source_kind": source_kind,
+        }
+        payload = (json.dumps(diagnostic, sort_keys=True) + "\n").encode("utf-8")
+        fingerprint = (
+            (digest or hashlib.sha256(path.name.encode("utf-8")).hexdigest())[:16]
+        )
+        safe_reason = (
+            re.sub(r"[^a-zA-Z0-9_.-]+", "-", reason).strip("-") or "invalid"
+        )
+        if already_quarantined:
+            # A staged artifact already lives in the board's one bounded
+            # quarantine directory. Reuse it so adversarial replacements can
+            # never drive recursive quarantine/quarantine/... authority.
+            quarantine_descriptor = os.dup(board_descriptor)
+            quarantine_dir = board_dir
+        else:
+            with contextlib.suppress(FileExistsError):
+                os.mkdir("quarantine", 0o700, dir_fd=board_descriptor)
+            quarantine_descriptor = os.open(
+                "quarantine",
+                board_flags,
+                dir_fd=board_descriptor,
+            )
+            quarantine_dir = board_dir / "quarantine"
+        for sequence in range(32):
+            name = f"{path.name}.{safe_reason}.{fingerprint}.{sequence}.quarantine"
+            try:
+                destination_descriptor = os.open(
+                    name,
+                    create_flags,
+                    0o600,
+                    dir_fd=quarantine_descriptor,
+                )
+            except FileExistsError:
+                continue
+            try:
+                with os.fdopen(destination_descriptor, "wb") as target:
+                    target.write(payload)
+                    target.flush()
+                    os.fsync(target.fileno())
+                    target_stat = os.fstat(target.fileno())
+                    if (
+                        stat.S_IMODE(target_stat.st_mode) != 0o600
+                        or target_stat.st_nlink != 1
+                    ):
+                        raise OSError("quarantine destination is not private and singly linked")
+            except Exception:
+                os.unlink(name, dir_fd=quarantine_descriptor)
+                raise
+            destination = quarantine_dir / name
+            break
+        else:
+            raise OSError("could not reserve a unique quarantine destination")
+        os.fsync(quarantine_descriptor)
+        retirement = _retire_terminal_entry(
+            path,
+            source_stat,
+            board_descriptor=board_descriptor,
+            quarantine_descriptor=quarantine_descriptor,
+            source_descriptor=source_descriptor,
+        )
+    except OSError as error:
+        # A bad quarantine target must not make an attacker-controlled pending
+        # path eligible for replay forever. The same identity-safe staging
+        # protocol works in the board directory when quarantine is unusable.
+        if board_descriptor is not None and source_stat is not None:
+            try:
+                retirement = _retire_terminal_entry(
+                    path,
+                    source_stat,
+                    board_descriptor=board_descriptor,
+                    quarantine_descriptor=(
+                        quarantine_descriptor
+                        if quarantine_descriptor is not None
+                        else board_descriptor
+                    ),
+                    source_descriptor=source_descriptor,
+                )
+            except OSError:
+                retirement = "deferred"
+        print(
+            f"terminal journal quarantine degraded safely: {type(error).__name__}: {error}",
+            file=sys.stderr,
+            flush=True,
+        )
+    finally:
+        if owns_source and source_descriptor is not None:
+            os.close(source_descriptor)
+        if quarantine_descriptor is not None:
+            os.close(quarantine_descriptor)
+        if owns_source and board_descriptor is not None:
+            os.close(board_descriptor)
+    print(
+        "quarantined terminal journal "
+        f"{path.name}: {reason}; identity={identity or 'unparseable'}; "
+        f"retirement={retirement}",
+        file=sys.stderr,
+        flush=True,
+    )
+    return destination

services/hermes/scripts/cli_lane_quota.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+"""Quota-aware soft routing for automatic CLI lane provider selection."""
+
+from __future__ import annotations
+
+import re
+import sys
+import urllib.request
+from dataclasses import dataclass
+from typing import Callable
+
+from cli_lane_config import (
+    QUOTA_METRICS_URL,
+    QUOTA_MIN_REMAINING_PERCENT_DEFAULT,
+    kanban_setting,
+)
+from cli_lane_metrics import (
+    record_provider_quota,
+    record_quota_fetch_failure,
+    record_soft_exclusion,
+)
+from cli_lane_routing import fresh_unavailable_provider
+
+
+# Exporter accounts are labeled by vendor; the lane routes by CLI provider.
+QUOTA_PROVIDER_LABELS = {"openai": "codex", "anthropic": "claude"}
+# Gate only on account-wide windows: a model-specific Claude window (for
+# example seven_day_opus) must not veto the whole provider.
+GATED_QUOTA_WINDOWS = {
+    "codex": None,
+    "claude": ("five_hour", "seven_day"),
+}
+_SAMPLE_PATTERN = re.compile(
+    r"^atlas_ai_quota_(remaining_percent|reset_timestamp_seconds|fetch_success)"
+    r"\{([^}]*)\} (-?[0-9.eE+]+)$"
+)
+_LABEL_PATTERN = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="([^"]*)"')
+
+
+@dataclass(frozen=True)
+class ProviderQuota:
+    """One provider's binding remaining quota as seen by the lane."""
+
+    remaining_percent: float
+    reset_timestamp: float | None
+
+
+@dataclass(frozen=True)
+class SelectionConstraint:
+    """One provider exclusion decided at a routing boundary, with evidence."""
+
+    exclude_provider: str | None
+    exclude_reason: str | None
+    source: str | None
+    notes: tuple[str, ...]
+
+
+def _gated_window(provider: str, labels: dict[str, str]) -> str | None:
+    """Return the window key when a sample belongs to a gated account limit."""
+    if labels.get("limit") != provider:
+        return None
+    window = labels.get("window", "")
+    allowed = GATED_QUOTA_WINDOWS[provider]
+    if allowed is not None and window not in allowed:
+        return None
+    return window
+
+
+def parse_quota_metrics(text: str) -> dict[str, ProviderQuota]:
+    """Reduce exporter samples to each provider's tightest account window."""
+    remaining: dict[tuple[str, str], float] = {}
+    resets: dict[tuple[str, str], float] = {}
+    fetch_success: dict[str, bool] = {}
+    for line in text.splitlines():
+        match = _SAMPLE_PATTERN.match(line.strip())
+        if not match:
+            continue
+        metric, raw_labels, raw_value = match.groups()
+        labels = dict(_LABEL_PATTERN.findall(raw_labels))
+        provider = QUOTA_PROVIDER_LABELS.get(labels.get("provider", ""))
+        if provider is None:
+            continue
+        value = float(raw_value)
+        if metric == "fetch_success":
+            fetch_success[provider] = value >= 1.0
+            continue
+        window = _gated_window(provider, labels)
+        if window is None:
+            continue
+        if metric == "remaining_percent":
+            remaining[(provider, window)] = value
+        else:
+            resets[(provider, window)] = value
+    snapshot: dict[str, ProviderQuota] = {}
+    for provider in QUOTA_PROVIDER_LABELS.values():
+        windows = {
+            window: value
+            for (candidate, window), value in remaining.items()
+            if candidate == provider
+        }
+        if not windows or not fetch_success.get(provider, True):
+            # Failed or stale exporter fetches leave this provider ungated.
+            continue
+        binding = min(windows, key=lambda window: (windows[window], window))
+        snapshot[provider] = ProviderQuota(
+            remaining_percent=windows[binding],
+            reset_timestamp=resets.get((provider, binding)),
+        )
+    return snapshot
+
+
+def fetch_quota_snapshot(
+    url: str = QUOTA_METRICS_URL,
+    open_url: Callable = urllib.request.urlopen,
+) -> dict[str, ProviderQuota]:
+    """Read the in-pod usage exporter; empty means the signal is unavailable."""
+    try:
+        with open_url(url, timeout=10) as response:
+            text = response.read(1 << 20).decode("utf-8", errors="replace")
+        snapshot = parse_quota_metrics(text)
+    except (OSError, ValueError) as error:
+        record_quota_fetch_failure()
+        print(
+            f"quota snapshot unavailable; routing fails open: {type(error).__name__}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return {}
+    if not snapshot:
+        record_quota_fetch_failure()
+        print(
+            "quota snapshot carried no gated windows; routing fails open",
+            file=sys.stderr,
+            flush=True,
+        )
+    return snapshot
+
+
+def quota_soft_exclusion(
+    snapshot: dict[str, ProviderQuota], threshold: float
+) -> tuple[str | None, str | None]:
+    """Choose at most one provider to soft-exclude from new automatic work."""
+    below = {
+        provider: quota.remaining_percent
+        for provider, quota in snapshot.items()
+        if quota.remaining_percent < threshold
+    }
+    if not below:
+        return None, None
+    excluded = min(below, key=lambda provider: (below[provider], provider))
+    if len(below) == len(snapshot) and len(snapshot) > 1:
+        preferred = next(name for name in snapshot if name != excluded)
+        note = (
+            "Quota degraded: every provider is below the "
+            f"{threshold:.6g}% remaining threshold; preferring {preferred} "
+            f"({snapshot[preferred].remaining_percent:.6g}% left) over "
+            f"{excluded} ({below[excluded]:.6g}% left) for new auto work."
+        )
+        return excluded, note
+    note = (
+        f"Quota guard: {excluded} has {below[excluded]:.6g}% remaining "
+        f"(threshold {threshold:.6g}%); routing new auto work to the other "
+        "provider while active work finishes."
+    )
+    return excluded, note
+
+
+def selection_constraint(
+    assignee: str,
+    *,
+    snapshot_from: Callable[[], dict[str, ProviderQuota]] | None = None,
+    now: float | None = None,
+) -> SelectionConstraint:
+    """Combine provider health and quota gating for one selection boundary."""
+    if assignee != "cli-auto":
+        # Explicit lanes are operator decisions; they are never re-routed.
+        return SelectionConstraint(None, None, None, ())
+    snapshot = (snapshot_from or fetch_quota_snapshot)()
+    resets: dict[str, float] = {}
+    for provider, quota in snapshot.items():
+        record_provider_quota(provider, quota.remaining_percent, quota.reset_timestamp)
+        if quota.reset_timestamp is not None:
+            resets[provider] = quota.reset_timestamp
+    health_excluded = fresh_unavailable_provider(now=now, quota_resets=resets)
+    if health_excluded is not None:
+        return SelectionConstraint(
+            health_excluded,
+            "is unavailable according to fresh native health",
+            "health",
+            (),
+        )
+    threshold = kanban_setting(
+        "provider_quota_min_remaining_percent", QUOTA_MIN_REMAINING_PERCENT_DEFAULT
+    )
+    excluded, note = quota_soft_exclusion(snapshot, threshold)
+    for provider in snapshot:
+        record_soft_exclusion(provider, provider == excluded)
+    if excluded is None or note is None:
+        return SelectionConstraint(None, None, None, ())
+    return SelectionConstraint(
+        excluded,
+        "is below its remaining-quota routing threshold",
+        "quota",
+        (note,),
+    )

services/hermes/scripts/cli_lane_records.py

@@ -0,0 +1,356 @@
+#!/usr/bin/env python3
+"""Bounded inode-pinned reads and validation for Hermes terminal records."""
+
+from __future__ import annotations
+
+import json
+import os
+import stat
+from pathlib import Path
+from typing import Any
+
+import cli_lane_goal
+from cli_lane_config import (
+    MAX_TERMINAL_RECORD_BYTES,
+    QUARANTINE_HASH_BYTES,
+    RESULT_SCHEMA,
+    Route,
+    TerminalIdentity,
+    TerminalRecoverySnapshot,
+    TerminalSnapshot,
+    canonical_run_id,
+    utc_now,
+)
+from cli_lane_files import (
+    _candidate_path,
+    _terminal_identity,
+    _terminal_path,
+    atomic_json,
+)
+
+
+def _read_bounded(descriptor: int, limit: int) -> bytes:
+    """Read at most ``limit`` bytes from a regular file descriptor."""
+    chunks: list[bytes] = []
+    remaining = max(0, limit)
+    while remaining:
+        chunk = os.read(descriptor, min(64 * 1024, remaining))
+        if not chunk:
+            break
+        chunks.append(chunk)
+        remaining -= len(chunk)
+    return b"".join(chunks)
+
+def _open_small_json_snapshot(path: Path) -> TerminalSnapshot | None:
+    """Open and pin one bounded, singly-linked JSON document."""
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
+    file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    directory = None
+    descriptor = None
+    try:
+        directory = os.open(path.parent, directory_flags)
+        descriptor = os.open(path.name, file_flags, dir_fd=directory)
+        opened = os.fstat(descriptor)
+        if (
+            not stat.S_ISREG(opened.st_mode)
+            or opened.st_nlink != 1
+            or opened.st_size > MAX_TERMINAL_RECORD_BYTES
+        ):
+            return None
+        payload = _read_bounded(descriptor, MAX_TERMINAL_RECORD_BYTES + 1)
+        if len(payload) > MAX_TERMINAL_RECORD_BYTES:
+            return None
+        current = os.stat(
+            path.name,
+            dir_fd=directory,
+            follow_symlinks=False,
+        )
+        if (
+            current.st_dev != opened.st_dev
+            or current.st_ino != opened.st_ino
+            or current.st_size != opened.st_size
+            or len(payload) != opened.st_size
+        ):
+            return None
+        value = json.loads(payload.decode("utf-8"))
+        if not isinstance(value, dict):
+            return None
+        snapshot = TerminalSnapshot(value, opened, descriptor, directory)
+        descriptor = None
+        directory = None
+        return snapshot
+    except (OSError, UnicodeError, json.JSONDecodeError):
+        return None
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        if directory is not None:
+            os.close(directory)
+
+def _open_terminal_recovery_snapshot(path: Path) -> TerminalRecoverySnapshot | None:
+    """Pin the entry recovery inspected, even when its payload is malformed."""
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
+    file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    directory = None
+    descriptor = None
+    try:
+        directory = os.open(path.parent, directory_flags)
+        observed = os.stat(
+            path.name,
+            dir_fd=directory,
+            follow_symlinks=False,
+        )
+        if not stat.S_ISREG(observed.st_mode):
+            snapshot = TerminalRecoverySnapshot(
+                None,
+                observed,
+                None,
+                directory,
+                b"",
+                "non-regular",
+            )
+            directory = None
+            return snapshot
+        try:
+            descriptor = os.open(path.name, file_flags, dir_fd=directory)
+        except OSError:
+            snapshot = TerminalRecoverySnapshot(
+                None,
+                observed,
+                None,
+                directory,
+                b"",
+                "open-failed",
+            )
+            directory = None
+            return snapshot
+        opened = os.fstat(descriptor)
+        if opened.st_dev != observed.st_dev or opened.st_ino != observed.st_ino:
+            os.close(descriptor)
+            descriptor = None
+            snapshot = TerminalRecoverySnapshot(
+                None,
+                observed,
+                None,
+                directory,
+                b"",
+                "identity-changed-during-open",
+            )
+            directory = None
+            return snapshot
+        observed = opened
+        if observed.st_nlink != 1:
+            snapshot = TerminalRecoverySnapshot(
+                None,
+                observed,
+                descriptor,
+                directory,
+                b"",
+                "hardlinked",
+            )
+            descriptor = None
+            directory = None
+            return snapshot
+        if observed.st_size > MAX_TERMINAL_RECORD_BYTES:
+            prefix = _read_bounded(descriptor, QUARANTINE_HASH_BYTES)
+            snapshot = TerminalRecoverySnapshot(
+                None,
+                observed,
+                descriptor,
+                directory,
+                prefix,
+                "oversized",
+            )
+            descriptor = None
+            directory = None
+            return snapshot
+        payload = _read_bounded(descriptor, MAX_TERMINAL_RECORD_BYTES + 1)
+        after_read = os.fstat(descriptor)
+        unchanged = (
+            after_read.st_dev == observed.st_dev
+            and after_read.st_ino == observed.st_ino
+            and after_read.st_size == observed.st_size
+            and after_read.st_mtime_ns == observed.st_mtime_ns
+            and after_read.st_ctime_ns == observed.st_ctime_ns
+            and len(payload) == observed.st_size
+        )
+        document = None
+        invalid_reason = "unstable-payload"
+        if unchanged:
+            try:
+                parsed = json.loads(payload.decode("utf-8"))
+                if isinstance(parsed, dict):
+                    document = parsed
+                    invalid_reason = None
+                else:
+                    invalid_reason = "non-object-payload"
+            except (UnicodeError, json.JSONDecodeError):
+                invalid_reason = "malformed-payload"
+        snapshot = TerminalRecoverySnapshot(
+            document,
+            after_read,
+            descriptor,
+            directory,
+            payload[:QUARANTINE_HASH_BYTES],
+            invalid_reason,
+        )
+        descriptor = None
+        directory = None
+        return snapshot
+    except OSError:
+        return None
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        if directory is not None:
+            os.close(directory)
+
+def _open_terminal_snapshot(
+    path: Path,
+    identity: TerminalIdentity,
+) -> TerminalSnapshot | None:
+    """Open a terminal journal only after its lexical identity is verified."""
+    if _terminal_identity(path) != identity:
+        return None
+    return _open_small_json_snapshot(path)
+
+def _load_terminal_json(path: Path, identity: TerminalIdentity) -> dict[str, Any]:
+    """Read one small, singly-linked journal through its validated directory."""
+    snapshot = _open_terminal_snapshot(path, identity)
+    if snapshot is None:
+        return {}
+    try:
+        return snapshot.document
+    finally:
+        snapshot.close()
+
+def _load_small_json(path: Path) -> dict[str, Any]:
+    """Read one bounded immutable evidence document without following links."""
+    snapshot = _open_small_json_snapshot(path)
+    if snapshot is None:
+        return {}
+    try:
+        return snapshot.document
+    finally:
+        snapshot.close()
+
+def _persist_candidate(
+    state: dict[str, Any],
+    state_file: Path,
+    structured: dict[str, Any],
+    *,
+    route: Route,
+    returncode: int,
+    goal_turn: int,
+) -> Path:
+    """Persist every structured response before any judge can supersede it."""
+    sequence = max(0, int(state.get("candidate_sequence", 0) or 0)) + 1
+    path = _candidate_path(state_file, state.get("run_id"), sequence)
+    while path.exists():
+        sequence += 1
+        path = _candidate_path(state_file, state.get("run_id"), sequence)
+    atomic_json(
+        path,
+        {
+            "board": state.get("board"),
+            "task_id": state.get("task_id"),
+            "expected_run_id": state.get("run_id"),
+            "goal_turn": goal_turn,
+            "provider": route.provider,
+            "model": route.model,
+            "effort": route.effort,
+            "returncode": returncode,
+            "structured": structured,
+            "recorded_at": utc_now(),
+        },
+    )
+    state["candidate_sequence"] = sequence
+    state["last_candidate_file"] = str(path)
+    atomic_json(state_file, state)
+    return path
+
+def _write_terminal_record(
+    state_file: Path,
+    *,
+    board: str,
+    task_id: str,
+    run_id: Any,
+    structured: dict[str, Any],
+    summary: str,
+    metadata: dict[str, Any],
+) -> tuple[Path, dict[str, Any]]:
+    """Journal an accepted result before attempting the Kanban transaction."""
+    if type(run_id) is not int or canonical_run_id(run_id) is None:
+        raise ValueError("terminal run ID must be a positive SQLite int64")
+    path = _terminal_path(state_file, run_id)
+    record = {
+        "board": board,
+        "task_id": task_id,
+        "expected_run_id": run_id,
+        "result": json.dumps(structured, sort_keys=True),
+        "summary": summary,
+        "metadata": metadata,
+        "kanban_state": "pending",
+        "recorded_at": utc_now(),
+    }
+    atomic_json(path, record)
+    return path, record
+
+def _terminal_record_valid(
+    record: dict[str, Any],
+    identity: TerminalIdentity | None = None,
+) -> bool:
+    """Reject malformed or non-terminal replay journals without side effects."""
+    if not isinstance(record, dict):
+        return False
+    required_strings = ("board", "task_id", "result", "summary")
+    if not all(
+        isinstance(record.get(key), str) and record[key]
+        for key in required_strings
+    ):
+        return False
+    expected_run_id = record.get("expected_run_id")
+    if (
+        type(expected_run_id) is not int
+        or canonical_run_id(expected_run_id) is None
+    ):
+        return False
+    if not isinstance(record.get("metadata"), dict):
+        return False
+    try:
+        structured = json.loads(record["result"])
+    except (TypeError, json.JSONDecodeError):
+        return False
+    if not isinstance(structured, dict):
+        return False
+    required = set(RESULT_SCHEMA["required"])
+    if set(structured) != required:
+        return False
+    if (
+        type(structured["status"]) is not str
+        or structured["status"] not in cli_lane_goal.RESULT_STATUSES
+        or type(structured["summary"]) is not str
+        or not structured["summary"].strip()
+    ):
+        return False
+    for key in ("changed_files", "tests_run", "artifacts", "findings", "blockers"):
+        value = structured[key]
+        if type(value) is not list or any(type(item) is not str for item in value):
+            return False
+    valid = (
+        structured["status"] == "completed"
+        and structured["blockers"] == []
+        and cli_lane_goal.unfinished_result_reason(structured) is None
+        and record["summary"] == structured["summary"]
+        and record.get("kanban_state") in {"pending", "prepared", "committed"}
+    )
+    if not valid or identity is None:
+        return valid
+    return (
+        record.get("board") == identity.board
+        and record.get("task_id") == identity.task_id
+        and record.get("expected_run_id") == identity.run_id
+        and record.get("kanban_state") == identity.state
+    )

services/hermes/scripts/cli_lane_recovery.py

@@ -0,0 +1,460 @@
+#!/usr/bin/env python3
+"""Bounded replay of pending, prepared, and staged terminal journals."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+from cli_lane_board import _record_board_access_error
+from cli_lane_config import (
+    STATE_ROOT,
+    TerminalIdentity,
+    TerminalRecoverySnapshot,
+    canonical_run_id,
+)
+from cli_lane_evidence import (
+    _persist_conflict_evidence,
+    _persist_prepared_evidence,
+    _same_terminal_document,
+    _terminal_evidence_path,
+    _terminal_evidence_valid,
+    _write_json_noreplace,
+)
+from cli_lane_files import (
+    _LEGACY_RETIRE_NAME,
+    _RETIRE_NAME,
+    _SAFE_BOARD,
+    _terminal_evidence_identity,
+    _terminal_identity,
+    _terminal_path,
+    state_path,
+)
+from cli_lane_finalization import (
+    _finalize_document_db,
+    _finalize_terminal_record,
+    _promote_prepared_evidence,
+    _recover_exact_run,
+    _resolve_pending_after_winner,
+    _retire_snapshot,
+    _retire_snapshot_after_db,
+)
+from cli_lane_quarantine import _quarantine_terminal
+from cli_lane_records import (
+    _load_small_json,
+    _load_terminal_json,
+    _open_terminal_recovery_snapshot,
+    _terminal_record_valid,
+)
+
+
+def _terminal_entry_absent(path: Path) -> bool:
+    """Confirm absence through a nofollow directory descriptor."""
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
+    directory = None
+    try:
+        directory = os.open(path.parent, directory_flags)
+        os.stat(path.name, dir_fd=directory, follow_symlinks=False)
+        return False
+    except FileNotFoundError:
+        return True
+    except OSError:
+        return False
+    finally:
+        if directory is not None:
+            os.close(directory)
+
+def _staged_terminal_authority(
+    path: Path,
+    snapshot: TerminalRecoverySnapshot,
+) -> tuple[TerminalIdentity, Path] | None:
+    """Bind a staged inode to the canonical pending name it was moved from."""
+    try:
+        relative = path.relative_to(STATE_ROOT)
+    except ValueError:
+        return None
+    if len(relative.parts) == 2:
+        board, filename = relative.parts
+    elif len(relative.parts) == 3 and relative.parts[1] == "quarantine":
+        board, _quarantine, filename = relative.parts
+    else:
+        return None
+    match = _RETIRE_NAME.fullmatch(filename)
+    legacy = _LEGACY_RETIRE_NAME.fullmatch(filename)
+    document = snapshot.document
+    if (
+        (match is None and legacy is None)
+        or document is None
+        or not _SAFE_BOARD.fullmatch(board)
+        or board in {".", ".."}
+        or document.get("board") != board
+        or not isinstance(document.get("task_id"), str)
+        or type(document.get("expected_run_id")) is not int
+        or canonical_run_id(document.get("expected_run_id")) is None
+    ):
+        return None
+    state = document.get("kanban_state")
+    if state not in {"pending", "prepared", "committed", "conflict"}:
+        return None
+    identity = TerminalIdentity(
+        board, document["task_id"], document["expected_run_id"], state
+    )
+    if state in {"pending", "committed"}:
+        canonical = _terminal_path(
+            state_path(identity.board, identity.task_id),
+            identity.run_id,
+            state,
+        )
+        valid = _terminal_record_valid(document, identity)
+    else:
+        canonical = _terminal_evidence_path(identity, state, document)
+        valid = _terminal_evidence_valid(document, identity, state)
+    board_dir = STATE_ROOT / board
+    if (
+        canonical.parent != board_dir
+        or path.parent not in {board_dir, board_dir / "quarantine"}
+        or (
+            match is not None
+            and hashlib.sha256(canonical.name.encode("utf-8")).hexdigest()[:16]
+            != match.group("path_digest")
+        )
+        or not valid
+    ):
+        return None
+    return identity, canonical
+
+def _retirement_staging_paths() -> list[Path]:
+    """List every directory where identity-safe retirement can stage an inode."""
+    paths = list(STATE_ROOT.glob("*/.retire.*"))
+    paths.extend(STATE_ROOT.glob("*/quarantine/.retire.*"))
+    return sorted(set(paths))
+
+def _restore_staged_terminal_authority(
+    identity: TerminalIdentity,
+    canonical: Path,
+    document: dict[str, Any],
+) -> None:
+    """Restore one hidden accepted artifact without executing it prematurely."""
+    if identity.state == "pending":
+        _persist_prepared_evidence(identity, document)
+        return
+    if _write_json_noreplace(canonical, document):
+        return
+    existing = _load_small_json(canonical)
+    if identity.state in {"prepared", "conflict"}:
+        valid = _terminal_evidence_valid(existing, identity, identity.state)
+    else:
+        valid = _terminal_record_valid(existing, identity)
+    if valid and _same_terminal_document(existing, document):
+        return
+    if identity.state == "committed":
+        _persist_conflict_evidence(
+            identity,
+            document,
+            "staged committed evidence conflicts with the canonical first writer",
+        )
+        return
+    raise OSError("staged terminal evidence path contains a conflicting result")
+
+def _recover_retirement_staging() -> int:
+    """Promote valid hidden journals to durable prepared evidence."""
+    recovered = 0
+    for path in _retirement_staging_paths():
+        snapshot = _open_terminal_recovery_snapshot(path)
+        if snapshot is None:
+            continue
+        authority = _staged_terminal_authority(path, snapshot)
+        if authority is None:
+            try:
+                _quarantine_terminal(
+                    path,
+                    None,
+                    "untrusted-retirement-staging",
+                    snapshot=snapshot,
+                )
+            finally:
+                snapshot.close()
+            continue
+        identity, canonical = authority
+        try:
+            _restore_staged_terminal_authority(
+                identity,
+                canonical,
+                snapshot.document or {},
+            )
+            retirement = _retire_snapshot(
+                path,
+                snapshot,
+                authority_name=canonical.name,
+            )
+        except Exception as error:
+            _record_board_access_error(identity.board, error)
+            continue
+        finally:
+            snapshot.close()
+        if retirement in {"retired", "missing"}:
+            recovered += 1
+    return recovered
+
+def _drain_retirement_staging() -> int:
+    """Converge bounded replacement churn without spinning on storage failure."""
+    recovered = 0
+    previous: tuple[tuple[str, int, int, int, int], ...] | None = None
+    for _attempt in range(16):
+        surface = []
+        for path in _retirement_staging_paths():
+            try:
+                observed = path.stat(follow_symlinks=False)
+            except OSError:
+                continue
+            surface.append(
+                (
+                    str(path),
+                    observed.st_dev,
+                    observed.st_ino,
+                    observed.st_size,
+                    observed.st_mtime_ns,
+                )
+            )
+        signature = tuple(surface)
+        if not signature or signature == previous:
+            break
+        previous = signature
+        recovered += _recover_retirement_staging()
+    return recovered
+
+def _recover_prepared_finalizations(kanban_db: Any) -> int:
+    """Recover an accepted result staged before an interrupted DB boundary."""
+    recovered = 0
+    recovered_identities: set[tuple[str, str, int]] = set()
+    prepared_paths = list(STATE_ROOT.glob("*/*.terminal.prepared-*.json"))
+
+    def durable_order(path: Path) -> tuple[int, str]:
+        try:
+            return path.stat(follow_symlinks=False).st_mtime_ns, str(path)
+        except OSError:
+            return 2**63 - 1, str(path)
+
+    # When no DB winner exists yet, the first durably prepared result owns the
+    # run. The DB transaction remains the final arbiter across runner processes.
+    for path in sorted(prepared_paths, key=durable_order):
+        for _replacement_attempt in range(16):
+            identity = _terminal_evidence_identity(path)
+            snapshot = _open_terminal_recovery_snapshot(path)
+            if snapshot is None:
+                break
+            document = snapshot.document
+            if identity is None or identity.state != "prepared" or document is None:
+                try:
+                    _quarantine_terminal(
+                        path,
+                        identity,
+                        snapshot.invalid_reason or "malformed-prepared-evidence",
+                        snapshot=snapshot,
+                    )
+                finally:
+                    snapshot.close()
+                continue
+            expected = _terminal_evidence_path(identity, "prepared", document)
+            if (
+                expected.name != path.name
+                or not _terminal_evidence_valid(document, identity, "prepared")
+            ):
+                try:
+                    _quarantine_terminal(
+                        path,
+                        identity,
+                        "foreign-prepared-evidence",
+                        snapshot=snapshot,
+                    )
+                finally:
+                    snapshot.close()
+                continue
+            try:
+                outcome = _finalize_document_db(kanban_db, identity, document)
+            except Exception as error:
+                _record_board_access_error(identity.board, error)
+                snapshot.close()
+                break
+            try:
+                if outcome == "committed":
+                    _promote_prepared_evidence(
+                        identity,
+                        document,
+                        path,
+                        discard_prepared=False,
+                    )
+                    _retire_snapshot_after_db(path, snapshot)
+                    pending = _terminal_path(
+                        state_path(identity.board, identity.task_id),
+                        identity.run_id,
+                        "pending",
+                    )
+                    pending_identity = TerminalIdentity(
+                        identity.board,
+                        identity.task_id,
+                        identity.run_id,
+                        "pending",
+                    )
+                    _resolve_pending_after_winner(
+                        pending,
+                        pending_identity,
+                        document,
+                    )
+                    recovered_identity = (
+                        identity.board,
+                        identity.task_id,
+                        identity.run_id,
+                    )
+                    if recovered_identity not in recovered_identities:
+                        recovered_identities.add(recovered_identity)
+                        recovered += 1
+                elif outcome == "stale":
+                    _persist_conflict_evidence(
+                        identity,
+                        document,
+                        "prepared result no longer matches the authoritative task run",
+                    )
+                    _retire_snapshot_after_db(path, snapshot)
+                else:
+                    snapshot.close()
+                    break
+            except Exception as error:
+                _record_board_access_error(identity.board, error)
+                snapshot.close()
+                break
+            snapshot.close()
+    return recovered
+
+def recover_pending_finalizations() -> int:
+    """Replay accepted exact-run results before scheduling more provider work."""
+    from hermes_cli import kanban_db
+
+    _drain_retirement_staging()
+    recovered = _recover_prepared_finalizations(kanban_db)
+    for path in sorted(STATE_ROOT.glob("*/*.terminal.pending.json")):
+        identity = _terminal_identity(path)
+        last_invalid_reason = "malformed-name" if identity is None else None
+        retired_invalid = False
+        for _replacement_attempt in range(16):
+            snapshot = _open_terminal_recovery_snapshot(path)
+            if snapshot is None:
+                if (
+                    identity is not None
+                    and retired_invalid
+                    and _terminal_entry_absent(path)
+                ):
+                    _recover_exact_run(
+                        kanban_db,
+                        identity,
+                        last_invalid_reason or "malformed-payload",
+                    )
+                break
+            record = snapshot.document or {}
+            reason = None
+            if identity is None:
+                reason = "malformed-name"
+            elif not _terminal_record_valid(record):
+                reason = snapshot.invalid_reason or "malformed-payload"
+            elif not _terminal_record_valid(record, identity):
+                reason = "foreign-identity"
+            if reason is not None:
+                try:
+                    _quarantine_terminal(
+                        path,
+                        identity,
+                        reason,
+                        snapshot=snapshot,
+                    )
+                finally:
+                    snapshot.close()
+                retired_invalid = True
+                last_invalid_reason = reason
+                continue
+            assert identity is not None
+            try:
+                outcome = _finalize_terminal_record(
+                    kanban_db,
+                    path,
+                    record,
+                    snapshot=snapshot,
+                )
+            except Exception as error:
+                _record_board_access_error(identity.board, error)
+                break
+            if outcome == "committed":
+                recovered += 1
+                break
+            if outcome in {"invalid", "foreign"}:
+                # The pathname changed after the recovery snapshot closed.
+                # Reclassify and pin the new inode instead of quarantining it
+                # using authority derived from the older entry.
+                last_invalid_reason = outcome
+                continue
+            break
+        else:
+            print(
+                f"terminal journal replacement churn deferred for {path.name}",
+                file=sys.stderr,
+                flush=True,
+            )
+    # A quarantine or identity-safe retirement performed above can itself
+    # encounter replacement churn. Promote every resulting hidden generation
+    # back to first-class prepared/conflict/committed evidence before this
+    # recovery pass returns; DB replay, if still needed, occurs on the next
+    # bounded dispatcher pass.
+    _drain_retirement_staging()
+    return recovered
+
+def _has_pending_finalization(board: str, task_id: str, run_id: Any) -> bool:
+    """Keep an exact run claimed while its accepted result awaits replay."""
+    if type(run_id) is not int or canonical_run_id(run_id) is None:
+        return False
+    path = _terminal_path(state_path(board, task_id), run_id, "pending")
+    try:
+        path.stat()
+    except FileNotFoundError:
+        pass
+    except OSError:
+        return False
+    else:
+        identity = _terminal_identity(path)
+        if identity is not None:
+            record = _load_terminal_json(path, identity)
+            if _terminal_record_valid(record, identity):
+                return True
+    base = state_path(board, task_id)
+    for prepared in base.parent.glob(
+        f"{base.stem}.run-{run_id}.terminal.prepared-*.json"
+    ):
+        identity = _terminal_evidence_identity(prepared)
+        if identity is None or identity.state != "prepared":
+            continue
+        record = _load_small_json(prepared)
+        if _terminal_evidence_valid(record, identity, "prepared"):
+            return True
+    staged_paths = list(base.parent.glob(".retire.*"))
+    quarantine = base.parent / "quarantine"
+    if quarantine.is_dir() and not quarantine.is_symlink():
+        staged_paths.extend(quarantine.glob(".retire.*"))
+    for staged in staged_paths:
+        snapshot = _open_terminal_recovery_snapshot(staged)
+        if snapshot is None:
+            continue
+        try:
+            authority = _staged_terminal_authority(staged, snapshot)
+            if (
+                authority is not None
+                and authority[0].board == board
+                and authority[0].task_id == task_id
+                and authority[0].run_id == run_id
+                and authority[0].state in {"pending", "prepared", "committed"}
+            ):
+                return True
+        finally:
+            snapshot.close()
+    return False

services/hermes/scripts/cli_lane_retention.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""Bounded age, count, and byte retention for non-pending lane artifacts."""
+
+from __future__ import annotations
+
+import os
+import stat
+import sys
+import time
+from pathlib import Path
+
+from cli_lane_config import (
+    ARTIFACT_GC_INTERVAL_SECONDS,
+    ARTIFACT_RETENTION_AGE_SECONDS,
+    ARTIFACT_RETENTION_BYTES,
+    ARTIFACT_RETENTION_COUNT,
+    STATE_ROOT,
+)
+from cli_lane_evidence import _retire_terminal_entry, _terminal_evidence_valid
+from cli_lane_files import (
+    _fsync_directory,
+    _terminal_evidence_identity,
+    _terminal_identity,
+)
+from cli_lane_quarantine import _quarantine_terminal
+from cli_lane_recovery import _staged_terminal_authority
+from cli_lane_records import _open_terminal_recovery_snapshot, _terminal_record_valid
+
+
+def _artifact_gc_candidates(board_dir: Path) -> list[Path]:
+    """List only bounded artifact classes; pending journals are excluded."""
+    candidates: list[Path] = []
+    patterns = (
+        "*.provider-*.result.json",
+        "*.candidate-*.json",
+        "*.terminal.committed.json",
+        "*.terminal.conflict-*.json",
+        ".retire.*",
+    )
+    for pattern in patterns:
+        candidates.extend(board_dir.glob(pattern))
+    quarantine = board_dir / "quarantine"
+    if quarantine.is_dir():
+        candidates.extend(quarantine.glob("*.quarantine"))
+        candidates.extend(quarantine.glob(".retire.*"))
+    return candidates
+
+def _unlink_artifact_if_same(path: Path, observed: os.stat_result) -> bool:
+    """Remove only the exact artifact inode selected by retention."""
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
+    descriptor = None
+    source_descriptor = None
+    try:
+        descriptor = os.open(path.parent, directory_flags)
+        file_flags = getattr(os, "O_PATH", os.O_RDONLY) | getattr(os, "O_NOFOLLOW", 0)
+        source_descriptor = os.open(path.name, file_flags, dir_fd=descriptor)
+        pinned = os.fstat(source_descriptor)
+        if pinned.st_dev != observed.st_dev or pinned.st_ino != observed.st_ino:
+            return False
+        outcome = _retire_terminal_entry(
+            path,
+            observed,
+            board_descriptor=descriptor,
+            quarantine_descriptor=descriptor,
+            source_descriptor=source_descriptor,
+        )
+        return outcome == "retired"
+    except OSError:
+        return False
+    finally:
+        if source_descriptor is not None:
+            os.close(source_descriptor)
+        if descriptor is not None:
+            os.close(descriptor)
+
+
+def _quarantine_invalid_retained_terminal(path: Path) -> tuple[bool, bool]:
+    """Audit retained committed/conflict evidence on the bounded GC cadence."""
+    committed = path.name.endswith(".terminal.committed.json")
+    conflict = ".terminal.conflict-" in path.name and path.name.endswith(".json")
+    if not committed and not conflict:
+        return False, False
+    identity = (
+        _terminal_identity(path) if committed else _terminal_evidence_identity(path)
+    )
+    snapshot = _open_terminal_recovery_snapshot(path)
+    if snapshot is None:
+        return True, False
+    document = snapshot.document
+    valid = False
+    if identity is not None and document is not None:
+        valid = (
+            _terminal_record_valid(document, identity)
+            if committed
+            else _terminal_evidence_valid(document, identity, "conflict")
+        )
+    if valid:
+        snapshot.close()
+        return False, False
+    try:
+        _quarantine_terminal(
+            path,
+            identity,
+            snapshot.invalid_reason or "invalid-retained-terminal-evidence",
+            snapshot=snapshot,
+        )
+    finally:
+        snapshot.close()
+    try:
+        path.stat(follow_symlinks=False)
+    except FileNotFoundError:
+        return True, True
+    except OSError:
+        return True, False
+    return True, False
+
+def gc_lane_artifacts(
+    *,
+    now: float | None = None,
+    max_age_seconds: int = ARTIFACT_RETENTION_AGE_SECONDS,
+    max_count: int = ARTIFACT_RETENTION_COUNT,
+    max_bytes: int = ARTIFACT_RETENTION_BYTES,
+) -> int:
+    """Bound non-pending lane artifacts by age, count, and bytes per board."""
+    current = time.time() if now is None else now
+    removed = 0
+    for board_dir in STATE_ROOT.iterdir() if STATE_ROOT.is_dir() else ():
+        if not board_dir.is_dir() or board_dir.is_symlink():
+            continue
+        board_removed = 0
+        entries: list[tuple[Path, os.stat_result]] = []
+        for path in _artifact_gc_candidates(board_dir):
+            classified, quarantined = _quarantine_invalid_retained_terminal(path)
+            if classified:
+                if quarantined:
+                    removed += 1
+                    board_removed += 1
+                continue
+            if path.name.startswith(".retire."):
+                snapshot = _open_terminal_recovery_snapshot(path)
+                if snapshot is not None:
+                    try:
+                        if _staged_terminal_authority(path, snapshot) is not None:
+                            # Accepted authority is first promoted/replayed by
+                            # recovery; retention never deletes it directly.
+                            continue
+                    finally:
+                        snapshot.close()
+            try:
+                file_stat = path.stat(follow_symlinks=False)
+            except OSError:
+                continue
+            if not stat.S_ISREG(file_stat.st_mode):
+                if _unlink_artifact_if_same(path, file_stat):
+                    removed += 1
+                    board_removed += 1
+                continue
+            if max_age_seconds >= 0 and current - file_stat.st_mtime > max_age_seconds:
+                if _unlink_artifact_if_same(path, file_stat):
+                    removed += 1
+                    board_removed += 1
+            else:
+                entries.append((path, file_stat))
+        retained_count = 0
+        retained_bytes = 0
+        for path, file_stat in sorted(
+            entries, key=lambda item: item[1].st_mtime, reverse=True
+        ):
+            exceeds_count = max_count >= 0 and retained_count >= max_count
+            exceeds_bytes = (
+                max_bytes >= 0 and retained_bytes + file_stat.st_size > max_bytes
+            )
+            if exceeds_count or exceeds_bytes:
+                if _unlink_artifact_if_same(path, file_stat):
+                    removed += 1
+                    board_removed += 1
+                continue
+            retained_count += 1
+            retained_bytes += file_stat.st_size
+        if board_removed:
+            _fsync_directory(board_dir)
+    return removed
+
+LAST_ARTIFACT_GC = 0.0
+
+def maybe_gc_lane_artifacts(*, now: float | None = None) -> int:
+    """Run artifact retention on a bounded cadence, not every dispatcher tick."""
+    global LAST_ARTIFACT_GC
+    current = time.time() if now is None else now
+    if current - LAST_ARTIFACT_GC < ARTIFACT_GC_INTERVAL_SECONDS:
+        return 0
+    LAST_ARTIFACT_GC = current
+    try:
+        return gc_lane_artifacts(now=current)
+    except OSError as error:
+        print(
+            f"lane artifact retention deferred: {type(error).__name__}: {error}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return 0

services/hermes/scripts/cli_lane_routing.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""Switchyard route selection and provider-health exclusion for CLI workers."""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+import urllib.error
+import urllib.request
+from typing import Any, Callable
+
+from cli_lane_config import (
+    EFFORTS,
+    PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS,
+    PROVIDER_HEALTH_MAX_AGE_SECONDS,
+    PROVIDER_HEALTH_PATHS,
+    SWITCHYARD_URL,
+    Route,
+)
+from cli_lane_files import load_json
+
+
+def parse_assignee(assignee: str) -> tuple[str | None, str | None]:
+    """Parse external lane overrides while leaving cli-auto fully automatic."""
+    value = str(assignee or "").strip().lower()
+    if value == "cli-auto":
+        return None, None
+    match = re.fullmatch(r"cli-(codex|claude)-(low|medium|high|xhigh)", value)
+    if not match:
+        raise ValueError(f"unsupported external lane assignee: {assignee}")
+    return match.group(1), match.group(2)
+
+def _health_number(value: Any) -> float | None:
+    """Return a plain numeric health field, rejecting booleans and strings."""
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    return float(value)
+
+def _quota_reset_readmits(
+    health: dict[str, Any], current: float, reset_timestamp: float | None
+) -> bool:
+    """A passed provider quota reset ends a capacity cooldown early."""
+    if health.get("authenticated") is False:
+        # An authentication failure is not repaired by a quota window reset.
+        return False
+    failed_at = _health_number(health.get("failed_at"))
+    return (
+        reset_timestamp is not None
+        and failed_at is not None
+        and failed_at < reset_timestamp <= current
+    )
+
+def fresh_unavailable_provider(
+    now: float | None = None,
+    quota_resets: dict[str, float] | None = None,
+) -> str | None:
+    """Return one recently proven-down provider for automatic route exclusion."""
+    current = time.time() if now is None else now
+    resets = quota_resets or {}
+    unavailable: list[str] = []
+    for provider, path in PROVIDER_HEALTH_PATHS.items():
+        health = load_json(path)
+        if health.get("state") not in ("unavailable", "capacity-limited"):
+            continue
+        cooldown_until = _health_number(health.get("cooldown_until"))
+        if cooldown_until is not None:
+            # Lane-recorded failures re-enter only after the full cooldown has
+            # elapsed (hysteresis), a quota reset passed, or fresh success
+            # rewrote the state above.
+            if current >= cooldown_until or _quota_reset_readmits(
+                health, current, resets.get(provider)
+            ):
+                continue
+            unavailable.append(provider)
+            continue
+        try:
+            age = current - path.stat().st_mtime
+        except OSError:
+            continue
+        max_age = (
+            PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS
+            if health.get("authenticated") is False
+            else PROVIDER_HEALTH_MAX_AGE_SECONDS
+        )
+        if 0 <= age <= max_age:
+            unavailable.append(provider)
+    # If both providers are down, keep the normal attempt/fallback path so the
+    # card records authoritative current errors instead of trusting snapshots.
+    return unavailable[0] if len(unavailable) == 1 else None
+
+def _decode_worker_target(value: str) -> tuple[str, str, str]:
+    """Decode the selected model header emitted by a worker decision route."""
+    parts = value.split("/", 3)
+    if len(parts) != 4 or parts[0] != "worker":
+        raise RuntimeError(f"invalid Switchyard worker target: {value}")
+    provider, model, effort = parts[1:]
+    if provider not in {"codex", "claude"} or effort not in EFFORTS:
+        raise RuntimeError(f"unsupported Switchyard worker target: {value}")
+    return provider, model, effort
+
+def select_route(
+    prompt: str,
+    assignee: str,
+    *,
+    exclude_provider: str | None = None,
+    exclude_reason: str | None = None,
+    switchyard_url: str = SWITCHYARD_URL,
+    open_request: Callable[..., Any] = urllib.request.urlopen,
+) -> Route:
+    """Ask Switchyard to select one native CLI worker at this boundary."""
+    started = time.monotonic()
+    manual_provider, manual_effort = parse_assignee(assignee)
+    if manual_provider and manual_effort:
+        route_id = f"atlas/worker/manual/{manual_provider}/{manual_effort}"
+        source = "switchyard-manual"
+    else:
+        route_id = "atlas/worker/auto/maximum"
+        source = "switchyard-classifier"
+    context = prompt
+    if exclude_provider:
+        reason = exclude_reason or "failed or exhausted capacity at this boundary"
+        context += (
+            f"\n\nRouting constraint: the {exclude_provider} provider {reason}. "
+            "Do not select it."
+        )
+    payload = json.dumps(
+        {
+            "model": route_id,
+            "messages": [{"role": "user", "content": context}],
+            "stream": False,
+            "max_tokens": 1,
+        }
+    ).encode("utf-8")
+    request = urllib.request.Request(
+        switchyard_url,
+        data=payload,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with open_request(request, timeout=60) as response:
+            selected = str(response.headers.get("x-model-router-selected-model") or "")
+            rationale = str(response.headers.get("x-model-router-rationale") or "")
+            response_body = response.read()
+    except (OSError, urllib.error.URLError) as exc:
+        raise RuntimeError(f"Switchyard worker routing failed: {exc}") from exc
+    provider, model, effort = _decode_worker_target(selected)
+    # Switchyard preserves the stable tier target in the selection header and
+    # top-level response model. The worker broker's assistant content contains
+    # the steward-resolved provider model required by the native CLI.
+    try:
+        response_document = json.loads(response_body)
+        resolved_target = str(response_document["choices"][0]["message"]["content"] or "")
+        resolved_provider, resolved_model, resolved_effort = _decode_worker_target(resolved_target)
+        required_prefix = "gpt-" if provider == "codex" else "claude-"
+        if (
+            resolved_provider == provider
+            and resolved_effort == effort
+            and resolved_model.startswith(required_prefix)
+        ):
+            model = resolved_model
+    except (AttributeError, IndexError, KeyError, RuntimeError, TypeError, ValueError, json.JSONDecodeError):
+        pass
+    if exclude_provider and provider == exclude_provider:
+        alternate = "claude" if provider == "codex" else "codex"
+        guarded = select_route(
+            prompt,
+            f"cli-{alternate}-{effort}",
+            switchyard_url=switchyard_url,
+            open_request=open_request,
+        )
+        return Route(
+            provider=guarded.provider,
+            model=guarded.model,
+            effort=guarded.effort,
+            profile=guarded.profile,
+            classifier=f"{source}-health-guard",
+            reason=(
+                f"Switchyard selected excluded {exclude_provider}; preserved "
+                f"its {effort} effort on healthy-provider route. {guarded.reason}"
+            ),
+            latency_ms=int((time.monotonic() - started) * 1000),
+            fallback_chain=(),
+        )
+    return Route(
+        provider=provider,
+        model=model,
+        effort=effort,
+        profile=f"{provider}-{effort}",
+        classifier=source,
+        reason=rationale or f"Switchyard selected {selected}",
+        latency_ms=int((time.monotonic() - started) * 1000),
+        fallback_chain=(),
+    )

services/hermes/scripts/execution_pool_client.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+"""Ordinal-local signing and SCM mediation boundary for a model worker."""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+import urllib.error
+import urllib.request
+from http.server import BaseHTTPRequestHandler
+from pathlib import Path
+from typing import Any
+
+import cli_lane_goal
+from execution_pool_protocol import (
+    MAX_WIRE_BYTES,
+    PROTOCOL_VERSION,
+    BoundedHTTPServer,
+    ProtocolError,
+    canonical_json,
+    parse_wire,
+    read_key,
+    sign_envelope,
+    verify_envelope,
+)
+from execution_pool_scm import Boundary as SCMBoundary
+
+
+KEY_PATH = Path(
+    os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/pool-access/execution-pool-key")
+)
+COORDINATOR = os.environ.get(
+    "HERMES_EXECUTION_POOL_URL",
+    "http://hermes-execution-pool.hermes.svc.cluster.local:9007",
+).rstrip("/")
+ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
+PORT = int(os.environ.get("HERMES_EXECUTION_CLIENT_PORT", "9009"))
+RESULT_FIELDS = frozenset(
+    {"status", "summary", "changed_files", "tests_run", "artifacts", "findings", "blockers"}
+)
+
+
+def _binding(value: dict[str, Any]) -> dict[str, Any]:
+    return {
+        name: value[name]
+        for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt")
+    }
+
+
+def _validate_result(payload: Any) -> dict[str, Any]:
+    if not isinstance(payload, dict):
+        raise ProtocolError("terminal result payload must be an object")
+    structured = payload.get("structured")
+    if not isinstance(structured, dict) or set(structured) != RESULT_FIELDS:
+        raise ProtocolError("terminal result fields do not match the reviewed schema")
+    if structured.get("status") not in cli_lane_goal.RESULT_STATUSES:
+        raise ProtocolError("terminal result status is invalid")
+    if not isinstance(structured.get("summary"), str) or not structured["summary"].strip():
+        raise ProtocolError("terminal result summary is required")
+    for name in RESULT_FIELDS - {"status", "summary"}:
+        value = structured.get(name)
+        if not isinstance(value, list) or any(not isinstance(item, str) for item in value):
+            raise ProtocolError(f"terminal result {name} must be a text list")
+    return payload
+
+
+class ClientBoundary:
+    """Keep HMAC and SCM authority outside the model-facing container."""
+
+    def __init__(self, key: bytes, scm: SCMBoundary | None = None):
+        self.key = key
+        self.scm = scm or SCMBoundary(key)
+        self.current: dict[str, Any] | None = None
+        self.lock = threading.RLock()
+
+    def _post(self, path: str, envelope: dict[str, Any]) -> dict[str, Any]:
+        request = urllib.request.Request(
+            COORDINATOR + path,
+            data=canonical_json(envelope),
+            method="POST",
+            headers={"Content-Type": "application/json", "Cache-Control": "no-store"},
+        )
+        with urllib.request.urlopen(request, timeout=60) as response:
+            body = response.read(MAX_WIRE_BYTES + 1)
+        if len(body) > MAX_WIRE_BYTES:
+            raise ProtocolError("coordinator response exceeds the wire limit")
+        return verify_envelope(self.key, json.loads(body))
+
+    def poll(self) -> dict[str, Any]:
+        poll_binding = {
+            "board": "",
+            "task_id": "",
+            "run_id": "",
+            "worker_ordinal": ORDINAL,
+            "attempt": 0,
+        }
+        response = self._post(
+            "/v1/poll",
+            sign_envelope(self.key, "poll", poll_binding, {"ready": True}),
+        )
+        with self.lock:
+            if response["kind"] == "ack":
+                self.current = None
+                return {"assignment": None}
+            if response["kind"] != "assignment" or response["worker_ordinal"] != ORDINAL:
+                raise ProtocolError("coordinator returned a foreign assignment")
+            checkout = self.scm.checkout(response)
+            self.current = response
+            assignment = {
+                **_binding(response),
+                "payload": response["payload"],
+                "workspace": checkout["workspace"],
+                "baseline_sha": checkout["baseline_sha"],
+                "protocol_version": PROTOCOL_VERSION,
+            }
+        return {"assignment": assignment}
+
+    def _current_for(self, supplied: Any) -> tuple[dict[str, Any], dict[str, Any]]:
+        if not isinstance(supplied, dict):
+            raise ProtocolError("local request binding must be an object")
+        if self.current is None or supplied != _binding(self.current):
+            raise ProtocolError("local request does not own the current assignment")
+        return self.current, dict(supplied)
+
+    def heartbeat(self, request: dict[str, Any]) -> dict[str, Any]:
+        payload = request.get("payload")
+        if not isinstance(payload, dict):
+            raise ProtocolError("heartbeat payload must be an object")
+        with self.lock:
+            _assignment, binding = self._current_for(request.get("binding"))
+            response = self._post(
+                "/v1/heartbeat",
+                sign_envelope(self.key, "heartbeat", binding, payload),
+            )
+            if response["kind"] != "ack" or _binding(response) != binding:
+                raise ProtocolError("coordinator acknowledgement binding changed")
+        return {"ack": response["payload"]}
+
+    def _submit(
+        self, assignment: dict[str, Any], request: dict[str, Any],
+        structured: dict[str, Any],
+    ) -> None:
+        """Publish the run's work, downgrading the result rather than losing it.
+
+        A refused push or draft used to unwind the whole worker, so the run's
+        actual output was discarded and the task was blocked with only the SCM
+        error. The commits live on the durable workspace volume, so the result is
+        still reported here -- as blocked, carrying the reason -- which keeps the
+        prior work recoverable and reviewable by a human.
+        """
+        try:
+            submission = self.scm.submit(assignment, request)
+        except (OSError, ProtocolError, RuntimeError, ValueError) as error:
+            structured["status"] = "blocked"
+            reason = (
+                "Distributed SCM submission failed; the commits remain on this "
+                f"ordinal's workspace: {type(error).__name__}: {error}"
+            )
+            if reason not in structured["blockers"]:
+                structured["blockers"].append(reason)
+            return
+        pull = str(submission.get("pull_request") or "")
+        branch = str(submission.get("branch") or "")
+        for artifact in (pull, f"branch:{branch}" if branch else ""):
+            if artifact and artifact not in structured["artifacts"]:
+                structured["artifacts"].append(artifact)
+
+    def finish(self, request: dict[str, Any]) -> dict[str, Any]:
+        payload = _validate_result(request.get("payload"))
+        with self.lock:
+            assignment, binding = self._current_for(request.get("binding"))
+            structured = payload["structured"]
+            if structured["status"] == "completed" and int(payload.get("returncode", 1)) == 0:
+                self._submit(assignment, request, structured)
+            response = self._post(
+                "/v1/result", sign_envelope(self.key, "result", binding, payload)
+            )
+            if response["kind"] != "ack" or _binding(response) != binding:
+                raise ProtocolError("coordinator acknowledgement binding changed")
+            if response["payload"].get("accepted"):
+                self.current = None
+        return {"ack": response["payload"], "structured": structured}
+
+
+def handler_factory(boundary: ClientBoundary) -> type[BaseHTTPRequestHandler]:
+    class Handler(BaseHTTPRequestHandler):
+        server_version = f"hermes-execution-mediator/{PROTOCOL_VERSION}"
+
+        def _reply(self, status: int, value: dict[str, Any]) -> None:
+            body = canonical_json(value)
+            self.send_response(status)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(body)))
+            self.send_header("Cache-Control", "no-store")
+            self.end_headers()
+            self.wfile.write(body)
+
+        def do_GET(self) -> None:  # noqa: N802
+            value = {"ready": True, "protocol_version": PROTOCOL_VERSION}
+            self._reply(200, value) if self.path == "/ready" else self._reply(
+                404, {"error": "not found"}
+            )
+
+        def do_POST(self) -> None:  # noqa: N802
+            try:
+                length = int(self.headers.get("Content-Length", "0"))
+                request = (
+                    parse_wire(self.rfile.read(length))
+                    if 0 < length <= MAX_WIRE_BYTES
+                    else None
+                )
+                allowed = {"operation", "binding", "payload", "title", "body"}
+                if not request or set(request) - allowed:
+                    raise ProtocolError("invalid local mediator request")
+                operation = str(request.get("operation") or "")
+                routes = {
+                    "poll": boundary.poll,
+                    "heartbeat": lambda: boundary.heartbeat(request),
+                    "finish": lambda: boundary.finish(request),
+                }
+                if operation not in routes:
+                    raise ProtocolError("unsupported local mediator operation")
+                self._reply(200, routes[operation]())
+            except (ProtocolError, OSError, ValueError, urllib.error.URLError) as error:
+                self._reply(409, {"error": str(error)[:2000]})
+
+        def log_message(self, _format: str, *_arguments: Any) -> None:
+            return
+
+    return Handler
+
+
+def main() -> int:
+    if ORDINAL not in range(3):
+        raise SystemExit("HERMES_WORKER_ORDINAL must be 0, 1, or 2")
+    key = read_key(KEY_PATH)
+    BoundedHTTPServer(
+        ("0.0.0.0", PORT),
+        handler_factory(ClientBoundary(key)),
+        max_workers=4,
+    ).serve_forever()
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover - exercised by the container entrypoint
+    raise SystemExit(main())

services/hermes/scripts/execution_pool_coordinator.py

@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+"""Coordinator-only Kanban claim/finalize bridge for three Hermes workers."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import threading
+import time
+from http.server import BaseHTTPRequestHandler
+from pathlib import Path
+from typing import Any
+
+import cli_lane_goal
+from cli_lane_config import canonical_run_id
+from execution_pool_project import resolve_assignment
+from execution_pool_protocol import (
+    MAX_ACTIVITY_BYTES,
+    ProtocolError,
+    derive_ordinal_key,
+    envelope_ordinal,
+    sign_envelope,
+    verify_envelope,
+)
+from execution_pool_store import PoolStore
+
+REDACTIONS = (
+    re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/-]{12,}"),
+    re.compile(
+        r"(?i)(authorization|token|secret|password|api[_-]?key|"
+        r"refresh[_-]?token|access[_-]?token)[\"']?\s*[:=]\s*[\"']?[^\s\"',;}]+"
+    ),
+    re.compile(r"\b(?:sk-ant-|sk-proj-|rt_)[A-Za-z0-9_-]{12,}\b"),
+    re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"),
+    re.compile(r"(?i)https://[^/@\s]+@scm\.bstein\.dev"),
+)
+
+
+def _task_value(task: Any, name: str, default: Any = None) -> Any:
+    return getattr(task, name, default)
+
+
+def resolve_scm(task: Any, board: str = "titan-iac") -> tuple[str, str, str]:
+    """Resolve SCM only through the canonical board/project registry."""
+    return resolve_assignment(board, task)
+
+
+def assignment_payload(
+    kanban_db: Any, connection: Any, task: Any, board: str = "titan-iac"
+) -> dict[str, Any]:
+    context = kanban_db.build_worker_context(connection, str(_task_value(task, "id")))
+    if not isinstance(context, str):
+        context = json.dumps(context, default=str, sort_keys=True)
+    encoded = context.encode("utf-8")
+    if len(encoded) > 32 * 1024:
+        raise RuntimeError("Kanban worker context exceeds the 32KiB assignment limit")
+    repo_url, branch, base_branch = resolve_scm(task, board)
+    runtime = int(_task_value(task, "max_runtime_seconds", 0) or 12 * 60 * 60)
+    runtime = max(60, min(runtime, 12 * 60 * 60))
+    return {
+        "context": context,
+        "assignee": str(_task_value(task, "assignee", "cli-auto") or "cli-auto"),
+        "repo_url": repo_url,
+        "branch": branch,
+        "base_branch": base_branch,
+        "max_runtime_seconds": runtime,
+        "deadline_unix": int(time.time()) + runtime,
+        "goal_mode": bool(_task_value(task, "goal_mode", False)),
+        "goal_max_turns": max(1, min(int(_task_value(task, "goal_max_turns", 1) or 1), 12)),
+    }
+
+
+def sanitize_activity(value: Any) -> str:
+    """Bound and redact worker output before it reaches the shared UI log."""
+    text = str(value or "").replace("\x00", "")
+    text = "".join(character for character in text if character in "\n\t" or ord(character) >= 32)
+    for pattern in REDACTIONS:
+        text = pattern.sub("[REDACTED]", text)
+    return text.encode("utf-8")[:MAX_ACTIVITY_BYTES].decode("utf-8", "ignore")
+
+
+def _append_activity(kanban_db: Any, envelope: dict[str, Any]) -> None:
+    payload = envelope["payload"]
+    if not isinstance(payload, dict):
+        raise ProtocolError("heartbeat payload must be an object")
+    activity = sanitize_activity(payload.get("activity"))
+    if not activity:
+        return
+    path = Path(kanban_db.worker_log_path(envelope["task_id"], board=envelope["board"]))
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if path.is_symlink():
+        raise ProtocolError("Kanban activity log must not be a symlink")
+    flags = os.O_APPEND | os.O_CREAT | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0)
+    descriptor = os.open(path, flags, 0o600)
+    try:
+        os.write(descriptor, activity.encode("utf-8"))
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+def _append_terminal_activity(kanban_db: Any, record: dict[str, Any]) -> None:
+    """Append one terminal batch exactly once across result retries/restarts."""
+    payload = record.get("result")
+    activity = payload.get("final_activity") if isinstance(payload, dict) else ""
+    activity = sanitize_activity(activity)
+    if not activity:
+        return
+    marker = f"\n[execution-pool-result:{record['result_digest']}]\n"
+    path = Path(kanban_db.worker_log_path(record["task_id"], board=record["board"]))
+    if path.is_symlink():
+        raise ProtocolError("Kanban activity log must not be a symlink")
+    if path.exists():
+        flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+        descriptor = os.open(path, flags)
+        try:
+            size = os.fstat(descriptor).st_size
+            os.lseek(descriptor, max(0, size - 64 * 1024), os.SEEK_SET)
+            if marker.encode() in os.read(descriptor, 64 * 1024):
+                return
+        finally:
+            os.close(descriptor)
+    _append_activity(
+        kanban_db,
+        {**record, "payload": {"activity": marker + activity}},
+    )
+
+
+class Coordinator:
+    """Own all transitions between durable assignments and Hermes Kanban."""
+
+    def __init__(self, key: bytes, store: PoolStore):
+        self.master_key = key
+        self.store = store
+        self._kanban_lock = threading.RLock()
+
+    def _key(self, envelope: dict[str, Any]) -> bytes:
+        return derive_ordinal_key(self.master_key, envelope_ordinal(envelope))
+
+    def _verify(self, envelope: dict[str, Any], kind: str) -> dict[str, Any]:
+        return verify_envelope(self._key(envelope), envelope, expected_kind=kind)
+
+    def _sign(
+        self, kind: str, binding: dict[str, Any], payload: dict[str, Any]
+    ) -> dict[str, Any]:
+        key = derive_ordinal_key(self.master_key, int(binding["worker_ordinal"]))
+        return sign_envelope(key, kind, binding, payload)
+
+    @staticmethod
+    def _binding(record: dict[str, Any]) -> dict[str, Any]:
+        return {name: record[name] for name in (
+            "board", "task_id", "run_id", "worker_ordinal", "attempt"
+        )}
+
+    def poll(self, envelope: dict[str, Any]) -> dict[str, Any]:
+        verified = self._verify(envelope, "poll")
+        record = self.store.offer(int(verified["worker_ordinal"]))
+        if record is None:
+            binding = {**self._binding({
+                "board": "", "task_id": "", "run_id": "",
+                "worker_ordinal": verified["worker_ordinal"], "attempt": 0,
+            })}
+            return self._sign("ack", binding, {"assignment": None})
+        return self._sign("assignment", self._binding(record), record["payload"])
+
+    def heartbeat(self, envelope: dict[str, Any]) -> dict[str, Any]:
+        verified = self._verify(envelope, "heartbeat")
+        accepted, duplicate = self.store.heartbeat(verified)
+        from hermes_cli import kanban_db
+
+        payload = verified["payload"]
+        note = sanitize_activity(payload.get("note") if isinstance(payload, dict) else "")[:1000]
+        with self._kanban_lock, kanban_db.scoped_current_board(verified["board"]):
+            connection = kanban_db.connect(board=verified["board"])
+            try:
+                run_id = canonical_run_id(verified["run_id"])
+                if run_id is None:
+                    raise ProtocolError("worker run ID is not a canonical SQLite run")
+                alive = kanban_db.heartbeat_worker(
+                    connection, verified["task_id"],
+                    note=note or "distributed worker active",
+                    expected_run_id=run_id,
+                )
+                if not alive:
+                    raise ProtocolError("Kanban run no longer owns this worker")
+                if not duplicate:
+                    _append_activity(kanban_db, verified)
+            finally:
+                connection.close()
+        return self._sign(
+            "ack", self._binding(verified),
+            {"accepted": accepted, "duplicate": duplicate},
+        )
+
+    def result(self, envelope: dict[str, Any]) -> dict[str, Any]:
+        verified = self._verify(envelope, "result")
+        record, duplicate = self.store.accept_result(verified)
+        if record.get("state") == "result":
+            self.finalize(record)
+        return self._sign(
+            "ack", self._binding(verified),
+            {"accepted": True, "duplicate": duplicate},
+        )
+
+    def finalize(self, record: dict[str, Any]) -> None:
+        """Apply a result only if this exact run remains authoritative."""
+        from hermes_cli import kanban_db
+
+        binding = self._binding(record)
+        run_id = canonical_run_id(binding["run_id"])
+        if run_id is None:
+            self.store.finalize(binding, "stale")
+            return
+        payload = record.get("result")
+        if not isinstance(payload, dict):
+            raise ProtocolError("result payload must be an object")
+        structured = payload.get("structured")
+        if not isinstance(structured, dict):
+            structured = {}
+        with self._kanban_lock, kanban_db.scoped_current_board(binding["board"]):
+            connection = kanban_db.connect(board=binding["board"])
+            try:
+                task = kanban_db.get_task(connection, binding["task_id"])
+                current_run_id = canonical_run_id(
+                    _task_value(task, "current_run_id", None)
+                ) if task is not None else None
+                if current_run_id is None or current_run_id != run_id:
+                    self.store.finalize(binding, "stale")
+                    return
+                _append_terminal_activity(kanban_db, record)
+                metadata = {
+                    "executor": "distributed-execution-pool",
+                    "worker_ordinal": binding["worker_ordinal"],
+                    "attempt": binding["attempt"],
+                    "node": str(payload.get("node") or "unknown")[:128],
+                    "route": payload.get("route") if isinstance(payload.get("route"), dict) else {},
+                    "provider_sessions": payload.get("provider_sessions")
+                    if isinstance(payload.get("provider_sessions"), dict) else {},
+                    "changed_files": structured.get("changed_files", []),
+                    "tests_run": structured.get("tests_run", []),
+                    "artifacts": structured.get("artifacts", []),
+                    "findings": structured.get("findings", []),
+                    "blockers": structured.get("blockers", []),
+                }
+                problem = cli_lane_goal.unfinished_result_reason(structured)
+                if (
+                    structured.get("status") == "completed"
+                    and int(payload.get("returncode", 1)) == 0
+                    and problem is None
+                ):
+                    changed = kanban_db.complete_task(
+                        connection, binding["task_id"],
+                        result=json.dumps(structured, sort_keys=True),
+                        summary=str(structured.get("summary") or "Completed"),
+                        metadata=metadata, expected_run_id=run_id,
+                    )
+                else:
+                    reason = problem or "; ".join(map(str, structured.get("blockers", [])))
+                    reason = reason or str(structured.get("summary") or "worker failed")
+                    changed = kanban_db.block_task(
+                        connection, binding["task_id"], reason=reason,
+                        kind="transient" if payload.get("capacity_failure") else "capability",
+                        expected_run_id=run_id,
+                    )
+                self.store.finalize(binding, "finalized" if changed else "stale")
+            finally:
+                connection.close()
+
+    @staticmethod
+    def assignment_payload(
+        kanban_db: Any, connection: Any, task: Any, board: str
+    ) -> dict[str, Any]:
+        """Expose payload construction to the maintenance passes."""
+        return assignment_payload(kanban_db, connection, task, board)
+
+    def recover_results(self) -> None:
+        """Re-apply every durable result that Kanban has not confirmed yet."""
+        from execution_pool_maintenance import recover_results
+
+        recover_results(self)
+
+    def expire_leases(self) -> None:
+        """Fence dead attempts; after bounded retries, surface and release the run."""
+        from execution_pool_maintenance import expire_leases
+
+        expire_leases(self)
+
+    def reconcile(self) -> None:
+        """Recover the narrow claim/assignment crash gaps without double execution."""
+        from execution_pool_maintenance import reconcile
+
+        reconcile(self)
+
+    def dispatch(self) -> None:
+        """Claim up to the free ordinal count and materialize assignments."""
+        from execution_pool_maintenance import dispatch
+
+        dispatch(self)
+
+
+def handler_factory(coordinator: Coordinator) -> type[BaseHTTPRequestHandler]:
+    """Compatibility export for tests and the mounted coordinator entry point."""
+    from execution_pool_server import handler_factory as factory
+
+    return factory(coordinator)
+
+
+def main() -> int:
+    from execution_pool_server import run
+
+    return run(Coordinator)
+
+
+if __name__ == "__main__":  # pragma: no cover - exercised by the container entrypoint
+    raise SystemExit(main())

services/hermes/scripts/execution_pool_maintenance.py

@@ -0,0 +1,371 @@
+#!/usr/bin/env python3
+"""Recoverable maintenance passes for the Hermes execution-pool coordinator.
+
+Every pass here runs against a shared Kanban SQLite database that other writers
+(the agent UI, the local CLI lane, crons) hold concurrently, so *any* board call
+can fail transiently. The rules this module enforces:
+
+* No pass ever leaves a claimed run without a durable row that some later pass
+  will revisit. A Kanban write that fails leaves the row in its retryable state.
+* No single poisoned row, task, or board aborts the work queued behind it.
+* A durable row only reaches a terminal state on authoritative evidence about
+  its Kanban run, because terminal rows are the only ones ever collected.
+* A coordinator-side fault is never converted into a Kanban mutation. Only a
+  genuine capability failure of the run itself parks that exact run.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+import sys
+from typing import Any
+
+import cli_lane_dispatch
+from cli_lane_config import canonical_run_id
+from execution_pool_project import ProjectPolicyError, distributed_workspace_eligible
+from execution_pool_protocol import ProtocolError
+
+
+RECOVERABLE_BOARD_ERRORS = (OSError, sqlite3.Error)
+DEFERRALS: dict[str, str] = {}
+MAX_DEFERRALS = 64
+
+
+def _task_value(task: Any, name: str, default: Any = None) -> Any:
+    return getattr(task, name, default)
+
+
+def _defer(context: str, error: BaseException) -> None:
+    """Report one recoverable pool-side failure once per distinct detail.
+
+    The maintenance loop re-runs every few seconds, so an unchanged failure is
+    reported once rather than on every pass. The table is bounded because its
+    keys include run identities.
+    """
+    detail = f"{type(error).__name__}: {error}"
+    if DEFERRALS.get(context) == detail:
+        return
+    if len(DEFERRALS) >= MAX_DEFERRALS:
+        DEFERRALS.clear()
+    DEFERRALS[context] = detail
+    print(f"pool work deferred for {context}: {detail}", file=sys.stderr, flush=True)
+
+
+def _settled(context: str) -> None:
+    DEFERRALS.pop(context, None)
+
+
+def _context(record: dict[str, Any]) -> str:
+    return f"{record['board']}/{record['task_id']}#{record['run_id']}"
+
+
+def _block_exact(
+    kanban_db: Any,
+    board: str,
+    task_id: str,
+    run_id: int | None,
+    reason: str,
+    *,
+    kind: str = "capability",
+) -> None:
+    """Park exactly one run, never letting the write itself escape."""
+    try:
+        with kanban_db.scoped_current_board(board):
+            connection = kanban_db.connect(board=board)
+            try:
+                kanban_db.block_task(
+                    connection, task_id, reason=reason, kind=kind,
+                    expected_run_id=run_id,
+                )
+            finally:
+                connection.close()
+    except Exception as error:  # noqa: BLE001 - a failed park must not wedge the pass
+        _defer(f"{board}/{task_id} block", error)
+
+
+def recover_results(pool: Any) -> None:
+    """Re-apply each durable result in isolation from its neighbours."""
+    for record in pool.store.pending_results():
+        try:
+            pool.finalize(record)
+            _settled(_context(record))
+        except Exception as error:  # noqa: BLE001 - one bad result must not block the rest
+            print(
+                f"result recovery deferred: {type(error).__name__}: {error}",
+                file=sys.stderr,
+                flush=True,
+            )
+
+
+def _lease_failure_state(
+    kanban_db: Any, connection: Any, binding: dict[str, Any], run_id: int
+) -> str:
+    """Classify one exhausted lease from authoritative Kanban evidence only."""
+    task = kanban_db.get_task(connection, binding["task_id"])
+    current = (
+        canonical_run_id(_task_value(task, "current_run_id", None))
+        if task is not None
+        else None
+    )
+    if current != run_id:
+        return "stale"
+    if str(_task_value(task, "status", "") or "") != "running":
+        return "finalized"
+    if kanban_db.block_task(
+        connection,
+        binding["task_id"],
+        reason=(
+            "Distributed worker lease expired after "
+            f"{binding['attempt']} fenced attempts"
+        ),
+        kind="transient",
+        expected_run_id=run_id,
+    ):
+        return "finalized"
+    # Kanban refused the exact-run park yet still reports the run as running, so
+    # there is no terminal evidence. Keep the row for the next pass to retry.
+    raise ProtocolError("Kanban refused the exact-run lease park")
+
+
+def record_lease_failure(pool: Any, record: dict[str, Any]) -> None:
+    """Record one exhausted lease against its exact Kanban run, idempotently.
+
+    The row stays ``lease_failed`` unless this pass obtains authoritative
+    evidence: the run moved on or vanished (``stale``), Kanban already parked it
+    (``finalized``), or this call parked it (``finalized``). Anything else --
+    including a refused write or an unreadable board -- leaves the row for the
+    next pass, so no run is ever silently dropped.
+    """
+    from hermes_cli import kanban_db
+
+    binding = pool._binding(record)
+    run_id = canonical_run_id(binding["run_id"])
+    if run_id is None:
+        pool.store.finalize(binding, "stale")
+        return
+    with pool._kanban_lock, kanban_db.scoped_current_board(binding["board"]):
+        connection = kanban_db.connect(board=binding["board"])
+        try:
+            state = _lease_failure_state(kanban_db, connection, binding, run_id)
+        finally:
+            connection.close()
+    pool.store.finalize(binding, state)
+    _settled(_context(record))
+
+
+def recover_lease_failures(pool: Any) -> None:
+    """Retry every released-but-unrecorded lease failure, each in isolation."""
+    for record in pool.store.failed_leases():
+        try:
+            record_lease_failure(pool, record)
+        except Exception as error:  # noqa: BLE001 - retried on the next pass
+            _defer(f"{_context(record)} lease park", error)
+
+
+def expire_leases(pool: Any) -> None:
+    """Fence dead attempts, then drain every unconfirmed terminal record."""
+    pool.store.expire_leases()
+    recover_lease_failures(pool)
+
+
+def _release_moved_run(pool: Any, kanban_db: Any, record: dict[str, Any]) -> None:
+    """Release one durable assignment whose Kanban run is no longer ours."""
+    with kanban_db.scoped_current_board(record["board"]):
+        connection = kanban_db.connect(board=record["board"])
+        try:
+            task = kanban_db.get_task(connection, record["task_id"])
+            current_run_id = (
+                canonical_run_id(_task_value(task, "current_run_id", None))
+                if task
+                else None
+            )
+            current = str(current_run_id or "")
+            status = str(_task_value(task, "status", "") or "") if task else ""
+        finally:
+            connection.close()
+    if current != record["run_id"] or status != "running":
+        pool.store.finalize(record, "stale")
+
+
+def _adopt_task(
+    pool: Any,
+    kanban_db: Any,
+    connection: Any,
+    board: str,
+    task: Any,
+    ordinals: list[int],
+    known: set[tuple[str, str, str]],
+) -> None:
+    """Adopt one orphaned pool-eligible run onto a free ordinal."""
+    task_id = str(_task_value(task, "id", "") or "")
+    raw_run_id = canonical_run_id(_task_value(task, "current_run_id", None))
+    run_id = str(raw_run_id or "")
+    assignee = str(_task_value(task, "assignee", "") or "")
+    if (
+        not task_id
+        or not run_id
+        or str(_task_value(task, "status", "")) != "running"
+        or not assignee.startswith("cli-")
+        or not distributed_workspace_eligible(task)
+        or (board, task_id, run_id) in known
+    ):
+        return
+    try:
+        payload = pool.assignment_payload(kanban_db, connection, task, board)
+    except RECOVERABLE_BOARD_ERRORS as error:
+        _defer(f"{board}/{task_id} recovery payload", error)
+        return
+    except Exception as error:  # noqa: BLE001 - a real capability failure of this run
+        _block_exact(
+            kanban_db, board, task_id, raw_run_id,
+            "Distributed assignment recovery failed: "
+            f"{type(error).__name__}: {error}",
+        )
+        return
+    binding = {
+        "board": board, "task_id": task_id, "run_id": run_id,
+        "worker_ordinal": ordinals[0], "attempt": 1,
+    }
+    try:
+        pool.store.add(binding, payload)
+    except ProtocolError as error:
+        # A durable row already owns this identity or ordinal. Leave the ordinal
+        # free and leave Kanban untouched; a later pass revisits it.
+        _defer(f"{board}/{task_id} adoption", error)
+        return
+    known.add((board, task_id, run_id))
+    ordinals.pop(0)
+
+
+def _adopt_board(
+    pool: Any,
+    kanban_db: Any,
+    board: str,
+    ordinals: list[int],
+    known: set[tuple[str, str, str]],
+) -> None:
+    """Adopt this board's orphaned runs without touching any other board."""
+    with kanban_db.scoped_current_board(board):
+        connection = kanban_db.connect(board=board)
+        try:
+            for task in kanban_db.list_tasks(connection):
+                if not ordinals:
+                    return
+                _adopt_task(pool, kanban_db, connection, board, task, ordinals, known)
+        finally:
+            connection.close()
+
+
+def reconcile(pool: Any) -> None:
+    """Recover the narrow claim/assignment crash gaps without double execution."""
+    from hermes_cli import kanban_db
+
+    for record in pool.store.active_assignments():
+        try:
+            _release_moved_run(pool, kanban_db, record)
+        except Exception as error:  # noqa: BLE001 - other assignments still reconcile
+            _defer(f"{_context(record)} release", error)
+    ordinals = pool.store.available_ordinals()
+    if not ordinals:
+        return
+    known = pool.store.known_runs()
+    try:
+        boards = kanban_db.list_boards(include_archived=False)
+    except RECOVERABLE_BOARD_ERRORS as error:
+        _defer("board-registry", error)
+        return
+    _settled("board-registry")
+    for raw_board in boards:
+        board = cli_lane_dispatch._board_slug(raw_board)
+        if not board or not ordinals:
+            continue
+        try:
+            _adopt_board(pool, kanban_db, board, ordinals, known)
+            _settled(f"{board} adoption")
+        except Exception as error:  # noqa: BLE001 - later boards must still run
+            _defer(f"{board} adoption", error)
+
+
+def _materialize(
+    pool: Any,
+    kanban_db: Any,
+    board: str,
+    task_id: str,
+    ordinal: int,
+    known: set[tuple[str, str, str]],
+) -> None:
+    """Turn one freshly claimed run into exactly one durable assignment."""
+    database_run_id: int | None = None
+    try:
+        with kanban_db.scoped_current_board(board):
+            connection = kanban_db.connect(board=board)
+            try:
+                task = kanban_db.get_task(connection, task_id)
+                if task is None:
+                    return
+                raw_run = canonical_run_id(
+                    _task_value(task, "current_run_id", None)
+                )
+                if not distributed_workspace_eligible(task):
+                    if raw_run is not None:
+                        kanban_db.block_task(
+                            connection, task_id,
+                            reason=(
+                                "Distributed claim fenced because an existing "
+                                "workspace is owned by the local lane"
+                            ),
+                            kind="capability", expected_run_id=raw_run,
+                        )
+                    return
+                if raw_run is None:
+                    raise ProjectPolicyError("claimed task has no canonical run ID")
+                database_run_id = raw_run
+                payload = pool.assignment_payload(kanban_db, connection, task, board)
+            finally:
+                connection.close()
+        run_id = str(database_run_id)
+        if (board, task_id, run_id) in known:
+            return
+        pool.store.add(
+            {
+                "board": board, "task_id": task_id, "run_id": run_id,
+                "worker_ordinal": ordinal, "attempt": 1,
+            },
+            payload,
+        )
+        known.add((board, task_id, run_id))
+        _settled(f"{board}/{task_id} dispatch")
+    except (ProtocolError, *RECOVERABLE_BOARD_ERRORS) as error:
+        # Coordinator-side storage/identity conditions. Never convert these into
+        # a Kanban capability block; reconcile() adopts the claim on a later pass.
+        _defer(f"{board}/{task_id} dispatch", error)
+    except Exception as error:  # noqa: BLE001 - a real capability failure of this run
+        _block_exact(
+            kanban_db, board, task_id, database_run_id,
+            "Distributed assignment preparation failed: "
+            f"{type(error).__name__}: {error}",
+        )
+
+
+def dispatch(pool: Any) -> None:
+    """Claim up to the free ordinal count and materialize assignments."""
+    from hermes_cli import kanban_db
+
+    ordinals = pool.store.available_ordinals()
+    if not ordinals:
+        return
+    try:
+        # Only storage faults are absorbed here. An incompatible claim API must
+        # still surface, because silently skipping the eligibility predicate is
+        # how the pool would start claiming the local lane's owned workspaces.
+        claimed = cli_lane_dispatch.claim_ready(
+            set(), len(ordinals),
+            lambda _board, task: distributed_workspace_eligible(task),
+        )
+    except RECOVERABLE_BOARD_ERRORS as error:
+        _defer("claim-ready", error)
+        return
+    _settled("claim-ready")
+    known = pool.store.known_runs()
+    for ordinal, (board, task_id) in zip(ordinals, claimed, strict=False):
+        _materialize(pool, kanban_db, board, task_id, ordinal, known)