Merge Hermes stack (PRs #15, routing, #22, #18, #19, #12, #21, in-pod supervisor) — chain tip 9749e650

The composer routing chip showed bare AUTO/FAST/BALANCED/DEEP/MAXIMUM.
Relabel it to Automatic/Fast/Balanced/Deep/Maximum and align the
auto-route model option labels to Automatic · Fast/Balanced/Deep/Maximum
so they match the panels.js profile-default mapping and read as a
recognizable model name. Manual model names (Claude · Opus, etc.) are
unchanged. Update the image-build guard and its quality test to match.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

ci/scripts/semgrep_report.py

@@ -52,7 +52,7 @@ def build_report(
     results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
     errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else []
 
-    severity_counts = {severity: 0 for severity in SEVERITIES}
+    severity_counts = dict.fromkeys(SEVERITIES, 0)
     blocking_findings = 0
     for finding in results:
         severity = _finding_severity(finding)

clusters/atlas/flux-system/applications/hermes/kustomization.yaml

@@ -30,6 +30,12 @@ spec:
       kind: Deployment
       name: hermes-agent
       namespace: hermes
+    # hermes-execution-worker and its mediators are deliberately absent. They run
+    # at scavenger priority on a best-effort pool whose first start installs the
+    # provider CLIs, so gating this Kustomization's 10m health window on them
+    # would stall hermes-chat and hermes-observer-bindings, which dependsOn
+    # hermes. The pool reports its own health through the worker readiness probe,
+    # the per-ordinal mediator /ready endpoint, and the coordinator on :9007.
     - apiVersion: apps/v1
       kind: DaemonSet
       name: hermes-node-ssh-access

dockerfiles/Dockerfile.hermes-agent

@@ -1557,6 +1557,36 @@ function RootRedirect() {
 NODE
 RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 9 ;; *) exit 2 ;; esac
 
+COPY dockerfiles/patch-hermes-execution-safety.py /tmp/patch-hermes-execution-safety.py
+COPY dockerfiles/hermes-execution-safety-regression.py /tmp/hermes-execution-safety-regression.py
+COPY dockerfiles/hermes_execution_patch_support.py /tmp/hermes_execution_patch_support.py
+COPY dockerfiles/patch_hermes_run_safety.py /tmp/patch_hermes_run_safety.py
+COPY dockerfiles/patch_hermes_decomposition_safety.py /tmp/patch_hermes_decomposition_safety.py
+COPY dockerfiles/hermes_execution_regression_support.py /tmp/hermes_execution_regression_support.py
+COPY dockerfiles/hermes_run_safety_regression.py /tmp/hermes_run_safety_regression.py
+COPY dockerfiles/hermes_decomposition_safety_regression.py /tmp/hermes_decomposition_safety_regression.py
+COPY dockerfiles/hermes_lane_compatibility_regression.py /tmp/hermes_lane_compatibility_regression.py
+COPY services/hermes/scripts/cli_lane_*.py /tmp/hermes-lane-regression/
+RUN HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
+        HERMES_COMPATIBILITY_MODE=legacy \
+        /opt/hermes/.venv/bin/python /tmp/hermes_lane_compatibility_regression.py \
+    && /opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py \
+    && HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
+        HERMES_COMPATIBILITY_MODE=patched \
+        /opt/hermes/.venv/bin/python /tmp/hermes_lane_compatibility_regression.py \
+    && HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
+        /opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py \
+    && rm /tmp/patch-hermes-execution-safety.py \
+        /tmp/hermes-execution-safety-regression.py \
+        /tmp/hermes_execution_patch_support.py \
+        /tmp/patch_hermes_run_safety.py \
+        /tmp/patch_hermes_decomposition_safety.py \
+        /tmp/hermes_execution_regression_support.py \
+        /tmp/hermes_run_safety_regression.py \
+        /tmp/hermes_decomposition_safety_regression.py \
+        /tmp/hermes_lane_compatibility_regression.py \
+    && find /tmp/hermes-lane-regression -depth -delete
+
 COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate
 
 RUN rm -f /tmp/hermes-agent.Dockerfile /tmp/hermes-kaniko-heredoc-runner.py
@@ -1617,6 +1647,8 @@ RUN cd /opt/hermes/web \
         /opt/hermes/agent/turn_context.py \
         /opt/hermes/agent/conversation_loop.py \
         /opt/hermes/hermes_cli/oneshot.py \
+        /opt/hermes/hermes_cli/kanban_decompose.py \
+        /opt/hermes/gateway/kanban_watchers.py \
         /opt/hermes/tools/delegate_tool.py \
         /opt/hermes/tools/web_tools.py \
         /opt/hermes/tools/python_sandbox_tool.py \

dockerfiles/Dockerfile.hermes-agent.dockerignore

@@ -7,3 +7,16 @@
 !dockerfiles/hermes-session-migrate.py
 !dockerfiles/Dockerfile.hermes-agent
 !dockerfiles/hermes-kaniko-heredoc-runner.py
+!dockerfiles/patch-hermes-execution-safety.py
+!dockerfiles/hermes-execution-safety-regression.py
+!dockerfiles/hermes_execution_patch_support.py
+!dockerfiles/patch_hermes_run_safety.py
+!dockerfiles/patch_hermes_decomposition_safety.py
+!dockerfiles/hermes_execution_regression_support.py
+!dockerfiles/hermes_run_safety_regression.py
+!dockerfiles/hermes_decomposition_safety_regression.py
+!dockerfiles/hermes_lane_compatibility_regression.py
+!services/
+!services/hermes/
+!services/hermes/scripts/
+!services/hermes/scripts/cli_lane_*.py

dockerfiles/Dockerfile.hermes-webui

@@ -111,7 +111,7 @@ RUN /opt/hermes/.venv/bin/python -c 'import cryptography, yaml' \
     && grep -Fq "capability.provider!=='local_command'" /opt/hermes-webui/static/atlas-voice.js \
     && grep -Fq 'data-priority="maximum"' /opt/hermes-webui/static/index.html \
     && grep -Fq 'routing_priority:priority' /opt/hermes-webui/static/atlas-router.js \
-    && grep -Fq "'atlas/auto/fast':'AUTO · Fast'" /opt/hermes-webui/static/atlas-router.js \
+    && grep -Fq "'atlas/auto/fast':'Automatic · Fast'" /opt/hermes-webui/static/atlas-router.js \
     && grep -Fq 'explicit_reasoning_effort' /opt/hermes-webui/api/gateway_chat.py \
     && /opt/hermes/.venv/bin/python -m py_compile \
         /opt/hermes-webui/api/routes.py \

dockerfiles/hermes-execution-safety-regression.py

@@ -0,0 +1,23 @@
+"""Run the composable Hermes execution-safety build regressions."""
+
+import unittest
+
+from hermes_decomposition_safety_regression import DecompositionSafetyTests
+from hermes_run_safety_regression import ExactRunSafetyTests
+
+
+def main() -> int:
+    """Run both transaction domains and return a build-friendly status."""
+    loader = unittest.defaultTestLoader
+    suite = unittest.TestSuite(
+        (
+            loader.loadTestsFromTestCase(ExactRunSafetyTests),
+            loader.loadTestsFromTestCase(DecompositionSafetyTests),
+        )
+    )
+    result = unittest.TextTestRunner(verbosity=2).run(suite)
+    return 0 if result.wasSuccessful() else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dockerfiles/hermes-webui-router-patch.py

@@ -23,7 +23,7 @@ replace_once(
     '''            <div class="composer-ws-wrap">\n''',
     '''            <div class="composer-routing-wrap" id="composerRoutingWrap">
               <button class="composer-routing-chip" id="composerRoutingChip" type="button" title="Automatic routing priority" aria-haspopup="true" aria-expanded="false" aria-controls="composerRoutingDropdown">
-                <span class="composer-routing-label" id="composerRoutingLabel">AUTO</span>
+                <span class="composer-routing-label" id="composerRoutingLabel">Automatic</span>
                 <span class="composer-routing-chevron" aria-hidden="true">⌄</span>
               </button>
             </div>

dockerfiles/hermes-webui-router.js

@@ -2,14 +2,14 @@
   'use strict';
 
   const PRIORITIES=new Set(['auto','fast','balanced','deep','maximum']);
-  const LABELS={auto:'AUTO',fast:'FAST',balanced:'BALANCED',deep:'DEEP',maximum:'MAXIMUM'};
+  const LABELS={auto:'Automatic',fast:'Fast',balanced:'Balanced',deep:'Deep',maximum:'Maximum'};
   const STORAGE_KEY='atlas.hermes.routing-priority';
   const EXPLICIT_EFFORT_KEY='atlas.hermes.explicit-reasoning';
   const ROUTE_LABELS={
-    'atlas/auto/fast':'AUTO · Fast',
-    'atlas/auto/balanced':'AUTO · Balanced',
-    'atlas/auto/deep':'AUTO · Deep',
-    'atlas/auto/maximum':'AUTO · Maximum',
+    'atlas/auto/fast':'Automatic · Fast',
+    'atlas/auto/balanced':'Automatic · Balanced',
+    'atlas/auto/deep':'Automatic · Deep',
+    'atlas/auto/maximum':'Automatic · Maximum',
     'atlas/manual/codex/luna':'Codex · Luna',
     'atlas/manual/codex/terra':'Codex · Terra',
     'atlas/manual/codex/sol':'Codex · SOL',

dockerfiles/hermes_decomposition_safety_regression.py

@@ -0,0 +1,120 @@
+"""Regress automatic decomposition against existing execution history."""
+
+from __future__ import annotations
+
+
+from hermes_cli import kanban_db
+from hermes_cli import kanban_decompose
+from hermes_execution_regression_support import ExecutionSafetyTestCase
+
+
+class DecompositionSafetyTests(ExecutionSafetyTestCase):
+    """Verify only unexecuted triage roots may decompose automatically."""
+
+    def test_automatic_decomposer_skips_executed_triage_task(self) -> None:
+        task_id = self._route_executed_task_to_triage()
+        completions = self._client(
+            {"fanout": False, "title": "must not run", "body": "must not run"}
+        )
+        before = len(kanban_db.list_tasks(self.connection))
+
+        outcome = kanban_decompose.decompose_task(task_id, automatic=True)
+
+        self.assertFalse(outcome.ok)
+        self.assertIn("execution history", outcome.reason)
+        self.assertEqual(completions.calls, 0)
+        self.assertEqual(self._status(task_id), "triage")
+        self.assertEqual(len(kanban_db.list_tasks(self.connection)), before)
+
+    def test_manual_decomposition_remains_available_after_execution(self) -> None:
+        task_id = self._route_executed_task_to_triage()
+        completions = self._client(
+            {
+                "fanout": False,
+                "title": "operator-approved retry",
+                "body": "Retain one bounded task.",
+                "assignee": "default",
+            }
+        )
+
+        outcome = kanban_decompose.decompose_task(task_id, author="operator")
+
+        self.assertTrue(outcome.ok)
+        self.assertFalse(outcome.fanout)
+        self.assertEqual(completions.calls, 1)
+        self.assertEqual(self._status(task_id), "ready")
+
+    def test_fresh_triage_task_still_auto_promotes(self) -> None:
+        task_id = kanban_db.create_task(
+            self.connection,
+            title="fresh objective",
+            triage=True,
+        )
+        completions = self._client(
+            {
+                "fanout": False,
+                "title": "specified objective",
+                "body": "One bounded task.",
+                "assignee": "default",
+            }
+        )
+
+        outcome = kanban_decompose.decompose_task(task_id, automatic=True)
+
+        self.assertTrue(outcome.ok)
+        self.assertEqual(completions.calls, 1)
+        self.assertEqual(self._status(task_id), "ready")
+
+    def test_execution_history_gained_during_single_llm_call_blocks_commit(self) -> None:
+        task_id = kanban_db.create_task(
+            self.connection,
+            title="concurrent single objective",
+            triage=True,
+        )
+        completions = self._client(
+            {
+                "fanout": False,
+                "title": "redundant rewrite",
+                "body": "must not be committed",
+                "assignee": "default",
+            },
+            before_response=lambda: self._insert_run(task_id),
+        )
+
+        outcome = kanban_decompose.decompose_task(task_id, automatic=True)
+
+        self.assertFalse(outcome.ok)
+        self.assertIn("gained execution history", outcome.reason)
+        self.assertEqual(completions.calls, 1)
+        self.assertEqual(self._status(task_id), "triage")
+
+    def test_execution_history_gained_during_fanout_llm_call_blocks_commit(self) -> None:
+        task_id = kanban_db.create_task(
+            self.connection,
+            title="concurrent triage objective",
+            triage=True,
+        )
+
+        completions = self._client(
+            {
+                "fanout": True,
+                "tasks": [
+                    {
+                        "title": "redundant child",
+                        "body": "must not be created",
+                        "assignee": "default",
+                        "parents": [],
+                    }
+                ],
+            },
+            before_response=lambda: self._insert_run(task_id),
+        )
+        before = len(kanban_db.list_tasks(self.connection))
+
+        outcome = kanban_decompose.decompose_task(task_id, automatic=True)
+
+        self.assertFalse(outcome.ok)
+        self.assertIn("gained execution history", outcome.reason)
+        self.assertEqual(completions.calls, 1)
+        self.assertEqual(self._status(task_id), "triage")
+        self.assertEqual(len(kanban_db.list_tasks(self.connection)), before)

dockerfiles/hermes_execution_patch_support.py

@@ -0,0 +1,12 @@
+"""Fail-closed helpers shared by Hermes source patches."""
+
+
+
+def replace_once(source: str, before: str, after: str, label: str) -> str:
+    """Replace one anchored upstream fragment and fail closed on source drift."""
+    count = source.count(before)
+    if count != 1:
+        raise SystemExit(
+            f"Hermes {label} patch context changed: expected 1, found {count}"
+        )
+    return source.replace(before, after, 1)

dockerfiles/hermes_execution_regression_support.py

@@ -0,0 +1,132 @@
+"""Shared isolated database fixture for Hermes execution-safety regressions."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest import mock
+
+_HERMES_HOME = tempfile.TemporaryDirectory(prefix="hermes-execution-test-home-")
+os.environ["HERMES_HOME"] = _HERMES_HOME.name
+_LANE_SOURCE = Path(
+    os.environ.get("HERMES_CLI_LANE_SOURCE", "/tmp/hermes-lane-regression")
+)
+sys.path.insert(0, str(_LANE_SOURCE))
+
+from agent import auxiliary_client  # noqa: E402
+from cli_lane_config import TerminalIdentity  # noqa: E402
+from cli_lane_finalization import _finalize_document_db  # noqa: E402
+from hermes_cli import kanban_db  # noqa: E402
+from hermes_cli import kanban_decompose  # noqa: E402
+
+__all__ = ("ExecutionSafetyTestCase", "TerminalIdentity", "_finalize_document_db")
+
+
+class _FakeCompletions:
+    def __init__(self, payload: dict, before_response=None) -> None:
+        self.payload = payload
+        self.before_response = before_response
+        self.calls = 0
+
+    def create(self, **_kwargs):
+        self.calls += 1
+        if self.before_response is not None:
+            self.before_response()
+        return SimpleNamespace(
+            choices=[
+                SimpleNamespace(
+                    message=SimpleNamespace(content=json.dumps(self.payload))
+                )
+            ]
+        )
+
+
+class _FakeClient:
+    def __init__(self, completions: _FakeCompletions) -> None:
+        self.chat = SimpleNamespace(completions=completions)
+
+
+class ExecutionSafetyTestCase(unittest.TestCase):
+    """Exercise patched upstream APIs against an isolated real database."""
+
+    def setUp(self) -> None:
+        self.connection = kanban_db.connect()
+        self.patches = [
+            mock.patch.object(
+                kanban_decompose,
+                "_build_roster",
+                return_value=([], {"default"}),
+            ),
+            mock.patch.object(
+                kanban_decompose,
+                "_load_config",
+                return_value={"kanban": {"auto_promote_children": True}},
+            ),
+            mock.patch.object(
+                auxiliary_client,
+                "get_auxiliary_extra_body",
+                return_value={},
+            ),
+        ]
+        for patch in self.patches:
+            patch.start()
+
+    def tearDown(self) -> None:
+        for patch in reversed(self.patches):
+            patch.stop()
+        self.connection.close()
+
+    def _client(self, payload: dict, before_response=None) -> _FakeCompletions:
+        completions = _FakeCompletions(payload, before_response)
+        client = _FakeClient(completions)
+        patch = mock.patch.object(
+            auxiliary_client,
+            "get_text_auxiliary_client",
+            return_value=(client, "test-decomposer"),
+        )
+        patch.start()
+        self.addCleanup(patch.stop)
+        return completions
+
+    def _status(self, task_id: str) -> str:
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertIsNotNone(task)
+        return task.status
+
+    def _route_executed_task_to_triage(self) -> str:
+        task_id = kanban_db.create_task(
+            self.connection,
+            title="already implemented parent",
+        )
+        for attempt in range(kanban_db.BLOCK_RECURRENCE_LIMIT):
+            if attempt:
+                self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+            self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+            self.assertTrue(
+                kanban_db.block_task(
+                    self.connection,
+                    task_id,
+                    reason="same unavailable capability",
+                    kind="capability",
+                )
+            )
+        self.assertEqual(self._status(task_id), "triage")
+        self.assertEqual(
+            len(kanban_db.list_runs(self.connection, task_id)),
+            kanban_db.BLOCK_RECURRENCE_LIMIT,
+        )
+        return task_id
+
+    def _insert_run(self, task_id: str) -> None:
+        with kanban_db.connect_closing() as connection:
+            kanban_db._synthesize_ended_run(
+                connection,
+                task_id,
+                outcome="reclaimed",
+                summary="concurrent execution evidence",
+            )

dockerfiles/hermes_lane_compatibility_regression.py

@@ -0,0 +1,120 @@
+"""Exercise mounted CLI scripts against the exact image Kanban API."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+
+_HOME = tempfile.TemporaryDirectory(prefix="hermes-lane-compatibility-")
+os.environ["HERMES_HOME"] = _HOME.name
+os.environ["HERMES_CLI_LANE_CONCURRENCY"] = "1"
+lane_source = Path(os.environ["HERMES_CLI_LANE_SOURCE"])
+sys.path.insert(0, str(lane_source))
+
+from cli_lane_capabilities import (  # noqa: E402
+    detect_kanban_capabilities,
+    initialize_kanban_capabilities,
+    runtime_health,
+)
+from cli_lane_config import TerminalIdentity  # noqa: E402
+from cli_lane_dispatch import recover_orphans  # noqa: E402
+from cli_lane_finalization import _finalize_document_db  # noqa: E402
+from hermes_cli import kanban_db  # noqa: E402
+
+
+class MountedLaneCompatibilityTests(unittest.TestCase):
+    """Verify safe behavior before and after the image API patch."""
+
+    def setUp(self) -> None:
+        self.connection = kanban_db.connect()
+
+    def tearDown(self) -> None:
+        self.connection.close()
+
+    @staticmethod
+    def _document(summary: str) -> dict:
+        return {
+            "result": json.dumps(
+                {
+                    "status": "completed",
+                    "summary": summary,
+                    "changed_files": [],
+                    "tests_run": ["mixed source compatibility"],
+                    "artifacts": [],
+                    "findings": [],
+                    "blockers": [],
+                },
+                sort_keys=True,
+            ),
+            "summary": summary,
+            "metadata": {"matrix": os.environ["HERMES_COMPATIBILITY_MODE"]},
+        }
+
+    def test_exact_image_api_matrix(self) -> None:
+        mode = os.environ["HERMES_COMPATIBILITY_MODE"]
+        capabilities = initialize_kanban_capabilities(kanban_db)
+        self.assertTrue(capabilities.exact_run_completion)
+        self.assertEqual(capabilities.ready, mode == "patched")
+        self.assertEqual(runtime_health()["state"], "ready" if mode == "patched" else "deferred")
+        self.assertLess(Path(_HOME.name, "cli-lanes/runtime-health.json").stat().st_size, 2048)
+
+    def test_current_active_run_completion_stays_safe(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="active accepted result")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        run_id = kanban_db.get_task(self.connection, task_id).current_run_id
+        outcome = _finalize_document_db(
+            kanban_db,
+            TerminalIdentity("default", task_id, run_id, "pending"),
+            self._document("active result"),
+        )
+        self.assertEqual(outcome, "committed")
+        self.assertEqual(kanban_db.get_task(self.connection, task_id).status, "done")
+
+    def test_ended_run_replay_is_enabled_only_after_patch(self) -> None:
+        mode = os.environ["HERMES_COMPATIBILITY_MODE"]
+        task_id = kanban_db.create_task(self.connection, title="ended accepted result")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        run_id = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="simulate post-journal error",
+                expected_run_id=run_id,
+            )
+        )
+        outcome = _finalize_document_db(
+            kanban_db,
+            TerminalIdentity("default", task_id, run_id, "pending"),
+            self._document("ended result"),
+        )
+        self.assertEqual(outcome, "committed" if mode == "patched" else "deferred")
+
+    def test_old_reclaim_api_is_never_called_unguarded(self) -> None:
+        capabilities = detect_kanban_capabilities(kanban_db)
+        if capabilities.exact_run_reclaim:
+            self.skipTest("patched API exercises guarded reclaim in the exact-run suite")
+        with mock.patch.object(
+            kanban_db,
+            "reclaim_task",
+            side_effect=AssertionError("legacy unguarded reclaim called"),
+        ), mock.patch("cli_lane_dispatch.recover_pending_finalizations", return_value=0):
+            recover_orphans()
+
+
+def main() -> int:
+    """Run the compatibility matrix for the selected exact image source."""
+    result = unittest.TextTestRunner(verbosity=2).run(
+        unittest.defaultTestLoader.loadTestsFromTestCase(MountedLaneCompatibilityTests)
+    )
+    return 0 if result.wasSuccessful() else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dockerfiles/hermes_run_safety_regression.py

@@ -0,0 +1,467 @@
+"""Regress exact-run replay, provenance, and reclaim safety."""
+
+from __future__ import annotations
+
+import threading
+from unittest import mock
+
+from hermes_cli import kanban_db
+from hermes_execution_regression_support import (
+    ExecutionSafetyTestCase,
+    TerminalIdentity,
+    _finalize_document_db,
+)
+from cli_lane_files import state_path
+from cli_lane_records import _write_terminal_record
+from cli_lane_recovery import recover_pending_finalizations
+
+
+class ExactRunSafetyTests(ExecutionSafetyTestCase):
+    """Verify exact-run transactions reject every stale concurrent actor."""
+
+    def _lane_document(self, summary: str) -> dict:
+        """Return the DB-bearing portion of one accepted lane journal."""
+        return {
+            "result": summary,
+            "summary": summary,
+            "metadata": {"source": "exact-run-regression"},
+        }
+
+    def test_lane_marks_replaced_ended_run_stale_without_mutation(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="replaced journal")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        first_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="first run ended after persistence",
+                expected_run_id=first_run,
+            )
+        )
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        second_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="replacement run also ended",
+                expected_run_id=second_run,
+            )
+        )
+        before = kanban_db.get_task(self.connection, task_id)
+
+        outcome = _finalize_document_db(
+            kanban_db,
+            TerminalIdentity("default", task_id, first_run, "pending"),
+            self._lane_document("stale first result"),
+        )
+
+        after = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(outcome, "stale")
+        self.assertEqual(after.status, before.status)
+        self.assertEqual(after.current_run_id, before.current_run_id)
+        self.assertEqual(after.result, before.result)
+        self.assertEqual(after.completed_run_id, before.completed_run_id)
+
+    def test_lane_marks_nonexistent_ended_run_stale_without_mutation(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="no such run")
+        before = kanban_db.get_task(self.connection, task_id)
+
+        outcome = _finalize_document_db(
+            kanban_db,
+            TerminalIdentity("default", task_id, 42, "pending"),
+            self._lane_document("orphan result"),
+        )
+
+        after = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(outcome, "stale")
+        self.assertEqual(after.status, before.status)
+        self.assertEqual(after.result, before.result)
+        self.assertEqual(kanban_db.list_runs(self.connection, task_id), [])
+
+    def test_full_lane_recovery_conflicts_a_replaced_ended_run(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="recovery conflict")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        first_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="first accepted run ended",
+                expected_run_id=first_run,
+            )
+        )
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        second_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="replacement ended",
+                expected_run_id=second_run,
+            )
+        )
+        before = kanban_db.get_task(self.connection, task_id)
+        state_file = state_path("default", task_id)
+        pending, _record = _write_terminal_record(
+            state_file,
+            board="default",
+            task_id=task_id,
+            run_id=first_run,
+            structured={
+                "status": "completed",
+                "summary": "accepted stale result",
+                "changed_files": [],
+                "tests_run": ["real SQLite recovery regression"],
+                "artifacts": [],
+                "findings": [],
+                "blockers": [],
+            },
+            summary="accepted stale result",
+            metadata={"source": "exact-run-regression"},
+        )
+
+        self.assertEqual(recover_pending_finalizations(), 0)
+
+        after = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(after.status, before.status)
+        self.assertEqual(after.current_run_id, before.current_run_id)
+        self.assertEqual(after.result, before.result)
+        self.assertFalse(pending.exists())
+        conflicts = list(
+            state_file.parent.glob(
+                f"{state_file.stem}.run-{first_run}.terminal.conflict-*.json"
+            )
+        )
+        self.assertEqual(len(conflicts), 1)
+
+    def test_latest_ended_run_can_replay_a_durable_completion(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="journaled result")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        run_id = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="legacy post-journal failure",
+                kind="capability",
+                expected_run_id=run_id,
+            )
+        )
+
+        with mock.patch.object(
+            kanban_db, "_fire_kanban_lifecycle_hook"
+        ) as lifecycle:
+            self.assertTrue(
+                kanban_db.complete_task(
+                    self.connection,
+                    task_id,
+                    result="durable terminal result",
+                    summary="durable terminal result",
+                    replay_ended_run_id=run_id,
+                )
+            )
+        lifecycle.assert_called_once()
+        self.assertEqual(lifecycle.call_args.args[:2], ("kanban_task_completed", task_id))
+        self.assertEqual(lifecycle.call_args.kwargs["run_id"], run_id)
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(task.status, "done")
+        self.assertEqual(task.completed_run_id, run_id)
+        runs = kanban_db.list_runs(self.connection, task_id)
+        self.assertEqual(len(runs), 1)
+        self.assertEqual(runs[0].id, run_id)
+        self.assertEqual(runs[0].status, "done")
+        self.assertEqual(runs[0].outcome, "completed")
+        self.assertEqual(runs[0].summary, "durable terminal result")
+        completed_events = self.connection.execute(
+            "SELECT run_id FROM task_events "
+            "WHERE task_id = ? AND kind = 'completed' ORDER BY id",
+            (task_id,),
+        ).fetchall()
+        self.assertEqual([row["run_id"] for row in completed_events], [run_id])
+
+    def test_unchanged_triage_can_replay_its_exact_latest_ended_run(self) -> None:
+        task_id = self._route_executed_task_to_triage()
+        runs = kanban_db.list_runs(self.connection, task_id)
+        run_id = runs[-1].id
+
+        self.assertTrue(
+            kanban_db.complete_task(
+                self.connection,
+                task_id,
+                result="accepted before the loop breaker routed triage",
+                summary="accepted before the loop breaker routed triage",
+                replay_ended_run_id=run_id,
+            )
+        )
+
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(task.status, "done")
+        self.assertEqual(task.completed_run_id, run_id)
+        updated_runs = kanban_db.list_runs(self.connection, task_id)
+        self.assertEqual(len(updated_runs), kanban_db.BLOCK_RECURRENCE_LIMIT)
+        self.assertEqual(updated_runs[-1].id, run_id)
+        self.assertEqual(updated_runs[-1].outcome, "completed")
+
+    def test_triage_status_transition_prevents_stale_exact_run_replay(self) -> None:
+        task_id = self._route_executed_task_to_triage()
+        run_id = kanban_db.list_runs(self.connection, task_id)[-1].id
+        self.assertTrue(
+            kanban_db.specify_triage_task(
+                self.connection,
+                task_id,
+                title="operator changed the objective",
+                author="operator",
+            )
+        )
+
+        self.assertFalse(
+            kanban_db.complete_task(
+                self.connection,
+                task_id,
+                result="stale accepted result",
+                replay_ended_run_id=run_id,
+            )
+        )
+        self.assertNotEqual(self._status(task_id), "done")
+
+    def test_decomposition_lineage_prevents_replay_even_if_status_returns(self) -> None:
+        task_id = self._route_executed_task_to_triage()
+        run_id = kanban_db.list_runs(self.connection, task_id)[-1].id
+        children = kanban_db.decompose_triage_task(
+            self.connection,
+            task_id,
+            root_assignee="default",
+            children=[{"title": "new child objective"}],
+            author="operator",
+            auto_promote=False,
+        )
+        self.assertEqual(len(children or []), 1)
+        with kanban_db.write_txn(self.connection):
+            self.connection.execute(
+                "UPDATE tasks SET status = 'ready' WHERE id = ?",
+                (task_id,),
+            )
+
+        self.assertFalse(
+            kanban_db.complete_task(
+                self.connection,
+                task_id,
+                result="pre-decomposition accepted result",
+                replay_ended_run_id=run_id,
+            )
+        )
+        self.assertEqual(self._status(task_id), "ready")
+
+    def test_older_ended_run_cannot_complete_over_a_replacement(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="replacement guard")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        old_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="first run",
+                kind="capability",
+                expected_run_id=old_run,
+            )
+        )
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        replacement_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertNotEqual(old_run, replacement_run)
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="replacement run",
+                kind="capability",
+                expected_run_id=replacement_run,
+            )
+        )
+
+        self.assertFalse(
+            kanban_db.complete_task(
+                self.connection,
+                task_id,
+                result="stale result",
+                replay_ended_run_id=old_run,
+            )
+        )
+        self.assertIn(self._status(task_id), {"blocked", "triage"})
+
+    def test_equal_result_bytes_remain_bound_to_the_completing_run(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="same result retries")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        first_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="first attempt ended",
+                expected_run_id=first_run,
+            )
+        )
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        second_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.complete_task(
+                self.connection,
+                task_id,
+                result="identical result bytes",
+                summary="second run wins",
+                expected_run_id=second_run,
+            )
+        )
+
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(task.completed_run_id, second_run)
+        self.assertNotEqual(task.completed_run_id, first_run)
+        runs = {run.id: run for run in kanban_db.list_runs(self.connection, task_id)}
+        self.assertEqual(runs[first_run].outcome, "blocked")
+        self.assertEqual(runs[second_run].outcome, "completed")
+
+    def test_exact_run_reclaim_succeeds_for_the_authoritative_run(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="recover exact run")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        run_id = kanban_db.get_task(self.connection, task_id).current_run_id
+
+        self.assertTrue(
+            kanban_db.reclaim_task(
+                self.connection,
+                task_id,
+                reason="invalid exact journal",
+                expected_run_id=run_id,
+            )
+        )
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(task.status, "ready")
+        self.assertIsNone(task.current_run_id)
+
+    def test_stale_reclaim_before_transaction_preserves_replacement_run(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="replacement before txn")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        old_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="old recovery run",
+                expected_run_id=old_run,
+            )
+        )
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        replacement = kanban_db.get_task(self.connection, task_id)
+        replacement_run = replacement.current_run_id
+        self.assertNotEqual(old_run, replacement_run)
+        signals = []
+
+        self.assertFalse(
+            kanban_db.reclaim_task(
+                self.connection,
+                task_id,
+                reason="stale journal",
+                expected_run_id=old_run,
+                signal_fn=lambda *args: signals.append(args),
+            )
+        )
+        latest = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(latest.status, "running")
+        self.assertEqual(latest.current_run_id, replacement_run)
+        self.assertEqual(latest.claim_lock, replacement.claim_lock)
+        self.assertEqual(signals, [])
+
+    def test_reclaim_update_guard_preserves_run_changed_inside_transaction(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="replacement in txn")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        old_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        replacement = {}
+
+        def install_replacement(*_args, **_kwargs):
+            cursor = self.connection.execute(
+                "INSERT INTO task_runs (task_id, status, claim_lock, started_at) "
+                "VALUES (?, 'running', ?, strftime('%s','now'))",
+                (task_id, "replacement-lock"),
+            )
+            replacement["run_id"] = int(cursor.lastrowid)
+            self.connection.execute(
+                "UPDATE tasks SET current_run_id = ?, claim_lock = ? WHERE id = ?",
+                (replacement["run_id"], "replacement-lock", task_id),
+            )
+            return {}
+
+        with mock.patch.object(
+            kanban_db,
+            "_terminate_reclaimed_worker",
+            side_effect=install_replacement,
+        ):
+            self.assertFalse(
+                kanban_db.reclaim_task(
+                    self.connection,
+                    task_id,
+                    reason="journal for old run",
+                    expected_run_id=old_run,
+                )
+            )
+
+        latest = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(latest.status, "running")
+        self.assertEqual(latest.current_run_id, replacement["run_id"])
+        self.assertEqual(latest.claim_lock, "replacement-lock")
+
+    def test_concurrent_exact_reclaim_and_completion_have_one_winner(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="concurrent finalizer")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        run_id = kanban_db.get_task(self.connection, task_id).current_run_id
+        barrier = threading.Barrier(2)
+        outcomes = {}
+        errors = []
+
+        def reclaim() -> None:
+            try:
+                with kanban_db.connect_closing() as connection:
+                    barrier.wait()
+                    outcomes["reclaim"] = kanban_db.reclaim_task(
+                        connection,
+                        task_id,
+                        reason="concurrent invalid journal",
+                        expected_run_id=run_id,
+                    )
+            except BaseException as error:  # pragma: no cover - assertion relay
+                errors.append(error)
+
+        def finalize() -> None:
+            try:
+                with kanban_db.connect_closing() as connection:
+                    barrier.wait()
+                    outcomes["complete"] = kanban_db.complete_task(
+                        connection,
+                        task_id,
+                        result="durable winner",
+                        summary="durable winner",
+                        expected_run_id=run_id,
+                    )
+            except BaseException as error:  # pragma: no cover - assertion relay
+                errors.append(error)
+
+        threads = [threading.Thread(target=reclaim), threading.Thread(target=finalize)]
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join(timeout=10)
+            self.assertFalse(thread.is_alive())
+
+        self.assertEqual(errors, [])
+        self.assertEqual(set(outcomes), {"reclaim", "complete"})
+        self.assertEqual(sum(bool(value) for value in outcomes.values()), 1)
+        latest = kanban_db.get_task(self.connection, task_id)
+        self.assertIn(latest.status, {"done", "ready"})
+        self.assertIsNone(latest.current_run_id)
+        runs = kanban_db.list_runs(self.connection, task_id)
+        self.assertEqual(len(runs), 1)
+        self.assertIsNotNone(runs[0].ended_at)

dockerfiles/patch-hermes-execution-safety.py

@@ -0,0 +1,6 @@
+"""Apply the composable Hermes execution-safety source patches."""
+
+# Import order is intentional: the decomposition DB contract builds on the
+# exact-run schema and transaction changes. Each module fails closed on drift.
+import patch_hermes_run_safety  # noqa: F401
+import patch_hermes_decomposition_safety  # noqa: F401

dockerfiles/patch_hermes_decomposition_safety.py

@@ -0,0 +1,262 @@
+"""Patch transactional automatic-decomposition execution-history guards."""
+
+import os
+from pathlib import Path
+
+from hermes_execution_patch_support import replace_once
+
+source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes"))
+db_path = source_root / "hermes_cli/kanban_db.py"
+db = db_path.read_text(encoding="utf-8")
+
+specify_signature_before = '''def specify_triage_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    title: Optional[str] = None,
+    body: Optional[str] = None,
+    assignee: Optional[str] = None,
+    author: Optional[str] = None,
+) -> bool:
+'''
+specify_signature_after = '''def specify_triage_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    title: Optional[str] = None,
+    body: Optional[str] = None,
+    assignee: Optional[str] = None,
+    author: Optional[str] = None,
+    require_no_runs: bool = False,
+) -> bool:
+'''
+db = replace_once(
+    db,
+    specify_signature_before,
+    specify_signature_after,
+    "transactional specify signature",
+)
+
+specify_guard_before = '''        if existing is None:
+            return False
+        sets: list[str] = ["status = 'todo'"]
+'''
+specify_guard_after = '''        if existing is None:
+            return False
+        if require_no_runs and conn.execute(
+            "SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1",
+            (task_id,),
+        ).fetchone() is not None:
+            return False
+        sets: list[str] = ["status = 'todo'"]
+'''
+db = replace_once(
+    db,
+    specify_guard_before,
+    specify_guard_after,
+    "transactional specify execution-history guard",
+)
+
+decompose_signature_before = '''def decompose_triage_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    root_assignee: Optional[str],
+    children: list[dict],
+    author: Optional[str] = None,
+    auto_promote: bool = True,
+) -> Optional[list[str]]:
+'''
+decompose_signature_after = '''def decompose_triage_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    root_assignee: Optional[str],
+    children: list[dict],
+    author: Optional[str] = None,
+    auto_promote: bool = True,
+    require_no_runs: bool = False,
+) -> Optional[list[str]]:
+'''
+db = replace_once(
+    db,
+    decompose_signature_before,
+    decompose_signature_after,
+    "transactional decomposition signature",
+)
+
+decompose_guard_before = '''        if root_row["status"] != "triage":
+            return None
+        tenant = root_row["tenant"]
+'''
+decompose_guard_after = '''        if root_row["status"] != "triage":
+            return None
+        if require_no_runs and conn.execute(
+            "SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1",
+            (task_id,),
+        ).fetchone() is not None:
+            return None
+        tenant = root_row["tenant"]
+'''
+db = replace_once(
+    db,
+    decompose_guard_before,
+    decompose_guard_after,
+    "transactional decomposition execution-history guard",
+)
+db_path.write_text(db, encoding="utf-8")
+
+decompose_path = source_root / "hermes_cli/kanban_decompose.py"
+decompose = decompose_path.read_text(encoding="utf-8")
+
+helper_anchor = '''    if chosen not in valid_names:
+        return default_assignee
+    return chosen
+
+
+'''
+helper_replacement = helper_anchor + '''def _has_execution_history(task_id: str) -> bool:
+    """Return whether a task has ever entered a worker run."""
+    with kb.connect_closing() as conn:
+        return bool(kb.list_runs(conn, task_id))
+
+
+'''
+decompose = replace_once(
+    decompose,
+    helper_anchor,
+    helper_replacement,
+    "auto-decompose execution-history helper",
+)
+
+signature_before = '''def decompose_task(
+    task_id: str,
+    *,
+    author: Optional[str] = None,
+    timeout: Optional[int] = None,
+) -> DecomposeOutcome:
+'''
+signature_after = '''def decompose_task(
+    task_id: str,
+    *,
+    author: Optional[str] = None,
+    timeout: Optional[int] = None,
+    automatic: bool = False,
+) -> DecomposeOutcome:
+'''
+decompose = replace_once(
+    decompose,
+    signature_before,
+    signature_after,
+    "auto-decompose function signature",
+)
+
+status_before = '''    if task.status != "triage":
+        return DecomposeOutcome(
+            task_id, False, f"task is not in triage (status={task.status!r})"
+        )
+
+    cfg = _load_config()
+'''
+status_after = '''    if task.status != "triage":
+        return DecomposeOutcome(
+            task_id, False, f"task is not in triage (status={task.status!r})"
+        )
+    if automatic and _has_execution_history(task_id):
+        return DecomposeOutcome(
+            task_id,
+            False,
+            "task has execution history and requires deliberate manual triage",
+        )
+
+    cfg = _load_config()
+'''
+decompose = replace_once(
+    decompose,
+    status_before,
+    status_after,
+    "auto-decompose preflight guard",
+)
+
+single_before = '''                author=audit_author,
+            )
+        if not ok:
+'''
+single_after = '''                author=audit_author,
+                require_no_runs=automatic,
+            )
+        if not ok:
+            if automatic and _has_execution_history(task_id):
+                return DecomposeOutcome(
+                    task_id,
+                    False,
+                    "task gained execution history and requires deliberate manual triage",
+                )
+'''
+decompose = replace_once(
+    decompose,
+    single_before,
+    single_after,
+    "auto-decompose single-task commit guard",
+)
+
+fanout_before = '''                author=audit_author,
+                auto_promote=auto_promote,
+            )
+'''
+fanout_after = '''                author=audit_author,
+                auto_promote=auto_promote,
+                require_no_runs=automatic,
+            )
+'''
+decompose = replace_once(
+    decompose,
+    fanout_before,
+    fanout_after,
+    "auto-decompose transactional fanout guard",
+)
+
+fanout_outcome_before = '''    if child_ids is None:
+        return DecomposeOutcome(
+            task_id, False, "task moved out of triage before decomposition",
+        )
+'''
+fanout_outcome_after = '''    if child_ids is None:
+        if automatic and _has_execution_history(task_id):
+            return DecomposeOutcome(
+                task_id,
+                False,
+                "task gained execution history and requires deliberate manual triage",
+            )
+        return DecomposeOutcome(
+            task_id, False, "task moved out of triage before decomposition",
+        )
+'''
+decompose = replace_once(
+    decompose,
+    fanout_outcome_before,
+    fanout_outcome_after,
+    "auto-decompose fanout rejection reason",
+)
+decompose_path.write_text(decompose, encoding="utf-8")
+
+
+watcher_path = source_root / "gateway/kanban_watchers.py"
+watcher = watcher_path.read_text(encoding="utf-8")
+call_before = '''                            outcome = _decomp.decompose_task(
+                                tid, author="auto-decomposer",
+                            )
+'''
+call_after = '''                            outcome = _decomp.decompose_task(
+                                tid,
+                                author="auto-decomposer",
+                                automatic=True,
+                            )
+'''
+watcher = replace_once(
+    watcher,
+    call_before,
+    call_after,
+    "gateway automatic-decomposition call",
+)
+watcher_path.write_text(watcher, encoding="utf-8")

dockerfiles/patch_hermes_run_safety.py

@@ -0,0 +1,480 @@
+"""Patch exact-run completion, provenance, and reclaim transactions."""
+
+import os
+from pathlib import Path
+
+from hermes_execution_patch_support import replace_once
+
+
+
+source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes"))
+db_path = source_root / "hermes_cli/kanban_db.py"
+db = db_path.read_text(encoding="utf-8")
+
+complete_signature_before = '''def complete_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    result: Optional[str] = None,
+    summary: Optional[str] = None,
+    metadata: Optional[dict] = None,
+    created_cards: Optional[Iterable[str]] = None,
+    expected_run_id: Optional[int] = None,
+) -> bool:
+'''
+complete_signature_after = '''def complete_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    result: Optional[str] = None,
+    summary: Optional[str] = None,
+    metadata: Optional[dict] = None,
+    created_cards: Optional[Iterable[str]] = None,
+    expected_run_id: Optional[int] = None,
+    replay_ended_run_id: Optional[int] = None,
+) -> bool:
+'''
+db = replace_once(
+    db,
+    complete_signature_before,
+    complete_signature_after,
+    "exact ended-run completion signature",
+)
+
+complete_branch_before = '''    with write_txn(conn):
+        if expected_run_id is None:
+            cur = conn.execute(
+                """
+                UPDATE tasks
+                   SET status       = 'done',
+                       result       = ?,
+                       completed_at = ?,
+                       claim_lock   = NULL,
+                       claim_expires= NULL,
+                       worker_pid   = NULL,
+                       block_kind   = NULL,
+                       block_recurrences = 0
+                 WHERE id = ?
+                   AND status IN ('running', 'ready', 'blocked', 'scheduled')
+                """,
+                (result, now, task_id),
+            )
+        else:
+'''
+complete_branch_after = '''    # Journal recovery may arrive after a legacy error path ended the
+    # accepted run. Verify that it is still the latest run in this same
+    # transaction; a replacement attempt must make the replay fail closed.
+    with write_txn(conn):
+        replayed_run_id = None
+        if expected_run_id is not None and replay_ended_run_id is not None:
+            raise ValueError("expected_run_id and replay_ended_run_id are mutually exclusive")
+        if replay_ended_run_id is not None:
+            replayed_run_id = int(replay_ended_run_id)
+            # Replay is permitted only while the exact run-ending transition
+            # remains the latest status mutation. Comments and diagnostics do
+            # not invalidate durable accepted work, but an unblock, explicit
+            # triage specification, decomposition, reclaim, or newer attempt
+            # does. BEGIN IMMEDIATE keeps this lineage check and the guarded
+            # task/run updates in one transaction.
+            last_transition = conn.execute(
+                """
+                SELECT kind, run_id
+                  FROM task_events
+                 WHERE task_id = ?
+                   AND kind IN (
+                       'archived', 'block_loop_detected', 'blocked', 'claimed',
+                       'completed', 'decomposed', 'dependency_wait', 'gave_up',
+                       'promoted', 'promoted_manual', 'reclaimed', 'scheduled',
+                       'specified', 'stale', 'timed_out', 'unblocked'
+                   )
+                 ORDER BY id DESC
+                 LIMIT 1
+                """,
+                (task_id,),
+            ).fetchone()
+            if (
+                last_transition is None
+                or last_transition["run_id"] != replayed_run_id
+                or last_transition["kind"] not in {
+                    "block_loop_detected",
+                    "blocked",
+                    "reclaimed",
+                    "stale",
+                    "timed_out",
+                    "gave_up",
+                }
+            ):
+                return False
+            cur = conn.execute(
+                """
+                UPDATE tasks
+                   SET status       = 'done',
+                       result       = ?,
+                       completed_at = ?,
+                       claim_lock   = NULL,
+                       claim_expires= NULL,
+                       worker_pid   = NULL,
+                       block_kind   = NULL,
+                       block_recurrences = 0
+                 WHERE id = ?
+                   AND status IN ('ready', 'blocked', 'triage')
+                   AND current_run_id IS NULL
+                   AND ? = (
+                       SELECT id FROM task_runs
+                        WHERE task_id = ?
+                        ORDER BY id DESC
+                        LIMIT 1
+                   )
+                   AND EXISTS (
+                       SELECT 1 FROM task_runs
+                        WHERE id = ?
+                          AND task_id = ?
+                          AND ended_at IS NOT NULL
+                   )
+                """,
+                (
+                    result,
+                    now,
+                    task_id,
+                    replayed_run_id,
+                    task_id,
+                    replayed_run_id,
+                    task_id,
+                ),
+            )
+        elif expected_run_id is None:
+            cur = conn.execute(
+                """
+                UPDATE tasks
+                   SET status       = 'done',
+                       result       = ?,
+                       completed_at = ?,
+                       claim_lock   = NULL,
+                       claim_expires= NULL,
+                       worker_pid   = NULL,
+                       block_kind   = NULL,
+                       block_recurrences = 0
+                 WHERE id = ?
+                   AND status IN ('running', 'ready', 'blocked', 'scheduled')
+                """,
+                (result, now, task_id),
+            )
+        else:
+'''
+db = replace_once(
+    db,
+    complete_branch_before,
+    complete_branch_after,
+    "transactional exact ended-run completion",
+)
+
+run_completion_before = '''        run_id = _end_run(
+            conn, task_id,
+            outcome="completed", status="done",
+            summary=summary if summary is not None else result,
+            metadata=metadata,
+        )
+        # If complete_task was called on a never-claimed task (ready or
+        # blocked → done with no run in flight), synthesize a
+        # zero-duration run so the handoff fields are persisted in
+        # attempt history instead of silently lost.
+        if run_id is None and (summary or metadata or result):
+            run_id = _synthesize_ended_run(
+                conn, task_id,
+                outcome="completed",
+                summary=summary if summary is not None else result,
+                metadata=metadata,
+            )
+'''
+run_completion_after = '''        if replayed_run_id is None:
+            run_id = _end_run(
+                conn, task_id,
+                outcome="completed", status="done",
+                summary=summary if summary is not None else result,
+                metadata=metadata,
+            )
+            # If complete_task was called on a never-claimed task (ready or
+            # blocked → done with no run in flight), synthesize a
+            # zero-duration run so the handoff fields are persisted in
+            # attempt history instead of silently lost.
+            if run_id is None and (summary or metadata or result):
+                run_id = _synthesize_ended_run(
+                    conn, task_id,
+                    outcome="completed",
+                    summary=summary if summary is not None else result,
+                    metadata=metadata,
+                )
+        else:
+            run_id = replayed_run_id
+            replayed = conn.execute(
+                """
+                UPDATE task_runs
+                   SET status = 'done',
+                       outcome = 'completed',
+                       summary = ?,
+                       error = NULL,
+                       metadata = ?,
+                       claim_lock = NULL,
+                       claim_expires = NULL,
+                       worker_pid = NULL
+                 WHERE id = ?
+                   AND task_id = ?
+                   AND ended_at IS NOT NULL
+                """,
+                (
+                    summary if summary is not None else result,
+                    json.dumps(metadata, ensure_ascii=False) if metadata else None,
+                    run_id,
+                    task_id,
+                ),
+            )
+            if replayed.rowcount != 1:
+                raise RuntimeError("exact ended-run completion lost its run identity")
+        conn.execute(
+            "UPDATE tasks SET completed_run_id = ? WHERE id = ?",
+            (run_id, task_id),
+        )
+'''
+db = replace_once(
+    db,
+    run_completion_before,
+    run_completion_after,
+    "exact completion run provenance",
+)
+
+reclaim_before = '''def reclaim_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    reason: Optional[str] = None,
+    signal_fn=None,
+) -> bool:
+    """Operator-driven reclaim: release the claim and reset to ``ready``.
+
+    Unlike :func:`release_stale_claims` which only acts on tasks whose
+    ``claim_expires`` has passed, this function reclaims immediately
+    regardless of TTL. Intended for the dashboard/CLI recovery flow
+    when an operator wants to abort a running worker without waiting
+    for the TTL to expire (e.g. after seeing a hallucination warning).
+
+    Returns True if a reclaim happened, False if the task isn't in a
+    reclaimable state (not running, or doesn't exist).
+    """
+    row = conn.execute(
+        "SELECT status, claim_lock, worker_pid FROM tasks WHERE id = ?",
+        (task_id,),
+    ).fetchone()
+    if not row:
+        return False
+    if row["status"] != "running" and row["claim_lock"] is None:
+        # Nothing to reclaim — already ready / blocked / done.
+        return False
+    prev_lock = row["claim_lock"]
+    termination = _terminate_reclaimed_worker(
+        row["worker_pid"], prev_lock, signal_fn=signal_fn,
+    )
+    with write_txn(conn):
+        cur = conn.execute(
+            "UPDATE tasks SET status = 'ready', claim_lock = NULL, "
+            "claim_expires = NULL, worker_pid = NULL "
+            "WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
+            "AND claim_lock IS ?",
+            (task_id, prev_lock),
+        )
+        if cur.rowcount != 1:
+            return False
+        run_id = _end_run(
+            conn, task_id,
+            outcome="reclaimed", status="reclaimed",
+            error=(
+                f"manual_reclaim: {reason}" if reason
+                else f"manual_reclaim lock={prev_lock}"
+            ),
+            metadata=termination,
+        )
+        payload = {
+            "manual": True,
+            "reason": reason,
+            "prev_lock": prev_lock,
+        }
+        payload.update(termination)
+        _append_event(
+            conn, task_id, "reclaimed",
+            payload,
+            run_id=run_id,
+        )
+    # Operator intervention — they've looked at the task, so the
+    # consecutive-failures counter is now stale. Give the next retry
+    # a fresh budget. (_clear_failure_counter opens its own write_txn,
+    # so it runs after the enclosing one commits.)
+    _clear_failure_counter(conn, task_id)
+    return True
+'''
+reclaim_after = '''def reclaim_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    reason: Optional[str] = None,
+    signal_fn=None,
+    expected_run_id: Optional[int] = None,
+) -> bool:
+    """Operator-driven reclaim: release the claim and reset to ``ready``.
+
+    Unlike :func:`release_stale_claims` which only acts on tasks whose
+    ``claim_expires`` has passed, this function reclaims immediately
+    regardless of TTL. Intended for the dashboard/CLI recovery flow
+    when an operator wants to abort a running worker without waiting
+    for the TTL to expire (e.g. after seeing a hallucination warning).
+
+    When ``expected_run_id`` is supplied, the run identity is checked after
+    ``BEGIN IMMEDIATE`` and included in the guarded update. A replacement run
+    is therefore neither signalled nor reclaimed by stale recovery evidence.
+
+    Returns True if a reclaim happened, False if the task isn't in a
+    reclaimable state (not running, or doesn't exist), or its run changed.
+    """
+    with write_txn(conn):
+        row = conn.execute(
+            "SELECT status, claim_lock, worker_pid, current_run_id "
+            "FROM tasks WHERE id = ?",
+            (task_id,),
+        ).fetchone()
+        if not row:
+            return False
+        if expected_run_id is not None and row["current_run_id"] != expected_run_id:
+            return False
+        if row["status"] != "running" and row["claim_lock"] is None:
+            # Nothing to reclaim — already ready / blocked / done.
+            return False
+        prev_lock = row["claim_lock"]
+        termination = _terminate_reclaimed_worker(
+            row["worker_pid"], prev_lock, signal_fn=signal_fn,
+        )
+        if expected_run_id is None:
+            cur = conn.execute(
+                "UPDATE tasks SET status = 'ready', claim_lock = NULL, "
+                "claim_expires = NULL, worker_pid = NULL "
+                "WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
+                "AND claim_lock IS ?",
+                (task_id, prev_lock),
+            )
+        else:
+            cur = conn.execute(
+                "UPDATE tasks SET status = 'ready', claim_lock = NULL, "
+                "claim_expires = NULL, worker_pid = NULL "
+                "WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
+                "AND claim_lock IS ? AND current_run_id = ?",
+                (task_id, prev_lock, int(expected_run_id)),
+            )
+        if cur.rowcount != 1:
+            return False
+        run_id = _end_run(
+            conn, task_id,
+            outcome="reclaimed", status="reclaimed",
+            error=(
+                f"manual_reclaim: {reason}" if reason
+                else f"manual_reclaim lock={prev_lock}"
+            ),
+            metadata=termination,
+        )
+        payload = {
+            "manual": True,
+            "reason": reason,
+            "prev_lock": prev_lock,
+        }
+        payload.update(termination)
+        _append_event(
+            conn, task_id, "reclaimed",
+            payload,
+            run_id=run_id,
+        )
+    # Operator intervention — they've looked at the task, so the
+    # consecutive-failures counter is now stale. Give the next retry
+    # a fresh budget. (_clear_failure_counter opens its own write_txn,
+    # so it runs after the enclosing one commits.)
+    _clear_failure_counter(conn, task_id)
+    return True
+'''
+db = replace_once(
+    db,
+    reclaim_before,
+    reclaim_after,
+    "transactional exact-run reclaim",
+)
+
+task_field_before = '''    current_run_id: Optional[int] = None
+    workflow_template_id: Optional[str] = None
+'''
+task_field_after = '''    current_run_id: Optional[int] = None
+    # Durable provenance for the run that authoritatively completed this task.
+    completed_run_id: Optional[int] = None
+    workflow_template_id: Optional[str] = None
+'''
+db = replace_once(
+    db,
+    task_field_before,
+    task_field_after,
+    "completed-run task field",
+)
+
+task_row_before = '''            current_run_id=(
+                row["current_run_id"] if "current_run_id" in keys else None
+            ),
+            workflow_template_id=(
+'''
+task_row_after = '''            current_run_id=(
+                row["current_run_id"] if "current_run_id" in keys else None
+            ),
+            completed_run_id=(
+                row["completed_run_id"] if "completed_run_id" in keys else None
+            ),
+            workflow_template_id=(
+'''
+db = replace_once(
+    db,
+    task_row_before,
+    task_row_after,
+    "completed-run row mapping",
+)
+
+task_schema_before = '''    -- run is in-flight). Denormalised for cheap reads.
+    current_run_id       INTEGER,
+    -- Forward-compat for v2 workflow routing. In v1 the kernel writes
+'''
+task_schema_after = '''    -- run is in-flight). Denormalised for cheap reads.
+    current_run_id       INTEGER,
+    -- Immutable provenance for the task result's authoritative run.
+    completed_run_id     INTEGER,
+    -- Forward-compat for v2 workflow routing. In v1 the kernel writes
+'''
+db = replace_once(
+    db,
+    task_schema_before,
+    task_schema_after,
+    "completed-run schema",
+)
+
+task_migration_before = '''    if "current_run_id" not in cols:
+        _add_column_if_missing(
+            conn, "tasks", "current_run_id", "current_run_id INTEGER"
+        )
+    if "workflow_template_id" not in cols:
+'''
+task_migration_after = '''    if "current_run_id" not in cols:
+        _add_column_if_missing(
+            conn, "tasks", "current_run_id", "current_run_id INTEGER"
+        )
+    if "completed_run_id" not in cols:
+        _add_column_if_missing(
+            conn, "tasks", "completed_run_id", "completed_run_id INTEGER"
+        )
+    if "workflow_template_id" not in cols:
+'''
+db = replace_once(
+    db,
+    task_migration_before,
+    task_migration_after,
+    "completed-run migration",
+)
+
+db_path.write_text(db, encoding="utf-8")

docs/hermes_full_handoff_acceptance.md

@@ -0,0 +1,341 @@
+# Hermes full-handoff acceptance and release runbook
+
+`scripts/ops/hermes_handoff_acceptance.py` is a fail-closed release gate. It
+contains 63 checks in 15 groups. An empty catalog, an unknown result status, a
+mandatory `FAIL` or `NOT_RUN`, a harness error, or incomplete evidence makes the
+decision `NO_GO`. The four ephemeral checks are non-mandatory `NOT_RUN` in the
+default mode and are replaced with mandatory results only in armed mode.
+
+This runbook describes evidence collection. It does not authorize a merge,
+publish, deployment, Flux reconcile, credential read, or live cluster mutation.
+Human review of PR #19 and of the resulting report is required.
+
+## Required release inputs
+
+Every value that can go stale must be supplied explicitly. Resolve `main`, PR
+heads, the image digest, build source SHA, deployment revision, and chat config
+revision from their authoritative read-only sources immediately before the run.
+Do not copy the snapshot in the dependency section into a later run.
+
+```sh
+scripts/ops/hermes_handoff_acceptance.py \
+  --context atlas-operator \
+  --remote-main-sha "$REMOTE_MAIN_SHA" \
+  --reviewed-head-sha "$REVIEWED_PR19_HEAD" \
+  --agent-image "$AGENT_GIT_SHA_BUILD_TAG_AT_SHA256" \
+  --build-sha "$IMAGE_BUILD_SOURCE_SHA" \
+  --deployment-revision "$RUNNING_DEPLOYMENT_REVISION" \
+  --chat-config-revision "$RUNNING_CHAT_CONFIG_REVISION" \
+  --dependency-head "12=$PR12_HEAD" \
+  --dependency-head "14=$PR14_HEAD" \
+  --dependency-head "15=$PR15_HEAD" \
+  --dependency-head "16=$PR16_HEAD" \
+  --dependency-head "17=$PR17_HEAD" \
+  --dependency-head "18=$PR18_HEAD" \
+  --dependency-head "20=$PR20_HEAD" \
+  --output build/hermes-handoff-acceptance.json
+```
+
+The fixed topology is three pool workers on three distinct nodes. `--node-count`
+and `--pool-replicas` cannot lower that requirement. Pool node/ordinal assignment,
+Telegram session continuity, and the chat revision are mandatory. Deliberate Flux
+suspensions must be named with repeatable `--expected-suspension` arguments.
+
+Exit status is `0` only for `GO`, `1` for `NO_GO`, and `2` for invalid execution
+bounds. JSON is written to `--output` or stdout; the human summary goes to stderr
+unless `--json-only` is passed.
+
+## Structural safety boundary
+
+The catalog is validated before a `Runner` exists. Invalid inputs, an empty or
+unsafe catalog, and an invalid armed preflight therefore create no subprocess.
+The command policy is checked both before and after executable resolution.
+
+Default mode permits a narrow read grammar:
+
+- Kubernetes reads use `name` or reviewed JSONPath projections. Secret resources,
+  raw API reads, full JSON/YAML, `describe`, `cluster-info dump`, credential-bearing
+  projections, `auth reconcile`, config operations, all mutations, and every
+  dry-run mutation trick are rejected before spawn.
+- Git is read-only. `fetch`, configuration/helper overrides, alternate worktrees,
+  external diff helpers, and push are rejected.
+- Gitea paths are pinned to `atlas/titan-iac` on an exact segment boundary, so a
+  look-alike repository such as `atlas/titan-iac-evil` is refused, and any
+  relative segment — including a percent-encoded `%2e%2e` — is rejected before
+  the request is built.
+- Shell commands must be rendered from frozen templates, use fixed executable
+  paths, and cannot name service-account, Vault, runtime-access, SSH, or other
+  credential roots.
+- Impersonation is structurally impossible. `--as`, `--as-group`, and `--as-uid`
+  are refused for every argv, in every mode, from every vantage, and the inner
+  command of a `kubectl exec` is re-checked under the same rule rather than being
+  granted an exemption. `validate_catalog` refuses an impersonating step from any
+  vantage, not only the operator. A future audited self-probe that genuinely needs
+  impersonation is a reviewed code change, not a configuration flag.
+- The binary allowlist is exactly the set with pinned release digests:
+  `kubectl`, `git`, `sh`, `hermes`, and the Gitea client. `flux` and `helm` are
+  not on it; Flux and Helm status is read through `kubectl` instead, so no
+  allowlisted binary can be admitted that the executor would refuse to attest.
+- `PATH` is replaced with a fixed path. Every outer executable is resolved to an
+  allowed real path, checked for unsafe write mode, and compared with a pinned
+  SHA-256 digest. A caller-authored `kubectl`, `git`, or helper cannot fabricate
+  evidence.
+
+The executor uses a new process group, drains stdout and stderr concurrently,
+and retains at most the configured bytes while the child is running. A monotonic
+absolute timeout terminates and reaps the group, including descendants that keep
+pipes open. Timeout, deadline, and output-byte values reject booleans,
+zero/negative values, non-finite numbers, and values outside fixed bounds. There
+is no concurrency knob: `run_catalog` is strictly sequential and says so rather
+than accepting a bound it would ignore.
+Truncation discards the affected stream and makes every evaluator `NOT_RUN`.
+Bulk-evidence steps have their bytes replaced before they are recorded, but they
+keep the resolved executable path and its attested SHA-256, so withholding output
+never withholds the provenance of the binary that produced it.
+
+Credential screening runs before truncation. Command, output, error, identity,
+reason, evidence, JSON, and human-summary fields are screened recursively,
+including mapping keys and short/numeric sensitive values. The complete rendered
+report is screened again; a remaining credential shape is replaced by a bounded
+`NO_GO` stub rather than being written. No credential prefix is retained at a
+truncation boundary.
+
+## Vantages and evidence
+
+The operator must use an external read-only kubeconfig. The self probe runs in
+the actual Hermes agent pod. Both identities are obtained with
+`kubectl auth whoami`; equal or missing principals make the run `NO_GO`. Required
+Switchyard and chat/Telegram pod vantages must also resolve. Node coverage is
+derived from all daemonset pod node names, not one selected pod.
+
+Dangerous Kubernetes permissions are checked only with side-effect-free
+`kubectl auth can-i` reviews. The default catalog never constructs a Secret,
+TokenRequest, mutation, attach, port-forward, or arbitrary exec request. The one
+forge administrative denial check performs a read-only GET and accepts only an
+explicit authorization refusal; generic `404`, `not found`, and `no route` text
+are not denial evidence.
+
+An absence is only evidence when the probe observed something. Both name rules
+resolve their step through one guard, so a step that exits `0` with no lines — or
+with only separators — is `NOT_RUN`, never `PASS`. This is reachable in practice
+two ways, and both are pinned by regressions: a POSIX pipeline reports the status
+of its *last* stage, so a missing upstream binary in the frozen `env_names`
+template exits `0` with no bytes; and `kubectl -o jsonpath` exits `0` when the
+structure it walks has drifted, whether a parent field, a leaf field, or a
+container filter. Because all five absence checks are mandatory, silence costs
+the release its `GO`. The pool claim projection emits one `<volume>=<claim>` line
+per template volume so a volume without a PVC still counts as an observation.
+
+The strict evaluators reject duplicate-key JSON, raw control characters,
+`NaN`/infinity, boolean/number equality coercion, future timestamps, stale
+provider evidence, and every malformed routing or Flux row. Routing freshness is
+required separately for each provider, lane, and effort. Flux objects must be
+non-suspended, current at `observedGeneration`, and explicitly `Ready=True`;
+unknown or reconciling state is not healthy.
+
+`release.exact-lineage-is-running` atomically binds:
+
+- fixed remote `main` SHA and local `origin/main`;
+- open draft PR #19, base `main`, existing feature ref, and exact reviewed head;
+- the exact `:git-<source-SHA>-build-<n>@sha256:<digest>` image identity,
+  with the source SHA equal to the release `main` SHA;
+- current Deployment generation and revision;
+- the one active ReplicaSet revision/hash; every Ready pod image and imageID;
+- the Flux-applied `main` revision.
+
+The pool evaluator requires exactly ordinals 0, 1, and 2, all Ready, on three
+distinct nodes and claims, with service-account token automount disabled. The
+chat checks always require durable Telegram sessions and the exact running chat
+configuration revision.
+
+## Armed ephemeral mode
+
+Armed mode is the only write path and runs only after the complete default
+catalog reports `GO`.
+
+```sh
+scripts/ops/hermes_handoff_acceptance.py \
+  <all required release inputs above> \
+  --arm-ephemeral-push \
+  --confirm "ARM EPHEMERAL HERMES HANDOFF PUSH" \
+  --ephemeral-token acceptance-20260817a \
+  --output build/hermes-handoff-acceptance-armed.json
+```
+
+Before any runner, network request, or cluster vantage is created, preflight
+binds the working directory to the existing linked PR #19 worktree, the exact
+feature branch and reviewed local HEAD, the fixed HTTPS Atlas origin, base
+`main`, confirmation phrase, and one unique ephemeral ref. The command policy is
+then pinned to that exact ref. It allows only its push, one exact draft-create
+payload, cleanup of the exactly discovered PR number, and deletion of that ref.
+
+The run reserves at least 300 seconds (up to 6000) for cleanup. It verifies the
+pushed SHA, then identifies the pull request from two independent sources: the
+create response itself, and the pull index read page by page to its last page
+(50 per page, 20 pages maximum) rather than a single truncated first page. Either
+source alone names a pull request this run definitely made, because preflight
+proved the ephemeral ref did not exist. Every number found from either source is
+registered and closed, so a failed or unusable index can no longer leave an
+orphan open. A `GO` still requires exactly one open draft with the expected base,
+head ref, base SHA, and head SHA, and the two sources must agree.
+
+Cleanup requires successful PR close, successful branch deletion, a successful
+empty `ls-remote`, and a strict re-read of the same closed PR. Malformed or
+partial cleanup is `FAIL`; it is never inferred from an error. When anything is
+uncertain the `draft-pull-request` evidence carries `residue_ref` and an exact
+`manual_cleanup` command list — the `PATCH .../pulls/<n> --field state=closed`
+calls for every number found, the `DELETE .../branches/<ref>` call, and, when no
+number could be determined at all, the exact ref to sweep the index for by hand.
+
+Force, force-with-lease, fan-out, protected refs, other repositories, other PR
+numbers, arbitrary PATCH payloads, and repository administration remain denied.
+
+## Dependency, merge, and conflict graph
+
+Authoritative snapshot observed 2026-08-17 after refreshing Gitea and Git against
+remote `main` `d8f2d818b9a552ea6c2d7fe86554be829bd5ffff`:
+
+| PR | Head SHA | Draft | Base | Role |
+| --- | --- | --- | --- | --- |
+| #12 | `ed43dbaa7f094e1841aa760862e80015deb9d80c` | no | `main` | AI usage exporter repair |
+| #14 | `8c6e3acdacb223f68b1716f0a74bd543e18890e6` | yes | `main` | bounded forge client and least-privilege RBAC |
+| #15 | `73fefbb5d92ef07371a9a997f22d848acd2d06ed` | yes | `main` | result-decomposition reliability |
+| #16 | `e8c26ecf85a0c802d378fffffb98b7df43b78242` | no | `main` | unprivileged image builder |
+| #17 | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | yes | `main` | three-lane placement |
+| #18 | `689bcb6e48414854447988ba174143a307d308b9` | yes | `main` | distributed three-worker pool |
+| #19 | `c808baff40a12a90f43c2e5e39b94d139f88ea29` | yes | `main` | head this repair replaces on the same branch |
+| #20 | `a242dcc786576ae1a18000cb4941836ff610dff2` | yes | `main` | CLI process-group reaping |
+
+The repaired #19 SHA cannot be embedded in its own commit without changing that
+SHA. Resolve it from Gitea after push and require it as `--reviewed-head-sha`;
+the PR handoff records the exact value. All eight heads in this table, including
+the repaired #19, merge cleanly with `main` individually. Pairwise
+`git merge-tree --write-tree` over all 28 combinations found conflicts for:
+#12+#17, #12+#18, #12+#19, #14+#15, #14+#16, #14+#17, #14+#19, #14+#20, #15+#16,
+#15+#17, #15+#19, #16+#19, #16+#20, #17+#18, and #17+#20. Gitea's
+`mergeable=false` on a draft is not treated as conflict evidence.
+
+This graph differs from the previous snapshot in exactly two ways. #12 and #18
+both advanced, which adds #12+#18, #17+#18, and #12+#19. And #19 no longer
+duplicates the repo-wide hygiene legacy-exception mechanism that #18 also
+carries, which makes #18+#19 clean. Recompute before every integration step;
+nothing here is durable.
+
+Use this dependency order, refreshing every head and re-running merge-tree at
+each step:
+
+1. #14 first so the release never restores or prolongs cluster-admin authority.
+2. #16, then #15, resolving their Dockerfile and quality-contract changes
+   additively. Never take one PR's quality contract wholesale.
+3. #20 after #15; it depends on the CLI lifecycle work and must be rebased across
+   the current #14/#16 contract and helper changes.
+4. #12 before #17, resolving their Deployment overlap once.
+5. #17, then #18, after the authority, image, reliability, and reaping work.
+6. #19 last as the integration/acceptance change, only after #12/#14-#18/#20 are
+   merged and their exact supplied heads are ancestors of refreshed `main`.
+
+An exact post-repair merge-tree check reports #19 clean with current `main`, #17,
+#18, and #20, and conflicting with #12, #14, #15, and #16 on these files:
+
+| Pair | Overlapping files |
+| --- | --- |
+| #12+#19 | `testing/tests/test_hermes_auto_router.py` |
+| #14+#19 | `testing/quality_contract.json`, `testing/quality_coverage.py`, `testing/tests/test_quality_coverage_helpers.py` |
+| #15+#19 | `testing/tests/conftest.py` |
+| #16+#19 | `testing/quality_contract.json` |
+
+Every one of those overlaps is additive on the #19 side. Resolve the contract by
+union — take every PR's managed modules, lint paths, tracked files, and branch
+tracked files, and never take one PR's quality contract wholesale. #19 adds two
+fixture-isolation fixtures to `conftest.py` and two lines to the auto-router test;
+keep both sides. Preserve branch coverage, every tracked module, fixture
+isolation, and all desired-state changes during conflict resolution.
+
+Recompute this graph from the Gitea API immediately before integration; do not
+infer conflict status from Gitea's draft `mergeable` field.
+
+The #16/#19 integration must keep both the immutable digest and the source tag
+(`newTag: git-<main-SHA>-build-<n>` plus the digest) in rendered image identity.
+Digest-only rendering loses the attested build-source link and therefore
+correctly remains `NO_GO` under the exact-lineage evaluator.
+
+## Safe rollback guidance
+
+Every rollback is a reviewed forward fix or selective Git revert followed by
+the normal Flux workflow. Never restore `hermes-agent-cluster-admin`, even when
+reverting unrelated portions of #14. If a rollback would widen RBAC, stop and
+write a forward least-privilege fix instead.
+
+Reverting #18 removes the StatefulSet desired state, but its
+`volumeClaimTemplates` PVCs are retained by the Kubernetes default because no
+delete retention policy is set. Do not claim Flux pruning deletes those PVCs.
+Any PVC deletion is a separate, explicit, human-reviewed data-destruction task
+with exact targets and recovery implications.
+
+Image rollback means a reviewed commit restoring a previously attested digest;
+never retag an immutable image. Do not use direct `kubectl` edits as a durable
+rollback.
+
+## Quality and release evidence
+
+The canonical quality contract explicitly manages, lints, and tracks all 16
+`scripts/ops/hermes_handoff_*.py` modules. It enforces at least 95% line and
+branch coverage per tracked file and the 500-physical-line source cap. Contract
+tests derive the module set so a future file cannot silently escape the gate.
+This change is purely additive to the contract: it registers its own modules and
+changes no repo-wide hygiene semantics.
+
+The gate's `hygiene` step is red on `main` and stays red here. Exactly four
+pre-existing test files exceed the 500-line cap —
+`scripts/tests/test_dashboards_render_atlas.py` (516),
+`testing/tests/test_hermes_chat_quality.py` (2242),
+`testing/tests/test_hermes_cli_lanes.py` (1971), and
+`testing/tests/test_hermes_coordinator.py` (510). All four are byte-identical at
+merge-base `30259b52` and at current `origin/main`, none is touched by this
+change, and the same four are the only issues a pristine `main` reports. Splitting
+them, or grandfathering them, is a repo-wide contract decision that belongs to the
+canonical contract change carried by PR #14/#15, not to this acceptance harness.
+Every other gate step — docs, smell, unit, coverage — is green, and every file
+this change adds or edits is under the cap.
+
+```sh
+python3 -m testing.quality_gate --profile local --build-dir build
+python3 -m testing.quality_handoff_mutation
+python3 -m ruff check --select F,B,SIM,C4,UP --ignore B017,UP015,UP035 \
+  scripts/ops/hermes_handoff_*.py testing/quality_coverage.py \
+  testing/quality_handoff_mutation.py testing/tests/test_hermes_handoff_*.py
+python3 -m ruff format --check \
+  scripts/ops/hermes_handoff_*.py testing/quality_coverage.py \
+  testing/quality_handoff_mutation.py testing/tests/test_hermes_handoff_*.py
+python3 -m compileall -q scripts/ops testing
+kustomize build services/hermes | kubectl apply --dry-run=client --validate=false -f -
+git diff --check origin/main...HEAD
+```
+
+The format scope names the files this change owns. A wider `testing/quality_*.py`
+glob picks up `testing/quality_gate.py`, which is unmodified here and not
+format-clean, so the documented command would fail for a reason this change did
+not cause. `ruff format` is evidence, not a gate step: the canonical gate runs
+`ruff check` only.
+
+The deterministic mutation gate must kill every policy, redaction, verdict,
+numeric, truncation, provenance, and cleanup mutant, including one that restores
+the zero-evidence pass, one that restores the bare-prefix repository pin, one
+that stops re-checking a `kubectl exec` inner command, and one that drops binary
+attestation from an unrecorded step. The adversarial suite includes large
+concurrent stdout/stderr, malformed JSON and bytes, zero state, partial failure,
+absolute timeout, descendant-held pipes, malformed cleanup, ambiguous creates,
+and no-subprocess/no-mutation spies. Render/client dry-run, exact conflict checks,
+and bounded diff/full-tree secret scans must also pass.
+
+The provider-health and Gitea-token-sensitive tests use hermetic fixtures. Their
+result must be identical whether synthetic runtime files exist or are absent;
+tests must never read the mounted production token.
+
+## Release decision
+
+The handoff remains `NO_GO` until all dependency heads are merged, the exact
+lineage check passes from an external operator, both default and armed reports
+are complete, the branch and PR cleanup is verified, the canonical and
+adversarial gates are green, and a human has reviewed PR #19 plus the artifacts.
+The harness never merges, deploys, reconciles, publishes, or approves anything.

scripts/ops/hermes_handoff_acceptance.py

@@ -0,0 +1,354 @@
+#!/usr/bin/env python3
+"""Fail-closed acceptance harness for the Hermes full handoff.
+
+Read-only by default. The harness collects evidence from two genuinely separate
+places — an external operator vantage and the Hermes agent probing itself from
+inside its own pod — classifies every check PASS / FAIL / NOT_RUN /
+NOT_APPLICABLE, and emits machine-readable JSON alongside a short human summary.
+
+Any mandatory FAIL or NOT_RUN is a NO_GO, and so is a harness-level problem: an
+unreachable vantage, a catalog whose evidence no longer exists, an expired
+deadline, or two vantages that turn out to be the same principal. Nothing here
+rounds an absence of evidence up to a pass.
+
+    scripts/ops/hermes_handoff_acceptance.py --help
+
+The release runbook lists every required freshness and lineage input.
+
+Mutation lives behind a separate arming flag, needs an exact confirmation
+phrase and a unique ref, refuses protected push targets before any network
+call, and verifies its own cleanup. A default run reports those checks NOT_RUN.
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from hermes_handoff_catalog import Targets  # noqa: E402
+from hermes_handoff_ephemeral import (  # noqa: E402
+    CONFIRMATION,
+    ArmingError,
+    ArmRequest,
+    preflight,
+    run_armed,
+)
+from hermes_handoff_exec import (  # noqa: E402
+    DEFAULT_COMMAND_TIMEOUT,
+    DEFAULT_RUN_DEADLINE,
+    Runner,
+)
+from hermes_handoff_model import GO, Report, unscreened_fields, utc_now  # noqa: E402
+from hermes_handoff_policy import (  # noqa: E402
+    ARMED,
+    EXPECTED_BASE,
+    EXPECTED_REMOTE,
+    EXPECTED_REPO,
+    READ_ONLY,
+)
+from hermes_handoff_redaction import DEFAULT_MAX_BYTES  # noqa: E402
+from hermes_handoff_run import (  # noqa: E402
+    build_catalog,
+    build_report,
+    resolve_vantages,
+    unavailable_report,
+    validate_catalog,
+    validate_targets,
+)
+
+EXIT_GO = 0
+EXIT_NO_GO = 1
+EXIT_USAGE = 2
+
+
+def dependency_head(value: str) -> tuple[int, str]:
+    """Parse one `PR=SHA` dependency-head binding."""
+    try:
+        number, sha = value.split("=", 1)
+        parsed = int(number)
+    except (TypeError, ValueError) as exc:
+        raise argparse.ArgumentTypeError(
+            "dependency head must be PR=40-character-SHA"
+        ) from exc
+    if (
+        parsed <= 0
+        or len(sha) != 40
+        or any(character not in "0123456789abcdef" for character in sha)
+    ):
+        raise argparse.ArgumentTypeError(
+            "dependency head must be PR=lowercase-40-character-SHA"
+        )
+    return (parsed, sha)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Return the command-line interface for the harness."""
+    parser = argparse.ArgumentParser(
+        prog="hermes_handoff_acceptance",
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--namespace", default="hermes", help="Hermes namespace.")
+    parser.add_argument(
+        "--context", help="Kubernetes context for the operator vantage."
+    )
+    parser.add_argument("--kubeconfig", help="Kubeconfig for the operator vantage.")
+    parser.add_argument(
+        "--output", type=Path, help="Write the JSON report here instead of stdout."
+    )
+    parser.add_argument(
+        "--json-only", action="store_true", help="Suppress the human summary."
+    )
+    parser.add_argument(
+        "--command-timeout",
+        type=float,
+        default=DEFAULT_COMMAND_TIMEOUT,
+        help="Per-command seconds.",
+    )
+    parser.add_argument(
+        "--deadline",
+        type=float,
+        default=DEFAULT_RUN_DEADLINE,
+        help="Whole-run seconds.",
+    )
+    parser.add_argument(
+        "--max-output-bytes",
+        type=int,
+        default=DEFAULT_MAX_BYTES,
+        help="Per-capture byte budget.",
+    )
+    parser.add_argument(
+        "--baseline-commit",
+        default=Targets.baseline_commit,
+        help="Commit main must descend from.",
+    )
+    parser.add_argument(
+        "--dependency-pr",
+        type=int,
+        action="append",
+        default=None,
+        help="Pull request that must be merged before this release is certifiable (repeatable).",
+    )
+    parser.add_argument(
+        "--dependency-head",
+        type=dependency_head,
+        action="append",
+        default=None,
+        metavar="PR=SHA",
+        help="Exact current dependency PR head (repeat for every dependency).",
+    )
+    parser.add_argument(
+        "--remote-main-sha", default="", help="Exact observed origin/main SHA."
+    )
+    parser.add_argument(
+        "--reviewed-head-sha", default="", help="Exact reviewed PR #19 head SHA."
+    )
+    parser.add_argument(
+        "--agent-image",
+        default="",
+        help="Exact agent :git-<source>-build-<n>@sha256 identity.",
+    )
+    parser.add_argument(
+        "--build-sha", default="", help="Exact source SHA used for the image build."
+    )
+    parser.add_argument(
+        "--deployment-revision", default="", help="Exact running Deployment revision."
+    )
+    parser.add_argument(
+        "--expected-suspension",
+        action="append",
+        default=None,
+        metavar="NAMESPACE/NAME",
+        help="Flux object allowed to be suspended (repeatable).",
+    )
+    parser.add_argument(
+        "--node-count",
+        type=int,
+        default=3,
+        help="Nodes the fleet must cover (fixed at 3).",
+    )
+    parser.add_argument(
+        "--pool-replicas", type=int, default=3, help="Expected pool worker count."
+    )
+    parser.add_argument(
+        "--chat-config-revision",
+        default="",
+        help="Expected chat pod-template config revision.",
+    )
+    parser.add_argument(
+        "--chat-ordinal",
+        type=int,
+        default=0,
+        help="Chat tenant ordinal that carries the Telegram identity.",
+    )
+    parser.add_argument(
+        "--pool-worker-env",
+        action="append",
+        default=None,
+        help="Environment variable the pool workers must declare (repeatable).",
+    )
+    parser.add_argument(
+        "--max-evidence-age",
+        type=int,
+        default=86_400,
+        help="Seconds before provider and routing evidence is treated as stale.",
+    )
+    armed = parser.add_argument_group("ephemeral mutation mode")
+    armed.add_argument(
+        "--arm-ephemeral-push",
+        action="store_true",
+        help="Permit one ephemeral branch push and draft pull request, then remove both.",
+    )
+    armed.add_argument(
+        "--confirm", default="", help=f"Exact confirmation phrase: {CONFIRMATION!r}."
+    )
+    armed.add_argument(
+        "--ephemeral-token", default="", help="Unique 8-64 character ref suffix."
+    )
+    return parser
+
+
+def targets_from_args(arguments: argparse.Namespace, now: dt.datetime) -> Targets:
+    """Return the catalog targets a parsed command line describes."""
+    defaults = Targets()
+    return Targets(
+        namespace=arguments.namespace,
+        baseline_commit=arguments.baseline_commit,
+        dependency_pull_requests=tuple(
+            arguments.dependency_pr or defaults.dependency_pull_requests
+        ),
+        dependency_heads=tuple(arguments.dependency_head or ()),
+        remote_main_sha=arguments.remote_main_sha,
+        reviewed_head_sha=arguments.reviewed_head_sha,
+        agent_image=arguments.agent_image,
+        build_sha=arguments.build_sha,
+        deployment_revision=arguments.deployment_revision,
+        expected_suspensions=tuple(arguments.expected_suspension or ()),
+        node_count=arguments.node_count,
+        pool_replicas=arguments.pool_replicas,
+        chat_config_revision=arguments.chat_config_revision,
+        chat_ordinal=arguments.chat_ordinal,
+        pool_worker_env=tuple(arguments.pool_worker_env or defaults.pool_worker_env),
+        max_evidence_age_seconds=arguments.max_evidence_age,
+        now=now,
+    )
+
+
+def emit(report: Report, arguments: argparse.Namespace, stream=None) -> list[str]:
+    """Write the report, refusing to publish one that still looks credential-shaped."""
+    payload = report.as_dict()
+    offenders = unscreened_fields(payload)
+    if offenders:
+        payload = {
+            "harness": payload["harness"],
+            "decision": "NO_GO",
+            "harness_errors": [
+                "the rendered report failed its own credential screening at "
+                + ", ".join(offenders[:8])
+            ],
+        }
+    rendered = json.dumps(payload, indent=2, sort_keys=True, default=str)
+    if arguments.output:
+        arguments.output.parent.mkdir(parents=True, exist_ok=True)
+        arguments.output.write_text(rendered + "\n", encoding="utf-8")
+    else:
+        print(rendered, file=stream or sys.stdout)
+    if not arguments.json_only:
+        print(report.render_summary(), file=sys.stderr)
+    return offenders
+
+
+def _armed_results(arguments: argparse.Namespace, runner: Runner, vantages) -> list:
+    request = ArmRequest(
+        repo=EXPECTED_REPO,
+        remote=EXPECTED_REMOTE,
+        token=arguments.ephemeral_token,
+        confirmation=arguments.confirm,
+        base=EXPECTED_BASE,
+        expected_head=arguments.reviewed_head_sha,
+        expected_base_sha=arguments.remote_main_sha,
+    )
+    preflight(request, EXPECTED_REPO, Path.cwd())
+    vantage = vantages["operator"]
+    return run_armed(runner, vantage, request)
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run the acceptance sweep and return the process exit status."""
+    arguments = build_parser().parse_args(argv)
+    now = dt.datetime.now(dt.timezone.utc)
+    started_at = utc_now(lambda: now)
+    targets = targets_from_args(arguments, now)
+    mode = ARMED if arguments.arm_ephemeral_push else READ_ONLY
+
+    specs = build_catalog(targets)
+    startup_problems = [*validate_targets(targets), *validate_catalog(specs)]
+    if arguments.arm_ephemeral_push:
+        try:
+            preflight(
+                ArmRequest(
+                    repo=EXPECTED_REPO,
+                    remote=EXPECTED_REMOTE,
+                    token=arguments.ephemeral_token,
+                    confirmation=arguments.confirm,
+                    base=EXPECTED_BASE,
+                    expected_head=arguments.reviewed_head_sha,
+                    expected_base_sha=arguments.remote_main_sha,
+                ),
+                EXPECTED_REPO,
+                Path.cwd(),
+            )
+        except ArmingError as exc:
+            startup_problems.append(f"arming refused: {exc}")
+    if startup_problems:
+        report = unavailable_report(
+            mode, started_at, "; ".join(startup_problems), targets
+        )
+        emit(report, arguments)
+        return EXIT_NO_GO
+
+    try:
+        runner = Runner(
+            mode=mode,
+            command_timeout=arguments.command_timeout,
+            deadline_seconds=arguments.deadline,
+            max_bytes=arguments.max_output_bytes,
+        )
+    except ValueError as exc:
+        report = unavailable_report(
+            mode, started_at, f"invalid execution bound: {exc}", targets
+        )
+        emit(report, arguments)
+        return EXIT_USAGE
+    vantages, records = resolve_vantages(
+        runner, targets, arguments.kubeconfig, arguments.context
+    )
+    report = build_report(runner, targets, specs, vantages, records, mode, started_at)
+    if arguments.arm_ephemeral_push:
+        if report.decision != GO:
+            report.harness_errors.append(
+                "armed mutation was not started because read-only acceptance did not GO"
+            )
+        else:
+            armed_results = _armed_results(arguments, runner, vantages)
+            armed_ids = {result.spec.id for result in armed_results}
+            report.results = [
+                result for result in report.results if result.spec.id not in armed_ids
+            ]
+            report.results.extend(armed_results)
+            report.results.sort(key=lambda result: result.spec.id)
+            report.finished_at = utc_now()
+    offenders = emit(report, arguments)
+    if offenders:
+        return EXIT_NO_GO
+    return EXIT_GO if report.decision == GO else EXIT_NO_GO
+
+
+if __name__ == "__main__":  # pragma: no cover - exercised through main()
+    raise SystemExit(main())

scripts/ops/hermes_handoff_arming.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""Arming, provenance, and push-target rules for the armed ephemeral mode.
+
+Nothing here touches the network or spawns a process. It is the half of armed
+mode that must be true before a single command exists: the confirmation phrase,
+the fixed repository and base, the one ephemeral ref grammar, and a local
+worktree whose HEAD and ``origin/main`` are byte-exact with the reviewed release.
+"""
+
+from __future__ import annotations
+
+import configparser
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from hermes_handoff_policy import (
+    EXPECTED_BASE,
+    EXPECTED_REMOTE,
+    EXPECTED_REPO,
+    arm_ephemeral_policy,
+)
+
+CONFIRMATION = "ARM EPHEMERAL HERMES HANDOFF PUSH"
+REF_PREFIX = "ephemeral/hermes-handoff-acceptance"
+REF_RE = re.compile(rf"{REF_PREFIX}/[a-z0-9][a-z0-9-]{{7,63}}\Z")
+TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9-]{7,63}\Z")
+SHA_RE = re.compile(r"[0-9a-f]{40}\Z")
+EXPECTED_HEAD_REF = "feature/hermes-full-handoff-acceptance"
+EXPECTED_REMOTE_URL = "https://scm.bstein.dev/atlas/titan-iac.git"
+PROTECTED_NAMES = frozenset(
+    {
+        "default",
+        "develop",
+        "head",
+        "main",
+        "master",
+        "prod",
+        "production",
+        "release",
+        "stable",
+        "trunk",
+    }
+)
+
+
+class ArmingError(ValueError):
+    """Raised when an ephemeral run is not safe to start."""
+
+
+@dataclass(frozen=True)
+class ArmRequest:
+    """Every fixed and caller-confirmed input to one armed run."""
+
+    repo: str
+    remote: str
+    token: str
+    confirmation: str
+    base: str = EXPECTED_BASE
+    expected_head: str = ""
+    expected_base_sha: str = ""
+
+    @property
+    def ref(self) -> str:
+        return f"{REF_PREFIX}/{self.token}"
+
+
+def normalise_branch(name: str) -> str:
+    """Return a branch name stripped of ref decoration, for comparison."""
+    return name.strip().removeprefix("refs/heads/").strip("/").lower()
+
+
+def assert_push_target_allowed(ref: str) -> None:
+    """Raise unless ``ref`` is the one ephemeral branch grammar."""
+    if normalise_branch(ref) in PROTECTED_NAMES:
+        raise ArmingError(f"refusing to push to protected branch {ref!r}")
+    if not REF_RE.fullmatch(ref):
+        raise ArmingError(f"push target {ref!r} is not an ephemeral acceptance ref")
+
+
+def bounded_read(path: Path, limit: int = 1024 * 1024) -> str:
+    data = path.read_bytes()
+    if not data or len(data) > limit:
+        raise ArmingError(f"local Git metadata is empty or exceeds {limit} bytes")
+    return data.decode("utf-8", "strict")
+
+
+def _attest_worktree(root: Path, expected_head: str, expected_base_sha: str) -> None:
+    """Bind cwd, common Git metadata, branch, remote, and exact local HEAD."""
+    resolved = root.resolve(strict=True)
+    marker = resolved / ".git"
+    if not marker.is_file():
+        raise ArmingError(
+            "armed mode must run at the root of the existing linked worktree"
+        )
+    line = bounded_read(marker, 4096).strip()
+    if not line.startswith("gitdir: "):
+        raise ArmingError("linked-worktree Git metadata is malformed")
+    gitdir = Path(line.removeprefix("gitdir: ")).resolve(strict=True)
+    common = (gitdir / bounded_read(gitdir / "commondir", 4096).strip()).resolve(
+        strict=True
+    )
+    head = bounded_read(gitdir / "HEAD", 4096).strip()
+    expected_ref = f"refs/heads/{EXPECTED_HEAD_REF}"
+    if head != f"ref: {expected_ref}":
+        raise ArmingError("armed mode must run from the existing PR #19 feature branch")
+    ref_path = common / expected_ref
+    actual_head = bounded_read(ref_path, 4096).strip() if ref_path.is_file() else ""
+    if actual_head != expected_head:
+        raise ArmingError(
+            "local worktree HEAD does not match the exact reviewed PR head"
+        )
+    remote_ref = common / "refs" / "remotes" / EXPECTED_REMOTE / EXPECTED_BASE
+    remote_main = bounded_read(remote_ref, 4096).strip() if remote_ref.is_file() else ""
+    if remote_main != expected_base_sha:
+        raise ArmingError("local origin/main does not match the exact release base SHA")
+    parser = configparser.RawConfigParser(interpolation=None)
+    try:
+        parser.read_string(bounded_read(common / "config"))
+        remote_url = parser.get(f'remote "{EXPECTED_REMOTE}"', "url")
+    except (configparser.Error, KeyError) as exc:
+        raise ArmingError("local Git remote metadata is malformed") from exc
+    if remote_url != EXPECTED_REMOTE_URL:
+        raise ArmingError("origin is not the fixed Atlas titan-iac HTTPS repository")
+
+
+def preflight(
+    request: ArmRequest, expected_repo: str, worktree: Path | None = None
+) -> None:
+    """Validate arming and local provenance before any subprocess/network call."""
+    if request.confirmation != CONFIRMATION:
+        raise ArmingError("confirmation phrase does not match")
+    if expected_repo != EXPECTED_REPO or request.repo != EXPECTED_REPO:
+        raise ArmingError(
+            "armed repository is not the fixed Atlas titan-iac repository"
+        )
+    if (
+        request.remote != EXPECTED_REMOTE
+        or normalise_branch(request.base) != EXPECTED_BASE
+    ):
+        raise ArmingError("armed remote and pull-request base are fixed to origin/main")
+    if not TOKEN_RE.fullmatch(request.token):
+        raise ArmingError(
+            "ephemeral token must be 8-64 lowercase alphanumeric or dash characters"
+        )
+    if not SHA_RE.fullmatch(request.expected_head) or not SHA_RE.fullmatch(
+        request.expected_base_sha
+    ):
+        raise ArmingError("exact reviewed head and release base SHAs are required")
+    assert_push_target_allowed(request.ref)
+    _attest_worktree(
+        worktree or Path.cwd(), request.expected_head, request.expected_base_sha
+    )
+    arm_ephemeral_policy(request.ref)

scripts/ops/hermes_handoff_catalog.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""Declarative acceptance catalog: baseline, identity, routing, scopes, access.
+
+The catalog is data on purpose. A reviewer should be able to read what the
+release is asserted to satisfy without reading any plumbing, and the harness
+should be able to prove structural properties over every entry — both vantages
+represented, no impersonation from outside a pod, and no mutation command of
+any kind — before it runs a single command.
+
+Catalog drift is a NO_GO, not a pass. If a manifest, a field name, or a workload
+is renamed, the affected checks report ``NOT_RUN`` because the evidence they
+name is not there, and the release stops until either the platform or the
+catalog is corrected.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from hermes_handoff_model import ALWAYS, READ, CheckSpec, Step
+from hermes_handoff_policy import projection
+
+OPERATOR = "operator"
+SELF = "self"
+SWITCHYARD = "switchyard"
+NODE = "node"
+CHAT = "chat"
+VANTAGE_NAMES = (OPERATOR, SELF, SWITCHYARD, NODE, CHAT)
+
+# `name<TAB>generation<TAB>observedGeneration<TAB>suspend<TAB>conditions`.
+FLUX_PROJECTION = projection(
+    (
+        "jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{'\\t'}"
+        "{.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}"
+        "{range .status.conditions[*]}{.type}={.status},{end}{'\\n'}{end}"
+    ).replace("'", '"')
+)
+
+ENV_NAME_PROJECTION = projection(
+    (
+        "jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{'\\n'}{end}{end}"
+    ).replace("'", '"')
+)
+
+INIT_NAME_PROJECTION = projection(
+    (
+        "jsonpath={range .spec.template.spec.initContainers[*]}{.name}{'\\n'}{end}"
+    ).replace("'", '"')
+)
+
+NODE_NAME_PROJECTION = projection(
+    "jsonpath={range .items[*]}{.spec.nodeName}{'\\n'}{end}".replace("'", '"')
+)
+
+POOL_PROJECTION = projection(
+    (
+        "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.spec.nodeName}{'\\t'}"
+        "{.status.phase}{'\\t'}{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}"
+        "{'\\t'}{range .spec.volumes[*]}{.persistentVolumeClaim.claimName}{','}{end}{'\\n'}{end}"
+    ).replace("'", '"')
+)
+
+PROVIDER_API_KEY_NAMES = (
+    "ANTHROPIC_API_KEY",
+    "ANTHROPIC_AUTH_TOKEN",
+    "ANTHROPIC_BASE_URL",
+    "AZURE_OPENAI_API_KEY",
+    "CLAUDE_API_KEY",
+    "CLAUDE_CODE_OAUTH_TOKEN",
+    "GEMINI_API_KEY",
+    "GOOGLE_API_KEY",
+    "OPENAI_API_KEY",
+    "OPENAI_BASE_URL",
+)
+FORGE_CREDENTIAL_NAMES = (
+    "GITEA_PASSWORD",
+    "GITEA_TOKEN",
+    "GITEA_USERNAME",
+    "GIT_PASSWORD",
+)
+
+CODEX_HEALTH = "/opt/data/provider-health/codex.json"
+CLAUDE_HEALTH = "/opt/data/provider-health/claude.json"
+CODEX_HEALTH_FIELDS = "authenticated,transport,state,model,effort,latency_ms,checked_at"
+CLAUDE_HEALTH_FIELDS = (
+    "authenticated,api_provider,auth_method,transport,state,subscription_type,"
+    "model,effort,latency_ms,checked_at"
+)
+ROUTING_LOG = "/var/lib/switchyard/routing.jsonl"
+
+
+@dataclass(frozen=True)
+class Targets:
+    """Runtime names and expected values the catalog is written against.
+
+    Everything a site can legitimately rename lives here so the assertions stay
+    readable and a rename is a configuration change rather than a code change.
+    """
+
+    namespace: str = "hermes"
+    agent_deployment: str = "hermes-agent"
+    agent_container: str = "hermes"
+    chat_statefulset: str = "hermes-chat-tenant"
+    chat_container: str = "hermes"
+    chat_ordinal: int = 0
+    chat_router_deployment: str = "hermes-chat-router"
+    triage_deployment: str = "hermes"
+    triage_container: str = "hermes"
+    switchyard_deployment: str = "hermes-switchyard"
+    switchyard_container: str = "switchyard"
+    node_daemonset: str = "hermes-node-ssh-access"
+    node_container: str = "key-reconciler"
+    pool_statefulset: str = "hermes-execution-worker"
+    pool_replicas: int = 3
+    pool_selector: str = "app=hermes-execution-worker"
+    coordinator_claims: tuple[str, ...] = (
+        "hermes-agent-home",
+        "hermes-agent-workspace",
+    )
+    node_account: str = "hermes-agent"
+    node_account_identity: str = "1200:1200:/home/hermes-agent:/bin/bash"
+    node_count: int = 3
+    repo: str = "atlas/titan-iac"
+    remote: str = "origin"
+    baseline_commit: str = "ab346f55509d584e457fe26cf90be3078f7a375c"
+    dependency_pull_requests: tuple[int, ...] = (12, 14, 15, 16, 17, 18, 20)
+    dependency_heads: tuple[tuple[int, str], ...] = ()
+    remote_main_sha: str = ""
+    reviewed_pr_number: int = 19
+    reviewed_head_ref: str = "feature/hermes-full-handoff-acceptance"
+    reviewed_head_sha: str = ""
+    agent_image: str = ""
+    build_sha: str = ""
+    deployment_revision: str = ""
+    expected_suspensions: tuple[str, ...] = ()
+    harbor_namespace: str = "harbor"
+    harbor_immutability_job: str = "hermes-agent-immutability"
+    builder_namespace: str = "jenkins"
+    builder_serviceaccount: str = "hermes-image-builder"
+    builder_pipeline: str = "ci/Jenkinsfile.hermes-agent-image"
+    builder_capabilities: str = (
+        'add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]'
+    )
+    agent_init_containers: tuple[str, ...] = ("patch-web-session-activity",)
+    chat_init_containers: tuple[str, ...] = ("patch-api-server-sessions",)
+    chat_config_revision: str = ""
+    pool_worker_env: tuple[str, ...] = ("HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL")
+    max_evidence_age_seconds: int = 86_400
+    routing_tail_lines: int = 600
+    routing_tail_bytes: int = 512 * 1024
+    listing_bytes: int = 512 * 1024
+    git_askpass: str = "/opt/coordinator/gitea_askpass.sh"
+    now: Any = None
+    extra: dict[str, Any] = field(default_factory=dict)
+
+
+def step(
+    key: str,
+    vantage: str,
+    *argv: str,
+    kind: str = READ,
+    optional: bool = False,
+    record: bool = True,
+    max_bytes: int | None = None,
+) -> Step:
+    """Return a probe step addressed to one vantage."""
+    return Step(
+        key=key,
+        vantage=vantage,
+        argv=tuple(argv),
+        kind=kind,
+        optional=optional,
+        record=record,
+        max_bytes=max_bytes,
+    )
+
+
+def check(
+    identifier: str,
+    title: str,
+    group: str,
+    rule: str,
+    steps: Sequence[Step],
+    expect: dict[str, Any] | None = None,
+    mandatory: bool = True,
+    scope: str = ALWAYS,
+    rationale: str = "",
+) -> CheckSpec:
+    """Return one catalog entry."""
+    return CheckSpec(
+        id=identifier,
+        title=title,
+        group=group,
+        rule=rule,
+        steps=tuple(steps),
+        expect=expect or {},
+        mandatory=mandatory,
+        scope=scope,
+        rationale=rationale,
+    )

scripts/ops/hermes_handoff_checks_access.py

@@ -0,0 +1,339 @@
+#!/usr/bin/env python3
+"""Acceptance checks for what Hermes may and may not do to the cluster.
+
+Denials use Kubernetes authorization reviews only.  Live denied attempts were
+removed because a policy regression could turn a supposed probe into a real
+credential read or mutation; the default harness never constructs such argv.
+"""
+
+from __future__ import annotations
+
+from hermes_handoff_catalog import (
+    FLUX_PROJECTION,
+    OPERATOR,
+    SELF,
+    Targets,
+    check,
+    step,
+)
+from hermes_handoff_model import REVIEW, CheckSpec
+
+
+def _access(targets: Targets) -> list[CheckSpec]:
+    namespace = targets.namespace
+    return [
+        check(
+            "access.no-cluster-admin-binding-for-agent",
+            "No ClusterRoleBinding grants the agent service account cluster-admin",
+            "access-denied",
+            "names_absent",
+            [
+                step(
+                    "bindings",
+                    OPERATOR,
+                    "kubectl",
+                    "get",
+                    "clusterrolebindings",
+                    "-o",
+                    "name",
+                )
+            ],
+            {"step": "bindings", "contains": ("hermes-agent-cluster-admin",)},
+            rationale="The binding inventory and in-pod authorization reviews are independent desired/enforced views.",
+        ),
+        check(
+            "access.secrets-are-denied",
+            "Hermes cannot read Secrets cluster-wide",
+            "access-denied",
+            "review_denied",
+            [
+                step(
+                    "review",
+                    SELF,
+                    "kubectl",
+                    "auth",
+                    "can-i",
+                    "get",
+                    "secrets",
+                    "--all-namespaces",
+                    kind=REVIEW,
+                )
+            ],
+            rationale="A live Secret request is never constructed, even when an RBAC regression would allow it.",
+        ),
+        check(
+            "access.service-account-tokens-are-denied",
+            "Hermes cannot mint a service-account token",
+            "access-denied",
+            "review_denied",
+            [
+                step(
+                    "review-token",
+                    SELF,
+                    "kubectl",
+                    "auth",
+                    "can-i",
+                    "create",
+                    "serviceaccounts/token",
+                    "--namespace",
+                    namespace,
+                    kind=REVIEW,
+                ),
+                step(
+                    "review-serviceaccount",
+                    SELF,
+                    "kubectl",
+                    "auth",
+                    "can-i",
+                    "create",
+                    "serviceaccounts",
+                    "--namespace",
+                    namespace,
+                    kind=REVIEW,
+                ),
+            ],
+            rationale="Token creation has no credential-safe live probe, so both relevant grants must be denied by review.",
+        ),
+        check(
+            "access.impersonation-is-denied",
+            "Hermes cannot impersonate another identity",
+            "access-denied",
+            "review_denied",
+            [
+                step(
+                    "review",
+                    SELF,
+                    "kubectl",
+                    "auth",
+                    "can-i",
+                    "impersonate",
+                    "users",
+                    "--all-namespaces",
+                    kind=REVIEW,
+                )
+            ],
+            rationale="The default harness never constructs an impersonated request.",
+        ),
+        check(
+            "access.workload-mutation-is-denied",
+            "Hermes cannot mutate its own workload",
+            "access-denied",
+            "review_denied",
+            [
+                step(
+                    "review",
+                    SELF,
+                    "kubectl",
+                    "auth",
+                    "can-i",
+                    "patch",
+                    "deployments",
+                    "--namespace",
+                    namespace,
+                    kind=REVIEW,
+                )
+            ],
+            rationale="Client-side and server-side dry-run mutation tricks are excluded structurally.",
+        ),
+        check(
+            "access.pod-exec-is-denied",
+            "Hermes cannot exec into an arbitrary pod",
+            "access-denied",
+            "review_denied",
+            [
+                step(
+                    "review",
+                    SELF,
+                    "kubectl",
+                    "auth",
+                    "can-i",
+                    "create",
+                    "pods/exec",
+                    "--all-namespaces",
+                    kind=REVIEW,
+                )
+            ],
+            rationale="No cross-pod exec request is created by a read-only run.",
+        ),
+        check(
+            "access.attach-portforward-and-kube-system-writes-are-denied",
+            "Hermes holds no attach, port-forward, or control-plane write authority",
+            "access-denied",
+            "review_denied",
+            [
+                step(
+                    "review-attach",
+                    SELF,
+                    "kubectl",
+                    "auth",
+                    "can-i",
+                    "create",
+                    "pods/attach",
+                    "--all-namespaces",
+                    kind=REVIEW,
+                ),
+                step(
+                    "review-portforward",
+                    SELF,
+                    "kubectl",
+                    "auth",
+                    "can-i",
+                    "create",
+                    "pods/portforward",
+                    "--all-namespaces",
+                    kind=REVIEW,
+                ),
+                step(
+                    "review-create",
+                    SELF,
+                    "kubectl",
+                    "auth",
+                    "can-i",
+                    "create",
+                    "configmaps",
+                    "--namespace",
+                    "kube-system",
+                    kind=REVIEW,
+                ),
+            ],
+            rationale="Every authority is reviewed without issuing the request it would authorize.",
+        ),
+        check(
+            "access.required-reads-are-allowed",
+            "Hermes retains the reads its operator role depends on",
+            "access-allowed",
+            "allowed",
+            [
+                step(
+                    "watch-review",
+                    SELF,
+                    "kubectl",
+                    "auth",
+                    "can-i",
+                    "watch",
+                    "pods",
+                    "--namespace",
+                    namespace,
+                    kind=REVIEW,
+                ),
+                step(
+                    "pods",
+                    SELF,
+                    "kubectl",
+                    "--namespace",
+                    namespace,
+                    "get",
+                    "pods",
+                    "-o",
+                    "name",
+                ),
+                step("namespaces", SELF, "kubectl", "get", "namespaces", "-o", "name"),
+                step(
+                    "deployments",
+                    SELF,
+                    "kubectl",
+                    "--namespace",
+                    namespace,
+                    "get",
+                    "deployments",
+                    "-o",
+                    "name",
+                ),
+                step("nodes", SELF, "kubectl", "get", "nodes", "-o", "name"),
+            ],
+        ),
+        check(
+            "access.flux-and-helm-status-are-allowed",
+            "Hermes can read Flux and Helm reconciliation status",
+            "access-allowed",
+            "allowed",
+            [
+                step(
+                    "kustomizations",
+                    SELF,
+                    "kubectl",
+                    "get",
+                    "kustomizations.kustomize.toolkit.fluxcd.io",
+                    "--all-namespaces",
+                    "-o",
+                    "name",
+                    record=False,
+                ),
+                step(
+                    "helmreleases",
+                    SELF,
+                    "kubectl",
+                    "get",
+                    "helmreleases.helm.toolkit.fluxcd.io",
+                    "--all-namespaces",
+                    "-o",
+                    "name",
+                    record=False,
+                ),
+            ],
+        ),
+        check(
+            "access.namespace-view-agrees-across-vantages",
+            "The operator and in-pod vantages see the same namespace inventory",
+            "access-allowed",
+            "vantages_agree",
+            [
+                step(
+                    "operator",
+                    OPERATOR,
+                    "kubectl",
+                    "get",
+                    "namespaces",
+                    "-o",
+                    "name",
+                    record=False,
+                ),
+                step(
+                    "self",
+                    SELF,
+                    "kubectl",
+                    "get",
+                    "namespaces",
+                    "-o",
+                    "name",
+                    record=False,
+                ),
+            ],
+            {"steps": ("operator", "self")},
+            rationale="A disagreement here means one vantage is not seeing the cluster the other is certifying.",
+        ),
+    ]
+
+
+def _gitops(targets: Targets) -> list[CheckSpec]:
+    return [
+        check(
+            f"gitops.{kind.split('.')[0]}-reconcile-cleanly",
+            f"No unexpected {label} suspension or unhealthy reconciliation",
+            "gitops",
+            "flux_health",
+            [
+                step(
+                    "objects",
+                    OPERATOR,
+                    "kubectl",
+                    "get",
+                    kind,
+                    "--all-namespaces",
+                    "-o",
+                    FLUX_PROJECTION,
+                    record=False,
+                )
+            ],
+            {"step": "objects", "expected_suspensions": targets.expected_suspensions},
+        )
+        for kind, label in (
+            ("kustomizations.kustomize.toolkit.fluxcd.io", "Kustomization"),
+            ("helmreleases.helm.toolkit.fluxcd.io", "HelmRelease"),
+        )
+    ]
+
+
+def access_checks(targets: Targets) -> list[CheckSpec]:
+    """Return the access-denied, access-allowed, and GitOps health checks."""
+    return [*_access(targets), *_gitops(targets)]

scripts/ops/hermes_handoff_checks_delivery.py

@@ -0,0 +1,374 @@
+#!/usr/bin/env python3
+"""Acceptance checks for forge authority, node accounts, images, and regressions.
+
+What the platform is allowed to do outside its own namespace: which forge
+authority a worker holds, what a node account is and is not, how an image is
+built and pinned, and whether the fixes the release depends on are actually
+running rather than merely merged.
+
+Where a property has no side-effect-free live probe — a merge that must never be
+attempted, a branch protection the harness identity cannot read — the entry says
+so in its rationale and asserts the nearest fact that *is* observable, rather
+than quietly asserting nothing.
+"""
+
+from __future__ import annotations
+
+from hermes_handoff_catalog import (
+    FORGE_CREDENTIAL_NAMES,
+    INIT_NAME_PROJECTION,
+    NODE_NAME_PROJECTION,
+    OPERATOR,
+    SELF,
+    Targets,
+    check,
+    step,
+)
+from hermes_handoff_model import ATTEMPT, CheckSpec
+from hermes_handoff_policy import GITEA_CLIENT, projection, shell
+
+
+def _forge(targets: Targets) -> list[CheckSpec]:
+    api = f"/api/v1/repos/{targets.repo}"
+    return [
+        check(
+            "forge.worker-holds-no-git-credential-variable",
+            "No forge credential is exported into the worker process",
+            "forge",
+            "names_absent",
+            [step("env", SELF, *shell("env_names"))],
+            {"step": "env", "names": FORGE_CREDENTIAL_NAMES},
+            rationale="Authentication reaches Git through an askpass helper that reads runtime state, never through the environment.",
+        ),
+        check(
+            "forge.identity-is-not-an-administrator",
+            "The forge identity the worker uses holds no administrative rights",
+            "forge",
+            "json_field",
+            [step("user", SELF, GITEA_CLIENT, "GET", "/api/v1/user")],
+            {"step": "user", "fields": {"is_admin": False}},
+        ),
+        check(
+            "forge.repository-authority-is-push-only",
+            "The worker can push a branch but cannot administer the repository",
+            "forge",
+            "json_field",
+            [step("repo", SELF, GITEA_CLIENT, "GET", api)],
+            {
+                "step": "repo",
+                "fields": {
+                    "permissions.admin": False,
+                    "permissions.push": True,
+                    "permissions.pull": True,
+                },
+            },
+            rationale="Merge, approve, close, and protection changes all require authority this identity is proven not to hold.",
+        ),
+        check(
+            "forge.administrative-route-is-refused",
+            "An administrative forge route is refused for this identity",
+            "forge",
+            "denied",
+            [
+                step(
+                    "attempt",
+                    SELF,
+                    GITEA_CLIENT,
+                    "GET",
+                    f"{api}/branch_protections",
+                    kind=ATTEMPT,
+                )
+            ],
+            rationale="A live refusal, not a configuration reading. The same lack of authority is what blocks a merge route.",
+        ),
+    ]
+
+
+def _nodes(targets: Targets) -> list[CheckSpec]:
+    return [
+        check(
+            "nodes.hardening-covers-every-node",
+            "The node account audit runs on every node in the cluster",
+            "nodes",
+            "distinct_count",
+            [
+                step(
+                    "coverage",
+                    OPERATOR,
+                    "kubectl",
+                    "--namespace",
+                    targets.namespace,
+                    "get",
+                    "pods",
+                    "--selector",
+                    f"app={targets.node_daemonset}",
+                    "-o",
+                    NODE_NAME_PROJECTION,
+                )
+            ],
+            {
+                "step": "coverage",
+                "minimum": targets.node_count,
+                "total_equals": targets.node_count,
+            },
+            rationale="A hardened fleet with one unenrolled node is an unhardened fleet.",
+        ),
+    ]
+
+
+def _build(targets: Targets) -> list[CheckSpec]:
+    pipeline = f"{targets.remote}/main:{targets.builder_pipeline}"
+    return [
+        check(
+            "build.builder-service-account-is-tokenless",
+            "The image builder service account mounts no API token",
+            "build",
+            "stdout_matches",
+            [
+                step(
+                    "serviceaccount",
+                    OPERATOR,
+                    "kubectl",
+                    "--namespace",
+                    targets.builder_namespace,
+                    "get",
+                    "serviceaccount",
+                    targets.builder_serviceaccount,
+                    "-o",
+                    projection("jsonpath={.automountServiceAccountToken}"),
+                )
+            ],
+            {"step": "serviceaccount", "equals": "false"},
+        ),
+        check(
+            "build.builder-runs-unprivileged-with-exact-capabilities",
+            "The build pod is unprivileged and adds only the expected capabilities",
+            "build",
+            "names_present",
+            [step("pipeline", OPERATOR, "git", "show", pipeline, record=False)],
+            {
+                "step": "pipeline",
+                "names": (
+                    targets.builder_capabilities,
+                    "privileged: false",
+                    "allowPrivilegeEscalation: false",
+                    "automountServiceAccountToken: false",
+                    'drop: ["ALL"]',
+                ),
+            },
+            rationale="Exact lines, not a substring sweep: a widened capability set has to change one of these lines to take effect.",
+        ),
+        check(
+            "build.harbor-immutability-rule-is-applied",
+            "The Harbor immutable-tag rule for the agent image has been applied",
+            "build",
+            "lines_match",
+            [
+                step(
+                    "job",
+                    OPERATOR,
+                    "kubectl",
+                    "--namespace",
+                    targets.harbor_namespace,
+                    "get",
+                    "job",
+                    targets.harbor_immutability_job,
+                    "-o",
+                    projection("jsonpath={.status.succeeded}"),
+                )
+            ],
+            {"step": "job", "pattern": r"^[1-9][0-9]*$", "minimum": 1},
+            rationale="The job verifies the rule it creates, so a completed run is evidence the rule matched the expected contract.",
+        ),
+        _release_lineage(targets),
+    ]
+
+
+def _release_lineage(targets: Targets) -> CheckSpec:
+    """Bind the exact reviewed release from Git through the Ready pods."""
+    container = targets.agent_container
+    namespace = targets.namespace
+    deployment = f"deploy/{targets.agent_deployment}"
+    deployment_projection = projection(
+        (
+            "jsonpath={.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}"
+            "{.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}"
+            "{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}"
+            f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}'
+        ).replace("'", '"')
+    )
+    pod_projection = projection(
+        (
+            "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.status.phase}{'\\t'}"
+            "{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}"
+            f'{{.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}'
+            f'{{.status.containerStatuses[?(@.name=="{container}")].imageID}}{{"\\t"}}'
+            "{.metadata.labels.pod-template-hash}{'\\n'}{end}"
+        ).replace("'", '"')
+    )
+    replicaset_projection = projection(
+        (
+            "jsonpath={range .items[*]}{.metadata.name}{'\\t'}"
+            "{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}"
+            "{.status.readyReplicas}{'\\t'}{.status.availableReplicas}{'\\t'}"
+            f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}'
+            "{.metadata.labels.pod-template-hash}{'\\n'}{end}"
+        ).replace("'", '"')
+    )
+    flux_projection = projection(
+        (
+            "jsonpath={.metadata.name}{'\\t'}{.metadata.generation}{'\\t'}"
+            "{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}"
+            "{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}"
+            "{.status.lastAppliedRevision}"
+        ).replace("'", '"')
+    )
+    return check(
+        "release.exact-lineage-is-running",
+        "Remote main, reviewed PR, image, build, Flux, and running revision are identical",
+        "release",
+        "release_lineage",
+        [
+            step(
+                "remote-main",
+                OPERATOR,
+                "git",
+                "ls-remote",
+                targets.remote,
+                "refs/heads/main",
+            ),
+            step(
+                "reviewed-pr",
+                OPERATOR,
+                GITEA_CLIENT,
+                "GET",
+                f"/api/v1/repos/{targets.repo}/pulls/{targets.reviewed_pr_number}",
+            ),
+            step(
+                "build-source",
+                OPERATOR,
+                "git",
+                "rev-parse",
+                f"{targets.remote}/main",
+            ),
+            step(
+                "deployment",
+                OPERATOR,
+                "kubectl",
+                "--namespace",
+                namespace,
+                "get",
+                deployment,
+                "-o",
+                deployment_projection,
+            ),
+            step(
+                "pods",
+                OPERATOR,
+                "kubectl",
+                "--namespace",
+                namespace,
+                "get",
+                "pods",
+                "--selector",
+                f"app={targets.agent_deployment}",
+                "-o",
+                pod_projection,
+            ),
+            step(
+                "replicasets",
+                OPERATOR,
+                "kubectl",
+                "--namespace",
+                namespace,
+                "get",
+                "replicasets",
+                "--selector",
+                f"app={targets.agent_deployment}",
+                "-o",
+                replicaset_projection,
+            ),
+            step(
+                "flux",
+                OPERATOR,
+                "kubectl",
+                "--namespace",
+                "flux-system",
+                "get",
+                "kustomization/hermes",
+                "-o",
+                flux_projection,
+            ),
+        ],
+        {
+            "main_sha": targets.remote_main_sha,
+            "head_sha": targets.reviewed_head_sha,
+            "head_ref": targets.reviewed_head_ref,
+            "pr_number": targets.reviewed_pr_number,
+            "image": targets.agent_image,
+            "build_sha": targets.build_sha,
+            "deployment_revision": targets.deployment_revision,
+        },
+        rationale="A PASS requires one exact release identity across every source and running object.",
+    )
+
+
+def _reliability(targets: Targets) -> list[CheckSpec]:
+    return [
+        check(
+            "reliability.finalization-and-replay-patches-are-deployed",
+            "The agent workload runs the finalization, replay, and session patches",
+            "reliability",
+            "names_present",
+            [
+                step(
+                    "init",
+                    OPERATOR,
+                    "kubectl",
+                    "--namespace",
+                    targets.namespace,
+                    "get",
+                    f"deploy/{targets.agent_deployment}",
+                    "-o",
+                    INIT_NAME_PROJECTION,
+                )
+            ],
+            {"step": "init", "names": targets.agent_init_containers},
+            rationale="The regression suites live in the repository; this asserts the fixes they cover are actually running.",
+        ),
+        check(
+            "reliability.regression-suites-are-present-on-main",
+            "The decomposition and worker-recovery regression suites are on main",
+            "reliability",
+            "names_present",
+            [
+                step(
+                    "suites",
+                    OPERATOR,
+                    "git",
+                    "ls-tree",
+                    "--name-only",
+                    f"{targets.remote}/main",
+                    "testing/tests/",
+                )
+            ],
+            {
+                "step": "suites",
+                "names": (
+                    "testing/tests/test_hermes_cli_lanes.py",
+                    "testing/tests/test_hermes_coordinator.py",
+                    "testing/tests/test_hermes_worker_recovery.py",
+                ),
+            },
+        ),
+    ]
+
+
+def delivery_checks(targets: Targets) -> list[CheckSpec]:
+    """Return the forge, node, build, and reliability checks."""
+    return [
+        *_forge(targets),
+        *_nodes(targets),
+        *_build(targets),
+        *_reliability(targets),
+    ]

scripts/ops/hermes_handoff_checks_platform.py

@@ -0,0 +1,387 @@
+#!/usr/bin/env python3
+"""Acceptance checks for the release baseline, provider identity, and routing.
+
+These are the facts a handoff rests on before any authority question is asked:
+that the tree under test is the merged one, that both providers authenticate
+through the subscriptions they are supposed to, that Switchyard actually decided
+the routes it claims to, and that chat, agent, and triage are still three
+distinct scopes rather than one surface wearing three hostnames.
+"""
+
+from __future__ import annotations
+
+from hermes_handoff_catalog import (
+    CLAUDE_HEALTH,
+    CLAUDE_HEALTH_FIELDS,
+    CODEX_HEALTH,
+    CODEX_HEALTH_FIELDS,
+    ENV_NAME_PROJECTION,
+    OPERATOR,
+    PROVIDER_API_KEY_NAMES,
+    ROUTING_LOG,
+    SELF,
+    SWITCHYARD,
+    Targets,
+    check,
+    step,
+)
+from hermes_handoff_model import CheckSpec
+from hermes_handoff_policy import GITEA_CLIENT, projection, shell
+
+
+def _baseline(targets: Targets) -> list[CheckSpec]:
+    api = f"/api/v1/repos/{targets.repo}"
+    dependency_heads = dict(targets.dependency_heads)
+    checks = [
+        check(
+            "baseline.local-origin-main-is-current",
+            "The local origin/main ref equals the fixed remote-main SHA",
+            "baseline",
+            "stdout_matches",
+            [step("main", OPERATOR, "git", "rev-parse", f"{targets.remote}/main")],
+            {"step": "main", "equals": targets.remote_main_sha},
+            rationale="The ancestry check is meaningful only after the local tracking ref is bound to the remote observation.",
+        ),
+        check(
+            "baseline.origin-main-descends-merged-work",
+            "origin/main descends from the merged Hermes isolation baseline",
+            "baseline",
+            "allowed",
+            [
+                step(
+                    "ancestor",
+                    OPERATOR,
+                    "git",
+                    "merge-base",
+                    "--is-ancestor",
+                    targets.baseline_commit,
+                    f"{targets.remote}/main",
+                )
+            ],
+            rationale="Everything downstream assumes the merged worker-isolation baseline is present.",
+        ),
+        check(
+            "baseline.pending-pull-requests-recorded",
+            "Exact open pull-request heads are recorded, not assumed",
+            "baseline",
+            "json_record",
+            [
+                step(
+                    "open",
+                    OPERATOR,
+                    GITEA_CLIENT,
+                    "GET",
+                    f"{api}/pulls?state=open&limit=50",
+                )
+            ],
+            {
+                "step": "open",
+                "record": {
+                    "numbers": "[].number",
+                    "head_refs": "[].head.ref",
+                    "head_shas": "[].head.sha",
+                    "base_shas": "[].base.sha",
+                    "drafts": "[].draft",
+                    "mergeable": "[].mergeable",
+                },
+            },
+            rationale="The runbook has to name the heads it was verified against; stale heads are how a release verifies the wrong tree.",
+        ),
+        check(
+            "baseline.operator-checkout-clean",
+            "The operator checkout has no uncommitted drift",
+            "baseline",
+            "stdout_matches",
+            [step("status", OPERATOR, "git", "status", "--porcelain")],
+            {"step": "status", "equals": ""},
+            mandatory=False,
+            rationale="Advisory: a dirty checkout does not invalidate cluster evidence, but it does muddy what was compared.",
+        ),
+    ]
+    checks += [
+        check(
+            f"baseline.dependency-pr-{number}-merged",
+            f"Pull request #{number} is merged into main",
+            "baseline",
+            "json_field",
+            [step("pr", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls/{number}")],
+            {
+                "step": "pr",
+                "fields": {
+                    "merged": True,
+                    "base.ref": "main",
+                    "head.sha": dependency_heads.get(number, ""),
+                },
+            },
+            rationale="This harness certifies the post-merge platform; an unmerged dependency means it is measuring something else.",
+        )
+        for number in targets.dependency_pull_requests
+    ]
+    return checks
+
+
+def _identity(targets: Targets) -> list[CheckSpec]:
+    codex = shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS)
+    claude = shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS)
+    agent = f"deploy/{targets.agent_deployment}"
+    return [
+        check(
+            "identity.codex-is-chatgpt-subscription",
+            "Codex authenticates through the ChatGPT subscription, not an API key",
+            "identity",
+            "json_field",
+            [step("health", SELF, *codex)],
+            {
+                "step": "health",
+                "fields": {
+                    "authenticated": True,
+                    "transport": "codex-chatgpt-subscription",
+                    "state": "available",
+                },
+            },
+            rationale="Only allow-listed status fields are read; the credential itself is never opened.",
+        ),
+        check(
+            "identity.codex-evidence-is-fresh",
+            "Codex authentication evidence is current",
+            "identity",
+            "json_recent",
+            [step("health", SELF, *codex)],
+            {
+                "step": "health",
+                "now": targets.now,
+                "fields": {"checked_at": targets.max_evidence_age_seconds},
+            },
+            rationale="Stale provider evidence describes a state the release no longer has.",
+        ),
+        check(
+            "identity.claude-is-firstparty-subscription",
+            "Claude authenticates as a claude.ai first-party subscription",
+            "identity",
+            "json_field",
+            [step("health", SELF, *claude)],
+            {
+                "step": "health",
+                "fields": {
+                    "authenticated": True,
+                    "api_provider": "firstParty",
+                    "auth_method": "claude.ai",
+                    "transport": "claude-code-cli-subscription",
+                    "state": "available",
+                },
+            },
+        ),
+        check(
+            "identity.claude-evidence-is-fresh",
+            "Claude authentication evidence is current",
+            "identity",
+            "json_recent",
+            [step("health", SELF, *claude)],
+            {
+                "step": "health",
+                "now": targets.now,
+                "fields": {"checked_at": targets.max_evidence_age_seconds},
+            },
+        ),
+        check(
+            "identity.no-provider-api-key-in-pod",
+            "No provider API-key variable is set in the running agent process",
+            "identity",
+            "names_absent",
+            [step("env", SELF, *shell("env_names"))],
+            {
+                "step": "env",
+                "names": PROVIDER_API_KEY_NAMES,
+                "contains": ("API_KEY",),
+            },
+            rationale="Names only. The probe lists variable names and never their values.",
+        ),
+        check(
+            "identity.no-provider-api-key-in-manifest",
+            "No provider API-key variable is declared on the agent workload",
+            "identity",
+            "names_absent",
+            [
+                step(
+                    "env",
+                    OPERATOR,
+                    "kubectl",
+                    "--namespace",
+                    targets.namespace,
+                    "get",
+                    agent,
+                    "-o",
+                    ENV_NAME_PROJECTION,
+                )
+            ],
+            {"step": "env", "names": PROVIDER_API_KEY_NAMES, "contains": ("API_KEY",)},
+            rationale="The operator counterpart to the in-pod probe: desired state and running state are asserted separately.",
+        ),
+    ]
+
+
+def _routing(targets: Targets) -> list[CheckSpec]:
+    switchyard = f"deploy/{targets.switchyard_deployment}"
+    tail = shell("tail_lines", path=ROUTING_LOG, limit=str(targets.routing_tail_lines))
+    routing_step = step(
+        "routing", SWITCHYARD, *tail, record=False, max_bytes=targets.routing_tail_bytes
+    )
+    return [
+        check(
+            "routing.switchyard-is-available",
+            "Switchyard has a ready replica serving every model-call boundary",
+            "routing",
+            "lines_match",
+            [
+                step(
+                    "deployment",
+                    OPERATOR,
+                    "kubectl",
+                    "--namespace",
+                    targets.namespace,
+                    "get",
+                    switchyard,
+                    "-o",
+                    projection("jsonpath={.status.readyReplicas}"),
+                )
+            ],
+            {"step": "deployment", "pattern": r"^[1-9][0-9]*$", "minimum": 1},
+        ),
+        check(
+            "routing.provider-effort-and-fallback-evidence",
+            "Routing evidence covers both providers, all graded efforts, and a real fallback",
+            "routing",
+            "routing_evidence",
+            [routing_step],
+            {
+                "step": "routing",
+                "providers": ("codex", "claude"),
+                "efforts": ("medium", "high", "xhigh"),
+                "lanes": ("route", "worker"),
+                "require_fallback_evidence": True,
+                "max_age_seconds": targets.max_evidence_age_seconds,
+                "now": targets.now,
+            },
+            rationale="The routing log records the decided route per boundary; the tail is parsed here and never pasted into the report.",
+        ),
+        check(
+            "routing.codex-latency-recorded",
+            "Codex boundary latency is measured, not assumed",
+            "routing",
+            "json_numeric",
+            [
+                step(
+                    "health",
+                    SELF,
+                    *shell(
+                        "json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS
+                    ),
+                )
+            ],
+            {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}},
+        ),
+        check(
+            "routing.claude-latency-recorded",
+            "Claude boundary latency is measured, not assumed",
+            "routing",
+            "json_numeric",
+            [
+                step(
+                    "health",
+                    SELF,
+                    *shell(
+                        "json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS
+                    ),
+                )
+            ],
+            {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}},
+        ),
+    ]
+
+
+def _scope_profile(
+    targets: Targets, name: str, workload: str, container: str, expected: str
+) -> CheckSpec:
+    profile_projection = projection(
+        f'jsonpath={{.spec.template.spec.containers[?(@.name=="{container}")]'
+        '.env[?(@.name=="HERMES_AUTO_ROUTER_PROFILE")].value}'
+    )
+    return check(
+        f"scopes.{name}-profile-is-distinct",
+        f"The {name} surface runs under its own routing profile",
+        "scopes",
+        "stdout_matches",
+        [
+            step(
+                "profile",
+                OPERATOR,
+                "kubectl",
+                "--namespace",
+                targets.namespace,
+                "get",
+                workload,
+                "-o",
+                profile_projection,
+            )
+        ],
+        {"step": "profile", "equals": expected},
+        rationale="Chat, agent, and triage must not collapse into one scope; each states its own profile.",
+    )
+
+
+def _scopes(targets: Targets) -> list[CheckSpec]:
+    return [
+        _scope_profile(
+            targets,
+            "agent",
+            f"deploy/{targets.agent_deployment}",
+            targets.agent_container,
+            "agent",
+        ),
+        _scope_profile(
+            targets,
+            "chat",
+            f"statefulset/{targets.chat_statefulset}",
+            targets.chat_container,
+            "chat",
+        ),
+        _scope_profile(
+            targets,
+            "triage",
+            f"deploy/{targets.triage_deployment}",
+            targets.triage_container,
+            "triage",
+        ),
+        check(
+            "scopes.surfaces-have-distinct-hosts",
+            "Chat, agent, and triage are published on distinct hostnames",
+            "scopes",
+            "distinct_count",
+            [
+                step(
+                    "hosts",
+                    OPERATOR,
+                    "kubectl",
+                    "--namespace",
+                    targets.namespace,
+                    "get",
+                    "ingress",
+                    "-o",
+                    projection(
+                        'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}'
+                    ),
+                )
+            ],
+            {"step": "hosts", "minimum": 3},
+        ),
+    ]
+
+
+def platform_checks(targets: Targets) -> list[CheckSpec]:
+    """Return the baseline, identity, routing, and scope checks."""
+    return [
+        *_baseline(targets),
+        *_identity(targets),
+        *_routing(targets),
+        *_scopes(targets),
+    ]

scripts/ops/hermes_handoff_checks_workers.py

@@ -0,0 +1,376 @@
+#!/usr/bin/env python3
+"""Acceptance checks for the distributed worker pool and the visible surfaces.
+
+A pool is only distributed if its workers occupy distinct nodes and distinct
+volumes and leave state ownership with the coordinator; a replica count alone
+satisfies none of that. A surface is only working if the work it did is visible
+in it, which is why the session and Kanban checks look at the same data the UI
+renders rather than at whether a pod is Running.
+
+The ephemeral entries are placeholders in a default run. They report NOT_RUN
+until the mutation mode is separately armed, and the armed mode replaces them
+with real, mandatory results.
+"""
+
+from __future__ import annotations
+
+from hermes_handoff_catalog import (
+    CHAT,
+    INIT_NAME_PROJECTION,
+    OPERATOR,
+    POOL_PROJECTION,
+    SELF,
+    Targets,
+    check,
+)
+from hermes_handoff_catalog import step as step
+from hermes_handoff_model import EPHEMERAL, CheckSpec
+from hermes_handoff_policy import projection
+
+KANBAN_STATUSES = [
+    "archived",
+    "blocked",
+    "done",
+    "ready",
+    "review",
+    "running",
+    "scheduled",
+    "todo",
+    "triage",
+]
+
+
+def _pool(targets: Targets) -> list[CheckSpec]:
+    selector = targets.pool_selector
+    return [
+        check(
+            "pool.three-workers-have-distinct-nodes-and-volumes",
+            "Exactly three Ready ordinal workers occupy distinct nodes and claims",
+            "pool",
+            "pool_topology",
+            [
+                step(
+                    "statefulset",
+                    OPERATOR,
+                    "kubectl",
+                    "--namespace",
+                    targets.namespace,
+                    "get",
+                    f"statefulset/{targets.pool_statefulset}",
+                    "-o",
+                    projection(
+                        "jsonpath={.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}{.status.currentReplicas}{'\\t'}{.spec.template.spec.automountServiceAccountToken}".replace(
+                            "'", '"'
+                        )
+                    ),
+                ),
+                step(
+                    "workers",
+                    OPERATOR,
+                    "kubectl",
+                    "--namespace",
+                    targets.namespace,
+                    "get",
+                    "pods",
+                    "--selector",
+                    selector,
+                    "-o",
+                    POOL_PROJECTION,
+                ),
+            ],
+            {
+                "state_step": "statefulset",
+                "worker_step": "workers",
+                "replicas": targets.pool_replicas,
+                "name": targets.pool_statefulset,
+            },
+            rationale="Replica count, Ready state, ordinal coverage, node placement, PVC isolation, and tokenlessness are one fail-closed assertion.",
+        ),
+        check(
+            "pool.coordinator-retains-sole-state-ownership",
+            "No pool worker mounts the coordinator's durable state",
+            "pool",
+            "names_absent",
+            [
+                step(
+                    "claims",
+                    OPERATOR,
+                    "kubectl",
+                    "--namespace",
+                    targets.namespace,
+                    "get",
+                    f"statefulset/{targets.pool_statefulset}",
+                    "-o",
+                    projection(
+                        "jsonpath={range .spec.template.spec.volumes[*]}{.name}{'='}"
+                        "{.persistentVolumeClaim.claimName}{'\\n'}{end}".replace(
+                            "'", '"'
+                        )
+                    ),
+                )
+            ],
+            {"step": "claims", "contains": targets.coordinator_claims},
+            rationale="Two writers on one SQLite state file is the duplicate-work and lost-result failure this pool exists to avoid. One line per template volume keeps a drifted or empty projection NOT_RUN instead of a silent pass.",
+        ),
+        *_pool_env(targets),
+    ]
+
+
+def _pool_env(targets: Targets) -> list[CheckSpec]:
+    """Return the mandatory assignment-safety configuration check."""
+    return [
+        check(
+            "pool.assignment-safety-knobs-are-configured",
+            "Pool workers carry the lease and de-duplication configuration",
+            "pool",
+            "names_present",
+            [
+                step(
+                    "env",
+                    OPERATOR,
+                    "kubectl",
+                    "--namespace",
+                    targets.namespace,
+                    "get",
+                    f"statefulset/{targets.pool_statefulset}",
+                    "-o",
+                    projection(
+                        'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}'
+                    ),
+                )
+            ],
+            {"step": "env", "names": targets.pool_worker_env},
+            rationale="Deployed configuration only; the stale-claim and duplicate-dispatch behaviour itself is covered by the pool's own regression suite.",
+        ),
+        check(
+            "pool.assignment-fieldrefs-are-bound",
+            "Worker node and ordinal identities come from fixed downward-API fields",
+            "pool",
+            "names_present",
+            [
+                step(
+                    "source",
+                    OPERATOR,
+                    "git",
+                    "show",
+                    f"{targets.remote}/main:services/hermes/execution-worker-statefulset.yaml",
+                    record=False,
+                )
+            ],
+            {
+                "step": "source",
+                "names": (
+                    "name: HERMES_WORKER_ORDINAL",
+                    "fieldPath: metadata.labels['apps.kubernetes.io/pod-index']",
+                    "name: HERMES_WORKER_NODE",
+                    "fieldPath: spec.nodeName",
+                ),
+            },
+            rationale="This prevents caller-authored ordinal/node values from impersonating another worker assignment.",
+        ),
+    ]
+
+
+def _surfaces(targets: Targets) -> list[CheckSpec]:
+    checks = [
+        check(
+            "surfaces.kanban-activity-is-visible",
+            "Durable Kanban activity is visible to the agent surface",
+            "surfaces",
+            "all_of_field",
+            [
+                step(
+                    "tasks",
+                    SELF,
+                    "hermes",
+                    "kanban",
+                    "list",
+                    "--json",
+                    record=False,
+                    max_bytes=targets.listing_bytes,
+                )
+            ],
+            {"step": "tasks", "key_field": "id", "fields": {"status": KANBAN_STATUSES}},
+            rationale="Non-empty and well-formed. An empty board is NOT_RUN, because it cannot distinguish a quiet week from a broken read.",
+        ),
+        check(
+            "surfaces.durable-sessions-render-without-blank-rows",
+            "Durable sessions are listed with a preview, a source, and an id",
+            "surfaces",
+            "lines_match",
+            [
+                step(
+                    "sessions",
+                    SELF,
+                    "hermes",
+                    "sessions",
+                    "list",
+                    "--limit",
+                    "20",
+                    record=False,
+                )
+            ],
+            {"step": "sessions", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
+            rationale="A row whose leading column is blank is the session-render regression seen from the data side.",
+        ),
+        check(
+            "surfaces.agent-ui-session-patches-are-deployed",
+            "The agent surface runs the session-activity rendering patches",
+            "surfaces",
+            "names_present",
+            [
+                step(
+                    "init",
+                    OPERATOR,
+                    "kubectl",
+                    "--namespace",
+                    targets.namespace,
+                    "get",
+                    f"deploy/{targets.agent_deployment}",
+                    "-o",
+                    INIT_NAME_PROJECTION,
+                )
+            ],
+            {
+                "step": "init",
+                "names": ("patch-web-session-activity", "patch-api-server-sessions"),
+            },
+            rationale="Blank session rows were a rendering regression; this asserts the patches that fixed it are in the running pod.",
+        ),
+        check(
+            "surfaces.chat-session-patches-are-deployed",
+            "The chat surface runs the API-session continuity patches",
+            "surfaces",
+            "names_present",
+            [
+                step(
+                    "init",
+                    OPERATOR,
+                    "kubectl",
+                    "--namespace",
+                    targets.namespace,
+                    "get",
+                    f"statefulset/{targets.chat_statefulset}",
+                    "-o",
+                    INIT_NAME_PROJECTION,
+                )
+            ],
+            {"step": "init", "names": targets.chat_init_containers},
+        ),
+        check(
+            "surfaces.chat-telegram-topic-state-is-durable",
+            "Telegram topic and tenant state is held on durable storage",
+            "surfaces",
+            "names_present",
+            [
+                step(
+                    "claims",
+                    OPERATOR,
+                    "kubectl",
+                    "--namespace",
+                    targets.namespace,
+                    "get",
+                    f"deploy/{targets.chat_router_deployment}",
+                    "-o",
+                    projection(
+                        'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}'
+                    ),
+                )
+            ],
+            {"step": "claims", "contains": ("state",)},
+            rationale="Topic selection survives a router restart only if the tenant state file is on a claim rather than in the pod.",
+        ),
+    ]
+    checks.append(
+        check(
+            "surfaces.chat-telegram-sessions-are-continuous",
+            "The selected chat tenant still holds durable Telegram-sourced sessions",
+            "surfaces",
+            "lines_match",
+            [
+                step(
+                    "telegram",
+                    CHAT,
+                    "hermes",
+                    "sessions",
+                    "list",
+                    "--source",
+                    "telegram",
+                    "--limit",
+                    "20",
+                    record=False,
+                )
+            ],
+            {"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
+            rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.",
+        )
+    )
+    checks.append(
+        check(
+            "surfaces.chat-runs-the-telegram-topic-revision",
+            "The chat surface runs the durable Telegram topic configuration",
+            "surfaces",
+            "stdout_matches",
+            [
+                step(
+                    "revision",
+                    OPERATOR,
+                    "kubectl",
+                    "--namespace",
+                    targets.namespace,
+                    "get",
+                    f"statefulset/{targets.chat_statefulset}",
+                    "-o",
+                    projection(
+                        "jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}"
+                    ),
+                )
+            ],
+            {"step": "revision", "equals": targets.chat_config_revision},
+            rationale="Telegram topic continuity is carried by the config revision the pod template pins.",
+        )
+    )
+    return checks
+
+
+def _ephemeral() -> list[CheckSpec]:
+    return [
+        check(
+            f"ephemeral.{name}",
+            title,
+            "ephemeral",
+            "not_armed",
+            [],
+            {"reason": reason},
+            mandatory=False,
+            scope=EPHEMERAL,
+            rationale="Mutating checks are separately armed. A default run reports them NOT_RUN and never touches the forge.",
+        )
+        for name, title, reason in (
+            (
+                "feature-branch-push",
+                "A unique ephemeral feature branch can be pushed and removed",
+                "ephemeral mutation mode is not armed",
+            ),
+            (
+                "draft-pull-request",
+                "A draft pull request can be opened and closed against the ephemeral branch",
+                "ephemeral mutation mode is not armed",
+            ),
+            (
+                "protected-branch-refusal",
+                "Arming against a protected branch is refused before any network call",
+                "ephemeral mutation mode is not armed",
+            ),
+            (
+                "cleanup-verified",
+                "Every ephemeral ref and pull request is removed and the removal is verified",
+                "ephemeral mutation mode is not armed",
+            ),
+        )
+    ]
+
+
+def worker_checks(targets: Targets) -> list[CheckSpec]:
+    """Return the pool, surface, and ephemeral-placeholder checks."""
+    return [*_pool(targets), *_surfaces(targets), *_ephemeral()]

scripts/ops/hermes_handoff_ephemeral.py

@@ -0,0 +1,439 @@
+#!/usr/bin/env python3
+"""Discoverable, residue-surfacing, self-cleaning armed ephemeral run.
+
+The create response — not the index — is the authoritative name of a pull
+request this harness made, and the index is read to its last page. When either
+source is uncertain the run fails closed and reports the exact residue plus the
+exact commands that remove it, so an operator is never left guessing what was
+created.
+
+Arming, provenance, and push-target rules live in :mod:`hermes_handoff_arming`
+and are re-exported here for callers that treat armed mode as one surface.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+from hermes_handoff_arming import (
+    CONFIRMATION,
+    REF_PREFIX,
+    ArmingError,
+    ArmRequest,
+    assert_push_target_allowed,
+    bounded_read,
+    normalise_branch,
+    preflight,
+)
+from hermes_handoff_exec import Outcome, Runner, Vantage
+from hermes_handoff_model import EPHEMERAL, FAIL, NOT_RUN, PASS, CheckResult, CheckSpec
+from hermes_handoff_policy import (
+    EXPECTED_BASE,
+    GITEA_CLIENT,
+    register_ephemeral_pull,
+)
+from hermes_handoff_rules import strict_json
+
+__all__ = [
+    "CONFIRMATION",
+    "DISCOVERY_PAGE_BUDGET",
+    "DISCOVERY_PAGE_SIZE",
+    "REF_PREFIX",
+    "ArmRequest",
+    "ArmingError",
+    "assert_push_target_allowed",
+    "bounded_read",
+    "normalise_branch",
+    "preflight",
+    "run_armed",
+]
+
+DRAFT_TITLE_PREFIX = "WIP: "
+DISCOVERY_PAGE_SIZE = 50
+DISCOVERY_PAGE_BUDGET = 20
+
+
+def _spec(identifier: str, title: str) -> CheckSpec:
+    return CheckSpec(
+        id=f"ephemeral.{identifier}",
+        title=title,
+        group="ephemeral",
+        rule="armed",
+        steps=(),
+        mandatory=True,
+        scope=EPHEMERAL,
+    )
+
+
+def _result(
+    identifier: str,
+    title: str,
+    status: str,
+    reason: str,
+    outcomes: Sequence[Outcome] = (),
+    evidence: dict[str, Any] | None = None,
+) -> CheckResult:
+    return CheckResult(
+        spec=_spec(identifier, title),
+        status=status,
+        reason=reason,
+        evidence=evidence or {},
+        outcomes=list(outcomes),
+    )
+
+
+def _protected_refusal_result() -> CheckResult:
+    accepted: list[str] = []
+    for candidate in ("main", "master", "refs/heads/main", "HEAD", f"{REF_PREFIX}/x"):
+        try:
+            assert_push_target_allowed(candidate)
+        except ArmingError:
+            continue
+        accepted.append(candidate)
+    return _result(
+        "protected-branch-refusal",
+        "Arming against a protected branch is refused before any network call",
+        FAIL if accepted else PASS,
+        f"guard accepted {accepted}"
+        if accepted
+        else "every protected and malformed target was refused",
+        evidence={"accepted": accepted},
+    )
+
+
+def _detail(outcome: Outcome) -> str:
+    return (
+        outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}"
+    )
+
+
+def _pr_problems(payload: Any, request: ArmRequest) -> list[str]:
+    if not isinstance(payload, dict):
+        return ["pull request is not an object"]
+    wanted = {
+        "state": "open",
+        "draft": True,
+        "merged": False,
+        "base": {"ref": EXPECTED_BASE, "sha": request.expected_base_sha},
+        "head": {"ref": request.ref, "sha": request.expected_head},
+    }
+    problems: list[str] = []
+    if (
+        not isinstance(payload.get("number"), int)
+        or isinstance(payload.get("number"), bool)
+        or payload["number"] <= 0
+    ):
+        problems.append("number is malformed")
+    for field in ("state", "draft", "merged"):
+        if (
+            type(payload.get(field)) is not type(wanted[field])
+            or payload.get(field) != wanted[field]
+        ):
+            problems.append(f"{field} does not match")
+    for group, values in (("base", wanted["base"]), ("head", wanted["head"])):
+        observed = payload.get(group)
+        if not isinstance(observed, dict):
+            problems.append(f"{group} is malformed")
+            continue
+        for field, value in values.items():
+            if observed.get(field) != value:
+                problems.append(f"{group}.{field} does not match")
+    return problems
+
+
+def _created_number(outcome: Outcome, request: ArmRequest) -> tuple[int, str]:
+    """Return the number the create response itself reports.
+
+    The create response is the only source that names a pull request the harness
+    definitely made. Relying on the index alone means an index that fails or
+    answers ambiguously leaves an unnamed, unclosed pull request behind.
+    """
+    if not outcome.ok:
+        return (0, f"pull-request creation failed: {_detail(outcome)}")
+    try:
+        payload = strict_json(outcome.stdout)
+    except ValueError as exc:
+        return (0, f"pull-request creation returned malformed JSON: {exc}")
+    problems = _pr_problems(payload, request)
+    if problems:
+        return (0, f"created pull request is invalid: {', '.join(problems)}")
+    return (payload["number"], "")
+
+
+def _index_pages(
+    runner: Runner, vantage: Vantage, api: str, reserve: float
+) -> tuple[list[Outcome], list[Any], str]:
+    """Read every page of the pull index; a truncated page can hide a match."""
+    outcomes: list[Outcome] = []
+    rows: list[Any] = []
+    for page in range(1, DISCOVERY_PAGE_BUDGET + 1):
+        outcome = runner.run(
+            (
+                GITEA_CLIENT,
+                "GET",
+                f"{api}/pulls?state=all&limit={DISCOVERY_PAGE_SIZE}&page={page}",
+            ),
+            vantage,
+            reserve_seconds=reserve,
+        )
+        outcomes.append(outcome)
+        if not outcome.ok:
+            return (outcomes, [], f"discovery page {page} failed: {_detail(outcome)}")
+        try:
+            payload = strict_json(outcome.stdout)
+        except ValueError as exc:
+            return (outcomes, [], f"discovery page {page} is malformed JSON: {exc}")
+        if not isinstance(payload, list):
+            return (outcomes, [], f"discovery page {page} did not return an array")
+        rows.extend(payload)
+        if len(payload) < DISCOVERY_PAGE_SIZE:
+            return (outcomes, rows, "")
+    return (outcomes, [], "pull-request discovery exceeded its page budget")
+
+
+def _residue(request: ArmRequest, numbers: Sequence[int], api: str) -> dict[str, Any]:
+    """Name the exact residue and the exact commands that remove it by hand."""
+    manual = [
+        f"{GITEA_CLIENT} PATCH {api}/pulls/{number} --field state=closed"
+        for number in numbers
+    ]
+    if not numbers:
+        manual.append(
+            f"{GITEA_CLIENT} GET {api}/pulls?state=all&limit={DISCOVERY_PAGE_SIZE}"
+            f" then close by hand every pull whose head ref is {request.ref}"
+        )
+    manual.append(f"{GITEA_CLIENT} DELETE {api}/branches/{request.ref}")
+    return {"residue_ref": request.ref, "manual_cleanup": manual}
+
+
+def _match_pulls(rows: Sequence[Any], request: ArmRequest) -> tuple[list[int], str]:
+    matches: list[int] = []
+    for index, item in enumerate(rows):
+        if (
+            not isinstance(item, dict)
+            or not isinstance(item.get("head"), dict)
+            or not isinstance(item.get("base"), dict)
+        ):
+            return ([], f"pull-request discovery row {index + 1} is malformed")
+        if item["head"].get("ref") != request.ref:
+            continue
+        problems = _pr_problems(item, request)
+        if problems:
+            return ([], f"discovered pull request is invalid: {', '.join(problems)}")
+        matches.append(item["number"])
+    if len(matches) != 1:
+        return (
+            matches,
+            f"expected one exact draft pull request, discovered {len(matches)}",
+        )
+    return (matches, "")
+
+
+def _closed_pr_problems(payload: Any, request: ArmRequest) -> list[str]:
+    """Validate the same exact PR identity with state changed to closed."""
+    problems = _pr_problems(payload, request)
+    if isinstance(payload, dict) and payload.get("state") == "closed":
+        problems = [
+            problem for problem in problems if problem != "state does not match"
+        ]
+    elif "state does not match" not in problems:
+        problems.append("state is not closed")
+    return problems
+
+
+def _cleanup(
+    runner: Runner,
+    vantage: Vantage,
+    request: ArmRequest,
+    api: str,
+    numbers: Sequence[int],
+    discovery_error: str = "",
+) -> CheckResult:
+    outcomes: list[Outcome] = []
+    problems = [discovery_error] if discovery_error else []
+    for number in sorted(set(numbers)):
+        closed = runner.run(
+            (GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"),
+            vantage,
+        )
+        outcomes.append(closed)
+        if not closed.ok:
+            problems.append(f"pull request #{number} close failed")
+            continue
+        try:
+            payload = strict_json(closed.stdout)
+        except ValueError:
+            problems.append(f"pull request #{number} close response is malformed")
+            continue
+        expected = _closed_pr_problems(payload, request)
+        if expected:
+            problems.append(f"pull request #{number} close response does not match")
+    deleted = runner.run(
+        (GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage
+    )
+    outcomes.append(deleted)
+    if not deleted.ok:
+        problems.append("ephemeral branch deletion failed")
+    remaining = runner.run(
+        ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage
+    )
+    outcomes.append(remaining)
+    if not remaining.ok or remaining.stdout.strip():
+        problems.append("ephemeral branch absence could not be verified")
+    for number in sorted(set(numbers)):
+        observed = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage)
+        outcomes.append(observed)
+        if not observed.ok:
+            problems.append(f"pull request #{number} closure could not be verified")
+            continue
+        try:
+            payload = strict_json(observed.stdout)
+        except ValueError:
+            problems.append(f"pull request #{number} verification is malformed")
+            continue
+        expected = _closed_pr_problems(payload, request)
+        if expected:
+            problems.append(f"pull request #{number} is not exactly closed")
+    return _result(
+        "cleanup-verified",
+        "Every ephemeral ref and pull request is removed and the removal is verified",
+        FAIL if problems else PASS,
+        "; ".join(problems)
+        if problems
+        else "the exact branch and draft pull request are gone",
+        outcomes,
+        {"ref": request.ref, "pull_requests": sorted(set(numbers))},
+    )
+
+
+def _nothing_created(branch: CheckResult, draft_reason: str) -> list[CheckResult]:
+    """Report the two results that follow a run which created no artifact."""
+    return [
+        branch,
+        _result(
+            "draft-pull-request",
+            "A draft pull request can be opened and closed",
+            NOT_RUN,
+            draft_reason,
+        ),
+        _result(
+            "cleanup-verified",
+            "Every ephemeral artifact is removed",
+            NOT_RUN,
+            "nothing was created",
+        ),
+    ]
+
+
+def run_armed(
+    runner: Runner, vantage: Vantage, request: ArmRequest
+) -> list[CheckResult]:
+    """Create, discover, validate, and clean one exact ephemeral branch/PR."""
+    guard = _protected_refusal_result()
+    branch_title = "A unique ephemeral feature branch can be pushed and removed"
+    if guard.status != PASS:
+        unrun = _result("feature-branch-push", branch_title, NOT_RUN, "guard failed")
+        return [guard, *_nothing_created(unrun, "protected-ref guard failed")]
+    api = f"/api/v1/repos/{request.repo}"
+    reserve = min(6000.0, max(300.0, runner.command_timeout * 10))
+    existing = runner.run(
+        ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"),
+        vantage,
+        reserve_seconds=reserve,
+    )
+    if not existing.ok or existing.stdout.strip():
+        reason = (
+            _detail(existing) if not existing.ok else "the ephemeral ref already exists"
+        )
+        refused = _result("feature-branch-push", branch_title, FAIL, reason, [existing])
+        return [guard, *_nothing_created(refused, "branch preflight failed")]
+    local_head = runner.run(
+        ("git", "rev-parse", "HEAD"), vantage, reserve_seconds=reserve
+    )
+    pushed = (
+        runner.run(
+            ("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"),
+            vantage,
+            reserve_seconds=reserve,
+        )
+        if local_head.ok and local_head.stdout.strip() == request.expected_head
+        else Outcome(
+            argv=("git", "push"),
+            vantage=vantage.name,
+            error="local HEAD changed after static preflight",
+        )
+    )
+    verified = runner.run(
+        ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"),
+        vantage,
+        reserve_seconds=reserve,
+    )
+    branch_ok = verified.ok and verified.stdout.split() == [
+        request.expected_head,
+        f"refs/heads/{request.ref}",
+    ]
+    branch_result = _result(
+        "feature-branch-push",
+        branch_title,
+        PASS if branch_ok else FAIL,
+        "the exact reviewed HEAD was pushed"
+        if branch_ok
+        else "push could not be verified exactly",
+        [local_head, pushed, verified],
+        {"ref": request.ref, "head": request.expected_head},
+    )
+    numbers: list[int] = []
+    problems: list[str] = ["branch creation was not verified"]
+    create_outcomes: list[Outcome] = []
+    if branch_ok:
+        created = runner.run(
+            (
+                GITEA_CLIENT,
+                "POST",
+                f"{api}/pulls",
+                "--field",
+                f"title={DRAFT_TITLE_PREFIX}Hermes handoff acceptance ephemeral probe",
+                "--field",
+                f"head={request.ref}",
+                "--field",
+                f"base={EXPECTED_BASE}",
+                "--field",
+                "body=Ephemeral acceptance probe. Closed and deleted by the harness.",
+            ),
+            vantage,
+            reserve_seconds=reserve,
+        )
+        number, create_error = _created_number(created, request)
+        pages, rows, index_error = _index_pages(runner, vantage, api, reserve)
+        found, match_error = (
+            _match_pulls(rows, request) if not index_error else ([], "")
+        )
+        create_outcomes = [created, *pages]
+        # Either source alone names a pull request this run definitely made: the
+        # ephemeral ref was proven absent in preflight, so any pull on it is ours.
+        numbers = sorted({item for item in (number, *found) if item})
+        problems = [text for text in (index_error, match_error) if text]
+        if number and found and found != [number]:
+            problems.append(f"created #{number} is not the discovered {found}")
+        if not numbers:
+            problems.append(
+                create_error or "no exact draft pull request was identified"
+            )
+        for item in numbers:
+            register_ephemeral_pull(item)
+    exact = len(numbers) == 1 and not problems
+    draft_result = _result(
+        "draft-pull-request",
+        "A draft pull request can be opened and closed against the ephemeral branch",
+        PASS if exact else FAIL,
+        f"discovered exact draft pull request #{numbers[0]}"
+        if exact
+        else "; ".join(problems),
+        create_outcomes,
+        {"pull_requests": numbers}
+        if exact
+        else {"pull_requests": numbers, **_residue(request, numbers, api)},
+    )
+    cleanup = _cleanup(runner, vantage, request, api, numbers, "; ".join(problems))
+    return [guard, branch_result, draft_result, cleanup]

scripts/ops/hermes_handoff_evaluators.py

@@ -0,0 +1,497 @@
+#!/usr/bin/env python3
+"""Text-shaped evaluators for the handoff acceptance harness.
+
+These rules cover evidence that arrives as a refusal, an exit status, or a list
+of names. The ``denied`` rule is the load-bearing one: it refuses to classify on
+an authorization review alone — ``kubectl auth can-i`` reports what RBAC says,
+not what the API server does — so a real request must have been made and
+refused before it will pass.
+
+An absence is only evidence when the probe observed something, so a name step
+that exits 0 with no lines is ``NOT_RUN``: drift produces silence, not a pass.
+
+Importing this module also registers the JSON-shaped rules, so a caller only has
+to import one place to have the full catalog vocabulary available.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from hermes_handoff_exec import Outcome
+from hermes_handoff_json_rules import evaluate_flux_health  # noqa: F401  (registers rules)
+from hermes_handoff_model import ATTEMPT, FAIL, NOT_RUN, PASS, REVIEW, CheckSpec
+from hermes_handoff_rules import (
+    EVALUATORS,
+    Evaluation,
+    dotted,
+    evaluate,
+    evaluator,
+    exact_equal,
+    lines,
+    looks_denied,
+    missing_steps,
+    step_keys,
+    strict_json,
+)
+
+__all__ = ["EVALUATORS", "Evaluation", "evaluate", "evaluator"]
+
+
+@evaluator("denied")
+def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
+    """Pass only when the review says no and a real request was refused."""
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    attempts = step_keys(spec, ATTEMPT)
+    if not attempts:
+        return Evaluation(
+            NOT_RUN, "deny checks require a real attempt, not only a review"
+        )
+    evidence: dict[str, Any] = {}
+    for key in step_keys(spec, REVIEW):
+        verdict = outcomes[key].stdout.strip().lower()
+        evidence[key] = verdict
+        if verdict != "no":
+            return Evaluation(
+                FAIL,
+                f"authorization review {key} reported {verdict or 'nothing'}",
+                evidence,
+            )
+    for key in attempts:
+        outcome = outcomes[key]
+        if outcome.ok:
+            return Evaluation(
+                FAIL, f"attempt {key} succeeded but must be refused", evidence
+            )
+        if not looks_denied(outcome):
+            return Evaluation(
+                NOT_RUN,
+                f"attempt {key} failed without a recognisable refusal (rc={outcome.returncode})",
+                evidence,
+            )
+        evidence[key] = "refused"
+    return Evaluation(PASS, "review and live attempt were both refused", evidence)
+
+
+@evaluator("review_denied")
+def evaluate_review_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
+    """Pass only when every side-effect-free authorization review says no."""
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    reviews = step_keys(spec, REVIEW)
+    if not reviews:
+        return Evaluation(NOT_RUN, "a denial check requires an authorization review")
+    evidence: dict[str, str] = {}
+    for key in reviews:
+        verdict = outcomes[key].stdout.strip().lower()
+        evidence[key] = verdict
+        if verdict != "no":
+            return Evaluation(
+                FAIL,
+                f"authorization review {key} reported {verdict or 'nothing'}",
+                evidence,
+            )
+    return Evaluation(PASS, "every authorization review was denied", evidence)
+
+
+@evaluator("allowed")
+def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
+    """Pass when every required step succeeded and every review said yes."""
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    evidence: dict[str, Any] = {}
+    for step in spec.steps:
+        outcome = outcomes[step.key]
+        if step.kind == REVIEW:
+            verdict = outcome.stdout.strip().lower()
+            evidence[step.key] = verdict
+            if verdict != "yes":
+                return Evaluation(
+                    FAIL,
+                    f"authorization review {step.key} reported {verdict or 'nothing'}",
+                    evidence,
+                )
+            continue
+        if not outcome.ok:
+            return Evaluation(
+                FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence
+            )
+        evidence[step.key] = "succeeded"
+    return Evaluation(PASS, "required access is present", evidence)
+
+
+def _matched(observed: set[str], patterns: tuple[str, ...]) -> set[str]:
+    return {name for pattern in patterns for name in observed if pattern in name}
+
+
+def _line_step(
+    spec: CheckSpec, outcomes: dict[str, Outcome]
+) -> tuple[set[str], str] | Evaluation:
+    """Return a step's distinct lines, or the NOT_RUN that silence must produce."""
+    key = spec.expect["step"]
+    outcome = outcomes[key]
+    if not outcome.ok:
+        return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
+    observed = set(lines(outcome))
+    if not observed:
+        return Evaluation(NOT_RUN, f"step {key} exited 0 with no lines to examine")
+    return (observed, key)
+
+
+@evaluator("names_absent")
+def evaluate_names_absent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
+    """Pass when a step observed names and none of them is forbidden."""
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    resolved = _line_step(spec, outcomes)
+    if isinstance(resolved, Evaluation):
+        return resolved
+    observed, _ = resolved
+    offenders = sorted(
+        (observed & set(spec.expect.get("names", ())))
+        | _matched(observed, tuple(spec.expect.get("contains", ())))
+    )
+    evidence = {"observed_count": len(observed), "offenders": offenders}
+    if offenders:
+        return Evaluation(
+            FAIL, f"forbidden names present: {', '.join(offenders)}", evidence
+        )
+    return Evaluation(PASS, "no forbidden names are present", evidence)
+
+
+@evaluator("names_present")
+def evaluate_names_present(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
+    """Pass when every required name, and every required substring, appears."""
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    resolved = _line_step(spec, outcomes)
+    if isinstance(resolved, Evaluation):
+        return resolved
+    observed, _ = resolved
+    missing = sorted(set(spec.expect.get("names", ())) - observed)
+    missing += sorted(
+        pattern
+        for pattern in spec.expect.get("contains", ())
+        if not any(pattern in name for name in observed)
+    )
+    evidence = {"observed_count": len(observed), "missing": missing}
+    if missing:
+        return Evaluation(
+            FAIL, f"required names absent: {', '.join(missing)}", evidence
+        )
+    return Evaluation(PASS, "every required name is present", evidence)
+
+
+@evaluator("stdout_matches")
+def evaluate_stdout_matches(
+    spec: CheckSpec, outcomes: dict[str, Outcome]
+) -> Evaluation:
+    """Compare a step's trimmed stdout against an expected value or set."""
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    key = spec.expect["step"]
+    outcome = outcomes[key]
+    if not outcome.ok:
+        return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
+    actual = outcome.stdout.strip()
+    expected = spec.expect["equals"]
+    allowed = expected if isinstance(expected, list) else [expected]
+    evidence = {"observed": actual}
+    if actual not in allowed:
+        return Evaluation(
+            FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence
+        )
+    return Evaluation(PASS, "output matches the expected value", evidence)
+
+
+@evaluator("distinct_count")
+def evaluate_distinct_count(
+    spec: CheckSpec, outcomes: dict[str, Outcome]
+) -> Evaluation:
+    """Pass when a step yields enough distinct rows — and, optionally, exactly N.
+
+    Distinctness is the point for placement checks: three worker pods pinned to
+    one node satisfy a replica count and defeat the isolation the replica count
+    was standing in for.
+    """
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    key = spec.expect["step"]
+    outcome = outcomes[key]
+    if not outcome.ok:
+        return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
+    values = lines(outcome)
+    distinct = sorted(set(values))
+    minimum = spec.expect["minimum"]
+    evidence = {
+        "total": len(values),
+        "distinct": len(distinct),
+        "values": distinct[:16],
+    }
+    if not values:
+        return Evaluation(NOT_RUN, f"step {key} produced no rows to count", evidence)
+    if len(distinct) < minimum:
+        return Evaluation(
+            FAIL,
+            f"{len(distinct)} distinct values, expected at least {minimum}",
+            evidence,
+        )
+    total_equals = spec.expect.get("total_equals")
+    if total_equals is not None and len(values) != total_equals:
+        return Evaluation(
+            FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence
+        )
+    return Evaluation(PASS, f"{len(distinct)} distinct values observed", evidence)
+
+
+@evaluator("lines_match")
+def evaluate_lines_match(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
+    """Pass when every data row of a tabular listing matches a shape.
+
+    Some surfaces only speak tables. A blank leading column is exactly what the
+    session-render regression looked like, so asserting the shape of each row is
+    a real check rather than a proxy for one.
+    """
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    key = spec.expect["step"]
+    outcome = outcomes[key]
+    if not outcome.ok:
+        return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
+    pattern = re.compile(spec.expect["pattern"])
+    rows = outcome.stdout.splitlines()[spec.expect.get("skip", 0) :]
+    rows = [row for row in rows if row.strip()]
+    offenders = [row[:60] for row in rows if not pattern.search(row)]
+    evidence = {"rows": len(rows), "offenders": offenders[:8]}
+    if len(rows) < spec.expect.get("minimum", 1):
+        return Evaluation(
+            NOT_RUN,
+            f"step {key} produced {len(rows)} data rows, expected at least "
+            f"{spec.expect.get('minimum', 1)}",
+            evidence,
+        )
+    if offenders:
+        return Evaluation(
+            FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence
+        )
+    return Evaluation(PASS, f"all {len(rows)} rows match the expected shape", evidence)
+
+
+@evaluator("vantages_agree")
+def evaluate_vantages_agree(
+    spec: CheckSpec, outcomes: dict[str, Outcome]
+) -> Evaluation:
+    """Pass only when two independent observations report the same fact.
+
+    A disagreement is a finding in its own right: it usually means the manifest
+    on record and the workload actually running have drifted apart, and either
+    one alone would have looked clean.
+    """
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    left_key, right_key = spec.expect["steps"]
+    observations: dict[str, list[str]] = {}
+    for key in (left_key, right_key):
+        outcome = outcomes[key]
+        if not outcome.ok:
+            return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
+        observations[key] = sorted(lines(outcome))
+    evidence = {key: value[:16] for key, value in observations.items()}
+    if observations[left_key] != observations[right_key]:
+        return Evaluation(
+            FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence
+        )
+    if not observations[left_key]:
+        return Evaluation(
+            NOT_RUN, "both vantages returned nothing to compare", evidence
+        )
+    return Evaluation(PASS, "both vantages report the same state", evidence)
+
+
+@evaluator("not_armed")
+def evaluate_not_armed(spec: CheckSpec, _outcomes: dict[str, Outcome]) -> Evaluation:
+    """Report the fixed NOT_RUN a default, read-only run must produce."""
+    reason = spec.expect.get("reason", "ephemeral mutation mode is not armed")
+    return Evaluation(NOT_RUN, reason, {"armed": False})
+
+
+# This validator is one compact, linear state machine so every exit observes
+# only fully validated state.
+# fmt: off
+@evaluator("pool_topology")
+def evaluate_pool_topology(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
+    """Require exactly three Ready ordinal workers on distinct nodes and claims."""
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    state = outcomes[spec.expect["state_step"]].stdout.strip().split("\t")
+    if len(state) != 4:
+        return Evaluation(NOT_RUN, "StatefulSet projection is malformed")
+    expected = spec.expect.get("replicas", 3)
+    try:
+        replicas, ready, current = (int(value) for value in state[:3])
+    except ValueError:
+        return Evaluation(NOT_RUN, "StatefulSet replica fields are malformed")
+    evidence: dict[str, Any] = {"replicas": replicas, "ready": ready, "current": current,
+                                "automountServiceAccountToken": state[3]}
+    if (replicas, ready, current, state[3].lower()) != (expected, expected, expected, "false"):
+        return Evaluation(FAIL, "worker replica or tokenless state does not match the contract", evidence)
+    workers: dict[int, tuple[str, str]] = {}
+    names: list[str] = []
+    for number, row in enumerate(lines(outcomes[spec.expect["worker_step"]]), 1):
+        columns = row.split("\t")
+        if len(columns) != 5:
+            return Evaluation(NOT_RUN, f"worker row {number} is malformed", evidence)
+        name, node, phase, ready_state, claims = columns
+        match = re.fullmatch(rf"{re.escape(spec.expect['name'])}-(\d+)", name)
+        claim_list = [claim for claim in claims.split(",") if claim]
+        if (not match or not node or phase != "Running" or ready_state != "True"
+                or len(claim_list) != 1):
+            return Evaluation(FAIL, f"worker row {number} is not a Ready isolated worker", evidence)
+        ordinal = int(match.group(1))
+        if ordinal in workers:
+            return Evaluation(FAIL, f"worker ordinal {ordinal} is duplicated", evidence)
+        workers[ordinal] = (node, claim_list[0])
+        names.append(name)
+    evidence.update({"workers": sorted(names), "nodes": sorted({v[0] for v in workers.values()}),
+                     "claims": sorted({v[1] for v in workers.values()})})
+    if set(workers) != set(range(expected)):
+        return Evaluation(FAIL, f"worker ordinals are not exactly 0..{expected - 1}", evidence)
+    if (len({v[0] for v in workers.values()}) != expected
+            or len({v[1] for v in workers.values()}) != expected):
+        return Evaluation(FAIL, "workers do not occupy distinct nodes and claims", evidence)
+    return Evaluation(PASS, "three Ready workers have distinct nodes, ordinals, and claims", evidence)
+# fmt: on
+
+
+_SHA_RE = re.compile(r"[0-9a-f]{40}\Z")
+_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}\Z")
+_IMAGE_RE = re.compile(
+    r"[^@\s]+:git-([0-9a-f]{40})-build-([1-9][0-9]*)@(sha256:[0-9a-f]{64})\Z"
+)
+
+
+def _strict_object(outcome: Outcome, label: str) -> dict[str, Any] | Evaluation:
+    try:
+        payload = strict_json(outcome.stdout)
+    except ValueError as exc:
+        return Evaluation(NOT_RUN, f"{label} JSON is malformed: {exc}")
+    if not isinstance(payload, dict):
+        return Evaluation(NOT_RUN, f"{label} JSON is not an object")
+    return payload
+
+
+# fmt: off
+@evaluator("release_lineage")
+def evaluate_release_lineage(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
+    """Atomically bind Git, PR, image, build, Flux, and running revision evidence."""
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    expected = spec.expect
+    main_sha = expected["main_sha"]
+    head_sha = expected["head_sha"]
+    image = expected["image"]
+    build_sha = expected["build_sha"]
+    revision = str(expected["deployment_revision"])
+    digest = image.rsplit("@", 1)[-1]
+    image_match = _IMAGE_RE.fullmatch(image)
+    if not (_SHA_RE.fullmatch(main_sha) and _SHA_RE.fullmatch(head_sha)
+            and _SHA_RE.fullmatch(build_sha) and _DIGEST_RE.fullmatch(digest)
+            and image_match and image_match.group(1) == build_sha and build_sha == main_sha):
+        return Evaluation(NOT_RUN, "release lineage expectations are malformed")
+    remote = outcomes["remote-main"].stdout.strip().split()
+    if remote != [main_sha, "refs/heads/main"]:
+        return Evaluation(FAIL, "remote main does not match the fixed expected SHA",
+                          {"remote_main": remote[:2]})
+    pr = _strict_object(outcomes["reviewed-pr"], "reviewed PR")
+    if isinstance(pr, Evaluation):
+        return pr
+    fields = {
+        "number": expected["pr_number"],
+        "state": "open",
+        "draft": True,
+        "merged": False,
+        "base.ref": "main",
+        "base.sha": main_sha,
+        "head.ref": expected["head_ref"],
+        "head.sha": head_sha,
+    }
+    for path, wanted in fields.items():
+        try:
+            actual = dotted(pr, path)
+        except (KeyError, ValueError, IndexError):
+            return Evaluation(NOT_RUN, f"reviewed PR is missing {path}")
+        if not exact_equal(actual, wanted):
+            return Evaluation(FAIL, f"reviewed PR {path} does not match", {path: actual})
+    if outcomes["build-source"].stdout.strip() != build_sha:
+        return Evaluation(FAIL, "build source SHA does not match the fixed release build SHA")
+    deployment = outcomes["deployment"].stdout.strip().split("\t")
+    if len(deployment) != 6:
+        return Evaluation(NOT_RUN, "Deployment projection is malformed")
+    generation, observed, replicas, ready, deployed_revision, deployed_image = deployment
+    try:
+        current = (int(generation) > 0 and generation == observed
+                   and int(replicas) > 0 and replicas == ready)
+    except ValueError:
+        return Evaluation(NOT_RUN, "Deployment numeric fields are malformed")
+    if not current or (deployed_revision, deployed_image) != (revision, image):
+        return Evaluation(FAIL, "Deployment lineage does not match the release contract")
+    replica_hash = ""
+    for number, row in enumerate(lines(outcomes["replicasets"]), 1):
+        columns = row.split("\t")
+        if len(columns) != 6:
+            return Evaluation(NOT_RUN, f"ReplicaSet row {number} is malformed")
+        _, rs_revision, rs_ready, rs_available, rs_image, rs_hash = columns
+        if rs_revision != revision:
+            continue
+        if (rs_ready != replicas or rs_available != replicas or rs_image != image or not rs_hash):
+            return Evaluation(FAIL, "active ReplicaSet does not match the Deployment lineage")
+        if replica_hash:
+            return Evaluation(FAIL, "more than one ReplicaSet claims the running revision")
+        replica_hash = rs_hash
+    if not replica_hash:
+        return Evaluation(FAIL, "running Deployment revision has no matching ReplicaSet")
+    pod_rows = lines(outcomes["pods"])
+    if len(pod_rows) != int(replicas):
+        return Evaluation(FAIL, "running pod count does not match the Deployment")
+    pods: list[str] = []
+    for number, row in enumerate(pod_rows, 1):
+        columns = row.split("\t")
+        if len(columns) != 6:
+            return Evaluation(NOT_RUN, f"pod row {number} is malformed")
+        name, phase, ready_state, pod_image, image_id, pod_hash = columns
+        if phase != "Running" or ready_state != "True":
+            return Evaluation(FAIL, f"pod {name or number} is not Ready")
+        if (pod_image, pod_hash) != (image, replica_hash) or not image_id.endswith(f"@{digest}"):
+            return Evaluation(FAIL, f"pod {name or number} lineage does not match")
+        pods.append(name)
+    flux = outcomes["flux"].stdout.strip().split("\t")
+    if len(flux) != 6:
+        return Evaluation(NOT_RUN, "Flux lineage projection is malformed")
+    name, generation, observed, suspended, ready_state, applied = flux
+    if (not name or generation != observed or suspended.lower() == "true" or ready_state != "True"):
+        return Evaluation(FAIL, "Flux source is not current and Ready")
+    match = re.search(r"sha1:([0-9a-f]{40})\Z", applied)
+    if not match or match.group(1) != main_sha:
+        return Evaluation(FAIL, "Flux applied revision is not the expected remote main")
+    return Evaluation(PASS, "Git, PR, image, build, Flux, and running revision are bound",
+                      {"main_sha": main_sha, "head_sha": head_sha, "image_digest": digest,
+                       "build_sha": build_sha, "deployment_revision": revision, "pods": pods})
+# fmt: on
+
+
+# Two observations of the same fact agree whether or not they came from
+# different vantages; the dual-vantage case is just the important one.
+EVALUATORS["steps_agree"] = evaluate_vantages_agree

scripts/ops/hermes_handoff_exec.py

@@ -0,0 +1,482 @@
+#!/usr/bin/env python3
+"""Bounded, screened, provenance-attested command execution.
+
+It bounds pipes, times out and reaps the process group, and discards truncated output.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import os
+import selectors
+import shlex
+import shutil
+import signal
+import stat
+import subprocess
+import time
+from collections.abc import Callable, Mapping, Sequence
+from contextlib import suppress
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from hermes_handoff_policy import GITEA_CLIENT, READ_ONLY, PolicyError, check_argv
+from hermes_handoff_redaction import (
+    DEFAULT_MAX_BYTES,
+    TRUNCATION_NOTE,
+    redact,
+    safe_text,
+    scrub,
+)
+
+DEFAULT_COMMAND_TIMEOUT = 25.0
+DEFAULT_RUN_DEADLINE = 900.0
+DEFAULT_CLEANUP_RESERVE = 90.0
+MAX_COMMAND_TIMEOUT = 600.0
+MAX_RUN_DEADLINE = 7200.0
+MAX_OUTPUT_BYTES = 8 * 1024 * 1024
+READ_CHUNK = 64 * 1024
+PIPE_DRAIN_SECONDS = 0.25
+
+SAFE_PATH = "/opt/data/tools/bin:/opt/hermes/.venv/bin:/opt/coordinator:/usr/local/bin:/usr/bin:/bin"
+# These exact allowlists are intentionally dense: reviewers compare them as a
+# single provenance boundary rather than as extensible application config.
+# fmt: off
+ENV_PASSTHROUGH = ("KUBECONFIG", "LANG", "LC_ALL", "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE")
+FIXED_ENVIRONMENT = {"GIT_ASKPASS": "/opt/coordinator/gitea_askpass.sh",
+                     "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_NOSYSTEM": "1",
+                     "GIT_TERMINAL_PROMPT": "0", "GITEA_BASE_URL": "https://scm.bstein.dev",
+                     "LC_ALL": "C", "PATH": SAFE_PATH}
+EXPECTED_PATHS = {
+    "kubectl": {"/opt/data/tools/bin/kubectl", "/usr/local/bin/kubectl", "/usr/bin/kubectl"},
+    "git": {"/usr/bin/git", "/bin/git"},
+    "hermes": {"/opt/hermes/.venv/bin/hermes"},
+    "sh": {"/usr/bin/sh", "/bin/sh"},
+    GITEA_CLIENT: {GITEA_CLIENT},
+}
+EXPECTED_SHA256 = {
+    "kubectl": {"3d514dbae5dc8c09f773df0ef0f5d449dfad05b3aca5c96b13565f886df345fd"},
+    "git": {"a0e562e4bd3c4c79379e91d8c07a10104b2cefe8fac966dc6bd4874a57a807f3"},
+    "sh": {"367967c823a0c391e5049b15a67c6a0a629c88b9b6dcdca75ef13ac9d65334b1"},
+    "hermes": {"c8419290ef7f1a95f59eadcea9f58f446b1e6535a7f4ffee9f1c0cb1172335e4"},
+    GITEA_CLIENT: {"f0943db4e81ce96908e94b0f0e78aa4b5ef95d6c4f7803f9f74a43e761576d85"},
+}
+
+DEADLINE_ERROR = "deadline-exceeded"
+TIMEOUT_ERROR = "timeout"
+OPERATOR = "operator"
+SELF = "self"
+CONTEXT_AWARE_BINARIES = ("kubectl",)
+POD_COMMAND_PATHS = {"kubectl": "/usr/local/bin/kubectl", "hermes": "/opt/hermes/.venv/bin/hermes",
+                     "git": "/usr/bin/git", "sh": "/bin/sh", GITEA_CLIENT: GITEA_CLIENT}
+# fmt: on
+
+
+@dataclass(frozen=True)
+class Attestation:
+    """The exact outer executable used for a probe."""
+
+    path: str
+    sha256: str
+
+
+@dataclass(frozen=True)
+class Capture:
+    """Raw bounded process result returned by an executor implementation."""
+
+    returncode: int
+    stdout: bytes = b""
+    stderr: bytes = b""
+    stdout_truncated: bool = False
+    stderr_truncated: bool = False
+    timed_out: bool = False
+
+
+@dataclass(frozen=True)
+class Outcome:
+    """The screened, bounded result of one probe command."""
+
+    argv: tuple[str, ...]
+    vantage: str
+    returncode: int | None = None
+    stdout: str = ""
+    stderr: str = ""
+    truncated: bool = False
+    duration_ms: int = 0
+    error: str | None = None
+    executable_path: str = ""
+    executable_sha256: str = ""
+
+    @property
+    def ok(self) -> bool:
+        return self.error is None and self.returncode == 0 and not self.truncated
+
+    @property
+    def ran(self) -> bool:
+        return self.error is None
+
+    @property
+    def combined(self) -> str:
+        return "\n".join(part for part in (self.stdout, self.stderr) if part)
+
+    def as_dict(self) -> dict[str, object]:
+        return scrub(
+            {
+                "command": redact(shlex.join(self.argv)),
+                "vantage": self.vantage,
+                "returncode": self.returncode,
+                "stdout": self.stdout,
+                "stderr": self.stderr,
+                "truncated": self.truncated,
+                "duration_ms": self.duration_ms,
+                "error": self.error,
+                "executable_path": self.executable_path,
+                "executable_sha256": self.executable_sha256,
+            }
+        )
+
+
+@dataclass(frozen=True)
+class Vantage:
+    """A named identity and transport used to observe a probe."""
+
+    name: str
+    prefix: tuple[str, ...] = ()
+    inserts: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
+    env: Mapping[str, str] = field(default_factory=dict)
+    command_paths: Mapping[str, str] = field(default_factory=dict)
+    description: str = ""
+
+    def wrap(self, argv: Sequence[str]) -> tuple[str, ...]:
+        command = tuple(argv)
+        if command:
+            replacement = self.command_paths.get(command[0])
+            if replacement:
+                command = (replacement, *command[1:])
+            extra = self.inserts.get(command[0].rsplit("/", 1)[-1])
+            if extra:
+                command = (command[0], *extra, *command[1:])
+        return (*self.prefix, *command)
+
+
+def _finite_number(name: str, value: Any, minimum: float, maximum: float) -> float:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        raise ValueError(f"{name} must be numeric")
+    converted = float(value)
+    if not math.isfinite(converted) or not minimum <= converted <= maximum:
+        raise ValueError(
+            f"{name} must be finite and between {minimum:g} and {maximum:g}"
+        )
+    return converted
+
+
+def _byte_budget(value: Any) -> int:
+    if (
+        isinstance(value, bool)
+        or not isinstance(value, int)
+        or not 1 <= value <= MAX_OUTPUT_BYTES
+    ):
+        raise ValueError(
+            f"max_bytes must be an integer between 1 and {MAX_OUTPUT_BYTES}"
+        )
+    return value
+
+
+def build_environment(
+    vantage: Vantage, source: Mapping[str, str] | None = None
+) -> dict[str, str]:
+    """Build a deterministic child environment without caller-controlled PATH."""
+    origin = os.environ if source is None else source
+    environment = {name: origin[name] for name in ENV_PASSTHROUGH if name in origin}
+    environment.update(FIXED_ENVIRONMENT)
+    environment.update(vantage.env)
+    environment["PATH"] = SAFE_PATH
+    environment["GITEA_BASE_URL"] = FIXED_ENVIRONMENT["GITEA_BASE_URL"]
+    environment["GIT_ASKPASS"] = FIXED_ENVIRONMENT["GIT_ASKPASS"]
+    return environment
+
+
+# fmt: off
+def operator_vantage(kubeconfig: str | None = None, context: str | None = None) -> Vantage:
+    inserts = dict.fromkeys(CONTEXT_AWARE_BINARIES, ("--context", context)) if context else {}
+    detail = " ".join(part for part in (f"context={context}" if context else "",
+                      "kubeconfig-pinned" if kubeconfig else "") if part)
+    return Vantage(name=OPERATOR, inserts=inserts,
+                    env={"KUBECONFIG": kubeconfig} if kubeconfig else {},
+                    description=f"operator {detail}".strip())
+
+
+def pod_vantage(namespace: str, pod: str, container: str, operator: Vantage | None = None) -> Vantage:
+    reach = operator.inserts.get("kubectl", ()) if operator else ()
+    return Vantage(name=SELF, prefix=("kubectl", *reach, "exec", "--namespace", namespace,
+                    pod, "--container", container, "--"),
+                    env=dict(operator.env) if operator else {}, command_paths=POD_COMMAND_PATHS,
+                    description=f"in-pod {namespace}/{pod}[{container}]")
+# fmt: on
+
+
+def attest_executable(command: str, environment: Mapping[str, str]) -> Attestation:
+    """Resolve and hash an executable from a fixed, trusted path set."""
+    name = command if command == GITEA_CLIENT else command.rsplit("/", 1)[-1]
+    candidate = (
+        command if "/" in command else shutil.which(command, path=environment["PATH"])
+    )
+    if not candidate:
+        raise OSError(f"trusted executable is unavailable: {name}")
+    resolved = str(Path(candidate).resolve(strict=True))
+    allowed = {
+        str(Path(item).resolve(strict=False))
+        for item in EXPECTED_PATHS.get(name, set())
+    }
+    if resolved not in allowed:
+        raise OSError(f"executable resolved outside its trusted paths: {name}")
+    details = os.stat(resolved)
+    if (
+        not stat.S_ISREG(details.st_mode)
+        or details.st_mode & 0o022
+        or not details.st_mode & 0o111
+    ):
+        raise OSError(f"trusted executable has unsafe ownership mode: {name}")
+    digest = hashlib.sha256()
+    with open(resolved, "rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    observed = digest.hexdigest()
+    expected = EXPECTED_SHA256.get(name)
+    if not expected or observed not in expected:
+        raise OSError(f"executable digest is not release-attested: {name}")
+    return Attestation(resolved, observed)
+
+
+def _append_bounded(buffer: bytearray, chunk: bytes, limit: int) -> bool:
+    remaining = limit + 1 - len(buffer)
+    if remaining > 0:
+        buffer.extend(chunk[:remaining])
+    return len(buffer) > limit or len(chunk) > remaining
+
+
+def _signal_group(process: subprocess.Popen[bytes], signum: int) -> None:
+    with suppress(ProcessLookupError):
+        os.killpg(process.pid, signum)
+
+
+def execute_bounded(
+    argv: Sequence[str],
+    environment: Mapping[str, str],
+    timeout: float,
+    max_bytes: int,
+    clock: Callable[[], float] = time.monotonic,
+    popen: Callable[..., subprocess.Popen[bytes]] = subprocess.Popen,
+) -> Capture:
+    """Execute one process with bounded live pipe capture and group cleanup."""
+    process = popen(
+        list(argv),
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env=dict(environment),
+        text=False,
+        close_fds=True,
+        start_new_session=True,
+    )
+    streams = {"stdout": process.stdout, "stderr": process.stderr}
+    buffers = {"stdout": bytearray(), "stderr": bytearray()}
+    truncated = {"stdout": False, "stderr": False}
+    selector = selectors.DefaultSelector()
+    for name, stream in streams.items():
+        if (
+            stream is not None
+        ):  # pragma: no branch - PIPE was requested for both streams
+            os.set_blocking(stream.fileno(), False)
+            selector.register(stream, selectors.EVENT_READ, name)
+    expires = clock() + timeout
+    timed_out = False
+    try:
+        while process.poll() is None:
+            remaining = expires - clock()
+            if remaining <= 0:
+                timed_out = True
+                break
+            for key, _ in selector.select(min(remaining, 0.1)):
+                try:
+                    chunk = os.read(key.fileobj.fileno(), READ_CHUNK)
+                except BlockingIOError:  # pragma: no cover - readiness race, retry only
+                    continue
+                if not chunk:
+                    selector.unregister(key.fileobj)
+                    continue
+                name = key.data
+                truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes)
+        _signal_group(process, signal.SIGKILL if timed_out else signal.SIGTERM)
+        drain_until = clock() + PIPE_DRAIN_SECONDS
+        while selector.get_map() and clock() < drain_until:
+            for key, _ in selector.select(max(0.0, drain_until - clock())):
+                try:
+                    chunk = os.read(key.fileobj.fileno(), READ_CHUNK)
+                except BlockingIOError:  # pragma: no cover - readiness race, retry only
+                    continue
+                if not chunk:
+                    selector.unregister(key.fileobj)
+                    continue
+                name = key.data
+                truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes)
+        _signal_group(process, signal.SIGKILL)
+        returncode = process.wait(timeout=PIPE_DRAIN_SECONDS)
+    finally:
+        selector.close()
+        for stream in streams.values():
+            if (
+                stream is not None
+            ):  # pragma: no branch - PIPE was requested for both streams
+                stream.close()
+        if (
+            process.poll() is None
+        ):  # pragma: no branch - fail-safe for wait/selector exceptions
+            _signal_group(process, signal.SIGKILL)
+            process.wait()
+    return Capture(
+        returncode=returncode,
+        stdout=bytes(buffers["stdout"][:max_bytes]),
+        stderr=bytes(buffers["stderr"][:max_bytes]),
+        stdout_truncated=truncated["stdout"],
+        stderr_truncated=truncated["stderr"],
+        timed_out=timed_out,
+    )
+
+
+def _injected_executor(
+    spawn: Callable[..., subprocess.CompletedProcess[Any]],
+) -> Callable[..., Capture]:
+    """Adapt deterministic test doubles without weakening the production path."""
+
+    def execute(
+        argv: Sequence[str],
+        environment: Mapping[str, str],
+        timeout: float,
+        max_bytes: int,
+    ) -> Capture:
+        completed = spawn(
+            list(argv),
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            check=False,
+            env=dict(environment),
+        )
+        stdout = str(completed.stdout or "").encode()
+        stderr = str(completed.stderr or "").encode()
+        return Capture(
+            completed.returncode,
+            stdout[:max_bytes],
+            stderr[:max_bytes],
+            len(stdout) > max_bytes,
+            len(stderr) > max_bytes,
+        )
+
+    return execute
+
+
+def _screen_capture(data: bytes, truncated: bool, budget: int) -> str:
+    if truncated:
+        return TRUNCATION_NOTE.encode()[:budget].decode()
+    return safe_text(data.decode("utf-8", "replace"), budget)[0]
+
+
+class Runner:
+    """Run probes under policy, provenance, timeout, deadline, and byte budgets."""
+
+    def __init__(
+        self,
+        mode: str = READ_ONLY,
+        command_timeout: float = DEFAULT_COMMAND_TIMEOUT,
+        deadline_seconds: float = DEFAULT_RUN_DEADLINE,
+        max_bytes: int = DEFAULT_MAX_BYTES,
+        clock: Callable[[], float] = time.monotonic,
+        executor: Callable[..., Capture] | None = None,
+        spawn: Callable[..., subprocess.CompletedProcess[Any]] | None = None,
+        environ: Mapping[str, str] | None = None,
+        attestor: Callable[[str, Mapping[str, str]], Attestation] = attest_executable,
+    ) -> None:
+        self.mode = mode
+        self.command_timeout = _finite_number(
+            "command_timeout", command_timeout, 0.05, MAX_COMMAND_TIMEOUT
+        )
+        deadline = _finite_number(
+            "deadline_seconds", deadline_seconds, 0.05, MAX_RUN_DEADLINE
+        )
+        self.max_bytes = _byte_budget(max_bytes)
+        self._clock = clock
+        self._executor = executor or (
+            _injected_executor(spawn) if spawn else execute_bounded
+        )
+        self._environ = environ
+        self._attestor = attestor
+        self._deadline_at = clock() + deadline
+        self.commands_run = 0
+
+    @property
+    def remaining_seconds(self) -> float:
+        return self._deadline_at - self._clock()
+
+    def run(
+        self,
+        argv: Sequence[str],
+        vantage: Vantage,
+        max_bytes: int | None = None,
+        reserve_seconds: float = 0.0,
+    ) -> Outcome:
+        budget_bytes = self.max_bytes if max_bytes is None else _byte_budget(max_bytes)
+        reserve = _finite_number(
+            "reserve_seconds", reserve_seconds, 0.0, MAX_RUN_DEADLINE
+        )
+        addressed = vantage.wrap(argv)
+        try:
+            check_argv(addressed, self.mode)
+        except PolicyError as exc:
+            return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}")
+        budget = min(self.command_timeout, self.remaining_seconds - reserve)
+        if budget <= 0:
+            return Outcome(argv=addressed, vantage=vantage.name, error=DEADLINE_ERROR)
+        environment = build_environment(vantage, self._environ)
+        try:
+            attestation = self._attestor(addressed[0], environment)
+        except (OSError, ValueError) as exc:
+            return Outcome(
+                argv=addressed, vantage=vantage.name, error=f"attestation: {exc}"
+            )
+        executed = (attestation.path, *addressed[1:])
+        try:
+            check_argv(executed, self.mode)
+        except PolicyError as exc:
+            return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}")
+        started = self._clock()
+        self.commands_run += 1
+        try:
+            capture = self._executor(executed, environment, budget, budget_bytes)
+        except subprocess.TimeoutExpired:
+            capture = Capture(-signal.SIGKILL, timed_out=True)
+        except OSError as exc:
+            return Outcome(
+                argv=addressed,
+                vantage=vantage.name,
+                error=f"spawn: {exc.strerror or type(exc).__name__}",
+                executable_path=attestation.path,
+                executable_sha256=attestation.sha256,
+            )
+        was_truncated = capture.stdout_truncated or capture.stderr_truncated
+        stdout = _screen_capture(capture.stdout, capture.stdout_truncated, budget_bytes)
+        stderr = _screen_capture(capture.stderr, capture.stderr_truncated, budget_bytes)
+        return Outcome(
+            argv=addressed,
+            vantage=vantage.name,
+            returncode=capture.returncode,
+            stdout=stdout,
+            stderr=stderr,
+            truncated=was_truncated,
+            duration_ms=int((self._clock() - started) * 1000),
+            error=TIMEOUT_ERROR if capture.timed_out else None,
+            executable_path=attestation.path,
+            executable_sha256=attestation.sha256,
+        )

scripts/ops/hermes_handoff_json_rules.py

@@ -0,0 +1,474 @@
+#!/usr/bin/env python3
+"""JSON-shaped evaluators for the handoff acceptance harness.
+
+These rules cover the evidence that arrives as structured output — Kubernetes
+objects, forge pull requests, provider health files, and the two append-only
+evidence logs the platform keeps. They share the fail-closed contract of every
+other rule: an absent path, an unparsable payload, or an empty result set is
+``NOT_RUN``, never a quiet pass.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import math
+import re
+from typing import Any
+
+from hermes_handoff_exec import Outcome
+from hermes_handoff_model import FAIL, NOT_RUN, PASS, CheckSpec
+from hermes_handoff_rules import (
+    Evaluation,
+    dotted,
+    evaluator,
+    exact_equal,
+    is_list,
+    missing_steps,
+    parsed_step,
+    strict_json,
+)
+
+_ROUTE_PART_RE = re.compile(r"[a-z0-9][a-z0-9_.-]{0,63}\Z")
+_FLUX_NAME_RE = re.compile(
+    r"[a-z0-9]([-a-z0-9.]*[a-z0-9])?/[a-z0-9]([-a-z0-9.]*[a-z0-9])?\Z"
+)
+
+
+def _timestamp(value: Any) -> dt.datetime:
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError("timestamp is not a non-empty string")
+    observed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
+    if observed.tzinfo is None:
+        raise ValueError("timestamp has no timezone")
+    return observed.astimezone(dt.timezone.utc)
+
+
+def _now(value: Any) -> dt.datetime:
+    if not isinstance(value, dt.datetime) or value.tzinfo is None:
+        raise ValueError("run clock is not timezone-aware")
+    return value.astimezone(dt.timezone.utc)
+
+
+@evaluator("json_field")
+def evaluate_json_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
+    """Compare dotted JSON paths of one step's output against expected values."""
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    parsed = parsed_step(spec, outcomes)
+    if isinstance(parsed, Evaluation):
+        return parsed
+    payload, key = parsed
+    evidence: dict[str, Any] = {}
+    mismatches: list[str] = []
+    for path, expected in spec.expect["fields"].items():
+        try:
+            actual = dotted(payload, path)
+        except (KeyError, ValueError, IndexError):
+            return Evaluation(
+                NOT_RUN, f"path {path} is absent from step {key}", evidence
+            )
+        evidence[path] = actual
+        allowed = expected if isinstance(expected, list) else [expected]
+        if not any(exact_equal(actual, item) for item in allowed):
+            mismatches.append(f"{path}={actual!r} expected one of {allowed!r}")
+    if mismatches:
+        return Evaluation(FAIL, "; ".join(mismatches), evidence)
+    return Evaluation(PASS, "observed fields match the release contract", evidence)
+
+
+@evaluator("json_numeric")
+def evaluate_json_numeric(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
+    """Compare numeric JSON paths against inclusive minimum/maximum bounds."""
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    parsed = parsed_step(spec, outcomes)
+    if isinstance(parsed, Evaluation):
+        return parsed
+    payload, key = parsed
+    evidence: dict[str, Any] = {}
+    problems: list[str] = []
+    for path, bounds in spec.expect["fields"].items():
+        try:
+            actual = dotted(payload, path)
+        except (KeyError, ValueError, IndexError):
+            return Evaluation(
+                NOT_RUN, f"path {path} is absent from step {key}", evidence
+            )
+        if (
+            not isinstance(actual, (int, float))
+            or isinstance(actual, bool)
+            or not math.isfinite(float(actual))
+        ):
+            return Evaluation(
+                NOT_RUN, f"path {path} is not numeric in step {key}", evidence
+            )
+        evidence[path] = actual
+        if "min" in bounds and actual < bounds["min"]:
+            problems.append(f"{path}={actual} below minimum {bounds['min']}")
+        if "max" in bounds and actual > bounds["max"]:
+            problems.append(f"{path}={actual} above maximum {bounds['max']}")
+    if problems:
+        return Evaluation(FAIL, "; ".join(problems), evidence)
+    return Evaluation(PASS, "numeric fields are within the expected bounds", evidence)
+
+
+@evaluator("json_recent")
+def evaluate_json_recent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
+    """Pass when a timestamp field is fresh enough to be evidence about now.
+
+    Stale provider or routing evidence is worse than none: it describes a state
+    the release no longer has. A timestamp older than the allowed age is a FAIL,
+    and one the harness cannot parse is NOT_RUN.
+    """
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    parsed = parsed_step(spec, outcomes)
+    if isinstance(parsed, Evaluation):
+        return parsed
+    payload, key = parsed
+    try:
+        now = _now(spec.expect["now"])
+    except (TypeError, ValueError) as exc:
+        return Evaluation(NOT_RUN, f"invalid run clock: {exc}")
+    evidence: dict[str, Any] = {}
+    stale: list[str] = []
+    for path, max_age in spec.expect["fields"].items():
+        try:
+            raw = dotted(payload, path)
+        except (KeyError, ValueError, IndexError):
+            return Evaluation(
+                NOT_RUN, f"path {path} is absent from step {key}", evidence
+            )
+        try:
+            observed = _timestamp(raw)
+        except (TypeError, ValueError):
+            return Evaluation(
+                NOT_RUN, f"path {path} is not a timezone-aware ISO timestamp", evidence
+            )
+        age = (now - observed).total_seconds()
+        evidence[path] = {"age_seconds": int(age)}
+        if age < 0:
+            stale.append(f"{path} is {abs(int(age))}s in the future")
+        elif age > max_age:
+            stale.append(f"{path} is {int(age)}s old, limit {max_age}s")
+    if stale:
+        return Evaluation(FAIL, "; ".join(stale), evidence)
+    return Evaluation(PASS, "observed timestamps are fresh", evidence)
+
+
+@evaluator("all_of_field")
+def evaluate_all_of_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
+    """Pass when every selected element of a JSON array matches expected fields."""
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    parsed = parsed_step(spec, outcomes)
+    if isinstance(parsed, Evaluation):
+        return parsed
+    payload, key = parsed
+    try:
+        elements = dotted(payload, spec.expect.get("array", ""))
+    except (KeyError, ValueError, IndexError):
+        return Evaluation(NOT_RUN, f"array path is absent from step {key}")
+    if not is_list(elements):
+        return Evaluation(NOT_RUN, f"step {key} did not return an array")
+    key_field = spec.expect.get("key_field", "")
+    wanted = spec.expect.get("keys")
+    exempt = set(spec.expect.get("exempt_keys", ()))
+    seen: list[Any] = []
+    problems: list[str] = []
+    evidence: dict[str, Any] = {}
+    for element in elements:
+        identity = dotted(element, key_field) if key_field else None
+        if wanted is not None and identity not in wanted:
+            continue
+        if identity in exempt:
+            continue
+        seen.append(identity)
+        for path in spec.expect.get("non_empty", ()):
+            try:
+                value = dotted(element, path)
+            except (KeyError, ValueError, IndexError):
+                value = None
+            if value in (None, "", [], {}):
+                problems.append(f"{identity}: {path} is empty")
+        for path, expected in spec.expect.get("fields", {}).items():
+            try:
+                actual = dotted(element, path)
+            except (KeyError, ValueError, IndexError):
+                actual = None
+            allowed = expected if isinstance(expected, list) else [expected]
+            if not any(exact_equal(actual, item) for item in allowed):
+                problems.append(
+                    f"{identity}: {path}={actual!r} expected one of {allowed!r}"
+                )
+    evidence["examined"] = seen[:32]
+    if wanted is not None:
+        absent = [item for item in wanted if item not in seen]
+        if absent:
+            return Evaluation(
+                NOT_RUN, f"expected keys absent from step {key}: {absent}", evidence
+            )
+    elif not seen:
+        return Evaluation(NOT_RUN, f"step {key} returned nothing to examine", evidence)
+    if problems:
+        return Evaluation(FAIL, "; ".join(problems[:8]), evidence)
+    return Evaluation(
+        PASS, f"{len(seen)} element(s) match the release contract", evidence
+    )
+
+
+@evaluator("json_record")
+def evaluate_json_record(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
+    """Capture named JSON paths as evidence; pass when all of them resolve.
+
+    Recording is a release requirement in its own right — the runbook has to
+    name the exact heads it was verified against — so an unreachable source is a
+    NOT_RUN rather than an empty section.
+    """
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    parsed = parsed_step(spec, outcomes)
+    if isinstance(parsed, Evaluation):
+        return parsed
+    payload, key = parsed
+    evidence: dict[str, Any] = {}
+    for label, path in spec.expect["record"].items():
+        try:
+            evidence[label] = dotted(payload, path)
+        except (KeyError, ValueError, IndexError):
+            return Evaluation(
+                NOT_RUN, f"path {path} is absent from step {key}", evidence
+            )
+    return Evaluation(PASS, "state recorded", evidence)
+
+
+@evaluator("routing_evidence")
+def evaluate_routing_evidence(
+    spec: CheckSpec, outcomes: dict[str, Outcome]
+) -> Evaluation:
+    """Summarise a Switchyard routing log tail into provider/effort coverage.
+
+    Route identifiers are ``<lane>/<provider>/<family>/<effort>``, so provider,
+    model family, effort tier, and lane all come from the same field. Fallback
+    and failure evidence is a separate ``fallback_reason`` on the record that
+    was retried. Every row must be complete strict JSON; a clipped or malformed
+    line makes the evidence unusable rather than being silently discarded.
+    """
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    key = spec.expect["step"]
+    outcome = outcomes[key]
+    if not outcome.ok:
+        return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
+    records: list[dict[str, Any]] = []
+    routes: list[list[str]] = []
+    stamps_by_dimension: dict[tuple[str, str], dt.datetime] = {}
+    now_value = spec.expect.get("now")
+    try:
+        now = _now(now_value) if now_value is not None else None
+    except (TypeError, ValueError) as exc:
+        return Evaluation(NOT_RUN, f"invalid routing run clock: {exc}")
+    for row_number, line in enumerate(outcome.stdout.splitlines(), 1):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            record = strict_json(line)
+        except ValueError as exc:
+            return Evaluation(
+                NOT_RUN, f"routing row {row_number} is malformed JSON: {exc}"
+            )
+        if not isinstance(record, dict):
+            return Evaluation(NOT_RUN, f"routing row {row_number} is not an object")
+        try:
+            observed = _timestamp(record.get("ts"))
+        except (TypeError, ValueError) as exc:
+            return Evaluation(
+                NOT_RUN, f"routing row {row_number} has invalid timestamp: {exc}"
+            )
+        if now is not None and observed > now:
+            return Evaluation(FAIL, f"routing row {row_number} has a future timestamp")
+        model = record.get("model")
+        if not isinstance(model, str) or not model:
+            return Evaluation(NOT_RUN, f"routing row {row_number} has no model")
+        route = model.split("/")
+        if len(route) == 4 and all(_ROUTE_PART_RE.fullmatch(part) for part in route):
+            routes.append(route)
+            for label, value in (
+                ("provider", route[1]),
+                ("effort", route[3]),
+                ("lane", route[0]),
+            ):
+                key_dimension = (label, value)
+                stamps_by_dimension[key_dimension] = max(
+                    observed, stamps_by_dimension.get(key_dimension, observed)
+                )
+        elif record.get("tier") != "classifier":
+            return Evaluation(
+                NOT_RUN, f"routing row {row_number} has malformed route {model!r}"
+            )
+        fallback = record.get("fallback_reason")
+        if fallback is not None and (
+            not isinstance(fallback, str) or not fallback.strip()
+        ):
+            return Evaluation(
+                NOT_RUN, f"routing row {row_number} has malformed fallback_reason"
+            )
+        record["_observed"] = observed
+        records.append(record)
+    fallbacks = [record for record in records if record.get("fallback_reason")]
+    stamps = sorted(record["_observed"] for record in records)
+    evidence: dict[str, Any] = {
+        "records": len(records),
+        "routed": len(routes),
+        "lanes": sorted({route[0] for route in routes}),
+        "providers": sorted({route[1] for route in routes}),
+        "families": sorted({route[2] for route in routes}),
+        "efforts": sorted({route[3] for route in routes}),
+        "fallback_events": len(fallbacks),
+        "fallback_reasons": sorted(
+            {str(record["fallback_reason"]) for record in fallbacks}
+        ),
+        "latest_ts": stamps[-1].isoformat() if stamps else "",
+    }
+    if not records:
+        return Evaluation(
+            NOT_RUN, "routing log tail contained no parsable records", evidence
+        )
+    missing = [
+        f"{label}:{item}"
+        for label, wanted in (
+            ("provider", spec.expect.get("providers", ())),
+            ("effort", spec.expect.get("efforts", ())),
+            ("lane", spec.expect.get("lanes", ())),
+        )
+        for item in wanted
+        if item not in evidence[f"{label}s"]
+    ]
+    if spec.expect.get("require_fallback_evidence") and not fallbacks:
+        missing.append("fallback:none-observed")
+    max_age = spec.expect.get("max_age_seconds")
+    if max_age and now:
+        ages: dict[str, int] = {}
+        for label, wanted in (
+            ("provider", spec.expect.get("providers", ())),
+            ("effort", spec.expect.get("efforts", ())),
+            ("lane", spec.expect.get("lanes", ())),
+        ):
+            for item in wanted:
+                stamp = stamps_by_dimension.get((label, item))
+                if stamp is None:
+                    continue
+                age = int((now - stamp).total_seconds())
+                ages[f"{label}:{item}"] = age
+                if age > max_age:
+                    missing.append(
+                        f"freshness:{label}:{item}:{age}s-exceeds-{max_age}s"
+                    )
+        evidence["age_seconds"] = ages
+    if missing:
+        return Evaluation(
+            FAIL, f"routing evidence missing {', '.join(missing)}", evidence
+        )
+    return Evaluation(PASS, "routing evidence covers the required lanes", evidence)
+
+
+@evaluator("flux_health")
+def evaluate_flux_health(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
+    """Classify a compact Flux projection into suspensions and unhealthy objects.
+
+    Each row carries name, generation, observed generation, suspension, and all
+    condition states. A non-suspended object must be current, explicitly
+    ``Ready=True``, not ``Healthy=False/Unknown``, and not reconciling. Suspended
+    objects are judged against the exact expected-suspension list.
+    """
+    blocked = missing_steps(spec, outcomes)
+    if blocked:
+        return blocked
+    key = spec.expect["step"]
+    outcome = outcomes[key]
+    if not outcome.ok:
+        return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
+    expected = set(spec.expect.get("expected_suspensions", ()))
+    suspended: list[str] = []
+    unhealthy: list[str] = []
+    total = 0
+    for row_number, line in enumerate(outcome.stdout.splitlines(), 1):
+        if not line.strip():
+            continue
+        columns = line.split("\t")
+        if len(columns) != 5:
+            return Evaluation(
+                NOT_RUN, f"Flux row {row_number} has {len(columns)} columns"
+            )
+        name, generation, observed_generation, suspend, conditions = columns
+        name = name.strip()
+        if not _FLUX_NAME_RE.fullmatch(name):
+            return Evaluation(
+                NOT_RUN, f"Flux row {row_number} has malformed object name"
+            )
+        try:
+            current_generation = int(generation)
+            reconciled_generation = int(observed_generation)
+        except ValueError:
+            return Evaluation(
+                NOT_RUN, f"Flux row {row_number} has malformed generations"
+            )
+        if current_generation <= 0 or current_generation != reconciled_generation:
+            return Evaluation(
+                FAIL, f"Flux row {row_number} has stale observedGeneration"
+            )
+        if suspend.strip().lower() not in {"", "false", "true"}:
+            return Evaluation(
+                NOT_RUN, f"Flux row {row_number} has malformed suspension state"
+            )
+        total += 1
+        if suspend.strip().lower() == "true":
+            suspended.append(name)
+            continue
+        states: dict[str, str] = {}
+        for pair in filter(None, conditions.strip().strip(",").split(",")):
+            if pair.count("=") != 1:
+                return Evaluation(
+                    NOT_RUN, f"Flux row {row_number} has malformed condition"
+                )
+            condition, state = pair.split("=", 1)
+            if (
+                not condition
+                or state not in {"True", "False", "Unknown"}
+                or condition in states
+            ):
+                return Evaluation(
+                    NOT_RUN, f"Flux row {row_number} has malformed condition state"
+                )
+            states[condition] = state
+        healthy = states.get("Healthy")
+        if (
+            states.get("Ready") != "True"
+            or healthy in {"False", "Unknown"}
+            or states.get("Reconciling") == "True"
+        ):
+            unhealthy.append(name)
+    unexpected = sorted(set(suspended) - expected)
+    evidence = {
+        "total": total,
+        "suspended": sorted(suspended),
+        "unexpected_suspensions": unexpected,
+        "unhealthy": sorted(unhealthy),
+    }
+    if not total:
+        return Evaluation(NOT_RUN, f"step {key} returned no Flux objects", evidence)
+    problems = []
+    if unexpected:
+        problems.append(f"unexpected suspensions: {', '.join(unexpected)}")
+    if unhealthy:
+        problems.append(f"unhealthy: {', '.join(sorted(unhealthy)[:8])}")
+    if problems:
+        return Evaluation(FAIL, "; ".join(problems), evidence)
+    return Evaluation(PASS, f"{total} Flux objects reconciled as expected", evidence)

scripts/ops/hermes_handoff_model.py

@@ -0,0 +1,290 @@
+#!/usr/bin/env python3
+"""Result model, aggregation, and rendering for the handoff acceptance harness.
+
+The classification is deliberately four-valued. ``NOT_APPLICABLE`` is the only
+way a check can be excused, and a check can only reach it by matching a scope
+rule that a reviewer wrote down; anything the harness merely failed to observe
+is ``NOT_RUN``. Because a mandatory ``NOT_RUN`` is a NO_GO exactly like a
+``FAIL``, an unreachable cluster, an exhausted deadline, or a probe nobody
+maintained can never round up to a release.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from hermes_handoff_exec import Outcome
+from hermes_handoff_redaction import contains_credential_shape, safe_text, scrub
+
+PASS = "PASS"
+FAIL = "FAIL"
+NOT_RUN = "NOT_RUN"
+NOT_APPLICABLE = "NOT_APPLICABLE"
+STATUSES = (PASS, FAIL, NOT_RUN, NOT_APPLICABLE)
+
+GO = "GO"
+NO_GO = "NO_GO"
+
+ALWAYS = "always"
+EPHEMERAL = "ephemeral"
+
+HARNESS = "hermes-handoff-acceptance"
+SCHEMA_VERSION = 1
+
+
+REVIEW = "review"
+ATTEMPT = "attempt"
+READ = "read"
+STEP_KINDS = (REVIEW, ATTEMPT, READ)
+
+
+@dataclass(frozen=True)
+class Step:
+    """One command a check needs, and the vantage it must be observed from.
+
+    ``kind`` separates a self access review (``review``) from a real request the
+    identity is expected to be refused (``attempt``). A deny check needs both:
+    an authorization review states policy, only an attempt proves the API server
+    enforces it.
+
+    ``record`` is cleared for bulk evidence — a routing log tail, a large object
+    listing — which the harness parses but must not paste into the artifact.
+    ``max_bytes`` raises the capture budget for that same bulk evidence, where a
+    default-sized truncation would silently narrow the window being examined.
+    """
+
+    key: str
+    vantage: str
+    argv: tuple[str, ...]
+    kind: str = READ
+    optional: bool = False
+    record: bool = True
+    max_bytes: int | None = None
+
+
+@dataclass(frozen=True)
+class CheckSpec:
+    """A declarative acceptance check.
+
+    ``rule`` names an evaluator; ``expect`` carries its parameters. Keeping the
+    catalog declarative means a reviewer reads what is asserted, not how it is
+    plumbed, and the harness can prove structural properties over every check
+    (no impersonation, no mutation, both vantages present) before running any.
+    """
+
+    id: str
+    title: str
+    group: str
+    rule: str
+    steps: tuple[Step, ...] = ()
+    expect: Mapping[str, Any] = field(default_factory=dict)
+    mandatory: bool = True
+    scope: str = ALWAYS
+    rationale: str = ""
+
+    @property
+    def vantages(self) -> tuple[str, ...]:
+        """Return the distinct vantages this check draws evidence from."""
+        seen: list[str] = []
+        for step in self.steps:
+            if step.vantage not in seen:
+                seen.append(step.vantage)
+        return tuple(seen)
+
+
+@dataclass
+class CheckResult:
+    """The classified outcome of one acceptance check."""
+
+    spec: CheckSpec
+    status: str
+    reason: str = ""
+    evidence: dict[str, Any] = field(default_factory=dict)
+    outcomes: list[Outcome] = field(default_factory=list)
+
+    @property
+    def blocking(self) -> bool:
+        """Report whether this result alone forces NO_GO."""
+        if self.status not in STATUSES:
+            return True
+        return self.spec.mandatory and self.status in {FAIL, NOT_RUN}
+
+    def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]:
+        """Return a JSON-serialisable, screened view of the result."""
+        payload: dict[str, Any] = {
+            "id": self.spec.id,
+            "title": self.spec.title,
+            "group": self.spec.group,
+            "rule": self.spec.rule,
+            "scope": self.spec.scope,
+            "mandatory": self.spec.mandatory,
+            "vantages": list(self.spec.vantages),
+            "status": self.status,
+            "reason": self.reason,
+            "blocking": self.blocking,
+            "evidence": self.evidence,
+        }
+        if self.spec.rationale:
+            payload["rationale"] = self.spec.rationale
+        if include_outcomes:
+            payload["commands"] = [outcome.as_dict() for outcome in self.outcomes]
+        return scrub(payload)
+
+
+@dataclass
+class VantageRecord:
+    """What the harness proved about one vantage before trusting its evidence."""
+
+    name: str
+    description: str
+    identity: str = ""
+    available: bool = False
+    detail: str = ""
+
+    def as_dict(self) -> dict[str, Any]:
+        """Return a JSON-serialisable view of the vantage."""
+        return scrub(
+            {
+                "name": self.name,
+                "description": self.description,
+                "identity": self.identity,
+                "available": self.available,
+                "detail": self.detail,
+            }
+        )
+
+
+@dataclass
+class Report:
+    """The whole acceptance run: metadata, per-check results, and a verdict."""
+
+    mode: str
+    started_at: str
+    baseline: dict[str, Any] = field(default_factory=dict)
+    vantages: list[VantageRecord] = field(default_factory=list)
+    results: list[CheckResult] = field(default_factory=list)
+    harness_errors: list[str] = field(default_factory=list)
+    finished_at: str = ""
+
+    @property
+    def counts(self) -> dict[str, int]:
+        """Return the number of results in each status."""
+        tally = dict.fromkeys(STATUSES, 0)
+        for result in self.results:
+            if result.status in tally:
+                tally[result.status] += 1
+        return tally
+
+    @property
+    def invalid_statuses(self) -> list[str]:
+        """Return check ids whose status is outside the report schema."""
+        return [
+            result.spec.id for result in self.results if result.status not in STATUSES
+        ]
+
+    @property
+    def blocking(self) -> list[CheckResult]:
+        """Return every result that forces NO_GO, most severe first."""
+        order = {FAIL: 0, NOT_RUN: 1}
+        return sorted(
+            (result for result in self.results if result.blocking),
+            key=lambda result: (order.get(result.status, 2), result.spec.id),
+        )
+
+    @property
+    def decision(self) -> str:
+        """Return GO only when nothing mandatory failed, skipped, or errored."""
+        if (
+            not self.results
+            or self.invalid_statuses
+            or self.harness_errors
+            or self.blocking
+        ):
+            return NO_GO
+        return GO
+
+    def group_summary(self) -> dict[str, dict[str, int]]:
+        """Return per-group status counts, for the human summary."""
+        groups: dict[str, dict[str, int]] = {}
+        for result in self.results:
+            bucket = groups.setdefault(result.spec.group, dict.fromkeys(STATUSES, 0))
+            bucket[result.status] = bucket.get(result.status, 0) + 1
+        return groups
+
+    def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]:
+        """Return the machine-readable report."""
+        return scrub(
+            {
+                "harness": HARNESS,
+                "schema_version": SCHEMA_VERSION,
+                "mode": self.mode,
+                "started_at": self.started_at,
+                "finished_at": self.finished_at,
+                "decision": self.decision,
+                "counts": self.counts,
+                "baseline": scrub(self.baseline),
+                "vantages": [vantage.as_dict() for vantage in self.vantages],
+                "harness_errors": [scrub(error) for error in self.harness_errors],
+                "invalid_statuses": self.invalid_statuses,
+                "blocking": [result.spec.id for result in self.blocking],
+                "checks": [result.as_dict(include_outcomes) for result in self.results],
+            }
+        )
+
+    def render_summary(self, width: int = 88) -> str:
+        """Return the concise human summary printed alongside the JSON."""
+        counts = self.counts
+        lines = [
+            f"{HARNESS}: {self.decision}",
+            f"  mode={self.mode} started={self.started_at} checks={len(self.results)}",
+            "  " + "  ".join(f"{status}={counts[status]}" for status in STATUSES),
+        ]
+        for vantage in self.vantages:
+            state = "available" if vantage.available else "unavailable"
+            identity = vantage.identity or "unknown"
+            lines.append(f"  vantage {vantage.name}: {state} identity={identity}")
+        for error in self.harness_errors:
+            lines.append(f"  harness error: {error}")
+        if self.blocking:
+            lines.append("  blocking:")
+            for result in self.blocking:
+                lines.append(f"    {result.status:<15}{result.spec.id}")
+                lines.append(f"    {'':<15}{result.reason[:width]}")
+        groups = self.group_summary()
+        lines.append("  groups:")
+        for name in sorted(groups):
+            bucket = groups[name]
+            detail = " ".join(
+                f"{status}={bucket[status]}" for status in STATUSES if bucket[status]
+            )
+            lines.append(f"    {name:<18}{detail}")
+        return safe_text("\n".join(lines), 32 * 1024)[0]
+
+
+def utc_now(clock: Any = None) -> str:
+    """Return an ISO-8601 UTC timestamp, injectable for deterministic tests."""
+    now = clock() if clock else dt.datetime.now(dt.timezone.utc)
+    return now.replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def unscreened_fields(payload: Any, path: str = "$") -> list[str]:
+    """Return JSON pointers whose text still looks credential-shaped.
+
+    The report is already screened field by field; this is the invariant check
+    that runs before anything is written, so a screening gap fails the run
+    instead of publishing an artifact nobody re-read.
+    """
+    offenders: list[str] = []
+    if isinstance(payload, str):
+        if contains_credential_shape(payload):
+            offenders.append(path)
+    elif isinstance(payload, Mapping):
+        for key, value in payload.items():
+            offenders.extend(unscreened_fields(value, f"{path}.{key}"))
+    elif isinstance(payload, Sequence) and not isinstance(payload, (str, bytes)):
+        for index, value in enumerate(payload):
+            offenders.extend(unscreened_fields(value, f"{path}[{index}]"))
+    return offenders

scripts/ops/hermes_handoff_policy.py

@@ -0,0 +1,493 @@
+#!/usr/bin/env python3
+"""Fail-closed command policy for the Hermes handoff acceptance harness.
+
+Before spawn it admits only credential-safe, structurally read-only grammar;
+shell is limited to frozen templates with absolute tool paths.
+
+Impersonation is structurally impossible: ``--as`` and its siblings are refused
+for every argv, in every mode, from every vantage, and the inner command of a
+``kubectl exec`` is re-checked under the same rule rather than being trusted.
+Only binaries with a pinned release digest are on the allowlist, so nothing can
+be admitted here that the executor would then refuse to attest.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterator
+from urllib.parse import unquote
+
+READ_ONLY = "read-only"
+ARMED = "ephemeral-armed"
+MODES = (READ_ONLY, ARMED)
+
+GITEA_CLIENT = "/opt/coordinator/gitea_api.py"
+EXPECTED_REPO = "atlas/titan-iac"
+EXPECTED_REMOTE = "origin"
+EXPECTED_BASE = "main"
+
+# fmt: off
+FORBIDDEN_ARGS = ("--as", "--as-group", "--as-uid", "--client-certificate",
+                  "--client-key", "--password", "--token", "--username")
+# fmt: on
+IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid")
+FORBIDDEN_BINARIES = frozenset(
+    {"age", "curl", "gpg", "openssl", "sops", "vault", "wget"}
+)
+ALLOWED_BINARIES = frozenset({"kubectl", "git", "hermes", "sh", GITEA_CLIENT})
+TRUSTED_EXECUTABLE_ROOTS = (
+    "/bin/",
+    "/usr/bin/",
+    "/usr/local/bin/",
+    "/opt/data/tools/bin/",
+    "/opt/hermes/.venv/bin/",
+    "/opt/coordinator/",
+)
+TRUSTED_INNER_PATHS = {
+    "/bin/sh": "sh",
+    "/usr/bin/git": "git",
+    "/usr/bin/sh": "sh",
+    "/usr/local/bin/kubectl": "kubectl",
+    "/opt/hermes/.venv/bin/hermes": "hermes",
+    GITEA_CLIENT: GITEA_CLIENT,
+}
+
+VALUE_FLAGS = frozenset(
+    {
+        "-C",
+        "-c",
+        "-l",
+        "-n",
+        "-o",
+        "--cluster",
+        "--container",
+        "--context",
+        "--field-selector",
+        "--kubeconfig",
+        "--namespace",
+        "--output",
+        "--request-timeout",
+        "--selector",
+        "--server",
+        "--since",
+        "--tail",
+        "--user",
+    }
+)
+
+KUBECTL_READ_SUBCOMMANDS = frozenset(
+    {
+        "api-resources",
+        "api-versions",
+        "explain",
+        "get",
+        "top",
+        "version",
+    }
+)
+KUBECTL_EXEC_SUBCOMMANDS = frozenset({"exec"})
+KUBECTL_SUBCOMMANDS = KUBECTL_READ_SUBCOMMANDS | KUBECTL_EXEC_SUBCOMMANDS | {"auth"}
+SECRET_RESOURCES = frozenset({"secret", "secrets"})
+UNSAFE_PROJECTION_PARTS = (
+    ".args",
+    ".command",
+    ".data",
+    ".envfrom",
+    ".secretkeyref",
+    ".status.token",
+    ".stringdata",
+    ".valuefrom",
+    "clientconfig",
+    "password",
+    "privatekey",
+)
+SAFE_ENV_VALUE = 'env[?(@.name=="hermes_auto_router_profile")].value'
+SAFE_JSON_FIELDS = frozenset(["authenticated", "transport", "state", "model", "effort", "latency_ms", "checked_at", "api_provider", "auth_method", "subscription_type"])  # fmt: skip
+
+GIT_READ_SUBCOMMANDS = frozenset(
+    {
+        "cat-file",
+        "diff",
+        "for-each-ref",
+        "log",
+        "ls-remote",
+        "ls-tree",
+        "merge-base",
+        "rev-list",
+        "rev-parse",
+        "show",
+        "status",
+    }
+)
+GIT_FORCE_FLAGS = frozenset(
+    {"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"}
+)
+GIT_UNSAFE_FLAGS = (
+    "-c",
+    "--config-env",
+    "--exec-path",
+    "--upload-pack",
+    "--receive-pack",
+)
+GITEA_READ_METHODS = frozenset({"GET"})
+HERMES_READ_COMMANDS = frozenset(
+    {("kanban", "list"), ("kanban", "show"), ("sessions", "list"), ("status",)}
+)
+
+# These locations contain credential bytes even when a caller claims it will
+# only inspect metadata.  They are rejected for every argv, including a frozen
+# shell rendering and the inner command of kubectl exec.
+SECRET_PATH_RE = re.compile(
+    r"(?:^|[\s'\"=])(?:/[^\s'\"]*)?(?:"
+    r"var/run/secrets|run/secrets|vault/secrets|runtime-access|scm-access|pool-access|"
+    r"\.credentials\.json|auth\.json|\.git-credentials|authorized_keys|"
+    r"id_[a-z0-9]+|\.netrc|shadow|gshadow|k3s\.yaml"
+    r")(?:$|[/\s'\"])",
+    re.IGNORECASE,
+)
+_PARAMETER_RE = re.compile(r"[A-Za-z0-9/][A-Za-z0-9._/@:,-]{0,255}\Z")
+_PLACEHOLDER_RE = re.compile(r"\{([a-z_]+)\}")
+_SAFE_API_PATH_RE = re.compile(r"/[A-Za-z0-9][A-Za-z0-9._~:/?&=%+-]{0,511}\Z")
+
+SHELL_TEMPLATES: dict[str, str] = {
+    "env_names": (
+        "/usr/bin/env | /usr/bin/sed -n "
+        "'s/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/\\1/p' | /usr/bin/sort"
+    ),
+    "json_fields": (
+        '/opt/hermes/.venv/bin/python -c "import json,sys;d=json.load(open(sys.argv[1]));'
+        'print(json.dumps({{k:d.get(k) for k in sys.argv[2].split(chr(44))}}))" '
+        "{path} {fields}"
+    ),
+    "tail_lines": "/usr/bin/tail -n {limit} {path}",
+}
+
+
+class PolicyError(ValueError):
+    """Raised when a requested command falls outside the safe boundary."""
+
+
+def template_parameters(template_name: str) -> set[str]:
+    """Return the parameter names a frozen template requires."""
+    template = SHELL_TEMPLATES.get(template_name)
+    if template is None:
+        raise PolicyError(f"unknown shell template: {template_name}")
+    return set(_PLACEHOLDER_RE.findall(template))
+
+
+def render_shell(template_name: str, **parameters: str) -> str:
+    """Return a frozen shell template with validated parameters substituted."""
+    required = template_parameters(template_name)
+    if set(parameters) != required:
+        raise PolicyError(
+            f"shell template {template_name} expects parameters {sorted(required)}"
+        )
+    for key, value in parameters.items():
+        if not _PARAMETER_RE.fullmatch(value):
+            raise PolicyError(f"unsafe shell parameter for {template_name}.{key}")
+        if SECRET_PATH_RE.search(value):
+            raise PolicyError("shell templates may not name credential paths")
+    if (
+        template_name == "json_fields"
+        and not set(parameters["fields"].split(",")) <= SAFE_JSON_FIELDS
+    ):
+        raise PolicyError("json_fields requested a credential-bearing or unknown field")
+    return SHELL_TEMPLATES[template_name].format(**parameters)
+
+
+_RENDERED: set[str] = set()
+_APPROVED_PROJECTIONS: set[str] = set()
+_ARMED_REF = ""
+_ARMED_PULLS: set[int] = set()
+
+
+# fmt: off
+def arm_ephemeral_policy(ref: str) -> None:
+    global _ARMED_REF
+    if not re.fullmatch(r"ephemeral/hermes-handoff-acceptance/[a-z0-9][a-z0-9-]{7,63}", ref):
+        raise PolicyError("armed policy requires one exact ephemeral ref")
+    _ARMED_REF = ref
+    _ARMED_PULLS.clear()
+
+
+def register_ephemeral_pull(number: int) -> None:
+    if not _ARMED_REF or isinstance(number, bool) or not isinstance(number, int) or number <= 0:
+        raise PolicyError("ephemeral pull registration is malformed")
+    _ARMED_PULLS.add(number)
+
+
+def shell(template_name: str, **parameters: str) -> tuple[str, ...]:
+    """Return a validated ``sh -c`` argv built from a frozen template."""
+    script = render_shell(template_name, **parameters)
+    _RENDERED.add(script)
+    return ("sh", "-c", script)
+
+
+def projection(value: str) -> str:
+    normalized = value.lower().replace(" ", "")
+    if not normalized.startswith("jsonpath=") or any(part in normalized for part in UNSAFE_PROJECTION_PARTS):
+        raise PolicyError("Kubernetes projection is not credential-safe")
+    if ".metadata.annotations" in normalized and not any(key in normalized for key in (
+            "deployment\\.kubernetes\\.io/revision", "ai\\.bstein\\.dev/config-rev")):
+        raise PolicyError("Kubernetes annotations are not on the reviewed allowlist")
+    if ".metadata.labels" in normalized and "pod-template-hash" not in normalized:
+        raise PolicyError("Kubernetes labels are not on the reviewed allowlist")
+    if ".env" in normalized and ".value" in normalized and SAFE_ENV_VALUE not in normalized:
+        raise PolicyError("Kubernetes projection may expose environment values")
+    _APPROVED_PROJECTIONS.add(normalized)
+    return value
+# fmt: on
+
+
+def positionals(argv: tuple[str, ...], start: int = 1) -> Iterator[str]:
+    """Yield positional arguments, skipping flags and their values."""
+    index = start
+    while index < len(argv):
+        item = argv[index]
+        if item == "--":
+            return
+        if item.startswith("-"):
+            index += 2 if item in VALUE_FLAGS else 1
+            continue
+        yield item
+        index += 1
+
+
+def _binary_name(value: str) -> str:
+    if value == GITEA_CLIENT or (
+        value.startswith("/opt/coordinator/") and value.endswith("/gitea_api.py")
+    ):
+        return GITEA_CLIENT
+    if "/" not in value:
+        return value
+    mapped = TRUSTED_INNER_PATHS.get(value)
+    if mapped:
+        return mapped
+    if "/../" in value or value.endswith("/..") or "/./" in value:
+        raise PolicyError(f"executable path is not normalized: {value}")
+    if not value.startswith(TRUSTED_EXECUTABLE_ROOTS):
+        raise PolicyError(f"executable path is not trusted: {value}")
+    return value.rsplit("/", 1)[-1]
+
+
+def _reject_forbidden_arguments(argv: tuple[str, ...]) -> None:
+    """Refuse credential and impersonation arguments up to the ``--`` boundary.
+
+    Everything after ``--`` is the inner command of a ``kubectl exec``, which
+    :func:`_check_kubectl` submits to :func:`check_argv` again, so it is held to
+    exactly this rule rather than being exempted from it.
+    """
+    boundary = argv.index("--") if "--" in argv else len(argv)
+    for argument in argv[:boundary]:
+        for forbidden in FORBIDDEN_ARGS:
+            if argument == forbidden or argument.startswith(f"{forbidden}="):
+                raise PolicyError(f"forbidden argument: {forbidden}")
+        if argument == "--raw" or argument.startswith("--raw="):
+            raise PolicyError("raw API reads are forbidden")
+    for argument in argv:
+        if argument == "--dry-run" or argument.startswith("--dry-run="):
+            raise PolicyError("dry-run mutation commands are forbidden")
+
+
+def _reject_secret_paths(argv: tuple[str, ...]) -> None:
+    if any(SECRET_PATH_RE.search(argument) for argument in argv):
+        raise PolicyError("probes may not name a credential path")
+
+
+def _resource_name(value: str) -> str:
+    return value.lower().split("/", 1)[0].split(".", 1)[0]
+
+
+def _output_value(argv: tuple[str, ...]) -> str:
+    """Return kubectl's output argument in a single canonical form."""
+    for index, item in enumerate(argv[1:], 1):
+        if item in {"-o", "--output"} and index + 1 < len(argv):
+            return argv[index + 1]
+        if item.startswith(("-o=", "--output=")):
+            return item.split("=", 1)[1]
+    return ""
+
+
+def _check_projection(output: str) -> None:
+    """Reject full or credential-bearing Kubernetes output projections."""
+    lowered = output.lower().replace(" ", "")
+    if lowered in {"json", "yaml", "wide"} or lowered.startswith("custom-columns"):
+        raise PolicyError(
+            "full Kubernetes object output is forbidden; use a safe projection"
+        )
+    if lowered and not (lowered == "name" or lowered.startswith("jsonpath=")):
+        raise PolicyError(
+            "Kubernetes output must be name or an approved JSONPath projection"
+        )
+    if lowered.startswith("jsonpath="):
+        if any(part in lowered for part in UNSAFE_PROJECTION_PARTS):
+            raise PolicyError("Kubernetes projection may expose runtime credentials")
+        if ".env" in lowered and SAFE_ENV_VALUE not in lowered:
+            # Environment names are safe; arbitrary `.value` reads are not.
+            without_names = lowered.replace("{.name}", "")
+            if ".value" in without_names:
+                raise PolicyError("Kubernetes projection may expose environment values")
+        if lowered not in _APPROVED_PROJECTIONS:
+            raise PolicyError(
+                "Kubernetes JSONPath projection is not on the reviewed allowlist"
+            )
+
+
+def _check_kubectl(argv: tuple[str, ...], mode: str) -> None:
+    words = list(positionals(argv))
+    if not words or words[0] not in KUBECTL_SUBCOMMANDS:
+        raise PolicyError("kubectl subcommand is outside the read-only boundary")
+    subcommand = words[0]
+    if subcommand == "auth":
+        if len(words) < 2 or words[1] not in {"can-i", "whoami"}:
+            raise PolicyError("kubectl auth is limited to can-i and whoami")
+        return
+    if subcommand == "exec":
+        if "--" not in argv:
+            raise PolicyError("kubectl exec requires an explicit -- command boundary")
+        inner = argv[argv.index("--") + 1 :]
+        if not inner:
+            raise PolicyError("kubectl exec requires an inner command")
+        if inner[0] not in TRUSTED_INNER_PATHS:
+            raise PolicyError(
+                "kubectl exec inner command must use a fixed trusted path"
+            )
+        check_argv(inner, mode)
+        return
+    resources = words[1].split(",") if len(words) > 1 else []
+    if any(_resource_name(resource) in SECRET_RESOURCES for resource in resources):
+        raise PolicyError("Secret objects may not be read by the harness")
+    output = _output_value(argv)
+    if subcommand == "get" and not output:
+        raise PolicyError("kubectl get requires a reviewed output projection")
+    _check_projection(output)
+
+
+def _check_git(argv: tuple[str, ...], mode: str) -> None:
+    unsafe = False
+    for item in argv[1:]:
+        if item == "-C" or item.startswith(("--git-dir", "--work-tree", "--namespace")):
+            unsafe = True
+        if item == "-c" or (item.startswith("-c") and len(item) > 2):
+            unsafe = True
+        if any(
+            item == flag or item.startswith(f"{flag}=") for flag in GIT_UNSAFE_FLAGS[1:]
+        ):
+            unsafe = True
+    if unsafe:
+        raise PolicyError("git configuration and helper overrides are forbidden")
+    subcommand = next(positionals(argv), None)
+    if subcommand is None:
+        raise PolicyError("git requires a subcommand")
+    if subcommand in GIT_READ_SUBCOMMANDS:
+        return
+    if subcommand == "push" and mode == ARMED:
+        if GIT_FORCE_FLAGS & set(argv):
+            raise PolicyError("git push may not force or fan out across refs")
+        words = list(positionals(argv))
+        if len(words) != 3 or words[1] != EXPECTED_REMOTE:
+            raise PolicyError("armed git push is pinned to origin and one refspec")
+        destination = words[2].partition(":")[2]
+        if not _ARMED_REF or destination != f"refs/heads/{_ARMED_REF}":
+            raise PolicyError("armed git push is not the preflighted ephemeral ref")
+        return
+    raise PolicyError(f"git {subcommand} is not permitted in {mode} mode")
+
+
+def _check_hermes(argv: tuple[str, ...]) -> None:
+    words = tuple(positionals(argv))
+    if not words:
+        raise PolicyError("hermes requires a subcommand")
+    if words[:2] not in HERMES_READ_COMMANDS and words[:1] not in HERMES_READ_COMMANDS:
+        raise PolicyError(
+            f"hermes {' '.join(words[:2])} is outside the read-only boundary"
+        )
+
+
+def _reject_dot_segments(path: str) -> None:
+    """Refuse any relative segment, including a percent-encoded one.
+
+    The repository pin below is a textual prefix test, so a single ``..`` would
+    let a pinned path address another repository or the token endpoints once a
+    router normalised it. Decode first: ``%2e%2e`` is the same segment.
+    """
+    decoded = unquote(unquote(path)).split("?", 1)[0]
+    if any(segment in {"", ".", ".."} for segment in decoded.split("/")[1:]):
+        raise PolicyError("gitea_api.py paths may not contain relative segments")
+
+
+def _check_gitea(argv: tuple[str, ...], mode: str) -> None:
+    if len(argv) < 3:
+        raise PolicyError("gitea_api.py requires a method and a path")
+    method, path = argv[1].upper(), argv[2]
+    if not path.startswith("/api/v1/") or not _SAFE_API_PATH_RE.fullmatch(path):
+        raise PolicyError("gitea_api.py requires a bounded API path")
+    _reject_dot_segments(path)
+    repo_root = f"/api/v1/repos/{EXPECTED_REPO}"
+    if (
+        path != "/api/v1/user"
+        and path != repo_root
+        and not path.startswith((f"{repo_root}/", f"{repo_root}?"))
+    ):
+        raise PolicyError("gitea_api.py is pinned to the Atlas titan-iac repository")
+    if method in GITEA_READ_METHODS:
+        return
+    if mode == ARMED:
+        fields = [
+            argv[index + 1] for index, item in enumerate(argv[:-1]) if item == "--field"
+        ]
+        if method == "POST" and path == f"{repo_root}/pulls" and _ARMED_REF:
+            required = {
+                "title=WIP: Hermes handoff acceptance ephemeral probe",
+                f"head={_ARMED_REF}",
+                f"base={EXPECTED_BASE}",
+                "body=Ephemeral acceptance probe. Closed and deleted by the harness.",
+            }
+            if len(fields) == len(required) and set(fields) == required:
+                return
+        match = re.fullmatch(rf"{re.escape(repo_root)}/pulls/([1-9][0-9]*)", path)
+        if (
+            method == "PATCH"
+            and match
+            and int(match.group(1)) in _ARMED_PULLS
+            and fields == ["state=closed"]
+        ):
+            return
+        if (
+            method == "DELETE"
+            and _ARMED_REF
+            and path == f"{repo_root}/branches/{_ARMED_REF}"
+            and len(argv) == 3
+        ):
+            return
+    raise PolicyError(f"gitea_api.py {method} is not permitted in {mode} mode")
+
+
+def _check_shell(argv: tuple[str, ...]) -> None:
+    if len(argv) != 3 or argv[1] != "-c" or argv[2] not in _RENDERED:
+        raise PolicyError("sh is only reachable as sh -c <frozen template>")
+
+
+def check_argv(argv: tuple[str, ...], mode: str = READ_ONLY) -> None:
+    """Raise :class:`PolicyError` unless ``argv`` is safe in ``mode``."""
+    if mode not in MODES:
+        raise PolicyError(f"unknown mode: {mode}")
+    if not argv:
+        raise PolicyError("empty command")
+    binary = _binary_name(argv[0])
+    if binary in FORBIDDEN_BINARIES:
+        raise PolicyError(f"forbidden binary: {binary}")
+    if binary not in ALLOWED_BINARIES:
+        raise PolicyError(f"binary is not on the harness allowlist: {binary}")
+    _reject_forbidden_arguments(argv)
+    _reject_secret_paths(argv)
+    if binary == "sh":
+        _check_shell(argv)
+    elif binary == "kubectl":
+        _check_kubectl(argv, mode)
+    elif binary == "git":
+        _check_git(argv, mode)
+    elif binary == "hermes":
+        _check_hermes(argv)
+    else:
+        _check_gitea(argv, mode)

scripts/ops/hermes_handoff_redaction.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+"""Credential redaction and output bounding for the handoff acceptance harness.
+
+Every string the harness records — command output, failure reasons, evidence
+values — passes through :func:`redact` before it reaches a report. Screening is
+a fail-closed accident barrier, not proof that text is secret-free: a short
+secret under an innocuous key is not reliably distinguishable from prose. The
+harness therefore also never asks for a secret in the first place. Probes read
+names, shapes, and status fields; they never read environment values, token
+files, private keys, cookies, or Vault responses.
+
+Bounding is part of the same barrier. Screening always precedes truncation so a
+credential that crosses the byte boundary cannot leave a useful prefix behind.
+The process runner additionally discards all captured bytes when a live stream
+exceeds its cap; the fixed truncation marker is the only recorded output.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+REDACTED = "[redacted]"
+DEFAULT_MAX_BYTES = 64 * 1024
+TRUNCATION_NOTE = "[truncated]"
+
+# Keys whose value is never safe to record, matched against a normalised
+# (lowercased, separator-stripped) form of the key.
+SENSITIVE_KEY_SUFFIXES = (
+    "accesskey",
+    "accesskeyid",
+    "apikey",
+    "authtoken",
+    "clientsecret",
+    "connectionstring",
+    "cookie",
+    "credential",
+    "credentials",
+    "passphrase",
+    "passwd",
+    "password",
+    "privatekey",
+    "secret",
+    "secretkey",
+    "sessionkey",
+    "signature",
+    "signingkey",
+    "token",
+    "webhook",
+)
+SENSITIVE_KEY_EXACT = frozenset(
+    {
+        "auth",
+        "authorization",
+        "bearer",
+        "pat",
+        "pw",
+        "sas",
+        "setcookie",
+    }
+)
+# A bare `key:` is a structural field name across Kubernetes and TOML far more
+# often than it is a credential, so it is screened by value shape and by the
+# token patterns rather than by name. The compound forms above still match.
+
+# Ordinary prose values shorter than this threshold are not independently
+# treated as secret-shaped. Values under sensitive keys are screened at every
+# length, including numeric values.
+MIN_SECRET_LENGTH = 8
+NON_SECRET_LITERALS = frozenset(
+    {"true", "false", "null", "none", "yes", "no", "0", "1", ""}
+)
+_KEY_NOISE_RE = re.compile(r"[^a-z0-9]+")
+
+# Ordered: structural matches first so a nested token is not partially rewritten
+# by a looser rule.
+_PATTERNS: tuple[re.Pattern[str], ...] = (
+    re.compile(
+        r"-----BEGIN[ A-Z0-9-]{0,64}-----.*?-----END[ A-Z0-9-]{0,64}-----",
+        re.DOTALL,
+    ),
+    re.compile(r"-----BEGIN[ A-Z0-9-]{0,64}-----.*", re.DOTALL),
+    re.compile(r"(?i)\b(?:authorization|proxy-authorization)\s*:\s*\S.*"),
+    re.compile(r"(?i)\b(?:set-)?cookie\s*:\s*\S.*"),
+    re.compile(r"(?i)\b(?:bearer|basic|token)\s+[A-Za-z0-9+/_=.:~-]{8,}"),
+    re.compile(r"(?i)\b[a-z][a-z0-9+.-]{1,20}://[^\s/@:]{1,128}:[^\s/@]{1,256}@"),
+    re.compile(
+        r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}"
+    ),
+    re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}"),
+    re.compile(r"(?<![A-Za-z0-9])gh[pousr]_[A-Za-z0-9]*"),
+    re.compile(r"(?<![A-Za-z0-9])github_pat_[A-Za-z0-9_]*"),
+    re.compile(r"(?<![A-Za-z0-9])glpat-[A-Za-z0-9_-]*"),
+    re.compile(r"(?<![A-Za-z0-9])xox[baprs]-[A-Za-z0-9-]*"),
+    re.compile(r"(?<![A-Za-z0-9])sk-(?:ant-|proj-)?[A-Za-z0-9_-]*"),
+    re.compile(r"(?<![A-Za-z0-9])(?:gta|gto|gitea|forgejo)_[A-Za-z0-9_-]*"),
+    re.compile(r"(?<![A-Za-z0-9])hvs\.[A-Za-z0-9_-]*"),
+    re.compile(r"(?<![A-Za-z0-9])(?:AKIA|ASIA)[0-9A-Z]*"),
+    re.compile(r"(?<![A-Za-z0-9])AIza[0-9A-Za-z_-]{35}"),
+    re.compile(r"(?<![A-Za-z0-9])npm_[A-Za-z0-9]{16,}"),
+)
+
+# `key: value` / `key=value` where the key names a credential.
+_ASSIGNMENT_RE = re.compile(
+    r"""(?x)
+    (?P<quote>["']?)
+    (?P<key>[A-Za-z][A-Za-z0-9_.\ -]{0,63})
+    (?P=quote)
+    (?P<gap>\s{0,8}[:=]\s{0,8})
+    (?P<value>"[^"\n]{0,4096}"|'[^'\n]{0,4096}'|[^\s,;}\n][^,;}\n]{0,4096})
+    """
+)
+
+# Long, mixed-class runs. Deliberately conservative: prose and hex digests of
+# fewer than 44 characters are left alone so commit SHAs stay readable.
+_ENTROPY_RE = re.compile(
+    r"(?<![A-Za-z0-9+/_=-])[A-Za-z0-9+/_=-]{44,}(?![A-Za-z0-9+/_=-])"
+)
+
+
+def normalise_key(key: str) -> str:
+    """Return a key lowercased with separators removed, for suffix matching."""
+    return _KEY_NOISE_RE.sub("", key.lower())
+
+
+def is_sensitive_key(key: str) -> bool:
+    """Report whether a key name means its value must never be recorded."""
+    compact = normalise_key(key)
+    if not compact:
+        return False
+    if compact in SENSITIVE_KEY_EXACT:
+        return True
+    return compact.endswith(SENSITIVE_KEY_SUFFIXES)
+
+
+def _is_high_entropy(candidate: str) -> bool:
+    """Report whether a long run mixes enough character classes to be a token."""
+    classes = sum(
+        (
+            any(character.islower() for character in candidate),
+            any(character.isupper() for character in candidate),
+            any(character.isdigit() for character in candidate),
+            any(character in "+/_=-" for character in candidate),
+        )
+    )
+    return classes >= 4
+
+
+def could_hold_secret(value: str) -> bool:
+    """Report whether a raw value is long and opaque enough to be a credential."""
+    stripped = value.strip().strip("\"'")
+    if stripped.lower() in NON_SECRET_LITERALS or len(stripped) < MIN_SECRET_LENGTH:
+        return False
+    try:
+        float(stripped)
+    except ValueError:
+        return True
+    return False
+
+
+def _redact_assignment(match: re.Match[str]) -> str:
+    value = match.group("value")
+    stripped = value.strip().strip("\"'").lower()
+    if not is_sensitive_key(match.group("key")) or stripped in NON_SECRET_LITERALS:
+        return match.group(0)
+    # Preserve the original quoting so screened JSON stays parsable JSON.
+    quoted = value.strip()[:1] in {'"', "'"}
+    replacement = f'"{REDACTED}"' if quoted else REDACTED
+    key_quote = match.group("quote")
+    return (
+        f"{key_quote}{match.group('key')}{key_quote}{match.group('gap')}{replacement}"
+    )
+
+
+def _redact_entropy(match: re.Match[str]) -> str:
+    return REDACTED if _is_high_entropy(match.group(0)) else match.group(0)
+
+
+def redact(text: str) -> str:
+    """Return ``text`` with credential-shaped substrings replaced."""
+    if not text:
+        return text
+    for pattern in _PATTERNS:
+        text = pattern.sub(REDACTED, text)
+    text = _ASSIGNMENT_RE.sub(_redact_assignment, text)
+    return _ENTROPY_RE.sub(_redact_entropy, text)
+
+
+def bound(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]:
+    """Truncate ``text`` to a UTF-8 byte budget, reporting whether it was cut."""
+    if max_bytes <= 0:
+        return ("", bool(text))
+    encoded = text.encode("utf-8", "replace")
+    if len(encoded) <= max_bytes:
+        return (text, False)
+    marker = TRUNCATION_NOTE.encode()[:max_bytes]
+    content_budget = max(
+        0, max_bytes - len(marker) - (1 if len(marker) < max_bytes else 0)
+    )
+    kept = encoded[:content_budget].decode("utf-8", "ignore")
+    separator = "\n" if kept and len(marker) < max_bytes else ""
+    return (f"{kept}{separator}{marker.decode()}", True)
+
+
+def safe_text(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]:
+    """Redact then bound, so truncation never exposes a credential prefix."""
+    return bound(redact(text), max_bytes)
+
+
+def scrub(value: Any, max_bytes: int = DEFAULT_MAX_BYTES) -> Any:
+    """Recursively bound and redact a JSON-shaped value.
+
+    Mapping keys are themselves screened; values under a sensitive key are
+    dropped outright rather than pattern-screened, because a short secret
+    would survive screening.
+    """
+    if isinstance(value, str):
+        return safe_text(value, max_bytes)[0]
+    if isinstance(value, dict):
+        screened: dict[str, Any] = {}
+        for key, item in value.items():
+            raw_key = str(key)
+            safe_key = safe_text(raw_key, max_bytes)[0]
+            sensitive_value = (
+                is_sensitive_key(raw_key)
+                and item is not None
+                and not isinstance(item, bool)
+                and item != ""
+            )
+            screened[safe_key] = REDACTED if sensitive_value else scrub(item, max_bytes)
+        return screened
+    if isinstance(value, (list, tuple)):
+        return [scrub(item, max_bytes) for item in value]
+    return value
+
+
+def contains_credential_shape(text: str) -> bool:
+    """Report whether screening would rewrite ``text``.
+
+    Used by the harness self-test to prove a rendered report is clean before it
+    is written, and by the unit suite as an assertion helper.
+    """
+    return redact(text) != text

scripts/ops/hermes_handoff_rules.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+"""Shared primitives and dispatch for handoff acceptance evaluators.
+
+Evaluators are registered by name so the check catalog stays declarative: an
+entry says *what* it asserts, not how to plumb it. The dispatch here is the one
+place that turns an unexpected exception into ``NOT_RUN`` — a harness bug must
+never be indistinguishable from a passing release.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from hermes_handoff_exec import Outcome
+from hermes_handoff_model import NOT_RUN, STATUSES, CheckSpec
+
+DENIAL_MARKERS = (
+    "forbidden",
+    "is not allowed",
+    "unauthorized",
+    "permission denied",
+    "cannot get",
+    "cannot list",
+    "cannot create",
+    "cannot delete",
+    "cannot patch",
+    "not permitted",
+    "gitea api returned http 403",
+)
+
+
+@dataclass
+class Evaluation:
+    """A classification plus the small facts that justify it."""
+
+    status: str
+    reason: str
+    evidence: dict[str, Any] = field(default_factory=dict)
+
+
+Evaluator = Callable[[CheckSpec, dict[str, Outcome]], Evaluation]
+EVALUATORS: dict[str, Evaluator] = {}
+
+
+def evaluator(name: str) -> Callable[[Evaluator], Evaluator]:
+    """Register an evaluator under the rule name catalog entries use."""
+
+    def register(function: Evaluator) -> Evaluator:
+        EVALUATORS[name] = function
+        return function
+
+    return register
+
+
+def missing_steps(
+    spec: CheckSpec, outcomes: Mapping[str, Outcome]
+) -> Evaluation | None:
+    """Return NOT_RUN when a required step never produced usable evidence."""
+    for step in spec.steps:
+        if step.optional:
+            continue
+        outcome = outcomes.get(step.key)
+        if outcome is None:
+            return Evaluation(NOT_RUN, f"step {step.key} was never executed")
+        if outcome.truncated:
+            return Evaluation(NOT_RUN, f"step {step.key} output was truncated")
+        if not outcome.ran:
+            return Evaluation(NOT_RUN, f"step {step.key} did not run: {outcome.error}")
+    return None
+
+
+def looks_denied(outcome: Outcome) -> bool:
+    """Report whether a real request was refused by policy rather than failing."""
+    if outcome.ok:
+        return False
+    haystack = outcome.combined.lower()
+    return any(marker in haystack for marker in DENIAL_MARKERS)
+
+
+def step_keys(spec: CheckSpec, kind: str) -> tuple[str, ...]:
+    """Return the keys of every step of one kind."""
+    return tuple(step.key for step in spec.steps if step.kind == kind)
+
+
+def is_list(value: Any) -> bool:
+    """Report whether a value is a JSON array rather than a string."""
+    return isinstance(value, Sequence) and not isinstance(value, (str, bytes))
+
+
+def exact_equal(actual: Any, expected: Any) -> bool:
+    """Compare JSON values without Python's bool/int type confusion."""
+    return type(actual) is type(expected) and actual == expected
+
+
+def dotted(payload: Any, path: str) -> Any:
+    """Return a value addressed by a dotted path.
+
+    ``a.b[0].c`` indexes; ``a[].b`` maps the remainder of the path over a list,
+    which is how the catalog records one field from every pull request or every
+    Flux object without a bespoke parser per check.
+    """
+    current = payload
+    segments = [segment for segment in path.split(".") if segment]
+    for position, raw in enumerate(segments):
+        name, bracket, indexes = raw.partition("[")
+        if name:
+            if not isinstance(current, Mapping) or name not in current:
+                raise KeyError(path)
+            current = current[name]
+        if not bracket:
+            continue
+        for chunk in indexes.rstrip("]").split("]["):
+            if not is_list(current):
+                raise KeyError(path)
+            if chunk == "":
+                remainder = ".".join(segments[position + 1 :])
+                return [
+                    dotted(item, remainder) if remainder else item for item in current
+                ]
+            offset = int(chunk)
+            if offset >= len(current):
+                raise KeyError(path)
+            current = current[offset]
+    return current
+
+
+def strict_json(text: str) -> Any:
+    """Decode strict JSON, rejecting constants and duplicate object keys."""
+    text = text.strip()
+    if not text:
+        raise ValueError("empty output")
+
+    def reject_constant(value: str) -> None:
+        raise ValueError(f"non-finite JSON constant {value}")
+
+    def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+        result: dict[str, Any] = {}
+        for key, value in pairs:
+            if key in result:
+                raise ValueError(f"duplicate JSON key {key!r}")
+            result[key] = value
+        return result
+
+    return json.loads(
+        text,
+        strict=True,
+        parse_constant=reject_constant,
+        object_pairs_hook=unique_object,
+    )
+
+
+def parse_json(outcome: Outcome) -> Any:
+    """Parse one outcome as strict JSON."""
+    return strict_json(outcome.stdout)
+
+
+def lines(outcome: Outcome) -> list[str]:
+    """Return non-empty, stripped stdout lines."""
+    return [line.strip() for line in outcome.stdout.splitlines() if line.strip()]
+
+
+def parsed_step(
+    spec: CheckSpec, outcomes: dict[str, Outcome]
+) -> tuple[Any, str] | Evaluation:
+    """Return the parsed payload of ``expect['step']`` or the NOT_RUN to report."""
+    key = spec.expect["step"]
+    outcome = outcomes[key]
+    if not outcome.ok:
+        return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
+    try:
+        return (parse_json(outcome), key)
+    except ValueError as exc:
+        return Evaluation(NOT_RUN, f"step {key} did not return JSON: {exc}")
+
+
+def evaluate(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
+    """Dispatch to the named evaluator, converting any surprise into NOT_RUN."""
+    rule = EVALUATORS.get(spec.rule)
+    if rule is None:
+        return Evaluation(NOT_RUN, f"unknown evaluator: {spec.rule}")
+    truncated = sorted(key for key, outcome in outcomes.items() if outcome.truncated)
+    if truncated:
+        return Evaluation(
+            NOT_RUN, f"truncated evidence from step(s): {', '.join(truncated)}"
+        )
+    try:
+        result = rule(spec, outcomes)
+    except Exception as exc:  # noqa: BLE001 - a harness bug must never read as PASS
+        return Evaluation(
+            NOT_RUN, f"evaluator {spec.rule} raised {type(exc).__name__}: {exc}"
+        )
+    if result.status not in STATUSES:
+        return Evaluation(
+            NOT_RUN, f"evaluator {spec.rule} returned unknown status {result.status!r}"
+        )
+    return result

scripts/ops/hermes_handoff_run.py

@@ -0,0 +1,484 @@
+#!/usr/bin/env python3
+"""Orchestration for the Hermes full-handoff acceptance harness.
+
+Three things happen before any check runs, and each of them can stop the run on
+its own:
+
+* the catalog is validated structurally — every rule exists, every live-denial
+  evaluator carries a read-only attempt, no operator-side step can impersonate;
+* each vantage is resolved and asked who it is; and
+* the operator and in-pod identities are compared. If they are the same
+  principal there is only one vantage, the dual-vantage evidence would be a
+  restatement rather than a corroboration, and the run is a NO_GO.
+
+Everything after that is uniform: run a check's steps against its vantage, hand
+the outcomes to the named evaluator, and record the classification. A vantage
+that could not be resolved yields ``NOT_RUN`` for every check that needs it,
+which is exactly the fail-closed answer.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable, Mapping, Sequence
+
+from hermes_handoff_catalog import (
+    CHAT,
+    NODE,
+    OPERATOR,
+    SELF,
+    SWITCHYARD,
+    VANTAGE_NAMES,
+    Targets,
+)
+from hermes_handoff_checks_access import access_checks
+from hermes_handoff_checks_delivery import delivery_checks
+from hermes_handoff_checks_platform import platform_checks
+from hermes_handoff_checks_workers import worker_checks
+from hermes_handoff_evaluators import EVALUATORS, evaluate
+from hermes_handoff_exec import Outcome, Runner, Vantage, operator_vantage, pod_vantage
+from hermes_handoff_model import (
+    ATTEMPT,
+    EPHEMERAL,
+    NOT_RUN,
+    STEP_KINDS,
+    CheckResult,
+    CheckSpec,
+    Report,
+    VantageRecord,
+    utc_now,
+)
+from hermes_handoff_policy import (
+    EXPECTED_REMOTE,
+    EXPECTED_REPO,
+    IMPERSONATION_ARGS,
+    PolicyError,
+    check_argv,
+    projection,
+)
+
+UNRECORDED = "[output not recorded]"
+
+
+def build_catalog(targets: Targets) -> list[CheckSpec]:
+    """Return the whole acceptance catalog for a set of targets."""
+    return [
+        *platform_checks(targets),
+        *access_checks(targets),
+        *delivery_checks(targets),
+        *worker_checks(targets),
+    ]
+
+
+def validate_catalog(specs: Sequence[CheckSpec]) -> list[str]:
+    """Return the structural problems that make a catalog unsafe to run."""
+    problems: list[str] = []
+    if not specs:
+        return ["catalog is empty"]
+    seen: set[str] = set()
+    fake_operator = operator_vantage()
+    fake_pod = pod_vantage("hermes", "acceptance-probe", "probe", fake_operator)
+    for spec in specs:
+        if spec.id in seen:
+            problems.append(f"{spec.id}: duplicate check id")
+        seen.add(spec.id)
+        if spec.rule not in EVALUATORS and spec.scope != EPHEMERAL:
+            problems.append(f"{spec.id}: unknown rule {spec.rule!r}")
+        step_names: set[str] = set()
+        for step in spec.steps:
+            if step.key in step_names:
+                problems.append(f"{spec.id}: duplicate step key {step.key!r}")
+            step_names.add(step.key)
+            if step.vantage not in VANTAGE_NAMES:
+                problems.append(
+                    f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}"
+                )
+            if step.kind not in STEP_KINDS:
+                problems.append(
+                    f"{spec.id}: step {step.key} has unknown kind {step.kind!r}"
+                )
+            if set(step.argv) & set(IMPERSONATION_ARGS):
+                problems.append(
+                    f"{spec.id}: step {step.key} impersonates from the "
+                    f"{step.vantage} vantage"
+                )
+            try:
+                addressed = (
+                    fake_operator if step.vantage == OPERATOR else fake_pod
+                ).wrap(step.argv)
+                check_argv(addressed)
+            except PolicyError as exc:
+                problems.append(f"{spec.id}: step {step.key} violates policy: {exc}")
+        if spec.rule == "denied" and not any(
+            step.kind == ATTEMPT for step in spec.steps
+        ):
+            problems.append(
+                f"{spec.id}: a deny check needs a real attempt, not only a review"
+            )
+    return problems
+
+
+def validate_targets(targets: Targets) -> list[str]:
+    """Return release-input problems before a runner or network call exists."""
+    problems: list[str] = []
+    sha = re.compile(r"[0-9a-f]{40}\Z")
+    image = re.compile(
+        r"[^@\s]+:git-([0-9a-f]{40})-build-[1-9][0-9]*@sha256:[0-9a-f]{64}\Z"
+    )
+    if targets.repo != EXPECTED_REPO or targets.remote != EXPECTED_REMOTE:
+        problems.append(
+            "repository and remote must be the fixed Atlas titan-iac origin"
+        )
+    if (
+        targets.reviewed_pr_number != 19
+        or targets.reviewed_head_ref != "feature/hermes-full-handoff-acceptance"
+    ):
+        problems.append(
+            "reviewed PR identity must remain fixed to draft PR #19 and its existing branch"
+        )
+    for label, value in (
+        ("baseline commit", targets.baseline_commit),
+        ("remote main SHA", targets.remote_main_sha),
+        ("reviewed head SHA", targets.reviewed_head_sha),
+        ("build SHA", targets.build_sha),
+    ):
+        if not sha.fullmatch(value):
+            problems.append(f"{label} must be an exact lowercase 40-character SHA")
+    image_match = image.fullmatch(targets.agent_image)
+    if not image_match or image_match.group(1) != targets.build_sha:
+        problems.append("agent image tag/digest must bind the exact build SHA")
+    if targets.build_sha != targets.remote_main_sha:
+        problems.append("build SHA must equal the exact release main SHA")
+    if not re.fullmatch(r"[1-9][0-9]{0,8}", str(targets.deployment_revision)):
+        problems.append("deployment revision must be a positive integer string")
+    if targets.node_count != 3 or targets.pool_replicas != 3:
+        problems.append("node and pool counts are fixed at exactly three")
+    if not targets.chat_config_revision.strip():
+        problems.append(
+            "Telegram continuity and its chat config revision are mandatory"
+        )
+    if set(targets.pool_worker_env) != {"HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL"}:
+        problems.append("pool assignment evidence must include worker node and ordinal")
+    heads = dict(targets.dependency_heads)
+    if len(heads) != len(targets.dependency_heads) or set(heads) != set(
+        targets.dependency_pull_requests
+    ):
+        problems.append("every dependency PR must have one exact current head")
+    elif any(not sha.fullmatch(value) for value in heads.values()):
+        problems.append("dependency heads must be exact lowercase 40-character SHAs")
+    if (
+        not isinstance(targets.max_evidence_age_seconds, int)
+        or targets.max_evidence_age_seconds <= 0
+    ):
+        problems.append("maximum evidence age must be a positive integer")
+    return problems
+
+
+def resolve_identity(runner: Runner, vantage: Vantage) -> tuple[str, str]:
+    """Return the authenticated username of a vantage and any failure detail."""
+    outcome = runner.run(
+        (
+            "kubectl",
+            "auth",
+            "whoami",
+            "-o",
+            projection("jsonpath={.status.userInfo.username}"),
+        ),
+        vantage,
+    )
+    if not outcome.ok:
+        return ("", outcome.error or outcome.combined[:200])
+    username = outcome.stdout.strip()
+    if not username or any(character.isspace() for character in username):
+        return ("", "identity response carried no username")
+    return (username, "")
+
+
+def _first_pod(
+    runner: Runner, operator: Vantage, namespace: str, selector: str
+) -> tuple[str, str]:
+    outcome = runner.run(
+        (
+            "kubectl",
+            "--namespace",
+            namespace,
+            "get",
+            "pods",
+            "--selector",
+            selector,
+            "--field-selector",
+            "status.phase=Running",
+            "-o",
+            "name",
+        ),
+        operator,
+    )
+    if not outcome.ok:
+        return ("", outcome.error or outcome.combined[:200])
+    names = [
+        line.strip().removeprefix("pod/")
+        for line in outcome.stdout.splitlines()
+        if line.strip()
+    ]
+    if not names:
+        return ("", f"no running pod matches {selector}")
+    return (sorted(names)[0], "")
+
+
+def resolve_vantages(
+    runner: Runner,
+    targets: Targets,
+    kubeconfig: str | None = None,
+    context: str | None = None,
+) -> tuple[dict[str, Vantage], list[VantageRecord]]:
+    """Resolve every vantage the catalog names, recording what each one is."""
+    operator = operator_vantage(kubeconfig, context)
+    username, detail = resolve_identity(runner, operator)
+    records = [
+        VantageRecord(
+            name=OPERATOR,
+            description=operator.description or "external read-only operator",
+            identity=username,
+            available=bool(username),
+            detail=detail,
+        )
+    ]
+    vantages: dict[str, Vantage] = {OPERATOR: operator}
+    if not username:
+        return (vantages, records)
+
+    for name, selector, container, description in (
+        (
+            SELF,
+            f"app={targets.agent_deployment}",
+            targets.agent_container,
+            "the Hermes agent itself",
+        ),
+        (
+            SWITCHYARD,
+            f"app={targets.switchyard_deployment}",
+            targets.switchyard_container,
+            "the Switchyard routing evidence log",
+        ),
+        (
+            NODE,
+            f"app={targets.node_daemonset}",
+            targets.node_container,
+            "a node hardening probe",
+        ),
+    ):
+        pod, failure = _first_pod(runner, operator, targets.namespace, selector)
+        if not pod:
+            records.append(
+                VantageRecord(
+                    name=name, description=description, available=False, detail=failure
+                )
+            )
+            continue
+        vantage = pod_vantage(targets.namespace, pod, container, operator)
+        identity, identity_detail = (
+            resolve_identity(runner, vantage) if name == SELF else ("", "")
+        )
+        records.append(
+            VantageRecord(
+                name=name,
+                description=f"{description} ({targets.namespace}/{pod})",
+                identity=identity,
+                available=True,
+                detail=identity_detail,
+            )
+        )
+        vantages[name] = vantage
+
+    chat_pod = f"{targets.chat_statefulset}-{targets.chat_ordinal}"
+    present = runner.run(
+        (
+            "kubectl",
+            "--namespace",
+            targets.namespace,
+            "get",
+            f"pod/{chat_pod}",
+            "-o",
+            "name",
+        ),
+        operator,
+    )
+    if present.ok and present.stdout.strip():
+        vantages[CHAT] = pod_vantage(
+            targets.namespace, chat_pod, targets.chat_container, operator
+        )
+        records.append(
+            VantageRecord(
+                name=CHAT,
+                description=f"the chat surface ({targets.namespace}/{chat_pod})",
+                available=True,
+            )
+        )
+    else:
+        records.append(
+            VantageRecord(
+                name=CHAT,
+                description=f"the chat surface ({targets.namespace}/{chat_pod})",
+                available=False,
+                detail=present.error
+                or present.combined[:200]
+                or "chat tenant pod not found",
+            )
+        )
+    return (vantages, records)
+
+
+def vantage_problems(records: Iterable[VantageRecord]) -> list[str]:
+    """Return the identity problems that invalidate dual-vantage evidence."""
+    by_name = {record.name: record for record in records}
+    operator = by_name.get(OPERATOR)
+    hermes = by_name.get(SELF)
+    problems: list[str] = []
+    if operator is None or not operator.available:
+        problems.append(
+            "the operator vantage is unavailable; no evidence can be collected"
+        )
+        return problems
+    if hermes is None or not hermes.available:
+        problems.append(
+            "the in-pod Hermes vantage is unavailable; self-probe evidence is missing"
+        )
+        return problems
+    if not hermes.identity:
+        problems.append("the in-pod vantage did not report an identity")
+    elif operator.identity == hermes.identity:
+        problems.append(
+            "the operator and in-pod vantages authenticate as the same principal "
+            f"({operator.identity}); run the harness with a separate operator kubeconfig"
+        )
+    for name, label in ((SWITCHYARD, "Switchyard"), (CHAT, "chat/Telegram")):
+        record = by_name.get(name)
+        if record is None or not record.available:
+            problems.append(f"the required {label} vantage is unavailable")
+    return problems
+
+
+def run_check(
+    runner: Runner, spec: CheckSpec, vantages: Mapping[str, Vantage]
+) -> CheckResult:
+    """Run one check's steps and classify the result."""
+    outcomes: dict[str, Outcome] = {}
+    recorded: list[Outcome] = []
+    for step in spec.steps:
+        vantage = vantages.get(step.vantage)
+        if vantage is None:
+            outcome = Outcome(
+                argv=step.argv,
+                vantage=step.vantage,
+                error=f"vantage {step.vantage} is unavailable",
+            )
+        else:
+            outcome = runner.run(step.argv, vantage, step.max_bytes)
+        outcomes[step.key] = outcome
+        recorded.append(
+            outcome
+            if step.record
+            else Outcome(
+                argv=outcome.argv,
+                vantage=outcome.vantage,
+                returncode=outcome.returncode,
+                stdout=UNRECORDED if outcome.stdout else "",
+                stderr=UNRECORDED if outcome.stderr else "",
+                truncated=outcome.truncated,
+                duration_ms=outcome.duration_ms,
+                error=outcome.error,
+                executable_path=outcome.executable_path,
+                executable_sha256=outcome.executable_sha256,
+            )
+        )
+    evaluation = evaluate(spec, outcomes)
+    return CheckResult(
+        spec=spec,
+        status=evaluation.status,
+        reason=evaluation.reason,
+        evidence=evaluation.evidence,
+        outcomes=recorded,
+    )
+
+
+def run_catalog(
+    runner: Runner, specs: Sequence[CheckSpec], vantages: Mapping[str, Vantage]
+) -> list[CheckResult]:
+    """Run every check in the catalog.
+
+    Nothing is skipped once the deadline passes: the runner returns a
+    deadline outcome for each remaining command, which classifies as
+    ``NOT_RUN``, so a truncated sweep is visible in the report rather than
+    silently shorter.
+    """
+    return [run_check(runner, spec, vantages) for spec in specs]
+
+
+def build_report(
+    runner: Runner,
+    targets: Targets,
+    specs: Sequence[CheckSpec],
+    vantages: Mapping[str, Vantage],
+    records: Sequence[VantageRecord],
+    mode: str,
+    started_at: str,
+    extra_results: Sequence[CheckResult] = (),
+) -> Report:
+    """Run the catalog and assemble the report, including structural problems."""
+    problems = validate_catalog(specs) + vantage_problems(records)
+    report = Report(
+        mode=mode,
+        started_at=started_at,
+        baseline={
+            "namespace": targets.namespace,
+            "baseline_commit": targets.baseline_commit,
+            "dependency_pull_requests": list(targets.dependency_pull_requests),
+            "expected_suspensions": list(targets.expected_suspensions),
+            "node_count": targets.node_count,
+            "pool_replicas": targets.pool_replicas,
+        },
+        vantages=list(records),
+        harness_errors=problems,
+    )
+    armed_ids = {result.spec.id for result in extra_results}
+    runnable = [spec for spec in specs if spec.id not in armed_ids]
+    if problems:
+        report.results = [
+            CheckResult(spec=spec, status=NOT_RUN, reason="harness preflight failed")
+            for spec in runnable
+        ]
+        report.results.extend(extra_results)
+    else:
+        report.results = [*run_catalog(runner, runnable, vantages), *extra_results]
+    report.results.sort(key=lambda result: result.spec.id)
+    if runner.remaining_seconds <= 0:
+        report.harness_errors.append(
+            "the run deadline expired before every check completed"
+        )
+    report.finished_at = utc_now()
+    return report
+
+
+def unavailable_report(
+    mode: str, started_at: str, reason: str, targets: Targets
+) -> Report:
+    """Return a NO_GO report for a run that could not start at all."""
+    return Report(
+        mode=mode,
+        started_at=started_at,
+        finished_at=utc_now(),
+        baseline={"namespace": targets.namespace},
+        harness_errors=[reason],
+        results=[
+            CheckResult(
+                spec=CheckSpec(
+                    id="harness.startup",
+                    title="The harness could start and resolve its vantages",
+                    group="harness",
+                    rule="allowed",
+                ),
+                status=NOT_RUN,
+                reason=reason,
+            )
+        ],
+    )

scripts/tests/test_dashboard_render_support.py

@@ -0,0 +1,25 @@
+import importlib.util
+
+import pathlib
+
+def load_module():
+    path = pathlib.Path(__file__).resolve().parents[1] / "render" / "dashboards_render_atlas.py"
+    spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+def flatten_panels(panels):
+    flat = []
+    for panel in panels:
+        flat.append(panel)
+        flat.extend(panel.get("panels", []))
+    return flat
+
+__all__ = (
+    "flatten_panels",
+    "importlib",
+    "load_module",
+    "pathlib",
+)

scripts/tests/test_dashboards_render_atlas_drilldowns.py

@@ -1,6 +1,6 @@
 """Detailed Atlas Jobs dashboard collapse and drilldown contracts."""
 
-from scripts.tests.test_dashboards_render_atlas import load_module
+from scripts.tests.test_dashboard_render_support import load_module
 
 
 def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():

scripts/tests/test_dashboards_render_jobs.py

@@ -0,0 +1,281 @@
+"""Atlas testing and jobs dashboard rendering contracts."""
+
+from __future__ import annotations
+
+from test_dashboard_render_support import (
+    flatten_panels,
+    load_module,
+)
+
+
+def test_testing_suite_variable_uses_configured_suite_values_only():
+    mod = load_module()
+    variable = mod.testing_suite_variable()
+    configured_matcher = "|".join(mod.PLATFORM_TEST_SUITE_NAMES)
+    legacy_names = {"bstein-home", "data-prepper", "titan-iac", "pegasus-health"}
+    out_of_scope_names = {"arcanagon", "typhon"}
+
+    assert variable["allValue"] == configured_matcher
+    assert not any(alias in variable["query"] for alias in legacy_names)
+    assert not any(alias in variable["allValue"] for alias in legacy_names)
+    assert not any(name in variable["query"] for name in out_of_scope_names)
+    assert not any(name in variable["allValue"] for name in out_of_scope_names)
+    assert [option["value"] for option in variable["options"]] == mod.PLATFORM_TEST_SUITE_NAMES
+    assert "lesavka" in variable["allValue"]
+    assert any(option["value"] == "lesavka" for option in variable["options"])
+    assert not any(
+        option["value"] in out_of_scope_names for option in variable["options"]
+    )
+
+def test_testing_dashboard_is_public_but_jobs_dashboard_remains_internal():
+    mod = load_module()
+    jobs = mod.build_jobs_dashboard()
+    testing = mod.build_testing_dashboard()
+
+    assert jobs["folderUid"] == mod.PRIVATE_FOLDER
+    assert jobs["editable"] is True
+    assert testing["uid"] == "atlas-testing"
+    assert testing["folderUid"] == mod.PUBLIC_DASHBOARD_FOLDER
+    assert testing["editable"] is False
+
+def test_jobs_dashboard_separates_current_gate_health_from_reliability():
+    mod = load_module()
+    dashboard = mod.build_jobs_dashboard()
+    panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
+    variables_by_name = {item["name"]: item for item in dashboard["templating"]["list"]}
+
+    test_variable = variables_by_name["test"]
+    assert test_variable["query"].startswith("query_result(")
+    assert "platform_quality:test_case_health_rate:percent_1h" in test_variable["query"]
+    assert "platform_quality_gate_test_case_result" not in test_variable["query"]
+    assert test_variable["regex"] == '/test="([^"]+)"/'
+    branch_variable = variables_by_name["branch"]
+    assert branch_variable["type"] == "custom"
+    assert branch_variable["allValue"] == ".*"
+    assert "platform_quality_gate_build_info" not in branch_variable["query"]
+    assert "origin/main : origin/main" in branch_variable["query"]
+
+    assert "Current Gate Health (%)" in panels_by_title
+    assert "Suite Freshness (24h)" in panels_by_title
+    assert "Latest Gate Health by Suite" in panels_by_title
+    assert "Latest Gate Checks Passing by Suite" not in panels_by_title
+    assert "CI Run Success by Suite (24h)" in panels_by_title
+    assert "CI Run Success by Suite (7d rolling)" in panels_by_title
+    assert "Daily Run Volume (Selected Scope)" in panels_by_title
+    assert "Coverage History by Suite" in panels_by_title
+    assert "Files <=500 LOC History by Suite" in panels_by_title
+    assert "Run Reliability History by Suite" not in panels_by_title
+    assert "Current Gate Health by Suite" not in panels_by_title
+    assert "Coverage & LOC Compliance History" not in panels_by_title
+    assert "Run Status Mix (30d)" not in panels_by_title
+    assert "Failures by Suite (24h)" not in panels_by_title
+    assert "Success Rate by Suite (24h)" not in panels_by_title
+
+    current_gate_expr = panels_by_title["Latest Gate Health by Suite"]["targets"][0]["expr"]
+    assert 'check)' in current_gate_expr
+    assert "platform_quality:check_status:present_1h" in current_gate_expr
+    assert "platform_quality:test_category_health_rate:percent_1h" in current_gate_expr
+    assert "- 1" in current_gate_expr
+    assert '.*_quality_gate_checks_total' not in current_gate_expr
+    assert "last_over_time" not in current_gate_expr
+    assert 'label_replace' not in current_gate_expr
+    assert f'check=~"{mod.PLATFORM_TEST_ALWAYS_REQUIRED_CHECK_REGEX}"' in current_gate_expr
+    assert f'suite=~"{mod.PLATFORM_TEST_SUPPLY_CHAIN_REQUIRED_MATCHER}"' in current_gate_expr
+    assert 'check="supply_chain"' in current_gate_expr
+    assert f'check=~"{mod.PLATFORM_TEST_STANDARD_CHECK_REGEX}"' not in current_gate_expr
+    assert 'status=~"ok|passed|success|not_applicable|skipped|na|n/a"' in current_gate_expr
+    assert 'status!~"ok|passed|success|not_applicable|skipped|na|n/a"' in current_gate_expr
+    assert "unless on(suite, check)" in current_gate_expr
+    assert "tlast_over_time" not in current_gate_expr
+    assert panels_by_title["Current Gate Health (%)"]["gridPos"] == {"h": 4, "w": 3, "x": 0, "y": 0}
+    current_gate_stat_expr = panels_by_title["Current Gate Health (%)"]["targets"][0]["expr"]
+    assert "platform_quality:test_category_health_rate:percent_1h" in current_gate_stat_expr
+    assert "- 1" not in current_gate_stat_expr
+    assert panels_by_title["Suite Freshness (24h)"]["gridPos"] == {"h": 4, "w": 3, "x": 15, "y": 0}
+    suite_freshness_expr = panels_by_title["Suite Freshness (24h)"]["targets"][0]["expr"]
+    assert "platform_quality:suite_runs:increase_24h" in suite_freshness_expr
+    assert "max_over_time(platform_quality_gate_runs_total" not in suite_freshness_expr
+    assert "[7d:1h]" in panels_by_title["CI Run Success Rate (7d)"]["targets"][0]["expr"]
+    assert panels_by_title["Latest Gate Health by Suite"]["gridPos"]["w"] == 6
+    assert panels_by_title["CI Run Success by Suite (24h)"]["gridPos"]["w"] == 6
+    assert panels_by_title["Coverage by Suite (Latest, gate 95)"]["gridPos"] == {"h": 7, "w": 6, "x": 12, "y": 4}
+    assert panels_by_title["Files <=500 LOC by Suite (Latest)"]["gridPos"] == {"h": 7, "w": 6, "x": 18, "y": 4}
+
+    reliability_panel = panels_by_title["CI Run Success by Suite (24h)"]
+    reliability_expr = reliability_panel["targets"][0]["expr"]
+    assert "platform_quality:suite_runs:increase_24h" in reliability_expr
+    assert "platform_quality_gate_runs_total" not in reliability_expr
+    assert "> 0" in reliability_expr
+    assert "- 1" in reliability_expr
+    assert reliability_expr.startswith("sort(")
+    assert reliability_panel["fieldConfig"]["defaults"]["mappings"] == [
+        {"type": "value", "options": {"-1": {"text": "no runs"}}}
+    ]
+
+    rolling_panel = panels_by_title["CI Run Success by Suite (7d rolling)"]
+    assert rolling_panel["type"] == "state-timeline"
+    assert "[7d:1h]" in rolling_panel["targets"][0]["expr"]
+    category_panel = panels_by_title["Test Category Health History"]
+    assert category_panel["type"] == "state-timeline"
+    assert "category" in category_panel["targets"][0]["expr"]
+    assert category_panel["targets"][0]["format"] == "time_series"
+    assert category_panel["targets"][0]["range"] is True
+    assert "platform_quality:test_category_health_rate:percent_1h" in category_panel["targets"][0]["expr"]
+    assert 'test!=""' not in category_panel["targets"][0]["expr"]
+    assert f'category=~"{mod.PLATFORM_TEST_CATEGORY_REGEX}"' in category_panel["targets"][0]["expr"]
+    assert "installer" not in mod.PLATFORM_TEST_CATEGORY_REGEX
+    assert "Use the Suite filter" in category_panel["description"]
+    assert "skipped tests are healthy" in category_panel["description"]
+
+    tests_failure_expr = panels_by_title["Tests Failure Rate"]["targets"][0]["expr"]
+    assert "platform_quality:check_failed_flag:present_1h" in tests_failure_expr
+    assert "platform_quality:check_seen_flag:present_1h" in tests_failure_expr
+    assert "platform_quality:check_status:present_1h" not in tests_failure_expr
+
+    tests_healthy_expr = panels_by_title["Tests Healthy Rate"]["targets"][0]["expr"]
+    assert "platform_quality:check_healthy_flag:present_1h" in tests_healthy_expr
+    assert "platform_quality:check_seen_flag:present_1h" in tests_healthy_expr
+
+    problematic_expr = panels_by_title["Problematic Tests Over Time (Top failures)"]["targets"][0]["expr"]
+    assert "platform_quality:test_case_status:count_1h" in problematic_expr
+    assert "platform_quality_gate_test_case_result" not in problematic_expr
+    assert "[24h:1h]" in problematic_expr
+    assert ">= 2" in problematic_expr
+    assert "@ end()" in problematic_expr
+
+    coverage_panel = panels_by_title["Coverage History by Suite"]
+    loc_panel = panels_by_title["Files <=500 LOC History by Suite"]
+    assert coverage_panel["type"] == "state-timeline"
+    assert loc_panel["type"] == "state-timeline"
+    assert coverage_panel["targets"][0]["expr"] != loc_panel["targets"][0]["expr"]
+    assert "platform_quality:suite_coverage_percent:latest_1h" in coverage_panel["targets"][0]["expr"]
+    assert "platform_quality:suite_source_files_total:latest_1h" in loc_panel["targets"][0]["expr"]
+    assert "platform_quality:suite_source_lines_over_500_total:latest_1h" in loc_panel["targets"][0]["expr"]
+    assert "tlast_over_time" not in coverage_panel["targets"][0]["expr"]
+    assert "tlast_over_time" not in loc_panel["targets"][0]["expr"]
+    assert "platform_quality_gate_workspace_line_coverage_percent" not in panels_by_title["Coverage by Suite (Latest, gate 95)"]["targets"][0]["expr"]
+    assert "platform_quality:suite_coverage_percent:latest_1h" in panels_by_title["Avg Coverage (%)"]["targets"][0]["expr"]
+    assert panels_by_title["Daily Run Volume (Selected Scope)"]["gridPos"] == {"h": 8, "w": 8, "x": 0, "y": 27}
+    assert coverage_panel["gridPos"] == {"h": 8, "w": 8, "x": 8, "y": 27}
+    assert loc_panel["gridPos"] == {"h": 8, "w": 8, "x": 16, "y": 27}
+
+    run_volume_panel = panels_by_title["Daily Run Volume (Selected Scope)"]
+    assert run_volume_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
+    assert "[$__interval]" not in run_volume_panel["targets"][0]["expr"]
+
+def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors():
+    mod = load_module()
+    dashboard = mod.build_jobs_dashboard()
+    panels = flatten_panels(dashboard["panels"])
+    bar_gauges = [panel for panel in panels if panel["type"] == "bargauge"]
+
+    assert bar_gauges
+    assert all(panel["options"]["displayMode"] == "basic" for panel in bar_gauges)
+    assert all(
+        panel["fieldConfig"]["defaults"]["color"]["mode"] == "thresholds"
+        for panel in bar_gauges
+    )
+
+    reliability_panel = next(
+        panel for panel in panels if panel["title"] == "CI Run Success by Suite (24h)"
+    )
+    threshold_steps = reliability_panel["fieldConfig"]["defaults"]["thresholds"]["steps"]
+
+    assert {"color": "dark-yellow", "value": 93} in threshold_steps
+    assert {"color": "dark-blue", "value": 100} in threshold_steps
+def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
+    mod = load_module()
+    dashboard = mod.build_jobs_dashboard()
+    panels = dashboard["panels"]
+    rows = [panel for panel in panels if panel["type"] == "row"]
+    visible_query_panels = [panel for panel in panels if panel["type"] != "row"]
+    nested_panels_by_title = {
+        child["title"]: child
+        for row in rows
+        for child in row.get("panels", [])
+    }
+
+    assert len(panels) == 18
+    assert len(visible_query_panels) == 12
+    assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12
+    assert all(
+        panel["title"] != "Coverage Gap to 95% by Suite"
+        for panel in visible_query_panels
+    )
+    assert [row["title"] for row in rows] == [
+        "CI Runs And Test Result History",
+        "Check Failure Rates By Suite",
+        "Check Healthy Rates By Suite",
+        "Test Drilldowns And Problem Tests",
+        "Telemetry Completeness And Branches",
+        "SonarQube Project Health",
+    ]
+    assert all(row["collapsed"] for row in rows)
+
+    assert "Coverage Failure Rate" in nested_panels_by_title
+    assert "Supply Chain Healthy Rate" in nested_panels_by_title
+    assert "Test Category Health History" in nested_panels_by_title
+    assert "Selected Test Pass Rate History" in nested_panels_by_title
+    assert "Coverage Metrics Present by Suite" in nested_panels_by_title
+    assert "SonarQube API Up" in nested_panels_by_title
+
+    failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"]
+    assert failure_rate_panel["type"] == "state-timeline"
+    assert failure_rate_panel["fieldConfig"]["defaults"]["unit"] == "percent"
+    assert failure_rate_panel["fieldConfig"]["defaults"]["max"] == 100
+    assert failure_rate_panel["fieldConfig"]["defaults"]["thresholds"]["steps"][0]["color"] == "dark-blue"
+    assert "increase(" not in failure_rate_panel["targets"][0]["expr"]
+    assert "platform_quality:check_failed_flag:present_1h" in failure_rate_panel["targets"][0]["expr"]
+    assert "platform_quality:check_seen_flag:present_1h" in failure_rate_panel["targets"][0]["expr"]
+    assert "platform_quality:check_status:present_1h" not in failure_rate_panel["targets"][0]["expr"]
+    assert '.*_quality_gate_checks_total' not in failure_rate_panel["targets"][0]["expr"]
+    assert "last_over_time" not in failure_rate_panel["targets"][0]["expr"]
+    assert 'label_replace' not in failure_rate_panel["targets"][0]["expr"]
+    assert "0 *" in failure_rate_panel["targets"][0]["expr"]
+    assert "and on(suite)" not in failure_rate_panel["targets"][0]["expr"]
+
+    pass_rate_panel = nested_panels_by_title["Selected Test Pass Rate History"]
+    assert pass_rate_panel["type"] == "state-timeline"
+    assert "platform_quality:test_case_pass_rate:percent_1h" in pass_rate_panel["targets"][0]["expr"]
+    assert "platform_quality_gate_test_case_result" not in pass_rate_panel["targets"][0]["expr"]
+
+    pass_fail_panel = nested_panels_by_title["Selected Test Pass/Fail History"]
+    assert pass_fail_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
+    assert all(
+        "platform_quality:test_case_status:count_1h" in target["expr"]
+        for target in pass_fail_panel["targets"]
+    )
+
+    problematic_panel = nested_panels_by_title["Problematic Tests Over Time (Top failures)"]
+    assert problematic_panel["type"] == "state-timeline"
+    assert problematic_panel["gridPos"]["w"] == 24
+    assert 'test!=""' in problematic_panel["targets"][0]["expr"]
+    assert "vector(0)" not in problematic_panel["targets"][0]["expr"]
+    assert problematic_panel["fieldConfig"]["defaults"]["thresholds"]["steps"] == [
+        {"color": "dark-blue", "value": None},
+        {"color": "dark-green", "value": 2},
+        {"color": "dark-yellow", "value": 3},
+        {"color": "dark-orange", "value": 5},
+        {"color": "dark-red", "value": 8},
+    ]
+    assert "rolling 24h failure count" in problematic_panel["description"]
+    assert "at least two recent failures" in problematic_panel["description"]
+
+    sonar_mix_panel = nested_panels_by_title["Sonar Gate Status Mix (Selected)"]
+    sonar_health_panel = nested_panels_by_title["Sonar Gate Health by Project"]
+    assert sonar_mix_panel["gridPos"]["w"] == 4
+    assert sonar_health_panel["gridPos"]["w"] == 8
+    assert sonar_health_panel["type"] == "state-timeline"
+    assert "platform_quality:sonar_gate_health_percent:latest_1h" in sonar_health_panel["targets"][0]["expr"]
+    assert "sonarqube_project_quality_gate_pass" not in sonar_health_panel["targets"][0]["expr"]
+
+    branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"]
+    recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"]
+    assert branch_panel["gridPos"]["x"] == 12
+    assert recent_branch_panel["gridPos"]["x"] == 18
+    assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"]
+    assert "[7d:1h]" in branch_panel["targets"][0]["expr"]
+    assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent"
+    assert "unless on(suite)" in branch_panel["targets"][0]["expr"]
+    assert "> bool 0" in branch_panel["targets"][0]["expr"]
+    assert branch_panel["targets"][0]["expr"].startswith("sort(")

scripts/tests/test_dashboards_render_overview.py

@@ -1,22 +1,11 @@
-import importlib.util
-import pathlib
+"""Atlas overview dashboard rendering contracts."""
 
+from __future__ import annotations
 
-def load_module():
-    path = pathlib.Path(__file__).resolve().parents[1] / "render" / "dashboards_render_atlas.py"
-    spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path)
-    module = importlib.util.module_from_spec(spec)
-    assert spec.loader is not None
-    spec.loader.exec_module(module)
-    return module
-
-
-def flatten_panels(panels):
-    flat = []
-    for panel in panels:
-        flat.append(panel)
-        flat.extend(panel.get("panels", []))
-    return flat
+from test_dashboard_render_support import (
+    flatten_panels,
+    load_module,
+)
 
 
 def test_table_panel_options_and_filterable():
@@ -39,7 +28,6 @@ def test_table_panel_options_and_filterable():
     assert panel["options"]["showHeader"] is True
     assert panel["targets"][0]["format"] == "table"
 
-
 def test_node_filter_and_expr_helpers():
     mod = load_module()
     expr = mod.node_filter("titan-.*")
@@ -51,7 +39,6 @@ def test_node_filter_and_expr_helpers():
     assert "* 100, 0), 100)" in cpu_expr
     assert "node_memory_MemAvailable_bytes" in mem_expr
 
-
 def test_overview_availability_panel_uses_recorded_365d_rollup():
     mod = load_module()
     dashboard = mod.build_overview()
@@ -76,7 +63,6 @@ def test_overview_availability_panel_uses_recorded_365d_rollup():
     assert "daily rollup job publishes one annual sample" in panel["description"]
     assert "keeps it for up to 48 hours" in panel["description"]
 
-
 def test_overview_uses_readable_quality_power_and_gitops_panels():
     mod = load_module()
     dashboard = mod.build_overview()
@@ -195,7 +181,6 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
     assert "continuously sampled" in panels_by_title["Namespace GPU Utilization"]["description"]
     assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
 
-
 def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
     mod = load_module()
     dashboard = mod.build_gpu_dashboard()
@@ -212,7 +197,6 @@ def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_famili
     assert "nvidia_process_gpu_sm_util_percent" in process_expr
     assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
 
-
 def test_overview_and_testing_panels_all_have_concise_descriptions():
     mod = load_module()
 
@@ -221,7 +205,6 @@ def test_overview_and_testing_panels_all_have_concise_descriptions():
         assert all(panel.get("description") for panel in panels if panel["type"] != "row")
         assert "85.7" not in repr(dashboard)
 
-
 def test_render_configmap_writes(tmp_path):
     mod = load_module()
     mod.DASHBOARD_DIR = tmp_path / "dash"
@@ -236,183 +219,3 @@ def test_render_configmap_writes(tmp_path):
     content = (tmp_path / "cm.yaml").read_text()
     assert "kind: ConfigMap" in content
     assert f"{uid}.json" in content
-
-
-def test_testing_suite_variable_uses_configured_suite_values_only():
-    mod = load_module()
-    variable = mod.testing_suite_variable()
-    configured_matcher = "|".join(mod.PLATFORM_TEST_SUITE_NAMES)
-    legacy_names = {"bstein-home", "data-prepper", "titan-iac", "pegasus-health"}
-    out_of_scope_names = {"arcanagon", "typhon"}
-
-    assert variable["allValue"] == configured_matcher
-    assert not any(alias in variable["query"] for alias in legacy_names)
-    assert not any(alias in variable["allValue"] for alias in legacy_names)
-    assert not any(name in variable["query"] for name in out_of_scope_names)
-    assert not any(name in variable["allValue"] for name in out_of_scope_names)
-    assert [option["value"] for option in variable["options"]] == mod.PLATFORM_TEST_SUITE_NAMES
-    assert "lesavka" in variable["allValue"]
-    assert any(option["value"] == "lesavka" for option in variable["options"])
-    assert not any(
-        option["value"] in out_of_scope_names for option in variable["options"]
-    )
-
-
-def test_testing_dashboard_is_public_but_jobs_dashboard_remains_internal():
-    mod = load_module()
-    jobs = mod.build_jobs_dashboard()
-    testing = mod.build_testing_dashboard()
-
-    assert jobs["folderUid"] == mod.PRIVATE_FOLDER
-    assert jobs["editable"] is True
-    assert testing["uid"] == "atlas-testing"
-    assert testing["folderUid"] == mod.PUBLIC_DASHBOARD_FOLDER
-    assert testing["editable"] is False
-
-
-def test_jobs_dashboard_separates_current_gate_health_from_reliability():
-    mod = load_module()
-    dashboard = mod.build_jobs_dashboard()
-    panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
-    variables_by_name = {item["name"]: item for item in dashboard["templating"]["list"]}
-
-    test_variable = variables_by_name["test"]
-    assert test_variable["query"].startswith("query_result(")
-    assert "platform_quality:test_case_health_rate:percent_1h" in test_variable["query"]
-    assert "platform_quality_gate_test_case_result" not in test_variable["query"]
-    assert test_variable["regex"] == '/test="([^"]+)"/'
-    branch_variable = variables_by_name["branch"]
-    assert branch_variable["type"] == "custom"
-    assert branch_variable["allValue"] == ".*"
-    assert "platform_quality_gate_build_info" not in branch_variable["query"]
-    assert "origin/main : origin/main" in branch_variable["query"]
-
-    assert "Current Gate Health (%)" in panels_by_title
-    assert "Suite Freshness (24h)" in panels_by_title
-    assert "Latest Gate Health by Suite" in panels_by_title
-    assert "Latest Gate Checks Passing by Suite" not in panels_by_title
-    assert "CI Run Success by Suite (24h)" in panels_by_title
-    assert "CI Run Success by Suite (7d rolling)" in panels_by_title
-    assert "Daily Run Volume (Selected Scope)" in panels_by_title
-    assert "Coverage History by Suite" in panels_by_title
-    assert "Files <=500 LOC History by Suite" in panels_by_title
-    assert "Run Reliability History by Suite" not in panels_by_title
-    assert "Current Gate Health by Suite" not in panels_by_title
-    assert "Coverage & LOC Compliance History" not in panels_by_title
-    assert "Run Status Mix (30d)" not in panels_by_title
-    assert "Failures by Suite (24h)" not in panels_by_title
-    assert "Success Rate by Suite (24h)" not in panels_by_title
-
-    current_gate_expr = panels_by_title["Latest Gate Health by Suite"]["targets"][0]["expr"]
-    assert 'check)' in current_gate_expr
-    assert "platform_quality:check_status:present_1h" in current_gate_expr
-    assert "platform_quality:test_category_health_rate:percent_1h" in current_gate_expr
-    assert "- 1" in current_gate_expr
-    assert '.*_quality_gate_checks_total' not in current_gate_expr
-    assert "last_over_time" not in current_gate_expr
-    assert 'label_replace' not in current_gate_expr
-    assert f'check=~"{mod.PLATFORM_TEST_ALWAYS_REQUIRED_CHECK_REGEX}"' in current_gate_expr
-    assert f'suite=~"{mod.PLATFORM_TEST_SUPPLY_CHAIN_REQUIRED_MATCHER}"' in current_gate_expr
-    assert 'check="supply_chain"' in current_gate_expr
-    assert f'check=~"{mod.PLATFORM_TEST_STANDARD_CHECK_REGEX}"' not in current_gate_expr
-    assert 'status=~"ok|passed|success|not_applicable|skipped|na|n/a"' in current_gate_expr
-    assert 'status!~"ok|passed|success|not_applicable|skipped|na|n/a"' in current_gate_expr
-    assert "unless on(suite, check)" in current_gate_expr
-    assert "tlast_over_time" not in current_gate_expr
-    assert panels_by_title["Current Gate Health (%)"]["gridPos"] == {"h": 4, "w": 3, "x": 0, "y": 0}
-    current_gate_stat_expr = panels_by_title["Current Gate Health (%)"]["targets"][0]["expr"]
-    assert "platform_quality:test_category_health_rate:percent_1h" in current_gate_stat_expr
-    assert "- 1" not in current_gate_stat_expr
-    assert panels_by_title["Suite Freshness (24h)"]["gridPos"] == {"h": 4, "w": 3, "x": 15, "y": 0}
-    suite_freshness_expr = panels_by_title["Suite Freshness (24h)"]["targets"][0]["expr"]
-    assert "platform_quality:suite_runs:increase_24h" in suite_freshness_expr
-    assert "max_over_time(platform_quality_gate_runs_total" not in suite_freshness_expr
-    assert "[7d:1h]" in panels_by_title["CI Run Success Rate (7d)"]["targets"][0]["expr"]
-    assert panels_by_title["Latest Gate Health by Suite"]["gridPos"]["w"] == 6
-    assert panels_by_title["CI Run Success by Suite (24h)"]["gridPos"]["w"] == 6
-    assert panels_by_title["Coverage by Suite (Latest, gate 95)"]["gridPos"] == {"h": 7, "w": 6, "x": 12, "y": 4}
-    assert panels_by_title["Files <=500 LOC by Suite (Latest)"]["gridPos"] == {"h": 7, "w": 6, "x": 18, "y": 4}
-
-    reliability_panel = panels_by_title["CI Run Success by Suite (24h)"]
-    reliability_expr = reliability_panel["targets"][0]["expr"]
-    assert "platform_quality:suite_runs:increase_24h" in reliability_expr
-    assert "platform_quality_gate_runs_total" not in reliability_expr
-    assert "> 0" in reliability_expr
-    assert "- 1" in reliability_expr
-    assert reliability_expr.startswith("sort(")
-    assert reliability_panel["fieldConfig"]["defaults"]["mappings"] == [
-        {"type": "value", "options": {"-1": {"text": "no runs"}}}
-    ]
-
-    rolling_panel = panels_by_title["CI Run Success by Suite (7d rolling)"]
-    assert rolling_panel["type"] == "state-timeline"
-    assert "[7d:1h]" in rolling_panel["targets"][0]["expr"]
-    category_panel = panels_by_title["Test Category Health History"]
-    assert category_panel["type"] == "state-timeline"
-    assert "category" in category_panel["targets"][0]["expr"]
-    assert category_panel["targets"][0]["format"] == "time_series"
-    assert category_panel["targets"][0]["range"] is True
-    assert "platform_quality:test_category_health_rate:percent_1h" in category_panel["targets"][0]["expr"]
-    assert 'test!=""' not in category_panel["targets"][0]["expr"]
-    assert f'category=~"{mod.PLATFORM_TEST_CATEGORY_REGEX}"' in category_panel["targets"][0]["expr"]
-    assert "installer" not in mod.PLATFORM_TEST_CATEGORY_REGEX
-    assert "Use the Suite filter" in category_panel["description"]
-    assert "skipped tests are healthy" in category_panel["description"]
-
-    tests_failure_expr = panels_by_title["Tests Failure Rate"]["targets"][0]["expr"]
-    assert "platform_quality:check_failed_flag:present_1h" in tests_failure_expr
-    assert "platform_quality:check_seen_flag:present_1h" in tests_failure_expr
-    assert "platform_quality:check_status:present_1h" not in tests_failure_expr
-
-    tests_healthy_expr = panels_by_title["Tests Healthy Rate"]["targets"][0]["expr"]
-    assert "platform_quality:check_healthy_flag:present_1h" in tests_healthy_expr
-    assert "platform_quality:check_seen_flag:present_1h" in tests_healthy_expr
-
-    problematic_expr = panels_by_title["Problematic Tests Over Time (Top failures)"]["targets"][0]["expr"]
-    assert "platform_quality:test_case_status:count_1h" in problematic_expr
-    assert "platform_quality_gate_test_case_result" not in problematic_expr
-    assert "[24h:1h]" in problematic_expr
-    assert ">= 2" in problematic_expr
-    assert "@ end()" in problematic_expr
-
-    coverage_panel = panels_by_title["Coverage History by Suite"]
-    loc_panel = panels_by_title["Files <=500 LOC History by Suite"]
-    assert coverage_panel["type"] == "state-timeline"
-    assert loc_panel["type"] == "state-timeline"
-    assert coverage_panel["targets"][0]["expr"] != loc_panel["targets"][0]["expr"]
-    assert "platform_quality:suite_coverage_percent:latest_1h" in coverage_panel["targets"][0]["expr"]
-    assert "platform_quality:suite_source_files_total:latest_1h" in loc_panel["targets"][0]["expr"]
-    assert "platform_quality:suite_source_lines_over_500_total:latest_1h" in loc_panel["targets"][0]["expr"]
-    assert "tlast_over_time" not in coverage_panel["targets"][0]["expr"]
-    assert "tlast_over_time" not in loc_panel["targets"][0]["expr"]
-    assert "platform_quality_gate_workspace_line_coverage_percent" not in panels_by_title["Coverage by Suite (Latest, gate 95)"]["targets"][0]["expr"]
-    assert "platform_quality:suite_coverage_percent:latest_1h" in panels_by_title["Avg Coverage (%)"]["targets"][0]["expr"]
-    assert panels_by_title["Daily Run Volume (Selected Scope)"]["gridPos"] == {"h": 8, "w": 8, "x": 0, "y": 27}
-    assert coverage_panel["gridPos"] == {"h": 8, "w": 8, "x": 8, "y": 27}
-    assert loc_panel["gridPos"] == {"h": 8, "w": 8, "x": 16, "y": 27}
-
-    run_volume_panel = panels_by_title["Daily Run Volume (Selected Scope)"]
-    assert run_volume_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
-    assert "[$__interval]" not in run_volume_panel["targets"][0]["expr"]
-
-
-def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors():
-    mod = load_module()
-    dashboard = mod.build_jobs_dashboard()
-    panels = flatten_panels(dashboard["panels"])
-    bar_gauges = [panel for panel in panels if panel["type"] == "bargauge"]
-
-    assert bar_gauges
-    assert all(panel["options"]["displayMode"] == "basic" for panel in bar_gauges)
-    assert all(
-        panel["fieldConfig"]["defaults"]["color"]["mode"] == "thresholds"
-        for panel in bar_gauges
-    )
-
-    reliability_panel = next(
-        panel for panel in panels if panel["title"] == "CI Run Success by Suite (24h)"
-    )
-    threshold_steps = reliability_panel["fieldConfig"]["defaults"]["thresholds"]["steps"]
-
-    assert {"color": "dark-yellow", "value": 93} in threshold_steps
-    assert {"color": "dark-blue", "value": 100} in threshold_steps

scripts/tests/test_mailu_sync_edges.py

@@ -0,0 +1,96 @@
+"""Retry, attribute-corruption, and skip-path edges for the Mailu sync job."""
+
+from test_mailu_sync import _FakeResponse, _FakeSession, load_sync_module
+
+def test_retry_helpers_with_zero_attempts_do_nothing(monkeypatch):
+    sync = load_sync_module(monkeypatch)
+    monkeypatch.setattr(
+        sync.psycopg2,
+        "connect",
+        lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not connect")),
+    )
+
+    assert sync.retry_request(
+        "request",
+        lambda: (_ for _ in ()).throw(AssertionError("must not run")),
+        attempts=0,
+    ) is None
+    assert sync.retry_db_connect(attempts=0) is None
+
+
+def test_kc_update_attributes_replaces_a_corrupt_attribute_map(monkeypatch):
+    sync = load_sync_module(monkeypatch)
+    current_resp = _FakeResponse({"username": "u1", "attributes": "corrupt"})
+    ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}})
+    sync.SESSION = _FakeSession(_FakeResponse({}), [current_resp, ok_resp])
+
+    sync.kc_update_attributes(
+        "token",
+        {"id": "u1", "username": "u1"},
+        {"mailu_app_password": "abc"},
+    )
+
+    assert sync.SESSION.put_called and sync.SESSION.get_calls == 2
+
+
+def test_main_skips_disabled_users_and_settled_attribute_sets(monkeypatch):
+    sync = load_sync_module(monkeypatch)
+    monkeypatch.setattr(sync.bcrypt_sha256, "hash", lambda password: f"hash:{password}")
+    users = [
+        {
+            "id": "u_disabled",
+            "username": "disabled",
+            "email": "disabled@example.com",
+            "enabled": False,
+            "attributes": {"mailu_enabled": ["true"]},
+        },
+        {
+            "id": "u_settled",
+            "username": "settled",
+            "email": "settled@example.com",
+            "attributes": {
+                "mailu_enabled": ["true"],
+                "mailu_email": ["settled@example.com"],
+                "mailu_app_password": ["keepme"],
+            },
+        },
+    ]
+    updated = []
+
+    class _Cursor:
+        def __init__(self):
+            self.executions = []
+
+        def execute(self, sql, params):
+            self.executions.append(params)
+
+        def close(self):
+            return None
+
+    class _Conn:
+        def __init__(self):
+            self.autocommit = False
+            self._cursor = _Cursor()
+
+        def cursor(self, cursor_factory=None):
+            return self._cursor
+
+        def close(self):
+            return None
+
+    conn = _Conn()
+    monkeypatch.setattr(sync, "get_kc_token", lambda: "tok")
+    monkeypatch.setattr(sync, "kc_get_users", lambda token: users)
+    monkeypatch.setattr(
+        sync,
+        "kc_update_attributes",
+        lambda token, user, attrs: updated.append(user["id"]),
+    )
+    monkeypatch.setattr(sync.psycopg2, "connect", lambda **kwargs: conn)
+
+    sync.main()
+
+    assert updated == []
+    assert [params["email"] for params in conn._cursor.executions] == [
+        "settled@example.com"
+    ]

scripts/tests/test_mailu_sync_listener.py

@@ -144,3 +144,47 @@ def test_listener_log_message_is_quiet(monkeypatch):
     handler = listener.Handler.__new__(listener.Handler)
 
     assert handler.log_message("ignored %s", "value") is None
+
+
+def test_listener_post_treats_non_object_payloads_as_plain_triggers(monkeypatch):
+    listener = load_listener_module(monkeypatch)
+    called = []
+    monkeypatch.setattr(
+        listener,
+        "_trigger_sync_async",
+        lambda force=False: called.append(force) or True,
+    )
+    handler = _handler_for(listener, "[1, 2, 3]")
+
+    handler.do_POST()
+
+    assert called == [False]
+    assert handler.responses == [202]
+
+
+def test_listener_module_serves_forever_when_run_directly(monkeypatch):
+    import http.server
+    import runpy
+
+    served = []
+
+    class _FakeServer:
+        def __init__(self, address, handler):
+            served.append((address, handler.__name__))
+
+        def serve_forever(self):
+            served.append("serving")
+
+    monkeypatch.setenv("MAILU_SYNC_WAIT_TIMEOUT_SEC", "0")
+    monkeypatch.setattr(http.server, "ThreadingHTTPServer", _FakeServer)
+    module_path = (
+        pathlib.Path(__file__).resolve().parents[2]
+        / "services"
+        / "mailu"
+        / "scripts"
+        / "mailu_sync_listener.py"
+    )
+
+    runpy.run_path(str(module_path), run_name="__main__")
+
+    assert served == [(("", 8080), "Handler"), "serving"]

services/hermes-scm-broker/deployment.yaml

@@ -17,6 +17,7 @@ spec:
       labels:
         app: hermes-scm-broker
       annotations:
+        ai.bstein.dev/config-rev: scm-boundary-v2
         vault.hashicorp.com/agent-inject: "true"
         vault.hashicorp.com/role: hermes-scm-broker
         vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea
@@ -91,7 +92,7 @@ spec:
       volumes:
         - name: broker-code
           configMap:
-            name: hermes-scm-boundary
+            name: hermes-scm-boundary-v2
             defaultMode: 0555
         - name: tmp
           emptyDir:

services/hermes-scm-broker/networkpolicy.yaml

@@ -17,6 +17,12 @@ spec:
           podSelector:
             matchLabels:
               app: hermes-agent
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: hermes
+          podSelector:
+            matchLabels:
+              app: hermes-execution-mediator
       ports:
         - {protocol: TCP, port: 9081}
   egress:

services/hermes/NOTES.md

@@ -361,6 +361,60 @@ Use this short explanation:
   leading different real incidents, improving the skill after failures, and
   teaching the architecture without prompts.
 
+## Distributed execution-pool rollout boundary
+
+- Do not reconcile the pool until the approved credential owner has provisioned
+  six distinct Vault fields: `execution_worker_{0,1,2}_claude_credentials_json`
+  and `execution_worker_{0,1,2}_codex_auth_json`. Each ordinal needs an
+  independent account or refresh-token lineage; copying one rotating refresh
+  token into multiple fields recreates the lost-update failure this design
+  prevents.
+- Worker credential files live on separate ordinal-owned RWO `provider-access`
+  claims. Provider refresh updates these durable private copies across Pod
+  restarts; they are never synchronized back to Vault. Rotate one bootstrap
+  credential at a time through the reviewed Vault workflow and reinitialize
+  only that ordinal after human approval.
+- Existing task worktrees remain on the single local owner lane. Only tasks
+  without `workspace_path` enter the distributed pool, where each ordinal owns
+  one RWO checkout. Repository and base-branch identity comes from the canonical
+  board registry rather than task-supplied metadata.
+- Model Pods have no pool key, broker mount, broker egress, or Kubernetes token.
+  Ordinal mediator Deployments are separate network identities that co-mount the
+  matching workspace PVC; they alone authenticate exact-run results and reach the
+  PR14 SCM broker.
+- The ordinal workspace claim is `ReadWriteMany` because the worker and its
+  mediator both mount it. With `ReadWriteOnce` a drain or preemption that moved
+  only one of them left the other holding the attachment on the old node, and
+  the ordinal deadlocked on Multi-Attach until an operator deleted a Pod.
+  Colocation is now a scheduling preference and both Pods run at `scavenger`
+  priority, so either can be evicted and rescheduled on its own. There is
+  deliberately no PodDisruptionBudget: this is a best-effort pool and a budget
+  would impede node drains. Claims only one Pod mounts stay `ReadWriteOnce`.
+- `lease_failed` is a *retryable* durable state, not a terminal one. It means the
+  ordinal was released but Kanban has not confirmed the outcome, so every
+  maintenance pass retries it. Only `finalized` and `stale` are terminal, and
+  only terminal rows are garbage-collected, so no run is dropped before the pool
+  knows what happened to it. If the log keeps reporting a deferred lease park for
+  the same run, Kanban is refusing an exact-run park on a task it still reports
+  as running: inspect that task rather than deleting the row.
+- The broker accepts branch creation only. A pool retry that adds commits
+  therefore publishes a new `<branch>-attempt-<n>` (or, failing that,
+  `<branch>-<head12>`) ref in the same reviewed namespace instead of updating the
+  ref a previous attempt published, and adopts an existing ref that already
+  points at the exact head rather than pushing again. Expect one draft per
+  published ref; each is still human-reviewed. A submission that cannot land
+  blocks the run with the reason and leaves the commits on the ordinal's
+  workspace, so prior work is recoverable.
+- Provider CLIs install once per pinned version onto a durable per-ordinal
+  `tools` claim, re-verified against the real binaries so a pruned cache
+  reinstalls. The pool is intentionally absent from the `hermes` Kustomization's
+  `healthChecks`: gating that 10m window on a best-effort pool would stall
+  `hermes-chat` and `hermes-observer-bindings`, which `dependsOn: hermes`.
+- A hashed execution-pool ConfigMap, protocol-version readiness checks, and
+  versioned SCM boundary name make code/config changes controlled rollouts. A
+  rollout is still a human-reviewed operation; this repository change does not
+  reconcile or deploy it.
+
 ## Your shortest path to fluency
 
 1. Explain the request diagram without looking.

services/hermes/agent-configmap.yaml

@@ -87,7 +87,27 @@ data:
       max_in_progress_per_profile: 1
       auto_decompose: true
       auto_decompose_per_tick: 2
+      # In-pod autonomous supervisor (kanban_supervisor.py). Default false so it
+      # stays inert until the external codex-shepherd is retired; re-read each
+      # tick like auto_decompose. When true it drives the cross-card
+      # implement->review->repair->re-review chain by creating Kanban cards
+      # only (subscription lanes), never merging/approving. Bounded by the
+      # cycle and concurrent-chain ceilings below.
+      auto_supervise: false
+      supervise_interval_seconds: 30
+      supervise_max_cycles: 5
+      supervise_max_chains: 20
+      supervise_review_assignee: cli-claude-xhigh
+      supervise_repair_assignee: cli-auto
       dispatch_stale_timeout_seconds: 14400
+      # Steady-state quota-aware routing for the direct CLI lane: below this
+      # remaining-percent a provider stops receiving NEW cli-auto work while
+      # active work finishes; lane-observed capacity/auth failures exclude a
+      # provider until the cooldown elapses, its quota window resets, or a
+      # fresh success is recorded.
+      provider_quota_min_remaining_percent: 15
+      provider_capacity_cooldown_seconds: 300
+      provider_auth_cooldown_seconds: 3600
 
     model_catalog:
       enabled: true

services/hermes/agent-deployment.yaml

@@ -25,7 +25,7 @@ spec:
         ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
         ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
         ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
-        ai.bstein.dev/config-rev: "20260816-auth-health-quarantine-v6"
+        ai.bstein.dev/config-rev: "20260818-ai-usage-exporter-split-v3"
         prometheus.io/scrape: "true"
         prometheus.io/path: /metrics
         prometheus.io/port: "9010"
@@ -864,6 +864,8 @@ spec:
               . /opt/data/.env
               set +a
               exec /opt/hermes/.venv/bin/python /opt/coordinator/cli_lane_runner.py
+          ports:
+            - {name: lane-metrics, containerPort: 9011, protocol: TCP}
           env:
             - {name: HERMES_HOME, value: /opt/data}
             - {name: HERMES_AUTH_FILE, value: /runtime-access/hermes-auth.json}
@@ -873,6 +875,9 @@ spec:
             - {name: KUBECONFIG, value: /opt/data/home/.kube/config}
             - {name: PYTHONPATH, value: /opt/hermes}
             - {name: HERMES_CLI_LANE_CONCURRENCY, value: "2"}
+            - {name: HERMES_CLI_HEALTH_MAX_AGE_SECONDS, value: "60"}
+            - {name: HERMES_CLI_LANE_METRICS_PORT, value: "9011"}
+            - {name: HERMES_CLI_QUOTA_METRICS_URL, value: "http://127.0.0.1:9010/metrics"}
             - {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
             - {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
           securityContext:
@@ -895,6 +900,15 @@ spec:
             - {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true}
             - {name: scm-boundary, mountPath: /opt/scm, readOnly: true}
             - {name: tmp, mountPath: /tmp}
+          readinessProbe:
+            exec:
+              command:
+                - /opt/hermes/.venv/bin/python
+                - /opt/coordinator/cli_lane_capabilities.py
+            initialDelaySeconds: 2
+            periodSeconds: 5
+            timeoutSeconds: 5
+            failureThreshold: 3
           resources:
             requests: {cpu: 100m, memory: 256Mi}
             limits: {cpu: "2", memory: 6Gi}
@@ -927,6 +941,37 @@ spec:
           resources:
             requests: {cpu: 25m, memory: 64Mi}
             limits: {cpu: 250m, memory: 512Mi}
+        - name: kanban-supervisor
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
+          imagePullPolicy: IfNotPresent
+          # Autonomous cross-card review->repair->re-review driver. Reads/writes
+          # only the local Kanban DB under /opt/data; creates Kanban cards that
+          # route through the existing subscription lanes. No runtime-access
+          # mount and no provider client: it holds no metered/API-key path. Inert
+          # until kanban.auto_supervise is set true in the deployed config.
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/kanban_supervisor.py]
+          env:
+            - {name: HERMES_HOME, value: /opt/data}
+            - {name: HOME, value: /opt/data/home}
+            - {name: PYTHONPATH, value: /opt/hermes}
+            - {name: PYTHONDONTWRITEBYTECODE, value: "1"}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - {name: home, mountPath: /opt/data}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+            - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
+          resources:
+            requests: {cpu: 10m, memory: 64Mi}
+            limits: {cpu: 250m, memory: 256Mi}
         - name: credential-sync
           image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
           imagePullPolicy: IfNotPresent
@@ -1196,7 +1241,7 @@ spec:
               - {key: openai.yaml, path: agents/openai.yaml}
         - name: scm-boundary
           configMap:
-            name: hermes-scm-boundary
+            name: hermes-scm-boundary-v2
             defaultMode: 0555
         - name: image-policy
           configMap:

services/hermes/chat-statefulset.yaml

@@ -204,6 +204,9 @@ spec:
             - |
               /opt/hermes/.venv/bin/python /opt/coordinator/patch_api_server_sessions.py \
                 /opt/hermes/gateway/platforms/api_server.py /patched/api_server.py
+              grep -Fq 'conversation_history = compact_telegram_history(conversation_history)' /patched/api_server.py
+              grep -Fq 'full_history = compact_telegram_history(full_history)' /patched/api_server.py
+              grep -Fq 'conversation_history_snapshot = compact_telegram_history(' /patched/api_server.py
               /opt/hermes/.venv/bin/python /opt/coordinator/migrate_telegram_api_sessions.py \
                 /opt/data/state.db /opt/data/response_store.db
           securityContext:
@@ -284,6 +287,7 @@ spec:
             - {name: runtime-access, mountPath: /runtime-access}
             - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
             - {name: api-server-patch, mountPath: /opt/hermes/gateway/platforms/api_server.py, subPath: api_server.py}
+            - {name: coordinator, mountPath: /opt/hermes/gateway/platforms/telegram_continuity.py, subPath: migrate_telegram_api_sessions.py, readOnly: true}
             - {name: stream-recovery-patch, mountPath: /opt/hermes/agent/conversation_loop.py, subPath: conversation_loop.py}
             - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/environments/local.py, subPath: local.py}
             - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py}

services/hermes/execution-coordinator-patch.yaml

@@ -0,0 +1,66 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hermes-agent
+  namespace: hermes
+spec:
+  template:
+    metadata:
+      annotations:
+        vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
+        vault.hashicorp.com/agent-inject-template-execution-pool-key: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
+          {{- end }}
+    spec:
+      containers:
+        - name: cli-lane-runner
+          env:
+            - {name: HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY, value: "true"}
+            - {name: HERMES_CLI_LANE_CONCURRENCY, value: "1"}
+        - name: execution-pool-coordinator
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
+          imagePullPolicy: IfNotPresent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_coordinator.py]
+          env:
+            - {name: HERMES_HOME, value: /opt/data}
+            - {name: HOME, value: /opt/data/home}
+            - {name: PYTHONPATH, value: /opt/hermes}
+            - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
+            - {name: PATH, value: /opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: home, mountPath: /opt/data}
+            - {name: runtime-access, mountPath: /pool-access/execution-pool-key, subPath: execution-pool-key, readOnly: true}
+            - {name: execution-pool-code, mountPath: /opt/coordinator, readOnly: true}
+            - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
+            - {name: tmp, mountPath: /tmp}
+          ports:
+            - {name: execution-pool, containerPort: 9007, protocol: TCP}
+          startupProbe:
+            httpGet: {path: /ready, port: execution-pool}
+            periodSeconds: 5
+            failureThreshold: 60
+          readinessProbe:
+            httpGet: {path: /ready, port: execution-pool}
+            periodSeconds: 10
+          livenessProbe:
+            httpGet: {path: /ready, port: execution-pool}
+            initialDelaySeconds: 30
+            periodSeconds: 30
+          resources:
+            requests: {cpu: 50m, memory: 128Mi}
+            limits: {cpu: 500m, memory: 512Mi}
+      volumes:
+        - name: execution-pool-code
+          configMap:
+            name: hermes-execution-pool
+            defaultMode: 0555

services/hermes/execution-mediator.yaml

@@ -0,0 +1,476 @@
+# Privileged HMAC and SCM mediation runs outside the model Pods. Each mediator is
+# colocated with exactly one ordinal's RWO workspace but has a distinct network
+# identity, private HMAC derivation, and durable integrity state.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata: {name: hermes-execution-mediator-state-0, namespace: hermes}
+spec:
+  accessModes: [ReadWriteOnce]
+  storageClassName: astreae
+  resources: {requests: {storage: 1Gi}}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata: {name: hermes-execution-mediator-state-1, namespace: hermes}
+spec:
+  accessModes: [ReadWriteOnce]
+  storageClassName: astreae
+  resources: {requests: {storage: 1Gi}}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata: {name: hermes-execution-mediator-state-2, namespace: hermes}
+spec:
+  accessModes: [ReadWriteOnce]
+  storageClassName: astreae
+  resources: {requests: {storage: 1Gi}}
+---
+apiVersion: v1
+kind: Service
+metadata: {name: hermes-execution-mediator-0, namespace: hermes}
+spec:
+  selector: {app: hermes-execution-mediator, pool-ordinal: "0"}
+  ports: [{name: mediator, port: 9009, targetPort: mediator}]
+---
+apiVersion: v1
+kind: Service
+metadata: {name: hermes-execution-mediator-1, namespace: hermes}
+spec:
+  selector: {app: hermes-execution-mediator, pool-ordinal: "1"}
+  ports: [{name: mediator, port: 9009, targetPort: mediator}]
+---
+apiVersion: v1
+kind: Service
+metadata: {name: hermes-execution-mediator-2, namespace: hermes}
+spec:
+  selector: {app: hermes-execution-mediator, pool-ordinal: "2"}
+  ports: [{name: mediator, port: 9009, targetPort: mediator}]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hermes-execution-mediator-0
+  namespace: hermes
+  labels: {app: hermes-execution-mediator, pool-ordinal: "0"}
+spec:
+  replicas: 1
+  strategy: {type: Recreate}
+  selector:
+    matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
+  template:
+    metadata:
+      labels: {app: hermes-execution-mediator, pool-ordinal: "0"}
+      annotations:
+        ai.bstein.dev/config-rev: execution-pool-v2-mediated
+        ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: hermes-execution-worker
+        vault.hashicorp.com/agent-inject-containers: stage-mediator-access
+        vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
+        vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
+        vault.hashicorp.com/agent-inject-template-execution-pool-key: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
+          {{- end }}
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/agent-init-first: "true"
+        vault.hashicorp.com/agent-requests-cpu: 2m
+        vault.hashicorp.com/agent-requests-mem: 16Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
+    spec:
+      serviceAccountName: hermes-execution-worker
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      # Priority parity with the worker: a pressured node evicts the pair
+      # symmetrically instead of moving only the lower-priority half, and the
+      # pool never preempts anything else in the cluster.
+      priorityClassName: scavenger
+      securityContext:
+        fsGroup: 10000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile: {type: RuntimeDefault}
+      affinity:
+        # Colocation with the worker is a throughput preference, not a
+        # requirement. The shared workspace claim is ReadWriteMany, so this
+        # mediator still schedules and serves when its worker is Pending or
+        # placed elsewhere -- which is what makes a drain self-healing.
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - {key: kubernetes.io/arch, operator: In, values: [arm64]}
+                  - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
+                  - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    statefulset.kubernetes.io/pod-name: hermes-execution-worker-0
+                topologyKey: kubernetes.io/hostname
+      initContainers:
+        - name: stage-mediator-access
+          image: registry.bstein.dev/bstein/hermes-agent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
+          env:
+            - {name: HERMES_WORKER_ORDINAL, value: "0"}
+            - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsUser: 0
+            runAsGroup: 0
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: pool-access, mountPath: /pool-access}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+          resources:
+            requests: {cpu: 2m, memory: 16Mi}
+            limits: {cpu: 100m, memory: 64Mi}
+      containers:
+        - name: execution-mediator
+          image: registry.bstein.dev/bstein/hermes-agent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
+          env:
+            - {name: HERMES_WORKER_ORDINAL, value: "0"}
+            - {name: HERMES_WORKER_ROOT, value: /workspace}
+            - {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
+            - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
+            - {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
+          ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
+          startupProbe:
+            httpGet: {path: /ready, port: mediator}
+            periodSeconds: 5
+            failureThreshold: 60
+          readinessProbe:
+            httpGet: {path: /ready, port: mediator}
+            periodSeconds: 10
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: workspace, mountPath: /workspace}
+            - {name: scm-state, mountPath: /scm-state}
+            - {name: pool-access, mountPath: /pool-access, readOnly: true}
+            - {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+            - {name: tmp, mountPath: /tmp}
+          resources:
+            requests: {cpu: 2m, memory: 64Mi}
+            limits: {cpu: 250m, memory: 256Mi}
+      volumes:
+        - name: workspace
+          persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-0}
+        - name: scm-state
+          persistentVolumeClaim: {claimName: hermes-execution-mediator-state-0}
+        - name: pool-access
+          emptyDir: {medium: Memory, sizeLimit: 1Mi}
+        - name: scm-broker-client
+          configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
+        - name: coordinator
+          configMap: {name: hermes-execution-pool, defaultMode: 0555}
+        - name: tmp
+          emptyDir: {sizeLimit: 64Mi}
+        - name: vault-auth-token
+          projected:
+            defaultMode: 0600
+            sources:
+              - serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
+              - configMap:
+                  name: kube-root-ca.crt
+                  items: [{key: ca.crt, path: ca.crt}]
+              - downwardAPI:
+                  items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hermes-execution-mediator-1
+  namespace: hermes
+  labels: {app: hermes-execution-mediator, pool-ordinal: "1"}
+spec:
+  replicas: 1
+  strategy: {type: Recreate}
+  selector:
+    matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
+  template:
+    metadata:
+      labels: {app: hermes-execution-mediator, pool-ordinal: "1"}
+      annotations:
+        ai.bstein.dev/config-rev: execution-pool-v2-mediated
+        ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: hermes-execution-worker
+        vault.hashicorp.com/agent-inject-containers: stage-mediator-access
+        vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
+        vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
+        vault.hashicorp.com/agent-inject-template-execution-pool-key: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
+          {{- end }}
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/agent-init-first: "true"
+        vault.hashicorp.com/agent-requests-cpu: 2m
+        vault.hashicorp.com/agent-requests-mem: 16Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
+    spec:
+      serviceAccountName: hermes-execution-worker
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      # Priority parity with the worker: a pressured node evicts the pair
+      # symmetrically instead of moving only the lower-priority half, and the
+      # pool never preempts anything else in the cluster.
+      priorityClassName: scavenger
+      securityContext:
+        fsGroup: 10000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile: {type: RuntimeDefault}
+      affinity:
+        # Colocation with the worker is a throughput preference, not a
+        # requirement. The shared workspace claim is ReadWriteMany, so this
+        # mediator still schedules and serves when its worker is Pending or
+        # placed elsewhere -- which is what makes a drain self-healing.
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - {key: kubernetes.io/arch, operator: In, values: [arm64]}
+                  - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
+                  - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    statefulset.kubernetes.io/pod-name: hermes-execution-worker-1
+                topologyKey: kubernetes.io/hostname
+      initContainers:
+        - name: stage-mediator-access
+          image: registry.bstein.dev/bstein/hermes-agent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
+          env:
+            - {name: HERMES_WORKER_ORDINAL, value: "1"}
+            - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsUser: 0
+            runAsGroup: 0
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: pool-access, mountPath: /pool-access}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+          resources:
+            requests: {cpu: 2m, memory: 16Mi}
+            limits: {cpu: 100m, memory: 64Mi}
+      containers:
+        - name: execution-mediator
+          image: registry.bstein.dev/bstein/hermes-agent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
+          env:
+            - {name: HERMES_WORKER_ORDINAL, value: "1"}
+            - {name: HERMES_WORKER_ROOT, value: /workspace}
+            - {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
+            - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
+            - {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
+          ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
+          startupProbe:
+            httpGet: {path: /ready, port: mediator}
+            periodSeconds: 5
+            failureThreshold: 60
+          readinessProbe:
+            httpGet: {path: /ready, port: mediator}
+            periodSeconds: 10
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: workspace, mountPath: /workspace}
+            - {name: scm-state, mountPath: /scm-state}
+            - {name: pool-access, mountPath: /pool-access, readOnly: true}
+            - {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+            - {name: tmp, mountPath: /tmp}
+          resources:
+            requests: {cpu: 2m, memory: 64Mi}
+            limits: {cpu: 250m, memory: 256Mi}
+      volumes:
+        - name: workspace
+          persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-1}
+        - name: scm-state
+          persistentVolumeClaim: {claimName: hermes-execution-mediator-state-1}
+        - name: pool-access
+          emptyDir: {medium: Memory, sizeLimit: 1Mi}
+        - name: scm-broker-client
+          configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
+        - name: coordinator
+          configMap: {name: hermes-execution-pool, defaultMode: 0555}
+        - name: tmp
+          emptyDir: {sizeLimit: 64Mi}
+        - name: vault-auth-token
+          projected:
+            defaultMode: 0600
+            sources:
+              - serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
+              - configMap:
+                  name: kube-root-ca.crt
+                  items: [{key: ca.crt, path: ca.crt}]
+              - downwardAPI:
+                  items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hermes-execution-mediator-2
+  namespace: hermes
+  labels: {app: hermes-execution-mediator, pool-ordinal: "2"}
+spec:
+  replicas: 1
+  strategy: {type: Recreate}
+  selector:
+    matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
+  template:
+    metadata:
+      labels: {app: hermes-execution-mediator, pool-ordinal: "2"}
+      annotations:
+        ai.bstein.dev/config-rev: execution-pool-v2-mediated
+        ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: hermes-execution-worker
+        vault.hashicorp.com/agent-inject-containers: stage-mediator-access
+        vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
+        vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
+        vault.hashicorp.com/agent-inject-template-execution-pool-key: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
+          {{- end }}
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/agent-init-first: "true"
+        vault.hashicorp.com/agent-requests-cpu: 2m
+        vault.hashicorp.com/agent-requests-mem: 16Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
+    spec:
+      serviceAccountName: hermes-execution-worker
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      # Priority parity with the worker: a pressured node evicts the pair
+      # symmetrically instead of moving only the lower-priority half, and the
+      # pool never preempts anything else in the cluster.
+      priorityClassName: scavenger
+      securityContext:
+        fsGroup: 10000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile: {type: RuntimeDefault}
+      affinity:
+        # Colocation with the worker is a throughput preference, not a
+        # requirement. The shared workspace claim is ReadWriteMany, so this
+        # mediator still schedules and serves when its worker is Pending or
+        # placed elsewhere -- which is what makes a drain self-healing.
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - {key: kubernetes.io/arch, operator: In, values: [arm64]}
+                  - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
+                  - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    statefulset.kubernetes.io/pod-name: hermes-execution-worker-2
+                topologyKey: kubernetes.io/hostname
+      initContainers:
+        - name: stage-mediator-access
+          image: registry.bstein.dev/bstein/hermes-agent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
+          env:
+            - {name: HERMES_WORKER_ORDINAL, value: "2"}
+            - {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsUser: 0
+            runAsGroup: 0
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: pool-access, mountPath: /pool-access}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+          resources:
+            requests: {cpu: 2m, memory: 16Mi}
+            limits: {cpu: 100m, memory: 64Mi}
+      containers:
+        - name: execution-mediator
+          image: registry.bstein.dev/bstein/hermes-agent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
+          env:
+            - {name: HERMES_WORKER_ORDINAL, value: "2"}
+            - {name: HERMES_WORKER_ROOT, value: /workspace}
+            - {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
+            - {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
+            - {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
+          ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
+          startupProbe:
+            httpGet: {path: /ready, port: mediator}
+            periodSeconds: 5
+            failureThreshold: 60
+          readinessProbe:
+            httpGet: {path: /ready, port: mediator}
+            periodSeconds: 10
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: workspace, mountPath: /workspace}
+            - {name: scm-state, mountPath: /scm-state}
+            - {name: pool-access, mountPath: /pool-access, readOnly: true}
+            - {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+            - {name: tmp, mountPath: /tmp}
+          resources:
+            requests: {cpu: 2m, memory: 64Mi}
+            limits: {cpu: 250m, memory: 256Mi}
+      volumes:
+        - name: workspace
+          persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-2}
+        - name: scm-state
+          persistentVolumeClaim: {claimName: hermes-execution-mediator-state-2}
+        - name: pool-access
+          emptyDir: {medium: Memory, sizeLimit: 1Mi}
+        - name: scm-broker-client
+          configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
+        - name: coordinator
+          configMap: {name: hermes-execution-pool, defaultMode: 0555}
+        - name: tmp
+          emptyDir: {sizeLimit: 64Mi}
+        - name: vault-auth-token
+          projected:
+            defaultMode: 0600
+            sources:
+              - serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
+              - configMap:
+                  name: kube-root-ca.crt
+                  items: [{key: ca.crt, path: ca.crt}]
+              - downwardAPI:
+                  items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]

services/hermes/execution-worker-networkpolicy.yaml

@@ -0,0 +1,206 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-worker-isolation
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-execution-worker
+  policyTypes: [Ingress, Egress]
+  ingress: []
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels: {kubernetes.io/metadata.name: kube-system}
+          podSelector:
+            matchLabels: {k8s-app: kube-dns}
+      ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}]
+    - to:
+        - namespaceSelector:
+            matchLabels: {kubernetes.io/metadata.name: vault}
+          podSelector:
+            matchLabels: {app: vault}
+      ports: [{protocol: TCP, port: 8200}]
+    - to:
+        - podSelector:
+            matchLabels: {app: hermes-switchyard}
+      ports: [{protocol: TCP, port: 9005}]
+    - to:
+        - ipBlock:
+            cidr: 0.0.0.0/0
+            except:
+              - 10.0.0.0/8
+              - 100.64.0.0/10
+              - 127.0.0.0/8
+              - 169.254.0.0/16
+              - 172.16.0.0/12
+              - 192.168.0.0/16
+              - 224.0.0.0/4
+      ports: [{protocol: TCP, port: 443}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-worker-mediator-0
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-execution-worker
+      apps.kubernetes.io/pod-index: "0"
+  policyTypes: [Egress]
+  egress:
+    - to:
+        - podSelector:
+            matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
+      ports: [{protocol: TCP, port: 9009}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-worker-mediator-1
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-execution-worker
+      apps.kubernetes.io/pod-index: "1"
+  policyTypes: [Egress]
+  egress:
+    - to:
+        - podSelector:
+            matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
+      ports: [{protocol: TCP, port: 9009}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-worker-mediator-2
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-execution-worker
+      apps.kubernetes.io/pod-index: "2"
+  policyTypes: [Egress]
+  egress:
+    - to:
+        - podSelector:
+            matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
+      ports: [{protocol: TCP, port: 9009}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-pool-ingress
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels: {app: hermes-agent}
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels: {app: hermes-execution-mediator}
+      ports: [{protocol: TCP, port: 9007}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-mediator-isolation
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels: {app: hermes-execution-mediator}
+  policyTypes: [Ingress, Egress]
+  ingress: []
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels: {kubernetes.io/metadata.name: kube-system}
+          podSelector:
+            matchLabels: {k8s-app: kube-dns}
+      ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}]
+    - to:
+        - namespaceSelector:
+            matchLabels: {kubernetes.io/metadata.name: vault}
+          podSelector:
+            matchLabels: {app: vault}
+      ports: [{protocol: TCP, port: 8200}]
+    - to:
+        - podSelector:
+            matchLabels: {app: hermes-agent}
+      ports: [{protocol: TCP, port: 9007}]
+    - to:
+        - namespaceSelector:
+            matchLabels: {kubernetes.io/metadata.name: hermes-scm}
+          podSelector:
+            matchLabels: {app: hermes-scm-broker}
+      ports: [{protocol: TCP, port: 9081}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-mediator-worker-0
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app: hermes-execution-worker
+              apps.kubernetes.io/pod-index: "0"
+      ports: [{protocol: TCP, port: 9009}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-mediator-worker-1
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app: hermes-execution-worker
+              apps.kubernetes.io/pod-index: "1"
+      ports: [{protocol: TCP, port: 9009}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-mediator-worker-2
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app: hermes-execution-worker
+              apps.kubernetes.io/pod-index: "2"
+      ports: [{protocol: TCP, port: 9009}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-execution-switchyard-ingress
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels: {app: hermes-switchyard}
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels: {app: hermes-execution-worker}
+      ports: [{protocol: TCP, port: 9005}]

services/hermes/execution-worker-rbac.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: hermes-execution-worker
+  namespace: hermes
+  labels:
+    app.kubernetes.io/name: hermes-execution-worker
+    app.kubernetes.io/part-of: hermes
+automountServiceAccountToken: false
+# Intentionally no RoleBinding or ClusterRoleBinding. The projected, bounded
+# token is mounted only into Vault-facing containers and grants no Kubernetes
+# API verbs to the model-facing execution worker.

services/hermes/execution-worker-statefulset.yaml

@@ -0,0 +1,291 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: hermes-execution-worker
+  namespace: hermes
+  labels:
+    app: hermes-execution-worker
+spec:
+  serviceName: hermes-execution-worker
+  replicas: 3
+  podManagementPolicy: Parallel
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: hermes-execution-worker
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      partition: 0
+  template:
+    metadata:
+      labels:
+        app: hermes-execution-worker
+        app.kubernetes.io/name: hermes-execution-worker
+        app.kubernetes.io/part-of: hermes
+      annotations:
+        ai.bstein.dev/role: fenced-execution-only
+        ai.bstein.dev/scm-boundary: mediated-pr14-broker-with-completion-gates
+        ai.bstein.dev/model-policy: Switchyard AUTO with cross-provider fallback
+        ai.bstein.dev/storage: separate durable RWO workspace and OAuth refresh ownership per ordinal
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: hermes-execution-worker
+        vault.hashicorp.com/agent-inject-containers: stage-worker-access
+        vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
+        vault.hashicorp.com/agent-inject-secret-claude-credentials-0: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-claude-credentials-0: "0600"
+        vault.hashicorp.com/agent-inject-template-claude-credentials-0: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ .Data.data.execution_worker_0_claude_credentials_json }}
+          {{- end }}
+        vault.hashicorp.com/agent-inject-secret-claude-credentials-1: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-claude-credentials-1: "0600"
+        vault.hashicorp.com/agent-inject-template-claude-credentials-1: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ .Data.data.execution_worker_1_claude_credentials_json }}
+          {{- end }}
+        vault.hashicorp.com/agent-inject-secret-claude-credentials-2: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-claude-credentials-2: "0600"
+        vault.hashicorp.com/agent-inject-template-claude-credentials-2: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ .Data.data.execution_worker_2_claude_credentials_json }}
+          {{- end }}
+        vault.hashicorp.com/agent-inject-secret-codex-auth-0: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-codex-auth-0: "0600"
+        vault.hashicorp.com/agent-inject-template-codex-auth-0: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ .Data.data.execution_worker_0_codex_auth_json }}
+          {{- end }}
+        vault.hashicorp.com/agent-inject-secret-codex-auth-1: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-codex-auth-1: "0600"
+        vault.hashicorp.com/agent-inject-template-codex-auth-1: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ .Data.data.execution_worker_1_codex_auth_json }}
+          {{- end }}
+        vault.hashicorp.com/agent-inject-secret-codex-auth-2: kv/data/atlas/hermes/agent-tokens
+        vault.hashicorp.com/agent-inject-perms-codex-auth-2: "0600"
+        vault.hashicorp.com/agent-inject-template-codex-auth-2: |
+          {{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
+          {{ .Data.data.execution_worker_2_codex_auth_json }}
+          {{- end }}
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/agent-init-first: "true"
+        vault.hashicorp.com/agent-requests-cpu: 2m
+        vault.hashicorp.com/agent-requests-mem: 16Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
+    spec:
+      serviceAccountName: hermes-execution-worker
+      priorityClassName: scavenger
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      terminationGracePeriodSeconds: 30
+      securityContext:
+        fsGroup: 10000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - {key: kubernetes.io/arch, operator: In, values: [arm64]}
+                  - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
+                  - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - {key: node-role.kubernetes.io/accelerator, operator: Exists}
+            - weight: 50
+              preference:
+                matchExpressions:
+                  - {key: hardware, operator: In, values: [rpi5]}
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: hermes-execution-worker
+              topologyKey: kubernetes.io/hostname
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app: hermes-execution-worker
+      initContainers:
+        - name: stage-worker-access
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
+          imagePullPolicy: IfNotPresent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-worker]
+          env:
+            - {name: HERMES_WORKER_ROOT, value: /workspace}
+            - {name: HERMES_PROVIDER_ACCESS_ROOT, value: /provider-access}
+            - name: HERMES_WORKER_ORDINAL
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsUser: 0
+            runAsGroup: 0
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: workspace, mountPath: /workspace}
+            - {name: provider-access, mountPath: /provider-access}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+          resources:
+            requests: {cpu: 2m, memory: 32Mi}
+            limits: {cpu: 100m, memory: 64Mi}
+        - name: install-provider-clis
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
+          imagePullPolicy: IfNotPresent
+          command: [/bin/sh, -ec]
+          args:
+            # The tools volume is durable, so this install runs once per pinned
+            # version rather than on every Pod start. The version marker is only
+            # trusted when both binaries are actually present and executable, so
+            # a partial or pruned cache reinstalls instead of failing closed
+            # forever. The install itself is time-bounded so a stalled registry
+            # surfaces as a fast init failure with backoff instead of an
+            # open-ended wait that a Flux health timeout would have to absorb.
+            - |
+              tools=/worker-data/tools
+              marker="${tools}/.cli-versions-0.147.0-2.1.226"
+              mkdir -p "${tools}/bin"
+              if [ ! -f "${marker}" ] || [ ! -x "${tools}/bin/codex" ] \
+                  || [ ! -x "${tools}/bin/claude" ]; then
+                rm -f "${marker}"
+                timeout 900 npm install --global --omit=dev --no-audit --no-fund \
+                  --fetch-timeout=120000 --fetch-retries=2 --prefix "${tools}" \
+                  @openai/codex@0.147.0 @anthropic-ai/claude-code@2.1.226
+                touch "${marker}"
+              fi
+              test -x "${tools}/bin/codex"
+              test -x "${tools}/bin/claude"
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            runAsNonRoot: true
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: tools, mountPath: /worker-data/tools}
+          resources:
+            requests: {cpu: 5m, memory: 64Mi}
+            limits: {cpu: "1", memory: 1Gi}
+      containers:
+        - name: execution-worker
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
+          imagePullPolicy: IfNotPresent
+          command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_worker.py]
+          env:
+            - {name: HERMES_HOME, value: /worker-data}
+            - {name: HERMES_WORKER_ROOT, value: /workspace}
+            - {name: HOME, value: /worker-data/home}
+            - {name: CODEX_HOME, value: /provider-access/codex}
+            - {name: CLAUDE_CONFIG_DIR, value: /provider-access/claude}
+            - {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
+            - {name: PYTHONPATH, value: /opt/hermes}
+            - {name: PATH, value: /worker-data/tools/bin:/opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
+            - name: HERMES_WORKER_ORDINAL
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
+            - name: HERMES_WORKER_NODE
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          startupProbe:
+            exec:
+              command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json && test -w /provider-access/claude/.credentials.json"]
+            periodSeconds: 5
+            failureThreshold: 60
+          readinessProbe:
+            exec:
+              command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json"]
+            periodSeconds: 10
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile: {type: RuntimeDefault}
+          volumeMounts:
+            - {name: workspace, mountPath: /workspace}
+            - {name: worker-data, mountPath: /worker-data}
+            - {name: tools, mountPath: /worker-data/tools, readOnly: true}
+            - {name: provider-access, mountPath: /provider-access}
+            - {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
+            - {name: tmp, mountPath: /tmp}
+          resources:
+            requests: {cpu: 5m, memory: 128Mi, ephemeral-storage: 1Gi}
+            limits: {cpu: "2", memory: 4Gi, ephemeral-storage: 8Gi}
+      volumes:
+        - name: worker-data
+          emptyDir: {sizeLimit: 128Mi}
+        - name: coordinator
+          configMap:
+            name: hermes-execution-pool
+            defaultMode: 0555
+        - name: tmp
+          emptyDir: {sizeLimit: 2Gi}
+        - name: vault-auth-token
+          projected:
+            defaultMode: 0600
+            sources:
+              - serviceAccountToken:
+                  audience: vault
+                  expirationSeconds: 3600
+                  path: token
+              - configMap:
+                  name: kube-root-ca.crt
+                  items:
+                    - {key: ca.crt, path: ca.crt}
+              - downwardAPI:
+                  items:
+                    - {path: namespace, fieldRef: {fieldPath: metadata.namespace}}
+  volumeClaimTemplates:
+    # The workspace is shared with this ordinal's mediator, which performs every
+    # SCM operation on it. ReadWriteMany removes the cross-node Multi-Attach
+    # deadlock that a drain or preemption used to create between the two Pods,
+    # so each can be evicted and rescheduled independently. Longhorn already
+    # serves the hermes-chat tenant workspaces this way on the same class.
+    - metadata:
+        name: workspace
+        labels:
+          app: hermes-execution-worker
+      spec:
+        accessModes: [ReadWriteMany]
+        storageClassName: astreae
+        resources:
+          requests:
+            storage: 30Gi
+    # Provider CLIs are installed once per pinned version onto a durable volume
+    # instead of being re-downloaded into an emptyDir on every Pod start.
+    - metadata:
+        name: tools
+        labels:
+          app: hermes-execution-worker
+      spec:
+        accessModes: [ReadWriteOnce]
+        storageClassName: astreae
+        resources:
+          requests:
+            storage: 2Gi
+    # OAuth refresh ownership is single-mounter by design: keep it ReadWriteOnce.
+    - metadata:
+        name: provider-access
+        labels:
+          app: hermes-execution-worker
+      spec:
+        accessModes: [ReadWriteOnce]
+        storageClassName: astreae
+        resources:
+          requests:
+            storage: 1Gi

services/hermes/kustomization.yaml

@@ -41,6 +41,13 @@ resources:
   - oauth2-proxy.yaml
   - agent-certificate.yaml
   - agent-ingress.yaml
+  - execution-worker-rbac.yaml
+  - execution-worker-statefulset.yaml
+  - execution-mediator.yaml
+  - execution-worker-networkpolicy.yaml
+
+patches:
+  - path: execution-coordinator-patch.yaml
 
 configMapGenerator:
   - name: hermes-chat-oauth-templates
@@ -55,13 +62,62 @@ configMapGenerator:
       - OPERATOR-RUNBOOK.md=NOTES.md
     options:
       disableNameSuffixHash: true
+  - name: hermes-execution-pool
+    namespace: hermes
+    files:
+      - cli_lane_board.py=scripts/cli_lane_board.py
+      - cli_lane_capabilities.py=scripts/cli_lane_capabilities.py
+      - cli_lane_config.py=scripts/cli_lane_config.py
+      - cli_lane_dispatch.py=scripts/cli_lane_dispatch.py
+      - cli_lane_evidence.py=scripts/cli_lane_evidence.py
+      - cli_lane_execution.py=scripts/cli_lane_execution.py
+      - cli_lane_files.py=scripts/cli_lane_files.py
+      - cli_lane_finalization.py=scripts/cli_lane_finalization.py
+      - cli_lane_goal.py=scripts/cli_lane_goal.py
+      - cli_lane_prompt.py=scripts/cli_lane_prompt.py
+      - cli_lane_provider.py=scripts/cli_lane_provider.py
+      - cli_lane_quarantine.py=scripts/cli_lane_quarantine.py
+      - cli_lane_records.py=scripts/cli_lane_records.py
+      - cli_lane_recovery.py=scripts/cli_lane_recovery.py
+      - cli_lane_retention.py=scripts/cli_lane_retention.py
+      - cli_lane_routing.py=scripts/cli_lane_routing.py
+      - cli_lane_runner.py=scripts/cli_lane_runner.py
+      - execution_pool_protocol.py=scripts/execution_pool_protocol.py
+      - execution_pool_store.py=scripts/execution_pool_store.py
+      - execution_pool_project.py=scripts/execution_pool_project.py
+      - execution_pool_coordinator.py=scripts/execution_pool_coordinator.py
+      - execution_pool_maintenance.py=scripts/execution_pool_maintenance.py
+      - execution_pool_server.py=scripts/execution_pool_server.py
+      - execution_pool_client.py=scripts/execution_pool_client.py
+      - execution_pool_worker.py=scripts/execution_pool_worker.py
+      - execution_pool_scm.py=scripts/execution_pool_scm.py
+      - gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py
+      - scm_broker_client.py=scm-common/scripts/scm_broker_client.py
+      - stage_runtime_access.py=scripts/stage_runtime_access.py
   - name: hermes-coordinator
     namespace: hermes
     files:
+      - ai_usage_codex.py=scripts/ai_usage_codex.py
       - ai_usage_exporter.py=scripts/ai_usage_exporter.py
+      - ai_usage_http.py=scripts/ai_usage_http.py
       - claude=scripts/claude
       - claude_command_policy.py=scripts/claude_command_policy.py
+      - cli_lane_board.py=scripts/cli_lane_board.py
+      - cli_lane_capabilities.py=scripts/cli_lane_capabilities.py
+      - cli_lane_config.py=scripts/cli_lane_config.py
+      - cli_lane_dispatch.py=scripts/cli_lane_dispatch.py
+      - cli_lane_evidence.py=scripts/cli_lane_evidence.py
+      - cli_lane_execution.py=scripts/cli_lane_execution.py
+      - cli_lane_files.py=scripts/cli_lane_files.py
+      - cli_lane_finalization.py=scripts/cli_lane_finalization.py
       - cli_lane_goal.py=scripts/cli_lane_goal.py
+      - cli_lane_prompt.py=scripts/cli_lane_prompt.py
+      - cli_lane_provider.py=scripts/cli_lane_provider.py
+      - cli_lane_quarantine.py=scripts/cli_lane_quarantine.py
+      - cli_lane_records.py=scripts/cli_lane_records.py
+      - cli_lane_recovery.py=scripts/cli_lane_recovery.py
+      - cli_lane_retention.py=scripts/cli_lane_retention.py
+      - cli_lane_routing.py=scripts/cli_lane_routing.py
       - cli_lane_runner.py=scripts/cli_lane_runner.py
       - codex=scripts/codex
       - configure_agent_clients.py=scripts/configure_agent_clients.py
@@ -77,6 +133,8 @@ configMapGenerator:
       - jenkins_build_evidence.py=scripts/jenkins_build_evidence.py
       - jenkins_image_build_trigger.py=scripts/jenkins_image_build_trigger.py
       - kanban_status_recovery.py=scripts/kanban_status_recovery.py
+      - kanban_supervisor.py=scripts/kanban_supervisor.py
+      - supervisor_policy.py=scripts/supervisor_policy.py
       - migrate_herdr_state.py=scripts/migrate_herdr_state.py
       - migrate_api_session_lineage.py=scripts/migrate_api_session_lineage.py
       - migrate_telegram_api_sessions.py=scripts/migrate_telegram_api_sessions.py

services/hermes/networkpolicy.yaml

@@ -119,6 +119,7 @@ spec:
               app: server
       ports:
         - {protocol: TCP, port: 9010}
+        - {protocol: TCP, port: 9011}
   egress:
     - to:
         - namespaceSelector:

services/hermes/oauth2-proxy.yaml

@@ -16,6 +16,9 @@ metadata:
 spec:
   selector:
     app: hermes-agent
+  # The agent dashboard and terminal must stay reachable while the
+  # cli-lane-runner readiness probe holds the shared pod NotReady.
+  publishNotReadyAddresses: true
   ports:
     - {name: http, port: 80, targetPort: auth-http}
 ---

services/hermes/router/main.go

@@ -27,8 +27,9 @@ type linkRecord struct {
 }
 
 type telegramTopic struct {
-	Label    string `json:"label"`
-	LastUsed int64  `json:"last_used"`
+	Label      string `json:"label"`
+	LastUsed   int64  `json:"last_used"`
+	Generation int64  `json:"generation,omitempty"`
 }
 
 type telegramTopicState struct {
@@ -372,7 +373,12 @@ func (router *tenantRouter) ServeHTTP(writer http.ResponseWriter, request *http.
 		return
 	}
 	proxy := httputil.NewSingleHostReverseProxy(target)
-	proxy.ModifyResponse = injectChatBridge
+	proxy.ModifyResponse = func(response *http.Response) error {
+		if err := boundSessionSnapshot(response); err != nil {
+			return err
+		}
+		return injectChatBridge(response)
+	}
 	proxy.ErrorHandler = func(writer http.ResponseWriter, _ *http.Request, proxyErr error) {
 		log.Printf("tenant slot %d unavailable", slot)
 		http.Error(writer, "private chat runtime is starting", http.StatusBadGateway)

services/hermes/router/main_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -126,6 +127,9 @@ func TestRouterProxiesWebUIAndAddsTelegramShortcut(t *testing.T) {
 	if !strings.Contains(response.Body.String(), "hermes-chat-bridge.js") || !strings.Contains(response.Body.String(), "hermes-chat-bridge.css") {
 		t.Fatal("Telegram shortcut assets were not injected")
 	}
+	if !strings.Contains(response.Body.String(), "hermes-session-continuity.js") {
+		t.Fatal("session continuity fallback was not injected")
+	}
 	assetRequest := httptest.NewRequest(http.MethodGet, "/hermes-chat-bridge.js", nil)
 	assetRequest.Header.Set("X-Forwarded-User", "subject")
 	assetResponse := httptest.NewRecorder()
@@ -145,6 +149,75 @@ func TestRouterProxiesWebUIAndAddsTelegramShortcut(t *testing.T) {
 	}
 }
 
+func TestSessionContinuityAssetHasAccessiblePollFallback(t *testing.T) {
+	router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(int) string { return "" })
+	if err != nil {
+		t.Fatal(err)
+	}
+	request := httptest.NewRequest(http.MethodGet, "/hermes-session-continuity.js", nil)
+	request.Header.Set("X-Forwarded-User", "subject")
+	response := httptest.NewRecorder()
+	router.ServeHTTP(response, request)
+	if response.Code != http.StatusOK {
+		t.Fatalf("got status %d", response.Code)
+	}
+	asset := response.Body.String()
+	for _, expected := range []string{
+		"aria-live", "aria-busy", "/api/sessions/", "/messages?limit=24&hermes_fallback=1",
+		"Latest stored activity", "no renderable messages",
+		"Session updates disconnected", "oauth2/start?rd=", "payload.session_id",
+		"session.ended_at == null", "latest.observed", "addEventListener('online'",
+		"addEventListener('pageshow'", "schedule(document.hidden", "fetch(",
+	} {
+		if !strings.Contains(asset, expected) {
+			t.Fatalf("session fallback omitted %q", expected)
+		}
+	}
+	for _, forbidden := range []string{
+		"location.replace", "location.reload", "history.replaceState", "/api/session?", "WebSocket",
+	} {
+		if strings.Contains(asset, forbidden) {
+			t.Fatalf("session fallback interferes with native continuity via %q", forbidden)
+		}
+	}
+}
+
+func TestSessionContinuityPollIsBoundedByRouter(t *testing.T) {
+	backend := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
+		messages := make([]map[string]int, 30)
+		for index := range messages {
+			messages[index] = map[string]int{"index": index}
+		}
+		writer.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(writer).Encode(map[string]any{
+			"session_id": "resolved", "messages": messages,
+		})
+	}))
+	defer backend.Close()
+	router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(int) string { return backend.URL })
+	if err != nil {
+		t.Fatal(err)
+	}
+	request := httptest.NewRequest(
+		http.MethodGet,
+		"/api/sessions/root/messages?limit=24&hermes_fallback=1",
+		nil,
+	)
+	request.Header.Set("X-Forwarded-User", "subject")
+	response := httptest.NewRecorder()
+	router.ServeHTTP(response, request)
+	if response.Code != http.StatusOK {
+		t.Fatalf("got status %d", response.Code)
+	}
+	var payload sessionSnapshot
+	if err := json.NewDecoder(response.Body).Decode(&payload); err != nil {
+		t.Fatal(err)
+	}
+	if len(payload.Messages) != sessionSnapshotItems || payload.TotalMessages != 30 {
+		t.Fatalf("router returned unbounded fallback: %#v", payload)
+	}
+}
+
 func TestTelegramPageExplainsOneTimeOperatorSetup(t *testing.T) {
 	router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(slot int) string { return "" })
 	if err != nil {
@@ -294,11 +367,52 @@ func TestPrivateFileBrowserIsTenantAuthenticatedAndHidesKanban(t *testing.T) {
 	assetRequest.Header.Set("X-Forwarded-User", "subject")
 	assetResponse := httptest.NewRecorder()
 	router.ServeHTTP(assetResponse, assetRequest)
-	if !strings.Contains(assetResponse.Body.String(), "endpoint('list'") || !strings.Contains(assetResponse.Body.String(), `'[data-panel="kanban"]'`) {
+	if !strings.Contains(assetResponse.Body.String(), "endpoint('list'") || !strings.Contains(assetResponse.Body.String(), `'[data-panel="' + panel + '"]'`) {
 		t.Fatal("file browsing or chat-only navigation policy is missing")
 	}
 }
 
+func TestChatBridgeHidesOnlyNonChatCockpitChrome(t *testing.T) {
+	router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(slot int) string { return "" })
+	if err != nil {
+		t.Fatal(err)
+	}
+	asset := func(path string) string {
+		request := httptest.NewRequest(http.MethodGet, path, nil)
+		request.Header.Set("X-Forwarded-User", "subject")
+		response := httptest.NewRecorder()
+		router.ServeHTTP(response, request)
+		if response.Code != http.StatusOK {
+			t.Fatalf("%s got status %d", path, response.Code)
+		}
+		return response.Body.String()
+	}
+	js := asset("/hermes-chat-bridge.js")
+	css := asset("/hermes-chat-bridge.css")
+
+	// The whole cockpit (Kanban plus every non-chat rail panel) is declutter-hidden
+	// both pre-JS (CSS, no flash) and by the runtime .hidden reinforcement.
+	for _, panel := range []string{"kanban", "logs", "insights", "memory", "skills", "workspaces", "todos", "tasks", "profiles"} {
+		if !strings.Contains(css, `[data-panel="`+panel+`"]`) {
+			t.Fatalf("bridge CSS does not hide cockpit panel %q", panel)
+		}
+		if !strings.Contains(js, `'`+panel+`'`) {
+			t.Fatalf("bridge JS does not mark cockpit panel %q hidden", panel)
+		}
+	}
+	if !strings.Contains(css, "display:none") || !strings.Contains(css, "#settingsModel") {
+		t.Fatal("bridge CSS must hide the cockpit panels and the redundant settings model selector")
+	}
+
+	// The clean chat experience (chat surface, history, composer, model chip)
+	// must never be part of the hide set.
+	for _, keep := range []string{`[data-panel="chat"]`, `[data-panel="history"]`, `[data-panel="sessions"]`, `[data-panel="composer"]`, "composerRoutingChip", "composerRoutingLabel"} {
+		if strings.Contains(css, keep) || strings.Contains(js, keep) {
+			t.Fatalf("declutter wrongly targets a core chat control: %q", keep)
+		}
+	}
+}
+
 func TestWebUIModelAndReasoningOverridesAreProxied(t *testing.T) {
 	backend := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
 		if request.Method != http.MethodPost {

services/hermes/router/session_continuity.go

@@ -0,0 +1,97 @@
+package main
+
+const sessionContinuityJS = `(() => {
+  const match = location.pathname.match(/^\/session\/([^/]+)\/?$/);
+  if (!match) return;
+  let sessionId;
+  try { sessionId = decodeURIComponent(match[1]); } catch (_) { sessionId = match[1]; }
+  const started = Date.now();
+  let timer = 0;
+  let request = null;
+  const card = document.createElement('aside');
+  card.id = 'hermes-session-continuity';
+  card.setAttribute('role', 'status');
+  card.setAttribute('aria-live', 'polite');
+  card.setAttribute('aria-atomic', 'true');
+  card.hidden = true;
+  card.style.cssText = 'position:fixed;right:16px;bottom:16px;z-index:1000;max-width:min(420px,calc(100vw - 32px));box-sizing:border-box;padding:10px 12px;border:1px solid #475569;border-radius:10px;background:#111827;color:#e5e7eb;font:13px/1.4 system-ui,sans-serif;box-shadow:0 8px 28px #0008';
+  const message = document.createElement('span');
+  const newChat = document.createElement('a');
+  newChat.href = '/';
+  newChat.textContent = ' Start a new chat.';
+  newChat.style.color = '#7dd3fc';
+  card.append(message, newChat);
+  document.body.appendChild(card);
+
+  const show = (text, busy, link) => {
+    message.textContent = text;
+    card.hidden = false;
+    card.setAttribute('aria-busy', busy ? 'true' : 'false');
+    newChat.hidden = !link;
+  };
+  const hide = () => { card.hidden = true; card.setAttribute('aria-busy', 'false'); };
+  const activityLabel = (messages) => {
+    const latest = messages[messages.length - 1] || {};
+    if (latest.tool_name) return 'tool ' + String(latest.tool_name).slice(0, 80);
+    if (Array.isArray(latest.tool_calls) && latest.tool_calls.length) {
+      const call = latest.tool_calls[latest.tool_calls.length - 1] || {};
+      return 'tool ' + String((call.function || {}).name || 'activity').slice(0, 80);
+    }
+    if (latest.activity_event) return String(latest.activity_event).replace(/[._-]+/g, ' ').slice(0, 80);
+    if (latest.observed && typeof latest.content === 'string') return latest.content.slice(0, 120);
+    return latest.role === 'assistant' ? 'assistant update' : latest.role === 'user' ? 'request stored' : 'working';
+  };
+  const authOrMissing = (response) => {
+    if (response.status === 401 || response.status === 403) {
+      const rd = location.pathname + location.search + location.hash;
+      location.assign('/oauth2/start?rd=' + encodeURIComponent(rd));
+      return 'auth';
+    }
+    if (response.status === 404) {
+      show('This session is unavailable to this account.', false, true);
+      schedule(10000);
+      return 'missing';
+    }
+    return '';
+  };
+  const schedule = (delay) => {
+    clearTimeout(timer);
+    timer = window.setTimeout(poll, delay);
+  };
+  async function poll() {
+    if (request) return;
+    request = new AbortController();
+    const timeout = window.setTimeout(() => request && request.abort(), 8000);
+    try {
+      const root = '/api/sessions/' + encodeURIComponent(sessionId);
+      const messageResponse = await fetch(root + '/messages?limit=24&hermes_fallback=1', {cache:'no-store', credentials:'same-origin', signal:request.signal});
+      if (authOrMissing(messageResponse)) return;
+      if (!messageResponse.ok) throw new Error('session messages poll failed');
+      const payload = await messageResponse.json();
+      const messages = Array.isArray(payload.messages) ? payload.messages : [];
+      const resolvedId = typeof payload.session_id === 'string' && payload.session_id ? payload.session_id : sessionId;
+      const detailResponse = await fetch('/api/sessions/' + encodeURIComponent(resolvedId), {cache:'no-store', credentials:'same-origin', signal:request.signal});
+      if (authOrMissing(detailResponse)) return;
+      if (!detailResponse.ok) throw new Error('session detail poll failed');
+      const session = await detailResponse.json();
+      const lastActive = Number(session.last_active || session.started_at || 0);
+      const durableWorker = session.source === 'api_server' && Boolean(session.parent_session_id);
+      const recentlyInteractive = Number.isFinite(lastActive) && Date.now() / 1000 - lastActive < 300;
+      const active = session.ended_at == null && messages.length > 0 && (durableWorker || recentlyInteractive);
+      if (active) show('Hermes is working. Latest stored activity: ' + activityLabel(messages) + '.', true, false);
+      else if (!messages.length && Date.now() - started >= 4000) show('This session has no renderable messages yet. It may be new or no longer available.', false, true);
+      else hide();
+      schedule(document.hidden ? 10000 : 2500);
+    } catch (_) {
+      show('Session updates disconnected. Retrying without changing this session…', true, false);
+      schedule(document.hidden ? 10000 : 3000);
+    } finally {
+      clearTimeout(timeout);
+      request = null;
+    }
+  }
+  addEventListener('online', () => schedule(0));
+  addEventListener('pageshow', () => schedule(0));
+  document.addEventListener('visibilitychange', () => schedule(0));
+  schedule(1200);
+})();`

services/hermes/router/session_snapshot.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+)
+
+const (
+	sessionSnapshotItems = 24
+	sessionSnapshotBytes = 8 << 20
+)
+
+type sessionSnapshot struct {
+	SessionID     string            `json:"session_id"`
+	Messages      []json.RawMessage `json:"messages"`
+	TotalMessages int               `json:"total_messages"`
+}
+
+// boundSessionSnapshot caps only the continuity fallback response. Native
+// WebUI requests remain untouched, including when an older backend ignores
+// its optional `limit` query parameter.
+func boundSessionSnapshot(response *http.Response) error {
+	request := response.Request
+	if request == nil || response.StatusCode != http.StatusOK ||
+		request.URL.Query().Get("hermes_fallback") != "1" ||
+		!strings.HasPrefix(request.URL.Path, "/api/sessions/") ||
+		!strings.HasSuffix(request.URL.Path, "/messages") {
+		return nil
+	}
+	body, err := io.ReadAll(io.LimitReader(response.Body, sessionSnapshotBytes+1))
+	if err != nil {
+		return err
+	}
+	_ = response.Body.Close()
+	if len(body) > sessionSnapshotBytes {
+		return errors.New("session snapshot exceeds safe response limit")
+	}
+	var payload sessionSnapshot
+	if err := json.Unmarshal(body, &payload); err != nil || payload.Messages == nil {
+		return errors.New("session snapshot is malformed")
+	}
+	total := len(payload.Messages)
+	if payload.TotalMessages > total {
+		total = payload.TotalMessages
+	}
+	if len(payload.Messages) > sessionSnapshotItems {
+		payload.Messages = payload.Messages[len(payload.Messages)-sessionSnapshotItems:]
+	}
+	payload.TotalMessages = total
+	body, err = json.Marshal(payload)
+	if err != nil {
+		return err
+	}
+	response.Body = io.NopCloser(strings.NewReader(string(body)))
+	response.ContentLength = int64(len(body))
+	response.Header.Set("Content-Length", strconv.Itoa(len(body)))
+	response.Header.Set("Content-Type", "application/json; charset=utf-8")
+	response.Header.Set("Cache-Control", "no-store")
+	response.Header.Del("ETag")
+	return nil
+}

services/hermes/router/session_snapshot_test.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"testing/iotest"
+)
+
+func snapshotResponse(target, body string) *http.Response {
+	request, _ := http.NewRequest(http.MethodGet, target, nil)
+	return &http.Response{
+		StatusCode: http.StatusOK,
+		Header:     make(http.Header),
+		Body:       io.NopCloser(strings.NewReader(body)),
+		Request:    request,
+	}
+}
+
+func TestBoundSessionSnapshotKeepsOnlyRecentMessages(t *testing.T) {
+	messages := make([]map[string]int, 30)
+	for index := range messages {
+		messages[index] = map[string]int{"index": index}
+	}
+	body, _ := json.Marshal(map[string]any{
+		"session_id": "resolved", "messages": messages,
+	})
+	response := snapshotResponse(
+		"http://tenant/api/sessions/root/messages?limit=24&hermes_fallback=1",
+		string(body),
+	)
+	response.Header.Set("ETag", "stale")
+	if err := boundSessionSnapshot(response); err != nil {
+		t.Fatal(err)
+	}
+	var bounded sessionSnapshot
+	if err := json.NewDecoder(response.Body).Decode(&bounded); err != nil {
+		t.Fatal(err)
+	}
+	if len(bounded.Messages) != 24 || bounded.TotalMessages != 30 {
+		t.Fatalf("snapshot was not bounded: %#v", bounded)
+	}
+	var first map[string]int
+	if err := json.Unmarshal(bounded.Messages[0], &first); err != nil || first["index"] != 6 {
+		t.Fatalf("snapshot did not retain the recent tail: %#v, %v", first, err)
+	}
+	if response.Header.Get("Cache-Control") != "no-store" || response.Header.Get("ETag") != "" {
+		t.Fatal("bounded snapshot retained cache metadata")
+	}
+}
+
+func TestBoundSessionSnapshotPreservesLargerReportedTotal(t *testing.T) {
+	response := snapshotResponse(
+		"http://tenant/api/sessions/root/messages?hermes_fallback=1",
+		`{"session_id":"leaf","messages":[],"total_messages":100}`,
+	)
+	if err := boundSessionSnapshot(response); err != nil {
+		t.Fatal(err)
+	}
+	var bounded sessionSnapshot
+	if err := json.NewDecoder(response.Body).Decode(&bounded); err != nil {
+		t.Fatal(err)
+	}
+	if bounded.SessionID != "leaf" || bounded.TotalMessages != 100 {
+		t.Fatalf("reported total was lost: %#v", bounded)
+	}
+}
+
+func TestBoundSessionSnapshotFailsClosedOnMalformedOrOversizedBody(t *testing.T) {
+	for name, body := range map[string]string{
+		"malformed": `{`,
+		"oversized": strings.Repeat("x", sessionSnapshotBytes+1),
+	} {
+		t.Run(name, func(t *testing.T) {
+			response := snapshotResponse(
+				"http://tenant/api/sessions/root/messages?hermes_fallback=1",
+				body,
+			)
+			if err := boundSessionSnapshot(response); err == nil {
+				t.Fatal("unsafe snapshot was accepted")
+			}
+		})
+	}
+	response := snapshotResponse(
+		"http://tenant/api/sessions/root/messages?hermes_fallback=1", `{}`,
+	)
+	response.Body = io.NopCloser(iotest.ErrReader(errors.New("read failed")))
+	if err := boundSessionSnapshot(response); err == nil {
+		t.Fatal("snapshot body read failure was ignored")
+	}
+}
+
+func TestBoundSessionSnapshotLeavesNativeAndErrorResponsesUntouched(t *testing.T) {
+	for _, response := range []*http.Response{
+		{StatusCode: http.StatusOK},
+		snapshotResponse("http://tenant/api/sessions/root/messages", `{}`),
+		snapshotResponse("http://tenant/api/sessions/root", `{}`),
+	} {
+		if err := boundSessionSnapshot(response); err != nil {
+			t.Fatal(err)
+		}
+	}
+	errorResponse := snapshotResponse(
+		"http://tenant/api/sessions/root/messages?hermes_fallback=1", `{}`,
+	)
+	errorResponse.StatusCode = http.StatusNotFound
+	if err := boundSessionSnapshot(errorResponse); err != nil {
+		t.Fatal(err)
+	}
+}

services/hermes/router/telegram.go

@@ -24,20 +24,21 @@ type telegramConfig struct {
 }
 
 type telegramBot struct {
-	config       telegramConfig
-	router       *tenantRouter
-	apiBase      string
-	fileBase     string
-	client       *http.Client
-	mediaClient  *http.Client
-	agentClient  *http.Client
-	mu           sync.RWMutex
-	botUsername  string
-	ready        bool
-	lastError    string
-	workLimit    chan struct{}
-	slotLocks    map[int]*sync.Mutex
-	slotLocksMux sync.Mutex
+	config        telegramConfig
+	router        *tenantRouter
+	apiBase       string
+	fileBase      string
+	client        *http.Client
+	mediaClient   *http.Client
+	agentClient   *http.Client
+	mu            sync.RWMutex
+	botUsername   string
+	ready         bool
+	lastError     string
+	workLimit     chan struct{}
+	activityEvery time.Duration
+	slotLocks     map[int]*sync.Mutex
+	slotLocksMux  sync.Mutex
 }
 
 type telegramUpdate struct {
@@ -96,15 +97,16 @@ func readTelegramConfig(path string) (telegramConfig, error) {
 
 func newTelegramBot(config telegramConfig, router *tenantRouter) *telegramBot {
 	return &telegramBot{
-		config:      config,
-		router:      router,
-		apiBase:     "https://api.telegram.org/bot" + config.BotToken,
-		fileBase:    "https://api.telegram.org/file/bot" + config.BotToken,
-		client:      &http.Client{Timeout: 70 * time.Second},
-		mediaClient: newTenantMediaClient(),
-		agentClient: &http.Client{Timeout: 15 * time.Minute},
-		workLimit:   make(chan struct{}, 4),
-		slotLocks:   map[int]*sync.Mutex{},
+		config:        config,
+		router:        router,
+		apiBase:       "https://api.telegram.org/bot" + config.BotToken,
+		fileBase:      "https://api.telegram.org/file/bot" + config.BotToken,
+		client:        &http.Client{Timeout: 70 * time.Second},
+		mediaClient:   newTenantMediaClient(),
+		agentClient:   &http.Client{Timeout: 15 * time.Minute},
+		workLimit:     make(chan struct{}, 4),
+		activityEvery: 4 * time.Second,
+		slotLocks:     map[int]*sync.Mutex{},
 	}
 }
 
@@ -238,16 +240,6 @@ func (bot *telegramBot) run() {
 	}
 }
 
-func commandParts(text string) (string, []string) {
-	fields := strings.Fields(strings.TrimSpace(text))
-	if len(fields) == 0 || !strings.HasPrefix(fields[0], "/") {
-		return "", nil
-	}
-	command := strings.TrimPrefix(strings.ToLower(fields[0]), "/")
-	command, _, _ = strings.Cut(command, "@")
-	return command, fields[1:]
-}
-
 func (bot *telegramBot) handleUpdate(update telegramUpdate) {
 	message := update.Message
 	if message == nil || message.From == nil || message.Chat.Type != "private" || message.Chat.ID != message.From.ID {
@@ -276,7 +268,7 @@ func (bot *telegramBot) handleUpdate(update telegramUpdate) {
 		return
 	}
 	if command == "help" {
-		_ = bot.sendText(message.Chat.ID, "Send text or a photo to chat with Hermes. Use /topic <name> to create or switch a durable topic, /topics to list topics, or /unlink to disconnect this Telegram account. Model and intensity controls are available in Hermes WebUI.")
+		_ = bot.sendText(message.Chat.ID, "Send text or a photo to chat with Hermes. Photos are available for analysis on the message that carries them. Image edits can reuse the latest image Hermes generated; inbound Telegram photos are not stored as editable artifacts. Use /topic <name> to create or switch a durable topic, /topics to list topics, or /unlink to disconnect this Telegram account. Model and intensity controls are available in Hermes WebUI.")
 		return
 	}
 	slot, linked := bot.router.telegramSlot(userID)
@@ -332,7 +324,9 @@ func (bot *telegramBot) handleUpdate(update telegramUpdate) {
 		_ = bot.sendText(message.Chat.ID, "Hermes could not save the active topic right now. Please try again shortly.")
 		return
 	}
-	_ = bot.sendAction(message.Chat.ID, "typing")
+	text = telegramInputPrompt(text, hasPhoto)
+	stopActivity := bot.startTelegramActivity(message.Chat.ID)
+	defer stopActivity()
 	var reply string
 	if hasPhoto {
 		media, fetchErr := bot.fetchTelegramPhoto(message.Photo)
@@ -429,6 +423,10 @@ func (bot *telegramBot) askTenantRequest(slot int, bodyReader io.Reader, updateI
 func (bot *telegramBot) sendAction(chatID int64, action string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
+	return bot.sendActionContext(ctx, chatID, action)
+}
+
+func (bot *telegramBot) sendActionContext(ctx context.Context, chatID int64, action string) error {
 	return bot.call(ctx, "sendChatAction", url.Values{
 		"chat_id": {strconv.FormatInt(chatID, 10)},
 		"action":  {action},

services/hermes/router/telegram_input.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -22,6 +23,93 @@ type telegramInputMedia struct {
 	Path string
 }
 
+func commandParts(text string) (string, []string) {
+	fields := strings.Fields(strings.TrimSpace(text))
+	if len(fields) == 0 || !strings.HasPrefix(fields[0], "/") {
+		return "", nil
+	}
+	command := strings.TrimPrefix(strings.ToLower(fields[0]), "/")
+	command, _, _ = strings.Cut(command, "@")
+	return command, fields[1:]
+}
+
+const inboundTelegramEditNotice = "Transport note: this Telegram photo is available for visual analysis on this turn, but it is not stored as a generated-image artifact. The image_edit_latest tools can reuse only an image Hermes generated. Do not claim an exact edit of this inbound photo; explain the limitation and offer either a new image inspired by it or an edit of the latest Hermes-generated image."
+
+func looksLikeImageEdit(text string) bool {
+	lower := strings.ToLower(strings.Join(strings.Fields(text), " "))
+	for _, phrase := range []string{
+		"edit this", "edit the image", "edit the photo", "change this image",
+		"change this photo", "modify this", "retouch", "turn this into",
+		"turn this cat", "turn this dog", "remove from the image",
+		"replace in the image", "edit it", "change it", "modify it", "crop it",
+		"resize it", "upscale it",
+	} {
+		if strings.Contains(lower, phrase) {
+			return true
+		}
+	}
+	hasTarget := false
+	for _, target := range []string{"image", "photo", "picture", "portrait", "selfie", "screenshot"} {
+		if strings.Contains(lower, target) {
+			hasTarget = true
+			break
+		}
+	}
+	if hasTarget {
+		for _, action := range []string{
+			"add ", "blur", "brighten", "change", "color", "convert", "crop",
+			"edit", "make ", "modify", "remove", "replace", "resize", "sharpen",
+			"transform", "turn ", "upscale",
+		} {
+			if strings.Contains(lower, action) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func telegramInputPrompt(text string, hasPhoto bool) string {
+	if hasPhoto {
+		return inboundTelegramEditNotice + "\n\nUser request: " + text
+	}
+	if !looksLikeImageEdit(text) {
+		return text
+	}
+	return "Transport note: no image is attached to this Telegram message. Use image_edit_latest only if Hermes previously generated an image in this private tenant. Otherwise do not claim access to an earlier inbound Telegram photo; offer a new image or ask the user to use Hermes WebUI with the source attachment.\n\nUser request: " + text
+}
+
+func (bot *telegramBot) startTelegramActivity(chatID int64) func() {
+	done := make(chan struct{})
+	stopped := make(chan struct{})
+	ctx, cancel := context.WithCancel(context.Background())
+	every := bot.activityEvery
+	if every <= 0 {
+		every = 4 * time.Second
+	}
+	go func() {
+		defer close(stopped)
+		ticker := time.NewTicker(every)
+		defer ticker.Stop()
+		for {
+			_ = bot.sendActionContext(ctx, chatID, "typing")
+			select {
+			case <-done:
+				return
+			case <-ticker.C:
+			}
+		}
+	}()
+	var once sync.Once
+	return func() {
+		once.Do(func() {
+			close(done)
+			cancel()
+		})
+		<-stopped
+	}
+}
+
 func largestTelegramPhoto(photos []telegramPhotoSize) (telegramPhotoSize, error) {
 	var selected telegramPhotoSize
 	for _, photo := range photos {

services/hermes/router/telegram_media.go

@@ -24,7 +24,11 @@ const (
 	tenantMediaLimit     = 50 << 20
 )
 
-var telegramMediaMarker = regexp.MustCompile(`MEDIA:([^\s\)\]]+)`)
+var (
+	telegramMediaMarker = regexp.MustCompile(`(?i)MEDIA:\s*([^\s\)\]\}"']+)`)
+	telegramMediaLabel  = regexp.MustCompile(`(?i)MEDIA:`)
+	telegramPrivatePath = regexp.MustCompile(`(?i)(?:/opt/data/(?:cache/images|workspace)|/workspace)/[^\s\)\]\}"']+`)
+)
 
 type telegramReply struct {
 	Text       string
@@ -50,6 +54,8 @@ func parseTelegramReply(text string) telegramReply {
 		}
 	}
 	cleaned := telegramMediaMarker.ReplaceAllString(text, "")
+	cleaned = telegramPrivatePath.ReplaceAllString(cleaned, "[private attachment]")
+	cleaned = telegramMediaLabel.ReplaceAllString(cleaned, "")
 	lines := strings.Split(cleaned, "\n")
 	compact := make([]string, 0, len(lines))
 	blank := false

services/hermes/router/telegram_media_test.go

@@ -124,6 +124,20 @@ func TestParseTelegramReplyRemovesOneOrMoreInternalMarkers(t *testing.T) {
 	}
 }
 
+func TestParseTelegramReplyScrubsMalformedMarkersAndBarePrivatePaths(t *testing.T) {
+	reply := parseTelegramReply(
+		"Delivered. MEDIA: /opt/data/cache/images/one.png and /workspace/private/two.png; MEDIA:",
+	)
+	if len(reply.MediaPaths) != 1 || reply.MediaPaths[0] != "/opt/data/cache/images/one.png" {
+		t.Fatalf("valid spaced marker was not retained for delivery: %#v", reply)
+	}
+	for _, unsafe := range []string{"MEDIA:", "/opt/data/", "/workspace/"} {
+		if strings.Contains(reply.Text, unsafe) {
+			t.Fatalf("unsafe transport detail %q leaked in %q", unsafe, reply.Text)
+		}
+	}
+}
+
 func TestNormalizeTenantMediaPath(t *testing.T) {
 	allowed := map[string]string{
 		"/workspace/./renders//portrait.png":          "/opt/data/workspace/renders/portrait.png",

services/hermes/router/telegram_test.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 )
@@ -273,6 +274,144 @@ func TestTelegramTopicValidationAndLegacyGeneralConversation(t *testing.T) {
 	}
 }
 
+func TestTelegramTopicContinuityExpiresOnlyAfterIdleWeek(t *testing.T) {
+	statePath := filepath.Join(t.TempDir(), "state.json")
+	base := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
+	now := base
+	router, err := newTenantRouter(statePath, 1, func(int) string { return "" })
+	if err != nil {
+		t.Fatal(err)
+	}
+	router.now = func() time.Time { return now }
+	first, err := router.selectTelegramTopic("123", "Week plan")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, elapsed := range []time.Duration{time.Hour, 24 * time.Hour} {
+		now = base.Add(elapsed)
+		resumed, touchErr := router.touchTelegramTopic("123")
+		if touchErr != nil || resumed.Conversation != first.Conversation {
+			t.Fatalf("topic changed after %s: %#v, %v", elapsed, resumed, touchErr)
+		}
+	}
+	// Exactly seven idle days remains the same durable conversation.
+	now = base.Add(8 * 24 * time.Hour)
+	exactWeek, err := router.touchTelegramTopic("123")
+	if err != nil || exactWeek.Conversation != first.Conversation {
+		t.Fatalf("topic changed at exact idle week: %#v, %v", exactWeek, err)
+	}
+	reloaded, err := newTenantRouter(statePath, 1, func(int) string { return "" })
+	if err != nil {
+		t.Fatal(err)
+	}
+	reloaded.now = func() time.Time { return now }
+	if active := reloaded.activeTelegramTopic("123"); active != exactWeek {
+		t.Fatalf("router restart changed topic: %#v != %#v", active, exactWeek)
+	}
+	now = now.Add(7*24*time.Hour + time.Second)
+	rotated, err := reloaded.touchTelegramTopic("123")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rotated.Conversation == first.Conversation || !strings.HasSuffix(rotated.Conversation, "-g1") {
+		t.Fatalf("expired topic did not rotate deterministically: %#v", rotated)
+	}
+	again, err := newTenantRouter(statePath, 1, func(int) string { return "" })
+	if err != nil {
+		t.Fatal(err)
+	}
+	if active := again.activeTelegramTopic("123"); active != rotated {
+		t.Fatalf("rotated identity changed after restart: %#v != %#v", active, rotated)
+	}
+}
+
+func TestTelegramActivityRepeatsAndStops(t *testing.T) {
+	var actions atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		if request.URL.Path != "/sendChatAction" {
+			t.Fatalf("unexpected Telegram method %s", request.URL.Path)
+		}
+		actions.Add(1)
+		writer.Header().Set("Content-Type", "application/json")
+		_, _ = writer.Write([]byte(`{"ok":true,"result":true}`))
+	}))
+	defer server.Close()
+	bot := &telegramBot{
+		apiBase:       server.URL,
+		client:        &http.Client{Timeout: time.Second},
+		activityEvery: 5 * time.Millisecond,
+	}
+	stop := bot.startTelegramActivity(42)
+	deadline := time.Now().Add(time.Second)
+	for actions.Load() < 2 && time.Now().Before(deadline) {
+		time.Sleep(time.Millisecond)
+	}
+	stop()
+	stoppedAt := actions.Load()
+	time.Sleep(20 * time.Millisecond)
+	if stoppedAt < 2 || actions.Load() != stoppedAt {
+		t.Fatalf("activity did not progress and stop: before=%d after=%d", stoppedAt, actions.Load())
+	}
+}
+
+func TestTelegramActivityStopCancelsInflightRequest(t *testing.T) {
+	started := make(chan struct{})
+	release := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) {
+		close(started)
+		select {
+		case <-request.Context().Done():
+		case <-release:
+		}
+	}))
+	defer server.Close()
+	bot := &telegramBot{
+		apiBase:       server.URL,
+		client:        &http.Client{Timeout: time.Minute},
+		activityEvery: time.Hour,
+	}
+	stop := bot.startTelegramActivity(42)
+	select {
+	case <-started:
+	case <-time.After(time.Second):
+		t.Fatal("activity request did not start")
+	}
+	stopped := make(chan struct{})
+	go func() {
+		stop()
+		close(stopped)
+	}()
+	select {
+	case <-stopped:
+	case <-time.After(time.Second):
+		t.Fatal("activity stop did not cancel the in-flight request")
+	}
+	close(release)
+}
+
+func TestTelegramImageEditPromptStatesTransportBoundary(t *testing.T) {
+	withPhoto := telegramInputPrompt("Edit this photo into a watercolor", true)
+	if !strings.Contains(withPhoto, "available for visual analysis") ||
+		!strings.Contains(withPhoto, "not stored as a generated-image artifact") {
+		t.Fatalf("inbound attachment boundary missing: %q", withPhoto)
+	}
+	withoutPhoto := telegramInputPrompt("Edit this photo into a watercolor", false)
+	if !strings.Contains(withoutPhoto, "no image is attached") ||
+		!strings.Contains(withoutPhoto, "image_edit_latest only if Hermes previously generated") {
+		t.Fatalf("prior generated image boundary missing: %q", withoutPhoto)
+	}
+	for _, request := range []string{"Make the photo brighter", "Crop it closer"} {
+		if got := telegramInputPrompt(request, false); !strings.Contains(got, "no image is attached") {
+			t.Fatalf("edit variant omitted attachment boundary: %q", got)
+		}
+	}
+	plain := telegramInputPrompt("Explain this photograph", true)
+	if !strings.Contains(plain, "available for visual analysis") ||
+		!strings.Contains(plain, "User request: Explain this photograph") {
+		t.Fatalf("photo analysis boundary missing: %q", plain)
+	}
+}
+
 func TestTelegramInputRejectsUnsafePathsAndNonImages(t *testing.T) {
 	for _, value := range []string{"", "../secret", "/absolute/image.jpg", `photos\image.jpg`, "photos/../../secret"} {
 		if _, err := safeTelegramFilePath(value); err == nil {

services/hermes/router/telegram_topics.go

@@ -5,12 +5,16 @@ import (
 	"encoding/hex"
 	"errors"
 	"sort"
+	"strconv"
 	"strings"
 	"unicode"
 	"unicode/utf8"
 )
 
-const telegramDefaultTopic = "General"
+const (
+	telegramDefaultTopic = "General"
+	telegramTopicIdleTTL = 7 * 24 * 60 * 60
+)
 
 type activeTelegramTopic struct {
 	Conversation string
@@ -42,12 +46,35 @@ func telegramTopicID(label string) string {
 }
 
 func telegramTopicConversation(topicID string) string {
+	return telegramTopicConversationGeneration(topicID, 0)
+}
+
+func telegramTopicConversationGeneration(topicID string, generation int64) string {
 	if topicID == "" || topicID == "general" {
 		// Preserve the original named conversation so existing Telegram context
 		// survives rollout into topic-aware routing.
-		return "telegram"
+		if generation == 0 {
+			return "telegram"
+		}
+		topicID = "general"
 	}
-	return "telegram-topic-" + topicID
+	conversation := "telegram-topic-" + topicID
+	if generation > 0 {
+		conversation += "-g" + strconv.FormatInt(generation, 10)
+	}
+	return conversation
+}
+
+func topicExpired(topic telegramTopic, now int64) bool {
+	return topic.LastUsed > 0 && now > topic.LastUsed && now-topic.LastUsed > telegramTopicIdleTTL
+}
+
+func touchTopic(topic telegramTopic, now int64) telegramTopic {
+	if topicExpired(topic, now) {
+		topic.Generation++
+	}
+	topic.LastUsed = now
+	return topic
 }
 
 func (router *tenantRouter) activeTelegramTopic(userID string) activeTelegramTopic {
@@ -63,7 +90,10 @@ func (router *tenantRouter) activeTelegramTopic(userID string) activeTelegramTop
 	if topic, found := state.Topics[topicID]; found && strings.TrimSpace(topic.Label) != "" {
 		label = topic.Label
 	}
-	return activeTelegramTopic{Conversation: telegramTopicConversation(topicID), Label: label}
+	return activeTelegramTopic{
+		Conversation: telegramTopicConversationGeneration(topicID, state.Topics[topicID].Generation),
+		Label:        label,
+	}
 }
 
 func (router *tenantRouter) selectTelegramTopic(userID, rawLabel string) (activeTelegramTopic, error) {
@@ -83,12 +113,18 @@ func (router *tenantRouter) selectTelegramTopic(userID, rawLabel string) (active
 		state.Topics = map[string]telegramTopic{}
 	}
 	state.Active = topicID
-	state.Topics[topicID] = telegramTopic{Label: label, LastUsed: router.now().Unix()}
+	topic := state.Topics[topicID]
+	topic.Label = label
+	topic = touchTopic(topic, router.now().Unix())
+	state.Topics[topicID] = topic
 	router.state.TelegramTopics[identity] = state
 	if err := router.saveLocked(); err != nil {
 		return activeTelegramTopic{}, err
 	}
-	return activeTelegramTopic{Conversation: telegramTopicConversation(topicID), Label: label}, nil
+	return activeTelegramTopic{
+		Conversation: telegramTopicConversationGeneration(topicID, topic.Generation),
+		Label:        label,
+	}, nil
 }
 
 func (router *tenantRouter) touchTelegramTopic(userID string) (activeTelegramTopic, error) {
@@ -108,13 +144,16 @@ func (router *tenantRouter) touchTelegramTopic(userID string) (activeTelegramTop
 	if strings.TrimSpace(topic.Label) == "" {
 		topic.Label = telegramDefaultTopic
 	}
-	topic.LastUsed = router.now().Unix()
+	topic = touchTopic(topic, router.now().Unix())
 	state.Topics[topicID] = topic
 	router.state.TelegramTopics[identity] = state
 	if err := router.saveLocked(); err != nil {
 		return activeTelegramTopic{}, err
 	}
-	return activeTelegramTopic{Conversation: telegramTopicConversation(topicID), Label: topic.Label}, nil
+	return activeTelegramTopic{
+		Conversation: telegramTopicConversationGeneration(topicID, topic.Generation),
+		Label:        topic.Label,
+	}, nil
 }
 
 func (router *tenantRouter) telegramTopics(userID string) []telegramTopic {

services/hermes/router/web.go

@@ -93,6 +93,8 @@ const privateFilesPage = `<!doctype html>
 </html>`
 
 const bridgeCSS = `
+[data-panel="kanban"],[data-panel="logs"],[data-panel="insights"],[data-panel="memory"],[data-panel="skills"],[data-panel="workspaces"],[data-panel="todos"],[data-panel="tasks"],[data-panel="profiles"]{display:none!important}
+#settingsModel,label:has(#settingsModel),.settings-row:has(#settingsModel){display:none!important}
 #hermes-files-sidebar,#hermes-telegram-sidebar{display:flex;align-items:center}
 .hermes-link-page{margin:0;min-height:100vh;display:grid;place-items:center;background:#0f172a;color:#e2e8f0;font:16px/1.5 system-ui,sans-serif}
 .hermes-link-card{width:min(620px,calc(100% - 40px));box-sizing:border-box;padding:32px;border:1px solid #334155;border-radius:18px;background:#111827;box-shadow:0 20px 60px #0006}
@@ -105,8 +107,11 @@ const bridgeCSS = `
 const bridgeJS = `(() => {
   const page = document.querySelector('[data-telegram-page]');
   const filesPage = document.querySelector('[data-files-page]');
+  const chatAdministrationPanels = ['kanban','logs','insights','memory','skills','workspaces','todos','tasks','profiles'];
   const hideChatAdministration = () => {
-    document.querySelectorAll('[data-panel="kanban"]').forEach((node) => { node.hidden = true; });
+    chatAdministrationPanels.forEach((panel) => {
+      document.querySelectorAll('[data-panel="' + panel + '"]').forEach((node) => { node.hidden = true; });
+    });
   };
   hideChatAdministration();
   if (!page && !filesPage) {
@@ -159,7 +164,7 @@ const bridgeJS = `(() => {
     const observer = new MutationObserver(() => {
       if (scheduled) return;
       scheduled = true;
-      requestAnimationFrame(() => { scheduled = false; installSidebarLinks(); });
+      requestAnimationFrame(() => { scheduled = false; hideChatAdministration(); installSidebarLinks(); });
     });
     installSidebarLinks();
     observer.observe(document.body, {childList:true, subtree:true});
@@ -354,6 +359,15 @@ func (router *tenantRouter) serveTelegramWeb(writer http.ResponseWriter, request
 		writer.Header().Set("Cache-Control", "public, max-age=3600")
 		_, _ = io.WriteString(writer, bridgeJS)
 		return true
+	case "/hermes-session-continuity.js":
+		if request.Method != http.MethodGet {
+			http.Error(writer, "method not allowed", http.StatusMethodNotAllowed)
+			return true
+		}
+		writer.Header().Set("Content-Type", "application/javascript; charset=utf-8")
+		writer.Header().Set("Cache-Control", "public, max-age=3600")
+		_, _ = io.WriteString(writer, sessionContinuityJS)
+		return true
 	case "/telegram":
 		if request.Method != http.MethodGet {
 			http.Error(writer, "method not allowed", http.StatusMethodNotAllowed)
@@ -455,6 +469,9 @@ func injectChatBridge(response *http.Response) error {
 		content = strings.Replace(content, "</head>", `<link rel="stylesheet" href="/hermes-chat-bridge.css?v=20260813-telegram-readiness-v1"></head>`, 1)
 		content = strings.Replace(content, "</body>", `<script src="/hermes-chat-bridge.js?v=20260813-telegram-readiness-v1" defer></script></body>`, 1)
 	}
+	if !strings.Contains(content, "hermes-session-continuity.js") {
+		content = strings.Replace(content, "</body>", `<script src="/hermes-session-continuity.js?v=20260817-v1" defer></script></body>`, 1)
+	}
 	response.Body = io.NopCloser(strings.NewReader(content))
 	response.ContentLength = int64(len(content))
 	response.Header.Set("Content-Length", strconv.Itoa(len(content)))

services/hermes/scm-common/kustomization.yaml

@@ -2,7 +2,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 configMapGenerator:
-  - name: hermes-scm-boundary
+  - name: hermes-scm-boundary-v2
     files:
       - deadline_http.py=scripts/deadline_http.py
       - git_pack_objects.py=scripts/git_pack_objects.py

services/hermes/scripts/ai_usage_codex.py

@@ -0,0 +1,138 @@
+"""Bounded Codex app-server query and child-process cleanup."""
+
+from __future__ import annotations
+
+import json
+import os
+import selectors
+import subprocess
+import time
+from typing import Any
+
+
+CODEX_BIN = os.environ.get("ATLAS_AI_CODEX_BIN", "/opt/data/tools/bin/codex")
+CODEX_HOME = os.environ.get("CODEX_HOME", "/runtime-access/codex")
+CODEX_CLEANUP_TIMEOUT_SECONDS = 5
+
+
+def _close_stream(stream: Any) -> bool:
+    """Close a subprocess pipe without allowing cleanup errors to escape."""
+    if stream is None:
+        return True
+    try:
+        stream.close()
+    except Exception:
+        return False
+    return True
+
+
+def _cleanup_codex_process(
+    process: subprocess.Popen[str], selector: selectors.BaseSelector | None
+) -> bool:
+    """Stop and reap a Codex child within two fixed waits, then close its pipes."""
+    cleanup_ok = True
+    if selector is not None:
+        try:
+            selector.close()
+        except Exception:
+            cleanup_ok = False
+
+    # Closing the request pipe first also gives a responsive app-server an EOF.
+    cleanup_ok = _close_stream(getattr(process, "stdin", None)) and cleanup_ok
+    try:
+        process.terminate()
+    except ProcessLookupError:
+        pass
+    except Exception:
+        cleanup_ok = False
+
+    needs_kill = False
+    try:
+        process.wait(timeout=CODEX_CLEANUP_TIMEOUT_SECONDS)
+    except subprocess.TimeoutExpired:
+        needs_kill = True
+    except Exception:
+        cleanup_ok = False
+        needs_kill = True
+
+    if needs_kill:
+        try:
+            process.kill()
+        except ProcessLookupError:
+            pass
+        except Exception:
+            cleanup_ok = False
+        try:
+            process.wait(timeout=CODEX_CLEANUP_TIMEOUT_SECONDS)
+        except Exception:
+            cleanup_ok = False
+
+    cleanup_ok = _close_stream(getattr(process, "stdout", None)) and cleanup_ok
+    cleanup_ok = _close_stream(getattr(process, "stderr", None)) and cleanup_ok
+    return cleanup_ok
+
+
+def query_codex(timeout: float = 20) -> tuple[dict[str, Any], dict[str, Any]]:
+    """Read Codex account quota and usage through its structured app-server protocol."""
+    process = subprocess.Popen(
+        [CODEX_BIN, "app-server", "--stdio"],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        text=True,
+        bufsize=1,
+        env={**os.environ, "CODEX_HOME": CODEX_HOME},
+    )
+    requests = (
+        {
+            "id": 1,
+            "method": "initialize",
+            "params": {
+                "clientInfo": {
+                    "name": "atlas-ai-usage-exporter",
+                    "title": "Atlas AI Usage Exporter",
+                    "version": "1.0.0",
+                },
+                "capabilities": {"experimentalApi": True},
+            },
+        },
+        {"method": "initialized", "params": {}},
+        {"id": 2, "method": "account/rateLimits/read", "params": None},
+        {"id": 3, "method": "account/usage/read", "params": None},
+    )
+    selector: selectors.BaseSelector | None = None
+    rate_result: dict[str, Any]
+    usage_result: dict[str, Any]
+    try:
+        if process.stdin is None or process.stdout is None:
+            raise RuntimeError("Codex app-server pipes are unavailable")
+        for message in requests:
+            process.stdin.write(json.dumps(message, separators=(",", ":")) + "\n")
+        process.stdin.flush()
+        selector = selectors.DefaultSelector()
+        selector.register(process.stdout, selectors.EVENT_READ)
+        responses: dict[int, dict[str, Any]] = {}
+        deadline = time.monotonic() + timeout
+        while len(responses) < 3 and time.monotonic() < deadline:
+            for key, _ in selector.select(min(1, max(0, deadline - time.monotonic()))):
+                line = key.fileobj.readline()
+                if not line:
+                    continue
+                message = json.loads(line)
+                if message.get("id") in (1, 2, 3):
+                    responses[int(message["id"])] = message
+        for response_id in (1, 2, 3):
+            response = responses.get(response_id)
+            if (
+                not response
+                or "error" in response
+                or not isinstance(response.get("result"), dict)
+            ):
+                raise RuntimeError(f"Codex app-server response {response_id} failed")
+        rate_result = responses[2]["result"]
+        usage_result = responses[3]["result"]
+    finally:
+        cleanup_ok = _cleanup_codex_process(process, selector)
+    if not cleanup_ok:
+        raise RuntimeError("Codex app-server cleanup failed")
+    return rate_result, usage_result

services/hermes/scripts/ai_usage_exporter.py

@@ -5,20 +5,18 @@ from __future__ import annotations
 
 import json
 import os
-import selectors
-import subprocess
 import threading
 import time
 from dataclasses import dataclass, field
 from datetime import UTC, date, datetime, timedelta
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 from typing import Any
 from urllib.request import Request, urlopen
 
+import ai_usage_codex as codex_query
+import ai_usage_http as http_engine
+
 
-CODEX_BIN = os.environ.get("ATLAS_AI_CODEX_BIN", "/opt/data/tools/bin/codex")
-CODEX_HOME = os.environ.get("CODEX_HOME", "/runtime-access/codex")
 CLAUDE_CREDENTIALS = Path(
     os.environ.get(
         "ATLAS_AI_CLAUDE_CREDENTIALS",
@@ -48,6 +46,8 @@ METRIC_HELP = {
     "atlas_ai_quota_used_percent": "Used percentage in a first-party coding CLI quota window.",
     "atlas_ai_quota_window_duration_seconds": "Nominal duration of a coding CLI quota window.",
 }
+POLL_STARTUP_GRACE_SECONDS = 45
+POLL_PROGRESS_BUDGET_SECONDS = 60
 
 
 @dataclass(frozen=True)
@@ -75,7 +75,9 @@ def _number(value: Any) -> float | None:
     if isinstance(value, bool) or not isinstance(value, (int, float)):
         return None
     converted = float(value)
-    return converted if converted == converted and abs(converted) != float("inf") else None
+    return (
+        converted if converted == converted and abs(converted) != float("inf") else None
+    )
 
 
 def _timestamp(value: Any) -> float | None:
@@ -112,7 +114,10 @@ def _codex_limit_name(limit_id: str, snapshot: dict[str, Any]) -> str:
 
 
 def parse_codex_payloads(
-    rate_response: dict[str, Any], usage_response: dict[str, Any], *, today: date | None = None
+    rate_response: dict[str, Any],
+    usage_response: dict[str, Any],
+    *,
+    today: date | None = None,
 ) -> list[Sample]:
     """Convert structured Codex app-server responses into bounded metrics."""
     samples: list[Sample] = []
@@ -140,15 +145,23 @@ def parse_codex_payloads(
             samples.extend(
                 (
                     Sample("atlas_ai_quota_used_percent", labels, used),
-                    Sample("atlas_ai_quota_remaining_percent", labels, max(0, 100 - used)),
+                    Sample(
+                        "atlas_ai_quota_remaining_percent", labels, max(0, 100 - used)
+                    ),
                 )
             )
             reset = _timestamp(raw_window.get("resetsAt"))
             duration = _number(raw_window.get("windowDurationMins"))
             if reset is not None:
-                samples.append(Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset))
+                samples.append(
+                    Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset)
+                )
             if duration is not None:
-                samples.append(Sample("atlas_ai_quota_window_duration_seconds", labels, duration * 60))
+                samples.append(
+                    Sample(
+                        "atlas_ai_quota_window_duration_seconds", labels, duration * 60
+                    )
+                )
 
     summary = usage_response.get("summary")
     if isinstance(summary, dict):
@@ -189,8 +202,16 @@ def parse_codex_payloads(
     anchor_day = max((start for start, _ in parsed_daily), default=current_day)
     for period, days in (("latest_day", 1), ("seven_day", 7), ("thirty_day", 30)):
         earliest = anchor_day - timedelta(days=days - 1)
-        value = sum(tokens for start, tokens in parsed_daily if earliest <= start <= anchor_day)
-        samples.append(Sample("atlas_ai_account_tokens", {"provider": "openai", "period": period}, value))
+        value = sum(
+            tokens for start, tokens in parsed_daily if earliest <= start <= anchor_day
+        )
+        samples.append(
+            Sample(
+                "atlas_ai_account_tokens",
+                {"provider": "openai", "period": period},
+                value,
+            )
+        )
     return samples
 
 
@@ -213,7 +234,9 @@ def parse_claude_payload(payload: dict[str, Any]) -> list[Sample]:
         )
         reset = _timestamp(window.get("resets_at"))
         if reset is not None:
-            samples.append(Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset))
+            samples.append(
+                Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset)
+            )
     extra = payload.get("extra_usage")
     if isinstance(extra, dict):
         enabled = extra.get("is_enabled")
@@ -228,64 +251,7 @@ def parse_claude_payload(payload: dict[str, Any]) -> list[Sample]:
     return samples
 
 
-def query_codex(timeout: float = 20) -> tuple[dict[str, Any], dict[str, Any]]:
-    """Read Codex account quota and usage through its structured app-server protocol."""
-    process = subprocess.Popen(
-        [CODEX_BIN, "app-server", "--stdio"],
-        stdin=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.DEVNULL,
-        text=True,
-        bufsize=1,
-        env={**os.environ, "CODEX_HOME": CODEX_HOME},
-    )
-    requests = (
-        {
-            "id": 1,
-            "method": "initialize",
-            "params": {
-                "clientInfo": {
-                    "name": "atlas-ai-usage-exporter",
-                    "title": "Atlas AI Usage Exporter",
-                    "version": "1.0.0",
-                },
-                "capabilities": {"experimentalApi": True},
-            },
-        },
-        {"method": "initialized", "params": {}},
-        {"id": 2, "method": "account/rateLimits/read", "params": None},
-        {"id": 3, "method": "account/usage/read", "params": None},
-    )
-    try:
-        if process.stdin is None or process.stdout is None:
-            raise RuntimeError("Codex app-server pipes are unavailable")
-        for message in requests:
-            process.stdin.write(json.dumps(message, separators=(",", ":")) + "\n")
-        process.stdin.flush()
-        selector = selectors.DefaultSelector()
-        selector.register(process.stdout, selectors.EVENT_READ)
-        responses: dict[int, dict[str, Any]] = {}
-        deadline = time.monotonic() + timeout
-        while len(responses) < 3 and time.monotonic() < deadline:
-            for key, _ in selector.select(min(1, max(0, deadline - time.monotonic()))):
-                line = key.fileobj.readline()
-                if not line:
-                    continue
-                message = json.loads(line)
-                if message.get("id") in (1, 2, 3):
-                    responses[int(message["id"])] = message
-        for response_id in (1, 2, 3):
-            response = responses.get(response_id)
-            if not response or "error" in response or not isinstance(response.get("result"), dict):
-                raise RuntimeError(f"Codex app-server response {response_id} failed")
-        return responses[2]["result"], responses[3]["result"]
-    finally:
-        process.terminate()
-        try:
-            process.wait(timeout=5)
-        except subprocess.TimeoutExpired:
-            process.kill()
-            process.wait(timeout=5)
+query_codex = codex_query.query_codex
 
 
 def query_claude() -> dict[str, Any]:
@@ -322,6 +288,24 @@ class Collector:
         self._lock = threading.Lock()
         self._providers = {name: ProviderState() for name in ("openai", "anthropic")}
 
+    def record_failure(
+        self,
+        provider: str,
+        *,
+        started: float | None = None,
+        monotonic_started: float | None = None,
+    ) -> None:
+        """Record a failed attempt while retaining the provider's last good samples."""
+        attempted_at = time.time() if started is None else started
+        duration_started = (
+            time.monotonic() if monotonic_started is None else monotonic_started
+        )
+        with self._lock:
+            state = self._providers[provider]
+            state.last_attempt = attempted_at
+            state.duration = max(0, time.monotonic() - duration_started)
+            state.fetch_success = False
+
     def refresh_provider(self, provider: str) -> None:
         """Refresh one provider while retaining the last good values on failure."""
         started = time.time()
@@ -333,19 +317,24 @@ class Collector:
                 samples = parse_claude_payload(query_claude())
             else:
                 raise ValueError("unknown provider")
-            success = True
-        except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as error:
-            print(f"{provider} quota collection deferred: {type(error).__name__}", flush=True)
-            samples = []
-            success = False
+        except Exception as error:
+            print(
+                f"{provider} quota collection deferred: {type(error).__name__}",
+                flush=True,
+            )
+            self.record_failure(
+                provider,
+                started=started,
+                monotonic_started=monotonic_started,
+            )
+            return
         with self._lock:
             state = self._providers[provider]
             state.last_attempt = started
             state.duration = time.monotonic() - monotonic_started
-            state.fetch_success = success
-            if success:
-                state.samples = samples
-                state.last_success = time.time()
+            state.fetch_success = True
+            state.samples = samples
+            state.last_success = time.time()
 
     def render(self) -> bytes:
         """Render the current provider states in Prometheus text format."""
@@ -360,10 +349,24 @@ class Collector:
             samples.extend(state.samples)
             samples.extend(
                 (
-                    Sample("atlas_ai_quota_fetch_success", labels, float(state.fetch_success)),
-                    Sample("atlas_ai_quota_last_attempt_timestamp_seconds", labels, state.last_attempt),
-                    Sample("atlas_ai_quota_last_success_timestamp_seconds", labels, state.last_success),
-                    Sample("atlas_ai_quota_fetch_duration_seconds", labels, state.duration),
+                    Sample(
+                        "atlas_ai_quota_fetch_success",
+                        labels,
+                        float(state.fetch_success),
+                    ),
+                    Sample(
+                        "atlas_ai_quota_last_attempt_timestamp_seconds",
+                        labels,
+                        state.last_attempt,
+                    ),
+                    Sample(
+                        "atlas_ai_quota_last_success_timestamp_seconds",
+                        labels,
+                        state.last_success,
+                    ),
+                    Sample(
+                        "atlas_ai_quota_fetch_duration_seconds", labels, state.duration
+                    ),
                 )
             )
         lines: list[str] = []
@@ -381,38 +384,112 @@ class Collector:
         return ("\n".join(lines) + "\n").encode("utf-8")
 
 
+class PollingEngine:
+    """Run isolated provider polls and track bounded forward progress."""
+
+    def __init__(
+        self,
+        collector: Collector,
+        *,
+        interval: float,
+        startup_grace: float = POLL_STARTUP_GRACE_SECONDS,
+        progress_timeout: float | None = None,
+        clock: Any = time.monotonic,
+    ) -> None:
+        self.collector = collector
+        self.interval = interval
+        self.startup_grace = startup_grace
+        self.progress_timeout = (
+            interval + POLL_PROGRESS_BUDGET_SECONDS
+            if progress_timeout is None
+            else progress_timeout
+        )
+        self._clock = clock
+        self._started_at = clock()
+        self._last_progress: float | None = None
+        self._progress_lock = threading.Lock()
+        self._thread: threading.Thread | None = None
+
+    def _mark_progress(self) -> None:
+        """Record completion of one bounded provider attempt."""
+        with self._progress_lock:
+            self._last_progress = self._clock()
+
+    def poll_once(self) -> None:
+        """Refresh every provider even if another provider fails unexpectedly."""
+        for provider in ("openai", "anthropic"):
+            started = time.time()
+            monotonic_started = time.monotonic()
+            try:
+                self.collector.refresh_provider(provider)
+            except Exception as error:
+                print(
+                    f"{provider} quota collection isolated: {type(error).__name__}",
+                    flush=True,
+                )
+                try:
+                    self.collector.record_failure(
+                        provider,
+                        started=started,
+                        monotonic_started=monotonic_started,
+                    )
+                except Exception as record_error:
+                    print(
+                        f"{provider} quota failure accounting deferred: "
+                        f"{type(record_error).__name__}",
+                        flush=True,
+                    )
+            finally:
+                self._mark_progress()
+
+    def run(self) -> None:
+        """Poll forever without allowing a cycle-level exception to stop the thread."""
+        while True:
+            try:
+                self.poll_once()
+            except Exception as error:
+                print(
+                    f"quota polling cycle deferred: {type(error).__name__}", flush=True
+                )
+            time.sleep(self.interval)
+
+    def start(self) -> None:
+        """Start the daemon poller exactly once."""
+        if self._thread is not None:
+            return
+        self._thread = threading.Thread(
+            target=self.run,
+            name="ai-usage-poller",
+            daemon=True,
+        )
+        self._thread.start()
+
+    def is_healthy(self) -> bool:
+        """Report thread liveness and progress, independent of provider success."""
+        thread = self._thread
+        if thread is None or not thread.is_alive():
+            return False
+        now = self._clock()
+        with self._progress_lock:
+            last_progress = self._last_progress
+        if last_progress is None:
+            return now - self._started_at <= self.startup_grace
+        return now - last_progress <= self.progress_timeout
+
+
+make_handler = http_engine.make_handler
+Server = http_engine.Server
+
+
 def main() -> int:
     """Poll quota APIs and serve only sanitized metrics and health endpoints."""
     collector = Collector()
     interval = max(60, int(os.environ.get("ATLAS_AI_USAGE_INTERVAL_SECONDS", "300")))
 
-    def polling_loop() -> None:
-        while True:
-            for provider in ("openai", "anthropic"):
-                collector.refresh_provider(provider)
-            time.sleep(interval)
-
-    class Handler(BaseHTTPRequestHandler):
-        def do_GET(self) -> None:  # noqa: N802
-            if self.path == "/metrics":
-                payload = collector.render()
-                self.send_response(200)
-                self.send_header("Content-Type", "text/plain; version=0.0.4")
-                self.send_header("Content-Length", str(len(payload)))
-                self.end_headers()
-                self.wfile.write(payload)
-            elif self.path == "/healthz":
-                self.send_response(200)
-                self.end_headers()
-            else:
-                self.send_error(404)
-
-        def log_message(self, _format: str, *_args: object) -> None:
-            return
-
-    threading.Thread(target=polling_loop, daemon=True).start()
+    poller = PollingEngine(collector, interval=interval)
+    poller.start()
     port = int(os.environ.get("ATLAS_AI_USAGE_PORT", "9010"))
-    server = ThreadingHTTPServer(("0.0.0.0", port), Handler)
+    server = Server(("0.0.0.0", port), make_handler(collector, poller))
     server.serve_forever()
     return 0
 

services/hermes/scripts/ai_usage_http.py

@@ -0,0 +1,49 @@
+"""Disconnect-safe HTTP serving for the AI usage exporter."""
+
+from __future__ import annotations
+
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import Any
+
+
+def make_handler(collector: Any, poller: Any) -> type[BaseHTTPRequestHandler]:
+    """Build an HTTP handler bound to one collector and polling engine."""
+
+    class Handler(BaseHTTPRequestHandler):
+        def _respond(self, status: int, payload: bytes, content_type: str) -> None:
+            self.send_response(status)
+            self.send_header("Content-Type", content_type)
+            self.send_header("Content-Length", str(len(payload)))
+            self.end_headers()
+            self.wfile.write(payload)
+
+        def do_GET(self) -> None:  # noqa: N802
+            try:
+                if self.path == "/metrics":
+                    self._respond(
+                        200,
+                        collector.render(),
+                        "text/plain; version=0.0.4",
+                    )
+                elif self.path == "/healthz":
+                    healthy = poller.is_healthy()
+                    self._respond(
+                        200 if healthy else 503,
+                        b"ok\n" if healthy else b"poller unhealthy\n",
+                        "text/plain; charset=utf-8",
+                    )
+                else:
+                    self.send_error(404)
+            except (BrokenPipeError, ConnectionResetError):
+                return
+
+        def log_message(self, _format: str, *_args: object) -> None:
+            return
+
+    return Handler
+
+
+class Server(ThreadingHTTPServer):
+    """Threaded metrics server that does not retain disconnected clients."""
+
+    daemon_threads = True

services/hermes/scripts/cli_lane_board.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""Bounded Kanban access and workspace resolution for Hermes CLI workers."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import sys
+import time
+from pathlib import Path
+from typing import Any, Callable
+
+from cli_lane_config import (
+    BOARD_CORRUPTION_ERRORS,
+    EXTERNAL_PREFIX,
+    KANBAN_STORAGE_ATTEMPTS,
+    WORKTREE_LOCK,
+)
+
+
+def _task_value(task: Any, name: str, default: Any = None) -> Any:
+    return getattr(task, name, default)
+
+def _task_context(kanban_db: Any, conn: Any, task_id: str) -> str:
+    value = kanban_db.build_worker_context(conn, task_id)
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, indent=2, default=str)
+
+def _resolve_workspace(kanban_db: Any, conn: Any, task: Any, board: str) -> Path:
+    # External coding lanes always receive their own linked worktree. A task
+    # that does not resolve to a Git repository is blocked rather than sharing
+    # a mutable checkout with another unattended worker.
+    # Git worktree creation updates common-repository metadata. Serialize that
+    # short materialization step while allowing the provider workers themselves
+    # to run concurrently in independent worktrees.
+    with WORKTREE_LOCK:
+        value, branch_name = kanban_db._resolve_worktree_workspace(task, board=board)
+    kanban_db.set_branch_name(conn, str(_task_value(task, "id")), branch_name)
+    return Path(value).resolve()
+
+def _record_board_access_error(board: str, error: Exception) -> None:
+    """Report one board access failure once while allowing other lanes to run."""
+    failure_kind = "storage" if isinstance(error, (OSError, sqlite3.Error)) else "access"
+    detail = f"{failure_kind} {type(error).__name__}: {error}"
+    if BOARD_CORRUPTION_ERRORS.get(board) == detail:
+        return
+    print(
+        f"temporarily skipping Kanban board {board!r}: {detail}",
+        file=sys.stderr,
+        flush=True,
+    )
+    BOARD_CORRUPTION_ERRORS[board] = detail
+
+def _board_call(
+    kanban_db: Any,
+    board: str,
+    operation: Callable[[Any], Any],
+) -> Any:
+    """Run one bounded Kanban operation on a fresh, promptly closed connection."""
+    last_storage_error: Exception | None = None
+    for attempt in range(KANBAN_STORAGE_ATTEMPTS):
+        conn = None
+        try:
+            with kanban_db.scoped_current_board(board):
+                conn = kanban_db.connect(board=board)
+                result = operation(conn)
+            BOARD_CORRUPTION_ERRORS.pop(board, None)
+            return result
+        except (OSError, sqlite3.Error) as error:
+            last_storage_error = error
+            _record_board_access_error(board, error)
+            if attempt + 1 < KANBAN_STORAGE_ATTEMPTS:
+                time.sleep(0.2 * (attempt + 1))
+        finally:
+            if conn is not None:
+                conn.close()
+    assert last_storage_error is not None
+    raise last_storage_error
+
+def _external(task: Any) -> bool:
+    return str(_task_value(task, "assignee", "") or "").startswith(EXTERNAL_PREFIX)

services/hermes/scripts/cli_lane_capabilities.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+"""Detect and publish the Kanban API boundary required by CLI lanes.
+
+During a rollout the coordinator container and the cli-lane-runner worker can
+run different hermes-agent image generations against one shared Kanban DB.
+The image APIs that differ on the decomposition/finalization path:
+
+* ``complete_task``: both generations accept ``expected_run_id`` (active
+  exact-run completion); only the patched image accepts
+  ``replay_ended_run_id`` (ended-run replay).
+* ``reclaim_task``: only the patched image accepts ``expected_run_id``
+  (guarded orphan reclaim after a runner restart).
+* ``specify_triage_task``/``decompose_triage_task``: only the patched image
+  accepts ``require_no_runs`` (refuses redefining tasks with run history).
+
+Worker-side handling per combination (proven in
+testing/tests/test_hermes_cli_capabilities.py):
+
+* legacy worker: health/readiness stay deferred, no new claims, no unguarded
+  reclaim, replay-needing results stay journaled for a patched successor.
+* patched worker: ready health, exact completion, guarded replay/reclaim; a
+  run voided by a legacy coordinator's unguarded mid-run decomposition
+  converges to stale/conflict evidence instead of completing.
+* patched coordinator: mid-run decomposition is refused entirely, so the
+  active exact run completes on either worker generation.
+"""
+
+from __future__ import annotations
+
+import inspect
+import json
+import os
+import sys
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Callable
+
+from cli_lane_config import STATE_ROOT, utc_now
+from cli_lane_files import atomic_json, load_json
+
+
+_CAPABILITIES: KanbanCapabilities | None = None
+_CAPABILITY_SOURCE: tuple[int, int] | None = None
+
+
+@dataclass(frozen=True)
+class KanbanCapabilities:
+    """Safety-relevant keyword support exposed by the image's Kanban DB."""
+
+    exact_run_completion: bool
+    ended_run_replay: bool
+    exact_run_reclaim: bool
+
+    @property
+    def ready(self) -> bool:
+        """Return whether dispatch and every recovery transition are safe."""
+        return (
+            self.exact_run_completion
+            and self.ended_run_replay
+            and self.exact_run_reclaim
+        )
+
+    @property
+    def deferred_features(self) -> tuple[str, ...]:
+        """List fixed, bounded capability names absent from the image API."""
+        missing = []
+        if not self.exact_run_completion:
+            missing.append("exact-run-completion")
+        if not self.ended_run_replay:
+            missing.append("ended-run-replay")
+        if not self.exact_run_reclaim:
+            missing.append("exact-run-reclaim")
+        return tuple(missing)
+
+
+def _explicit_keyword(function: Callable[..., Any], keyword: str) -> bool:
+    """Require an explicit keyword parameter rather than trusting ``**kwargs``."""
+    try:
+        parameter = inspect.signature(function).parameters.get(keyword)
+    except (TypeError, ValueError):
+        return False
+    return parameter is not None and parameter.kind in {
+        inspect.Parameter.POSITIONAL_OR_KEYWORD,
+        inspect.Parameter.KEYWORD_ONLY,
+    }
+
+
+def detect_kanban_capabilities(kanban_db: Any) -> KanbanCapabilities:
+    """Inspect the loaded image API without invoking a mutating DB operation."""
+    complete_task = getattr(kanban_db, "complete_task", None)
+    reclaim_task = getattr(kanban_db, "reclaim_task", None)
+    return KanbanCapabilities(
+        exact_run_completion=callable(complete_task)
+        and _explicit_keyword(complete_task, "expected_run_id"),
+        ended_run_replay=callable(complete_task)
+        and _explicit_keyword(complete_task, "replay_ended_run_id"),
+        exact_run_reclaim=callable(reclaim_task)
+        and _explicit_keyword(reclaim_task, "expected_run_id"),
+    )
+
+
+def _source_identity(kanban_db: Any) -> tuple[int, int]:
+    """Bind cached capability evidence to the two inspected callables."""
+    return (
+        id(getattr(kanban_db, "complete_task", None)),
+        id(getattr(kanban_db, "reclaim_task", None)),
+    )
+
+
+def _health_document(capabilities: KanbanCapabilities) -> dict[str, Any]:
+    """Return a bounded operator-readable compatibility health document."""
+    return {
+        "component": "cli-lane-runner",
+        "schema_version": 1,
+        "state": "ready" if capabilities.ready else "deferred",
+        "ready": capabilities.ready,
+        "active_run_completion_safe": capabilities.exact_run_completion,
+        "deferred_features": list(capabilities.deferred_features),
+        "capabilities": asdict(capabilities),
+        "observed_at": utc_now(),
+    }
+
+
+def initialize_kanban_capabilities(
+    kanban_db: Any,
+    *,
+    health_path: Path | None = None,
+) -> KanbanCapabilities:
+    """Detect startup compatibility and durably expose ready/deferred health."""
+    global _CAPABILITIES, _CAPABILITY_SOURCE
+    capabilities = detect_kanban_capabilities(kanban_db)
+    _CAPABILITIES = capabilities
+    _CAPABILITY_SOURCE = _source_identity(kanban_db)
+    destination = health_path or STATE_ROOT / "runtime-health.json"
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    atomic_json(destination, _health_document(capabilities), 0o600)
+    if not capabilities.ready:
+        print(
+            "CLI lane safety compatibility deferred; missing image APIs: "
+            + ", ".join(capabilities.deferred_features),
+            file=sys.stderr,
+            flush=True,
+        )
+    return capabilities
+
+
+def refresh_kanban_capabilities(
+    kanban_db: Any,
+    *,
+    health_path: Path | None = None,
+) -> KanbanCapabilities:
+    """Reinspect the live callables and refresh bounded readiness evidence.
+
+    ConfigMap script updates and image changes are deliberately not imported or
+    reloaded here.  A running process observes only the already-loaded module,
+    which prevents a half-old/half-new Python module graph.  Tests may replace
+    the two callables to model an image API transition at a loop boundary.
+    """
+    return initialize_kanban_capabilities(kanban_db, health_path=health_path)
+
+
+def kanban_capabilities(kanban_db: Any) -> KanbanCapabilities:
+    """Return startup evidence, redetecting only when the loaded API changes."""
+    if _CAPABILITIES is None or _source_identity(kanban_db) != _CAPABILITY_SOURCE:
+        return initialize_kanban_capabilities(kanban_db)
+    return _CAPABILITIES
+
+
+def runtime_health(path: Path | None = None) -> dict[str, Any]:
+    """Read the bounded health document for probes and operator diagnostics."""
+    document = load_json(path or STATE_ROOT / "runtime-health.json")
+    return document if isinstance(document, dict) else {}
+
+
+def readiness_issue(
+    path: Path | None = None,
+    *,
+    now: datetime | None = None,
+    maximum_age_seconds: float = 60.0,
+) -> str | None:
+    """Return a bounded readiness reason without exposing health contents."""
+    destination = path or STATE_ROOT / "runtime-health.json"
+    try:
+        document = json.loads(destination.read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        return "runtime health is missing"
+    except (OSError, UnicodeError, json.JSONDecodeError):
+        return "runtime health is malformed"
+    if not isinstance(document, dict) or document.get("schema_version") != 1:
+        return "runtime health is malformed"
+    if document.get("state") == "deferred" and document.get("ready") is False:
+        return "runtime compatibility is deferred"
+    capabilities = document.get("capabilities")
+    if (
+        document.get("state") != "ready"
+        or document.get("ready") is not True
+        or not isinstance(capabilities, dict)
+        or any(
+            capabilities.get(name) is not True
+            for name in (
+                "exact_run_completion",
+                "ended_run_replay",
+                "exact_run_reclaim",
+            )
+        )
+    ):
+        return "runtime health is malformed"
+    observed_at = document.get("observed_at")
+    if not isinstance(observed_at, str):
+        return "runtime health is malformed"
+    try:
+        observed = datetime.fromisoformat(observed_at.replace("Z", "+00:00"))
+    except ValueError:
+        return "runtime health is malformed"
+    if observed.tzinfo is None:
+        return "runtime health is malformed"
+    current = now or datetime.now(timezone.utc)
+    age = (
+        current.astimezone(timezone.utc) - observed.astimezone(timezone.utc)
+    ).total_seconds()
+    if age < -5.0 or age > maximum_age_seconds:
+        return "runtime health is stale"
+    return None
+
+
+def main() -> int:
+    """Provide the non-mutating Kubernetes readiness command."""
+    maximum_age = float(os.environ.get("HERMES_CLI_HEALTH_MAX_AGE_SECONDS", "60"))
+    issue = readiness_issue(maximum_age_seconds=maximum_age)
+    if issue:
+        print(issue, file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

services/hermes/scripts/cli_lane_config.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+"""Shared immutable configuration and result types for Hermes CLI lanes."""
+
+from __future__ import annotations
+
+import os
+import re
+import threading
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+import cli_lane_goal
+
+
+DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data"))
+SWITCHYARD_URL = os.environ.get(
+    "HERMES_SWITCHYARD_URL",
+    "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1/chat/completions",
+)
+STATE_ROOT = DATA_ROOT / "cli-lanes"
+CODEX_BIN = DATA_ROOT / "tools/bin/codex"
+CLAUDE_BIN = DATA_ROOT / "tools/bin/claude"
+CLAUDE_SETTINGS = DATA_ROOT / "home/.claude/settings.json"
+RESULT_SCHEMA_PATH = STATE_ROOT / "worker-result.schema.json"
+EFFORTS = ("low", "medium", "high", "xhigh")
+EXTERNAL_PREFIX = "cli-"
+DEFAULT_CLAIM_TTL = 7 * 24 * 60 * 60
+DEFAULT_MAX_RUNTIME = 12 * 60 * 60
+HEARTBEAT_SECONDS = 20
+PROVIDER_HEALTH_MAX_AGE_SECONDS = 5 * 60
+PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS = 12 * 60 * 60
+QUOTA_MIN_REMAINING_PERCENT_DEFAULT = 15.0
+CAPACITY_COOLDOWN_SECONDS_DEFAULT = 300.0
+AUTH_COOLDOWN_SECONDS_DEFAULT = 3600.0
+QUOTA_METRICS_URL = os.environ.get(
+    "HERMES_CLI_QUOTA_METRICS_URL", "http://127.0.0.1:9010/metrics"
+)
+KANBAN_STORAGE_ATTEMPTS = 5
+ARTIFACT_GC_INTERVAL_SECONDS = 5 * 60
+ARTIFACT_RETENTION_AGE_SECONDS = 30 * 24 * 60 * 60
+ARTIFACT_RETENTION_COUNT = 256
+ARTIFACT_RETENTION_BYTES = 128 * 1024 * 1024
+MAX_TERMINAL_RECORD_BYTES = 1024 * 1024
+QUARANTINE_HASH_BYTES = 64 * 1024
+SQLITE_RUN_ID_MAX = 2**63 - 1
+PROVIDER_HEALTH_PATHS = {
+    "codex": DATA_ROOT / "provider-health/codex.json",
+    "claude": DATA_ROOT / "provider-health/claude.json",
+}
+WORKTREE_LOCK = threading.Lock()
+BOARD_CORRUPTION_ERRORS: dict[str, str] = {}
+LAST_ARTIFACT_GC = 0.0
+CAPACITY_PATTERN = re.compile(
+    r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529"
+    r"|authentication|unauthorized|forbidden|oauth|token.*expired|401|403)",
+    re.I,
+)
+NO_CLAUDE_SESSION = "No conversation found with session ID:"
+CLAUDE_SESSION_COLLISION = "Session ID already in use"
+NO_CODEX_THREAD = "no rollout found for thread id"
+
+
+RESULT_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "additionalProperties": False,
+    "required": [
+        "status",
+        "summary",
+        "changed_files",
+        "tests_run",
+        "artifacts",
+        "findings",
+        "blockers",
+    ],
+    "properties": {
+        "status": {
+            "type": "string",
+            "enum": sorted(cli_lane_goal.RESULT_STATUSES),
+        },
+        "summary": {"type": "string"},
+        "changed_files": {"type": "array", "items": {"type": "string"}},
+        "tests_run": {"type": "array", "items": {"type": "string"}},
+        "artifacts": {"type": "array", "items": {"type": "string"}},
+        "findings": {
+            "type": "array",
+            "items": {"type": "string"},
+            "description": (
+                "Defects, risks, or observations discovered by a review or diagnosis. "
+                "Findings do not prevent the assigned review or diagnosis from completing."
+            ),
+        },
+        "blockers": {
+            "type": "array",
+            "items": {"type": "string"},
+            "description": (
+                "Concrete obstacles that prevent completion of the assigned task itself. "
+                "This must be empty when status is completed; review findings belong in findings."
+            ),
+        },
+    },
+}
+
+
+@dataclass(frozen=True)
+class Route:
+    """One provider/model/effort decision for an observable worker boundary."""
+
+    provider: str
+    model: str
+    effort: str
+    profile: str
+    classifier: str
+    reason: str
+    latency_ms: int
+    fallback_chain: tuple[str, ...]
+
+
+@dataclass
+class ProcessResult:
+    """Captured outcome from one provider CLI invocation."""
+
+    returncode: int
+    output: str
+    structured: dict[str, Any] | None
+    capacity_failure: bool
+
+
+class TerminalFinalizationPending(RuntimeError):
+    """An accepted worker result is durable but not committed to Kanban yet."""
+
+
+def canonical_run_id(value: object) -> int | None:
+    """Return one positive SQLite-safe run ID from an int or canonical decimal."""
+    if type(value) is int:
+        candidate = value
+    elif type(value) is str:
+        # Bound conversion before ``int`` so an attacker cannot hand Python an
+        # arbitrarily large decimal from an otherwise lexically valid name.
+        if not re.fullmatch(r"[1-9][0-9]{0,18}", value):
+            return None
+        candidate = int(value)
+    else:
+        return None
+    if not 1 <= candidate <= SQLITE_RUN_ID_MAX:
+        return None
+    return candidate
+
+
+@dataclass(frozen=True)
+class TerminalIdentity:
+    """Replay authority derived only from a journal's directory and filename."""
+
+    board: str
+    task_id: str
+    run_id: int
+    state: str
+
+    def __post_init__(self) -> None:
+        """Reject noncanonical identities before they can reach SQLite."""
+        if type(self.run_id) is not int or canonical_run_id(self.run_id) is None:
+            raise ValueError("terminal run identity is not a positive SQLite int64")
+
+
+@dataclass
+class TerminalSnapshot:
+    """One inode-bound, bounded journal read held open across finalization."""
+
+    document: dict[str, Any]
+    file_stat: os.stat_result
+    descriptor: int
+    directory_descriptor: int
+
+    def close(self) -> None:
+        """Release the pinned file and directory descriptors."""
+        os.close(self.descriptor)
+        os.close(self.directory_descriptor)
+
+
+@dataclass
+class TerminalRecoverySnapshot:
+    """One raw directory entry pinned before recovery classifies its payload."""
+
+    document: dict[str, Any] | None
+    file_stat: os.stat_result
+    descriptor: int | None
+    directory_descriptor: int
+    prefix: bytes
+    invalid_reason: str | None
+
+    def close(self) -> None:
+        """Release the descriptors retained across classification/quarantine."""
+        if self.descriptor is not None:
+            os.close(self.descriptor)
+            self.descriptor = None
+        if self.directory_descriptor >= 0:
+            os.close(self.directory_descriptor)
+            self.directory_descriptor = -1
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def kanban_setting(name: str, default: float) -> float:
+    """Read one numeric routing setting from the deployed kanban config block."""
+    try:
+        document = yaml.safe_load(
+            (DATA_ROOT / "config.yaml").read_text(encoding="utf-8")
+        )
+    except (OSError, yaml.YAMLError):
+        return default
+    kanban = document.get("kanban") if isinstance(document, dict) else None
+    value = kanban.get(name) if isinstance(kanban, dict) else None
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return default
+    return float(value)

services/hermes/scripts/cli_lane_dispatch.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""Concurrent board recovery, claim, and dispatch loop for CLI workers."""
+
+from __future__ import annotations
+
+import concurrent.futures
+import os
+import sys
+import time
+from collections.abc import Callable
+from typing import Any
+
+from cli_lane_board import _external, _record_board_access_error, _task_value
+from cli_lane_capabilities import (
+    initialize_kanban_capabilities,
+    kanban_capabilities,
+    refresh_kanban_capabilities,
+)
+from cli_lane_config import (
+    BOARD_CORRUPTION_ERRORS,
+    DEFAULT_CLAIM_TTL,
+    EXTERNAL_PREFIX,
+    RESULT_SCHEMA,
+    RESULT_SCHEMA_PATH,
+)
+from cli_lane_execution import execute_claim
+from cli_lane_files import atomic_json
+from cli_lane_metrics import start_metrics_server
+from cli_lane_recovery import _has_pending_finalization, recover_pending_finalizations
+from cli_lane_retention import maybe_gc_lane_artifacts
+
+
+def _board_slug(board: Any) -> str:
+    if isinstance(board, dict):
+        return str(board.get("slug") or board.get("id") or "")
+    return str(getattr(board, "slug", None) or getattr(board, "id", None) or board)
+
+
+def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None:
+    """Open one board without letting localized storage faults stop other lanes."""
+    try:
+        return kanban_db.connect(board=board)
+    except Exception as error:
+        _record_board_access_error(board, error)
+        return None
+
+
+def recover_orphans() -> None:
+    """Return external running tasks to ready after a runner/pod restart."""
+    from hermes_cli import kanban_db
+
+    recover_pending_finalizations()
+    if not kanban_capabilities(kanban_db).exact_run_reclaim:
+        return
+    try:
+        boards = kanban_db.list_boards(include_archived=False)
+    except Exception as error:
+        _record_board_access_error("board-registry", error)
+        return
+    for raw_board in boards:
+        board = _board_slug(raw_board)
+        if not board:
+            continue
+        with kanban_db.scoped_current_board(board):
+            conn = _connect_healthy_board(kanban_db, board)
+            if conn is None:
+                continue
+            try:
+                for task in kanban_db.list_tasks(conn):
+                    if (
+                        _external(task)
+                        and str(_task_value(task, "status", "")) == "running"
+                    ):
+                        task_id = str(_task_value(task, "id"))
+                        run_id = _task_value(task, "current_run_id", None)
+                        if not isinstance(run_id, int):
+                            continue
+                        if _has_pending_finalization(board, task_id, run_id):
+                            continue
+                        kanban_db.reclaim_task(
+                            conn,
+                            task_id,
+                            reason="direct CLI lane restarted; provider session will resume",
+                            expected_run_id=run_id,
+                        )
+                BOARD_CORRUPTION_ERRORS.pop(board, None)
+            except Exception as error:
+                _record_board_access_error(board, error)
+            finally:
+                conn.close()
+
+
+def claim_ready(
+    active: set[tuple[str, str]],
+    limit: int,
+    eligible: Callable[[str, Any], bool] | None = None,
+) -> list[tuple[str, str]]:
+    """Atomically claim external ready tasks across all non-archived boards."""
+    from hermes_cli import kanban_db
+
+    claimed: list[tuple[str, str]] = []
+    if limit <= 0:
+        return claimed
+    for raw_board in kanban_db.list_boards(include_archived=False):
+        board = _board_slug(raw_board)
+        if not board:
+            continue
+        with kanban_db.scoped_current_board(board):
+            conn = _connect_healthy_board(kanban_db, board)
+            if conn is None:
+                continue
+            try:
+                kanban_db.recompute_ready(conn)
+                tasks = kanban_db.list_tasks(conn)
+                BOARD_CORRUPTION_ERRORS.pop(board, None)
+                for task in tasks:
+                    task_id = str(_task_value(task, "id", ""))
+                    assignee = str(_task_value(task, "assignee", "") or "")
+                    if (
+                        task_id
+                        and not assignee
+                        and str(_task_value(task, "status", "")) == "ready"
+                        and kanban_db.assign_task(conn, task_id, "cli-auto")
+                    ):
+                        task = kanban_db.get_task(conn, task_id)
+                        assignee = "cli-auto"
+                    if (
+                        not task_id
+                        or (board, task_id) in active
+                        or not assignee.startswith(EXTERNAL_PREFIX)
+                        or str(_task_value(task, "status", "")) != "ready"
+                        or (eligible is not None and not eligible(board, task))
+                    ):
+                        continue
+                    try:
+                        result = kanban_db.claim_task(
+                            conn,
+                            task_id,
+                            ttl_seconds=DEFAULT_CLAIM_TTL,
+                            claimer="direct-cli-lane",
+                        )
+                    except Exception:
+                        continue
+                    if result is not None:
+                        claimed.append((board, task_id))
+                    if len(claimed) >= limit:
+                        return claimed
+            except Exception as error:
+                _record_board_access_error(board, error)
+            finally:
+                conn.close()
+    return claimed
+
+
+def main() -> int:
+    """Continuously bridge external Kanban lanes to provider CLIs.
+
+    The loop is exited only by process signals or unrecoverable exceptions;
+    every per-board and per-claim fault is bounded inside one pass.
+    """
+    from hermes_cli import kanban_db
+
+    RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
+    atomic_json(RESULT_SCHEMA_PATH, RESULT_SCHEMA, 0o644)
+    start_metrics_server()
+    capabilities = initialize_kanban_capabilities(kanban_db)
+    recover_orphans()
+    workers = max(1, min(int(os.environ.get("HERMES_CLI_LANE_CONCURRENCY", "4")), 8))
+    futures: dict[concurrent.futures.Future[None], tuple[str, str]] = {}
+    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
+        while True:
+            previous_capabilities = capabilities
+            capabilities = refresh_kanban_capabilities(kanban_db)
+            if capabilities.ready and not previous_capabilities.ready:
+                recover_orphans()
+            for future in list(futures):
+                if future.done():
+                    try:
+                        future.result()
+                    except Exception as error:
+                        print(
+                            f"worker future failed: {error}",
+                            file=sys.stderr,
+                            flush=True,
+                        )
+                    del futures[future]
+            recover_pending_finalizations()
+            maybe_gc_lane_artifacts()
+            active = set(futures.values())
+            try:
+                newly_claimed = (
+                    claim_ready(active, workers - len(futures))
+                    if capabilities.ready
+                    else []
+                )
+                BOARD_CORRUPTION_ERRORS.pop("board-registry", None)
+            except Exception as error:
+                _record_board_access_error("board-registry", error)
+                newly_claimed = []
+            for board, task_id in newly_claimed:
+                future = pool.submit(execute_claim, board, task_id)
+                futures[future] = (board, task_id)
+            time.sleep(5)

services/hermes/scripts/cli_lane_evidence.py

@@ -0,0 +1,294 @@
+#!/usr/bin/env python3
+"""Immutable terminal evidence creation and identity-safe retirement."""
+
+from __future__ import annotations
+
+import ctypes
+import contextlib
+import errno
+import hashlib
+import json
+import os
+import stat
+import uuid
+from pathlib import Path
+from typing import Any
+
+from cli_lane_config import MAX_TERMINAL_RECORD_BYTES, TerminalIdentity, utc_now
+from cli_lane_files import _terminal_evidence_identity, state_path
+from cli_lane_records import _load_small_json, _terminal_record_valid
+
+
+def _rename_noreplace(
+    source: str,
+    destination: str,
+    *,
+    source_dir: int,
+    destination_dir: int,
+) -> None:
+    """Atomically rename a directory entry without replacing a collision."""
+    renameat2 = getattr(ctypes.CDLL(None, use_errno=True), "renameat2", None)
+    if renameat2 is None:
+        raise OSError(errno.ENOSYS, "renameat2 is unavailable")
+    renameat2.argtypes = (
+        ctypes.c_int,
+        ctypes.c_char_p,
+        ctypes.c_int,
+        ctypes.c_char_p,
+        ctypes.c_uint,
+    )
+    renameat2.restype = ctypes.c_int
+    result = renameat2(
+        source_dir,
+        ctypes.c_char_p(os.fsencode(source)),
+        destination_dir,
+        ctypes.c_char_p(os.fsencode(destination)),
+        1,  # RENAME_NOREPLACE
+    )
+    if result != 0:
+        error = ctypes.get_errno()
+        raise OSError(error, os.strerror(error), source, destination)
+
+def _retire_terminal_entry(
+    path: Path,
+    source_stat: os.stat_result,
+    *,
+    board_descriptor: int,
+    quarantine_descriptor: int,
+    authority_name: str | None = None,
+    source_descriptor: int | None = None,
+) -> str:
+    """Retire only the inode previously inspected; preserve any replacement."""
+    pinned = os.fstat(source_descriptor) if source_descriptor is not None else source_stat
+    if pinned.st_dev != source_stat.st_dev or pinned.st_ino != source_stat.st_ino:
+        return "replacement"
+    try:
+        current = os.stat(
+            path.name,
+            dir_fd=board_descriptor,
+            follow_symlinks=False,
+        )
+    except FileNotFoundError:
+        return "missing"
+    if current.st_dev != pinned.st_dev or current.st_ino != pinned.st_ino:
+        return "replacement"
+    # A replacement can itself be staged repeatedly while recovery races a
+    # writer. Keep every generation bound to the original canonical pending
+    # name instead of hashing an intermediate .retire name and hiding it from
+    # the next recovery pass.
+    path_digest = hashlib.sha256(
+        (authority_name or path.name).encode("utf-8")
+    ).hexdigest()[:16]
+    source_digest = hashlib.sha256(
+        f"{pinned.st_dev:x}\0{pinned.st_ino:x}".encode()
+    ).hexdigest()[:16]
+    for sequence in range(32):
+        staging = f".retire.{path_digest}.{source_digest}.{sequence}"
+        try:
+            _rename_noreplace(
+                path.name,
+                staging,
+                source_dir=board_descriptor,
+                destination_dir=quarantine_descriptor,
+            )
+        except FileExistsError:
+            continue
+        break
+    else:
+        return "collision"
+    staged = os.stat(
+        staging,
+        dir_fd=quarantine_descriptor,
+        follow_symlinks=False,
+    )
+    if staged.st_dev == pinned.st_dev and staged.st_ino == pinned.st_ino:
+        os.unlink(staging, dir_fd=quarantine_descriptor)
+        os.fsync(quarantine_descriptor)
+        os.fsync(board_descriptor)
+        return "retired"
+    # The path changed after the pre-check but before the rename. Put the
+    # replacement back without overwriting an even newer entry.
+    try:
+        _rename_noreplace(
+            staging,
+            path.name,
+            source_dir=quarantine_descriptor,
+            destination_dir=board_descriptor,
+        )
+    except FileExistsError:
+        return "replacement-staged"
+    os.fsync(quarantine_descriptor)
+    os.fsync(board_descriptor)
+    return "replacement"
+
+def _write_json_noreplace(path: Path, value: dict[str, Any]) -> bool:
+    """Durably create one immutable JSON artifact without replacing a peer."""
+    payload = (json.dumps(value, indent=2, sort_keys=True) + "\n").encode("utf-8")
+    if len(payload) > MAX_TERMINAL_RECORD_BYTES + 4096:
+        raise ValueError("terminal evidence exceeds the bounded artifact limit")
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    os.chmod(path.parent, 0o700, follow_symlinks=False)
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
+    create_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    create_flags |= getattr(os, "O_NOFOLLOW", 0)
+    directory = os.open(path.parent, directory_flags)
+    temporary = f".{path.name}.{uuid.uuid4().hex}.tmp"
+    temporary_stat = None
+    descriptor = None
+    cleanup_descriptor = None
+    try:
+        descriptor = os.open(temporary, create_flags, 0o600, dir_fd=directory)
+        temporary_stat = os.fstat(descriptor)
+        with os.fdopen(descriptor, "wb") as stream:
+            descriptor = None
+            stream.write(payload)
+            stream.flush()
+            os.fsync(stream.fileno())
+            written = os.fstat(stream.fileno())
+            if stat.S_IMODE(written.st_mode) != 0o600 or written.st_nlink != 1:
+                raise OSError("terminal evidence is not private and singly linked")
+            cleanup_descriptor = os.dup(stream.fileno())
+        try:
+            _rename_noreplace(
+                temporary,
+                path.name,
+                source_dir=directory,
+                destination_dir=directory,
+            )
+        except FileExistsError:
+            return False
+        os.fsync(directory)
+        return True
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        if temporary_stat is not None:
+            with contextlib.suppress(OSError):
+                _retire_terminal_entry(
+                    Path(temporary),
+                    temporary_stat,
+                    board_descriptor=directory,
+                    quarantine_descriptor=directory,
+                    source_descriptor=cleanup_descriptor,
+                )
+        if cleanup_descriptor is not None:
+            os.close(cleanup_descriptor)
+        os.close(directory)
+
+def _terminal_document_digest(document: dict[str, Any]) -> str:
+    """Return the stable identity for one exact accepted result document."""
+    immutable = {
+        key: document.get(key)
+        for key in (
+            "board",
+            "task_id",
+            "expected_run_id",
+            "result",
+            "summary",
+            "metadata",
+        )
+    }
+    canonical = json.dumps(immutable, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:32]
+
+def _terminal_evidence_path(
+    identity: TerminalIdentity,
+    state: str,
+    document: dict[str, Any],
+) -> Path:
+    """Return a deterministic immutable prepared or conflict evidence path."""
+    if state not in {"prepared", "conflict"}:
+        raise ValueError(f"invalid terminal evidence state: {state}")
+    base = state_path(identity.board, identity.task_id)
+    digest = _terminal_document_digest(document)
+    return base.with_name(
+        f"{base.stem}.run-{identity.run_id}.terminal.{state}-{digest}.json"
+    )
+
+def _same_terminal_document(left: dict[str, Any], right: dict[str, Any]) -> bool:
+    """Compare the immutable result-bearing fields of terminal evidence."""
+    keys = ("board", "task_id", "expected_run_id", "result", "summary", "metadata")
+    return all(left.get(key) == right.get(key) for key in keys)
+
+def _terminal_evidence_valid(
+    document: dict[str, Any],
+    identity: TerminalIdentity,
+    state: str,
+) -> bool:
+    """Validate semantic terminal content plus its immutable evidence state."""
+    if document.get("kanban_state") != state:
+        return False
+    semantic = dict(document)
+    semantic_identity = identity
+    if state == "conflict":
+        semantic["kanban_state"] = "prepared"
+        semantic_identity = TerminalIdentity(
+            identity.board,
+            identity.task_id,
+            identity.run_id,
+            "prepared",
+        )
+    return _terminal_record_valid(semantic, semantic_identity)
+
+def _persist_prepared_evidence(
+    identity: TerminalIdentity,
+    document: dict[str, Any],
+) -> Path:
+    """Fsync an immutable accepted result before entering the DB boundary."""
+    prepared = dict(document)
+    prepared["kanban_state"] = "prepared"
+    prepared["prepared_at"] = utc_now()
+    path = _terminal_evidence_path(identity, "prepared", document)
+    if _write_json_noreplace(path, prepared):
+        return path
+    if _terminal_evidence_identity(path) != TerminalIdentity(
+        identity.board, identity.task_id, identity.run_id, "prepared"
+    ):
+        raise OSError("prepared terminal evidence identity is invalid")
+    existing = _load_small_json(path)
+    prepared_identity = TerminalIdentity(
+        identity.board,
+        identity.task_id,
+        identity.run_id,
+        "prepared",
+    )
+    if not _terminal_evidence_valid(
+        existing,
+        prepared_identity,
+        "prepared",
+    ) or not _same_terminal_document(existing, prepared):
+        raise OSError("prepared terminal evidence path contains a conflicting result")
+    return path
+
+def _persist_conflict_evidence(
+    identity: TerminalIdentity,
+    document: dict[str, Any],
+    reason: str,
+) -> Path:
+    """Preserve a losing valid result in full under deterministic retention."""
+    conflict = dict(document)
+    conflict["kanban_state"] = "conflict"
+    conflict["conflict_reason"] = reason
+    conflict["conflicted_at"] = utc_now()
+    path = _terminal_evidence_path(identity, "conflict", document)
+    if _write_json_noreplace(path, conflict):
+        return path
+    if _terminal_evidence_identity(path) != TerminalIdentity(
+        identity.board, identity.task_id, identity.run_id, "conflict"
+    ):
+        raise OSError("terminal conflict evidence identity is invalid")
+    existing = _load_small_json(path)
+    conflict_identity = TerminalIdentity(
+        identity.board,
+        identity.task_id,
+        identity.run_id,
+        "conflict",
+    )
+    if not _terminal_evidence_valid(
+        existing,
+        conflict_identity,
+        "conflict",
+    ) or not _same_terminal_document(existing, conflict):
+        raise OSError("terminal conflict evidence path contains a different result")
+    return path

services/hermes/scripts/cli_lane_execution.py

@@ -0,0 +1,462 @@
+#!/usr/bin/env python3
+"""Single-claim provider execution and durable Kanban result handoff."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import sqlite3
+import time
+from pathlib import Path
+
+import cli_lane_goal
+from cli_lane_board import _board_call, _resolve_workspace, _task_context, _task_value
+from cli_lane_config import (
+    DEFAULT_MAX_RUNTIME,
+    TerminalFinalizationPending,
+)
+from cli_lane_failover import _routed_or_blocked, capacity_failover
+from cli_lane_files import (
+    _terminal_identity,
+    _terminal_path,
+    atomic_json,
+    load_json,
+    state_path,
+)
+from cli_lane_finalization import _finalize_terminal_record, _recover_exact_run
+from cli_lane_health import record_provider_success
+from cli_lane_metrics import record_route_decision
+from cli_lane_prompt import build_prompt, git_handoff, workspace_artifacts
+from cli_lane_provider import run_provider
+from cli_lane_quota import selection_constraint
+from cli_lane_records import _persist_candidate, _write_terminal_record
+from cli_lane_recovery import _has_pending_finalization
+from cli_lane_routing import fresh_unavailable_provider, select_route
+
+
+def execute_claim(board: str, task_id: str) -> None:
+    """Execute one already-claimed task and commit its outcome to Kanban."""
+    from hermes_cli import kanban_db
+
+    run_id = None
+    with kanban_db.scoped_current_board(board):
+        conn = kanban_db.connect(board=board)
+        try:
+            task = kanban_db.get_task(conn, task_id)
+            if task is None:
+                return
+            run_id = _task_value(task, "current_run_id", None)
+            assignee = str(_task_value(task, "assignee", "cli-auto") or "cli-auto")
+            state_file = state_path(board, task_id)
+            state = load_json(state_file)
+            state.update(
+                {
+                    "board": board,
+                    "task_id": task_id,
+                    "run_id": run_id,
+                    "assignee": assignee,
+                }
+            )
+            atomic_json(state_file, state)
+            log_path = Path(kanban_db.worker_log_path(task_id, board=board))
+            workspace = _resolve_workspace(kanban_db, conn, task, board)
+            kanban_db.set_workspace_path(conn, task_id, str(workspace))
+            context = _task_context(kanban_db, conn, task_id)
+        except Exception as error:
+            failure_reason = (
+                f"Direct CLI lane preparation failed: {type(error).__name__}: {error}"
+            )
+            conn.close()
+            _board_call(
+                kanban_db,
+                board,
+                lambda fresh: kanban_db.block_task(
+                    fresh,
+                    task_id,
+                    reason=failure_reason,
+                    kind="capability",
+                    expected_run_id=run_id,
+                ),
+            )
+            return
+        finally:
+            conn.close()
+
+        def heartbeat(note: str) -> bool:
+            try:
+                return bool(
+                    _board_call(
+                        kanban_db,
+                        board,
+                        lambda fresh: kanban_db.heartbeat_worker(
+                            fresh,
+                            task_id,
+                            note=note,
+                            expected_run_id=run_id,
+                        ),
+                    )
+                )
+            except (OSError, sqlite3.Error):
+                # The claim TTL is deliberately long. Keep the expensive
+                # provider process alive during a transient volume stall and
+                # retry on the next heartbeat instead of losing its work.
+                return True
+
+        def comment(body: str) -> None:
+            with contextlib.suppress(OSError, sqlite3.Error):
+                _board_call(
+                    kanban_db,
+                    board,
+                    lambda fresh: kanban_db.add_comment(
+                        fresh,
+                        task_id,
+                        "cli-lane-runner",
+                        body,
+                    ),
+                )
+            # Route state is also written to the durable lane-state file;
+            # a later heartbeat or terminal result remains authoritative.
+
+        try:
+            previous_route = state.get("current_route")
+            constraint = selection_constraint(assignee)
+            unavailable_provider = (
+                constraint.exclude_provider if constraint.source == "health" else None
+            )
+            route = _routed_or_blocked(
+                kanban_db,
+                board,
+                task_id,
+                run_id,
+                lambda: select_route(
+                    context,
+                    assignee,
+                    exclude_provider=constraint.exclude_provider,
+                    exclude_reason=constraint.exclude_reason,
+                ),
+            )
+            if route is None:
+                return
+            for note in constraint.notes:
+                comment(note)
+            if constraint.exclude_provider:
+                comment(
+                    f"Provider routing guard ({constraint.source}) excluded "
+                    f"{constraint.exclude_provider} before automatic routing.",
+                )
+            comment(
+                f"CLI route: {route.provider}/{route.model} at {route.effort}; classifier={route.classifier}; {route.reason}",
+            )
+            resume_handoff = ""
+            if (
+                isinstance(previous_route, dict)
+                and previous_route.get("provider")
+                and previous_route.get("provider") != route.provider
+            ):
+                try:
+                    previous_output = log_path.read_text(encoding="utf-8")[-10000:]
+                except OSError:
+                    previous_output = "Previous provider log was unavailable after restart."
+                resume_handoff = git_handoff(workspace, previous_output)
+                comment(
+                    f"Restart-time provider change: {previous_route.get('provider')} -> {route.provider}; explicit workspace handoff attached.",
+                )
+            prompt = build_prompt(context, workspace, resume_handoff)
+            max_runtime = int(
+                _task_value(task, "max_runtime_seconds", 0) or DEFAULT_MAX_RUNTIME
+            )
+            goal_mode = bool(_task_value(task, "goal_mode", False))
+            goal_max_turns = max(1, int(_task_value(task, "goal_max_turns", 20) or 20))
+            goal_turn = max(1, int(state.get("goal_turn", 0) or 0) + 1)
+            deadline = time.monotonic() + max_runtime
+            while True:
+                state["goal_turn"] = goal_turn
+                atomic_json(state_file, state)
+                remaining = max(1, int(deadline - time.monotonic()))
+                result = run_provider(
+                    route,
+                    prompt,
+                    workspace,
+                    state,
+                    state_file,
+                    log_path,
+                    heartbeat,
+                    remaining,
+                )
+                candidate_file = None
+                if result.structured:
+                    candidate_file = _persist_candidate(
+                        state,
+                        state_file,
+                        dict(result.structured),
+                        route=route,
+                        returncode=result.returncode,
+                        goal_turn=goal_turn,
+                    )
+                if result.capacity_failure:
+                    boundary = capacity_failover(
+                        kanban_db,
+                        board=board,
+                        task_id=task_id,
+                        run_id=run_id,
+                        assignee=assignee,
+                        comment=comment,
+                        context=context,
+                        workspace=workspace,
+                        state=state,
+                        state_file=state_file,
+                        log_path=log_path,
+                        heartbeat=heartbeat,
+                        deadline=deadline,
+                        route=route,
+                        result=result,
+                        candidate_file=candidate_file,
+                        goal_turn=goal_turn,
+                    )
+                    route = boundary.route
+                    result = boundary.result
+                    candidate_file = boundary.candidate_file
+                    if boundary.unavailable_provider is not None:
+                        unavailable_provider = boundary.unavailable_provider
+                    if boundary.blocked:
+                        break
+                if result.returncode == 0:
+                    record_provider_success(route.provider)
+                structured = result.structured
+                if structured:
+                    structured = dict(structured)
+                    structured["artifacts"] = workspace_artifacts(
+                        workspace,
+                        structured.get("artifacts"),
+                    )
+                metadata = {
+                    "executor": "direct-cli-lane",
+                    "provider": route.provider,
+                    "model": route.model,
+                    "effort": route.effort,
+                    "classifier": route.classifier,
+                    "state_file": str(state_file),
+                    "codex_thread_id": state.get("codex_thread_id"),
+                    "claude_session_id": state.get("claude_session_id"),
+                    "goal_mode": goal_mode,
+                    "goal_turn": goal_turn,
+                }
+                if candidate_file is not None:
+                    metadata["candidate_file"] = str(candidate_file)
+                if structured:
+                    for key in (
+                        "changed_files",
+                        "tests_run",
+                        "artifacts",
+                        "findings",
+                        "blockers",
+                    ):
+                        value = structured.get(key)
+                        metadata[key] = value if isinstance(value, list) else []
+
+                completion_problem = None
+                if structured and result.returncode == 0:
+                    # Resolve the role from the card itself, before any
+                    # controller evidence is appended, and hand it to the gate
+                    # so a review is judged on its verdict rather than on prose
+                    # about the artifact it reviewed.
+                    task_role, role_source = cli_lane_goal.task_role(
+                        context,
+                        structured,
+                    )
+                    metadata["task_role"] = task_role
+                    metadata["task_role_source"] = role_source
+                    completion_problem = cli_lane_goal.unfinished_result_reason(
+                        structured,
+                        role=task_role,
+                    )
+                    if (
+                        structured.get("status") == "completed"
+                        and completion_problem is None
+                        and goal_mode
+                    ):
+                        heartbeat("local goal-completion judge active")
+                        rejection_history = state.get("goal_rejections", [])
+                        if not isinstance(rejection_history, list):
+                            rejection_history = []
+                        judge_context = context
+                        if goal_turn > 1 or rejection_history:
+                            judge_context += (
+                                f"\n\n{cli_lane_goal.CONTROLLER_EVIDENCE_HEADING}\n"
+                                f"current turn {goal_turn}/{goal_max_turns}; prior rejected "
+                                f"reports: {json.dumps(rejection_history[-5:])}."
+                            )
+                        accepted, judge_reason = cli_lane_goal.judge_goal_completion(
+                            judge_context,
+                            structured,
+                        )
+                        metadata["goal_judge_reason"] = judge_reason
+                        if not accepted:
+                            completion_problem = f"local goal judge requested continuation: {judge_reason}"
+
+                if (
+                    structured
+                    and structured.get("status") == "completed"
+                    and result.returncode == 0
+                    and completion_problem is None
+                ):
+                    terminal_file = _terminal_path(state_file, run_id, "pending")
+                    metadata["terminal_record"] = str(
+                        _terminal_path(state_file, run_id, "committed")
+                    )
+                    try:
+                        terminal_file, terminal_record = _write_terminal_record(
+                            state_file,
+                            board=board,
+                            task_id=task_id,
+                            run_id=run_id,
+                            structured=structured,
+                            summary=str(structured.get("summary") or "Completed"),
+                            metadata=metadata,
+                        )
+                        outcome = _finalize_terminal_record(
+                            kanban_db,
+                            terminal_file,
+                            terminal_record,
+                        )
+                    except Exception as error:
+                        identity = _terminal_identity(terminal_file)
+                        replayable = bool(
+                            identity is not None
+                            and _has_pending_finalization(
+                                identity.board,
+                                identity.task_id,
+                                identity.run_id,
+                            )
+                        )
+                        if not replayable:
+                            _recover_exact_run(
+                                kanban_db,
+                                identity,
+                                f"persistence raised {type(error).__name__}",
+                            )
+                        raise TerminalFinalizationPending(
+                            "accepted worker result remains outside the generic block path; "
+                            f"terminal replayable={replayable}; persistence/finalization "
+                            f"raised {type(error).__name__}"
+                        ) from error
+                    if outcome != "committed":
+                        raise TerminalFinalizationPending(
+                            "accepted worker result is durably journaled but "
+                            f"Kanban finalization is {outcome}"
+                        )
+                    record_route_decision(
+                        route.provider, route.effort, route.classifier, "completed"
+                    )
+                    break
+
+                can_continue = (
+                    goal_mode
+                    and completion_problem is not None
+                    and goal_turn < goal_max_turns
+                    and deadline - time.monotonic() > 30
+                )
+                if can_continue:
+                    rejection_history = state.get("goal_rejections", [])
+                    if not isinstance(rejection_history, list):
+                        rejection_history = []
+                    state["goal_rejections"] = [
+                        *rejection_history[-4:],
+                        completion_problem,
+                    ]
+                    goal_turn += 1
+                    comment(
+                        f"Goal completion rejected; continuing turn {goal_turn}/{goal_max_turns}: {completion_problem}",
+                    )
+                    escalation_context = (
+                        context
+                        + "\n\nThe previous worker turn missed its completion quality mark: "
+                        + completion_problem
+                        + "\nSelect a route that can finish and verify the remaining work."
+                    )
+                    excluded = unavailable_provider
+                    if excluded is None and assignee == "cli-auto":
+                        excluded = fresh_unavailable_provider()
+                    record_route_decision(
+                        route.provider, route.effort, route.classifier, "goal-continued"
+                    )
+                    next_route = _routed_or_blocked(
+                        kanban_db,
+                        board,
+                        task_id,
+                        run_id,
+                        lambda boundary=escalation_context, blocked=excluded: select_route(
+                            boundary,
+                            assignee,
+                            exclude_provider=blocked,
+                            exclude_reason="is unavailable according to fresh native health"
+                            if blocked
+                            else None,
+                        ),
+                    )
+                    if next_route is None:
+                        break
+                    comment(
+                        f"Goal route {goal_turn}/{goal_max_turns}: "
+                        f"{next_route.provider}/{next_route.model} at {next_route.effort}; "
+                        f"classifier={next_route.classifier}; {next_route.reason}",
+                    )
+                    handoff = (
+                        git_handoff(workspace, result.output)
+                        if next_route.provider != route.provider
+                        else ""
+                    )
+                    route = next_route
+                    prompt = build_prompt(
+                        context,
+                        workspace,
+                        handoff
+                        + "\nGoal-loop continuation: the previous final report was rejected because "
+                        + completion_problem
+                        + ". Reinspect live state, finish the outstanding work, and return new final evidence.",
+                    )
+                    continue
+
+                reason = completion_problem
+                if reason is None and structured:
+                    blockers = structured.get("blockers", [])
+                    reason = "; ".join(str(item) for item in blockers)
+                    reason = reason or str(structured.get("summary") or "")
+                if reason is None:
+                    reason = result.output[-4000:]
+                failure_reason = reason or (
+                    f"{route.provider} worker failed with exit {result.returncode}"
+                )
+                failure_kind = (
+                    "transient" if result.capacity_failure else "capability"
+                )
+                record_route_decision(
+                    route.provider, route.effort, route.classifier, f"blocked-{failure_kind}"
+                )
+                _board_call(
+                    kanban_db,
+                    board,
+                    lambda fresh, failure_reason=failure_reason, failure_kind=failure_kind: kanban_db.block_task(
+                        fresh,
+                        task_id,
+                        reason=failure_reason,
+                        kind=failure_kind,
+                        expected_run_id=run_id,
+                    ),
+                )
+                break
+        except TerminalFinalizationPending as error:
+            comment(str(error))
+        except Exception as error:
+            failure_reason = f"Direct CLI lane failed: {type(error).__name__}: {error}"
+            _board_call(
+                kanban_db,
+                board,
+                lambda fresh: kanban_db.block_task(
+                    fresh,
+                    task_id,
+                    reason=failure_reason,
+                    kind="capability",
+                    expected_run_id=run_id,
+                ),
+            )

services/hermes/scripts/cli_lane_failover.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""Bounded, fail-closed cross-provider failover for one capacity boundary."""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable
+
+from cli_lane_board import _board_call
+from cli_lane_config import ProcessResult, Route
+from cli_lane_health import classify_capacity_failure, record_provider_failure
+from cli_lane_metrics import (
+    record_provider_fallback,
+    record_route_decision,
+    record_router_selection_failure,
+)
+from cli_lane_prompt import build_prompt, git_handoff
+from cli_lane_provider import run_provider
+from cli_lane_records import _persist_candidate
+from cli_lane_routing import select_route
+
+
+@dataclass
+class FailoverOutcome:
+    """Where one capacity boundary left the claim's route and result."""
+
+    route: Route
+    result: ProcessResult
+    candidate_file: Path | None
+    unavailable_provider: str | None
+    blocked: bool
+
+
+def _block_transient(
+    kanban_db: Any, board: str, task_id: str, run_id: Any, reason: str
+) -> None:
+    """Park one card as retryable so it re-enters when providers recover."""
+    _board_call(
+        kanban_db,
+        board,
+        lambda fresh: kanban_db.block_task(
+            fresh,
+            task_id,
+            reason=reason,
+            kind="transient",
+            expected_run_id=run_id,
+        ),
+    )
+
+
+def _routed_or_blocked(
+    kanban_db: Any,
+    board: str,
+    task_id: str,
+    run_id: Any,
+    select: Callable[[], Route],
+) -> Route | None:
+    """Turn a Switchyard outage into a transient block, never a capability one."""
+    try:
+        return select()
+    except RuntimeError as error:
+        record_router_selection_failure()
+        _block_transient(
+            kanban_db,
+            board,
+            task_id,
+            run_id,
+            "Switchyard route selection is unavailable; the card retries "
+            f"when the router recovers: {error}",
+        )
+        return None
+
+
+def capacity_failover(
+    kanban_db: Any,
+    *,
+    board: str,
+    task_id: str,
+    run_id: Any,
+    assignee: str,
+    comment: Callable[[str], None],
+    context: str,
+    workspace: Path,
+    state: dict[str, Any],
+    state_file: Path,
+    log_path: Path,
+    heartbeat: Callable[[str], bool],
+    deadline: float,
+    route: Route,
+    result: ProcessResult,
+    candidate_file: Path | None,
+    goal_turn: int,
+) -> FailoverOutcome:
+    """Apply the capacity policy once: record, fail closed, or fail over."""
+    failure_class = classify_capacity_failure(result.output)
+    record_provider_failure(route.provider, failure_class)
+    record_route_decision(
+        route.provider, route.effort, route.classifier, "capacity-failure"
+    )
+    if assignee != "cli-auto":
+        # Fail closed: an operator pinned this provider, so a capacity or
+        # auth failure must wait for that provider, never switch away.
+        _block_transient(
+            kanban_db,
+            board,
+            task_id,
+            run_id,
+            f"Manually pinned provider {route.provider} hit a "
+            f"{failure_class} failure; the explicit {assignee} lane never "
+            "switches providers. Retry after recovery or reassign to cli-auto.",
+        )
+        return FailoverOutcome(route, result, candidate_file, None, True)
+    retry_context = (
+        context
+        + "\n\nRouting boundary: the first provider failed from capacity/authentication. "
+        + "Select the alternate hosted provider at an appropriate effort."
+    )
+    alternate = "claude" if route.provider == "codex" else "codex"
+    fallback = _routed_or_blocked(
+        kanban_db,
+        board,
+        task_id,
+        run_id,
+        lambda: select_route(retry_context, f"cli-{alternate}-{route.effort}"),
+    )
+    if fallback is None:
+        return FailoverOutcome(route, result, candidate_file, route.provider, True)
+    record_provider_fallback(route.provider, fallback.provider, failure_class)
+    comment(
+        f"Provider fallback: {route.provider} -> {fallback.provider} "
+        f"after a {failure_class} failure; Jetson reclassified the retry boundary.",
+    )
+    fallback_result = run_provider(
+        fallback,
+        build_prompt(
+            context,
+            workspace,
+            git_handoff(workspace, result.output),
+        ),
+        workspace,
+        state,
+        state_file,
+        log_path,
+        heartbeat,
+        max(1, int(deadline - time.monotonic())),
+    )
+    if fallback_result.capacity_failure:
+        # Bounded failover depth: two hosted providers, never a third
+        # metered or local path.
+        fallback_class = classify_capacity_failure(fallback_result.output)
+        record_provider_failure(fallback.provider, fallback_class)
+        record_route_decision(
+            fallback.provider, fallback.effort, fallback.classifier, "capacity-failure"
+        )
+        _block_transient(
+            kanban_db,
+            board,
+            task_id,
+            run_id,
+            "Both hosted providers failed at this boundary "
+            f"({route.provider}: {failure_class}; {fallback.provider}: "
+            f"{fallback_class}); the lane never falls to a metered or "
+            "local path. Waiting for provider recovery.",
+        )
+        return FailoverOutcome(
+            fallback, fallback_result, candidate_file, route.provider, True
+        )
+    if fallback_result.structured:
+        candidate_file = _persist_candidate(
+            state,
+            state_file,
+            dict(fallback_result.structured),
+            route=fallback,
+            returncode=fallback_result.returncode,
+            goal_turn=goal_turn,
+        )
+    return FailoverOutcome(
+        fallback, fallback_result, candidate_file, route.provider, False
+    )

services/hermes/scripts/cli_lane_files.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""Durable path and filesystem primitives for Hermes CLI lane journals."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import re
+import stat
+import uuid
+from pathlib import Path
+from typing import Any
+
+from cli_lane_config import STATE_ROOT, TerminalIdentity, canonical_run_id
+
+
+def atomic_json(path: Path, value: dict[str, Any], mode: int = 0o600) -> None:
+    """Durably replace a small state document without following temp symlinks."""
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    os.chmod(path.parent, 0o700, follow_symlinks=False)
+    temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    flags |= getattr(os, "O_NOFOLLOW", 0)
+    descriptor = None
+    try:
+        descriptor = os.open(temporary, flags, mode)
+        with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
+            descriptor = None
+            json.dump(value, stream, indent=2, sort_keys=True)
+            stream.write("\n")
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.chmod(temporary, mode, follow_symlinks=False)
+        os.replace(temporary, path)
+        _fsync_directory(path.parent)
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        with contextlib.suppress(FileNotFoundError):
+            temporary.unlink()
+
+def _fsync_directory(directory: Path) -> None:
+    """Persist directory-entry changes after an atomic rename or quarantine."""
+    flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    flags |= getattr(os, "O_NOFOLLOW", 0)
+    descriptor = os.open(directory, flags)
+    try:
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+def load_json(path: Path) -> dict[str, Any]:
+    try:
+        flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+        descriptor = os.open(path, flags)
+        with os.fdopen(descriptor, "r", encoding="utf-8") as stream:
+            value = json.load(stream)
+    except (OSError, UnicodeError, json.JSONDecodeError):
+        return {}
+    return value if isinstance(value, dict) else {}
+
+def state_path(board: str, task_id: str) -> Path:
+    safe_board = re.sub(r"[^a-zA-Z0-9_.-]+", "-", board)
+    safe_task = re.sub(r"[^a-zA-Z0-9_.-]+", "-", task_id)
+    return STATE_ROOT / safe_board / f"{safe_task}.json"
+
+def _result_path(state_file: Path, run_id: Any, sequence: int) -> Path:
+    """Return a unique provider-result path; completed turns are never reused."""
+    safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown"))
+    return state_file.with_name(
+        f"{state_file.stem}.run-{safe_run}.provider-{sequence}.result.json"
+    )
+
+def _candidate_path(state_file: Path, run_id: Any, sequence: int) -> Path:
+    """Return the durable path for one exact structured worker response."""
+    safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown"))
+    return state_file.with_name(
+        f"{state_file.stem}.run-{safe_run}.candidate-{sequence}.json"
+    )
+
+def _terminal_path(
+    state_file: Path,
+    run_id: Any,
+    state: str = "pending",
+) -> Path:
+    """Return the identity-bound replay path for an accepted terminal response."""
+    if state not in {"pending", "committed"}:
+        raise ValueError(f"invalid terminal journal state: {state}")
+    canonical = canonical_run_id(run_id)
+    if type(run_id) is not int or canonical is None:
+        raise ValueError("terminal run ID must be a positive SQLite int64")
+    return state_file.with_name(
+        f"{state_file.stem}.run-{canonical}.terminal.{state}.json"
+    )
+
+_TERMINAL_NAME = re.compile(
+    r"^(?P<task>[a-zA-Z0-9_.-]+)\.run-(?P<run>[0-9]+)\."
+    r"terminal\.(?P<state>pending|committed)\.json$"
+)
+_TERMINAL_EVIDENCE_NAME = re.compile(
+    r"^(?P<task>[a-zA-Z0-9_.-]+)\.run-(?P<run>[0-9]+)\."
+    r"terminal\.(?P<state>prepared|conflict)-(?P<digest>[a-f0-9]{32})\.json$"
+)
+_RETIRE_NAME = re.compile(
+    r"^\.retire\.(?P<path_digest>[a-f0-9]{16})\."
+    r"(?P<source_digest>[a-f0-9]{16})\.(?P<sequence>[0-9]+)$"
+)
+_LEGACY_RETIRE_NAME = re.compile(r"^\.retire\.[a-f0-9]{32}\.[0-9]+$")
+_SAFE_BOARD = re.compile(r"^[a-zA-Z0-9_.-]+$")
+
+
+def _terminal_identity(path: Path) -> TerminalIdentity | None:
+    """Parse replay authority lexically before opening a journal or board."""
+    try:
+        relative = path.relative_to(STATE_ROOT)
+    except ValueError:
+        return None
+    if len(relative.parts) != 2:
+        return None
+    board, filename = relative.parts
+    match = _TERMINAL_NAME.fullmatch(filename)
+    run_id = canonical_run_id(match.group("run")) if match else None
+    try:
+        board_stat = path.parent.stat(follow_symlinks=False)
+    except OSError:
+        return None
+    if (
+        not match
+        or run_id is None
+        or not stat.S_ISDIR(board_stat.st_mode)
+        or not _SAFE_BOARD.fullmatch(board)
+        or board in {".", ".."}
+        or match.group("task") in {".", ".."}
+    ):
+        return None
+    return TerminalIdentity(
+        board=board,
+        task_id=match.group("task"),
+        run_id=run_id,
+        state=match.group("state"),
+    )
+
+def _terminal_evidence_identity(path: Path) -> TerminalIdentity | None:
+    """Parse immutable prepared/conflict evidence authority from its path."""
+    try:
+        relative = path.relative_to(STATE_ROOT)
+    except ValueError:
+        return None
+    if len(relative.parts) != 2:
+        return None
+    board, filename = relative.parts
+    match = _TERMINAL_EVIDENCE_NAME.fullmatch(filename)
+    run_id = canonical_run_id(match.group("run")) if match else None
+    try:
+        board_stat = path.parent.stat(follow_symlinks=False)
+    except OSError:
+        return None
+    if (
+        not match
+        or run_id is None
+        or not stat.S_ISDIR(board_stat.st_mode)
+        or not _SAFE_BOARD.fullmatch(board)
+        or board in {".", ".."}
+        or match.group("task") in {".", ".."}
+    ):
+        return None
+    return TerminalIdentity(
+        board=board,
+        task_id=match.group("task"),
+        run_id=run_id,
+        state=match.group("state"),
+    )

services/hermes/scripts/cli_lane_finalization.py

@@ -0,0 +1,331 @@
+#!/usr/bin/env python3
+"""Exact-run database completion and terminal evidence publication."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Any
+
+from cli_lane_board import _board_call, _external, _record_board_access_error, _task_value
+from cli_lane_capabilities import kanban_capabilities
+from cli_lane_config import (
+    TerminalIdentity,
+    TerminalRecoverySnapshot,
+    TerminalSnapshot,
+    canonical_run_id,
+    utc_now,
+)
+from cli_lane_evidence import (
+    _persist_conflict_evidence,
+    _persist_prepared_evidence,
+    _retire_terminal_entry,
+    _same_terminal_document,
+    _write_json_noreplace,
+)
+from cli_lane_files import _terminal_identity, _terminal_path, state_path
+from cli_lane_records import (
+    _load_terminal_json,
+    _open_small_json_snapshot,
+    _open_terminal_snapshot,
+    _terminal_record_valid,
+)
+
+
+def _recover_exact_run(
+    kanban_db: Any,
+    identity: TerminalIdentity | None,
+    reason: str,
+) -> bool:
+    """Make an exact external run retryable after journal recovery fails."""
+    if identity is None or canonical_run_id(identity.run_id) is None:
+        return False
+    if not kanban_capabilities(kanban_db).exact_run_reclaim:
+        return False
+
+    def operation(conn: Any) -> bool:
+        task = kanban_db.get_task(conn, identity.task_id)
+        if task is None:
+            return False
+        if (
+            str(_task_value(task, "status", "")) != "running"
+            or _task_value(task, "current_run_id", None) != identity.run_id
+            or not _external(task)
+        ):
+            return False
+        return bool(
+            kanban_db.reclaim_task(
+                conn,
+                identity.task_id,
+                reason=f"terminal journal recovery failed ({reason}); exact run may retry",
+                expected_run_id=identity.run_id,
+            )
+        )
+
+    try:
+        recovered = bool(_board_call(kanban_db, identity.board, operation))
+    except Exception as error:
+        _record_board_access_error(identity.board, error)
+        return False
+    if recovered:
+        print(
+            f"reclaimed {identity.board}/{identity.task_id} run {identity.run_id} "
+            f"after terminal journal recovery: {reason}",
+            file=sys.stderr,
+            flush=True,
+        )
+    return recovered
+
+def _retire_snapshot(
+    path: Path,
+    snapshot: TerminalSnapshot,
+    *,
+    authority_name: str | None = None,
+) -> str:
+    """Retire only the directory entry still naming an opened snapshot inode."""
+    return _retire_terminal_entry(
+        path,
+        snapshot.file_stat,
+        board_descriptor=snapshot.directory_descriptor,
+        quarantine_descriptor=snapshot.directory_descriptor,
+        authority_name=authority_name,
+        source_descriptor=snapshot.descriptor,
+    )
+
+def _retire_snapshot_after_db(path: Path, snapshot: TerminalSnapshot) -> str:
+    """Keep a committed DB result recoverable across retirement fsync errors."""
+    try:
+        return _retire_snapshot(path, snapshot)
+    except OSError as error:
+        print(
+            f"terminal journal retirement deferred after DB commit: "
+            f"{type(error).__name__}: {error}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return "deferred"
+
+def _discard_evidence(path: Path) -> str:
+    """Identity-safely remove one immutable prepared evidence artifact."""
+    snapshot = _open_small_json_snapshot(path)
+    if snapshot is None:
+        return "missing"
+    try:
+        return _retire_snapshot(path, snapshot)
+    finally:
+        snapshot.close()
+
+def _promote_prepared_evidence(
+    identity: TerminalIdentity,
+    document: dict[str, Any],
+    prepared: Path,
+    *,
+    discard_prepared: bool = True,
+) -> tuple[Path, str]:
+    """Publish the DB-winning result without overwriting first-writer evidence."""
+    committed = _terminal_path(
+        state_path(identity.board, identity.task_id),
+        identity.run_id,
+        "committed",
+    )
+    committed_document = dict(document)
+    committed_document["kanban_state"] = "committed"
+    committed_document["committed_at"] = utc_now()
+    state = "committed"
+    if not _write_json_noreplace(committed, committed_document):
+        committed_identity = TerminalIdentity(
+            identity.board,
+            identity.task_id,
+            identity.run_id,
+            "committed",
+        )
+        existing = _load_terminal_json(committed, committed_identity)
+        if (
+            not _terminal_record_valid(existing, committed_identity)
+            or existing.get("kanban_state") != "committed"
+            or not _same_terminal_document(existing, committed_document)
+        ):
+            _persist_conflict_evidence(
+                identity,
+                document,
+                "canonical committed evidence already contains a different result",
+            )
+            state = "conflict"
+    if discard_prepared:
+        _discard_evidence(prepared)
+    return committed, state
+
+def _finalize_document_db(
+    kanban_db: Any,
+    identity: TerminalIdentity,
+    document: dict[str, Any],
+) -> str:
+    """Apply one validated exact-run result to Kanban under DB run guards."""
+    if canonical_run_id(identity.run_id) is None:
+        return "stale"
+
+    def operation(conn: Any) -> str:
+        task = kanban_db.get_task(conn, identity.task_id)
+        if task is None:
+            return "stale"
+        status = str(_task_value(task, "status", ""))
+        if status == "done":
+            return (
+                "committed"
+                if (
+                    _task_value(task, "completed_run_id", None) == identity.run_id
+                    and str(_task_value(task, "result", "") or "")
+                    == document["result"]
+                )
+                else "stale"
+            )
+        current_run_id = _task_value(task, "current_run_id", None)
+        completion_guard: dict[str, int]
+        if status == "running" and current_run_id == identity.run_id:
+            if not kanban_capabilities(kanban_db).exact_run_completion:
+                return "deferred"
+            completion_guard = {"expected_run_id": identity.run_id}
+        elif status in {"ready", "blocked", "triage"} and current_run_id is None:
+            # The journal may have survived an older post-persistence error
+            # path that ended its run. The patched DB verifies atomically that
+            # this is still the latest ended run before allowing completion.
+            if not kanban_capabilities(kanban_db).ended_run_replay:
+                return "deferred"
+            completion_guard = {"replay_ended_run_id": identity.run_id}
+        else:
+            return "stale"
+        completed = kanban_db.complete_task(
+            conn,
+            identity.task_id,
+            result=document["result"],
+            summary=document["summary"],
+            metadata=document["metadata"],
+            **completion_guard,
+        )
+        if completed:
+            return "committed"
+        # A duplicate finalizer can lose the guarded UPDATE to an identical
+        # first writer. Re-read rather than reporting a false pending state.
+        latest = kanban_db.get_task(conn, identity.task_id)
+        latest_status = (
+            str(_task_value(latest, "status", "")) if latest is not None else ""
+        )
+        if latest_status == "done":
+            return (
+                "committed"
+                if (
+                    _task_value(latest, "completed_run_id", None) == identity.run_id
+                    and str(_task_value(latest, "result", "") or "")
+                    == document["result"]
+                )
+                else "stale"
+            )
+        # A guarded completion can be retried only while this exact live run
+        # still owns the task. An ended, replaced, or nonexistent run cannot
+        # become authoritative later and must converge to conflict evidence.
+        if (
+            latest_status == "running"
+            and _task_value(latest, "current_run_id", None) == identity.run_id
+        ):
+            return "pending"
+        return "stale"
+
+    return str(_board_call(kanban_db, identity.board, operation))
+
+def _resolve_pending_after_winner(
+    path: Path,
+    identity: TerminalIdentity,
+    winning_document: dict[str, Any],
+) -> bool:
+    """Retire duplicates and preserve differing replacements as conflicts."""
+    for _attempt in range(8):
+        snapshot = _open_terminal_snapshot(path, identity)
+        if snapshot is None:
+            return False
+        document = snapshot.document
+        if not _terminal_record_valid(document, identity):
+            snapshot.close()
+            return False
+        differing = not _same_terminal_document(document, winning_document)
+        try:
+            if differing:
+                _persist_conflict_evidence(
+                    identity,
+                    document,
+                    "different valid result lost the exact-run first-writer race",
+                )
+            retirement = _retire_snapshot_after_db(path, snapshot)
+        finally:
+            snapshot.close()
+        if retirement in {"retired", "missing"}:
+            if differing:
+                print(
+                    f"preserved terminal result conflict for {identity.board}/"
+                    f"{identity.task_id} run {identity.run_id}",
+                    file=sys.stderr,
+                    flush=True,
+                )
+            return differing
+        if retirement not in {"replacement", "replacement-staged", "collision"}:
+            return differing
+    return False
+
+def _finalize_terminal_record(
+    kanban_db: Any,
+    path: Path,
+    _record: dict[str, Any] | None = None,
+    *,
+    snapshot: TerminalRecoverySnapshot | None = None,
+) -> str:
+    """Commit one inode-pinned exact-run journal, or leave it safely pending."""
+    identity = _terminal_identity(path)
+    if identity is None or identity.state != "pending":
+        if snapshot is not None:
+            snapshot.close()
+        return "invalid"
+    pinned: TerminalSnapshot | TerminalRecoverySnapshot | None = snapshot
+    if pinned is None:
+        pinned = _open_terminal_snapshot(path, identity)
+    if pinned is None or pinned.document is None:
+        if pinned is not None:
+            pinned.close()
+        return "invalid"
+    document = pinned.document
+    try:
+        if not _terminal_record_valid(document):
+            return "invalid"
+        if not _terminal_record_valid(document, identity):
+            return "foreign"
+        prepared = _persist_prepared_evidence(identity, document)
+        outcome = _finalize_document_db(kanban_db, identity, document)
+        if outcome == "committed":
+            _committed, evidence_state = _promote_prepared_evidence(
+                identity,
+                document,
+                prepared,
+            )
+            retirement = _retire_snapshot_after_db(path, pinned)
+        elif outcome == "stale":
+            _persist_conflict_evidence(
+                identity,
+                document,
+                "valid terminal result no longer matches the authoritative task run",
+            )
+            _discard_evidence(prepared)
+            retirement = _retire_snapshot_after_db(path, pinned)
+            outcome = "conflict"
+            evidence_state = "conflict"
+        else:
+            return outcome
+    finally:
+        pinned.close()
+    if outcome == "committed" and retirement not in {"retired", "missing"}:
+        _resolve_pending_after_winner(path, identity, document)
+    if evidence_state == "conflict":
+        print(
+            f"terminal first-writer conflict recorded for {identity.board}/"
+            f"{identity.task_id} run {identity.run_id}",
+            file=sys.stderr,
+            flush=True,
+        )
+    return outcome

services/hermes/scripts/cli_lane_goal.py

@@ -1,5 +1,49 @@
 #!/usr/bin/env python3
-"""Fail-closed completion checks for durable external Kanban workers."""
+"""Role-aware completion checks for durable external Kanban workers.
+
+Completion is judged against the action the card assigned, not against the
+health of whatever the card asked the worker to look at.
+
+* Implementation cards keep the fail-closed local goal judge: a report is
+  accepted only when the model agrees every explicit requirement finished.
+* Review, audit, and diagnostic cards deliver a *verdict*. A completed report
+  carrying a truthful SHIP or BLOCK verdict with evidence finalizes the card
+  even when the verdict says the reviewed artifact must not ship. Findings are
+  the deliverable, not a task blocker, so the card is never resumed with an
+  instruction to repair an implementation the reviewer may not touch.
+
+Three deliberate boundaries keep that regime honest. First, ``role`` belongs to
+the caller that holds the card: ``unfinished_result_reason`` cannot classify a
+card it was never handed, so every caller holding the objective MUST pass
+``role``; the role-blind default is the replay contract below, never a weaker
+classifier that could short-circuit the role-aware gate.
+
+Second, the verdict contract binds in single-shot mode exactly as in the goal
+loop: a review card's deliverable *is* its verdict, so the lane fails closed
+rather than record a verdictless review as done. Two single-shot outcomes change
+against main d8f2d818, deliberately - a completed review whose prose calls the
+*reviewed* artifact pending now completes instead of being discarded, and one
+carrying no usable verdict now blocks instead of completing.
+
+Third, a journalled terminal record stays valid across an upgrade. PR15 re-runs
+this gate on replay without a card, so ``role=None`` keeps the unfinished-work
+heuristic - that integrity check still works - and skips it only for a report
+that satisfies the whole review contract and changed no files. That is precisely
+the set the role-aware lane accepts as a review, so a record accepted under one
+version re-validates under every other instead of being quarantined into a
+re-dispatch of an already-accepted task.
+
+The task role comes from explicit card metadata (``Hermes-Task-Role: review``)
+whenever the card author supplies it, read from real newlines and from the
+literal ``\\n`` escapes the board stores in single-line bodies. Pre-contract
+cards fall back to a narrow inference needing a read-only scope, a requested
+SHIP/BLOCK verdict, no requested mutation deliverable, and a report that changed
+no files. A card whose deliverable is a findings list rather than a verdict
+deliberately stays on the implementation regime: the verdict *is* the review
+contract's gate, so inferring a review role for a card that never asked for one
+could only fail closed. A report that changed files never resolves to the review
+role, even when the card declares it.
+"""
 
 from __future__ import annotations
 
@@ -10,6 +54,11 @@ import urllib.error
 import urllib.request
 from typing import Any, Callable
 
+try:  # The lane runs inside the agent image, which owns the canonical redactor.
+    from agent.redact import redact_sensitive_text as _canonical_redact
+except Exception:  # pragma: no cover - only when the agent runtime is absent
+    _canonical_redact = None
+
 
 GOAL_JUDGE_URL = os.environ.get(
     "HERMES_GOAL_JUDGE_URL",
@@ -24,6 +73,46 @@ GOAL_JUDGE_TIMEOUT_SECONDS = max(
     min(180, int(os.environ.get("HERMES_GOAL_JUDGE_TIMEOUT_SECONDS", "120"))),
 )
 RESULT_STATUSES = frozenset({"completed", "blocked", "incomplete"})
+REVIEW_ROLE = "review"
+IMPLEMENTATION_ROLE = "implementation"
+TASK_ROLES = frozenset({REVIEW_ROLE, IMPLEMENTATION_ROLE})
+ROLE_ALIASES = {
+    "audit": REVIEW_ROLE,
+    "assessment": REVIEW_ROLE,
+    "diagnosis": REVIEW_ROLE,
+    "diagnostic": REVIEW_ROLE,
+    "inspection": REVIEW_ROLE,
+    "review": REVIEW_ROLE,
+    "reviewer": REVIEW_ROLE,
+    "verdict": REVIEW_ROLE,
+    "build": IMPLEMENTATION_ROLE,
+    "change": IMPLEMENTATION_ROLE,
+    "implement": IMPLEMENTATION_ROLE,
+    "implementation": IMPLEMENTATION_ROLE,
+    "repair": IMPLEMENTATION_ROLE,
+}
+JUDGE_REASON_LIMIT = 600
+MIN_REVIEW_SUMMARY_CHARS = 80
+READ_ONLY_GUARD = (
+    "the assigned action is a read-only review: report the verdict and evidence, "
+    "and do not modify the reviewed implementation"
+)
+# The runner appends its own turn/rejection evidence to the judge objective.
+# It is emitted under this heading so ``card_scope`` drops it with every other
+# non-card section instead of letting a quoted worker sentence reassign a role.
+CONTROLLER_EVIDENCE_HEADING = "## Hermes goal-controller evidence"
+# ``hermes_cli.kanban_db.build_worker_context`` (Hermes runtime 0.18.2) renders
+# the worker context as H2 sections and emits exactly CARD_SECTIONS followed by
+# HISTORY_SECTIONS; ``test_hermes_cli_review_context.py`` pins that contract
+# against the installed runtime. The cut below allow-lists the two card
+# sections, so an upstream rename fails closed - the card just ends early.
+CARD_SECTIONS = ("Body", "Attachments")
+HISTORY_SECTIONS = (
+    "Prior attempts on this task",
+    "Parent task results",
+    "Recent work by",
+    "Comment thread",
+)
 GOAL_JUDGE_SCHEMA: dict[str, Any] = {
     "type": "object",
     "additionalProperties": False,
@@ -43,6 +132,73 @@ UNFINISHED_EVIDENCE = re.compile(
     r"\b(?:tests?|checks?|build|validation|verification|commit|push)\b)",
     re.IGNORECASE | re.DOTALL,
 )
+HISTORY_SECTION = re.compile(
+    r"^##\s+(?!(?:Body|Attachments)\b)"
+    r"|^Authoritative Hermes goal-controller evidence\b",
+    re.IGNORECASE | re.MULTILINE,
+)
+# Explicit, machine-readable card metadata. This is the supported contract;
+# every pattern below it only keeps pre-contract cards working.
+ROLE_DIRECTIVE = re.compile(
+    r"^[\s>#*\-]*(?:hermes[\s_-]*)?task[\s_-]*role\s*[:=]\s*([A-Za-z_-]+)\s*$",
+    re.IGNORECASE | re.MULTILINE,
+)
+OUTPUT_DIRECTIVE = re.compile(
+    r"^[\s>#*\-]*(?:hermes[\s_-]*)?expected[\s_-]*output\s*[:=]\s*([A-Za-z_-]+)\s*$",
+    re.IGNORECASE | re.MULTILINE,
+)
+READ_ONLY_SCOPE = re.compile(
+    r"(?:\b(?:do|does|must|should|shall|may|will)\s+not\s+(?:\w+\s+){0,3}?"
+    r"(?:edit|modify|change|alter|patch|rewrite|implement|repair|author)\b"
+    r"|\bdon'?t\s+(?:\w+\s+){0,3}?(?:edit|modify|change|alter|patch)\b"
+    r"|\bwithout\s+(?:editing|modifying|changing|altering|patching)\b"
+    r"|\bno\s+(?:\w+\s+){0,2}?(?:edits?|changes?|modifications?)\b"
+    r"|\bmake\s+no\s+(?:code|file|source)?\s*changes\b"
+    r"|\bpristine\b|\bread[\s-]?only\b)",
+    re.IGNORECASE,
+)
+# A SHIP/BLOCK-shaped deliverable, not merely a mention of the word "verdict":
+# cards that audit some other component's verdict machinery must not be read as
+# owing one themselves.
+VERDICT_DELIVERABLE = re.compile(
+    r"(?:\bship\b[^.\n]{0,120}?\bblock\b|\bblock\b[^.\n]{0,120}?\bship\b"
+    r"|\bship\s*/\s*not-?ship\b|\bverdict\s+of\b"
+    r"|\b(?:return|record|report|give|provide|deliver|state|produce|emit)\b"
+    r"(?:\s+[-\w']+){0,6}?\s+verdict\b)",
+    re.IGNORECASE,
+)
+MUTATION_DELIVERABLE = re.compile(
+    r"(?:\bpushed\s+(?:sha|commit|head|branch|revision)\b"
+    r"|\bpush\s+(?:\w+\s+){0,3}?(?:branch|commit|change\w*|fix\w*)\b"
+    r"|\bcommit\s+(?:\w+\s+){0,4}?and\s+push\b|\bamend\b"
+    r"|\bopen\s+(?:a|an|one|the)\s+(?:new\s+)?(?:draft\s+)?(?:pr\b|pull\s+request)"
+    r"|\bdraft\s+pr\s+only\b)",
+    re.IGNORECASE,
+)
+VERDICT_DIRECTIVE = re.compile(
+    r"\b(?:final\s+|strict\s+|overall\s+)?(?:verdict|recommendation|decision)\s*"
+    r"(?:is\s*)?[:=—-]\s*[*_`\s]*(ship|block)\b",
+    re.IGNORECASE,
+)
+VERDICT_TOKEN = re.compile(r"(?<![A-Za-z])(SHIP|BLOCK)(?![A-Za-z])")
+CONTROL_CHARACTERS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
+# Fallback for environments without the agent runtime (repo-only checkouts).
+SECRET_LIKE = re.compile(
+    r"(?:gh[pousr]_[A-Za-z0-9]{16,}|xox[abprs]-[A-Za-z0-9-]{10,}"
+    r"|sk-[A-Za-z0-9_-]{16,}|AKIA[0-9A-Z]{16}"
+    r"|eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})"
+)
+# Two shapes the canonical redactor deliberately passes through but this lane
+# handles: ``scheme://user:secret@host`` clone URLs (upstream leaves web
+# userinfo alone so round-trip workflow URLs keep working) and an opaque
+# credential named in prose. Both are anchored on surrounding syntax so a 40-hex
+# git SHA - indistinguishable from a Gitea PAT alone - survives as evidence.
+URL_USERINFO = re.compile(r"(?<=://)([^/\s:@]{1,64}):([^/\s@]{1,256})@")
+CREDENTIAL_CONTEXT = re.compile(
+    r"((?:pat|token|secret|password|passwd|credential|api[_-]?key|authorization)"
+    r"[\"'\s:=]+(?:bearer\s+|token\s+)?)([A-Za-z0-9_.\-]{20,})",
+    re.IGNORECASE,
+)
 
 
 def _bounded(value: str, limit: int) -> str:
@@ -55,19 +211,155 @@ def _bounded(value: str, limit: int) -> str:
     return f"{value[:head]}{marker}{value[-(remaining - head):]}"
 
 
-def unfinished_result_reason(result: dict[str, Any]) -> str | None:
-    """Reject self-contradictory completion reports without model inference."""
-    status = str(result.get("status") or "")
+def redact(value: str) -> str:
+    """Redact with the runtime's canonical helper plus this lane's own gaps."""
+    text = SECRET_LIKE.sub("[redacted]", value)
+    text = URL_USERINFO.sub(r"\1:[redacted]@", text)
+    text = CREDENTIAL_CONTEXT.sub(r"\1[redacted]", text)
+    return _canonical_redact(text, force=True) if _canonical_redact else text
+
+
+def sanitize_reason(value: Any, limit: int = JUDGE_REASON_LIMIT) -> str:
+    """Return one bounded, single-line, secret-free judge reason."""
+    text = redact(str(value or ""))
+    text = " ".join(CONTROL_CHARACTERS.sub(" ", text).split())
+    if not text:
+        return "local judge supplied no reason"
+    return text if len(text) <= limit else f"{text[: limit - 3].rstrip()}..."
+
+
+def _strings(value: Any) -> list[str]:
+    """Return the non-empty string entries of a reported evidence list."""
+    if not isinstance(value, list):
+        return []
+    return [str(item).strip() for item in value if str(item).strip()]
+
+
+def card_scope(objective: str) -> str:
+    """Return the card the worker was assigned, without appended run history.
+
+    Literal ``\\n`` escapes are expanded first: the board stores most bodies on
+    a single physical line, and every pattern here is line-anchored or depends
+    on word boundaries the escape sequence would otherwise destroy. Expanding
+    before the cut is monotone - it can only move the first history heading
+    earlier - so it cannot smuggle appended history into the card. The
+    normalization is for analysis only; the judge objective is never rewritten.
+    """
+    text = str(objective or "").replace("\\r\\n", "\n").replace("\\n", "\n")
+    match = HISTORY_SECTION.search(text)
+    return text[: match.start()] if match else text
+
+
+def _declared_role(objective: str) -> str | None:
+    """Return the role a card declared through explicit worker metadata."""
+    for pattern in (ROLE_DIRECTIVE, OUTPUT_DIRECTIVE):
+        roles = {
+            ROLE_ALIASES[value]
+            for value in (
+                match.group(1).strip().lower() for match in pattern.finditer(objective)
+            )
+            if value in ROLE_ALIASES
+        }
+        if len(roles) == 1:
+            return roles.pop()
+        if roles:
+            # A card that declares two different roles has no usable contract.
+            # Hold it to the stricter implementation regime rather than letting
+            # the more permissive review path win an ambiguous declaration.
+            return IMPLEMENTATION_ROLE
+    return None
+
+
+def task_role(objective: str, result: dict[str, Any] | None = None) -> tuple[str, str]:
+    """Resolve the assigned task role and how that resolution was reached."""
+    text = card_scope(objective)
+    changed = _strings((result or {}).get("changed_files"))
+    declared = _declared_role(text)
+    if declared == REVIEW_ROLE and changed:
+        # Mutation evidence outranks a declaration: a card cannot label itself
+        # read-only and then self-certify a report that edited the tree.
+        return IMPLEMENTATION_ROLE, "conflict"
+    if declared is not None:
+        return declared, "directive"
+    if (
+        not changed
+        and READ_ONLY_SCOPE.search(text)
+        and VERDICT_DELIVERABLE.search(text)
+        and not MUTATION_DELIVERABLE.search(text)
+    ):
+        return REVIEW_ROLE, "inferred"
+    return IMPLEMENTATION_ROLE, "default"
+
+
+def review_verdict(result: dict[str, Any]) -> tuple[str | None, str | None]:
+    """Return the declared SHIP/BLOCK verdict, or why it cannot be trusted."""
+    declared: set[str] = set()
+    field = result.get("verdict")
+    if field is not None:
+        value = str(field).strip().upper()
+        if value not in {"SHIP", "BLOCK"}:
+            return None, f"review verdict field is not SHIP or BLOCK: {value[:40]!r}"
+        declared.add(value)
+    summary = str(result.get("summary") or "")
+    declared.update(match.upper() for match in VERDICT_DIRECTIVE.findall(summary))
+    if len(declared) > 1:
+        return None, "review report declares both SHIP and BLOCK verdicts"
+    if declared:
+        return declared.pop(), None
+    tokens = {match.upper() for match in VERDICT_TOKEN.findall(summary)}
+    if len(tokens) > 1:
+        return None, "review summary names both SHIP and BLOCK without a verdict line"
+    if tokens:
+        return tokens.pop(), None
+    return None, "review report declares no explicit SHIP or BLOCK verdict"
+
+
+def review_completion_problem(result: dict[str, Any]) -> str | None:
+    """Fail closed on a review report that does not carry a usable verdict."""
+    verdict, problem = review_verdict(result)
+    if problem or verdict is None:
+        return problem or "review report declares no explicit SHIP or BLOCK verdict"
     summary = str(result.get("summary") or "").strip()
-    blockers = result.get("blockers")
+    if len(summary) < MIN_REVIEW_SUMMARY_CHARS:
+        return f"{verdict} verdict carries no reviewable rationale in its summary"
+    findings = _strings(result.get("findings"))
+    if verdict == "BLOCK" and not findings:
+        return "BLOCK verdict reports no findings to justify it"
+    if not findings and not _strings(result.get("tests_run")) and not _strings(
+        result.get("artifacts")
+    ):
+        return f"{verdict} verdict reports no findings, tests, or artifacts as evidence"
+    return None
+
+
+def _completion_problem(result: dict[str, Any], role: str | None) -> str | None:
+    """Return the unguarded reason this report is not a finished deliverable."""
+    status = str(result.get("status") or "")
     if status == "incomplete":
+        summary = str(result.get("summary") or "").strip()
         return summary or "worker explicitly reported incomplete work"
     if status != "completed":
         return None
+    blockers = result.get("blockers")
     if isinstance(blockers, list) and any(str(item).strip() for item in blockers):
         return "worker reported blockers while claiming completion"
+    if role == REVIEW_ROLE:
+        # A review's prose describes the reviewed artifact, so the unfinished
+        # work heuristic below would read the artifact's state as the review's
+        # own. The verdict contract is the deterministic gate instead.
+        return review_completion_problem(result)
+    if (
+        role is None
+        and not _strings(result.get("changed_files"))
+        and review_completion_problem(result) is None
+    ):
+        # A replay caller holds no card. This is exactly the shape the
+        # role-aware lane accepts as a review, so honouring it here keeps a
+        # journalled record valid without weakening the heuristic for anything
+        # that is not already a complete review deliverable.
+        return None
     tests = result.get("tests_run")
-    evidence = [summary]
+    evidence = [str(result.get("summary") or "").strip()]
     if isinstance(tests, list):
         evidence.extend(str(item) for item in tests)
     match = UNFINISHED_EVIDENCE.search("\n".join(evidence))
@@ -76,18 +368,34 @@ def unfinished_result_reason(result: dict[str, Any]) -> str | None:
     return None
 
 
-def judge_goal_completion(
-    objective: str,
+def unfinished_result_reason(
     result: dict[str, Any],
     *,
-    url: str = GOAL_JUDGE_URL,
-    open_request: Callable[..., Any] = urllib.request.urlopen,
-) -> tuple[bool, str]:
-    """Use the local model as a fail-closed judge for explicit goal cards."""
-    deterministic_reason = unfinished_result_reason(result)
-    if deterministic_reason:
-        return False, deterministic_reason
-    payload = {
+    role: str | None = None,
+) -> str | None:
+    """Reject self-contradictory completion reports without model inference.
+
+    ``role`` must be supplied by any caller holding the card; see the module
+    docstring for the role-blind replay contract a caller without one gets.
+    """
+    problem = _completion_problem(result, role)
+    if problem is None:
+        return None
+    if role == REVIEW_ROLE:
+        # Every reason a reviewer may ever read carries the guard, so a resumed
+        # review is never told to repair the implementation it must not touch.
+        problem = f"{READ_ONLY_GUARD}; {problem}"
+    return sanitize_reason(problem)
+
+
+def _completion_claimed(result: dict[str, Any]) -> bool:
+    """Report whether the worker actually claimed a finished task."""
+    return str(result.get("status") or "") == "completed"
+
+
+def _judge_payload(objective: str, result: dict[str, Any], role: str) -> dict[str, Any]:
+    """Build the fail-closed local judge request for an implementation card."""
+    return {
         "model": GOAL_JUDGE_MODEL,
         "messages": [
             {
@@ -98,13 +406,16 @@ def judge_goal_completion(
                     "Return complete only when the report gives concrete evidence that every "
                     "explicit requirement finished. Return continue when any work, test, command, "
                     "commit, push, review, or verification is pending, in progress, omitted, or "
-                    "inconclusive. Do not trust the report status field."
+                    "inconclusive. Defects the worker reports about some other artifact are "
+                    "findings, not unfinished work on this task. Do not trust the report status "
+                    "field."
                 ),
             },
             {
                 "role": "user",
                 "content": json.dumps(
                     {
+                        "task_role": role,
                         "objective": _bounded(objective, 6000),
                         "worker_report": _bounded(
                             json.dumps(result, default=str), 4000
@@ -126,9 +437,40 @@ def judge_goal_completion(
             },
         },
     }
+
+
+def judge_goal_completion(
+    objective: str,
+    result: dict[str, Any],
+    *,
+    url: str = GOAL_JUDGE_URL,
+    open_request: Callable[..., Any] = urllib.request.urlopen,
+) -> tuple[bool, str]:
+    """Judge one worker report against the action its card actually assigned."""
+    role, source = task_role(objective, result)
+    reason = unfinished_result_reason(result, role=role)
+    if reason is None and not _completion_claimed(result):
+        reason = (
+            f"worker reported status {str(result.get('status') or 'unknown')!r} "
+            "rather than a finished task"
+        )
+        if role == REVIEW_ROLE:
+            reason = f"{READ_ONLY_GUARD}; {reason}"
+    if reason:
+        return False, sanitize_reason(reason)
+    if role == REVIEW_ROLE:
+        # The requested deliverable is present and self-consistent. A verdict
+        # that the reviewed artifact must not ship is a finished review, so the
+        # card finalizes instead of resuming a worker that may not repair it.
+        verdict, _ = review_verdict(result)
+        findings = len(_strings(result.get("findings")))
+        return True, sanitize_reason(
+            f"review deliverable complete: {verdict} verdict with {findings} finding(s); "
+            f"task role {role} resolved by {source}"
+        )
     request = urllib.request.Request(
         url,
-        data=json.dumps(payload).encode("utf-8"),
+        data=json.dumps(_judge_payload(objective, result, role)).encode("utf-8"),
         headers={"Content-Type": "application/json"},
         method="POST",
     )
@@ -139,7 +481,7 @@ def judge_goal_completion(
         verdict = json.loads(content)
         if verdict.get("verdict") not in {"complete", "continue"}:
             raise ValueError("local judge returned an invalid verdict")
-        reason = str(verdict.get("reason") or "local judge supplied no reason")
+        reason = sanitize_reason(verdict.get("reason"))
         return verdict.get("verdict") == "complete", reason
     except (
         AttributeError,
@@ -151,4 +493,6 @@ def judge_goal_completion(
         json.JSONDecodeError,
         urllib.error.URLError,
     ) as error:
-        return False, f"local completion judge unavailable: {type(error).__name__}: {error}"
+        return False, sanitize_reason(
+            f"local completion judge unavailable: {type(error).__name__}: {error}"
+        )

services/hermes/scripts/cli_lane_health.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""Lane-observed provider health with failure classification and cooldowns."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import time
+from typing import Any
+
+from cli_lane_config import (
+    AUTH_COOLDOWN_SECONDS_DEFAULT,
+    CAPACITY_COOLDOWN_SECONDS_DEFAULT,
+    PROVIDER_HEALTH_PATHS,
+    kanban_setting,
+    utc_now,
+)
+from cli_lane_files import load_json
+
+
+# Ordered from most to least specific so one output maps to one actionable
+# reason instead of the single conflated capacity signal.
+FAILURE_CLASSIFICATIONS = (
+    ("auth", re.compile(r"authentication|unauthorized|forbidden|oauth|token.*expired|401|403", re.I)),
+    ("rate-limit", re.compile(r"rate.?limit|429|529|overload", re.I)),
+    ("quota", re.compile(r"usage.?limit|quota|credit|exhaust|capacity", re.I)),
+)
+
+
+def classify_capacity_failure(output: str) -> str:
+    """Split the broad capacity regex into quota/auth/rate-limit/transport."""
+    for reason, pattern in FAILURE_CLASSIFICATIONS:
+        if pattern.search(output):
+            return reason
+    return "transport"
+
+
+def _write_provider_health(
+    provider: str, updates: dict[str, Any], clear: tuple[str, ...] = ()
+) -> None:
+    """Merge one lane observation into the shared provider health snapshot."""
+    path = PROVIDER_HEALTH_PATHS[provider]
+    value = load_json(path)
+    for key in clear:
+        value.pop(key, None)
+    value.update(updates)
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
+        temporary.write_text(
+            json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+        )
+        os.replace(temporary, path)
+    except OSError:
+        # Health snapshots are advisory; never fail the lane over one write.
+        pass
+
+
+def record_provider_success(provider: str) -> None:
+    """Publish fresh evidence of success so the provider re-enters routing."""
+    _write_provider_health(
+        provider,
+        {
+            "source": "cli-lane-runner",
+            "state": "available",
+            "authenticated": True,
+            "checked_at": utc_now(),
+        },
+        clear=("failure_reason", "failed_at", "cooldown_until"),
+    )
+
+
+def record_provider_failure(
+    provider: str, reason: str, now: float | None = None
+) -> float:
+    """Persist one lane-observed failure with its re-admission cooldown."""
+    current = time.time() if now is None else now
+    if reason == "auth":
+        cooldown = kanban_setting(
+            "provider_auth_cooldown_seconds", AUTH_COOLDOWN_SECONDS_DEFAULT
+        )
+        state, authenticated = "unavailable", False
+    else:
+        cooldown = kanban_setting(
+            "provider_capacity_cooldown_seconds", CAPACITY_COOLDOWN_SECONDS_DEFAULT
+        )
+        state, authenticated = "capacity-limited", True
+    cooldown_until = current + max(0.0, cooldown)
+    _write_provider_health(
+        provider,
+        {
+            "source": "cli-lane-runner",
+            "state": state,
+            "authenticated": authenticated,
+            "checked_at": utc_now(),
+            "failure_reason": reason,
+            "failed_at": current,
+            "cooldown_until": cooldown_until,
+        },
+    )
+    return cooldown_until

services/hermes/scripts/cli_lane_metrics.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+"""In-process Prometheus metrics for the direct CLI lane's routing decisions."""
+
+from __future__ import annotations
+
+import os
+import sys
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+
+METRIC_HELP = {
+    "hermes_cli_route_decisions_total": (
+        "Terminal outcome of each executed CLI lane route decision."
+    ),
+    "hermes_cli_provider_fallbacks_total": (
+        "Cross-provider failovers taken by the CLI lane, by failure reason."
+    ),
+    "hermes_cli_router_selection_failures_total": (
+        "Switchyard route selections that failed and blocked a card as transient."
+    ),
+    "hermes_cli_quota_fetch_failures_total": (
+        "Quota snapshot reads that failed; routing then fails open."
+    ),
+    "hermes_cli_quota_remaining_percent": (
+        "Binding remaining-quota percent per provider as seen by the CLI lane."
+    ),
+    "hermes_cli_quota_reset_timestamp_seconds": (
+        "Unix reset time of the binding quota window as seen by the CLI lane."
+    ),
+    "hermes_cli_provider_soft_excluded": (
+        "Whether new automatic work is currently routed away from a provider."
+    ),
+}
+COUNTER_NAMES = frozenset(name for name in METRIC_HELP if name.endswith("_total"))
+
+
+def _escape_label(value: str) -> str:
+    """Escape one Prometheus label value."""
+    return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
+
+
+class LaneMetricsRegistry:
+    """Thread-safe counter and gauge store rendered in Prometheus text format."""
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._values: dict[tuple[str, tuple[tuple[str, str], ...]], float] = {}
+
+    def _key(
+        self, name: str, labels: dict[str, str] | None
+    ) -> tuple[str, tuple[tuple[str, str], ...]]:
+        return name, tuple(sorted((labels or {}).items()))
+
+    def increment(
+        self, name: str, labels: dict[str, str] | None = None, amount: float = 1.0
+    ) -> None:
+        key = self._key(name, labels)
+        with self._lock:
+            self._values[key] = self._values.get(key, 0.0) + amount
+
+    def set_value(
+        self, name: str, labels: dict[str, str] | None, value: float
+    ) -> None:
+        with self._lock:
+            self._values[self._key(name, labels)] = value
+
+    def render(self) -> bytes:
+        """Render every recorded sample with stable ordering."""
+        with self._lock:
+            values = dict(self._values)
+        lines: list[str] = []
+        for name in sorted({name for name, _ in values}):
+            kind = "counter" if name in COUNTER_NAMES else "gauge"
+            lines.extend((f"# HELP {name} {METRIC_HELP[name]}", f"# TYPE {name} {kind}"))
+            for (sample_name, labels), value in sorted(values.items()):
+                if sample_name != name:
+                    continue
+                rendered = ",".join(
+                    f'{key}="{_escape_label(text)}"' for key, text in labels
+                )
+                body = f"{{{rendered}}}" if rendered else ""
+                lines.append(f"{name}{body} {value:.12g}")
+        return ("\n".join(lines) + "\n").encode("utf-8")
+
+
+METRICS = LaneMetricsRegistry()
+
+
+def record_route_decision(
+    provider: str, effort: str, classifier: str, outcome: str
+) -> None:
+    """Count one executed route with its terminal outcome."""
+    METRICS.increment(
+        "hermes_cli_route_decisions_total",
+        {
+            "provider": provider,
+            "effort": effort,
+            "classifier": classifier,
+            "outcome": outcome,
+        },
+    )
+
+
+def record_provider_fallback(
+    from_provider: str, to_provider: str, reason: str
+) -> None:
+    """Count one cross-provider failover with its classified reason."""
+    METRICS.increment(
+        "hermes_cli_provider_fallbacks_total",
+        {
+            "from_provider": from_provider,
+            "to_provider": to_provider,
+            "reason": reason,
+        },
+    )
+
+
+def record_router_selection_failure() -> None:
+    """Count one Switchyard outage observed at a selection boundary."""
+    METRICS.increment("hermes_cli_router_selection_failures_total")
+
+
+def record_quota_fetch_failure() -> None:
+    """Count one unavailable quota snapshot; routing continues fail-open."""
+    METRICS.increment("hermes_cli_quota_fetch_failures_total")
+
+
+def record_provider_quota(
+    provider: str, remaining_percent: float, reset_timestamp: float | None
+) -> None:
+    """Publish the binding quota window the lane based its routing on."""
+    labels = {"provider": provider}
+    METRICS.set_value(
+        "hermes_cli_quota_remaining_percent", labels, remaining_percent
+    )
+    if reset_timestamp is not None:
+        METRICS.set_value(
+            "hermes_cli_quota_reset_timestamp_seconds", labels, reset_timestamp
+        )
+
+
+def record_soft_exclusion(provider: str, excluded: bool) -> None:
+    """Publish whether new automatic work avoids this provider right now."""
+    METRICS.set_value(
+        "hermes_cli_provider_soft_excluded",
+        {"provider": provider},
+        1.0 if excluded else 0.0,
+    )
+
+
+def start_metrics_server(port: int | None = None) -> ThreadingHTTPServer | None:
+    """Serve lane metrics on the pod-local scrape port without blocking dispatch."""
+    selected_port = (
+        int(os.environ.get("HERMES_CLI_LANE_METRICS_PORT", "9011"))
+        if port is None
+        else port
+    )
+
+    class Handler(BaseHTTPRequestHandler):
+        def do_GET(self) -> None:  # noqa: N802
+            if self.path == "/metrics":
+                payload = METRICS.render()
+                self.send_response(200)
+                self.send_header("Content-Type", "text/plain; version=0.0.4")
+                self.send_header("Content-Length", str(len(payload)))
+                self.end_headers()
+                self.wfile.write(payload)
+            elif self.path == "/healthz":
+                self.send_response(200)
+                self.end_headers()
+            else:
+                self.send_error(404)
+
+        def log_message(self, _format: str, *_args: object) -> None:
+            return
+
+    try:
+        server = ThreadingHTTPServer(("0.0.0.0", selected_port), Handler)
+    except OSError as error:
+        print(
+            f"cli lane metrics server unavailable: {error}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return None
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    return server

services/hermes/scripts/cli_lane_prompt.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""Prompt, artifact, and structured-event helpers for CLI workers."""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+from pathlib import Path
+from typing import Any
+
+import cli_lane_goal
+from cli_lane_files import atomic_json
+
+
+def build_prompt(context: str, workspace: Path, handoff: str = "") -> str:
+    """Create a bounded worker contract with an explicit machine-readable result."""
+    return f"""You are a durable coding worker managed by Hermes Kanban.
+
+Work only on this objective and its acceptance criteria:
+
+{context}
+
+Workspace: {workspace}
+{handoff}
+
+Operate autonomously inside the workspace. Inspect before editing, preserve unrelated user changes, run proportionate tests, and do not claim completion without evidence. You have owner-level Kubernetes access in every namespace. Prefer Flux-tracked manifests for durable changes, but use kubectl, Flux, exec, port-forwarding, rollout operations, and existing Vault workflows when the objective or incident requires them. Persist any desired-state mutation back to Git. Do not force-push, hard-reset, clean untracked files, or expose credentials.
+
+Return a final JSON object matching the supplied schema. Use status=incomplete when required work, tests, commands, commits, pushes, or verification are still running or remain to be done. Use status=blocked only when an obstacle prevents completion of the assigned task itself. Never use status=completed for a progress report. For review or diagnostic tasks, put defects and risks in findings; those findings can make the reviewed change unfit to ship without blocking completion of the review. The blockers array must be empty whenever status is completed. List changed files, tests run, durable artifact paths, findings, and task blockers explicitly.
+"""
+
+def git_handoff(workspace: Path, prior_output: str) -> str:
+    """Build an explicit cross-provider handoff without transferring hidden state."""
+    def read_git(*args: str) -> str:
+        completed = subprocess.run(
+            ["git", "-C", str(workspace), *args],
+            text=True,
+            capture_output=True,
+            timeout=30,
+            check=False,
+        )
+        return (completed.stdout or completed.stderr).strip()[-12000:]
+
+    return (
+        "\nCross-provider handoff from a failed or exhausted worker:\n"
+        f"Git status:\n{read_git('status', '--short', '--branch')}\n\n"
+        f"Current diff summary:\n{read_git('diff', '--stat')}\n\n"
+        f"Prior worker tail:\n{prior_output[-10000:]}\n"
+        "Reinspect the workspace and verify all inherited claims before continuing."
+    )
+
+def workspace_artifacts(workspace: Path, values: Any) -> list[str]:
+    """Return existing regular artifacts contained by the task worktree."""
+    if not isinstance(values, list):
+        return []
+    root = workspace.resolve()
+    artifacts: list[str] = []
+    for value in values:
+        if not isinstance(value, str) or not value.strip():
+            continue
+        candidate = Path(value.strip()).expanduser()
+        if not candidate.is_absolute():
+            candidate = root / candidate
+        try:
+            resolved = candidate.resolve(strict=True)
+            resolved.relative_to(root)
+        except (OSError, RuntimeError, ValueError):
+            continue
+        if resolved.is_file():
+            artifacts.append(str(resolved))
+    return list(dict.fromkeys(artifacts))
+
+def _extract_json(value: Any) -> dict[str, Any] | None:
+    if isinstance(value, dict) and value.get("status") in cli_lane_goal.RESULT_STATUSES:
+        return value
+    if not isinstance(value, str):
+        return None
+    candidates = [value]
+    match = re.search(r"\{.*\}", value, re.DOTALL)
+    if match:
+        candidates.append(match.group(0))
+    for candidate in candidates:
+        try:
+            parsed = json.loads(candidate)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(parsed, dict) and parsed.get("status") in cli_lane_goal.RESULT_STATUSES:
+            return parsed
+    return None
+
+def _event_payload(provider: str, line: str, state: dict[str, Any], state_file: Path) -> dict[str, Any] | None:
+    """Persist provider session identifiers before interpreting final output."""
+    try:
+        event = json.loads(line)
+    except json.JSONDecodeError:
+        return None
+    if not isinstance(event, dict):
+        return None
+    if provider == "codex" and event.get("type") == "thread.started":
+        thread_id = event.get("thread_id") or event.get("thread", {}).get("id")
+        if thread_id:
+            state["codex_thread_id"] = str(thread_id)
+            atomic_json(state_file, state)
+    if provider == "claude" and event.get("session_id"):
+        state["claude_session_id"] = str(event["session_id"])
+        atomic_json(state_file, state)
+    for key in ("structured_output", "result", "output", "text"):
+        parsed = _extract_json(event.get(key))
+        if parsed:
+            return parsed
+    item = event.get("item")
+    if isinstance(item, dict):
+        for key in ("text", "content"):
+            parsed = _extract_json(item.get(key))
+            if parsed:
+                return parsed
+    return None

services/hermes/scripts/cli_lane_provider.py

@@ -0,0 +1,403 @@
+#!/usr/bin/env python3
+"""Native Codex and Claude process lifecycle for Hermes CLI workers."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import selectors
+import signal
+import subprocess
+import time
+import uuid
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any, Callable
+
+import cli_lane_goal
+from cli_lane_config import (
+    CAPACITY_PATTERN,
+    CLAUDE_BIN,
+    CLAUDE_SESSION_COLLISION,
+    CLAUDE_SETTINGS,
+    CODEX_BIN,
+    DATA_ROOT,
+    HEARTBEAT_SECONDS,
+    NO_CLAUDE_SESSION,
+    NO_CODEX_THREAD,
+    ProcessResult,
+    RESULT_SCHEMA,
+    RESULT_SCHEMA_PATH,
+    Route,
+    utc_now,
+)
+from cli_lane_files import _result_path, atomic_json, load_json
+from cli_lane_prompt import _event_payload
+
+
+def stream_process(
+    command: list[str],
+    *,
+    provider: str,
+    cwd: Path,
+    env: dict[str, str],
+    log_path: Path,
+    state: dict[str, Any],
+    state_file: Path,
+    heartbeat: Callable[[str], bool],
+    max_runtime: int,
+) -> ProcessResult:
+    """Stream JSONL to Kanban logs while maintaining the authoritative lease."""
+    process = subprocess.Popen(
+        command,
+        cwd=cwd,
+        env=env,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        bufsize=1,
+        start_new_session=True,
+    )
+    assert process.stdout is not None
+    selector = selectors.DefaultSelector()
+    selector.register(process.stdout, selectors.EVENT_READ)
+    started = time.monotonic()
+    last_heartbeat = 0.0
+    known_descendants: dict[int, tuple[int, int]] = {}
+    lines: list[str] = []
+    structured: dict[str, Any] | None = None
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    forced_failure = ""
+    with log_path.open("a", encoding="utf-8") as log:
+        log.write(f"\n[{utc_now()}] starting {provider} worker\n")
+        log.flush()
+        while process.poll() is None:
+            known_descendants.update(_descendant_processes(process.pid))
+            now = time.monotonic()
+            if now - started > max_runtime:
+                _terminate_worker_process(process, known_descendants)
+                forced_failure = "worker exceeded its maximum runtime"
+                lines.append(forced_failure + "\n")
+                break
+            if now - last_heartbeat >= HEARTBEAT_SECONDS:
+                if not heartbeat(f"{provider} worker active for {round(now - started)}s"):
+                    _terminate_worker_process(process, known_descendants)
+                    forced_failure = "Kanban lease was lost; provider process terminated"
+                    lines.append(forced_failure + "\n")
+                    break
+                last_heartbeat = now
+            for key, _ in selector.select(timeout=1.0):
+                line = key.fileobj.readline()
+                if not line:
+                    continue
+                lines.append(line)
+                if len(lines) > 4000:
+                    lines = lines[-4000:]
+                log.write(line)
+                log.flush()
+                parsed = _event_payload(provider, line, state, state_file)
+                structured = parsed or structured
+        _terminate_worker_process(process, known_descendants)
+        remainder = process.stdout.read()
+        if remainder:
+            lines.append(remainder)
+            log.write(remainder)
+            for line in remainder.splitlines():
+                parsed = _event_payload(provider, line, state, state_file)
+                structured = parsed or structured
+        if forced_failure:
+            log.write(forced_failure + "\n")
+        log.write(f"\n[{utc_now()}] {provider} exit={process.returncode}\n")
+    selector.close()
+    output = "".join(lines)[-100000:]
+    returncode = int(process.returncode if process.returncode is not None else 1)
+    return ProcessResult(
+        returncode=returncode,
+        output=output,
+        structured=structured,
+        capacity_failure=returncode != 0 and bool(CAPACITY_PATTERN.search(output)),
+    )
+
+def _process_record(pid: int) -> tuple[int, int, int] | None:
+    """Return one Linux process's parent, group, and start-time identity."""
+    try:
+        raw = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8")
+        fields = raw[raw.rfind(")") + 2 :].split()
+        return int(fields[1]), int(fields[2]), int(fields[19])
+    except (IndexError, OSError, ValueError):
+        return None
+
+def _descendant_processes(root_pid: int) -> dict[int, tuple[int, int]]:
+    """Snapshot descendants as pid -> (process group, start-time identity)."""
+    records: dict[int, tuple[int, int, int]] = {}
+    for entry in Path("/proc").iterdir():
+        if not entry.name.isdigit():
+            continue
+        pid = int(entry.name)
+        record = _process_record(pid)
+        if record is not None:
+            records[pid] = record
+    family = {root_pid}
+    changed = True
+    while changed:
+        changed = False
+        for pid, (parent, _, _) in records.items():
+            if pid not in family and parent in family:
+                family.add(pid)
+                changed = True
+    return {
+        pid: (records[pid][1], records[pid][2])
+        for pid in family
+        if pid != root_pid and pid in records
+    }
+
+def _process_identity_matches(pid: int, start_time: int) -> bool:
+    record = _process_record(pid)
+    return record is not None and record[2] == start_time
+
+def _signal_worker_tree(
+    root_pid: int,
+    descendants: dict[int, tuple[int, int]],
+    sig: signal.Signals,
+) -> None:
+    """Signal captured descendants across terminal-created process groups."""
+    groups = {root_pid}
+    for pid, (process_group, start_time) in descendants.items():
+        if _process_identity_matches(pid, start_time):
+            groups.add(process_group)
+    for process_group in groups:
+        with contextlib.suppress(ProcessLookupError):
+            os.killpg(process_group, sig)
+    for pid, (_, start_time) in descendants.items():
+        if not _process_identity_matches(pid, start_time):
+            continue
+        with contextlib.suppress(ProcessLookupError):
+            os.kill(pid, sig)
+
+def _terminate_worker_process(
+    process: subprocess.Popen[str],
+    known_descendants: dict[int, tuple[int, int]] | None = None,
+) -> None:
+    """Reap a worker and terminal descendants that created their own groups."""
+    descendants = dict(known_descendants or {})
+    descendants.update(_descendant_processes(process.pid))
+    if process.poll() is None:
+        _signal_worker_tree(process.pid, descendants, signal.SIGTERM)
+        with contextlib.suppress(subprocess.TimeoutExpired):
+            process.wait(timeout=10)
+
+    # Provider terminal tools create their own sessions, so killing only the
+    # native CLI's process group leaves test/build subprocesses orphaned. The
+    # identity check prevents a reused PID from targeting an unrelated worker.
+    _signal_worker_tree(process.pid, descendants, signal.SIGKILL)
+    if process.poll() is None:
+        process.wait(timeout=10)
+
+def _base_env() -> dict[str, str]:
+    """Build a worker environment while preserving Vault-backed CLI homes."""
+    env = os.environ.copy()
+    # Subscription CLIs only: a vendor API key in the environment could
+    # silently flip a worker onto a metered path, so drop every key here.
+    for secret in ("ANTHROPIC_API_KEY", "CLAUDE_API_KEY", "OPENAI_API_KEY", "API_SERVER_KEY"):
+        env.pop(secret, None)
+    env.update(
+        {
+            "HOME": str(DATA_ROOT / "home"),
+            "CODEX_HOME": os.environ.get("CODEX_HOME", "/runtime-access/codex"),
+            "CLAUDE_CONFIG_DIR": os.environ.get(
+                "CLAUDE_CONFIG_DIR", "/runtime-access/claude"
+            ),
+            "GIT_TERMINAL_PROMPT": "0",
+        }
+    )
+    env["PATH"] = f"{DATA_ROOT / 'tools/bin'}:/opt/coordinator:" + env.get("PATH", "")
+    return env
+
+def _codex_command(route: Route, prompt: str, workspace: Path, state: dict[str, Any], result_file: Path) -> list[str]:
+    base = [
+        str(CODEX_BIN),
+        "--dangerously-bypass-approvals-and-sandbox",
+        "--dangerously-bypass-hook-trust",
+        "exec",
+    ]
+    thread_id = str(state.get("codex_thread_id") or "")
+    common = [
+        "--json",
+        "-m",
+        route.model,
+        "-c",
+        f'model_reasoning_effort="{route.effort}"',
+        "--output-schema",
+        str(RESULT_SCHEMA_PATH),
+        "-o",
+        str(result_file),
+    ]
+    if thread_id:
+        return [*base, "resume", *common, thread_id, prompt]
+    return [
+        *base,
+        *common,
+        "-C",
+        str(workspace),
+        prompt,
+    ]
+
+def _claude_command(route: Route, prompt: str, state: dict[str, Any], resume: bool) -> list[str]:
+    session_id = str(state["claude_session_id"])
+    session = ["--resume", session_id] if resume else ["--session-id", session_id]
+    denied = [
+        "Bash(git push --force *)",
+        "Bash(git reset --hard *)",
+        "Bash(git clean -f *)",
+    ]
+    return [
+        str(CLAUDE_BIN),
+        "--dangerously-skip-permissions",
+        "--autocompact",
+        "auto",
+        "--settings",
+        str(CLAUDE_SETTINGS),
+        "--disallowedTools",
+        *denied,
+        "--model",
+        route.model,
+        "--effort",
+        route.effort,
+        "--output-format",
+        "stream-json",
+        "--verbose",
+        "--json-schema",
+        json.dumps(RESULT_SCHEMA, separators=(",", ":")),
+        *session,
+        "-p",
+        prompt,
+    ]
+
+def run_provider(
+    route: Route,
+    prompt: str,
+    workspace: Path,
+    state: dict[str, Any],
+    state_file: Path,
+    log_path: Path,
+    heartbeat: Callable[[str], bool],
+    max_runtime: int,
+) -> ProcessResult:
+    """Run or resume one provider session using its pinned structured CLI."""
+    state.setdefault("attempts", []).append({"route": asdict(route), "started_at": utc_now()})
+    if route.provider == "claude" and not state.get("claude_session_id"):
+        # The reservation is durable before Claude starts, closing the crash gap.
+        state["claude_session_id"] = str(uuid.uuid4())
+    state["current_route"] = asdict(route)
+    state["updated_at"] = utc_now()
+    env = _base_env()
+    if route.provider == "codex":
+        result_sequence = max(0, int(state.get("result_sequence", 0) or 0)) + 1
+        result_file = _result_path(state_file, state.get("run_id"), result_sequence)
+        while result_file.exists():
+            result_sequence += 1
+            result_file = _result_path(
+                state_file,
+                state.get("run_id"),
+                result_sequence,
+            )
+        result_file.parent.mkdir(parents=True, exist_ok=True)
+        result_file.touch(mode=0o600, exist_ok=False)
+        state["result_sequence"] = result_sequence
+        state["current_result_file"] = str(result_file)
+        atomic_json(state_file, state)
+        command = _codex_command(route, prompt, workspace, state, result_file)
+        result = stream_process(
+            command,
+            provider="codex",
+            cwd=workspace,
+            env=env,
+            log_path=log_path,
+            state=state,
+            state_file=state_file,
+            heartbeat=heartbeat,
+            max_runtime=max_runtime,
+        )
+        if (
+            state.get("codex_thread_id")
+            and result.returncode != 0
+            and NO_CODEX_THREAD in result.output.lower()
+        ):
+            state.pop("codex_thread_id", None)
+            result_sequence += 1
+            result_file = _result_path(
+                state_file,
+                state.get("run_id"),
+                result_sequence,
+            )
+            while result_file.exists():
+                result_sequence += 1
+                result_file = _result_path(
+                    state_file,
+                    state.get("run_id"),
+                    result_sequence,
+                )
+            result_file.touch(mode=0o600, exist_ok=False)
+            state["result_sequence"] = result_sequence
+            state["current_result_file"] = str(result_file)
+            atomic_json(state_file, state)
+            result = stream_process(
+                _codex_command(route, prompt, workspace, state, result_file),
+                provider="codex",
+                cwd=workspace,
+                env=env,
+                log_path=log_path,
+                state=state,
+                state_file=state_file,
+                heartbeat=heartbeat,
+                max_runtime=max_runtime,
+            )
+        file_result = load_json(result_file)
+        if file_result.get("status") in cli_lane_goal.RESULT_STATUSES:
+            result.structured = file_result
+            with contextlib.suppress(OSError):
+                result_file.chmod(0o600)
+        return result
+
+    atomic_json(state_file, state)
+    resume = bool(state.get("claude_started"))
+    result = stream_process(
+        _claude_command(route, prompt, state, resume),
+        provider="claude",
+        cwd=workspace,
+        env=env,
+        log_path=log_path,
+        state=state,
+        state_file=state_file,
+        heartbeat=heartbeat,
+        max_runtime=max_runtime,
+    )
+    if resume and result.returncode == 1 and NO_CLAUDE_SESSION in result.output:
+        result = stream_process(
+            _claude_command(route, prompt, state, False),
+            provider="claude",
+            cwd=workspace,
+            env=env,
+            log_path=log_path,
+            state=state,
+            state_file=state_file,
+            heartbeat=heartbeat,
+            max_runtime=max_runtime,
+        )
+    elif not resume and result.returncode == 1 and CLAUDE_SESSION_COLLISION in result.output:
+        result = stream_process(
+            _claude_command(route, prompt, state, True),
+            provider="claude",
+            cwd=workspace,
+            env=env,
+            log_path=log_path,
+            state=state,
+            state_file=state_file,
+            heartbeat=heartbeat,
+            max_runtime=max_runtime,
+        )
+    if result.returncode == 0 or NO_CLAUDE_SESSION not in result.output:
+        state["claude_started"] = True
+        atomic_json(state_file, state)
+    return result

services/hermes/scripts/cli_lane_quarantine.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+"""Bounded diagnostic quarantine for malformed terminal journals."""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import json
+import os
+import re
+import stat
+import sys
+from pathlib import Path
+
+from cli_lane_config import QUARANTINE_HASH_BYTES, STATE_ROOT, TerminalIdentity, TerminalRecoverySnapshot
+from cli_lane_evidence import _retire_terminal_entry
+from cli_lane_records import _read_bounded
+
+
+def _quarantine_terminal(
+    path: Path,
+    identity: TerminalIdentity | None,
+    reason: str,
+    *,
+    snapshot: TerminalRecoverySnapshot | None = None,
+) -> Path:
+    """Record bounded metadata, then retire only the inspected journal inode."""
+    board_dir = path.parent
+    try:
+        board_stat = board_dir.stat(follow_symlinks=False)
+    except OSError:
+        board_stat = None
+    if board_stat is None or not stat.S_ISDIR(board_stat.st_mode):
+        print(
+            f"refused terminal journal path through unsafe board directory: {path}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return path
+    board_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    board_flags |= getattr(os, "O_NOFOLLOW", 0)
+    file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    create_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    create_flags |= getattr(os, "O_NOFOLLOW", 0)
+    board_descriptor = None
+    quarantine_descriptor = None
+    source_descriptor = None
+    source_stat = None
+    destination = path
+    retirement = "not-attempted"
+    owns_source = snapshot is None
+    try:
+        relative = path.relative_to(STATE_ROOT)
+    except ValueError:
+        relative = Path()
+    already_quarantined = (
+        len(relative.parts) == 3 and relative.parts[1] == "quarantine"
+    )
+    try:
+        if snapshot is None:
+            board_descriptor = os.open(board_dir, board_flags)
+            source_stat = os.stat(
+                path.name,
+                dir_fd=board_descriptor,
+                follow_symlinks=False,
+            )
+        else:
+            board_descriptor = snapshot.directory_descriptor
+            source_descriptor = snapshot.descriptor
+            source_stat = snapshot.file_stat
+            pinned_board = os.fstat(board_descriptor)
+            if (
+                pinned_board.st_dev != board_stat.st_dev
+                or pinned_board.st_ino != board_stat.st_ino
+            ):
+                raise OSError("terminal recovery board identity changed")
+        safe_source = stat.S_ISREG(source_stat.st_mode) and source_stat.st_nlink == 1
+        digest = None
+        hashed_bytes = 0
+        hash_complete = False
+        if safe_source and snapshot is None:
+            source_descriptor = os.open(
+                path.name,
+                file_flags,
+                dir_fd=board_descriptor,
+            )
+            opened_stat = os.fstat(source_descriptor)
+            if (
+                opened_stat.st_dev != source_stat.st_dev
+                or opened_stat.st_ino != source_stat.st_ino
+                or opened_stat.st_nlink != 1
+            ):
+                raise OSError("terminal journal identity changed while opening")
+            # Keep this descriptor open through retirement. The staging entry
+            # must match the exact fstat-verified inode, not merely a path stat
+            # captured before a concurrent replacement.
+            source_stat = opened_stat
+            prefix = _read_bounded(source_descriptor, QUARANTINE_HASH_BYTES)
+            hashed_bytes = len(prefix)
+            hash_complete = opened_stat.st_size <= QUARANTINE_HASH_BYTES
+            digest = hashlib.sha256(prefix).hexdigest()
+        elif safe_source and source_descriptor is not None:
+            prefix = snapshot.prefix
+            hashed_bytes = len(prefix)
+            hash_complete = (
+                source_stat.st_size <= QUARANTINE_HASH_BYTES
+                and hashed_bytes == source_stat.st_size
+            )
+            digest = hashlib.sha256(prefix).hexdigest()
+        source_kind = (
+            "regular"
+            if safe_source
+            else "hardlink"
+            if stat.S_ISREG(source_stat.st_mode) and source_stat.st_nlink != 1
+            else "non-regular"
+        )
+        diagnostic = {
+            "hash_complete": hash_complete,
+            "hashed_bytes": hashed_bytes,
+            "original_name": path.name,
+            "reason": reason,
+            "sha256": digest,
+            "size": source_stat.st_size,
+            "source_kind": source_kind,
+        }
+        payload = (json.dumps(diagnostic, sort_keys=True) + "\n").encode("utf-8")
+        fingerprint = (
+            (digest or hashlib.sha256(path.name.encode("utf-8")).hexdigest())[:16]
+        )
+        safe_reason = (
+            re.sub(r"[^a-zA-Z0-9_.-]+", "-", reason).strip("-") or "invalid"
+        )
+        if already_quarantined:
+            # A staged artifact already lives in the board's one bounded
+            # quarantine directory. Reuse it so adversarial replacements can
+            # never drive recursive quarantine/quarantine/... authority.
+            quarantine_descriptor = os.dup(board_descriptor)
+            quarantine_dir = board_dir
+        else:
+            with contextlib.suppress(FileExistsError):
+                os.mkdir("quarantine", 0o700, dir_fd=board_descriptor)
+            quarantine_descriptor = os.open(
+                "quarantine",
+                board_flags,
+                dir_fd=board_descriptor,
+            )
+            quarantine_dir = board_dir / "quarantine"
+        for sequence in range(32):
+            name = f"{path.name}.{safe_reason}.{fingerprint}.{sequence}.quarantine"
+            try:
+                destination_descriptor = os.open(
+                    name,
+                    create_flags,
+                    0o600,
+                    dir_fd=quarantine_descriptor,
+                )
+            except FileExistsError:
+                continue
+            try:
+                with os.fdopen(destination_descriptor, "wb") as target:
+                    target.write(payload)
+                    target.flush()
+                    os.fsync(target.fileno())
+                    target_stat = os.fstat(target.fileno())
+                    if (
+                        stat.S_IMODE(target_stat.st_mode) != 0o600
+                        or target_stat.st_nlink != 1
+                    ):
+                        raise OSError("quarantine destination is not private and singly linked")
+            except Exception:
+                os.unlink(name, dir_fd=quarantine_descriptor)
+                raise
+            destination = quarantine_dir / name
+            break
+        else:
+            raise OSError("could not reserve a unique quarantine destination")
+        os.fsync(quarantine_descriptor)
+        retirement = _retire_terminal_entry(
+            path,
+            source_stat,
+            board_descriptor=board_descriptor,
+            quarantine_descriptor=quarantine_descriptor,
+            source_descriptor=source_descriptor,
+        )
+    except OSError as error:
+        # A bad quarantine target must not make an attacker-controlled pending
+        # path eligible for replay forever. The same identity-safe staging
+        # protocol works in the board directory when quarantine is unusable.
+        if board_descriptor is not None and source_stat is not None:
+            try:
+                retirement = _retire_terminal_entry(
+                    path,
+                    source_stat,
+                    board_descriptor=board_descriptor,
+                    quarantine_descriptor=(
+                        quarantine_descriptor
+                        if quarantine_descriptor is not None
+                        else board_descriptor
+                    ),
+                    source_descriptor=source_descriptor,
+                )
+            except OSError:
+                retirement = "deferred"
+        print(
+            f"terminal journal quarantine degraded safely: {type(error).__name__}: {error}",
+            file=sys.stderr,
+            flush=True,
+        )
+    finally:
+        if owns_source and source_descriptor is not None:
+            os.close(source_descriptor)
+        if quarantine_descriptor is not None:
+            os.close(quarantine_descriptor)
+        if owns_source and board_descriptor is not None:
+            os.close(board_descriptor)
+    print(
+        "quarantined terminal journal "
+        f"{path.name}: {reason}; identity={identity or 'unparseable'}; "
+        f"retirement={retirement}",
+        file=sys.stderr,
+        flush=True,
+    )
+    return destination

services/hermes/scripts/cli_lane_quota.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+"""Quota-aware soft routing for automatic CLI lane provider selection."""
+
+from __future__ import annotations
+
+import re
+import sys
+import urllib.request
+from dataclasses import dataclass
+from typing import Callable
+
+from cli_lane_config import (
+    QUOTA_METRICS_URL,
+    QUOTA_MIN_REMAINING_PERCENT_DEFAULT,
+    kanban_setting,
+)
+from cli_lane_metrics import (
+    record_provider_quota,
+    record_quota_fetch_failure,
+    record_soft_exclusion,
+)
+from cli_lane_routing import fresh_unavailable_provider
+
+
+# Exporter accounts are labeled by vendor; the lane routes by CLI provider.
+QUOTA_PROVIDER_LABELS = {"openai": "codex", "anthropic": "claude"}
+# Gate only on account-wide windows: a model-specific Claude window (for
+# example seven_day_opus) must not veto the whole provider.
+GATED_QUOTA_WINDOWS = {
+    "codex": None,
+    "claude": ("five_hour", "seven_day"),
+}
+_SAMPLE_PATTERN = re.compile(
+    r"^atlas_ai_quota_(remaining_percent|reset_timestamp_seconds|fetch_success)"
+    r"\{([^}]*)\} (-?[0-9.eE+]+)$"
+)
+_LABEL_PATTERN = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="([^"]*)"')
+
+
+@dataclass(frozen=True)
+class ProviderQuota:
+    """One provider's binding remaining quota as seen by the lane."""
+
+    remaining_percent: float
+    reset_timestamp: float | None
+
+
+@dataclass(frozen=True)
+class SelectionConstraint:
+    """One provider exclusion decided at a routing boundary, with evidence."""
+
+    exclude_provider: str | None
+    exclude_reason: str | None
+    source: str | None
+    notes: tuple[str, ...]
+
+
+def _gated_window(provider: str, labels: dict[str, str]) -> str | None:
+    """Return the window key when a sample belongs to a gated account limit."""
+    if labels.get("limit") != provider:
+        return None
+    window = labels.get("window", "")
+    allowed = GATED_QUOTA_WINDOWS[provider]
+    if allowed is not None and window not in allowed:
+        return None
+    return window
+
+
+def parse_quota_metrics(text: str) -> dict[str, ProviderQuota]:
+    """Reduce exporter samples to each provider's tightest account window."""
+    remaining: dict[tuple[str, str], float] = {}
+    resets: dict[tuple[str, str], float] = {}
+    fetch_success: dict[str, bool] = {}
+    for line in text.splitlines():
+        match = _SAMPLE_PATTERN.match(line.strip())
+        if not match:
+            continue
+        metric, raw_labels, raw_value = match.groups()
+        labels = dict(_LABEL_PATTERN.findall(raw_labels))
+        provider = QUOTA_PROVIDER_LABELS.get(labels.get("provider", ""))
+        if provider is None:
+            continue
+        value = float(raw_value)
+        if metric == "fetch_success":
+            fetch_success[provider] = value >= 1.0
+            continue
+        window = _gated_window(provider, labels)
+        if window is None:
+            continue
+        if metric == "remaining_percent":
+            remaining[(provider, window)] = value
+        else:
+            resets[(provider, window)] = value
+    snapshot: dict[str, ProviderQuota] = {}
+    for provider in QUOTA_PROVIDER_LABELS.values():
+        windows = {
+            window: value
+            for (candidate, window), value in remaining.items()
+            if candidate == provider
+        }
+        if not windows or not fetch_success.get(provider, True):
+            # Failed or stale exporter fetches leave this provider ungated.
+            continue
+        binding = min(windows, key=lambda window: (windows[window], window))
+        snapshot[provider] = ProviderQuota(
+            remaining_percent=windows[binding],
+            reset_timestamp=resets.get((provider, binding)),
+        )
+    return snapshot
+
+
+def fetch_quota_snapshot(
+    url: str = QUOTA_METRICS_URL,
+    open_url: Callable = urllib.request.urlopen,
+) -> dict[str, ProviderQuota]:
+    """Read the in-pod usage exporter; empty means the signal is unavailable."""
+    try:
+        with open_url(url, timeout=10) as response:
+            text = response.read(1 << 20).decode("utf-8", errors="replace")
+        snapshot = parse_quota_metrics(text)
+    except (OSError, ValueError) as error:
+        record_quota_fetch_failure()
+        print(
+            f"quota snapshot unavailable; routing fails open: {type(error).__name__}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return {}
+    if not snapshot:
+        record_quota_fetch_failure()
+        print(
+            "quota snapshot carried no gated windows; routing fails open",
+            file=sys.stderr,
+            flush=True,
+        )
+    return snapshot
+
+
+def quota_soft_exclusion(
+    snapshot: dict[str, ProviderQuota], threshold: float
+) -> tuple[str | None, str | None]:
+    """Choose at most one provider to soft-exclude from new automatic work."""
+    below = {
+        provider: quota.remaining_percent
+        for provider, quota in snapshot.items()
+        if quota.remaining_percent < threshold
+    }
+    if not below:
+        return None, None
+    excluded = min(below, key=lambda provider: (below[provider], provider))
+    if len(below) == len(snapshot) and len(snapshot) > 1:
+        preferred = next(name for name in snapshot if name != excluded)
+        note = (
+            "Quota degraded: every provider is below the "
+            f"{threshold:.6g}% remaining threshold; preferring {preferred} "
+            f"({snapshot[preferred].remaining_percent:.6g}% left) over "
+            f"{excluded} ({below[excluded]:.6g}% left) for new auto work."
+        )
+        return excluded, note
+    note = (
+        f"Quota guard: {excluded} has {below[excluded]:.6g}% remaining "
+        f"(threshold {threshold:.6g}%); routing new auto work to the other "
+        "provider while active work finishes."
+    )
+    return excluded, note
+
+
+def selection_constraint(
+    assignee: str,
+    *,
+    snapshot_from: Callable[[], dict[str, ProviderQuota]] | None = None,
+    now: float | None = None,
+) -> SelectionConstraint:
+    """Combine provider health and quota gating for one selection boundary."""
+    if assignee != "cli-auto":
+        # Explicit lanes are operator decisions; they are never re-routed.
+        return SelectionConstraint(None, None, None, ())
+    snapshot = (snapshot_from or fetch_quota_snapshot)()
+    resets: dict[str, float] = {}
+    for provider, quota in snapshot.items():
+        record_provider_quota(provider, quota.remaining_percent, quota.reset_timestamp)
+        if quota.reset_timestamp is not None:
+            resets[provider] = quota.reset_timestamp
+    health_excluded = fresh_unavailable_provider(now=now, quota_resets=resets)
+    if health_excluded is not None:
+        return SelectionConstraint(
+            health_excluded,
+            "is unavailable according to fresh native health",
+            "health",
+            (),
+        )
+    threshold = kanban_setting(
+        "provider_quota_min_remaining_percent", QUOTA_MIN_REMAINING_PERCENT_DEFAULT
+    )
+    excluded, note = quota_soft_exclusion(snapshot, threshold)
+    for provider in snapshot:
+        record_soft_exclusion(provider, provider == excluded)
+    if excluded is None or note is None:
+        return SelectionConstraint(None, None, None, ())
+    return SelectionConstraint(
+        excluded,
+        "is below its remaining-quota routing threshold",
+        "quota",
+        (note,),
+    )

services/hermes/scripts/cli_lane_records.py

@@ -0,0 +1,356 @@
+#!/usr/bin/env python3
+"""Bounded inode-pinned reads and validation for Hermes terminal records."""
+
+from __future__ import annotations
+
+import json
+import os
+import stat
+from pathlib import Path
+from typing import Any
+
+import cli_lane_goal
+from cli_lane_config import (
+    MAX_TERMINAL_RECORD_BYTES,
+    QUARANTINE_HASH_BYTES,
+    RESULT_SCHEMA,
+    Route,
+    TerminalIdentity,
+    TerminalRecoverySnapshot,
+    TerminalSnapshot,
+    canonical_run_id,
+    utc_now,
+)
+from cli_lane_files import (
+    _candidate_path,
+    _terminal_identity,
+    _terminal_path,
+    atomic_json,
+)
+
+
+def _read_bounded(descriptor: int, limit: int) -> bytes:
+    """Read at most ``limit`` bytes from a regular file descriptor."""
+    chunks: list[bytes] = []
+    remaining = max(0, limit)
+    while remaining:
+        chunk = os.read(descriptor, min(64 * 1024, remaining))
+        if not chunk:
+            break
+        chunks.append(chunk)
+        remaining -= len(chunk)
+    return b"".join(chunks)
+
+def _open_small_json_snapshot(path: Path) -> TerminalSnapshot | None:
+    """Open and pin one bounded, singly-linked JSON document."""
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
+    file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    directory = None
+    descriptor = None
+    try:
+        directory = os.open(path.parent, directory_flags)
+        descriptor = os.open(path.name, file_flags, dir_fd=directory)
+        opened = os.fstat(descriptor)
+        if (
+            not stat.S_ISREG(opened.st_mode)
+            or opened.st_nlink != 1
+            or opened.st_size > MAX_TERMINAL_RECORD_BYTES
+        ):
+            return None
+        payload = _read_bounded(descriptor, MAX_TERMINAL_RECORD_BYTES + 1)
+        if len(payload) > MAX_TERMINAL_RECORD_BYTES:
+            return None
+        current = os.stat(
+            path.name,
+            dir_fd=directory,
+            follow_symlinks=False,
+        )
+        if (
+            current.st_dev != opened.st_dev
+            or current.st_ino != opened.st_ino
+            or current.st_size != opened.st_size
+            or len(payload) != opened.st_size
+        ):
+            return None
+        value = json.loads(payload.decode("utf-8"))
+        if not isinstance(value, dict):
+            return None
+        snapshot = TerminalSnapshot(value, opened, descriptor, directory)
+        descriptor = None
+        directory = None
+        return snapshot
+    except (OSError, UnicodeError, json.JSONDecodeError):
+        return None
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        if directory is not None:
+            os.close(directory)
+
+def _open_terminal_recovery_snapshot(path: Path) -> TerminalRecoverySnapshot | None:
+    """Pin the entry recovery inspected, even when its payload is malformed."""
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
+    file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    directory = None
+    descriptor = None
+    try:
+        directory = os.open(path.parent, directory_flags)
+        observed = os.stat(
+            path.name,
+            dir_fd=directory,
+            follow_symlinks=False,
+        )
+        if not stat.S_ISREG(observed.st_mode):
+            snapshot = TerminalRecoverySnapshot(
+                None,
+                observed,
+                None,
+                directory,
+                b"",
+                "non-regular",
+            )
+            directory = None
+            return snapshot
+        try:
+            descriptor = os.open(path.name, file_flags, dir_fd=directory)
+        except OSError:
+            snapshot = TerminalRecoverySnapshot(
+                None,
+                observed,
+                None,
+                directory,
+                b"",
+                "open-failed",
+            )
+            directory = None
+            return snapshot
+        opened = os.fstat(descriptor)
+        if opened.st_dev != observed.st_dev or opened.st_ino != observed.st_ino:
+            os.close(descriptor)
+            descriptor = None
+            snapshot = TerminalRecoverySnapshot(
+                None,
+                observed,
+                None,
+                directory,
+                b"",
+                "identity-changed-during-open",
+            )
+            directory = None
+            return snapshot
+        observed = opened
+        if observed.st_nlink != 1:
+            snapshot = TerminalRecoverySnapshot(
+                None,
+                observed,
+                descriptor,
+                directory,
+                b"",
+                "hardlinked",
+            )
+            descriptor = None
+            directory = None
+            return snapshot
+        if observed.st_size > MAX_TERMINAL_RECORD_BYTES:
+            prefix = _read_bounded(descriptor, QUARANTINE_HASH_BYTES)
+            snapshot = TerminalRecoverySnapshot(
+                None,
+                observed,
+                descriptor,
+                directory,
+                prefix,
+                "oversized",
+            )
+            descriptor = None
+            directory = None
+            return snapshot
+        payload = _read_bounded(descriptor, MAX_TERMINAL_RECORD_BYTES + 1)
+        after_read = os.fstat(descriptor)
+        unchanged = (
+            after_read.st_dev == observed.st_dev
+            and after_read.st_ino == observed.st_ino
+            and after_read.st_size == observed.st_size
+            and after_read.st_mtime_ns == observed.st_mtime_ns
+            and after_read.st_ctime_ns == observed.st_ctime_ns
+            and len(payload) == observed.st_size
+        )
+        document = None
+        invalid_reason = "unstable-payload"
+        if unchanged:
+            try:
+                parsed = json.loads(payload.decode("utf-8"))
+                if isinstance(parsed, dict):
+                    document = parsed
+                    invalid_reason = None
+                else:
+                    invalid_reason = "non-object-payload"
+            except (UnicodeError, json.JSONDecodeError):
+                invalid_reason = "malformed-payload"
+        snapshot = TerminalRecoverySnapshot(
+            document,
+            after_read,
+            descriptor,
+            directory,
+            payload[:QUARANTINE_HASH_BYTES],
+            invalid_reason,
+        )
+        descriptor = None
+        directory = None
+        return snapshot
+    except OSError:
+        return None
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        if directory is not None:
+            os.close(directory)
+
+def _open_terminal_snapshot(
+    path: Path,
+    identity: TerminalIdentity,
+) -> TerminalSnapshot | None:
+    """Open a terminal journal only after its lexical identity is verified."""
+    if _terminal_identity(path) != identity:
+        return None
+    return _open_small_json_snapshot(path)
+
+def _load_terminal_json(path: Path, identity: TerminalIdentity) -> dict[str, Any]:
+    """Read one small, singly-linked journal through its validated directory."""
+    snapshot = _open_terminal_snapshot(path, identity)
+    if snapshot is None:
+        return {}
+    try:
+        return snapshot.document
+    finally:
+        snapshot.close()
+
+def _load_small_json(path: Path) -> dict[str, Any]:
+    """Read one bounded immutable evidence document without following links."""
+    snapshot = _open_small_json_snapshot(path)
+    if snapshot is None:
+        return {}
+    try:
+        return snapshot.document
+    finally:
+        snapshot.close()
+
+def _persist_candidate(
+    state: dict[str, Any],
+    state_file: Path,
+    structured: dict[str, Any],
+    *,
+    route: Route,
+    returncode: int,
+    goal_turn: int,
+) -> Path:
+    """Persist every structured response before any judge can supersede it."""
+    sequence = max(0, int(state.get("candidate_sequence", 0) or 0)) + 1
+    path = _candidate_path(state_file, state.get("run_id"), sequence)
+    while path.exists():
+        sequence += 1
+        path = _candidate_path(state_file, state.get("run_id"), sequence)
+    atomic_json(
+        path,
+        {
+            "board": state.get("board"),
+            "task_id": state.get("task_id"),
+            "expected_run_id": state.get("run_id"),
+            "goal_turn": goal_turn,
+            "provider": route.provider,
+            "model": route.model,
+            "effort": route.effort,
+            "returncode": returncode,
+            "structured": structured,
+            "recorded_at": utc_now(),
+        },
+    )
+    state["candidate_sequence"] = sequence
+    state["last_candidate_file"] = str(path)
+    atomic_json(state_file, state)
+    return path
+
+def _write_terminal_record(
+    state_file: Path,
+    *,
+    board: str,
+    task_id: str,
+    run_id: Any,
+    structured: dict[str, Any],
+    summary: str,
+    metadata: dict[str, Any],
+) -> tuple[Path, dict[str, Any]]:
+    """Journal an accepted result before attempting the Kanban transaction."""
+    if type(run_id) is not int or canonical_run_id(run_id) is None:
+        raise ValueError("terminal run ID must be a positive SQLite int64")
+    path = _terminal_path(state_file, run_id)
+    record = {
+        "board": board,
+        "task_id": task_id,
+        "expected_run_id": run_id,
+        "result": json.dumps(structured, sort_keys=True),
+        "summary": summary,
+        "metadata": metadata,
+        "kanban_state": "pending",
+        "recorded_at": utc_now(),
+    }
+    atomic_json(path, record)
+    return path, record
+
+def _terminal_record_valid(
+    record: dict[str, Any],
+    identity: TerminalIdentity | None = None,
+) -> bool:
+    """Reject malformed or non-terminal replay journals without side effects."""
+    if not isinstance(record, dict):
+        return False
+    required_strings = ("board", "task_id", "result", "summary")
+    if not all(
+        isinstance(record.get(key), str) and record[key]
+        for key in required_strings
+    ):
+        return False
+    expected_run_id = record.get("expected_run_id")
+    if (
+        type(expected_run_id) is not int
+        or canonical_run_id(expected_run_id) is None
+    ):
+        return False
+    if not isinstance(record.get("metadata"), dict):
+        return False
+    try:
+        structured = json.loads(record["result"])
+    except (TypeError, json.JSONDecodeError):
+        return False
+    if not isinstance(structured, dict):
+        return False
+    required = set(RESULT_SCHEMA["required"])
+    if set(structured) != required:
+        return False
+    if (
+        type(structured["status"]) is not str
+        or structured["status"] not in cli_lane_goal.RESULT_STATUSES
+        or type(structured["summary"]) is not str
+        or not structured["summary"].strip()
+    ):
+        return False
+    for key in ("changed_files", "tests_run", "artifacts", "findings", "blockers"):
+        value = structured[key]
+        if type(value) is not list or any(type(item) is not str for item in value):
+            return False
+    valid = (
+        structured["status"] == "completed"
+        and structured["blockers"] == []
+        and cli_lane_goal.unfinished_result_reason(structured) is None
+        and record["summary"] == structured["summary"]
+        and record.get("kanban_state") in {"pending", "prepared", "committed"}
+    )
+    if not valid or identity is None:
+        return valid
+    return (
+        record.get("board") == identity.board
+        and record.get("task_id") == identity.task_id
+        and record.get("expected_run_id") == identity.run_id
+        and record.get("kanban_state") == identity.state
+    )

services/hermes/scripts/cli_lane_recovery.py

@@ -0,0 +1,460 @@
+#!/usr/bin/env python3
+"""Bounded replay of pending, prepared, and staged terminal journals."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+from cli_lane_board import _record_board_access_error
+from cli_lane_config import (
+    STATE_ROOT,
+    TerminalIdentity,
+    TerminalRecoverySnapshot,
+    canonical_run_id,
+)
+from cli_lane_evidence import (
+    _persist_conflict_evidence,
+    _persist_prepared_evidence,
+    _same_terminal_document,
+    _terminal_evidence_path,
+    _terminal_evidence_valid,
+    _write_json_noreplace,
+)
+from cli_lane_files import (
+    _LEGACY_RETIRE_NAME,
+    _RETIRE_NAME,
+    _SAFE_BOARD,
+    _terminal_evidence_identity,
+    _terminal_identity,
+    _terminal_path,
+    state_path,
+)
+from cli_lane_finalization import (
+    _finalize_document_db,
+    _finalize_terminal_record,
+    _promote_prepared_evidence,
+    _recover_exact_run,
+    _resolve_pending_after_winner,
+    _retire_snapshot,
+    _retire_snapshot_after_db,
+)
+from cli_lane_quarantine import _quarantine_terminal
+from cli_lane_records import (
+    _load_small_json,
+    _load_terminal_json,
+    _open_terminal_recovery_snapshot,
+    _terminal_record_valid,
+)
+
+
+def _terminal_entry_absent(path: Path) -> bool:
+    """Confirm absence through a nofollow directory descriptor."""
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
+    directory = None
+    try:
+        directory = os.open(path.parent, directory_flags)
+        os.stat(path.name, dir_fd=directory, follow_symlinks=False)
+        return False
+    except FileNotFoundError:
+        return True
+    except OSError:
+        return False
+    finally:
+        if directory is not None:
+            os.close(directory)
+
+def _staged_terminal_authority(
+    path: Path,
+    snapshot: TerminalRecoverySnapshot,
+) -> tuple[TerminalIdentity, Path] | None:
+    """Bind a staged inode to the canonical pending name it was moved from."""
+    try:
+        relative = path.relative_to(STATE_ROOT)
+    except ValueError:
+        return None
+    if len(relative.parts) == 2:
+        board, filename = relative.parts
+    elif len(relative.parts) == 3 and relative.parts[1] == "quarantine":
+        board, _quarantine, filename = relative.parts
+    else:
+        return None
+    match = _RETIRE_NAME.fullmatch(filename)
+    legacy = _LEGACY_RETIRE_NAME.fullmatch(filename)
+    document = snapshot.document
+    if (
+        (match is None and legacy is None)
+        or document is None
+        or not _SAFE_BOARD.fullmatch(board)
+        or board in {".", ".."}
+        or document.get("board") != board
+        or not isinstance(document.get("task_id"), str)
+        or type(document.get("expected_run_id")) is not int
+        or canonical_run_id(document.get("expected_run_id")) is None
+    ):
+        return None
+    state = document.get("kanban_state")
+    if state not in {"pending", "prepared", "committed", "conflict"}:
+        return None
+    identity = TerminalIdentity(
+        board, document["task_id"], document["expected_run_id"], state
+    )
+    if state in {"pending", "committed"}:
+        canonical = _terminal_path(
+            state_path(identity.board, identity.task_id),
+            identity.run_id,
+            state,
+        )
+        valid = _terminal_record_valid(document, identity)
+    else:
+        canonical = _terminal_evidence_path(identity, state, document)
+        valid = _terminal_evidence_valid(document, identity, state)
+    board_dir = STATE_ROOT / board
+    if (
+        canonical.parent != board_dir
+        or path.parent not in {board_dir, board_dir / "quarantine"}
+        or (
+            match is not None
+            and hashlib.sha256(canonical.name.encode("utf-8")).hexdigest()[:16]
+            != match.group("path_digest")
+        )
+        or not valid
+    ):
+        return None
+    return identity, canonical
+
+def _retirement_staging_paths() -> list[Path]:
+    """List every directory where identity-safe retirement can stage an inode."""
+    paths = list(STATE_ROOT.glob("*/.retire.*"))
+    paths.extend(STATE_ROOT.glob("*/quarantine/.retire.*"))
+    return sorted(set(paths))
+
+def _restore_staged_terminal_authority(
+    identity: TerminalIdentity,
+    canonical: Path,
+    document: dict[str, Any],
+) -> None:
+    """Restore one hidden accepted artifact without executing it prematurely."""
+    if identity.state == "pending":
+        _persist_prepared_evidence(identity, document)
+        return
+    if _write_json_noreplace(canonical, document):
+        return
+    existing = _load_small_json(canonical)
+    if identity.state in {"prepared", "conflict"}:
+        valid = _terminal_evidence_valid(existing, identity, identity.state)
+    else:
+        valid = _terminal_record_valid(existing, identity)
+    if valid and _same_terminal_document(existing, document):
+        return
+    if identity.state == "committed":
+        _persist_conflict_evidence(
+            identity,
+            document,
+            "staged committed evidence conflicts with the canonical first writer",
+        )
+        return
+    raise OSError("staged terminal evidence path contains a conflicting result")
+
+def _recover_retirement_staging() -> int:
+    """Promote valid hidden journals to durable prepared evidence."""
+    recovered = 0
+    for path in _retirement_staging_paths():
+        snapshot = _open_terminal_recovery_snapshot(path)
+        if snapshot is None:
+            continue
+        authority = _staged_terminal_authority(path, snapshot)
+        if authority is None:
+            try:
+                _quarantine_terminal(
+                    path,
+                    None,
+                    "untrusted-retirement-staging",
+                    snapshot=snapshot,
+                )
+            finally:
+                snapshot.close()
+            continue
+        identity, canonical = authority
+        try:
+            _restore_staged_terminal_authority(
+                identity,
+                canonical,
+                snapshot.document or {},
+            )
+            retirement = _retire_snapshot(
+                path,
+                snapshot,
+                authority_name=canonical.name,
+            )
+        except Exception as error:
+            _record_board_access_error(identity.board, error)
+            continue
+        finally:
+            snapshot.close()
+        if retirement in {"retired", "missing"}:
+            recovered += 1
+    return recovered
+
+def _drain_retirement_staging() -> int:
+    """Converge bounded replacement churn without spinning on storage failure."""
+    recovered = 0
+    previous: tuple[tuple[str, int, int, int, int], ...] | None = None
+    for _attempt in range(16):
+        surface = []
+        for path in _retirement_staging_paths():
+            try:
+                observed = path.stat(follow_symlinks=False)
+            except OSError:
+                continue
+            surface.append(
+                (
+                    str(path),
+                    observed.st_dev,
+                    observed.st_ino,
+                    observed.st_size,
+                    observed.st_mtime_ns,
+                )
+            )
+        signature = tuple(surface)
+        if not signature or signature == previous:
+            break
+        previous = signature
+        recovered += _recover_retirement_staging()
+    return recovered
+
+def _recover_prepared_finalizations(kanban_db: Any) -> int:
+    """Recover an accepted result staged before an interrupted DB boundary."""
+    recovered = 0
+    recovered_identities: set[tuple[str, str, int]] = set()
+    prepared_paths = list(STATE_ROOT.glob("*/*.terminal.prepared-*.json"))
+
+    def durable_order(path: Path) -> tuple[int, str]:
+        try:
+            return path.stat(follow_symlinks=False).st_mtime_ns, str(path)
+        except OSError:
+            return 2**63 - 1, str(path)
+
+    # When no DB winner exists yet, the first durably prepared result owns the
+    # run. The DB transaction remains the final arbiter across runner processes.
+    for path in sorted(prepared_paths, key=durable_order):
+        for _replacement_attempt in range(16):
+            identity = _terminal_evidence_identity(path)
+            snapshot = _open_terminal_recovery_snapshot(path)
+            if snapshot is None:
+                break
+            document = snapshot.document
+            if identity is None or identity.state != "prepared" or document is None:
+                try:
+                    _quarantine_terminal(
+                        path,
+                        identity,
+                        snapshot.invalid_reason or "malformed-prepared-evidence",
+                        snapshot=snapshot,
+                    )
+                finally:
+                    snapshot.close()
+                continue
+            expected = _terminal_evidence_path(identity, "prepared", document)
+            if (
+                expected.name != path.name
+                or not _terminal_evidence_valid(document, identity, "prepared")
+            ):
+                try:
+                    _quarantine_terminal(
+                        path,
+                        identity,
+                        "foreign-prepared-evidence",
+                        snapshot=snapshot,
+                    )
+                finally:
+                    snapshot.close()
+                continue
+            try:
+                outcome = _finalize_document_db(kanban_db, identity, document)
+            except Exception as error:
+                _record_board_access_error(identity.board, error)
+                snapshot.close()
+                break
+            try:
+                if outcome == "committed":
+                    _promote_prepared_evidence(
+                        identity,
+                        document,
+                        path,
+                        discard_prepared=False,
+                    )
+                    _retire_snapshot_after_db(path, snapshot)
+                    pending = _terminal_path(
+                        state_path(identity.board, identity.task_id),
+                        identity.run_id,
+                        "pending",
+                    )
+                    pending_identity = TerminalIdentity(
+                        identity.board,
+                        identity.task_id,
+                        identity.run_id,
+                        "pending",
+                    )
+                    _resolve_pending_after_winner(
+                        pending,
+                        pending_identity,
+                        document,
+                    )
+                    recovered_identity = (
+                        identity.board,
+                        identity.task_id,
+                        identity.run_id,
+                    )
+                    if recovered_identity not in recovered_identities:
+                        recovered_identities.add(recovered_identity)
+                        recovered += 1
+                elif outcome == "stale":
+                    _persist_conflict_evidence(
+                        identity,
+                        document,
+                        "prepared result no longer matches the authoritative task run",
+                    )
+                    _retire_snapshot_after_db(path, snapshot)
+                else:
+                    snapshot.close()
+                    break
+            except Exception as error:
+                _record_board_access_error(identity.board, error)
+                snapshot.close()
+                break
+            snapshot.close()
+    return recovered
+
+def recover_pending_finalizations() -> int:
+    """Replay accepted exact-run results before scheduling more provider work."""
+    from hermes_cli import kanban_db
+
+    _drain_retirement_staging()
+    recovered = _recover_prepared_finalizations(kanban_db)
+    for path in sorted(STATE_ROOT.glob("*/*.terminal.pending.json")):
+        identity = _terminal_identity(path)
+        last_invalid_reason = "malformed-name" if identity is None else None
+        retired_invalid = False
+        for _replacement_attempt in range(16):
+            snapshot = _open_terminal_recovery_snapshot(path)
+            if snapshot is None:
+                if (
+                    identity is not None
+                    and retired_invalid
+                    and _terminal_entry_absent(path)
+                ):
+                    _recover_exact_run(
+                        kanban_db,
+                        identity,
+                        last_invalid_reason or "malformed-payload",
+                    )
+                break
+            record = snapshot.document or {}
+            reason = None
+            if identity is None:
+                reason = "malformed-name"
+            elif not _terminal_record_valid(record):
+                reason = snapshot.invalid_reason or "malformed-payload"
+            elif not _terminal_record_valid(record, identity):
+                reason = "foreign-identity"
+            if reason is not None:
+                try:
+                    _quarantine_terminal(
+                        path,
+                        identity,
+                        reason,
+                        snapshot=snapshot,
+                    )
+                finally:
+                    snapshot.close()
+                retired_invalid = True
+                last_invalid_reason = reason
+                continue
+            assert identity is not None
+            try:
+                outcome = _finalize_terminal_record(
+                    kanban_db,
+                    path,
+                    record,
+                    snapshot=snapshot,
+                )
+            except Exception as error:
+                _record_board_access_error(identity.board, error)
+                break
+            if outcome == "committed":
+                recovered += 1
+                break
+            if outcome in {"invalid", "foreign"}:
+                # The pathname changed after the recovery snapshot closed.
+                # Reclassify and pin the new inode instead of quarantining it
+                # using authority derived from the older entry.
+                last_invalid_reason = outcome
+                continue
+            break
+        else:
+            print(
+                f"terminal journal replacement churn deferred for {path.name}",
+                file=sys.stderr,
+                flush=True,
+            )
+    # A quarantine or identity-safe retirement performed above can itself
+    # encounter replacement churn. Promote every resulting hidden generation
+    # back to first-class prepared/conflict/committed evidence before this
+    # recovery pass returns; DB replay, if still needed, occurs on the next
+    # bounded dispatcher pass.
+    _drain_retirement_staging()
+    return recovered
+
+def _has_pending_finalization(board: str, task_id: str, run_id: Any) -> bool:
+    """Keep an exact run claimed while its accepted result awaits replay."""
+    if type(run_id) is not int or canonical_run_id(run_id) is None:
+        return False
+    path = _terminal_path(state_path(board, task_id), run_id, "pending")
+    try:
+        path.stat()
+    except FileNotFoundError:
+        pass
+    except OSError:
+        return False
+    else:
+        identity = _terminal_identity(path)
+        if identity is not None:
+            record = _load_terminal_json(path, identity)
+            if _terminal_record_valid(record, identity):
+                return True
+    base = state_path(board, task_id)
+    for prepared in base.parent.glob(
+        f"{base.stem}.run-{run_id}.terminal.prepared-*.json"
+    ):
+        identity = _terminal_evidence_identity(prepared)
+        if identity is None or identity.state != "prepared":
+            continue
+        record = _load_small_json(prepared)
+        if _terminal_evidence_valid(record, identity, "prepared"):
+            return True
+    staged_paths = list(base.parent.glob(".retire.*"))
+    quarantine = base.parent / "quarantine"
+    if quarantine.is_dir() and not quarantine.is_symlink():
+        staged_paths.extend(quarantine.glob(".retire.*"))
+    for staged in staged_paths:
+        snapshot = _open_terminal_recovery_snapshot(staged)
+        if snapshot is None:
+            continue
+        try:
+            authority = _staged_terminal_authority(staged, snapshot)
+            if (
+                authority is not None
+                and authority[0].board == board
+                and authority[0].task_id == task_id
+                and authority[0].run_id == run_id
+                and authority[0].state in {"pending", "prepared", "committed"}
+            ):
+                return True
+        finally:
+            snapshot.close()
+    return False

services/hermes/scripts/cli_lane_retention.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""Bounded age, count, and byte retention for non-pending lane artifacts."""
+
+from __future__ import annotations
+
+import os
+import stat
+import sys
+import time
+from pathlib import Path
+
+from cli_lane_config import (
+    ARTIFACT_GC_INTERVAL_SECONDS,
+    ARTIFACT_RETENTION_AGE_SECONDS,
+    ARTIFACT_RETENTION_BYTES,
+    ARTIFACT_RETENTION_COUNT,
+    STATE_ROOT,
+)
+from cli_lane_evidence import _retire_terminal_entry, _terminal_evidence_valid
+from cli_lane_files import (
+    _fsync_directory,
+    _terminal_evidence_identity,
+    _terminal_identity,
+)
+from cli_lane_quarantine import _quarantine_terminal
+from cli_lane_recovery import _staged_terminal_authority
+from cli_lane_records import _open_terminal_recovery_snapshot, _terminal_record_valid
+
+
+def _artifact_gc_candidates(board_dir: Path) -> list[Path]:
+    """List only bounded artifact classes; pending journals are excluded."""
+    candidates: list[Path] = []
+    patterns = (
+        "*.provider-*.result.json",
+        "*.candidate-*.json",
+        "*.terminal.committed.json",
+        "*.terminal.conflict-*.json",
+        ".retire.*",
+    )
+    for pattern in patterns:
+        candidates.extend(board_dir.glob(pattern))
+    quarantine = board_dir / "quarantine"
+    if quarantine.is_dir():
+        candidates.extend(quarantine.glob("*.quarantine"))
+        candidates.extend(quarantine.glob(".retire.*"))
+    return candidates
+
+def _unlink_artifact_if_same(path: Path, observed: os.stat_result) -> bool:
+    """Remove only the exact artifact inode selected by retention."""
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
+    descriptor = None
+    source_descriptor = None
+    try:
+        descriptor = os.open(path.parent, directory_flags)
+        file_flags = getattr(os, "O_PATH", os.O_RDONLY) | getattr(os, "O_NOFOLLOW", 0)
+        source_descriptor = os.open(path.name, file_flags, dir_fd=descriptor)
+        pinned = os.fstat(source_descriptor)
+        if pinned.st_dev != observed.st_dev or pinned.st_ino != observed.st_ino:
+            return False
+        outcome = _retire_terminal_entry(
+            path,
+            observed,
+            board_descriptor=descriptor,
+            quarantine_descriptor=descriptor,
+            source_descriptor=source_descriptor,
+        )
+        return outcome == "retired"
+    except OSError:
+        return False
+    finally:
+        if source_descriptor is not None:
+            os.close(source_descriptor)
+        if descriptor is not None:
+            os.close(descriptor)
+
+
+def _quarantine_invalid_retained_terminal(path: Path) -> tuple[bool, bool]:
+    """Audit retained committed/conflict evidence on the bounded GC cadence."""
+    committed = path.name.endswith(".terminal.committed.json")
+    conflict = ".terminal.conflict-" in path.name and path.name.endswith(".json")
+    if not committed and not conflict:
+        return False, False
+    identity = (
+        _terminal_identity(path) if committed else _terminal_evidence_identity(path)
+    )
+    snapshot = _open_terminal_recovery_snapshot(path)
+    if snapshot is None:
+        return True, False
+    document = snapshot.document
+    valid = False
+    if identity is not None and document is not None:
+        valid = (
+            _terminal_record_valid(document, identity)
+            if committed
+            else _terminal_evidence_valid(document, identity, "conflict")
+        )
+    if valid:
+        snapshot.close()
+        return False, False
+    try:
+        _quarantine_terminal(
+            path,
+            identity,
+            snapshot.invalid_reason or "invalid-retained-terminal-evidence",
+            snapshot=snapshot,
+        )
+    finally:
+        snapshot.close()
+    try:
+        path.stat(follow_symlinks=False)
+    except FileNotFoundError:
+        return True, True
+    except OSError:
+        return True, False
+    return True, False
+
+def gc_lane_artifacts(
+    *,
+    now: float | None = None,
+    max_age_seconds: int = ARTIFACT_RETENTION_AGE_SECONDS,
+    max_count: int = ARTIFACT_RETENTION_COUNT,
+    max_bytes: int = ARTIFACT_RETENTION_BYTES,
+) -> int:
+    """Bound non-pending lane artifacts by age, count, and bytes per board."""
+    current = time.time() if now is None else now
+    removed = 0
+    for board_dir in STATE_ROOT.iterdir() if STATE_ROOT.is_dir() else ():
+        if not board_dir.is_dir() or board_dir.is_symlink():
+            continue
+        board_removed = 0
+        entries: list[tuple[Path, os.stat_result]] = []
+        for path in _artifact_gc_candidates(board_dir):
+            classified, quarantined = _quarantine_invalid_retained_terminal(path)
+            if classified:
+                if quarantined:
+                    removed += 1
+                    board_removed += 1
+                continue
+            if path.name.startswith(".retire."):
+                snapshot = _open_terminal_recovery_snapshot(path)
+                if snapshot is not None:
+                    try:
+                        if _staged_terminal_authority(path, snapshot) is not None:
+                            # Accepted authority is first promoted/replayed by
+                            # recovery; retention never deletes it directly.
+                            continue
+                    finally:
+                        snapshot.close()
+            try:
+                file_stat = path.stat(follow_symlinks=False)
+            except OSError:
+                continue
+            if not stat.S_ISREG(file_stat.st_mode):
+                if _unlink_artifact_if_same(path, file_stat):
+                    removed += 1
+                    board_removed += 1
+                continue
+            if max_age_seconds >= 0 and current - file_stat.st_mtime > max_age_seconds:
+                if _unlink_artifact_if_same(path, file_stat):
+                    removed += 1
+                    board_removed += 1
+            else:
+                entries.append((path, file_stat))
+        retained_count = 0
+        retained_bytes = 0
+        for path, file_stat in sorted(
+            entries, key=lambda item: item[1].st_mtime, reverse=True
+        ):
+            exceeds_count = max_count >= 0 and retained_count >= max_count
+            exceeds_bytes = (
+                max_bytes >= 0 and retained_bytes + file_stat.st_size > max_bytes
+            )
+            if exceeds_count or exceeds_bytes:
+                if _unlink_artifact_if_same(path, file_stat):
+                    removed += 1
+                    board_removed += 1
+                continue
+            retained_count += 1
+            retained_bytes += file_stat.st_size
+        if board_removed:
+            _fsync_directory(board_dir)
+    return removed
+
+LAST_ARTIFACT_GC = 0.0
+
+def maybe_gc_lane_artifacts(*, now: float | None = None) -> int:
+    """Run artifact retention on a bounded cadence, not every dispatcher tick."""
+    global LAST_ARTIFACT_GC
+    current = time.time() if now is None else now
+    if current - LAST_ARTIFACT_GC < ARTIFACT_GC_INTERVAL_SECONDS:
+        return 0
+    LAST_ARTIFACT_GC = current
+    try:
+        return gc_lane_artifacts(now=current)
+    except OSError as error:
+        print(
+            f"lane artifact retention deferred: {type(error).__name__}: {error}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return 0

services/hermes/scripts/cli_lane_routing.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""Switchyard route selection and provider-health exclusion for CLI workers."""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+import urllib.error
+import urllib.request
+from typing import Any, Callable
+
+from cli_lane_config import (
+    EFFORTS,
+    PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS,
+    PROVIDER_HEALTH_MAX_AGE_SECONDS,
+    PROVIDER_HEALTH_PATHS,
+    SWITCHYARD_URL,
+    Route,
+)
+from cli_lane_files import load_json
+
+
+def parse_assignee(assignee: str) -> tuple[str | None, str | None]:
+    """Parse external lane overrides while leaving cli-auto fully automatic."""
+    value = str(assignee or "").strip().lower()
+    if value == "cli-auto":
+        return None, None
+    match = re.fullmatch(r"cli-(codex|claude)-(low|medium|high|xhigh)", value)
+    if not match:
+        raise ValueError(f"unsupported external lane assignee: {assignee}")
+    return match.group(1), match.group(2)
+
+def _health_number(value: Any) -> float | None:
+    """Return a plain numeric health field, rejecting booleans and strings."""
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    return float(value)
+
+def _quota_reset_readmits(
+    health: dict[str, Any], current: float, reset_timestamp: float | None
+) -> bool:
+    """A passed provider quota reset ends a capacity cooldown early."""
+    if health.get("authenticated") is False:
+        # An authentication failure is not repaired by a quota window reset.
+        return False
+    failed_at = _health_number(health.get("failed_at"))
+    return (
+        reset_timestamp is not None
+        and failed_at is not None
+        and failed_at < reset_timestamp <= current
+    )
+
+def fresh_unavailable_provider(
+    now: float | None = None,
+    quota_resets: dict[str, float] | None = None,
+) -> str | None:
+    """Return one recently proven-down provider for automatic route exclusion."""
+    current = time.time() if now is None else now
+    resets = quota_resets or {}
+    unavailable: list[str] = []
+    for provider, path in PROVIDER_HEALTH_PATHS.items():
+        health = load_json(path)
+        if health.get("state") not in ("unavailable", "capacity-limited"):
+            continue
+        cooldown_until = _health_number(health.get("cooldown_until"))
+        if cooldown_until is not None:
+            # Lane-recorded failures re-enter only after the full cooldown has
+            # elapsed (hysteresis), a quota reset passed, or fresh success
+            # rewrote the state above.
+            if current >= cooldown_until or _quota_reset_readmits(
+                health, current, resets.get(provider)
+            ):
+                continue
+            unavailable.append(provider)
+            continue
+        try:
+            age = current - path.stat().st_mtime
+        except OSError:
+            continue
+        max_age = (
+            PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS
+            if health.get("authenticated") is False
+            else PROVIDER_HEALTH_MAX_AGE_SECONDS
+        )
+        if 0 <= age <= max_age:
+            unavailable.append(provider)
+    # If both providers are down, keep the normal attempt/fallback path so the
+    # card records authoritative current errors instead of trusting snapshots.
+    return unavailable[0] if len(unavailable) == 1 else None
+
+def _decode_worker_target(value: str) -> tuple[str, str, str]:
+    """Decode the selected model header emitted by a worker decision route."""
+    parts = value.split("/", 3)
+    if len(parts) != 4 or parts[0] != "worker":
+        raise RuntimeError(f"invalid Switchyard worker target: {value}")
+    provider, model, effort = parts[1:]
+    if provider not in {"codex", "claude"} or effort not in EFFORTS:
+        raise RuntimeError(f"unsupported Switchyard worker target: {value}")
+    return provider, model, effort
+
+def select_route(
+    prompt: str,
+    assignee: str,
+    *,
+    exclude_provider: str | None = None,
+    exclude_reason: str | None = None,
+    switchyard_url: str = SWITCHYARD_URL,
+    open_request: Callable[..., Any] = urllib.request.urlopen,
+) -> Route:
+    """Ask Switchyard to select one native CLI worker at this boundary."""
+    started = time.monotonic()
+    manual_provider, manual_effort = parse_assignee(assignee)
+    if manual_provider and manual_effort:
+        route_id = f"atlas/worker/manual/{manual_provider}/{manual_effort}"
+        source = "switchyard-manual"
+    else:
+        route_id = "atlas/worker/auto/maximum"
+        source = "switchyard-classifier"
+    context = prompt
+    if exclude_provider:
+        reason = exclude_reason or "failed or exhausted capacity at this boundary"
+        context += (
+            f"\n\nRouting constraint: the {exclude_provider} provider {reason}. "
+            "Do not select it."
+        )
+    payload = json.dumps(
+        {
+            "model": route_id,
+            "messages": [{"role": "user", "content": context}],
+            "stream": False,
+            "max_tokens": 1,
+        }
+    ).encode("utf-8")
+    request = urllib.request.Request(
+        switchyard_url,
+        data=payload,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with open_request(request, timeout=60) as response:
+            selected = str(response.headers.get("x-model-router-selected-model") or "")
+            rationale = str(response.headers.get("x-model-router-rationale") or "")
+            response_body = response.read()
+    except (OSError, urllib.error.URLError) as exc:
+        raise RuntimeError(f"Switchyard worker routing failed: {exc}") from exc
+    provider, model, effort = _decode_worker_target(selected)
+    # Switchyard preserves the stable tier target in the selection header and
+    # top-level response model. The worker broker's assistant content contains
+    # the steward-resolved provider model required by the native CLI.
+    try:
+        response_document = json.loads(response_body)
+        resolved_target = str(response_document["choices"][0]["message"]["content"] or "")
+        resolved_provider, resolved_model, resolved_effort = _decode_worker_target(resolved_target)
+        required_prefix = "gpt-" if provider == "codex" else "claude-"
+        if (
+            resolved_provider == provider
+            and resolved_effort == effort
+            and resolved_model.startswith(required_prefix)
+        ):
+            model = resolved_model
+    except (AttributeError, IndexError, KeyError, RuntimeError, TypeError, ValueError, json.JSONDecodeError):
+        pass
+    if exclude_provider and provider == exclude_provider:
+        alternate = "claude" if provider == "codex" else "codex"
+        guarded = select_route(
+            prompt,
+            f"cli-{alternate}-{effort}",
+            switchyard_url=switchyard_url,
+            open_request=open_request,
+        )
+        return Route(
+            provider=guarded.provider,
+            model=guarded.model,
+            effort=guarded.effort,
+            profile=guarded.profile,
+            classifier=f"{source}-health-guard",
+            reason=(
+                f"Switchyard selected excluded {exclude_provider}; preserved "
+                f"its {effort} effort on healthy-provider route. {guarded.reason}"
+            ),
+            latency_ms=int((time.monotonic() - started) * 1000),
+            fallback_chain=(),
+        )
+    return Route(
+        provider=provider,
+        model=model,
+        effort=effort,
+        profile=f"{provider}-{effort}",
+        classifier=source,
+        reason=rationale or f"Switchyard selected {selected}",
+        latency_ms=int((time.monotonic() - started) * 1000),
+        fallback_chain=(),
+    )

services/hermes/scripts/execution_pool_client.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+"""Ordinal-local signing and SCM mediation boundary for a model worker."""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+import urllib.error
+import urllib.request
+from http.server import BaseHTTPRequestHandler
+from pathlib import Path
+from typing import Any
+
+import cli_lane_goal
+from execution_pool_protocol import (
+    MAX_WIRE_BYTES,
+    PROTOCOL_VERSION,
+    BoundedHTTPServer,
+    ProtocolError,
+    canonical_json,
+    parse_wire,
+    read_key,
+    sign_envelope,
+    verify_envelope,
+)
+from execution_pool_scm import Boundary as SCMBoundary
+
+
+KEY_PATH = Path(
+    os.environ.get("HERMES_EXECUTION_POOL_KEY_FILE", "/pool-access/execution-pool-key")
+)
+COORDINATOR = os.environ.get(
+    "HERMES_EXECUTION_POOL_URL",
+    "http://hermes-execution-pool.hermes.svc.cluster.local:9007",
+).rstrip("/")
+ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
+PORT = int(os.environ.get("HERMES_EXECUTION_CLIENT_PORT", "9009"))
+RESULT_FIELDS = frozenset(
+    {"status", "summary", "changed_files", "tests_run", "artifacts", "findings", "blockers"}
+)
+
+
+def _binding(value: dict[str, Any]) -> dict[str, Any]:
+    return {
+        name: value[name]
+        for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt")
+    }
+
+
+def _validate_result(payload: Any) -> dict[str, Any]:
+    if not isinstance(payload, dict):
+        raise ProtocolError("terminal result payload must be an object")
+    structured = payload.get("structured")
+    if not isinstance(structured, dict) or set(structured) != RESULT_FIELDS:
+        raise ProtocolError("terminal result fields do not match the reviewed schema")
+    if structured.get("status") not in cli_lane_goal.RESULT_STATUSES:
+        raise ProtocolError("terminal result status is invalid")
+    if not isinstance(structured.get("summary"), str) or not structured["summary"].strip():
+        raise ProtocolError("terminal result summary is required")
+    for name in RESULT_FIELDS - {"status", "summary"}:
+        value = structured.get(name)
+        if not isinstance(value, list) or any(not isinstance(item, str) for item in value):
+            raise ProtocolError(f"terminal result {name} must be a text list")
+    return payload
+
+
+class ClientBoundary:
+    """Keep HMAC and SCM authority outside the model-facing container."""
+
+    def __init__(self, key: bytes, scm: SCMBoundary | None = None):
+        self.key = key
+        self.scm = scm or SCMBoundary(key)
+        self.current: dict[str, Any] | None = None
+        self.lock = threading.RLock()
+
+    def _post(self, path: str, envelope: dict[str, Any]) -> dict[str, Any]:
+        request = urllib.request.Request(
+            COORDINATOR + path,
+            data=canonical_json(envelope),
+            method="POST",
+            headers={"Content-Type": "application/json", "Cache-Control": "no-store"},
+        )
+        with urllib.request.urlopen(request, timeout=60) as response:
+            body = response.read(MAX_WIRE_BYTES + 1)
+        if len(body) > MAX_WIRE_BYTES:
+            raise ProtocolError("coordinator response exceeds the wire limit")
+        return verify_envelope(self.key, json.loads(body))
+
+    def poll(self) -> dict[str, Any]:
+        poll_binding = {
+            "board": "",
+            "task_id": "",
+            "run_id": "",
+            "worker_ordinal": ORDINAL,
+            "attempt": 0,
+        }
+        response = self._post(
+            "/v1/poll",
+            sign_envelope(self.key, "poll", poll_binding, {"ready": True}),
+        )
+        with self.lock:
+            if response["kind"] == "ack":
+                self.current = None
+                return {"assignment": None}
+            if response["kind"] != "assignment" or response["worker_ordinal"] != ORDINAL:
+                raise ProtocolError("coordinator returned a foreign assignment")
+            checkout = self.scm.checkout(response)
+            self.current = response
+            assignment = {
+                **_binding(response),
+                "payload": response["payload"],
+                "workspace": checkout["workspace"],
+                "baseline_sha": checkout["baseline_sha"],
+                "protocol_version": PROTOCOL_VERSION,
+            }
+        return {"assignment": assignment}
+
+    def _current_for(self, supplied: Any) -> tuple[dict[str, Any], dict[str, Any]]:
+        if not isinstance(supplied, dict):
+            raise ProtocolError("local request binding must be an object")
+        if self.current is None or supplied != _binding(self.current):
+            raise ProtocolError("local request does not own the current assignment")
+        return self.current, dict(supplied)
+
+    def heartbeat(self, request: dict[str, Any]) -> dict[str, Any]:
+        payload = request.get("payload")
+        if not isinstance(payload, dict):
+            raise ProtocolError("heartbeat payload must be an object")
+        with self.lock:
+            _assignment, binding = self._current_for(request.get("binding"))
+            response = self._post(
+                "/v1/heartbeat",
+                sign_envelope(self.key, "heartbeat", binding, payload),
+            )
+            if response["kind"] != "ack" or _binding(response) != binding:
+                raise ProtocolError("coordinator acknowledgement binding changed")
+        return {"ack": response["payload"]}
+
+    def _submit(
+        self, assignment: dict[str, Any], request: dict[str, Any],
+        structured: dict[str, Any],
+    ) -> None:
+        """Publish the run's work, downgrading the result rather than losing it.
+
+        A refused push or draft used to unwind the whole worker, so the run's
+        actual output was discarded and the task was blocked with only the SCM
+        error. The commits live on the durable workspace volume, so the result is
+        still reported here -- as blocked, carrying the reason -- which keeps the
+        prior work recoverable and reviewable by a human.
+        """
+        try:
+            submission = self.scm.submit(assignment, request)
+        except (OSError, ProtocolError, RuntimeError, ValueError) as error:
+            structured["status"] = "blocked"
+            reason = (
+                "Distributed SCM submission failed; the commits remain on this "
+                f"ordinal's workspace: {type(error).__name__}: {error}"
+            )
+            if reason not in structured["blockers"]:
+                structured["blockers"].append(reason)
+            return
+        pull = str(submission.get("pull_request") or "")
+        branch = str(submission.get("branch") or "")
+        for artifact in (pull, f"branch:{branch}" if branch else ""):
+            if artifact and artifact not in structured["artifacts"]:
+                structured["artifacts"].append(artifact)
+
+    def finish(self, request: dict[str, Any]) -> dict[str, Any]:
+        payload = _validate_result(request.get("payload"))
+        with self.lock:
+            assignment, binding = self._current_for(request.get("binding"))
+            structured = payload["structured"]
+            if structured["status"] == "completed" and int(payload.get("returncode", 1)) == 0:
+                self._submit(assignment, request, structured)
+            response = self._post(
+                "/v1/result", sign_envelope(self.key, "result", binding, payload)
+            )
+            if response["kind"] != "ack" or _binding(response) != binding:
+                raise ProtocolError("coordinator acknowledgement binding changed")
+            if response["payload"].get("accepted"):
+                self.current = None
+        return {"ack": response["payload"], "structured": structured}
+
+
+def handler_factory(boundary: ClientBoundary) -> type[BaseHTTPRequestHandler]:
+    class Handler(BaseHTTPRequestHandler):
+        server_version = f"hermes-execution-mediator/{PROTOCOL_VERSION}"
+
+        def _reply(self, status: int, value: dict[str, Any]) -> None:
+            body = canonical_json(value)
+            self.send_response(status)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(body)))
+            self.send_header("Cache-Control", "no-store")
+            self.end_headers()
+            self.wfile.write(body)
+
+        def do_GET(self) -> None:  # noqa: N802
+            value = {"ready": True, "protocol_version": PROTOCOL_VERSION}
+            self._reply(200, value) if self.path == "/ready" else self._reply(
+                404, {"error": "not found"}
+            )
+
+        def do_POST(self) -> None:  # noqa: N802
+            try:
+                length = int(self.headers.get("Content-Length", "0"))
+                request = (
+                    parse_wire(self.rfile.read(length))
+                    if 0 < length <= MAX_WIRE_BYTES
+                    else None
+                )
+                allowed = {"operation", "binding", "payload", "title", "body"}
+                if not request or set(request) - allowed:
+                    raise ProtocolError("invalid local mediator request")
+                operation = str(request.get("operation") or "")
+                routes = {
+                    "poll": boundary.poll,
+                    "heartbeat": lambda: boundary.heartbeat(request),
+                    "finish": lambda: boundary.finish(request),
+                }
+                if operation not in routes:
+                    raise ProtocolError("unsupported local mediator operation")
+                self._reply(200, routes[operation]())
+            except (ProtocolError, OSError, ValueError, urllib.error.URLError) as error:
+                self._reply(409, {"error": str(error)[:2000]})
+
+        def log_message(self, _format: str, *_arguments: Any) -> None:
+            return
+
+    return Handler
+
+
+def main() -> int:
+    if ORDINAL not in range(3):
+        raise SystemExit("HERMES_WORKER_ORDINAL must be 0, 1, or 2")
+    key = read_key(KEY_PATH)
+    BoundedHTTPServer(
+        ("0.0.0.0", PORT),
+        handler_factory(ClientBoundary(key)),
+        max_workers=4,
+    ).serve_forever()
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover - exercised by the container entrypoint
+    raise SystemExit(main())

services/hermes/scripts/execution_pool_coordinator.py

@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+"""Coordinator-only Kanban claim/finalize bridge for three Hermes workers."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import threading
+import time
+from http.server import BaseHTTPRequestHandler
+from pathlib import Path
+from typing import Any
+
+import cli_lane_goal
+from cli_lane_config import canonical_run_id
+from execution_pool_project import resolve_assignment
+from execution_pool_protocol import (
+    MAX_ACTIVITY_BYTES,
+    ProtocolError,
+    derive_ordinal_key,
+    envelope_ordinal,
+    sign_envelope,
+    verify_envelope,
+)
+from execution_pool_store import PoolStore
+
+REDACTIONS = (
+    re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/-]{12,}"),
+    re.compile(
+        r"(?i)(authorization|token|secret|password|api[_-]?key|"
+        r"refresh[_-]?token|access[_-]?token)[\"']?\s*[:=]\s*[\"']?[^\s\"',;}]+"
+    ),
+    re.compile(r"\b(?:sk-ant-|sk-proj-|rt_)[A-Za-z0-9_-]{12,}\b"),
+    re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"),
+    re.compile(r"(?i)https://[^/@\s]+@scm\.bstein\.dev"),
+)
+
+
+def _task_value(task: Any, name: str, default: Any = None) -> Any:
+    return getattr(task, name, default)
+
+
+def resolve_scm(task: Any, board: str = "titan-iac") -> tuple[str, str, str]:
+    """Resolve SCM only through the canonical board/project registry."""
+    return resolve_assignment(board, task)
+
+
+def assignment_payload(
+    kanban_db: Any, connection: Any, task: Any, board: str = "titan-iac"
+) -> dict[str, Any]:
+    context = kanban_db.build_worker_context(connection, str(_task_value(task, "id")))
+    if not isinstance(context, str):
+        context = json.dumps(context, default=str, sort_keys=True)
+    encoded = context.encode("utf-8")
+    if len(encoded) > 32 * 1024:
+        raise RuntimeError("Kanban worker context exceeds the 32KiB assignment limit")
+    repo_url, branch, base_branch = resolve_scm(task, board)
+    runtime = int(_task_value(task, "max_runtime_seconds", 0) or 12 * 60 * 60)
+    runtime = max(60, min(runtime, 12 * 60 * 60))
+    return {
+        "context": context,
+        "assignee": str(_task_value(task, "assignee", "cli-auto") or "cli-auto"),
+        "repo_url": repo_url,
+        "branch": branch,
+        "base_branch": base_branch,
+        "max_runtime_seconds": runtime,
+        "deadline_unix": int(time.time()) + runtime,
+        "goal_mode": bool(_task_value(task, "goal_mode", False)),
+        "goal_max_turns": max(1, min(int(_task_value(task, "goal_max_turns", 1) or 1), 12)),
+    }
+
+
+def sanitize_activity(value: Any) -> str:
+    """Bound and redact worker output before it reaches the shared UI log."""
+    text = str(value or "").replace("\x00", "")
+    text = "".join(character for character in text if character in "\n\t" or ord(character) >= 32)
+    for pattern in REDACTIONS:
+        text = pattern.sub("[REDACTED]", text)
+    return text.encode("utf-8")[:MAX_ACTIVITY_BYTES].decode("utf-8", "ignore")
+
+
+def _append_activity(kanban_db: Any, envelope: dict[str, Any]) -> None:
+    payload = envelope["payload"]
+    if not isinstance(payload, dict):
+        raise ProtocolError("heartbeat payload must be an object")
+    activity = sanitize_activity(payload.get("activity"))
+    if not activity:
+        return
+    path = Path(kanban_db.worker_log_path(envelope["task_id"], board=envelope["board"]))
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if path.is_symlink():
+        raise ProtocolError("Kanban activity log must not be a symlink")
+    flags = os.O_APPEND | os.O_CREAT | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0)
+    descriptor = os.open(path, flags, 0o600)
+    try:
+        os.write(descriptor, activity.encode("utf-8"))
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+def _append_terminal_activity(kanban_db: Any, record: dict[str, Any]) -> None:
+    """Append one terminal batch exactly once across result retries/restarts."""
+    payload = record.get("result")
+    activity = payload.get("final_activity") if isinstance(payload, dict) else ""
+    activity = sanitize_activity(activity)
+    if not activity:
+        return
+    marker = f"\n[execution-pool-result:{record['result_digest']}]\n"
+    path = Path(kanban_db.worker_log_path(record["task_id"], board=record["board"]))
+    if path.is_symlink():
+        raise ProtocolError("Kanban activity log must not be a symlink")
+    if path.exists():
+        flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+        descriptor = os.open(path, flags)
+        try:
+            size = os.fstat(descriptor).st_size
+            os.lseek(descriptor, max(0, size - 64 * 1024), os.SEEK_SET)
+            if marker.encode() in os.read(descriptor, 64 * 1024):
+                return
+        finally:
+            os.close(descriptor)
+    _append_activity(
+        kanban_db,
+        {**record, "payload": {"activity": marker + activity}},
+    )
+
+
+class Coordinator:
+    """Own all transitions between durable assignments and Hermes Kanban."""
+
+    def __init__(self, key: bytes, store: PoolStore):
+        self.master_key = key
+        self.store = store
+        self._kanban_lock = threading.RLock()
+
+    def _key(self, envelope: dict[str, Any]) -> bytes:
+        return derive_ordinal_key(self.master_key, envelope_ordinal(envelope))
+
+    def _verify(self, envelope: dict[str, Any], kind: str) -> dict[str, Any]:
+        return verify_envelope(self._key(envelope), envelope, expected_kind=kind)
+
+    def _sign(
+        self, kind: str, binding: dict[str, Any], payload: dict[str, Any]
+    ) -> dict[str, Any]:
+        key = derive_ordinal_key(self.master_key, int(binding["worker_ordinal"]))
+        return sign_envelope(key, kind, binding, payload)
+
+    @staticmethod
+    def _binding(record: dict[str, Any]) -> dict[str, Any]:
+        return {name: record[name] for name in (
+            "board", "task_id", "run_id", "worker_ordinal", "attempt"
+        )}
+
+    def poll(self, envelope: dict[str, Any]) -> dict[str, Any]:
+        verified = self._verify(envelope, "poll")
+        record = self.store.offer(int(verified["worker_ordinal"]))
+        if record is None:
+            binding = {**self._binding({
+                "board": "", "task_id": "", "run_id": "",
+                "worker_ordinal": verified["worker_ordinal"], "attempt": 0,
+            })}
+            return self._sign("ack", binding, {"assignment": None})
+        return self._sign("assignment", self._binding(record), record["payload"])
+
+    def heartbeat(self, envelope: dict[str, Any]) -> dict[str, Any]:
+        verified = self._verify(envelope, "heartbeat")
+        accepted, duplicate = self.store.heartbeat(verified)
+        from hermes_cli import kanban_db
+
+        payload = verified["payload"]
+        note = sanitize_activity(payload.get("note") if isinstance(payload, dict) else "")[:1000]
+        with self._kanban_lock, kanban_db.scoped_current_board(verified["board"]):
+            connection = kanban_db.connect(board=verified["board"])
+            try:
+                run_id = canonical_run_id(verified["run_id"])
+                if run_id is None:
+                    raise ProtocolError("worker run ID is not a canonical SQLite run")
+                alive = kanban_db.heartbeat_worker(
+                    connection, verified["task_id"],
+                    note=note or "distributed worker active",
+                    expected_run_id=run_id,
+                )
+                if not alive:
+                    raise ProtocolError("Kanban run no longer owns this worker")
+                if not duplicate:
+                    _append_activity(kanban_db, verified)
+            finally:
+                connection.close()
+        return self._sign(
+            "ack", self._binding(verified),
+            {"accepted": accepted, "duplicate": duplicate},
+        )
+
+    def result(self, envelope: dict[str, Any]) -> dict[str, Any]:
+        verified = self._verify(envelope, "result")
+        record, duplicate = self.store.accept_result(verified)
+        if record.get("state") == "result":
+            self.finalize(record)
+        return self._sign(
+            "ack", self._binding(verified),
+            {"accepted": True, "duplicate": duplicate},
+        )
+
+    def finalize(self, record: dict[str, Any]) -> None:
+        """Apply a result only if this exact run remains authoritative."""
+        from hermes_cli import kanban_db
+
+        binding = self._binding(record)
+        run_id = canonical_run_id(binding["run_id"])
+        if run_id is None:
+            self.store.finalize(binding, "stale")
+            return
+        payload = record.get("result")
+        if not isinstance(payload, dict):
+            raise ProtocolError("result payload must be an object")
+        structured = payload.get("structured")
+        if not isinstance(structured, dict):
+            structured = {}
+        with self._kanban_lock, kanban_db.scoped_current_board(binding["board"]):
+            connection = kanban_db.connect(board=binding["board"])
+            try:
+                task = kanban_db.get_task(connection, binding["task_id"])
+                current_run_id = canonical_run_id(
+                    _task_value(task, "current_run_id", None)
+                ) if task is not None else None
+                if current_run_id is None or current_run_id != run_id:
+                    self.store.finalize(binding, "stale")
+                    return
+                _append_terminal_activity(kanban_db, record)
+                metadata = {
+                    "executor": "distributed-execution-pool",
+                    "worker_ordinal": binding["worker_ordinal"],
+                    "attempt": binding["attempt"],
+                    "node": str(payload.get("node") or "unknown")[:128],
+                    "route": payload.get("route") if isinstance(payload.get("route"), dict) else {},
+                    "provider_sessions": payload.get("provider_sessions")
+                    if isinstance(payload.get("provider_sessions"), dict) else {},
+                    "changed_files": structured.get("changed_files", []),
+                    "tests_run": structured.get("tests_run", []),
+                    "artifacts": structured.get("artifacts", []),
+                    "findings": structured.get("findings", []),
+                    "blockers": structured.get("blockers", []),
+                }
+                problem = cli_lane_goal.unfinished_result_reason(structured)
+                if (
+                    structured.get("status") == "completed"
+                    and int(payload.get("returncode", 1)) == 0
+                    and problem is None
+                ):
+                    changed = kanban_db.complete_task(
+                        connection, binding["task_id"],
+                        result=json.dumps(structured, sort_keys=True),
+                        summary=str(structured.get("summary") or "Completed"),
+                        metadata=metadata, expected_run_id=run_id,
+                    )
+                else:
+                    reason = problem or "; ".join(map(str, structured.get("blockers", [])))
+                    reason = reason or str(structured.get("summary") or "worker failed")
+                    changed = kanban_db.block_task(
+                        connection, binding["task_id"], reason=reason,
+                        kind="transient" if payload.get("capacity_failure") else "capability",
+                        expected_run_id=run_id,
+                    )
+                self.store.finalize(binding, "finalized" if changed else "stale")
+            finally:
+                connection.close()
+
+    @staticmethod
+    def assignment_payload(
+        kanban_db: Any, connection: Any, task: Any, board: str
+    ) -> dict[str, Any]:
+        """Expose payload construction to the maintenance passes."""
+        return assignment_payload(kanban_db, connection, task, board)
+
+    def recover_results(self) -> None:
+        """Re-apply every durable result that Kanban has not confirmed yet."""
+        from execution_pool_maintenance import recover_results
+
+        recover_results(self)
+
+    def expire_leases(self) -> None:
+        """Fence dead attempts; after bounded retries, surface and release the run."""
+        from execution_pool_maintenance import expire_leases
+
+        expire_leases(self)
+
+    def reconcile(self) -> None:
+        """Recover the narrow claim/assignment crash gaps without double execution."""
+        from execution_pool_maintenance import reconcile
+
+        reconcile(self)
+
+    def dispatch(self) -> None:
+        """Claim up to the free ordinal count and materialize assignments."""
+        from execution_pool_maintenance import dispatch
+
+        dispatch(self)
+
+
+def handler_factory(coordinator: Coordinator) -> type[BaseHTTPRequestHandler]:
+    """Compatibility export for tests and the mounted coordinator entry point."""
+    from execution_pool_server import handler_factory as factory
+
+    return factory(coordinator)
+
+
+def main() -> int:
+    from execution_pool_server import run
+
+    return run(Coordinator)
+
+
+if __name__ == "__main__":  # pragma: no cover - exercised by the container entrypoint
+    raise SystemExit(main())

services/hermes/scripts/execution_pool_maintenance.py

@@ -0,0 +1,371 @@
+#!/usr/bin/env python3
+"""Recoverable maintenance passes for the Hermes execution-pool coordinator.
+
+Every pass here runs against a shared Kanban SQLite database that other writers
+(the agent UI, the local CLI lane, crons) hold concurrently, so *any* board call
+can fail transiently. The rules this module enforces:
+
+* No pass ever leaves a claimed run without a durable row that some later pass
+  will revisit. A Kanban write that fails leaves the row in its retryable state.
+* No single poisoned row, task, or board aborts the work queued behind it.
+* A durable row only reaches a terminal state on authoritative evidence about
+  its Kanban run, because terminal rows are the only ones ever collected.
+* A coordinator-side fault is never converted into a Kanban mutation. Only a
+  genuine capability failure of the run itself parks that exact run.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+import sys
+from typing import Any
+
+import cli_lane_dispatch
+from cli_lane_config import canonical_run_id
+from execution_pool_project import ProjectPolicyError, distributed_workspace_eligible
+from execution_pool_protocol import ProtocolError
+
+
+RECOVERABLE_BOARD_ERRORS = (OSError, sqlite3.Error)
+DEFERRALS: dict[str, str] = {}
+MAX_DEFERRALS = 64
+
+
+def _task_value(task: Any, name: str, default: Any = None) -> Any:
+    return getattr(task, name, default)
+
+
+def _defer(context: str, error: BaseException) -> None:
+    """Report one recoverable pool-side failure once per distinct detail.
+
+    The maintenance loop re-runs every few seconds, so an unchanged failure is
+    reported once rather than on every pass. The table is bounded because its
+    keys include run identities.
+    """
+    detail = f"{type(error).__name__}: {error}"
+    if DEFERRALS.get(context) == detail:
+        return
+    if len(DEFERRALS) >= MAX_DEFERRALS:
+        DEFERRALS.clear()
+    DEFERRALS[context] = detail
+    print(f"pool work deferred for {context}: {detail}", file=sys.stderr, flush=True)
+
+
+def _settled(context: str) -> None:
+    DEFERRALS.pop(context, None)
+
+
+def _context(record: dict[str, Any]) -> str:
+    return f"{record['board']}/{record['task_id']}#{record['run_id']}"
+
+
+def _block_exact(
+    kanban_db: Any,
+    board: str,
+    task_id: str,
+    run_id: int | None,
+    reason: str,
+    *,
+    kind: str = "capability",
+) -> None:
+    """Park exactly one run, never letting the write itself escape."""
+    try:
+        with kanban_db.scoped_current_board(board):
+            connection = kanban_db.connect(board=board)
+            try:
+                kanban_db.block_task(
+                    connection, task_id, reason=reason, kind=kind,
+                    expected_run_id=run_id,
+                )
+            finally:
+                connection.close()
+    except Exception as error:  # noqa: BLE001 - a failed park must not wedge the pass
+        _defer(f"{board}/{task_id} block", error)
+
+
+def recover_results(pool: Any) -> None:
+    """Re-apply each durable result in isolation from its neighbours."""
+    for record in pool.store.pending_results():
+        try:
+            pool.finalize(record)
+            _settled(_context(record))
+        except Exception as error:  # noqa: BLE001 - one bad result must not block the rest
+            print(
+                f"result recovery deferred: {type(error).__name__}: {error}",
+                file=sys.stderr,
+                flush=True,
+            )
+
+
+def _lease_failure_state(
+    kanban_db: Any, connection: Any, binding: dict[str, Any], run_id: int
+) -> str:
+    """Classify one exhausted lease from authoritative Kanban evidence only."""
+    task = kanban_db.get_task(connection, binding["task_id"])
+    current = (
+        canonical_run_id(_task_value(task, "current_run_id", None))
+        if task is not None
+        else None
+    )
+    if current != run_id:
+        return "stale"
+    if str(_task_value(task, "status", "") or "") != "running":
+        return "finalized"
+    if kanban_db.block_task(
+        connection,
+        binding["task_id"],
+        reason=(
+            "Distributed worker lease expired after "
+            f"{binding['attempt']} fenced attempts"
+        ),
+        kind="transient",
+        expected_run_id=run_id,
+    ):
+        return "finalized"
+    # Kanban refused the exact-run park yet still reports the run as running, so
+    # there is no terminal evidence. Keep the row for the next pass to retry.
+    raise ProtocolError("Kanban refused the exact-run lease park")
+
+
+def record_lease_failure(pool: Any, record: dict[str, Any]) -> None:
+    """Record one exhausted lease against its exact Kanban run, idempotently.
+
+    The row stays ``lease_failed`` unless this pass obtains authoritative
+    evidence: the run moved on or vanished (``stale``), Kanban already parked it
+    (``finalized``), or this call parked it (``finalized``). Anything else --
+    including a refused write or an unreadable board -- leaves the row for the
+    next pass, so no run is ever silently dropped.
+    """
+    from hermes_cli import kanban_db
+
+    binding = pool._binding(record)
+    run_id = canonical_run_id(binding["run_id"])
+    if run_id is None:
+        pool.store.finalize(binding, "stale")
+        return
+    with pool._kanban_lock, kanban_db.scoped_current_board(binding["board"]):
+        connection = kanban_db.connect(board=binding["board"])
+        try:
+            state = _lease_failure_state(kanban_db, connection, binding, run_id)
+        finally:
+            connection.close()
+    pool.store.finalize(binding, state)
+    _settled(_context(record))
+
+
+def recover_lease_failures(pool: Any) -> None:
+    """Retry every released-but-unrecorded lease failure, each in isolation."""
+    for record in pool.store.failed_leases():
+        try:
+            record_lease_failure(pool, record)
+        except Exception as error:  # noqa: BLE001 - retried on the next pass
+            _defer(f"{_context(record)} lease park", error)
+
+
+def expire_leases(pool: Any) -> None:
+    """Fence dead attempts, then drain every unconfirmed terminal record."""
+    pool.store.expire_leases()
+    recover_lease_failures(pool)
+
+
+def _release_moved_run(pool: Any, kanban_db: Any, record: dict[str, Any]) -> None:
+    """Release one durable assignment whose Kanban run is no longer ours."""
+    with kanban_db.scoped_current_board(record["board"]):
+        connection = kanban_db.connect(board=record["board"])
+        try:
+            task = kanban_db.get_task(connection, record["task_id"])
+            current_run_id = (
+                canonical_run_id(_task_value(task, "current_run_id", None))
+                if task
+                else None
+            )
+            current = str(current_run_id or "")
+            status = str(_task_value(task, "status", "") or "") if task else ""
+        finally:
+            connection.close()
+    if current != record["run_id"] or status != "running":
+        pool.store.finalize(record, "stale")
+
+
+def _adopt_task(
+    pool: Any,
+    kanban_db: Any,
+    connection: Any,
+    board: str,
+    task: Any,
+    ordinals: list[int],
+    known: set[tuple[str, str, str]],
+) -> None:
+    """Adopt one orphaned pool-eligible run onto a free ordinal."""
+    task_id = str(_task_value(task, "id", "") or "")
+    raw_run_id = canonical_run_id(_task_value(task, "current_run_id", None))
+    run_id = str(raw_run_id or "")
+    assignee = str(_task_value(task, "assignee", "") or "")
+    if (
+        not task_id
+        or not run_id
+        or str(_task_value(task, "status", "")) != "running"
+        or not assignee.startswith("cli-")
+        or not distributed_workspace_eligible(task)
+        or (board, task_id, run_id) in known
+    ):
+        return
+    try:
+        payload = pool.assignment_payload(kanban_db, connection, task, board)
+    except RECOVERABLE_BOARD_ERRORS as error:
+        _defer(f"{board}/{task_id} recovery payload", error)
+        return
+    except Exception as error:  # noqa: BLE001 - a real capability failure of this run
+        _block_exact(
+            kanban_db, board, task_id, raw_run_id,
+            "Distributed assignment recovery failed: "
+            f"{type(error).__name__}: {error}",
+        )
+        return
+    binding = {
+        "board": board, "task_id": task_id, "run_id": run_id,
+        "worker_ordinal": ordinals[0], "attempt": 1,
+    }
+    try:
+        pool.store.add(binding, payload)
+    except ProtocolError as error:
+        # A durable row already owns this identity or ordinal. Leave the ordinal
+        # free and leave Kanban untouched; a later pass revisits it.
+        _defer(f"{board}/{task_id} adoption", error)
+        return
+    known.add((board, task_id, run_id))
+    ordinals.pop(0)
+
+
+def _adopt_board(
+    pool: Any,
+    kanban_db: Any,
+    board: str,
+    ordinals: list[int],
+    known: set[tuple[str, str, str]],
+) -> None:
+    """Adopt this board's orphaned runs without touching any other board."""
+    with kanban_db.scoped_current_board(board):
+        connection = kanban_db.connect(board=board)
+        try:
+            for task in kanban_db.list_tasks(connection):
+                if not ordinals:
+                    return
+                _adopt_task(pool, kanban_db, connection, board, task, ordinals, known)
+        finally:
+            connection.close()
+
+
+def reconcile(pool: Any) -> None:
+    """Recover the narrow claim/assignment crash gaps without double execution."""
+    from hermes_cli import kanban_db
+
+    for record in pool.store.active_assignments():
+        try:
+            _release_moved_run(pool, kanban_db, record)
+        except Exception as error:  # noqa: BLE001 - other assignments still reconcile
+            _defer(f"{_context(record)} release", error)
+    ordinals = pool.store.available_ordinals()
+    if not ordinals:
+        return
+    known = pool.store.known_runs()
+    try:
+        boards = kanban_db.list_boards(include_archived=False)
+    except RECOVERABLE_BOARD_ERRORS as error:
+        _defer("board-registry", error)
+        return
+    _settled("board-registry")
+    for raw_board in boards:
+        board = cli_lane_dispatch._board_slug(raw_board)
+        if not board or not ordinals:
+            continue
+        try:
+            _adopt_board(pool, kanban_db, board, ordinals, known)
+            _settled(f"{board} adoption")
+        except Exception as error:  # noqa: BLE001 - later boards must still run
+            _defer(f"{board} adoption", error)
+
+
+def _materialize(
+    pool: Any,
+    kanban_db: Any,
+    board: str,
+    task_id: str,
+    ordinal: int,
+    known: set[tuple[str, str, str]],
+) -> None:
+    """Turn one freshly claimed run into exactly one durable assignment."""
+    database_run_id: int | None = None
+    try:
+        with kanban_db.scoped_current_board(board):
+            connection = kanban_db.connect(board=board)
+            try:
+                task = kanban_db.get_task(connection, task_id)
+                if task is None:
+                    return
+                raw_run = canonical_run_id(
+                    _task_value(task, "current_run_id", None)
+                )
+                if not distributed_workspace_eligible(task):
+                    if raw_run is not None:
+                        kanban_db.block_task(
+                            connection, task_id,
+                            reason=(
+                                "Distributed claim fenced because an existing "
+                                "workspace is owned by the local lane"
+                            ),
+                            kind="capability", expected_run_id=raw_run,
+                        )
+                    return
+                if raw_run is None:
+                    raise ProjectPolicyError("claimed task has no canonical run ID")
+                database_run_id = raw_run
+                payload = pool.assignment_payload(kanban_db, connection, task, board)
+            finally:
+                connection.close()
+        run_id = str(database_run_id)
+        if (board, task_id, run_id) in known:
+            return
+        pool.store.add(
+            {
+                "board": board, "task_id": task_id, "run_id": run_id,
+                "worker_ordinal": ordinal, "attempt": 1,
+            },
+            payload,
+        )
+        known.add((board, task_id, run_id))
+        _settled(f"{board}/{task_id} dispatch")
+    except (ProtocolError, *RECOVERABLE_BOARD_ERRORS) as error:
+        # Coordinator-side storage/identity conditions. Never convert these into
+        # a Kanban capability block; reconcile() adopts the claim on a later pass.
+        _defer(f"{board}/{task_id} dispatch", error)
+    except Exception as error:  # noqa: BLE001 - a real capability failure of this run
+        _block_exact(
+            kanban_db, board, task_id, database_run_id,
+            "Distributed assignment preparation failed: "
+            f"{type(error).__name__}: {error}",
+        )
+
+
+def dispatch(pool: Any) -> None:
+    """Claim up to the free ordinal count and materialize assignments."""
+    from hermes_cli import kanban_db
+
+    ordinals = pool.store.available_ordinals()
+    if not ordinals:
+        return
+    try:
+        # Only storage faults are absorbed here. An incompatible claim API must
+        # still surface, because silently skipping the eligibility predicate is
+        # how the pool would start claiming the local lane's owned workspaces.
+        claimed = cli_lane_dispatch.claim_ready(
+            set(), len(ordinals),
+            lambda _board, task: distributed_workspace_eligible(task),
+        )
+    except RECOVERABLE_BOARD_ERRORS as error:
+        _defer("claim-ready", error)
+        return
+    _settled("claim-ready")
+    known = pool.store.known_runs()
+    for ordinal, (board, task_id) in zip(ordinals, claimed, strict=False):
+        _materialize(pool, kanban_db, board, task_id, ordinal, known)

services/hermes/scripts/execution_pool_project.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""Canonical Atlas project and Git-ref policy for distributed execution."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import stat
+import subprocess
+from pathlib import Path
+from typing import Any
+
+
+DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data"))
+PROJECT_ROOT = DATA_ROOT / "workspace/projects"
+BOARD_ROOT = DATA_ROOT / "kanban/boards"
+ATLAS_REPO = re.compile(
+    r"https://scm\.bstein\.dev/atlas/(?P<repo>[A-Za-z0-9][A-Za-z0-9_.-]{0,99})\.git\Z"
+)
+SAFE_PREFIXES = frozenset(
+    {"feature", "fix", "chore", "docs", "test", "refactor", "wt", "review", "hermes", "handoff"}
+)
+IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z")
+GIT = "/usr/bin/git"
+MAX_BOARD_BYTES = 64 * 1024
+
+
+class ProjectPolicyError(ValueError):
+    """Canonical project metadata or a requested ref failed closed."""
+
+
+def _task_value(task: Any, name: str, default: Any = None) -> Any:
+    return getattr(task, name, default)
+
+
+def _run_git(workdir: Path, *arguments: str) -> str:
+    completed = subprocess.run(
+        [GIT, "-C", str(workdir), *arguments],
+        check=False,
+        stdin=subprocess.DEVNULL,
+        capture_output=True,
+        text=True,
+        timeout=30,
+        env={
+            "HOME": "/nonexistent",
+            "PATH": "/usr/bin:/bin",
+            "GIT_CONFIG_NOSYSTEM": "1",
+            "GIT_TERMINAL_PROMPT": "0",
+        },
+    )
+    if completed.returncode:
+        raise ProjectPolicyError("canonical project Git metadata is unavailable")
+    return completed.stdout.strip()
+
+
+def validate_branch(value: object, *, feature: bool) -> str:
+    """Validate a complete local branch name with Git and a reviewed namespace."""
+    if not isinstance(value, str) or not value or len(value) > 200:
+        raise ProjectPolicyError("branch name exceeds the safe limit")
+    if not value.isascii() or value.startswith("-"):
+        raise ProjectPolicyError("branch name must be canonical ASCII")
+    prefix = value.split("/", 1)[0]
+    if feature and ("/" not in value or prefix not in SAFE_PREFIXES):
+        raise ProjectPolicyError("task branch is outside the reviewed namespace")
+    completed = subprocess.run(
+        [GIT, "check-ref-format", f"refs/heads/{value}"],
+        check=False,
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        timeout=10,
+        env={"PATH": "/usr/bin:/bin", "GIT_CONFIG_NOSYSTEM": "1"},
+    )
+    if completed.returncode:
+        raise ProjectPolicyError("branch name is not a valid Git ref")
+    return value
+
+
+def _read_board(board: str) -> dict[str, Any]:
+    if not IDENTIFIER.fullmatch(board):
+        raise ProjectPolicyError("board slug is invalid")
+    path = BOARD_ROOT / board / "board.json"
+    descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+    try:
+        info = os.fstat(descriptor)
+        if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= MAX_BOARD_BYTES:
+            raise ProjectPolicyError("board registry entry is not a bounded regular file")
+        raw = os.read(descriptor, MAX_BOARD_BYTES + 1)
+    finally:
+        os.close(descriptor)
+    try:
+        value = json.loads(raw)
+    except (UnicodeError, json.JSONDecodeError) as error:
+        raise ProjectPolicyError("board registry entry is malformed") from error
+    if not isinstance(value, dict) or value.get("slug") != board or value.get("archived") is True:
+        raise ProjectPolicyError("board registry identity is unavailable or archived")
+    return value
+
+
+def resolve_project(board: str) -> tuple[str, str, Path]:
+    """Resolve repo and base exclusively from the canonical board registry.
+
+    A board may be created before its primary checkout.  In that case the
+    Atlas repository naming contract and ``main`` are the registry defaults;
+    once a checkout exists, its credential-free origin and remote HEAD must
+    agree with that identity.  This keeps a missing checkout from silently
+    routing work to some other project's repository.
+    """
+    entry = _read_board(board)
+    raw_workdir = entry.get("default_workdir")
+    if not isinstance(raw_workdir, str) or not Path(raw_workdir).is_absolute():
+        raise ProjectPolicyError("board default_workdir is missing")
+    workdir = Path(raw_workdir).resolve(strict=False)
+    project_root = PROJECT_ROOT.resolve(strict=True)
+    try:
+        workdir.relative_to(project_root)
+    except ValueError as error:
+        raise ProjectPolicyError("board workdir is outside the Atlas project registry") from error
+    remote = f"https://scm.bstein.dev/atlas/{board}.git"
+    if not ATLAS_REPO.fullmatch(remote):
+        raise ProjectPolicyError("board repository identity is invalid")
+    if not workdir.exists():
+        return remote, "main", workdir
+    if not workdir.is_dir():
+        raise ProjectPolicyError("board checkout is not a directory")
+    checkout_remote = _run_git(workdir, "remote", "get-url", "origin")
+    if "@" in checkout_remote or checkout_remote != remote:
+        raise ProjectPolicyError("board origin disagrees with the Atlas registry")
+    try:
+        base = _run_git(workdir, "symbolic-ref", "--short", "refs/remotes/origin/HEAD")
+        if not base.startswith("origin/"):
+            raise ProjectPolicyError("origin HEAD is not canonical")
+        base = base.removeprefix("origin/")
+    except ProjectPolicyError:
+        base = "main"
+    return remote, validate_branch(base, feature=False), workdir
+
+
+def resolve_assignment(board: str, task: Any) -> tuple[str, str, str]:
+    """Resolve the exact repo/base and safe task branch for one board task."""
+    repo, base, _workdir = resolve_project(board)
+    task_id = str(_task_value(task, "id", "") or "")
+    if not IDENTIFIER.fullmatch(task_id):
+        raise ProjectPolicyError("task identity is invalid")
+    branch = str(_task_value(task, "branch_name", "") or f"wt/{task_id}")
+    return repo, validate_branch(branch, feature=True), base
+
+
+def distributed_workspace_eligible(task: Any) -> bool:
+    """Only pathless tasks migrate; an existing worktree stays with its local owner."""
+    return not str(_task_value(task, "workspace_path", "") or "").strip()

services/hermes/scripts/execution_pool_protocol.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+"""Authenticated, bounded, restart-safe Hermes execution-pool protocol."""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import os
+import re
+import stat
+import threading
+import time
+import uuid
+from http.server import ThreadingHTTPServer
+from pathlib import Path
+from typing import Any
+
+
+MAX_WIRE_BYTES = 64 * 1024
+MAX_ACTIVITY_BYTES = 12 * 1024
+MAX_CLOCK_SKEW = 30
+MAX_ENVELOPE_LIFETIME = 300
+PROTOCOL_VERSION = 2
+IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
+KINDS = frozenset({"poll", "assignment", "heartbeat", "result", "ack"})
+
+
+class ProtocolError(ValueError):
+    """A request failed the authenticated pool contract."""
+
+
+class BoundedHTTPServer(ThreadingHTTPServer):
+    """Bound concurrent requests and slow clients on internal pool channels."""
+
+    daemon_threads = True
+    request_queue_size = 8
+
+    def __init__(self, *args: Any, max_workers: int = 8, **kwargs: Any):
+        self._slots = threading.BoundedSemaphore(max(1, min(max_workers, 16)))
+        super().__init__(*args, **kwargs)
+
+    def get_request(self) -> tuple[Any, Any]:
+        request, address = super().get_request()
+        request.settimeout(15)
+        return request, address
+
+    def process_request(self, request: Any, client_address: Any) -> None:
+        if not self._slots.acquire(blocking=False):
+            self.shutdown_request(request)
+            return
+        try:
+            super().process_request(request, client_address)
+        except Exception:
+            self._slots.release()
+            raise
+
+    def process_request_thread(self, request: Any, client_address: Any) -> None:
+        try:
+            super().process_request_thread(request, client_address)
+        finally:
+            self._slots.release()
+
+
+def canonical_json(value: Any) -> bytes:
+    """Encode one value deterministically for digests and signatures."""
+    return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
+
+
+def atomic_json(path: Path, value: dict[str, Any], mode: int = 0o600) -> None:
+    """Durably replace one bounded pool document without following symlinks."""
+    path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+    if path.parent.is_symlink():
+        raise ProtocolError("pool state directory must not be a symlink")
+    temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
+    descriptor = os.open(temporary, flags, mode)
+    try:
+        remaining = memoryview(json.dumps(value, indent=2, sort_keys=True).encode() + b"\n")
+        while remaining:
+            remaining = remaining[os.write(descriptor, remaining) :]
+        os.fsync(descriptor)
+        os.fchmod(descriptor, mode)
+    finally:
+        os.close(descriptor)
+    try:
+        os.replace(temporary, path)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def payload_digest(payload: Any) -> str:
+    return hashlib.sha256(canonical_json(payload)).hexdigest()
+
+
+def derive_ordinal_key(master: bytes, ordinal: int) -> bytes:
+    """Derive one cryptographically isolated worker authority from the pool root."""
+    if ordinal not in range(3) or not 32 <= len(master) <= 4096:
+        raise ProtocolError("pool key derivation input is invalid")
+    context = f"hermes-execution-pool-v2:worker:{ordinal}".encode()
+    return hmac.new(master, context, hashlib.sha256).hexdigest().encode()
+
+
+def envelope_ordinal(envelope: Any) -> int:
+    """Read only the bounded ordinal needed to select a verification key."""
+    if not isinstance(envelope, dict):
+        raise ProtocolError("invalid pool message")
+    value = envelope.get("worker_ordinal")
+    if type(value) is not int or value not in range(3):
+        raise ProtocolError("worker binding is outside the pool")
+    return value
+
+
+def read_key(path: Path) -> bytes:
+    """Read a private regular file without following a final symlink."""
+    flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    try:
+        descriptor = os.open(path, flags)
+    except OSError as error:
+        raise ProtocolError(f"pool key is unavailable: {error}") from error
+    try:
+        info = os.fstat(descriptor)
+        if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077:
+            raise ProtocolError("pool key must be a private regular file")
+        value = os.read(descriptor, 4097).strip()
+    finally:
+        os.close(descriptor)
+    if len(value) < 32 or len(value) > 4096:
+        raise ProtocolError("pool key length is outside the safe range")
+    return value
+
+
+def _identifier(name: str, value: Any, *, allow_empty: bool = False) -> str:
+    if not isinstance(value, str):
+        raise ProtocolError(f"invalid {name}")
+    text = value
+    if allow_empty and not text:
+        return text
+    if not IDENTIFIER.fullmatch(text):
+        raise ProtocolError(f"invalid {name}")
+    return text
+
+
+def sign_envelope(
+    key: bytes,
+    kind: str,
+    binding: dict[str, Any],
+    payload: Any,
+    *,
+    now: int | None = None,
+    lifetime: int = 120,
+    delivery_id: str | None = None,
+) -> dict[str, Any]:
+    """Bind a message to the exact Kanban run, ordinal, and attempt."""
+    current = int(time.time()) if now is None else int(now)
+    lifetime = max(1, min(int(lifetime), MAX_ENVELOPE_LIFETIME))
+    envelope = {
+        "version": PROTOCOL_VERSION,
+        "kind": kind,
+        "board": str(binding.get("board") or ""),
+        "task_id": str(binding.get("task_id") or ""),
+        "run_id": str(binding.get("run_id") or ""),
+        "worker_ordinal": int(binding.get("worker_ordinal", -1)),
+        "attempt": int(binding.get("attempt", 0)),
+        "delivery_id": delivery_id or str(uuid.uuid4()),
+        "issued_at": current,
+        "expires_at": current + lifetime,
+        "payload_digest": payload_digest(payload),
+        "payload": payload,
+    }
+    if kind not in KINDS:
+        raise ProtocolError("unsupported message kind")
+    unsigned = canonical_json(envelope)
+    if len(unsigned) > MAX_WIRE_BYTES:
+        raise ProtocolError("pool message exceeds the wire limit")
+    envelope["signature"] = hmac.new(key, unsigned, hashlib.sha256).hexdigest()
+    return envelope
+
+
+def verify_envelope(
+    key: bytes,
+    envelope: Any,
+    *,
+    expected_kind: str | None = None,
+    now: int | None = None,
+) -> dict[str, Any]:
+    """Verify structure, lifetime, digest, and HMAC before using a message."""
+    if not isinstance(envelope, dict) or len(canonical_json(envelope)) > MAX_WIRE_BYTES:
+        raise ProtocolError("invalid or oversized pool message")
+    required = {"version", "kind", "board", "task_id", "run_id", "worker_ordinal", "attempt", "delivery_id", "issued_at", "expires_at", "payload_digest", "payload", "signature"}
+    if set(envelope) != required or envelope.get("version") != PROTOCOL_VERSION:
+        raise ProtocolError(
+            f"pool message fields do not match version {PROTOCOL_VERSION}"
+        )
+    kind = envelope["kind"]
+    if not isinstance(kind, str):
+        raise ProtocolError("unexpected message kind")
+    if kind not in KINDS or (expected_kind and kind != expected_kind):
+        raise ProtocolError("unexpected message kind")
+    empty_binding = kind in {"poll", "ack"}
+    _identifier("board", envelope["board"], allow_empty=empty_binding)
+    _identifier("task_id", envelope["task_id"], allow_empty=empty_binding)
+    _identifier("run_id", envelope["run_id"], allow_empty=empty_binding)
+    _identifier("delivery_id", envelope["delivery_id"])
+    numeric = tuple(envelope[name] for name in ("worker_ordinal", "attempt", "issued_at", "expires_at"))
+    if any(type(value) is not int for value in numeric):
+        raise ProtocolError("invalid numeric binding")
+    ordinal, attempt, issued, expires = numeric
+    if ordinal not in range(3) or attempt < 0:
+        raise ProtocolError("worker binding is outside the pool")
+    current = int(time.time()) if now is None else int(now)
+    if issued > current + MAX_CLOCK_SKEW or expires < current - MAX_CLOCK_SKEW:
+        raise ProtocolError("pool message is outside its validity window")
+    if expires <= issued or expires - issued > MAX_ENVELOPE_LIFETIME:
+        raise ProtocolError("pool message lifetime is invalid")
+    if envelope["payload_digest"] != payload_digest(envelope["payload"]):
+        raise ProtocolError("pool payload digest mismatch")
+    signature = str(envelope["signature"])
+    unsigned = dict(envelope)
+    unsigned.pop("signature")
+    expected = hmac.new(key, canonical_json(unsigned), hashlib.sha256).hexdigest()
+    if not hmac.compare_digest(signature, expected):
+        raise ProtocolError("pool message authentication failed")
+    return envelope
+
+
+def parse_wire(body: bytes) -> dict[str, Any]:
+    if not body or len(body) > MAX_WIRE_BYTES:
+        raise ProtocolError("empty or oversized request")
+    try:
+        value = json.loads(body)
+    except (UnicodeDecodeError, json.JSONDecodeError) as error:
+        raise ProtocolError("malformed JSON request") from error
+    if not isinstance(value, dict):
+        raise ProtocolError("request must be a JSON object")
+    return value
+
+
+def __getattr__(name: str) -> Any:
+    """Expose the durable store on the stable protocol surface.
+
+    ``PoolStore`` moved to :mod:`execution_pool_store` to keep both modules under
+    the managed line ceiling. It is resolved lazily here so the store can keep
+    importing this module's primitives without an import cycle.
+    """
+    if name == "PoolStore":
+        from execution_pool_store import PoolStore
+
+        return PoolStore
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

services/hermes/scripts/execution_pool_scm.py

@@ -0,0 +1,335 @@
+#!/usr/bin/env python3
+"""Assignment-bound Git gate routed exclusively through the PR14 SCM broker."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import stat
+import subprocess
+import threading
+import urllib.parse
+from pathlib import Path
+from typing import Any
+
+import scm_broker_client
+from execution_pool_project import ATLAS_REPO, validate_branch
+from execution_pool_protocol import ProtocolError, atomic_json, verify_envelope
+
+
+WORKSPACE_ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
+SCM_ROOT = Path(os.environ.get("HERMES_SCM_STATE_ROOT", "/scm-state"))
+ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
+BROKER_ORIGIN = scm_broker_client.BROKER_ORIGIN.rstrip("/")
+MAX_STATUS_BYTES = 4 * 1024 * 1024
+
+
+def _git_environment() -> dict[str, str]:
+    """Run Git without credentials, prompts, ambient config, or hook execution."""
+    return {
+        "HOME": "/nonexistent",
+        "PATH": "/usr/bin:/bin",
+        "GIT_CONFIG_NOSYSTEM": "1",
+        "GIT_TERMINAL_PROMPT": "0",
+    }
+
+
+def _run(*arguments: str, cwd: Path | None = None, timeout: int = 300) -> str:
+    command = [
+        "/usr/bin/git",
+        "-c",
+        "core.hooksPath=/dev/null",
+        "-c",
+        "core.fsmonitor=false",
+        *arguments,
+    ]
+    completed = subprocess.run(
+        command,
+        cwd=cwd,
+        env=_git_environment(),
+        stdin=subprocess.DEVNULL,
+        text=True,
+        capture_output=True,
+        timeout=timeout,
+        check=False,
+    )
+    if completed.returncode:
+        message = (completed.stderr or completed.stdout or "SCM operation failed")[-2000:]
+        raise RuntimeError(message.strip())
+    if len(completed.stdout.encode()) > MAX_STATUS_BYTES:
+        raise ProtocolError("SCM command output exceeds the safe limit")
+    return completed.stdout.strip()
+
+
+def _binding(envelope: dict[str, Any]) -> tuple[dict[str, Any], str, str, str]:
+    if envelope["kind"] != "assignment" or int(envelope["worker_ordinal"]) != ORDINAL:
+        raise ProtocolError("assignment does not belong to this worker ordinal")
+    payload = envelope.get("payload")
+    if not isinstance(payload, dict):
+        raise ProtocolError("assignment payload must be an object")
+    repo = str(payload.get("repo_url") or "")
+    match = ATLAS_REPO.fullmatch(repo)
+    if not match:
+        raise ProtocolError("assignment repository is outside Atlas")
+    try:
+        branch = validate_branch(payload.get("branch"), feature=True)
+        base = validate_branch(payload.get("base_branch"), feature=False)
+    except ValueError as error:
+        raise ProtocolError(str(error)) from error
+    return payload, match.group("repo"), branch, base
+
+
+def _broker_repo(repo: str) -> str:
+    return f"{BROKER_ORIGIN}/git/atlas/{repo}.git"
+
+
+def workspace_path(envelope: dict[str, Any]) -> Path:
+    """Derive a private ordinal path; no caller-provided path is accepted."""
+    parts = tuple(str(envelope[name]) for name in ("board", "task_id", "run_id"))
+    identifier = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z")
+    if any(not identifier.fullmatch(part) for part in parts):
+        raise ProtocolError("assignment path binding is invalid")
+    if WORKSPACE_ROOT.is_symlink():
+        raise ProtocolError("workspace root must not be a symlink")
+    workspace_root = WORKSPACE_ROOT.resolve()
+    root = WORKSPACE_ROOT / "runs"
+    if root.is_symlink():
+        raise ProtocolError("workspace run root must not be a symlink")
+    root.mkdir(mode=0o700, parents=True, exist_ok=True)
+    root = root.resolve()
+    root.relative_to(workspace_root)
+    candidate = root.joinpath(*parts)
+    current = root
+    for part in parts[:-1]:
+        current /= part
+        if current.is_symlink():
+            raise ProtocolError("workspace parent must not be a symlink")
+        current.mkdir(mode=0o700, exist_ok=True)
+    if candidate.is_symlink():
+        raise ProtocolError("workspace must not be a symlink")
+    candidate.resolve(strict=False).relative_to(root)
+    return candidate
+
+
+def _regular_text(path: Path, maximum: int) -> str:
+    descriptor = os.open(
+        path, os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0)
+    )
+    try:
+        info = os.fstat(descriptor)
+        if not stat.S_ISREG(info.st_mode) or info.st_size > maximum:
+            raise ProtocolError("private SCM state is invalid")
+        raw = os.read(descriptor, maximum + 1)
+    finally:
+        os.close(descriptor)
+    try:
+        return raw.decode().strip()
+    except UnicodeError as error:
+        raise ProtocolError("private SCM state is malformed") from error
+
+
+def _state_path(envelope: dict[str, Any]) -> Path:
+    if SCM_ROOT.is_symlink():
+        raise ProtocolError("private SCM root must not be a symlink")
+    root = SCM_ROOT.resolve()
+    root.mkdir(mode=0o700, parents=True, exist_ok=True)
+    name = "-".join(
+        str(envelope[name]) for name in ("board", "task_id", "run_id")
+    )
+    path = root / f"{name}.json"
+    if path.is_symlink():
+        raise ProtocolError("private SCM state must not be a symlink")
+    path.resolve(strict=False).relative_to(root)
+    return path
+
+
+def submission_refs(branch: str, attempt: int, head: str) -> tuple[str, ...]:
+    """Candidate push refs for one attempt, in order of preference.
+
+    The broker accepts branch *creation* only, so a retry that adds commits can
+    never update the ref a previous attempt already published, and no attempt may
+    ever move a protected ref. Each fallback is therefore a fresh name in the same
+    reviewed namespace -- bound first to the exact attempt, then to the exact
+    content -- so every push stays a creation and stays idempotent under replay.
+    """
+    candidates = (
+        branch,
+        f"{branch}-attempt-{max(1, int(attempt))}",
+        f"{branch}-{head[:12]}",
+    )
+    allowed: list[str] = []
+    for candidate in dict.fromkeys(candidates):
+        try:
+            allowed.append(validate_branch(candidate, feature=True))
+        except ValueError:
+            continue
+    if not allowed:
+        raise ProtocolError("no reviewed branch name is available for this attempt")
+    return tuple(allowed)
+
+
+def _remote_heads(destination: Path, refs: tuple[str, ...]) -> dict[str, str]:
+    """Read the broker's current head for each candidate ref, read-only."""
+    output = _run(
+        "ls-remote", "--heads", "hermes-broker",
+        *(f"refs/heads/{ref}" for ref in refs),
+        cwd=destination, timeout=300,
+    )
+    heads: dict[str, str] = {}
+    for line in output.splitlines():
+        fields = line.split()
+        if len(fields) == 2 and fields[1].startswith("refs/heads/"):
+            heads[fields[1].removeprefix("refs/heads/")] = fields[0]
+    return heads
+
+
+def _workspace_identity(destination: Path, repo: str, branch: str) -> str:
+    """Validate the checkout using bounded Git plumbing with all hooks disabled."""
+    if destination.is_symlink() or not (destination / ".git").is_dir():
+        raise ProtocolError("workspace Git metadata is unavailable")
+    ref_path = destination / ".git/refs/heads"
+    for part in branch.split("/"):
+        ref_path /= part
+        if ref_path.is_symlink():
+            raise ProtocolError("workspace branch ref must not be a symlink")
+    origin = _run("remote", "get-url", "origin", cwd=destination)
+    if origin != f"https://scm.bstein.dev/atlas/{repo}.git":
+        raise ProtocolError("workspace origin does not match assignment")
+    broker = _run("remote", "get-url", "hermes-broker", cwd=destination)
+    if broker != _broker_repo(repo):
+        raise ProtocolError("workspace broker remote does not match assignment")
+    current = _run("symbolic-ref", "--short", "HEAD", cwd=destination)
+    if current != branch:
+        raise ProtocolError("workspace branch does not match assignment")
+    head = _run("rev-parse", "--verify", "HEAD", cwd=destination)
+    if not re.fullmatch(r"[0-9a-f]{40,64}", head):
+        raise ProtocolError("workspace HEAD is invalid")
+    return head
+
+
+class Boundary:
+    """The only process allowed to turn model output into an SCM/result handoff."""
+
+    def __init__(self, key: bytes):
+        self.key = key
+        self.lock = threading.RLock()
+
+    def verify(self, raw: Any) -> dict[str, Any]:
+        return verify_envelope(self.key, raw, expected_kind="assignment")
+
+    def checkout(self, envelope: dict[str, Any]) -> dict[str, Any]:
+        _payload, repo, branch, base = _binding(envelope)
+        destination = workspace_path(envelope)
+        state_path = _state_path(envelope)
+        with self.lock:
+            if (destination / ".git").exists():
+                _workspace_identity(destination, repo, branch)
+                state = json.loads(_regular_text(state_path, 16 * 1024))
+                baseline = state.get("baseline_sha") if isinstance(state, dict) else None
+                if not isinstance(baseline, str) or not re.fullmatch(
+                    r"[0-9a-f]{40,64}", baseline
+                ):
+                    raise ProtocolError("private SCM baseline is unavailable")
+                return {"workspace": str(destination), "baseline_sha": baseline}
+            if destination.exists() and any(destination.iterdir()):
+                raise ProtocolError("workspace is non-empty and unmanaged")
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            broker = _broker_repo(repo)
+            try:
+                _run(
+                    "clone", "--single-branch", "--branch", branch, "--no-tags",
+                    broker, str(destination), timeout=900,
+                )
+            except RuntimeError as error:
+                if destination.exists() and any(destination.iterdir()):
+                    raise ProtocolError(
+                        "failed branch checkout left unmanaged workspace state"
+                    ) from error
+                if destination.exists():
+                    destination.rmdir()
+                _run(
+                    "clone", "--single-branch", "--branch", base, "--no-tags",
+                    broker, str(destination), timeout=900,
+                )
+                _run("checkout", "-b", branch, cwd=destination)
+            _run(
+                "remote", "set-url", "origin",
+                f"https://scm.bstein.dev/atlas/{repo}.git", cwd=destination,
+            )
+            _run("remote", "add", "hermes-broker", broker, cwd=destination)
+            _run("config", "user.name", "Hermes Execution Worker", cwd=destination)
+            _run("config", "user.email", "hermes@bstein.dev", cwd=destination)
+            baseline = _workspace_identity(destination, repo, branch)
+            atomic_json(
+                state_path,
+                {"baseline_sha": baseline, "repo": repo, "branch": branch},
+            )
+        return {"workspace": str(destination), "baseline_sha": baseline}
+
+    @staticmethod
+    def _draft(repo: str, branch: str, base: str, head: str, title: str, body: str) -> str:
+        query = urllib.parse.urlencode(
+            {"state": "open", "head": f"atlas:{branch}", "limit": 10}
+        )
+        existing = json.loads(
+            scm_broker_client.read(f"/api/v1/repos/atlas/{repo}/pulls?{query}")
+        )
+        if isinstance(existing, list) and existing:
+            return str(existing[0].get("html_url") or "")
+        created = json.loads(
+            scm_broker_client.create_draft(
+                repo,
+                base=base,
+                head=branch,
+                head_sha=head,
+                title=title,
+                body=body,
+            )
+        )
+        return str(created.get("html_url") or "")
+
+    def submit(self, envelope: dict[str, Any], request: dict[str, Any]) -> dict[str, Any]:
+        """Enforce clean/committed state, broker push, and reviewed draft creation."""
+        _payload, repo, branch, base = _binding(envelope)
+        destination = workspace_path(envelope)
+        title = str(request.get("title") or f"Hermes task {envelope['task_id']}").strip()
+        body = str(request.get("body") or "Automated Hermes draft.")
+        if not title or len(title.encode()) > 512 or len(body.encode()) > 32 * 1024:
+            raise ProtocolError("pull-request metadata exceeds the safe limit")
+        with self.lock:
+            head = _workspace_identity(destination, repo, branch)
+            state = json.loads(_regular_text(_state_path(envelope), 16 * 1024))
+            baseline = state.get("baseline_sha") if isinstance(state, dict) else ""
+            if not isinstance(baseline, str) or not re.fullmatch(
+                r"[0-9a-f]{40,64}", baseline
+            ):
+                raise ProtocolError("private SCM baseline is unavailable")
+            status = _run(
+                "status", "--porcelain=v1", "--untracked-files=all", cwd=destination
+            )
+            if status:
+                raise ProtocolError("workspace has uncommitted or untracked files")
+            candidates = submission_refs(branch, int(envelope["attempt"]), head)
+            heads = _remote_heads(destination, candidates)
+            # A candidate already at this exact head was published by an earlier
+            # attempt or an interrupted one: adopt it instead of pushing again, so
+            # replay is a no-op and prior work is never re-derived or dropped.
+            published = next(
+                (ref for ref in candidates if heads.get(ref) == head), None
+            )
+            ahead = int(_run("rev-list", "--count", f"{baseline}..{head}", cwd=destination))
+            if ahead <= 0 and published is None:
+                return {"workspace": str(destination), "branch": branch, "pull_request": ""}
+            target = published or next(
+                (ref for ref in candidates if ref not in heads), None
+            )
+            if target is None:
+                raise ProtocolError("every reviewed branch name for this attempt is taken")
+            if published is None:
+                _run(
+                    "push", "hermes-broker", f"HEAD:refs/heads/{target}",
+                    cwd=destination, timeout=900,
+                )
+            pull = self._draft(repo, target, base, head, title, body)
+        return {"workspace": str(destination), "branch": target, "pull_request": pull}

services/hermes/scripts/execution_pool_server.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""Versioned HTTP and maintenance loop for the execution-pool coordinator."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sqlite3
+import sys
+import threading
+import time
+from collections.abc import Iterable
+from http.server import BaseHTTPRequestHandler
+from pathlib import Path
+from typing import Any
+
+from execution_pool_protocol import (
+    MAX_WIRE_BYTES,
+    PROTOCOL_VERSION,
+    BoundedHTTPServer,
+    ProtocolError,
+    canonical_json,
+    parse_wire,
+    read_key,
+)
+from execution_pool_store import PoolStore
+
+
+DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data"))
+STATE_ROOT = DATA_ROOT / "execution-pool"
+KEY_PATH = Path(
+    os.environ.get(
+        "HERMES_EXECUTION_POOL_KEY_FILE", "/runtime-access/execution-pool-key"
+    )
+)
+PORT = int(os.environ.get("HERMES_EXECUTION_POOL_PORT", "9007"))
+RETENTION_SECONDS = int(
+    os.environ.get("HERMES_EXECUTION_POOL_RETENTION_SECONDS", "1209600")
+)
+
+
+def handler_factory(coordinator: Any) -> type[BaseHTTPRequestHandler]:
+    class Handler(BaseHTTPRequestHandler):
+        server_version = f"hermes-execution-pool/{PROTOCOL_VERSION}"
+
+        def _reply(self, status: int, value: dict[str, Any]) -> None:
+            body = canonical_json(value)
+            self.send_response(status)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(body)))
+            self.send_header("Cache-Control", "no-store")
+            self.end_headers()
+            self.wfile.write(body)
+
+        def do_GET(self) -> None:  # noqa: N802
+            if self.path != "/ready":
+                self._reply(404, {"error": "not found"})
+                return
+            try:
+                coordinator.store.available_ordinals()
+                self._reply(200, {"ready": True, "version": PROTOCOL_VERSION})
+            except (OSError, sqlite3.Error):
+                self._reply(503, {"ready": False, "version": PROTOCOL_VERSION})
+
+        def do_POST(self) -> None:  # noqa: N802
+            try:
+                length = int(self.headers.get("Content-Length", "0"))
+                if length <= 0 or length > MAX_WIRE_BYTES:
+                    raise ProtocolError("invalid content length")
+                envelope = parse_wire(self.rfile.read(length))
+                routes = {
+                    "/v1/poll": coordinator.poll,
+                    "/v1/heartbeat": coordinator.heartbeat,
+                    "/v1/result": coordinator.result,
+                }
+                if self.path not in routes:
+                    self._reply(404, {"error": "not found"})
+                    return
+                self._reply(200, routes[self.path](envelope))
+            except ProtocolError as error:
+                self._reply(409, {"error": str(error)})
+            except Exception as error:
+                print(
+                    f"pool request failed: {type(error).__name__}: {error}",
+                    file=sys.stderr,
+                    flush=True,
+                )
+                self._reply(503, {"error": "coordinator unavailable"})
+
+        def log_message(self, _format: str, *_arguments: Any) -> None:
+            return
+
+    return Handler
+
+
+def maintenance_cycle(operations: Iterable[Any]) -> int:
+    """Run one maintenance pass; a failing operation never aborts the others.
+
+    Startup uses this too. A durable row that one pass cannot process must not
+    stop the coordinator from binding its port, because the store lives on a PVC
+    and would otherwise turn a single bad row into a permanent crash loop.
+    """
+    deferred = 0
+    for operation in operations:
+        try:
+            operation()
+        except Exception as error:  # noqa: BLE001 - the loop is the recovery mechanism
+            deferred += 1
+            print(
+                f"pool maintenance deferred: {type(error).__name__}: {error}",
+                file=sys.stderr,
+                flush=True,
+            )
+    return deferred
+
+
+def run(coordinator_type: type[Any]) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--once", action="store_true")
+    args = parser.parse_args()
+    store = PoolStore(STATE_ROOT / "assignments.db")
+    coordinator = coordinator_type(read_key(KEY_PATH), store)
+    operations = (
+        coordinator.expire_leases,
+        coordinator.recover_results,
+        coordinator.reconcile,
+        coordinator.dispatch,
+        lambda: store.garbage_collect(RETENTION_SECONDS),
+    )
+    maintenance_cycle(operations)
+    if args.once:
+        return 0
+    server = BoundedHTTPServer(
+        ("0.0.0.0", PORT), handler_factory(coordinator), max_workers=8
+    )
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    while True:
+        maintenance_cycle(operations)
+        time.sleep(5)

services/hermes/scripts/execution_pool_store.py

@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+"""Durable, restart-safe assignment state for the Hermes execution pool.
+
+The store owns every transition of a claimed Kanban run through the pool. It
+deliberately keeps two invariants that the coordinator depends on for recovery:
+
+* ``lease_failed`` is a *retryable* state, not a terminal one. A row reaches it
+  when the pool has released the ordinal but has not yet confirmed the outcome
+  with Kanban, so the coordinator must keep retrying it.
+* Only ``finalized`` and ``stale`` are terminal, and only terminal rows are ever
+  garbage-collected. A row is therefore never removed before the pool holds
+  authoritative evidence about what happened to its run.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import threading
+import time
+from pathlib import Path
+from typing import Any
+
+from execution_pool_protocol import ProtocolError, canonical_json, payload_digest
+
+
+LIVE_STATES = ("assigned", "running", "result")
+TERMINAL_STATES = ("finalized", "stale")
+LEASE_FAILED = "lease_failed"
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS assignments (
+  board TEXT NOT NULL, task_id TEXT NOT NULL, run_id TEXT NOT NULL,
+  worker_ordinal INTEGER NOT NULL CHECK(worker_ordinal BETWEEN 0 AND 2),
+  attempt INTEGER NOT NULL, assignment_digest TEXT NOT NULL,
+  payload_json TEXT NOT NULL, state TEXT NOT NULL,
+  lease_until REAL NOT NULL DEFAULT 0, last_heartbeat REAL NOT NULL DEFAULT 0,
+  result_digest TEXT, result_json TEXT, created_at REAL NOT NULL,
+  updated_at REAL NOT NULL, PRIMARY KEY(board, task_id, run_id)
+);
+CREATE UNIQUE INDEX IF NOT EXISTS one_live_assignment_per_worker
+  ON assignments(worker_ordinal) WHERE state IN ('assigned','running','result');
+CREATE TABLE IF NOT EXISTS deliveries (
+  delivery_id TEXT PRIMARY KEY, kind TEXT NOT NULL, digest TEXT NOT NULL,
+  received_at REAL NOT NULL
+);
+"""
+
+
+class PoolStore:
+    """Coordinator-owned durable assignments; never stores provider secrets."""
+
+    def __init__(self, path: Path, lease_seconds: int = 90):
+        self.path = path
+        self.lease_seconds = max(60, min(int(lease_seconds), 600))
+        self._lock = threading.RLock()
+        path.parent.mkdir(parents=True, exist_ok=True)
+        self._initialize()
+
+    def _connect(self) -> sqlite3.Connection:
+        connection = sqlite3.connect(self.path, timeout=10, isolation_level=None)
+        connection.row_factory = sqlite3.Row
+        connection.execute("PRAGMA journal_mode=WAL")
+        connection.execute("PRAGMA synchronous=FULL")
+        connection.execute("PRAGMA busy_timeout=10000")
+        return connection
+
+    def _initialize(self) -> None:
+        with self._connect() as connection:
+            connection.executescript(SCHEMA)
+
+    @staticmethod
+    def _record(row: sqlite3.Row | None) -> dict[str, Any] | None:
+        if row is None:
+            return None
+        value = dict(row)
+        value["payload"] = json.loads(value.pop("payload_json"))
+        if value.get("result_json"):
+            value["result"] = json.loads(value["result_json"])
+        return value
+
+    def _select(self, sql: str) -> list[dict[str, Any]]:
+        with self._connect() as connection:
+            rows = connection.execute(sql).fetchall()
+        return [self._record(row) or {} for row in rows]
+
+    def add(self, binding: dict[str, Any], payload: dict[str, Any]) -> bool:
+        """Create exactly one assignment for a claimed run and free ordinal."""
+        now = time.time()
+        digest = payload_digest(payload)
+        values = (
+            binding["board"], binding["task_id"], binding["run_id"],
+            binding["worker_ordinal"], binding["attempt"], digest,
+            canonical_json(payload).decode(), "assigned", now, now,
+        )
+        with self._lock, self._connect() as connection:
+            try:
+                connection.execute("BEGIN IMMEDIATE")
+                connection.execute(
+                    """INSERT INTO assignments
+                    (board,task_id,run_id,worker_ordinal,attempt,assignment_digest,
+                     payload_json,state,created_at,updated_at)
+                    VALUES (?,?,?,?,?,?,?,?,?,?)""",
+                    values,
+                )
+                connection.commit()
+                return True
+            except sqlite3.IntegrityError as error:
+                connection.rollback()
+                existing = connection.execute(
+                    """SELECT assignment_digest,worker_ordinal,attempt FROM assignments
+                    WHERE board=? AND task_id=? AND run_id=?""",
+                    values[:3],
+                ).fetchone()
+                if existing and tuple(existing) == (
+                    digest, binding["worker_ordinal"], binding["attempt"]
+                ):
+                    return False
+                if existing:
+                    raise ProtocolError("conflicting duplicate assignment") from error
+                raise ProtocolError("worker ordinal already has a live assignment") from error
+
+    def available_ordinals(self) -> list[int]:
+        with self._connect() as connection:
+            rows = connection.execute(
+                "SELECT worker_ordinal FROM assignments WHERE state IN ('assigned','running','result')"
+            ).fetchall()
+        occupied = {int(row[0]) for row in rows}
+        return [ordinal for ordinal in range(3) if ordinal not in occupied]
+
+    def active_assignments(self) -> list[dict[str, Any]]:
+        return self._select(
+            "SELECT * FROM assignments WHERE state IN ('assigned','running','result')"
+        )
+
+    def known_runs(self) -> set[tuple[str, str, str]]:
+        """Every run identity this store already owns a row for, in any state.
+
+        Adoption must consult this rather than only the live states: re-adding a
+        run that still has a retrying or not-yet-collected row would collide with
+        ``PRIMARY KEY(board, task_id, run_id)`` under a different payload digest.
+        """
+        with self._connect() as connection:
+            rows = connection.execute(
+                "SELECT board,task_id,run_id FROM assignments"
+            ).fetchall()
+        return {(str(row[0]), str(row[1]), str(row[2])) for row in rows}
+
+    def offer(self, ordinal: int) -> dict[str, Any] | None:
+        """Return the ordinal's durable assignment, preserving restart identity."""
+        now = time.time()
+        with self._lock, self._connect() as connection:
+            connection.execute("BEGIN IMMEDIATE")
+            row = connection.execute(
+                """SELECT * FROM assignments WHERE worker_ordinal=?
+                AND state IN ('assigned','running') ORDER BY created_at LIMIT 1""",
+                (ordinal,),
+            ).fetchone()
+            if row is not None:
+                connection.execute(
+                    """UPDATE assignments SET state='running',lease_until=?,last_heartbeat=?,updated_at=?
+                    WHERE board=? AND task_id=? AND run_id=?""",
+                    (now + self.lease_seconds, now, now, row["board"], row["task_id"], row["run_id"]),
+                )
+            connection.commit()
+        return self._record(row)
+
+    def _matching(self, connection: sqlite3.Connection, envelope: dict[str, Any]) -> sqlite3.Row:
+        row = connection.execute(
+            "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?",
+            (envelope["board"], envelope["task_id"], envelope["run_id"]),
+        ).fetchone()
+        if row is None:
+            raise ProtocolError("assignment is unknown or stale")
+        if int(row["worker_ordinal"]) != int(envelope["worker_ordinal"]):
+            raise ProtocolError("worker ordinal does not own this assignment")
+        if int(row["attempt"]) != int(envelope["attempt"]):
+            raise ProtocolError("assignment attempt is stale")
+        return row
+
+    def heartbeat(self, envelope: dict[str, Any]) -> tuple[bool, bool]:
+        now = time.time()
+        delivery_digest = payload_digest(
+            {
+                name: envelope[name]
+                for name in (
+                    "kind", "board", "task_id", "run_id", "worker_ordinal",
+                    "attempt", "payload_digest",
+                )
+            }
+        )
+        with self._lock, self._connect() as connection:
+            connection.execute("BEGIN IMMEDIATE")
+            duplicate = connection.execute(
+                "SELECT digest FROM deliveries WHERE delivery_id=?",
+                (envelope["delivery_id"],),
+            ).fetchone()
+            row = self._matching(connection, envelope)
+            if row["state"] not in {"assigned", "running"}:
+                raise ProtocolError("assignment is no longer running")
+            if row["state"] == "running" and float(row["lease_until"]) < now:
+                raise ProtocolError("assignment lease expired")
+            if duplicate and duplicate[0] != delivery_digest:
+                raise ProtocolError("delivery identifier was reused")
+            if not duplicate:
+                connection.execute(
+                    "INSERT INTO deliveries VALUES (?,?,?,?)",
+                    (envelope["delivery_id"], "heartbeat", delivery_digest, now),
+                )
+            connection.execute(
+                "UPDATE assignments SET state='running',lease_until=?,last_heartbeat=?,updated_at=? WHERE board=? AND task_id=? AND run_id=?",
+                (now + self.lease_seconds, now, now, envelope["board"], envelope["task_id"], envelope["run_id"]),
+            )
+            connection.commit()
+        return True, bool(duplicate)
+
+    def accept_result(self, envelope: dict[str, Any]) -> tuple[dict[str, Any], bool]:
+        now = time.time()
+        with self._lock, self._connect() as connection:
+            connection.execute("BEGIN IMMEDIATE")
+            row = self._matching(connection, envelope)
+            digest = envelope["payload_digest"]
+            if row["result_digest"]:
+                if row["result_digest"] != digest:
+                    raise ProtocolError("conflicting result for completed delivery")
+                connection.rollback()
+                return self._record(row) or {}, True
+            if row["state"] not in {"assigned", "running"}:
+                raise ProtocolError("assignment cannot accept a result")
+            if row["state"] == "running" and float(row["lease_until"]) < now:
+                raise ProtocolError("assignment lease expired")
+            connection.execute(
+                """UPDATE assignments SET state='result',result_digest=?,result_json=?,updated_at=?
+                WHERE board=? AND task_id=? AND run_id=?""",
+                (digest, canonical_json(envelope["payload"]).decode(), now,
+                 envelope["board"], envelope["task_id"], envelope["run_id"]),
+            )
+            connection.commit()
+            row = connection.execute(
+                "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?",
+                (envelope["board"], envelope["task_id"], envelope["run_id"]),
+            ).fetchone()
+        return self._record(row) or {}, False
+
+    def pending_results(self) -> list[dict[str, Any]]:
+        return self._select(
+            "SELECT * FROM assignments WHERE state='result' ORDER BY updated_at"
+        )
+
+    def failed_leases(self) -> list[dict[str, Any]]:
+        """Rows whose ordinal is released but whose Kanban outcome is unconfirmed.
+
+        The coordinator drains this on every maintenance pass, so a Kanban write
+        that failed at expiry time is retried instead of stranding the run.
+        """
+        return self._select(
+            "SELECT * FROM assignments WHERE state='lease_failed' ORDER BY updated_at"
+        )
+
+    def expire_leases(
+        self, *, now: float | None = None, max_attempts: int = 3
+    ) -> list[dict[str, Any]]:
+        """Fence expired attempts and re-offer or terminally release their ordinals."""
+        current = time.time() if now is None else float(now)
+        maximum = max(1, min(int(max_attempts), 10))
+        changed: list[dict[str, Any]] = []
+        with self._lock, self._connect() as connection:
+            connection.execute("BEGIN IMMEDIATE")
+            rows = connection.execute(
+                """SELECT * FROM assignments WHERE state='running'
+                AND lease_until > 0 AND lease_until < ? ORDER BY updated_at""",
+                (current,),
+            ).fetchall()
+            for row in rows:
+                if int(row["attempt"]) >= maximum:
+                    state, attempt = LEASE_FAILED, int(row["attempt"])
+                else:
+                    state, attempt = "assigned", int(row["attempt"]) + 1
+                connection.execute(
+                    """UPDATE assignments SET state=?,attempt=?,lease_until=0,
+                    last_heartbeat=0,updated_at=? WHERE board=? AND task_id=? AND run_id=?
+                    AND attempt=? AND state='running'""",
+                    (
+                        state, attempt, current, row["board"], row["task_id"],
+                        row["run_id"], row["attempt"],
+                    ),
+                )
+                updated = connection.execute(
+                    "SELECT * FROM assignments WHERE board=? AND task_id=? AND run_id=?",
+                    (row["board"], row["task_id"], row["run_id"]),
+                ).fetchone()
+                if updated is not None:
+                    changed.append(self._record(updated) or {})
+            connection.commit()
+        return changed
+
+    def finalize(self, binding: dict[str, Any], state: str) -> bool:
+        """Apply one terminal state to the exact run, ordinal, and attempt."""
+        if state not in TERMINAL_STATES:
+            raise ProtocolError("invalid terminal assignment state")
+        with self._lock, self._connect() as connection:
+            cursor = connection.execute(
+                "UPDATE assignments SET state=?,updated_at=? WHERE board=? AND task_id=? AND run_id=? AND worker_ordinal=? AND attempt=?",
+                (state, time.time(), *(binding[name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt"))),
+            )
+            return bool(cursor.rowcount)
+
+    def garbage_collect(self, retention_seconds: int) -> int:
+        """Remove aged rows, and only ones with authoritative terminal evidence.
+
+        ``lease_failed`` is excluded on purpose: dropping a row whose Kanban
+        outcome was never confirmed would lose the pool's only record that the
+        run still needs to be surfaced.
+        """
+        cutoff = time.time() - max(3600, retention_seconds)
+        with self._lock, self._connect() as connection:
+            cursor = connection.execute(
+                "DELETE FROM assignments WHERE state IN ('finalized','stale') AND updated_at < ?",
+                (cutoff,),
+            )
+            connection.execute("DELETE FROM deliveries WHERE received_at < ?", (cutoff,))
+            return int(cursor.rowcount)

services/hermes/scripts/execution_pool_worker.py

@@ -0,0 +1,445 @@
+#!/usr/bin/env python3
+"""Run one fenced Hermes assignment on an ordinal-scoped durable workspace."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import shutil
+import stat
+import subprocess
+import time
+import urllib.error
+import urllib.request
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any
+
+import cli_lane_runner
+from execution_pool_protocol import (
+    ProtocolError,
+    atomic_json,
+    canonical_json,
+)
+
+
+ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
+ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
+CLIENT = os.environ.get(
+    "HERMES_EXECUTION_CLIENT_URL",
+    f"http://hermes-execution-mediator-{ORDINAL}.hermes.svc.cluster.local:9009",
+).rstrip("/")
+NODE = os.environ.get("HERMES_WORKER_NODE", "unknown")[:128]
+RUN_PART = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
+RETENTION_SECONDS = int(os.environ.get("HERMES_WORKER_RETENTION_SECONDS", "1209600"))
+
+
+def _post(url: str, value: dict[str, Any], timeout: int = 60) -> dict[str, Any]:
+    request = urllib.request.Request(
+        url, data=canonical_json(value), method="POST",
+        headers={"Content-Type": "application/json", "Cache-Control": "no-store"},
+    )
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        body = response.read(64 * 1024 + 1)
+    if len(body) > 64 * 1024:
+        raise ProtocolError("server response exceeds the wire limit")
+    document = json.loads(body)
+    if not isinstance(document, dict):
+        raise ProtocolError("server response is not an object")
+    return document
+
+
+def _client(operation: str, **values: Any) -> dict[str, Any]:
+    response = _post(f"{CLIENT}/v1/client", {"operation": operation, **values})
+    if response.get("error"):
+        raise ProtocolError(str(response["error"]))
+    return response
+
+
+def _poll() -> dict[str, Any] | None:
+    assignment = _client("poll").get("assignment")
+    if assignment is None:
+        return None
+    if (
+        not isinstance(assignment, dict)
+        or int(assignment["worker_ordinal"]) != ORDINAL
+        or int(assignment.get("protocol_version", 0)) != 2
+    ):
+        raise ProtocolError("local boundary returned a foreign assignment")
+    return assignment
+
+
+def _binding(assignment: dict[str, Any]) -> dict[str, Any]:
+    return {name: assignment[name] for name in (
+        "board", "task_id", "run_id", "worker_ordinal", "attempt"
+    )}
+
+
+def _state_path(assignment: dict[str, Any]) -> Path:
+    parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id"))
+    if any(not RUN_PART.fullmatch(part) for part in parts):
+        raise ProtocolError("assignment state path is invalid")
+    configured_root = ROOT / "session-state"
+    if configured_root.is_symlink():
+        raise ProtocolError("assignment state root must not be a symlink")
+    root = configured_root.resolve()
+    path = root.joinpath(*parts).with_suffix(".json")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if path.is_symlink() or any(parent.is_symlink() for parent in path.parents if parent != root.parent):
+        raise ProtocolError("assignment state path contains a symlink")
+    path.resolve(strict=False).relative_to(root)
+    return path
+
+
+def _bind_provider_sessions(assignment: dict[str, Any]) -> None:
+    """Attach provider session directories to this exact durable task/run."""
+    parts = tuple(str(assignment[name]) for name in ("board", "task_id", "run_id"))
+    if any(not RUN_PART.fullmatch(part) for part in parts):
+        raise ProtocolError("provider session binding is invalid")
+    configured_root = ROOT / "provider-state"
+    if configured_root.is_symlink() or not configured_root.is_dir():
+        raise ProtocolError("durable provider state root is unavailable")
+    provider_root = configured_root.resolve()
+    run_root = provider_root
+    for part in parts:
+        run_root /= part
+        if run_root.is_symlink():
+            raise ProtocolError("provider session path contains a symlink")
+        run_root.mkdir(mode=0o700, exist_ok=True)
+        run_root.resolve().relative_to(provider_root)
+    home = run_root / "home"
+    if home.is_symlink():
+        raise ProtocolError("provider HOME must not be a durable symlink")
+    home.mkdir(mode=0o700, parents=True, exist_ok=True)
+    runtime_home = cli_lane_runner.DATA_ROOT / "home"
+    if runtime_home.is_symlink():
+        runtime_home.unlink()
+    elif runtime_home.exists():
+        raise ProtocolError("provider HOME is not a task-bound symlink")
+    runtime_home.symlink_to(home)
+    (home / ".claude").mkdir(mode=0o700, exist_ok=True)
+    settings = home / ".claude/settings.json"
+    if not settings.exists():
+        atomic_json(settings, {}, 0o600)
+    bindings = {
+        Path(os.environ.get("CODEX_HOME", "/runtime-access/codex")) / "sessions":
+            run_root / "codex/sessions",
+        Path(os.environ.get("CLAUDE_CONFIG_DIR", "/runtime-access/claude")) / "projects":
+            run_root / "claude/projects",
+        Path(os.environ.get("CLAUDE_CONFIG_DIR", "/runtime-access/claude")) / "session-env":
+            run_root / "claude/session-env",
+        Path(os.environ.get("CLAUDE_CONFIG_DIR", "/runtime-access/claude")) / "todos":
+            run_root / "claude/todos",
+    }
+    for runtime, durable in bindings.items():
+        current = run_root
+        for part in durable.relative_to(run_root).parts:
+            current /= part
+            if current.is_symlink():
+                raise ProtocolError("provider session path contains a symlink")
+            current.mkdir(mode=0o700, exist_ok=True)
+            current.resolve().relative_to(provider_root)
+        if runtime.is_symlink():
+            runtime.unlink()
+        elif runtime.exists():
+            raise ProtocolError(f"provider session path is not a symlink: {runtime.name}")
+        runtime.symlink_to(durable)
+
+
+def _prompt(context: str, workspace: Path, binding: dict[str, Any]) -> str:
+    return f"""You are a durable coding worker managed by Hermes Kanban.
+
+Work only on this objective and its acceptance criteria:
+
+{context}
+
+Workspace: {workspace}
+Run binding: board={binding['board']} task={binding['task_id']} run={binding['run_id']} worker={ORDINAL}
+
+Operate autonomously only inside this private assigned checkout. Inspect before editing,
+preserve unrelated and untracked files, and fail closed rather than overwrite state you do
+not understand. You have no Kubernetes identity and no SCM credential. Do not attempt to
+read Secrets, mutate workloads, use exec/attach/port-forward, reach node roots, or bypass
+the reviewed SCM boundary. Commit intended changes locally on the assigned feature branch;
+the worker boundary handles the bounded push and draft pull request after validation.
+Switchyard owns provider/model/effort selection and cross-provider fallback.
+
+Return a final JSON object matching the supplied schema. Use status=incomplete when work,
+tests, commits, or verification remain. Use blocked only for a concrete task obstacle.
+Completed must have no blockers. List changed files, tests, artifacts, findings, and blockers.
+"""
+
+
+def _read_activity(log_path: Path, offset: int) -> tuple[str, int]:
+    flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0)
+    try:
+        descriptor = os.open(log_path, flags)
+    except FileNotFoundError:
+        return "", offset
+    try:
+        info = os.fstat(descriptor)
+        if not stat.S_ISREG(info.st_mode):
+            raise ProtocolError("worker log must be a regular file")
+        if info.st_size < offset:
+            offset = 0
+        os.lseek(descriptor, offset, os.SEEK_SET)
+        value = os.read(descriptor, 12 * 1024)
+        next_offset = os.lseek(descriptor, 0, os.SEEK_CUR)
+    finally:
+        os.close(descriptor)
+    return value.decode("utf-8", "replace"), next_offset
+
+
+def _git(workspace: Path, *arguments: str) -> str:
+    completed = subprocess.run(
+        ["git", "-C", str(workspace), *arguments], stdin=subprocess.DEVNULL,
+        text=True, capture_output=True, timeout=30, check=False,
+        env={**os.environ, "GIT_TERMINAL_PROMPT": "0", "GIT_ASKPASS": "/bin/false"},
+    )
+    if completed.returncode:
+        raise RuntimeError((completed.stderr or "Git inspection failed")[-1000:])
+    return completed.stdout.strip()
+
+
+def _bounded_result(value: dict[str, Any]) -> dict[str, Any]:
+    """Keep terminal evidence useful while fitting the authenticated wire cap."""
+    bounded: dict[str, Any] = {}
+    for name in ("status", "summary"):
+        bounded[name] = str(value.get(name) or "")[:8000]
+    for name in ("changed_files", "tests_run", "artifacts", "findings", "blockers"):
+        items = value.get(name)
+        bounded[name] = [str(item)[:2000] for item in items[:100]] if isinstance(items, list) else []
+    while len(canonical_json(bounded)) > 32 * 1024:
+        longest = max(
+            (name for name in bounded if isinstance(bounded[name], list) and bounded[name]),
+            key=lambda name: len(canonical_json(bounded[name])),
+            default="",
+        )
+        if longest:
+            bounded[longest].pop()
+        else:
+            bounded["summary"] = bounded["summary"][: len(bounded["summary"]) // 2]
+    return bounded
+
+
+def _refresh_assignment(binding: dict[str, Any]) -> dict[str, Any]:
+    fresh = _poll()
+    if fresh is None or _binding(fresh) != binding:
+        raise ProtocolError("assignment changed before SCM submission")
+    return fresh
+
+
+def garbage_collect(now: float | None = None) -> int:
+    """Remove only clean, terminal, assignment-derived workspaces after retention."""
+    current = time.time() if now is None else now
+    state_root = (ROOT / "session-state").resolve()
+    run_root = (ROOT / "runs").resolve()
+    removed = 0
+    for state_file in state_root.glob("*/*/*.json") if state_root.is_dir() else ():
+        state = cli_lane_runner.load_json(state_file)
+        terminal_at = float(state.get("terminal_at") or 0)
+        if terminal_at <= 0 or current - terminal_at < max(3600, RETENTION_SECONDS):
+            continue
+        workspace = Path(str(state.get("workspace") or ""))
+        try:
+            workspace.resolve(strict=True).relative_to(run_root)
+        except (OSError, RuntimeError, ValueError):
+            continue
+        if workspace.is_symlink() or _git(
+            workspace, "status", "--porcelain=v1", "--untracked-files=all"
+        ):
+            continue
+        shutil.rmtree(workspace)
+        state_file.unlink()
+        removed += 1
+    return removed
+
+
+def execute(assignment: dict[str, Any]) -> None:
+    binding = _binding(assignment)
+    payload = assignment["payload"]
+    if not isinstance(payload, dict):
+        raise ProtocolError("assignment payload is invalid")
+    _bind_provider_sessions(assignment)
+    workspace = Path(str(assignment.get("workspace") or "")).resolve(strict=True)
+    workspace.relative_to((ROOT / "runs").resolve())
+    state_file = _state_path(assignment)
+    state = cli_lane_runner.load_json(state_file)
+    state.update({**binding, "node": NODE, "workspace": str(workspace)})
+    state.setdefault("baseline_sha", str(assignment.get("baseline_sha") or ""))
+    atomic_json(state_file, state)
+    log_path = state_file.with_suffix(".log")
+    offset = 0
+    latest_route: dict[str, Any] = {}
+
+    def heartbeat(note: str) -> bool:
+        nonlocal offset
+        try:
+            activity, offset = _read_activity(log_path, offset)
+            response = _client(
+                "heartbeat", binding=binding,
+                payload={
+                    "note": f"worker={ORDINAL} node={NODE} {note}"[:1000],
+                    "activity": activity, "route": latest_route,
+                },
+            )
+            return bool(response.get("ack", {}).get("accepted"))
+        except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError):
+            return False
+
+    context = str(payload.get("context") or "")
+    assignee = str(payload.get("assignee") or "cli-auto")
+    route = cli_lane_runner.select_route(
+        context, assignee,
+        exclude_provider=cli_lane_runner.fresh_unavailable_provider()
+        if assignee == "cli-auto" else None,
+    )
+    latest_route = asdict(route)
+    if not heartbeat(
+        f"route={route.provider}/{route.model}/{route.effort}; assignment accepted"
+    ):
+        raise ProtocolError("assignment lease was lost before provider startup")
+    deadline = min(
+        int(payload.get("deadline_unix") or int(time.time()) + 60),
+        int(time.time()) + int(payload.get("max_runtime_seconds") or 60),
+    )
+    result = cli_lane_runner.run_provider(
+        route, _prompt(context, workspace, binding), workspace, state, state_file,
+        log_path, heartbeat, max(1, deadline - int(time.time())),
+    )
+    if result.capacity_failure and deadline - time.time() > 30:
+        alternate = "claude" if route.provider == "codex" else "codex"
+        fallback = cli_lane_runner.select_route(
+            context + "\nThe first provider failed from capacity or authentication.",
+            f"cli-{alternate}-{route.effort}",
+        )
+        latest_route = asdict(fallback)
+        heartbeat(f"fallback={route.provider}->{fallback.provider}")
+        result = cli_lane_runner.run_provider(
+            fallback,
+            _prompt(context, workspace, binding)
+            + cli_lane_runner.git_handoff(workspace, result.output),
+            workspace, state, state_file, log_path, heartbeat,
+            max(1, deadline - int(time.time())),
+        )
+        route = fallback
+    structured = dict(result.structured or {})
+    for name in ("changed_files", "tests_run", "artifacts", "findings", "blockers"):
+        if not isinstance(structured.get(name), list):
+            structured[name] = []
+    if structured.get("status") == "completed" and result.returncode == 0:
+        status = _git(workspace, "status", "--porcelain=v1", "--untracked-files=all")
+        if status:
+            structured["status"] = "incomplete"
+            structured["blockers"].append(
+                "Worker left uncommitted or untracked files; SCM submission failed closed."
+            )
+        else:
+            baseline = str(state.get("baseline_sha") or "")
+            if not re.fullmatch(r"[0-9a-f]{40,64}", baseline):
+                raise RuntimeError("durable SCM baseline is missing or invalid")
+            _refresh_assignment(binding)
+    structured = _bounded_result(structured)
+    activity, _ = _read_activity(log_path, offset)
+    result_payload = {
+        "structured": structured,
+        "returncode": result.returncode,
+        "capacity_failure": result.capacity_failure,
+        "node": NODE,
+        "route": asdict(route),
+        "provider_sessions": {
+            "codex_thread_id": state.get("codex_thread_id"),
+            "claude_session_id": state.get("claude_session_id"),
+        },
+        "final_activity": activity,
+    }
+    response = _client(
+        "finish",
+        binding=binding,
+        payload=result_payload,
+        title=str(structured.get("summary") or f"Hermes task {binding['task_id']}")[:240],
+        body=json.dumps(structured, indent=2, sort_keys=True)[:12000],
+    )
+    if not response.get("ack", {}).get("accepted"):
+        raise ProtocolError("coordinator did not accept the terminal result")
+    state["terminal_at"] = time.time()
+    atomic_json(state_file, state)
+
+
+def report_exception(assignment: dict[str, Any], error: Exception) -> bool:
+    """Surface one worker exception as an exact-run transient result."""
+    binding = _binding(assignment)
+    detail = f"{type(error).__name__}: {error}"[:1500]
+    structured = {
+        "status": "blocked",
+        "summary": "Distributed worker failed before a safe handoff.",
+        "changed_files": [],
+        "tests_run": [],
+        "artifacts": [],
+        "findings": [],
+        "blockers": [detail],
+    }
+    payload = {
+        "structured": structured,
+        "returncode": 1,
+        # This is an execution-infrastructure failure, not an objective capability
+        # verdict. The exact-run result releases the ordinal for another task.
+        "capacity_failure": True,
+        "node": NODE,
+        "route": {},
+        "provider_sessions": {},
+        "final_activity": detail,
+    }
+    try:
+        return bool(
+            _client("finish", binding=binding, payload=payload).get("ack", {}).get(
+                "accepted"
+            )
+        )
+    except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError):
+        return False
+
+
+def readiness() -> None:
+    if ORDINAL not in range(3):
+        raise ProtocolError("worker ordinal must be 0, 1, or 2")
+    for path in (ROOT, ROOT / "provider-state", cli_lane_runner.DATA_ROOT):
+        if not path.is_dir() or not os.access(path, os.W_OK):
+            raise ProtocolError(f"durable worker path is not writable: {path}")
+    for credential in (
+        Path(os.environ.get("CODEX_HOME", "/runtime-access/codex")) / "auth.json",
+        Path(os.environ.get("CLAUDE_CONFIG_DIR", "/runtime-access/claude")) / ".credentials.json",
+    ):
+        if not credential.is_file() or not os.access(credential, os.W_OK):
+            raise ProtocolError(f"subscription credential is unavailable or read-only: {credential.name}")
+    cli_lane_runner.RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
+    atomic_json(
+        cli_lane_runner.RESULT_SCHEMA_PATH, cli_lane_runner.RESULT_SCHEMA, 0o644
+    )
+    _poll()
+
+
+def main() -> int:
+    readiness()
+    while True:
+        assignment = None
+        try:
+            garbage_collect()
+            assignment = _poll()
+            if assignment is None:
+                time.sleep(5)
+                continue
+            execute(assignment)
+        except (OSError, RuntimeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as error:
+            surfaced = assignment is not None and report_exception(assignment, error)
+            print(
+                f"worker {ORDINAL} {'surfaced' if surfaced else 'deferred'}: "
+                f"{type(error).__name__}: {error}",
+                flush=True,
+            )
+            time.sleep(10)
+
+
+if __name__ == "__main__":  # pragma: no cover - exercised by the container entrypoint
+    raise SystemExit(main())

services/hermes/scripts/kanban_supervisor.py

@@ -0,0 +1,329 @@
+#!/usr/bin/env python3
+"""In-pod autonomous supervisor that sustains multi-hour unattended work.
+
+The deployed Hermes agent already decomposes objectives and iterates a within-
+card goal loop, but nothing in-pod spawns the *cross-card* review->repair->
+re-review follow-ups after a card finishes; today that chain only exists as an
+external Claude Code "codex-shepherd" session on an operator's workstation, so
+unattended runs stall once the first implementation card completes. This
+supervisor closes that gap from inside the pod.
+
+It runs a bounded poll loop that reads board state through ``hermes_cli.kanban_db``
+and, for each terminal card, applies exactly one follow-up:
+
+* a done implementation with a produced PR/commit and no existing review ->
+  create a review card (routed to a review assignee via the normal cli lane);
+* a review card that returned SHIP -> mark the implementation ready for a human
+  to merge (a comment/flag; the supervisor never merges, approves, or clears WIP);
+* a review card that returned BLOCK -> create a bounded repair card;
+* a repair card that produced a new commit -> create the re-review card.
+
+All decision logic lives in :mod:`supervisor_policy` (pure, unit tested). This
+module is only the I/O shell: config, board iteration, and turning a
+:class:`supervisor_policy.Decision` into ``create_task``/``block_task``/
+``add_comment`` calls. It imports no provider client and holds no metered path;
+the sole spawn is a Kanban card that routes through the existing subscription
+lanes. It is inert until ``kanban.auto_supervise`` is set true in the deployed
+config, re-read every tick like ``kanban.auto_decompose``.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable
+
+import yaml
+
+import supervisor_policy as policy
+
+DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data"))
+CONFIG_PATH = DATA_ROOT / "config.yaml"
+# Persistent record of terminal (ship / escalate) emissions so each fires at
+# most once across the stateless poll loop, and survives a pod restart.
+LEDGER_PATH = DATA_ROOT / "supervisor" / "emitted.json"
+
+DEFAULT_INTERVAL_SECONDS = 30
+DEFAULT_MAX_CYCLES = 5
+DEFAULT_MAX_CHAINS = 20
+DEFAULT_REVIEW_ASSIGNEE = "cli-claude-xhigh"
+DEFAULT_REPAIR_ASSIGNEE = "cli-auto"
+
+
+@dataclass(frozen=True)
+class Settings:
+    """Per-tick configuration, re-read from the deployed config each pass."""
+
+    enabled: bool
+    interval: int
+    limits: policy.Limits
+
+
+def _kanban_config() -> dict[str, Any]:
+    try:
+        document = yaml.safe_load(CONFIG_PATH.read_text(encoding="utf-8"))
+    except (OSError, yaml.YAMLError):
+        return {}
+    kanban = document.get("kanban") if isinstance(document, dict) else None
+    return kanban if isinstance(kanban, dict) else {}
+
+
+def _bool(value: Any, default: bool) -> bool:
+    return value if isinstance(value, bool) else default
+
+
+def _positive_int(value: Any, default: int) -> int:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return default
+    parsed = int(value)
+    return parsed if parsed > 0 else default
+
+
+def _text(value: Any, default: str) -> str:
+    return value if isinstance(value, str) and value.strip() else default
+
+
+def load_settings() -> Settings:
+    """Read the ``kanban`` config block; default to inert and conservative."""
+    cfg = _kanban_config()
+    return Settings(
+        enabled=_bool(cfg.get("auto_supervise"), False),
+        interval=_positive_int(
+            cfg.get("supervise_interval_seconds"), DEFAULT_INTERVAL_SECONDS
+        ),
+        limits=policy.Limits(
+            max_cycles=_positive_int(cfg.get("supervise_max_cycles"), DEFAULT_MAX_CYCLES),
+            max_chains=_positive_int(cfg.get("supervise_max_chains"), DEFAULT_MAX_CHAINS),
+            review_assignee=_text(
+                cfg.get("supervise_review_assignee"), DEFAULT_REVIEW_ASSIGNEE
+            ),
+            repair_assignee=_text(
+                cfg.get("supervise_repair_assignee"), DEFAULT_REPAIR_ASSIGNEE
+            ),
+        ),
+    )
+
+
+def _board_slug(board: Any) -> str:
+    if isinstance(board, dict):
+        return str(board.get("slug") or board.get("id") or "")
+    return str(getattr(board, "slug", None) or getattr(board, "id", None) or board or "")
+
+
+def _log(message: str) -> None:
+    print(f"kanban-supervisor: {message}", file=sys.stderr, flush=True)
+
+
+class Ledger:
+    """Durable set of emission keys the supervisor itself has already fired.
+
+    The poll loop is stateless and re-plans every terminal card each tick, so a
+    SHIP or a cycle-limit escalation would otherwise re-comment its target on
+    every pass. This ledger is the authoritative read-before-write guard: an
+    emission is performed only when its key is absent, then recorded, so each
+    fires at most once regardless of whether the underlying ``kanban_db`` moves
+    the source card out of its done state.
+    """
+
+    def __init__(self, path: Path):
+        self._path = path
+        self._keys = self._load(path)
+
+    @staticmethod
+    def _load(path: Path) -> set[str]:
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            return set()
+        return {str(key) for key in data} if isinstance(data, list) else set()
+
+    def has(self, key: str) -> bool:
+        return key in self._keys
+
+    def record(self, key: str) -> None:
+        self._keys.add(key)
+        try:
+            self._path.parent.mkdir(parents=True, exist_ok=True)
+            tmp = self._path.with_suffix(".tmp")
+            tmp.write_text(json.dumps(sorted(self._keys)), encoding="utf-8")
+            os.replace(tmp, self._path)
+        except OSError as error:
+            _log(f"could not persist emission ledger: {error}")
+
+
+def _iter_boards(kanban_db: Any) -> list[str]:
+    try:
+        boards = kanban_db.list_boards(include_archived=False)
+    except Exception as error:  # noqa: BLE001 - one bad registry never stops the tick
+        _log(f"could not list boards: {error}")
+        return []
+    slugs = [_board_slug(raw) for raw in boards]
+    return [slug for slug in slugs if slug]
+
+
+def _comment(kanban_db: Any, conn: Any, task_id: str, body: str) -> None:
+    try:
+        kanban_db.add_comment(conn, task_id, policy.SUPERVISOR_AUTHOR, body)
+    except Exception as error:  # noqa: BLE001 - a comment is best-effort signalling
+        _log(f"could not comment on {task_id}: {error}")
+
+
+def _mark_ready_for_human(kanban_db: Any, conn: Any, task_id: str, body: str) -> None:
+    """Flag the implementation as human-mergeable. Never merges or clears WIP."""
+    setter = getattr(kanban_db, "set_task_metadata", None) or getattr(
+        kanban_db, "update_task_metadata", None
+    )
+    if callable(setter):
+        try:
+            setter(conn, task_id, {"supervisor_ready_for_human_merge": True})
+        except Exception as error:  # noqa: BLE001 - the comment is the durable flag
+            _log(f"could not set ready flag on {task_id}: {error}")
+    _comment(kanban_db, conn, task_id, body)
+
+
+def _escalate(kanban_db: Any, conn: Any, task_id: str, reason: str) -> None:
+    body = (
+        f"supervisor fail-closed escalation: {reason}. Human attention required; "
+        "no follow-up card was auto-created."
+    )
+    try:
+        kanban_db.block_task(conn, task_id, reason=body, kind="supervisor")
+    except Exception as error:  # noqa: BLE001 - still record the human-visible comment
+        _log(f"could not block {task_id}: {error}")
+    _comment(kanban_db, conn, task_id, body)
+
+
+def _already_created(kanban_db: Any, conn: Any, payload: dict[str, Any]) -> bool:
+    """True if a card matching this spawn payload's stamp already exists."""
+    supervisor_stamp = (payload.get("metadata") or {}).get("supervisor") or {}
+    kind = supervisor_stamp.get("kind")
+    root = str(supervisor_stamp.get("root") or "")
+    commit = str(supervisor_stamp.get("head_commit") or "")
+    if not kind or not root:
+        return False
+    try:
+        tasks = list(kanban_db.list_tasks(conn))
+    except Exception as error:  # noqa: BLE001 - if we cannot confirm, do not skip
+        _log(f"could not re-scan before spawn retry: {error}")
+        return False
+    return policy.existing_followup(tasks, kind, root, commit)
+
+
+def _spawn(kanban_db: Any, conn: Any, decision: policy.Decision) -> None:
+    payload = dict(decision.payload or {})
+    try:
+        kanban_db.create_task(conn, **payload)
+    except TypeError:
+        # create_task may reject idempotency_key on older runtimes. That is
+        # raised at call binding, before any insert, but a post-insert TypeError
+        # is also possible, so re-run the dedup scan first: only retry the create
+        # when no matching card exists, so a partial insert is never doubled.
+        if not _already_created(kanban_db, conn, payload):
+            payload.pop("idempotency_key", None)
+            kanban_db.create_task(conn, **payload)
+    _comment(kanban_db, conn, decision.target_id, f"supervisor: {decision.reason}")
+
+
+def apply_decision(
+    kanban_db: Any, conn: Any, decision: policy.Decision, ledger: Ledger
+) -> bool:
+    """Execute one decision. Returns True when an action was taken.
+
+    Spawns are self-limiting across ticks (the created card fails the next dedup
+    scan). SHIP and escalate targets stay in their done state, so they are gated
+    on the persistent ledger and fire at most once per source card.
+    """
+    if decision.action == "none":
+        return False
+    policy.assert_safe(decision)
+    if decision.action == "spawn":
+        _spawn(kanban_db, conn, decision)
+        return True
+    key = f"{decision.action}:{decision.target_id}"
+    if ledger.has(key):
+        return False  # already emitted on a prior tick; never re-emit or re-block
+    if decision.action == "ship":
+        info = decision.payload or {}
+        commit = info.get("commit", "")
+        pr = info.get("pr", "") or "branch on record"
+        body = (
+            f"supervisor: review returned SHIP for commit {commit} ({pr}). "
+            "READY FOR HUMAN MERGE - a human must merge; the supervisor never "
+            "merges, approves, closes, or clears WIP."
+        )
+        _mark_ready_for_human(kanban_db, conn, decision.target_id, body)
+    else:  # escalate
+        _escalate(kanban_db, conn, decision.target_id, decision.reason)
+    ledger.record(key)
+    return True
+
+
+def supervise_board(
+    kanban_db: Any, board: str, limits: policy.Limits, ledger: Ledger
+) -> int:
+    actions = 0
+    with kanban_db.scoped_current_board(board):
+        conn = kanban_db.connect(board=board)
+        try:
+            tasks = list(kanban_db.list_tasks(conn))
+            for task in tasks:
+                decision = policy.plan(task, tasks, limits)
+                try:
+                    if apply_decision(kanban_db, conn, decision, ledger):
+                        actions += 1
+                except Exception as error:  # noqa: BLE001 - one card never stops others
+                    _log(f"action failed on board {board!r}: {error}")
+        finally:
+            conn.close()
+    return actions
+
+
+def supervise_once(
+    kanban_db: Any, limits: policy.Limits, ledger: Ledger | None = None
+) -> int:
+    """One bounded pass over every non-archived board."""
+    if ledger is None:
+        ledger = Ledger(LEDGER_PATH)
+    total = 0
+    for board in _iter_boards(kanban_db):
+        try:
+            total += supervise_board(kanban_db, board, limits, ledger)
+        except Exception as error:  # noqa: BLE001 - isolate a faulty board
+            _log(f"temporarily skipping board {board!r}: {error}")
+    return total
+
+
+def run_forever(
+    kanban_db: Any,
+    *,
+    sleep: Callable[[float], None] = time.sleep,
+    load: Callable[[], Settings] = load_settings,
+    max_ticks: int | None = None,
+) -> int:
+    """Poll loop. Inert while ``auto_supervise`` is false; re-reads config a tick."""
+    ticks = 0
+    while max_ticks is None or ticks < max_ticks:
+        settings = load()
+        if settings.enabled:
+            try:
+                supervise_once(kanban_db, settings.limits)
+            except Exception as error:  # noqa: BLE001 - never let one tick kill the loop
+                _log(f"tick failed: {error}")
+        sleep(settings.interval)
+        ticks += 1
+    return ticks
+
+
+def main(argv: list[str] | None = None) -> int:
+    from hermes_cli import kanban_db
+
+    run_forever(kanban_db)
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover - process entry point
+    sys.exit(main(sys.argv[1:]))