hermes: bind terminal commit to journal inode

dockerfiles/Dockerfile.hermes-agent

@@ -1544,6 +1544,28 @@ function RootRedirect() {
 }
 NODE
 
+COPY dockerfiles/patch-hermes-execution-safety.py /tmp/patch-hermes-execution-safety.py
+COPY dockerfiles/hermes-execution-safety-regression.py /tmp/hermes-execution-safety-regression.py
+COPY dockerfiles/hermes_execution_patch_support.py /tmp/hermes_execution_patch_support.py
+COPY dockerfiles/patch_hermes_run_safety.py /tmp/patch_hermes_run_safety.py
+COPY dockerfiles/patch_hermes_decomposition_safety.py /tmp/patch_hermes_decomposition_safety.py
+COPY dockerfiles/hermes_execution_regression_support.py /tmp/hermes_execution_regression_support.py
+COPY dockerfiles/hermes_run_safety_regression.py /tmp/hermes_run_safety_regression.py
+COPY dockerfiles/hermes_decomposition_safety_regression.py /tmp/hermes_decomposition_safety_regression.py
+COPY services/hermes/scripts/cli_lane_*.py /tmp/hermes-lane-regression/
+RUN /opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py \
+    && HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
+        /opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py \
+    && rm /tmp/patch-hermes-execution-safety.py \
+        /tmp/hermes-execution-safety-regression.py \
+        /tmp/hermes_execution_patch_support.py \
+        /tmp/patch_hermes_run_safety.py \
+        /tmp/patch_hermes_decomposition_safety.py \
+        /tmp/hermes_execution_regression_support.py \
+        /tmp/hermes_run_safety_regression.py \
+        /tmp/hermes_decomposition_safety_regression.py \
+    && find /tmp/hermes-lane-regression -depth -delete
+
 COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate
 
 RUN cd /opt/hermes/web \
@@ -1602,6 +1624,8 @@ RUN cd /opt/hermes/web \
         /opt/hermes/agent/turn_context.py \
         /opt/hermes/agent/conversation_loop.py \
         /opt/hermes/hermes_cli/oneshot.py \
+        /opt/hermes/hermes_cli/kanban_decompose.py \
+        /opt/hermes/gateway/kanban_watchers.py \
         /opt/hermes/tools/delegate_tool.py \
         /opt/hermes/tools/web_tools.py \
         /opt/hermes/tools/python_sandbox_tool.py \

dockerfiles/Dockerfile.hermes-agent.dockerignore

@@ -5,3 +5,15 @@
 !dockerfiles/hermes-public-extract/**
 !dockerfiles/hermes-session-activity-panel.tsx
 !dockerfiles/hermes-session-migrate.py
+!dockerfiles/patch-hermes-execution-safety.py
+!dockerfiles/hermes-execution-safety-regression.py
+!dockerfiles/hermes_execution_patch_support.py
+!dockerfiles/patch_hermes_run_safety.py
+!dockerfiles/patch_hermes_decomposition_safety.py
+!dockerfiles/hermes_execution_regression_support.py
+!dockerfiles/hermes_run_safety_regression.py
+!dockerfiles/hermes_decomposition_safety_regression.py
+!services/
+!services/hermes/
+!services/hermes/scripts/
+!services/hermes/scripts/cli_lane_*.py

dockerfiles/hermes-execution-safety-regression.py

@@ -0,0 +1,23 @@
+"""Run the composable Hermes execution-safety build regressions."""
+
+import unittest
+
+from hermes_decomposition_safety_regression import DecompositionSafetyTests
+from hermes_run_safety_regression import ExactRunSafetyTests
+
+
+def main() -> int:
+    """Run both transaction domains and return a build-friendly status."""
+    loader = unittest.defaultTestLoader
+    suite = unittest.TestSuite(
+        (
+            loader.loadTestsFromTestCase(ExactRunSafetyTests),
+            loader.loadTestsFromTestCase(DecompositionSafetyTests),
+        )
+    )
+    result = unittest.TextTestRunner(verbosity=2).run(suite)
+    return 0 if result.wasSuccessful() else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dockerfiles/hermes_decomposition_safety_regression.py

@@ -0,0 +1,120 @@
+"""Regress automatic decomposition against existing execution history."""
+
+from __future__ import annotations
+
+
+from hermes_cli import kanban_db
+from hermes_cli import kanban_decompose
+from hermes_execution_regression_support import ExecutionSafetyTestCase
+
+
+class DecompositionSafetyTests(ExecutionSafetyTestCase):
+    """Verify only unexecuted triage roots may decompose automatically."""
+
+    def test_automatic_decomposer_skips_executed_triage_task(self) -> None:
+        task_id = self._route_executed_task_to_triage()
+        completions = self._client(
+            {"fanout": False, "title": "must not run", "body": "must not run"}
+        )
+        before = len(kanban_db.list_tasks(self.connection))
+
+        outcome = kanban_decompose.decompose_task(task_id, automatic=True)
+
+        self.assertFalse(outcome.ok)
+        self.assertIn("execution history", outcome.reason)
+        self.assertEqual(completions.calls, 0)
+        self.assertEqual(self._status(task_id), "triage")
+        self.assertEqual(len(kanban_db.list_tasks(self.connection)), before)
+
+    def test_manual_decomposition_remains_available_after_execution(self) -> None:
+        task_id = self._route_executed_task_to_triage()
+        completions = self._client(
+            {
+                "fanout": False,
+                "title": "operator-approved retry",
+                "body": "Retain one bounded task.",
+                "assignee": "default",
+            }
+        )
+
+        outcome = kanban_decompose.decompose_task(task_id, author="operator")
+
+        self.assertTrue(outcome.ok)
+        self.assertFalse(outcome.fanout)
+        self.assertEqual(completions.calls, 1)
+        self.assertEqual(self._status(task_id), "ready")
+
+    def test_fresh_triage_task_still_auto_promotes(self) -> None:
+        task_id = kanban_db.create_task(
+            self.connection,
+            title="fresh objective",
+            triage=True,
+        )
+        completions = self._client(
+            {
+                "fanout": False,
+                "title": "specified objective",
+                "body": "One bounded task.",
+                "assignee": "default",
+            }
+        )
+
+        outcome = kanban_decompose.decompose_task(task_id, automatic=True)
+
+        self.assertTrue(outcome.ok)
+        self.assertEqual(completions.calls, 1)
+        self.assertEqual(self._status(task_id), "ready")
+
+    def test_execution_history_gained_during_single_llm_call_blocks_commit(self) -> None:
+        task_id = kanban_db.create_task(
+            self.connection,
+            title="concurrent single objective",
+            triage=True,
+        )
+        completions = self._client(
+            {
+                "fanout": False,
+                "title": "redundant rewrite",
+                "body": "must not be committed",
+                "assignee": "default",
+            },
+            before_response=lambda: self._insert_run(task_id),
+        )
+
+        outcome = kanban_decompose.decompose_task(task_id, automatic=True)
+
+        self.assertFalse(outcome.ok)
+        self.assertIn("gained execution history", outcome.reason)
+        self.assertEqual(completions.calls, 1)
+        self.assertEqual(self._status(task_id), "triage")
+
+    def test_execution_history_gained_during_fanout_llm_call_blocks_commit(self) -> None:
+        task_id = kanban_db.create_task(
+            self.connection,
+            title="concurrent triage objective",
+            triage=True,
+        )
+
+        completions = self._client(
+            {
+                "fanout": True,
+                "tasks": [
+                    {
+                        "title": "redundant child",
+                        "body": "must not be created",
+                        "assignee": "default",
+                        "parents": [],
+                    }
+                ],
+            },
+            before_response=lambda: self._insert_run(task_id),
+        )
+        before = len(kanban_db.list_tasks(self.connection))
+
+        outcome = kanban_decompose.decompose_task(task_id, automatic=True)
+
+        self.assertFalse(outcome.ok)
+        self.assertIn("gained execution history", outcome.reason)
+        self.assertEqual(completions.calls, 1)
+        self.assertEqual(self._status(task_id), "triage")
+        self.assertEqual(len(kanban_db.list_tasks(self.connection)), before)

dockerfiles/hermes_execution_patch_support.py

@@ -0,0 +1,12 @@
+"""Fail-closed helpers shared by Hermes source patches."""
+
+
+
+def replace_once(source: str, before: str, after: str, label: str) -> str:
+    """Replace one anchored upstream fragment and fail closed on source drift."""
+    count = source.count(before)
+    if count != 1:
+        raise SystemExit(
+            f"Hermes {label} patch context changed: expected 1, found {count}"
+        )
+    return source.replace(before, after, 1)

dockerfiles/hermes_execution_regression_support.py

@@ -0,0 +1,132 @@
+"""Shared isolated database fixture for Hermes execution-safety regressions."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest import mock
+
+_HERMES_HOME = tempfile.TemporaryDirectory(prefix="hermes-execution-test-home-")
+os.environ["HERMES_HOME"] = _HERMES_HOME.name
+_LANE_SOURCE = Path(
+    os.environ.get("HERMES_CLI_LANE_SOURCE", "/tmp/hermes-lane-regression")
+)
+sys.path.insert(0, str(_LANE_SOURCE))
+
+from agent import auxiliary_client  # noqa: E402
+from cli_lane_config import TerminalIdentity  # noqa: E402
+from cli_lane_finalization import _finalize_document_db  # noqa: E402
+from hermes_cli import kanban_db  # noqa: E402
+from hermes_cli import kanban_decompose  # noqa: E402
+
+__all__ = ("ExecutionSafetyTestCase", "TerminalIdentity", "_finalize_document_db")
+
+
+class _FakeCompletions:
+    def __init__(self, payload: dict, before_response=None) -> None:
+        self.payload = payload
+        self.before_response = before_response
+        self.calls = 0
+
+    def create(self, **_kwargs):
+        self.calls += 1
+        if self.before_response is not None:
+            self.before_response()
+        return SimpleNamespace(
+            choices=[
+                SimpleNamespace(
+                    message=SimpleNamespace(content=json.dumps(self.payload))
+                )
+            ]
+        )
+
+
+class _FakeClient:
+    def __init__(self, completions: _FakeCompletions) -> None:
+        self.chat = SimpleNamespace(completions=completions)
+
+
+class ExecutionSafetyTestCase(unittest.TestCase):
+    """Exercise patched upstream APIs against an isolated real database."""
+
+    def setUp(self) -> None:
+        self.connection = kanban_db.connect()
+        self.patches = [
+            mock.patch.object(
+                kanban_decompose,
+                "_build_roster",
+                return_value=([], {"default"}),
+            ),
+            mock.patch.object(
+                kanban_decompose,
+                "_load_config",
+                return_value={"kanban": {"auto_promote_children": True}},
+            ),
+            mock.patch.object(
+                auxiliary_client,
+                "get_auxiliary_extra_body",
+                return_value={},
+            ),
+        ]
+        for patch in self.patches:
+            patch.start()
+
+    def tearDown(self) -> None:
+        for patch in reversed(self.patches):
+            patch.stop()
+        self.connection.close()
+
+    def _client(self, payload: dict, before_response=None) -> _FakeCompletions:
+        completions = _FakeCompletions(payload, before_response)
+        client = _FakeClient(completions)
+        patch = mock.patch.object(
+            auxiliary_client,
+            "get_text_auxiliary_client",
+            return_value=(client, "test-decomposer"),
+        )
+        patch.start()
+        self.addCleanup(patch.stop)
+        return completions
+
+    def _status(self, task_id: str) -> str:
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertIsNotNone(task)
+        return task.status
+
+    def _route_executed_task_to_triage(self) -> str:
+        task_id = kanban_db.create_task(
+            self.connection,
+            title="already implemented parent",
+        )
+        for attempt in range(kanban_db.BLOCK_RECURRENCE_LIMIT):
+            if attempt:
+                self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+            self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+            self.assertTrue(
+                kanban_db.block_task(
+                    self.connection,
+                    task_id,
+                    reason="same unavailable capability",
+                    kind="capability",
+                )
+            )
+        self.assertEqual(self._status(task_id), "triage")
+        self.assertEqual(
+            len(kanban_db.list_runs(self.connection, task_id)),
+            kanban_db.BLOCK_RECURRENCE_LIMIT,
+        )
+        return task_id
+
+    def _insert_run(self, task_id: str) -> None:
+        with kanban_db.connect_closing() as connection:
+            kanban_db._synthesize_ended_run(
+                connection,
+                task_id,
+                outcome="reclaimed",
+                summary="concurrent execution evidence",
+            )

dockerfiles/hermes_run_safety_regression.py

@@ -0,0 +1,467 @@
+"""Regress exact-run replay, provenance, and reclaim safety."""
+
+from __future__ import annotations
+
+import threading
+from unittest import mock
+
+from hermes_cli import kanban_db
+from hermes_execution_regression_support import (
+    ExecutionSafetyTestCase,
+    TerminalIdentity,
+    _finalize_document_db,
+)
+from cli_lane_files import state_path
+from cli_lane_records import _write_terminal_record
+from cli_lane_recovery import recover_pending_finalizations
+
+
+class ExactRunSafetyTests(ExecutionSafetyTestCase):
+    """Verify exact-run transactions reject every stale concurrent actor."""
+
+    def _lane_document(self, summary: str) -> dict:
+        """Return the DB-bearing portion of one accepted lane journal."""
+        return {
+            "result": summary,
+            "summary": summary,
+            "metadata": {"source": "exact-run-regression"},
+        }
+
+    def test_lane_marks_replaced_ended_run_stale_without_mutation(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="replaced journal")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        first_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="first run ended after persistence",
+                expected_run_id=first_run,
+            )
+        )
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        second_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="replacement run also ended",
+                expected_run_id=second_run,
+            )
+        )
+        before = kanban_db.get_task(self.connection, task_id)
+
+        outcome = _finalize_document_db(
+            kanban_db,
+            TerminalIdentity("default", task_id, first_run, "pending"),
+            self._lane_document("stale first result"),
+        )
+
+        after = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(outcome, "stale")
+        self.assertEqual(after.status, before.status)
+        self.assertEqual(after.current_run_id, before.current_run_id)
+        self.assertEqual(after.result, before.result)
+        self.assertEqual(after.completed_run_id, before.completed_run_id)
+
+    def test_lane_marks_nonexistent_ended_run_stale_without_mutation(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="no such run")
+        before = kanban_db.get_task(self.connection, task_id)
+
+        outcome = _finalize_document_db(
+            kanban_db,
+            TerminalIdentity("default", task_id, 42, "pending"),
+            self._lane_document("orphan result"),
+        )
+
+        after = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(outcome, "stale")
+        self.assertEqual(after.status, before.status)
+        self.assertEqual(after.result, before.result)
+        self.assertEqual(kanban_db.list_runs(self.connection, task_id), [])
+
+    def test_full_lane_recovery_conflicts_a_replaced_ended_run(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="recovery conflict")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        first_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="first accepted run ended",
+                expected_run_id=first_run,
+            )
+        )
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        second_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="replacement ended",
+                expected_run_id=second_run,
+            )
+        )
+        before = kanban_db.get_task(self.connection, task_id)
+        state_file = state_path("default", task_id)
+        pending, _record = _write_terminal_record(
+            state_file,
+            board="default",
+            task_id=task_id,
+            run_id=first_run,
+            structured={
+                "status": "completed",
+                "summary": "accepted stale result",
+                "changed_files": [],
+                "tests_run": ["real SQLite recovery regression"],
+                "artifacts": [],
+                "findings": [],
+                "blockers": [],
+            },
+            summary="accepted stale result",
+            metadata={"source": "exact-run-regression"},
+        )
+
+        self.assertEqual(recover_pending_finalizations(), 0)
+
+        after = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(after.status, before.status)
+        self.assertEqual(after.current_run_id, before.current_run_id)
+        self.assertEqual(after.result, before.result)
+        self.assertFalse(pending.exists())
+        conflicts = list(
+            state_file.parent.glob(
+                f"{state_file.stem}.run-{first_run}.terminal.conflict-*.json"
+            )
+        )
+        self.assertEqual(len(conflicts), 1)
+
+    def test_latest_ended_run_can_replay_a_durable_completion(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="journaled result")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        run_id = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="legacy post-journal failure",
+                kind="capability",
+                expected_run_id=run_id,
+            )
+        )
+
+        with mock.patch.object(
+            kanban_db, "_fire_kanban_lifecycle_hook"
+        ) as lifecycle:
+            self.assertTrue(
+                kanban_db.complete_task(
+                    self.connection,
+                    task_id,
+                    result="durable terminal result",
+                    summary="durable terminal result",
+                    replay_ended_run_id=run_id,
+                )
+            )
+        lifecycle.assert_called_once()
+        self.assertEqual(lifecycle.call_args.args[:2], ("kanban_task_completed", task_id))
+        self.assertEqual(lifecycle.call_args.kwargs["run_id"], run_id)
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(task.status, "done")
+        self.assertEqual(task.completed_run_id, run_id)
+        runs = kanban_db.list_runs(self.connection, task_id)
+        self.assertEqual(len(runs), 1)
+        self.assertEqual(runs[0].id, run_id)
+        self.assertEqual(runs[0].status, "done")
+        self.assertEqual(runs[0].outcome, "completed")
+        self.assertEqual(runs[0].summary, "durable terminal result")
+        completed_events = self.connection.execute(
+            "SELECT run_id FROM task_events "
+            "WHERE task_id = ? AND kind = 'completed' ORDER BY id",
+            (task_id,),
+        ).fetchall()
+        self.assertEqual([row["run_id"] for row in completed_events], [run_id])
+
+    def test_unchanged_triage_can_replay_its_exact_latest_ended_run(self) -> None:
+        task_id = self._route_executed_task_to_triage()
+        runs = kanban_db.list_runs(self.connection, task_id)
+        run_id = runs[-1].id
+
+        self.assertTrue(
+            kanban_db.complete_task(
+                self.connection,
+                task_id,
+                result="accepted before the loop breaker routed triage",
+                summary="accepted before the loop breaker routed triage",
+                replay_ended_run_id=run_id,
+            )
+        )
+
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(task.status, "done")
+        self.assertEqual(task.completed_run_id, run_id)
+        updated_runs = kanban_db.list_runs(self.connection, task_id)
+        self.assertEqual(len(updated_runs), kanban_db.BLOCK_RECURRENCE_LIMIT)
+        self.assertEqual(updated_runs[-1].id, run_id)
+        self.assertEqual(updated_runs[-1].outcome, "completed")
+
+    def test_triage_status_transition_prevents_stale_exact_run_replay(self) -> None:
+        task_id = self._route_executed_task_to_triage()
+        run_id = kanban_db.list_runs(self.connection, task_id)[-1].id
+        self.assertTrue(
+            kanban_db.specify_triage_task(
+                self.connection,
+                task_id,
+                title="operator changed the objective",
+                author="operator",
+            )
+        )
+
+        self.assertFalse(
+            kanban_db.complete_task(
+                self.connection,
+                task_id,
+                result="stale accepted result",
+                replay_ended_run_id=run_id,
+            )
+        )
+        self.assertNotEqual(self._status(task_id), "done")
+
+    def test_decomposition_lineage_prevents_replay_even_if_status_returns(self) -> None:
+        task_id = self._route_executed_task_to_triage()
+        run_id = kanban_db.list_runs(self.connection, task_id)[-1].id
+        children = kanban_db.decompose_triage_task(
+            self.connection,
+            task_id,
+            root_assignee="default",
+            children=[{"title": "new child objective"}],
+            author="operator",
+            auto_promote=False,
+        )
+        self.assertEqual(len(children or []), 1)
+        with kanban_db.write_txn(self.connection):
+            self.connection.execute(
+                "UPDATE tasks SET status = 'ready' WHERE id = ?",
+                (task_id,),
+            )
+
+        self.assertFalse(
+            kanban_db.complete_task(
+                self.connection,
+                task_id,
+                result="pre-decomposition accepted result",
+                replay_ended_run_id=run_id,
+            )
+        )
+        self.assertEqual(self._status(task_id), "ready")
+
+    def test_older_ended_run_cannot_complete_over_a_replacement(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="replacement guard")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        old_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="first run",
+                kind="capability",
+                expected_run_id=old_run,
+            )
+        )
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        replacement_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertNotEqual(old_run, replacement_run)
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="replacement run",
+                kind="capability",
+                expected_run_id=replacement_run,
+            )
+        )
+
+        self.assertFalse(
+            kanban_db.complete_task(
+                self.connection,
+                task_id,
+                result="stale result",
+                replay_ended_run_id=old_run,
+            )
+        )
+        self.assertIn(self._status(task_id), {"blocked", "triage"})
+
+    def test_equal_result_bytes_remain_bound_to_the_completing_run(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="same result retries")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        first_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="first attempt ended",
+                expected_run_id=first_run,
+            )
+        )
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        second_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.complete_task(
+                self.connection,
+                task_id,
+                result="identical result bytes",
+                summary="second run wins",
+                expected_run_id=second_run,
+            )
+        )
+
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(task.completed_run_id, second_run)
+        self.assertNotEqual(task.completed_run_id, first_run)
+        runs = {run.id: run for run in kanban_db.list_runs(self.connection, task_id)}
+        self.assertEqual(runs[first_run].outcome, "blocked")
+        self.assertEqual(runs[second_run].outcome, "completed")
+
+    def test_exact_run_reclaim_succeeds_for_the_authoritative_run(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="recover exact run")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        run_id = kanban_db.get_task(self.connection, task_id).current_run_id
+
+        self.assertTrue(
+            kanban_db.reclaim_task(
+                self.connection,
+                task_id,
+                reason="invalid exact journal",
+                expected_run_id=run_id,
+            )
+        )
+        task = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(task.status, "ready")
+        self.assertIsNone(task.current_run_id)
+
+    def test_stale_reclaim_before_transaction_preserves_replacement_run(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="replacement before txn")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        old_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        self.assertTrue(
+            kanban_db.block_task(
+                self.connection,
+                task_id,
+                reason="old recovery run",
+                expected_run_id=old_run,
+            )
+        )
+        self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        replacement = kanban_db.get_task(self.connection, task_id)
+        replacement_run = replacement.current_run_id
+        self.assertNotEqual(old_run, replacement_run)
+        signals = []
+
+        self.assertFalse(
+            kanban_db.reclaim_task(
+                self.connection,
+                task_id,
+                reason="stale journal",
+                expected_run_id=old_run,
+                signal_fn=lambda *args: signals.append(args),
+            )
+        )
+        latest = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(latest.status, "running")
+        self.assertEqual(latest.current_run_id, replacement_run)
+        self.assertEqual(latest.claim_lock, replacement.claim_lock)
+        self.assertEqual(signals, [])
+
+    def test_reclaim_update_guard_preserves_run_changed_inside_transaction(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="replacement in txn")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        old_run = kanban_db.get_task(self.connection, task_id).current_run_id
+        replacement = {}
+
+        def install_replacement(*_args, **_kwargs):
+            cursor = self.connection.execute(
+                "INSERT INTO task_runs (task_id, status, claim_lock, started_at) "
+                "VALUES (?, 'running', ?, strftime('%s','now'))",
+                (task_id, "replacement-lock"),
+            )
+            replacement["run_id"] = int(cursor.lastrowid)
+            self.connection.execute(
+                "UPDATE tasks SET current_run_id = ?, claim_lock = ? WHERE id = ?",
+                (replacement["run_id"], "replacement-lock", task_id),
+            )
+            return {}
+
+        with mock.patch.object(
+            kanban_db,
+            "_terminate_reclaimed_worker",
+            side_effect=install_replacement,
+        ):
+            self.assertFalse(
+                kanban_db.reclaim_task(
+                    self.connection,
+                    task_id,
+                    reason="journal for old run",
+                    expected_run_id=old_run,
+                )
+            )
+
+        latest = kanban_db.get_task(self.connection, task_id)
+        self.assertEqual(latest.status, "running")
+        self.assertEqual(latest.current_run_id, replacement["run_id"])
+        self.assertEqual(latest.claim_lock, "replacement-lock")
+
+    def test_concurrent_exact_reclaim_and_completion_have_one_winner(self) -> None:
+        task_id = kanban_db.create_task(self.connection, title="concurrent finalizer")
+        self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
+        run_id = kanban_db.get_task(self.connection, task_id).current_run_id
+        barrier = threading.Barrier(2)
+        outcomes = {}
+        errors = []
+
+        def reclaim() -> None:
+            try:
+                with kanban_db.connect_closing() as connection:
+                    barrier.wait()
+                    outcomes["reclaim"] = kanban_db.reclaim_task(
+                        connection,
+                        task_id,
+                        reason="concurrent invalid journal",
+                        expected_run_id=run_id,
+                    )
+            except BaseException as error:  # pragma: no cover - assertion relay
+                errors.append(error)
+
+        def finalize() -> None:
+            try:
+                with kanban_db.connect_closing() as connection:
+                    barrier.wait()
+                    outcomes["complete"] = kanban_db.complete_task(
+                        connection,
+                        task_id,
+                        result="durable winner",
+                        summary="durable winner",
+                        expected_run_id=run_id,
+                    )
+            except BaseException as error:  # pragma: no cover - assertion relay
+                errors.append(error)
+
+        threads = [threading.Thread(target=reclaim), threading.Thread(target=finalize)]
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join(timeout=10)
+            self.assertFalse(thread.is_alive())
+
+        self.assertEqual(errors, [])
+        self.assertEqual(set(outcomes), {"reclaim", "complete"})
+        self.assertEqual(sum(bool(value) for value in outcomes.values()), 1)
+        latest = kanban_db.get_task(self.connection, task_id)
+        self.assertIn(latest.status, {"done", "ready"})
+        self.assertIsNone(latest.current_run_id)
+        runs = kanban_db.list_runs(self.connection, task_id)
+        self.assertEqual(len(runs), 1)
+        self.assertIsNotNone(runs[0].ended_at)

dockerfiles/patch-hermes-execution-safety.py

@@ -0,0 +1,6 @@
+"""Apply the composable Hermes execution-safety source patches."""
+
+# Import order is intentional: the decomposition DB contract builds on the
+# exact-run schema and transaction changes. Each module fails closed on drift.
+import patch_hermes_run_safety  # noqa: F401
+import patch_hermes_decomposition_safety  # noqa: F401

dockerfiles/patch_hermes_decomposition_safety.py

@@ -0,0 +1,262 @@
+"""Patch transactional automatic-decomposition execution-history guards."""
+
+import os
+from pathlib import Path
+
+from hermes_execution_patch_support import replace_once
+
+source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes"))
+db_path = source_root / "hermes_cli/kanban_db.py"
+db = db_path.read_text(encoding="utf-8")
+
+specify_signature_before = '''def specify_triage_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    title: Optional[str] = None,
+    body: Optional[str] = None,
+    assignee: Optional[str] = None,
+    author: Optional[str] = None,
+) -> bool:
+'''
+specify_signature_after = '''def specify_triage_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    title: Optional[str] = None,
+    body: Optional[str] = None,
+    assignee: Optional[str] = None,
+    author: Optional[str] = None,
+    require_no_runs: bool = False,
+) -> bool:
+'''
+db = replace_once(
+    db,
+    specify_signature_before,
+    specify_signature_after,
+    "transactional specify signature",
+)
+
+specify_guard_before = '''        if existing is None:
+            return False
+        sets: list[str] = ["status = 'todo'"]
+'''
+specify_guard_after = '''        if existing is None:
+            return False
+        if require_no_runs and conn.execute(
+            "SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1",
+            (task_id,),
+        ).fetchone() is not None:
+            return False
+        sets: list[str] = ["status = 'todo'"]
+'''
+db = replace_once(
+    db,
+    specify_guard_before,
+    specify_guard_after,
+    "transactional specify execution-history guard",
+)
+
+decompose_signature_before = '''def decompose_triage_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    root_assignee: Optional[str],
+    children: list[dict],
+    author: Optional[str] = None,
+    auto_promote: bool = True,
+) -> Optional[list[str]]:
+'''
+decompose_signature_after = '''def decompose_triage_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    root_assignee: Optional[str],
+    children: list[dict],
+    author: Optional[str] = None,
+    auto_promote: bool = True,
+    require_no_runs: bool = False,
+) -> Optional[list[str]]:
+'''
+db = replace_once(
+    db,
+    decompose_signature_before,
+    decompose_signature_after,
+    "transactional decomposition signature",
+)
+
+decompose_guard_before = '''        if root_row["status"] != "triage":
+            return None
+        tenant = root_row["tenant"]
+'''
+decompose_guard_after = '''        if root_row["status"] != "triage":
+            return None
+        if require_no_runs and conn.execute(
+            "SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1",
+            (task_id,),
+        ).fetchone() is not None:
+            return None
+        tenant = root_row["tenant"]
+'''
+db = replace_once(
+    db,
+    decompose_guard_before,
+    decompose_guard_after,
+    "transactional decomposition execution-history guard",
+)
+db_path.write_text(db, encoding="utf-8")
+
+decompose_path = source_root / "hermes_cli/kanban_decompose.py"
+decompose = decompose_path.read_text(encoding="utf-8")
+
+helper_anchor = '''    if chosen not in valid_names:
+        return default_assignee
+    return chosen
+
+
+'''
+helper_replacement = helper_anchor + '''def _has_execution_history(task_id: str) -> bool:
+    """Return whether a task has ever entered a worker run."""
+    with kb.connect_closing() as conn:
+        return bool(kb.list_runs(conn, task_id))
+
+
+'''
+decompose = replace_once(
+    decompose,
+    helper_anchor,
+    helper_replacement,
+    "auto-decompose execution-history helper",
+)
+
+signature_before = '''def decompose_task(
+    task_id: str,
+    *,
+    author: Optional[str] = None,
+    timeout: Optional[int] = None,
+) -> DecomposeOutcome:
+'''
+signature_after = '''def decompose_task(
+    task_id: str,
+    *,
+    author: Optional[str] = None,
+    timeout: Optional[int] = None,
+    automatic: bool = False,
+) -> DecomposeOutcome:
+'''
+decompose = replace_once(
+    decompose,
+    signature_before,
+    signature_after,
+    "auto-decompose function signature",
+)
+
+status_before = '''    if task.status != "triage":
+        return DecomposeOutcome(
+            task_id, False, f"task is not in triage (status={task.status!r})"
+        )
+
+    cfg = _load_config()
+'''
+status_after = '''    if task.status != "triage":
+        return DecomposeOutcome(
+            task_id, False, f"task is not in triage (status={task.status!r})"
+        )
+    if automatic and _has_execution_history(task_id):
+        return DecomposeOutcome(
+            task_id,
+            False,
+            "task has execution history and requires deliberate manual triage",
+        )
+
+    cfg = _load_config()
+'''
+decompose = replace_once(
+    decompose,
+    status_before,
+    status_after,
+    "auto-decompose preflight guard",
+)
+
+single_before = '''                author=audit_author,
+            )
+        if not ok:
+'''
+single_after = '''                author=audit_author,
+                require_no_runs=automatic,
+            )
+        if not ok:
+            if automatic and _has_execution_history(task_id):
+                return DecomposeOutcome(
+                    task_id,
+                    False,
+                    "task gained execution history and requires deliberate manual triage",
+                )
+'''
+decompose = replace_once(
+    decompose,
+    single_before,
+    single_after,
+    "auto-decompose single-task commit guard",
+)
+
+fanout_before = '''                author=audit_author,
+                auto_promote=auto_promote,
+            )
+'''
+fanout_after = '''                author=audit_author,
+                auto_promote=auto_promote,
+                require_no_runs=automatic,
+            )
+'''
+decompose = replace_once(
+    decompose,
+    fanout_before,
+    fanout_after,
+    "auto-decompose transactional fanout guard",
+)
+
+fanout_outcome_before = '''    if child_ids is None:
+        return DecomposeOutcome(
+            task_id, False, "task moved out of triage before decomposition",
+        )
+'''
+fanout_outcome_after = '''    if child_ids is None:
+        if automatic and _has_execution_history(task_id):
+            return DecomposeOutcome(
+                task_id,
+                False,
+                "task gained execution history and requires deliberate manual triage",
+            )
+        return DecomposeOutcome(
+            task_id, False, "task moved out of triage before decomposition",
+        )
+'''
+decompose = replace_once(
+    decompose,
+    fanout_outcome_before,
+    fanout_outcome_after,
+    "auto-decompose fanout rejection reason",
+)
+decompose_path.write_text(decompose, encoding="utf-8")
+
+
+watcher_path = source_root / "gateway/kanban_watchers.py"
+watcher = watcher_path.read_text(encoding="utf-8")
+call_before = '''                            outcome = _decomp.decompose_task(
+                                tid, author="auto-decomposer",
+                            )
+'''
+call_after = '''                            outcome = _decomp.decompose_task(
+                                tid,
+                                author="auto-decomposer",
+                                automatic=True,
+                            )
+'''
+watcher = replace_once(
+    watcher,
+    call_before,
+    call_after,
+    "gateway automatic-decomposition call",
+)
+watcher_path.write_text(watcher, encoding="utf-8")

dockerfiles/patch_hermes_run_safety.py

@@ -0,0 +1,480 @@
+"""Patch exact-run completion, provenance, and reclaim transactions."""
+
+import os
+from pathlib import Path
+
+from hermes_execution_patch_support import replace_once
+
+
+
+source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes"))
+db_path = source_root / "hermes_cli/kanban_db.py"
+db = db_path.read_text(encoding="utf-8")
+
+complete_signature_before = '''def complete_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    result: Optional[str] = None,
+    summary: Optional[str] = None,
+    metadata: Optional[dict] = None,
+    created_cards: Optional[Iterable[str]] = None,
+    expected_run_id: Optional[int] = None,
+) -> bool:
+'''
+complete_signature_after = '''def complete_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    result: Optional[str] = None,
+    summary: Optional[str] = None,
+    metadata: Optional[dict] = None,
+    created_cards: Optional[Iterable[str]] = None,
+    expected_run_id: Optional[int] = None,
+    replay_ended_run_id: Optional[int] = None,
+) -> bool:
+'''
+db = replace_once(
+    db,
+    complete_signature_before,
+    complete_signature_after,
+    "exact ended-run completion signature",
+)
+
+complete_branch_before = '''    with write_txn(conn):
+        if expected_run_id is None:
+            cur = conn.execute(
+                """
+                UPDATE tasks
+                   SET status       = 'done',
+                       result       = ?,
+                       completed_at = ?,
+                       claim_lock   = NULL,
+                       claim_expires= NULL,
+                       worker_pid   = NULL,
+                       block_kind   = NULL,
+                       block_recurrences = 0
+                 WHERE id = ?
+                   AND status IN ('running', 'ready', 'blocked', 'scheduled')
+                """,
+                (result, now, task_id),
+            )
+        else:
+'''
+complete_branch_after = '''    # Journal recovery may arrive after a legacy error path ended the
+    # accepted run. Verify that it is still the latest run in this same
+    # transaction; a replacement attempt must make the replay fail closed.
+    with write_txn(conn):
+        replayed_run_id = None
+        if expected_run_id is not None and replay_ended_run_id is not None:
+            raise ValueError("expected_run_id and replay_ended_run_id are mutually exclusive")
+        if replay_ended_run_id is not None:
+            replayed_run_id = int(replay_ended_run_id)
+            # Replay is permitted only while the exact run-ending transition
+            # remains the latest status mutation. Comments and diagnostics do
+            # not invalidate durable accepted work, but an unblock, explicit
+            # triage specification, decomposition, reclaim, or newer attempt
+            # does. BEGIN IMMEDIATE keeps this lineage check and the guarded
+            # task/run updates in one transaction.
+            last_transition = conn.execute(
+                """
+                SELECT kind, run_id
+                  FROM task_events
+                 WHERE task_id = ?
+                   AND kind IN (
+                       'archived', 'block_loop_detected', 'blocked', 'claimed',
+                       'completed', 'decomposed', 'dependency_wait', 'gave_up',
+                       'promoted', 'promoted_manual', 'reclaimed', 'scheduled',
+                       'specified', 'stale', 'timed_out', 'unblocked'
+                   )
+                 ORDER BY id DESC
+                 LIMIT 1
+                """,
+                (task_id,),
+            ).fetchone()
+            if (
+                last_transition is None
+                or last_transition["run_id"] != replayed_run_id
+                or last_transition["kind"] not in {
+                    "block_loop_detected",
+                    "blocked",
+                    "reclaimed",
+                    "stale",
+                    "timed_out",
+                    "gave_up",
+                }
+            ):
+                return False
+            cur = conn.execute(
+                """
+                UPDATE tasks
+                   SET status       = 'done',
+                       result       = ?,
+                       completed_at = ?,
+                       claim_lock   = NULL,
+                       claim_expires= NULL,
+                       worker_pid   = NULL,
+                       block_kind   = NULL,
+                       block_recurrences = 0
+                 WHERE id = ?
+                   AND status IN ('ready', 'blocked', 'triage')
+                   AND current_run_id IS NULL
+                   AND ? = (
+                       SELECT id FROM task_runs
+                        WHERE task_id = ?
+                        ORDER BY id DESC
+                        LIMIT 1
+                   )
+                   AND EXISTS (
+                       SELECT 1 FROM task_runs
+                        WHERE id = ?
+                          AND task_id = ?
+                          AND ended_at IS NOT NULL
+                   )
+                """,
+                (
+                    result,
+                    now,
+                    task_id,
+                    replayed_run_id,
+                    task_id,
+                    replayed_run_id,
+                    task_id,
+                ),
+            )
+        elif expected_run_id is None:
+            cur = conn.execute(
+                """
+                UPDATE tasks
+                   SET status       = 'done',
+                       result       = ?,
+                       completed_at = ?,
+                       claim_lock   = NULL,
+                       claim_expires= NULL,
+                       worker_pid   = NULL,
+                       block_kind   = NULL,
+                       block_recurrences = 0
+                 WHERE id = ?
+                   AND status IN ('running', 'ready', 'blocked', 'scheduled')
+                """,
+                (result, now, task_id),
+            )
+        else:
+'''
+db = replace_once(
+    db,
+    complete_branch_before,
+    complete_branch_after,
+    "transactional exact ended-run completion",
+)
+
+run_completion_before = '''        run_id = _end_run(
+            conn, task_id,
+            outcome="completed", status="done",
+            summary=summary if summary is not None else result,
+            metadata=metadata,
+        )
+        # If complete_task was called on a never-claimed task (ready or
+        # blocked → done with no run in flight), synthesize a
+        # zero-duration run so the handoff fields are persisted in
+        # attempt history instead of silently lost.
+        if run_id is None and (summary or metadata or result):
+            run_id = _synthesize_ended_run(
+                conn, task_id,
+                outcome="completed",
+                summary=summary if summary is not None else result,
+                metadata=metadata,
+            )
+'''
+run_completion_after = '''        if replayed_run_id is None:
+            run_id = _end_run(
+                conn, task_id,
+                outcome="completed", status="done",
+                summary=summary if summary is not None else result,
+                metadata=metadata,
+            )
+            # If complete_task was called on a never-claimed task (ready or
+            # blocked → done with no run in flight), synthesize a
+            # zero-duration run so the handoff fields are persisted in
+            # attempt history instead of silently lost.
+            if run_id is None and (summary or metadata or result):
+                run_id = _synthesize_ended_run(
+                    conn, task_id,
+                    outcome="completed",
+                    summary=summary if summary is not None else result,
+                    metadata=metadata,
+                )
+        else:
+            run_id = replayed_run_id
+            replayed = conn.execute(
+                """
+                UPDATE task_runs
+                   SET status = 'done',
+                       outcome = 'completed',
+                       summary = ?,
+                       error = NULL,
+                       metadata = ?,
+                       claim_lock = NULL,
+                       claim_expires = NULL,
+                       worker_pid = NULL
+                 WHERE id = ?
+                   AND task_id = ?
+                   AND ended_at IS NOT NULL
+                """,
+                (
+                    summary if summary is not None else result,
+                    json.dumps(metadata, ensure_ascii=False) if metadata else None,
+                    run_id,
+                    task_id,
+                ),
+            )
+            if replayed.rowcount != 1:
+                raise RuntimeError("exact ended-run completion lost its run identity")
+        conn.execute(
+            "UPDATE tasks SET completed_run_id = ? WHERE id = ?",
+            (run_id, task_id),
+        )
+'''
+db = replace_once(
+    db,
+    run_completion_before,
+    run_completion_after,
+    "exact completion run provenance",
+)
+
+reclaim_before = '''def reclaim_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    reason: Optional[str] = None,
+    signal_fn=None,
+) -> bool:
+    """Operator-driven reclaim: release the claim and reset to ``ready``.
+
+    Unlike :func:`release_stale_claims` which only acts on tasks whose
+    ``claim_expires`` has passed, this function reclaims immediately
+    regardless of TTL. Intended for the dashboard/CLI recovery flow
+    when an operator wants to abort a running worker without waiting
+    for the TTL to expire (e.g. after seeing a hallucination warning).
+
+    Returns True if a reclaim happened, False if the task isn't in a
+    reclaimable state (not running, or doesn't exist).
+    """
+    row = conn.execute(
+        "SELECT status, claim_lock, worker_pid FROM tasks WHERE id = ?",
+        (task_id,),
+    ).fetchone()
+    if not row:
+        return False
+    if row["status"] != "running" and row["claim_lock"] is None:
+        # Nothing to reclaim — already ready / blocked / done.
+        return False
+    prev_lock = row["claim_lock"]
+    termination = _terminate_reclaimed_worker(
+        row["worker_pid"], prev_lock, signal_fn=signal_fn,
+    )
+    with write_txn(conn):
+        cur = conn.execute(
+            "UPDATE tasks SET status = 'ready', claim_lock = NULL, "
+            "claim_expires = NULL, worker_pid = NULL "
+            "WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
+            "AND claim_lock IS ?",
+            (task_id, prev_lock),
+        )
+        if cur.rowcount != 1:
+            return False
+        run_id = _end_run(
+            conn, task_id,
+            outcome="reclaimed", status="reclaimed",
+            error=(
+                f"manual_reclaim: {reason}" if reason
+                else f"manual_reclaim lock={prev_lock}"
+            ),
+            metadata=termination,
+        )
+        payload = {
+            "manual": True,
+            "reason": reason,
+            "prev_lock": prev_lock,
+        }
+        payload.update(termination)
+        _append_event(
+            conn, task_id, "reclaimed",
+            payload,
+            run_id=run_id,
+        )
+    # Operator intervention — they've looked at the task, so the
+    # consecutive-failures counter is now stale. Give the next retry
+    # a fresh budget. (_clear_failure_counter opens its own write_txn,
+    # so it runs after the enclosing one commits.)
+    _clear_failure_counter(conn, task_id)
+    return True
+'''
+reclaim_after = '''def reclaim_task(
+    conn: sqlite3.Connection,
+    task_id: str,
+    *,
+    reason: Optional[str] = None,
+    signal_fn=None,
+    expected_run_id: Optional[int] = None,
+) -> bool:
+    """Operator-driven reclaim: release the claim and reset to ``ready``.
+
+    Unlike :func:`release_stale_claims` which only acts on tasks whose
+    ``claim_expires`` has passed, this function reclaims immediately
+    regardless of TTL. Intended for the dashboard/CLI recovery flow
+    when an operator wants to abort a running worker without waiting
+    for the TTL to expire (e.g. after seeing a hallucination warning).
+
+    When ``expected_run_id`` is supplied, the run identity is checked after
+    ``BEGIN IMMEDIATE`` and included in the guarded update. A replacement run
+    is therefore neither signalled nor reclaimed by stale recovery evidence.
+
+    Returns True if a reclaim happened, False if the task isn't in a
+    reclaimable state (not running, or doesn't exist), or its run changed.
+    """
+    with write_txn(conn):
+        row = conn.execute(
+            "SELECT status, claim_lock, worker_pid, current_run_id "
+            "FROM tasks WHERE id = ?",
+            (task_id,),
+        ).fetchone()
+        if not row:
+            return False
+        if expected_run_id is not None and row["current_run_id"] != expected_run_id:
+            return False
+        if row["status"] != "running" and row["claim_lock"] is None:
+            # Nothing to reclaim — already ready / blocked / done.
+            return False
+        prev_lock = row["claim_lock"]
+        termination = _terminate_reclaimed_worker(
+            row["worker_pid"], prev_lock, signal_fn=signal_fn,
+        )
+        if expected_run_id is None:
+            cur = conn.execute(
+                "UPDATE tasks SET status = 'ready', claim_lock = NULL, "
+                "claim_expires = NULL, worker_pid = NULL "
+                "WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
+                "AND claim_lock IS ?",
+                (task_id, prev_lock),
+            )
+        else:
+            cur = conn.execute(
+                "UPDATE tasks SET status = 'ready', claim_lock = NULL, "
+                "claim_expires = NULL, worker_pid = NULL "
+                "WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
+                "AND claim_lock IS ? AND current_run_id = ?",
+                (task_id, prev_lock, int(expected_run_id)),
+            )
+        if cur.rowcount != 1:
+            return False
+        run_id = _end_run(
+            conn, task_id,
+            outcome="reclaimed", status="reclaimed",
+            error=(
+                f"manual_reclaim: {reason}" if reason
+                else f"manual_reclaim lock={prev_lock}"
+            ),
+            metadata=termination,
+        )
+        payload = {
+            "manual": True,
+            "reason": reason,
+            "prev_lock": prev_lock,
+        }
+        payload.update(termination)
+        _append_event(
+            conn, task_id, "reclaimed",
+            payload,
+            run_id=run_id,
+        )
+    # Operator intervention — they've looked at the task, so the
+    # consecutive-failures counter is now stale. Give the next retry
+    # a fresh budget. (_clear_failure_counter opens its own write_txn,
+    # so it runs after the enclosing one commits.)
+    _clear_failure_counter(conn, task_id)
+    return True
+'''
+db = replace_once(
+    db,
+    reclaim_before,
+    reclaim_after,
+    "transactional exact-run reclaim",
+)
+
+task_field_before = '''    current_run_id: Optional[int] = None
+    workflow_template_id: Optional[str] = None
+'''
+task_field_after = '''    current_run_id: Optional[int] = None
+    # Durable provenance for the run that authoritatively completed this task.
+    completed_run_id: Optional[int] = None
+    workflow_template_id: Optional[str] = None
+'''
+db = replace_once(
+    db,
+    task_field_before,
+    task_field_after,
+    "completed-run task field",
+)
+
+task_row_before = '''            current_run_id=(
+                row["current_run_id"] if "current_run_id" in keys else None
+            ),
+            workflow_template_id=(
+'''
+task_row_after = '''            current_run_id=(
+                row["current_run_id"] if "current_run_id" in keys else None
+            ),
+            completed_run_id=(
+                row["completed_run_id"] if "completed_run_id" in keys else None
+            ),
+            workflow_template_id=(
+'''
+db = replace_once(
+    db,
+    task_row_before,
+    task_row_after,
+    "completed-run row mapping",
+)
+
+task_schema_before = '''    -- run is in-flight). Denormalised for cheap reads.
+    current_run_id       INTEGER,
+    -- Forward-compat for v2 workflow routing. In v1 the kernel writes
+'''
+task_schema_after = '''    -- run is in-flight). Denormalised for cheap reads.
+    current_run_id       INTEGER,
+    -- Immutable provenance for the task result's authoritative run.
+    completed_run_id     INTEGER,
+    -- Forward-compat for v2 workflow routing. In v1 the kernel writes
+'''
+db = replace_once(
+    db,
+    task_schema_before,
+    task_schema_after,
+    "completed-run schema",
+)
+
+task_migration_before = '''    if "current_run_id" not in cols:
+        _add_column_if_missing(
+            conn, "tasks", "current_run_id", "current_run_id INTEGER"
+        )
+    if "workflow_template_id" not in cols:
+'''
+task_migration_after = '''    if "current_run_id" not in cols:
+        _add_column_if_missing(
+            conn, "tasks", "current_run_id", "current_run_id INTEGER"
+        )
+    if "completed_run_id" not in cols:
+        _add_column_if_missing(
+            conn, "tasks", "completed_run_id", "completed_run_id INTEGER"
+        )
+    if "workflow_template_id" not in cols:
+'''
+db = replace_once(
+    db,
+    task_migration_before,
+    task_migration_after,
+    "completed-run migration",
+)
+
+db_path.write_text(db, encoding="utf-8")

scripts/tests/test_dashboard_render_support.py

@@ -0,0 +1,25 @@
+import importlib.util
+
+import pathlib
+
+def load_module():
+    path = pathlib.Path(__file__).resolve().parents[1] / "render" / "dashboards_render_atlas.py"
+    spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+def flatten_panels(panels):
+    flat = []
+    for panel in panels:
+        flat.append(panel)
+        flat.extend(panel.get("panels", []))
+    return flat
+
+__all__ = (
+    "flatten_panels",
+    "importlib",
+    "load_module",
+    "pathlib",
+)

scripts/tests/test_dashboards_render_jobs.py

@@ -1,241 +1,11 @@
-import importlib.util
-import pathlib
+"""Atlas testing and jobs dashboard rendering contracts."""
 
+from __future__ import annotations
 
-def load_module():
-    path = pathlib.Path(__file__).resolve().parents[1] / "render" / "dashboards_render_atlas.py"
-    spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path)
-    module = importlib.util.module_from_spec(spec)
-    assert spec.loader is not None
-    spec.loader.exec_module(module)
-    return module
-
-
-def flatten_panels(panels):
-    flat = []
-    for panel in panels:
-        flat.append(panel)
-        flat.extend(panel.get("panels", []))
-    return flat
-
-
-def test_table_panel_options_and_filterable():
-    mod = load_module()
-    panel = mod.table_panel(
-        1,
-        "test",
-        "metric",
-        {"h": 1, "w": 1, "x": 0, "y": 0},
-        unit="percent",
-        transformations=[{"id": "labelsToFields", "options": {}}],
-        instant=True,
-        options={"showColumnFilters": False},
-        filterable=False,
-        footer={"show": False, "fields": "", "calcs": []},
-        format="table",
-    )
-    assert panel["fieldConfig"]["defaults"]["unit"] == "percent"
-    assert panel["fieldConfig"]["defaults"]["custom"]["filterable"] is False
-    assert panel["options"]["showHeader"] is True
-    assert panel["targets"][0]["format"] == "table"
-
-
-def test_node_filter_and_expr_helpers():
-    mod = load_module()
-    expr = mod.node_filter("titan-.*")
-    assert "label_replace" in expr
-    cpu_expr = mod.node_cpu_expr("titan-.*")
-    mem_expr = mod.node_mem_expr("titan-.*")
-    assert "node_cpu_seconds_total" in cpu_expr
-    assert "clamp_max(clamp_min(" in cpu_expr
-    assert "* 100, 0), 100)" in cpu_expr
-    assert "node_memory_MemAvailable_bytes" in mem_expr
-
-
-def test_overview_availability_panel_uses_recorded_365d_rollup():
-    mod = load_module()
-    dashboard = mod.build_overview()
-    panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27)
-
-    assert panel["title"] == "Atlas Availability (365d)"
-    availability_expr = panel["targets"][0]["expr"]
-    assert (
-        'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])'
-        in availability_expr
-    )
-    assert 'code=~"5.."' in availability_expr
-    assert 'code=~"[1-5].."' in availability_expr
-    assert "atlas:availability:failures_1d" not in availability_expr
-    assert "atlas:availability:requests_1d" not in availability_expr
-    assert "sum_over_time" not in availability_expr
-    assert "kube_node_status_condition" not in availability_expr
-    assert "kube_deployment_status_replicas_available" not in availability_expr
-    assert panel["targets"][0]["instant"] is True
-    assert "Every server-side 5xx" in panel["description"]
-    assert "Replica counts, Grafana health" in panel["description"]
-    assert "daily rollup job publishes one annual sample" in panel["description"]
-    assert "keeps it for up to 48 hours" in panel["description"]
-
-
-def test_overview_uses_readable_quality_power_and_gitops_panels():
-    mod = load_module()
-    dashboard = mod.build_overview()
-    panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
-
-    assert dashboard["links"] == [
-        {"title": "Atlas Testing", "url": "/d/atlas-testing", "targetBlank": True}
-    ]
-    assert "atlas-jobs" not in repr(dashboard)
-    assert "Platform Test Success Rate" not in panels_by_title
-    assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
-    assert panels_by_title["Test Category Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 13}
-    assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
-    assert panels_by_title["Test Category Health"]["targets"][0]["format"] == "time_series"
-    assert panels_by_title["Test Category Health"]["targets"][0]["range"] is True
-    assert "${overview_suite:regex}" not in panels_by_title["Test Category Health"]["targets"][0]["expr"]
-    assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in panels_by_title["Test Category Health"]["targets"][0]["expr"]
-    assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Test Category Health"]["targets"][0]["expr"]
-    assert panels_by_title["Test Category Health"]["timeFrom"] == "24h"
-    assert f'category=~"{mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
-    assert "manual" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
-    assert "unit" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
-    assert panels_by_title["UPS History (Power Draw)"]["gridPos"] == {"h": 6, "w": 6, "x": 3, "y": 7}
-    assert panels_by_title["Ariadne Run Volume"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 7}
-    assert panels_by_title["Pyrphoros UPS Current"]["gridPos"]["w"] == 3
-    assert panels_by_title["Current Enclosure Climate"]["gridPos"]["w"] == 3
-    assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["placement"] == "bottom"
-    assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["displayMode"] == "list"
-    assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "line"
-    assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["fillOpacity"] == 22
-    assert all(target["expr"].startswith("max(") for target in panels_by_title["UPS History (Power Draw)"]["targets"])
-    ups_overrides = panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["overrides"]
-    ups_override_by_name = {override["matcher"]["options"]: override for override in ups_overrides}
-    assert ups_override_by_name["Pyrphoros"]["properties"] == [
-        {"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-blue"}},
-    ]
-    assert ups_override_by_name["Statera"]["properties"] == [
-        {"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-yellow"}},
-    ]
-    assert panels_by_title["Ariadne Run Volume"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
-    assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["placement"] == "bottom"
-    assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["displayMode"] == "list"
-    assert "Fan History (0-10)" not in panels_by_title
-    assert panels_by_title["Fan Intensity History"]["type"] == "state-timeline"
-    assert panels_by_title["Fan Intensity History"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 13}
-    assert panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["max"] == 10
-    assert panels_by_title["Fan Intensity History"]["targets"][0]["legendFormat"] == "{{fan}}"
-    fan_steps = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["thresholds"]["steps"]
-    assert [step["value"] for step in fan_steps] == [None, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-    assert fan_steps[0]["color"] == "#8f1d1d"
-    assert fan_steps[5]["color"] == "#d4b106"
-    assert fan_steps[-1]["color"] == "#1f60c4"
-    fan_mappings = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["mappings"][0]["options"]
-    assert fan_mappings["0"]["text"] == "Off"
-    assert fan_mappings["10"]["text"] == "10"
-    assert "Tent Interior" in panels_by_title["Fan Intensity History"]["targets"][0]["expr"]
-    assert panels_by_title["Fan Intensity History"]["options"]["legend"]["displayMode"] == "list"
-    assert panels_by_title["Fan Intensity History"]["options"]["legend"]["showLegend"] is False
-    assert panels_by_title["Fan Intensity History"]["options"]["mergeValues"] is False
-    assert panels_by_title["Fan Intensity History"]["options"]["showValue"] == "auto"
-
-    assert panels_by_title["Flux Source"]["type"] == "stat"
-    assert panels_by_title["Flux Source"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 7}
-    assert panels_by_title["Flux Source"]["targets"][0]["legendFormat"] == "{{branch}}"
-    assert panels_by_title["Current Gate Health"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 9}
-    assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Current Gate Health"]["targets"][0]["expr"]
-    assert panels_by_title["CI Run Success (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 11}
-    assert panels_by_title["Suites With Runs (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 15}
-    suites_reporting_expr = panels_by_title["Suites With Runs (24h)"]["targets"][0]["expr"]
-    assert "> bool 0" in suites_reporting_expr
-    assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in suites_reporting_expr
-    assert "bstein-home" not in suites_reporting_expr
-    assert "published quality-gate run" in panels_by_title["Suites With Runs (24h)"]["description"]
-    assert panels_by_title["Avg Coverage"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 17}
-    assert "LOC Clean Suites" not in panels_by_title
-    assert panels_by_title["GitOps Health"]["type"] == "state-timeline"
-    assert panels_by_title["GitOps Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 7}
-    gitops_expr = panels_by_title["GitOps Health"]["targets"][0]["expr"]
-    assert "Kustomizations Not Suspended" in gitops_expr
-    assert "HelmReleases Not Suspended" in gitops_expr
-    assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
-    assert panels_by_title["Test Category Health"]["options"]["legend"]["showLegend"] is False
-    assert panels_by_title["Test Category Health"]["options"]["mergeValues"] is False
-    assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto"
-    assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9
-    assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
-    assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
-    assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"])
-
-    pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"]
-    assert "backup-telemetry-missing" in pvc_backup_expr
-    assert 'pvc_backup_(count|last_success_timestamp_seconds|health_reason)' in pvc_backup_expr
-
-    gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
-    assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
-    assert "nvidia_gpu_device_utilization_percent" in gpu_expr
-    assert "jetson_gr3d_active_seconds_total" in gpu_expr
-    assert "increase" in gpu_expr
-    assert "sum_over_time" not in gpu_expr
-    assert "count_over_time" not in gpu_expr
-    assert "avg_over_time" in gpu_expr
-    assert "$__range" in gpu_expr
-    assert "$__range_s" in gpu_expr
-    assert "sum by (namespace)" in gpu_expr
-    assert 'namespace", "shared"' not in gpu_expr
-    assert "kube_pod_container_resource_requests" in gpu_expr
-    assert mod.GPU_RESOURCE_REGEX in gpu_expr
-    assert '"gpu_source", "nvidia"' in gpu_expr
-    assert '"gpu_source", "jetson"' in gpu_expr
-    assert "100 *" in gpu_expr
-    assert "100 -" not in gpu_expr
-    assert 'namespace", "unattributed"' in gpu_expr
-    assert 'namespace", "idle"' in gpu_expr
-    assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
-    assert "selected time range" in panels_by_title["Namespace GPU Utilization"]["description"]
-    assert "continuously sampled" in panels_by_title["Namespace GPU Utilization"]["description"]
-    assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
-
-
-def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
-    mod = load_module()
-    dashboard = mod.build_gpu_dashboard()
-    panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
-
-    node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"]
-    assert "nvidia_gpu_device_utilization_percent" in node_expr
-    assert "DCGM_FI_DEV_GPU_UTIL" in node_expr
-    assert "jetson_gr3d_freq_percent" in node_expr
-    assert "unless on(node)" in node_expr
-    assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
-
-    process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"]
-    assert "nvidia_process_gpu_sm_util_percent" in process_expr
-    assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
-
-
-def test_overview_and_testing_panels_all_have_concise_descriptions():
-    mod = load_module()
-
-    for dashboard in [mod.build_overview(), mod.build_jobs_dashboard(), mod.build_testing_dashboard()]:
-        panels = flatten_panels(dashboard["panels"])
-        assert all(panel.get("description") for panel in panels if panel["type"] != "row")
-        assert "85.7" not in repr(dashboard)
-
-
-def test_render_configmap_writes(tmp_path):
-    mod = load_module()
-    mod.DASHBOARD_DIR = tmp_path / "dash"
-    mod.ROOT = tmp_path
-    uid = "atlas-test"
-    info = {"configmap": tmp_path / "cm.yaml"}
-    data = {"title": "Atlas Test"}
-    mod.write_json(uid, data)
-    mod.render_configmap(uid, info)
-    json_path = mod.DASHBOARD_DIR / f"{uid}.json"
-    assert json_path.exists()
-    content = (tmp_path / "cm.yaml").read_text()
-    assert "kind: ConfigMap" in content
-    assert f"{uid}.json" in content
+from test_dashboard_render_support import (
+    flatten_panels,
+    load_module,
+)
 
 
 def test_testing_suite_variable_uses_configured_suite_values_only():
@@ -257,7 +27,6 @@ def test_testing_suite_variable_uses_configured_suite_values_only():
         option["value"] in out_of_scope_names for option in variable["options"]
     )
 
-
 def test_testing_dashboard_is_public_but_jobs_dashboard_remains_internal():
     mod = load_module()
     jobs = mod.build_jobs_dashboard()
@@ -269,7 +38,6 @@ def test_testing_dashboard_is_public_but_jobs_dashboard_remains_internal():
     assert testing["folderUid"] == mod.PUBLIC_DASHBOARD_FOLDER
     assert testing["editable"] is False
 
-
 def test_jobs_dashboard_separates_current_gate_health_from_reliability():
     mod = load_module()
     dashboard = mod.build_jobs_dashboard()
@@ -395,7 +163,6 @@ def test_jobs_dashboard_separates_current_gate_health_from_reliability():
     assert run_volume_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
     assert "[$__interval]" not in run_volume_panel["targets"][0]["expr"]
 
-
 def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors():
     mod = load_module()
     dashboard = mod.build_jobs_dashboard()
@@ -417,7 +184,6 @@ def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors():
     assert {"color": "dark-yellow", "value": 93} in threshold_steps
     assert {"color": "dark-blue", "value": 100} in threshold_steps
 
-
 def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
     mod = load_module()
     dashboard = mod.build_jobs_dashboard()

scripts/tests/test_dashboards_render_overview.py

@@ -0,0 +1,221 @@
+"""Atlas overview dashboard rendering contracts."""
+
+from __future__ import annotations
+
+from test_dashboard_render_support import (
+    flatten_panels,
+    load_module,
+)
+
+
+def test_table_panel_options_and_filterable():
+    mod = load_module()
+    panel = mod.table_panel(
+        1,
+        "test",
+        "metric",
+        {"h": 1, "w": 1, "x": 0, "y": 0},
+        unit="percent",
+        transformations=[{"id": "labelsToFields", "options": {}}],
+        instant=True,
+        options={"showColumnFilters": False},
+        filterable=False,
+        footer={"show": False, "fields": "", "calcs": []},
+        format="table",
+    )
+    assert panel["fieldConfig"]["defaults"]["unit"] == "percent"
+    assert panel["fieldConfig"]["defaults"]["custom"]["filterable"] is False
+    assert panel["options"]["showHeader"] is True
+    assert panel["targets"][0]["format"] == "table"
+
+def test_node_filter_and_expr_helpers():
+    mod = load_module()
+    expr = mod.node_filter("titan-.*")
+    assert "label_replace" in expr
+    cpu_expr = mod.node_cpu_expr("titan-.*")
+    mem_expr = mod.node_mem_expr("titan-.*")
+    assert "node_cpu_seconds_total" in cpu_expr
+    assert "clamp_max(clamp_min(" in cpu_expr
+    assert "* 100, 0), 100)" in cpu_expr
+    assert "node_memory_MemAvailable_bytes" in mem_expr
+
+def test_overview_availability_panel_uses_recorded_365d_rollup():
+    mod = load_module()
+    dashboard = mod.build_overview()
+    panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27)
+
+    assert panel["title"] == "Atlas Availability (365d)"
+    availability_expr = panel["targets"][0]["expr"]
+    assert (
+        'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])'
+        in availability_expr
+    )
+    assert 'code=~"5.."' in availability_expr
+    assert 'code=~"[1-5].."' in availability_expr
+    assert "atlas:availability:failures_1d" not in availability_expr
+    assert "atlas:availability:requests_1d" not in availability_expr
+    assert "sum_over_time" not in availability_expr
+    assert "kube_node_status_condition" not in availability_expr
+    assert "kube_deployment_status_replicas_available" not in availability_expr
+    assert panel["targets"][0]["instant"] is True
+    assert "Every server-side 5xx" in panel["description"]
+    assert "Replica counts, Grafana health" in panel["description"]
+    assert "daily rollup job publishes one annual sample" in panel["description"]
+    assert "keeps it for up to 48 hours" in panel["description"]
+
+def test_overview_uses_readable_quality_power_and_gitops_panels():
+    mod = load_module()
+    dashboard = mod.build_overview()
+    panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
+
+    assert dashboard["links"] == [
+        {"title": "Atlas Testing", "url": "/d/atlas-testing", "targetBlank": True}
+    ]
+    assert "atlas-jobs" not in repr(dashboard)
+    assert "Platform Test Success Rate" not in panels_by_title
+    assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
+    assert panels_by_title["Test Category Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 13}
+    assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
+    assert panels_by_title["Test Category Health"]["targets"][0]["format"] == "time_series"
+    assert panels_by_title["Test Category Health"]["targets"][0]["range"] is True
+    assert "${overview_suite:regex}" not in panels_by_title["Test Category Health"]["targets"][0]["expr"]
+    assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in panels_by_title["Test Category Health"]["targets"][0]["expr"]
+    assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Test Category Health"]["targets"][0]["expr"]
+    assert panels_by_title["Test Category Health"]["timeFrom"] == "24h"
+    assert f'category=~"{mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
+    assert "manual" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
+    assert "unit" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
+    assert panels_by_title["UPS History (Power Draw)"]["gridPos"] == {"h": 6, "w": 6, "x": 3, "y": 7}
+    assert panels_by_title["Ariadne Run Volume"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 7}
+    assert panels_by_title["Pyrphoros UPS Current"]["gridPos"]["w"] == 3
+    assert panels_by_title["Current Enclosure Climate"]["gridPos"]["w"] == 3
+    assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["placement"] == "bottom"
+    assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["displayMode"] == "list"
+    assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "line"
+    assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["fillOpacity"] == 22
+    assert all(target["expr"].startswith("max(") for target in panels_by_title["UPS History (Power Draw)"]["targets"])
+    ups_overrides = panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["overrides"]
+    ups_override_by_name = {override["matcher"]["options"]: override for override in ups_overrides}
+    assert ups_override_by_name["Pyrphoros"]["properties"] == [
+        {"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-blue"}},
+    ]
+    assert ups_override_by_name["Statera"]["properties"] == [
+        {"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-yellow"}},
+    ]
+    assert panels_by_title["Ariadne Run Volume"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
+    assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["placement"] == "bottom"
+    assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["displayMode"] == "list"
+    assert "Fan History (0-10)" not in panels_by_title
+    assert panels_by_title["Fan Intensity History"]["type"] == "state-timeline"
+    assert panels_by_title["Fan Intensity History"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 13}
+    assert panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["max"] == 10
+    assert panels_by_title["Fan Intensity History"]["targets"][0]["legendFormat"] == "{{fan}}"
+    fan_steps = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["thresholds"]["steps"]
+    assert [step["value"] for step in fan_steps] == [None, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+    assert fan_steps[0]["color"] == "#8f1d1d"
+    assert fan_steps[5]["color"] == "#d4b106"
+    assert fan_steps[-1]["color"] == "#1f60c4"
+    fan_mappings = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["mappings"][0]["options"]
+    assert fan_mappings["0"]["text"] == "Off"
+    assert fan_mappings["10"]["text"] == "10"
+    assert "Tent Interior" in panels_by_title["Fan Intensity History"]["targets"][0]["expr"]
+    assert panels_by_title["Fan Intensity History"]["options"]["legend"]["displayMode"] == "list"
+    assert panels_by_title["Fan Intensity History"]["options"]["legend"]["showLegend"] is False
+    assert panels_by_title["Fan Intensity History"]["options"]["mergeValues"] is False
+    assert panels_by_title["Fan Intensity History"]["options"]["showValue"] == "auto"
+
+    assert panels_by_title["Flux Source"]["type"] == "stat"
+    assert panels_by_title["Flux Source"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 7}
+    assert panels_by_title["Flux Source"]["targets"][0]["legendFormat"] == "{{branch}}"
+    assert panels_by_title["Current Gate Health"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 9}
+    assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Current Gate Health"]["targets"][0]["expr"]
+    assert panels_by_title["CI Run Success (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 11}
+    assert panels_by_title["Suites With Runs (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 15}
+    suites_reporting_expr = panels_by_title["Suites With Runs (24h)"]["targets"][0]["expr"]
+    assert "> bool 0" in suites_reporting_expr
+    assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in suites_reporting_expr
+    assert "bstein-home" not in suites_reporting_expr
+    assert "published quality-gate run" in panels_by_title["Suites With Runs (24h)"]["description"]
+    assert panels_by_title["Avg Coverage"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 17}
+    assert "LOC Clean Suites" not in panels_by_title
+    assert panels_by_title["GitOps Health"]["type"] == "state-timeline"
+    assert panels_by_title["GitOps Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 7}
+    gitops_expr = panels_by_title["GitOps Health"]["targets"][0]["expr"]
+    assert "Kustomizations Not Suspended" in gitops_expr
+    assert "HelmReleases Not Suspended" in gitops_expr
+    assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
+    assert panels_by_title["Test Category Health"]["options"]["legend"]["showLegend"] is False
+    assert panels_by_title["Test Category Health"]["options"]["mergeValues"] is False
+    assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto"
+    assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9
+    assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
+    assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
+    assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"])
+
+    pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"]
+    assert "backup-telemetry-missing" in pvc_backup_expr
+    assert 'pvc_backup_(count|last_success_timestamp_seconds|health_reason)' in pvc_backup_expr
+
+    gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
+    assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
+    assert "nvidia_gpu_device_utilization_percent" in gpu_expr
+    assert "jetson_gr3d_active_seconds_total" in gpu_expr
+    assert "increase" in gpu_expr
+    assert "sum_over_time" not in gpu_expr
+    assert "count_over_time" not in gpu_expr
+    assert "avg_over_time" in gpu_expr
+    assert "$__range" in gpu_expr
+    assert "$__range_s" in gpu_expr
+    assert "sum by (namespace)" in gpu_expr
+    assert 'namespace", "shared"' not in gpu_expr
+    assert "kube_pod_container_resource_requests" in gpu_expr
+    assert mod.GPU_RESOURCE_REGEX in gpu_expr
+    assert '"gpu_source", "nvidia"' in gpu_expr
+    assert '"gpu_source", "jetson"' in gpu_expr
+    assert "100 *" in gpu_expr
+    assert "100 -" not in gpu_expr
+    assert 'namespace", "unattributed"' in gpu_expr
+    assert 'namespace", "idle"' in gpu_expr
+    assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
+    assert "selected time range" in panels_by_title["Namespace GPU Utilization"]["description"]
+    assert "continuously sampled" in panels_by_title["Namespace GPU Utilization"]["description"]
+    assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
+
+def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
+    mod = load_module()
+    dashboard = mod.build_gpu_dashboard()
+    panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
+
+    node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"]
+    assert "nvidia_gpu_device_utilization_percent" in node_expr
+    assert "DCGM_FI_DEV_GPU_UTIL" in node_expr
+    assert "jetson_gr3d_freq_percent" in node_expr
+    assert "unless on(node)" in node_expr
+    assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
+
+    process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"]
+    assert "nvidia_process_gpu_sm_util_percent" in process_expr
+    assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
+
+def test_overview_and_testing_panels_all_have_concise_descriptions():
+    mod = load_module()
+
+    for dashboard in [mod.build_overview(), mod.build_jobs_dashboard(), mod.build_testing_dashboard()]:
+        panels = flatten_panels(dashboard["panels"])
+        assert all(panel.get("description") for panel in panels if panel["type"] != "row")
+        assert "85.7" not in repr(dashboard)
+
+def test_render_configmap_writes(tmp_path):
+    mod = load_module()
+    mod.DASHBOARD_DIR = tmp_path / "dash"
+    mod.ROOT = tmp_path
+    uid = "atlas-test"
+    info = {"configmap": tmp_path / "cm.yaml"}
+    data = {"title": "Atlas Test"}
+    mod.write_json(uid, data)
+    mod.render_configmap(uid, info)
+    json_path = mod.DASHBOARD_DIR / f"{uid}.json"
+    assert json_path.exists()
+    content = (tmp_path / "cm.yaml").read_text()
+    assert "kind: ConfigMap" in content
+    assert f"{uid}.json" in content

services/bstein-dev-home/kustomization.yaml

@@ -20,9 +20,9 @@ resources:
   - ingress.yaml
 images:
   - name: registry.bstein.dev/bstein/bstein-dev-home-frontend
-    newTag: 0.1.1-470 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"}
+    newTag: 0.1.1-471 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"}
   - name: registry.bstein.dev/bstein/bstein-dev-home-backend
-    newTag: 0.1.1-470 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"}
+    newTag: 0.1.1-471 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"}
 configMapGenerator:
   - name: chat-ai-gateway
     namespace: bstein-dev-home

services/hermes/kustomization.yaml

@@ -61,7 +61,21 @@ configMapGenerator:
       - ai_usage_exporter.py=scripts/ai_usage_exporter.py
       - claude=scripts/claude
       - claude_command_policy.py=scripts/claude_command_policy.py
+      - cli_lane_board.py=scripts/cli_lane_board.py
+      - cli_lane_config.py=scripts/cli_lane_config.py
+      - cli_lane_dispatch.py=scripts/cli_lane_dispatch.py
+      - cli_lane_evidence.py=scripts/cli_lane_evidence.py
+      - cli_lane_execution.py=scripts/cli_lane_execution.py
+      - cli_lane_files.py=scripts/cli_lane_files.py
+      - cli_lane_finalization.py=scripts/cli_lane_finalization.py
       - cli_lane_goal.py=scripts/cli_lane_goal.py
+      - cli_lane_prompt.py=scripts/cli_lane_prompt.py
+      - cli_lane_provider.py=scripts/cli_lane_provider.py
+      - cli_lane_quarantine.py=scripts/cli_lane_quarantine.py
+      - cli_lane_records.py=scripts/cli_lane_records.py
+      - cli_lane_recovery.py=scripts/cli_lane_recovery.py
+      - cli_lane_retention.py=scripts/cli_lane_retention.py
+      - cli_lane_routing.py=scripts/cli_lane_routing.py
       - cli_lane_runner.py=scripts/cli_lane_runner.py
       - codex=scripts/codex
       - configure_agent_clients.py=scripts/configure_agent_clients.py

services/hermes/scripts/cli_lane_board.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""Bounded Kanban access and workspace resolution for Hermes CLI workers."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import sys
+import time
+from pathlib import Path
+from typing import Any, Callable
+
+from cli_lane_config import (
+    BOARD_CORRUPTION_ERRORS,
+    EXTERNAL_PREFIX,
+    KANBAN_STORAGE_ATTEMPTS,
+    WORKTREE_LOCK,
+)
+
+
+def _task_value(task: Any, name: str, default: Any = None) -> Any:
+    return getattr(task, name, default)
+
+def _task_context(kanban_db: Any, conn: Any, task_id: str) -> str:
+    value = kanban_db.build_worker_context(conn, task_id)
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, indent=2, default=str)
+
+def _resolve_workspace(kanban_db: Any, conn: Any, task: Any, board: str) -> Path:
+    # External coding lanes always receive their own linked worktree. A task
+    # that does not resolve to a Git repository is blocked rather than sharing
+    # a mutable checkout with another unattended worker.
+    # Git worktree creation updates common-repository metadata. Serialize that
+    # short materialization step while allowing the provider workers themselves
+    # to run concurrently in independent worktrees.
+    with WORKTREE_LOCK:
+        value, branch_name = kanban_db._resolve_worktree_workspace(task, board=board)
+    kanban_db.set_branch_name(conn, str(_task_value(task, "id")), branch_name)
+    return Path(value).resolve()
+
+def _record_board_access_error(board: str, error: Exception) -> None:
+    """Report one board access failure once while allowing other lanes to run."""
+    failure_kind = "storage" if isinstance(error, (OSError, sqlite3.Error)) else "access"
+    detail = f"{failure_kind} {type(error).__name__}: {error}"
+    if BOARD_CORRUPTION_ERRORS.get(board) == detail:
+        return
+    print(
+        f"temporarily skipping Kanban board {board!r}: {detail}",
+        file=sys.stderr,
+        flush=True,
+    )
+    BOARD_CORRUPTION_ERRORS[board] = detail
+
+def _board_call(
+    kanban_db: Any,
+    board: str,
+    operation: Callable[[Any], Any],
+) -> Any:
+    """Run one bounded Kanban operation on a fresh, promptly closed connection."""
+    last_storage_error: Exception | None = None
+    for attempt in range(KANBAN_STORAGE_ATTEMPTS):
+        conn = None
+        try:
+            with kanban_db.scoped_current_board(board):
+                conn = kanban_db.connect(board=board)
+                result = operation(conn)
+            BOARD_CORRUPTION_ERRORS.pop(board, None)
+            return result
+        except (OSError, sqlite3.Error) as error:
+            last_storage_error = error
+            _record_board_access_error(board, error)
+            if attempt + 1 < KANBAN_STORAGE_ATTEMPTS:
+                time.sleep(0.2 * (attempt + 1))
+        finally:
+            if conn is not None:
+                conn.close()
+    assert last_storage_error is not None
+    raise last_storage_error
+
+def _external(task: Any) -> bool:
+    return str(_task_value(task, "assignee", "") or "").startswith(EXTERNAL_PREFIX)

services/hermes/scripts/cli_lane_config.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+"""Shared immutable configuration and result types for Hermes CLI lanes."""
+
+from __future__ import annotations
+
+import os
+import re
+import threading
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import cli_lane_goal
+
+
+DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data"))
+SWITCHYARD_URL = os.environ.get(
+    "HERMES_SWITCHYARD_URL",
+    "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1/chat/completions",
+)
+STATE_ROOT = DATA_ROOT / "cli-lanes"
+CODEX_BIN = DATA_ROOT / "tools/bin/codex"
+CLAUDE_BIN = DATA_ROOT / "tools/bin/claude"
+CLAUDE_SETTINGS = DATA_ROOT / "home/.claude/settings.json"
+RESULT_SCHEMA_PATH = STATE_ROOT / "worker-result.schema.json"
+EFFORTS = ("low", "medium", "high", "xhigh")
+EXTERNAL_PREFIX = "cli-"
+DEFAULT_CLAIM_TTL = 7 * 24 * 60 * 60
+DEFAULT_MAX_RUNTIME = 12 * 60 * 60
+HEARTBEAT_SECONDS = 20
+PROVIDER_HEALTH_MAX_AGE_SECONDS = 5 * 60
+PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS = 12 * 60 * 60
+KANBAN_STORAGE_ATTEMPTS = 5
+ARTIFACT_GC_INTERVAL_SECONDS = 5 * 60
+ARTIFACT_RETENTION_AGE_SECONDS = 30 * 24 * 60 * 60
+ARTIFACT_RETENTION_COUNT = 256
+ARTIFACT_RETENTION_BYTES = 128 * 1024 * 1024
+MAX_TERMINAL_RECORD_BYTES = 1024 * 1024
+QUARANTINE_HASH_BYTES = 64 * 1024
+SQLITE_RUN_ID_MAX = 2**63 - 1
+PROVIDER_HEALTH_PATHS = {
+    "codex": DATA_ROOT / "provider-health/codex.json",
+    "claude": DATA_ROOT / "provider-health/claude.json",
+}
+WORKTREE_LOCK = threading.Lock()
+BOARD_CORRUPTION_ERRORS: dict[str, str] = {}
+LAST_ARTIFACT_GC = 0.0
+CAPACITY_PATTERN = re.compile(
+    r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529|authentication|oauth|token.*expired)",
+    re.I,
+)
+NO_CLAUDE_SESSION = "No conversation found with session ID:"
+CLAUDE_SESSION_COLLISION = "Session ID already in use"
+NO_CODEX_THREAD = "no rollout found for thread id"
+
+
+RESULT_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "additionalProperties": False,
+    "required": [
+        "status",
+        "summary",
+        "changed_files",
+        "tests_run",
+        "artifacts",
+        "findings",
+        "blockers",
+    ],
+    "properties": {
+        "status": {
+            "type": "string",
+            "enum": sorted(cli_lane_goal.RESULT_STATUSES),
+        },
+        "summary": {"type": "string"},
+        "changed_files": {"type": "array", "items": {"type": "string"}},
+        "tests_run": {"type": "array", "items": {"type": "string"}},
+        "artifacts": {"type": "array", "items": {"type": "string"}},
+        "findings": {
+            "type": "array",
+            "items": {"type": "string"},
+            "description": (
+                "Defects, risks, or observations discovered by a review or diagnosis. "
+                "Findings do not prevent the assigned review or diagnosis from completing."
+            ),
+        },
+        "blockers": {
+            "type": "array",
+            "items": {"type": "string"},
+            "description": (
+                "Concrete obstacles that prevent completion of the assigned task itself. "
+                "This must be empty when status is completed; review findings belong in findings."
+            ),
+        },
+    },
+}
+
+
+@dataclass(frozen=True)
+class Route:
+    """One provider/model/effort decision for an observable worker boundary."""
+
+    provider: str
+    model: str
+    effort: str
+    profile: str
+    classifier: str
+    reason: str
+    latency_ms: int
+    fallback_chain: tuple[str, ...]
+
+
+@dataclass
+class ProcessResult:
+    """Captured outcome from one provider CLI invocation."""
+
+    returncode: int
+    output: str
+    structured: dict[str, Any] | None
+    capacity_failure: bool
+
+
+class TerminalFinalizationPending(RuntimeError):
+    """An accepted worker result is durable but not committed to Kanban yet."""
+
+
+def canonical_run_id(value: object) -> int | None:
+    """Return one positive SQLite-safe run ID from an int or canonical decimal."""
+    if type(value) is int:
+        candidate = value
+    elif type(value) is str:
+        # Bound conversion before ``int`` so an attacker cannot hand Python an
+        # arbitrarily large decimal from an otherwise lexically valid name.
+        if not re.fullmatch(r"[1-9][0-9]{0,18}", value):
+            return None
+        candidate = int(value)
+    else:
+        return None
+    if not 1 <= candidate <= SQLITE_RUN_ID_MAX:
+        return None
+    return candidate
+
+
+@dataclass(frozen=True)
+class TerminalIdentity:
+    """Replay authority derived only from a journal's directory and filename."""
+
+    board: str
+    task_id: str
+    run_id: int
+    state: str
+
+    def __post_init__(self) -> None:
+        """Reject noncanonical identities before they can reach SQLite."""
+        if type(self.run_id) is not int or canonical_run_id(self.run_id) is None:
+            raise ValueError("terminal run identity is not a positive SQLite int64")
+
+
+@dataclass
+class TerminalSnapshot:
+    """One inode-bound, bounded journal read held open across finalization."""
+
+    document: dict[str, Any]
+    file_stat: os.stat_result
+    descriptor: int
+    directory_descriptor: int
+
+    def close(self) -> None:
+        """Release the pinned file and directory descriptors."""
+        os.close(self.descriptor)
+        os.close(self.directory_descriptor)
+
+
+@dataclass
+class TerminalRecoverySnapshot:
+    """One raw directory entry pinned before recovery classifies its payload."""
+
+    document: dict[str, Any] | None
+    file_stat: os.stat_result
+    descriptor: int | None
+    directory_descriptor: int
+    prefix: bytes
+    invalid_reason: str | None
+
+    def close(self) -> None:
+        """Release the descriptors retained across classification/quarantine."""
+        if self.descriptor is not None:
+            os.close(self.descriptor)
+            self.descriptor = None
+        if self.directory_descriptor >= 0:
+            os.close(self.directory_descriptor)
+            self.directory_descriptor = -1
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()

services/hermes/scripts/cli_lane_dispatch.py

@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+"""Concurrent board recovery, claim, and dispatch loop for CLI workers."""
+
+from __future__ import annotations
+
+import concurrent.futures
+import os
+import sys
+import time
+from typing import Any
+
+from cli_lane_board import _external, _record_board_access_error, _task_value
+from cli_lane_config import (
+    BOARD_CORRUPTION_ERRORS,
+    DEFAULT_CLAIM_TTL,
+    EXTERNAL_PREFIX,
+    RESULT_SCHEMA,
+    RESULT_SCHEMA_PATH,
+)
+from cli_lane_execution import execute_claim
+from cli_lane_files import atomic_json
+from cli_lane_recovery import _has_pending_finalization, recover_pending_finalizations
+from cli_lane_retention import maybe_gc_lane_artifacts
+
+
+def _board_slug(board: Any) -> str:
+    if isinstance(board, dict):
+        return str(board.get("slug") or board.get("id") or "")
+    return str(getattr(board, "slug", None) or getattr(board, "id", None) or board)
+
+def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None:
+    """Open one board without letting localized storage faults stop other lanes."""
+    try:
+        return kanban_db.connect(board=board)
+    except Exception as error:
+        _record_board_access_error(board, error)
+        return None
+
+def recover_orphans() -> None:
+    """Return external running tasks to ready after a runner/pod restart."""
+    from hermes_cli import kanban_db
+
+    recover_pending_finalizations()
+    try:
+        boards = kanban_db.list_boards(include_archived=False)
+    except Exception as error:
+        _record_board_access_error("board-registry", error)
+        return
+    for raw_board in boards:
+        board = _board_slug(raw_board)
+        if not board:
+            continue
+        with kanban_db.scoped_current_board(board):
+            conn = _connect_healthy_board(kanban_db, board)
+            if conn is None:
+                continue
+            try:
+                for task in kanban_db.list_tasks(conn):
+                    if _external(task) and str(_task_value(task, "status", "")) == "running":
+                        task_id = str(_task_value(task, "id"))
+                        run_id = _task_value(task, "current_run_id", None)
+                        if not isinstance(run_id, int):
+                            continue
+                        if _has_pending_finalization(board, task_id, run_id):
+                            continue
+                        kanban_db.reclaim_task(
+                            conn,
+                            task_id,
+                            reason="direct CLI lane restarted; provider session will resume",
+                            expected_run_id=run_id,
+                        )
+                BOARD_CORRUPTION_ERRORS.pop(board, None)
+            except Exception as error:
+                _record_board_access_error(board, error)
+            finally:
+                conn.close()
+
+def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str]]:
+    """Atomically claim external ready tasks across all non-archived boards."""
+    from hermes_cli import kanban_db
+
+    claimed: list[tuple[str, str]] = []
+    if limit <= 0:
+        return claimed
+    for raw_board in kanban_db.list_boards(include_archived=False):
+        board = _board_slug(raw_board)
+        if not board:
+            continue
+        with kanban_db.scoped_current_board(board):
+            conn = _connect_healthy_board(kanban_db, board)
+            if conn is None:
+                continue
+            try:
+                kanban_db.recompute_ready(conn)
+                tasks = kanban_db.list_tasks(conn)
+                BOARD_CORRUPTION_ERRORS.pop(board, None)
+                for task in tasks:
+                    task_id = str(_task_value(task, "id", ""))
+                    assignee = str(_task_value(task, "assignee", "") or "")
+                    if task_id and not assignee and str(_task_value(task, "status", "")) == "ready":
+                        if kanban_db.assign_task(conn, task_id, "cli-auto"):
+                            task = kanban_db.get_task(conn, task_id)
+                            assignee = "cli-auto"
+                    if (
+                        not task_id
+                        or (board, task_id) in active
+                        or not assignee.startswith(EXTERNAL_PREFIX)
+                        or str(_task_value(task, "status", "")) != "ready"
+                    ):
+                        continue
+                    try:
+                        result = kanban_db.claim_task(
+                            conn,
+                            task_id,
+                            ttl_seconds=DEFAULT_CLAIM_TTL,
+                            claimer="direct-cli-lane",
+                        )
+                    except Exception:
+                        continue
+                    if result is not None:
+                        claimed.append((board, task_id))
+                    if len(claimed) >= limit:
+                        return claimed
+            except Exception as error:
+                _record_board_access_error(board, error)
+            finally:
+                conn.close()
+    return claimed
+
+def main() -> int:
+    """Continuously bridge external Kanban lanes to provider CLIs."""
+    RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
+    atomic_json(RESULT_SCHEMA_PATH, RESULT_SCHEMA, 0o644)
+    recover_orphans()
+    workers = max(1, min(int(os.environ.get("HERMES_CLI_LANE_CONCURRENCY", "4")), 8))
+    futures: dict[concurrent.futures.Future[None], tuple[str, str]] = {}
+    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
+        while True:
+            for future in list(futures):
+                if future.done():
+                    try:
+                        future.result()
+                    except Exception as error:
+                        print(f"worker future failed: {error}", file=sys.stderr, flush=True)
+                    del futures[future]
+            recover_pending_finalizations()
+            maybe_gc_lane_artifacts()
+            active = set(futures.values())
+            try:
+                newly_claimed = claim_ready(active, workers - len(futures))
+                BOARD_CORRUPTION_ERRORS.pop("board-registry", None)
+            except Exception as error:
+                _record_board_access_error("board-registry", error)
+                newly_claimed = []
+            for board, task_id in newly_claimed:
+                future = pool.submit(execute_claim, board, task_id)
+                futures[future] = (board, task_id)
+            time.sleep(5)
+    return 0

services/hermes/scripts/cli_lane_evidence.py

@@ -0,0 +1,286 @@
+#!/usr/bin/env python3
+"""Immutable terminal evidence creation and identity-safe retirement."""
+
+from __future__ import annotations
+
+import ctypes
+import errno
+import hashlib
+import json
+import os
+import stat
+import uuid
+from pathlib import Path
+from typing import Any
+
+from cli_lane_config import MAX_TERMINAL_RECORD_BYTES, TerminalIdentity, utc_now
+from cli_lane_files import _terminal_evidence_identity, state_path
+from cli_lane_records import _load_small_json, _terminal_record_valid
+
+
+def _rename_noreplace(
+    source: str,
+    destination: str,
+    *,
+    source_dir: int,
+    destination_dir: int,
+) -> None:
+    """Atomically rename a directory entry without replacing a collision."""
+    renameat2 = getattr(ctypes.CDLL(None, use_errno=True), "renameat2", None)
+    if renameat2 is None:
+        raise OSError(errno.ENOSYS, "renameat2 is unavailable")
+    renameat2.argtypes = (
+        ctypes.c_int,
+        ctypes.c_char_p,
+        ctypes.c_int,
+        ctypes.c_char_p,
+        ctypes.c_uint,
+    )
+    renameat2.restype = ctypes.c_int
+    result = renameat2(
+        source_dir,
+        ctypes.c_char_p(os.fsencode(source)),
+        destination_dir,
+        ctypes.c_char_p(os.fsencode(destination)),
+        1,  # RENAME_NOREPLACE
+    )
+    if result != 0:
+        error = ctypes.get_errno()
+        raise OSError(error, os.strerror(error), source, destination)
+
+def _retire_terminal_entry(
+    path: Path,
+    source_stat: os.stat_result,
+    *,
+    board_descriptor: int,
+    quarantine_descriptor: int,
+    authority_name: str | None = None,
+) -> str:
+    """Retire only the inode previously inspected; preserve any replacement."""
+    try:
+        current = os.stat(
+            path.name,
+            dir_fd=board_descriptor,
+            follow_symlinks=False,
+        )
+    except FileNotFoundError:
+        return "missing"
+    if current.st_dev != source_stat.st_dev or current.st_ino != source_stat.st_ino:
+        return "replacement"
+    # A replacement can itself be staged repeatedly while recovery races a
+    # writer. Keep every generation bound to the original canonical pending
+    # name instead of hashing an intermediate .retire name and hiding it from
+    # the next recovery pass.
+    path_digest = hashlib.sha256(
+        (authority_name or path.name).encode("utf-8")
+    ).hexdigest()[:16]
+    source_digest = hashlib.sha256(
+        f"{source_stat.st_dev:x}\0{source_stat.st_ino:x}".encode("utf-8")
+    ).hexdigest()[:16]
+    for sequence in range(32):
+        staging = f".retire.{path_digest}.{source_digest}.{sequence}"
+        try:
+            _rename_noreplace(
+                path.name,
+                staging,
+                source_dir=board_descriptor,
+                destination_dir=quarantine_descriptor,
+            )
+        except FileExistsError:
+            continue
+        break
+    else:
+        return "collision"
+    staged = os.stat(
+        staging,
+        dir_fd=quarantine_descriptor,
+        follow_symlinks=False,
+    )
+    if staged.st_dev == source_stat.st_dev and staged.st_ino == source_stat.st_ino:
+        os.unlink(staging, dir_fd=quarantine_descriptor)
+        os.fsync(quarantine_descriptor)
+        os.fsync(board_descriptor)
+        return "retired"
+    # The path changed after the pre-check but before the rename. Put the
+    # replacement back without overwriting an even newer entry.
+    try:
+        _rename_noreplace(
+            staging,
+            path.name,
+            source_dir=quarantine_descriptor,
+            destination_dir=board_descriptor,
+        )
+    except FileExistsError:
+        return "replacement-staged"
+    os.fsync(quarantine_descriptor)
+    os.fsync(board_descriptor)
+    return "replacement"
+
+def _write_json_noreplace(path: Path, value: dict[str, Any]) -> bool:
+    """Durably create one immutable JSON artifact without replacing a peer."""
+    payload = (json.dumps(value, indent=2, sort_keys=True) + "\n").encode("utf-8")
+    if len(payload) > MAX_TERMINAL_RECORD_BYTES + 4096:
+        raise ValueError("terminal evidence exceeds the bounded artifact limit")
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    os.chmod(path.parent, 0o700, follow_symlinks=False)
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
+    create_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    create_flags |= getattr(os, "O_NOFOLLOW", 0)
+    directory = os.open(path.parent, directory_flags)
+    temporary = f".{path.name}.{uuid.uuid4().hex}.tmp"
+    temporary_stat = None
+    descriptor = None
+    try:
+        descriptor = os.open(temporary, create_flags, 0o600, dir_fd=directory)
+        temporary_stat = os.fstat(descriptor)
+        with os.fdopen(descriptor, "wb") as stream:
+            descriptor = None
+            stream.write(payload)
+            stream.flush()
+            os.fsync(stream.fileno())
+            written = os.fstat(stream.fileno())
+            if stat.S_IMODE(written.st_mode) != 0o600 or written.st_nlink != 1:
+                raise OSError("terminal evidence is not private and singly linked")
+        try:
+            _rename_noreplace(
+                temporary,
+                path.name,
+                source_dir=directory,
+                destination_dir=directory,
+            )
+        except FileExistsError:
+            return False
+        os.fsync(directory)
+        return True
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        if temporary_stat is not None:
+            try:
+                _retire_terminal_entry(
+                    Path(temporary),
+                    temporary_stat,
+                    board_descriptor=directory,
+                    quarantine_descriptor=directory,
+                )
+            except OSError:
+                pass
+        os.close(directory)
+
+def _terminal_document_digest(document: dict[str, Any]) -> str:
+    """Return the stable identity for one exact accepted result document."""
+    immutable = {
+        key: document.get(key)
+        for key in (
+            "board",
+            "task_id",
+            "expected_run_id",
+            "result",
+            "summary",
+            "metadata",
+        )
+    }
+    canonical = json.dumps(immutable, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:32]
+
+def _terminal_evidence_path(
+    identity: TerminalIdentity,
+    state: str,
+    document: dict[str, Any],
+) -> Path:
+    """Return a deterministic immutable prepared or conflict evidence path."""
+    if state not in {"prepared", "conflict"}:
+        raise ValueError(f"invalid terminal evidence state: {state}")
+    base = state_path(identity.board, identity.task_id)
+    digest = _terminal_document_digest(document)
+    return base.with_name(
+        f"{base.stem}.run-{identity.run_id}.terminal.{state}-{digest}.json"
+    )
+
+def _same_terminal_document(left: dict[str, Any], right: dict[str, Any]) -> bool:
+    """Compare the immutable result-bearing fields of terminal evidence."""
+    keys = ("board", "task_id", "expected_run_id", "result", "summary", "metadata")
+    return all(left.get(key) == right.get(key) for key in keys)
+
+def _terminal_evidence_valid(
+    document: dict[str, Any],
+    identity: TerminalIdentity,
+    state: str,
+) -> bool:
+    """Validate semantic terminal content plus its immutable evidence state."""
+    if document.get("kanban_state") != state:
+        return False
+    semantic = dict(document)
+    semantic_identity = identity
+    if state == "conflict":
+        semantic["kanban_state"] = "prepared"
+        semantic_identity = TerminalIdentity(
+            identity.board,
+            identity.task_id,
+            identity.run_id,
+            "prepared",
+        )
+    return _terminal_record_valid(semantic, semantic_identity)
+
+def _persist_prepared_evidence(
+    identity: TerminalIdentity,
+    document: dict[str, Any],
+) -> Path:
+    """Fsync an immutable accepted result before entering the DB boundary."""
+    prepared = dict(document)
+    prepared["kanban_state"] = "prepared"
+    prepared["prepared_at"] = utc_now()
+    path = _terminal_evidence_path(identity, "prepared", document)
+    if _write_json_noreplace(path, prepared):
+        return path
+    if _terminal_evidence_identity(path) != TerminalIdentity(
+        identity.board, identity.task_id, identity.run_id, "prepared"
+    ):
+        raise OSError("prepared terminal evidence identity is invalid")
+    existing = _load_small_json(path)
+    prepared_identity = TerminalIdentity(
+        identity.board,
+        identity.task_id,
+        identity.run_id,
+        "prepared",
+    )
+    if not _terminal_evidence_valid(
+        existing,
+        prepared_identity,
+        "prepared",
+    ) or not _same_terminal_document(existing, prepared):
+        raise OSError("prepared terminal evidence path contains a conflicting result")
+    return path
+
+def _persist_conflict_evidence(
+    identity: TerminalIdentity,
+    document: dict[str, Any],
+    reason: str,
+) -> Path:
+    """Preserve a losing valid result in full under deterministic retention."""
+    conflict = dict(document)
+    conflict["kanban_state"] = "conflict"
+    conflict["conflict_reason"] = reason
+    conflict["conflicted_at"] = utc_now()
+    path = _terminal_evidence_path(identity, "conflict", document)
+    if _write_json_noreplace(path, conflict):
+        return path
+    if _terminal_evidence_identity(path) != TerminalIdentity(
+        identity.board, identity.task_id, identity.run_id, "conflict"
+    ):
+        raise OSError("terminal conflict evidence identity is invalid")
+    existing = _load_small_json(path)
+    conflict_identity = TerminalIdentity(
+        identity.board,
+        identity.task_id,
+        identity.run_id,
+        "conflict",
+    )
+    if not _terminal_evidence_valid(
+        existing,
+        conflict_identity,
+        "conflict",
+    ) or not _same_terminal_document(existing, conflict):
+        raise OSError("terminal conflict evidence path contains a different result")
+    return path

services/hermes/scripts/cli_lane_execution.py

@@ -0,0 +1,425 @@
+#!/usr/bin/env python3
+"""Single-claim provider execution and durable Kanban result handoff."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import time
+from pathlib import Path
+
+import cli_lane_goal
+from cli_lane_board import _board_call, _resolve_workspace, _task_context, _task_value
+from cli_lane_config import (
+    DEFAULT_MAX_RUNTIME,
+    TerminalFinalizationPending,
+)
+from cli_lane_files import (
+    _terminal_identity,
+    _terminal_path,
+    atomic_json,
+    load_json,
+    state_path,
+)
+from cli_lane_finalization import _finalize_terminal_record, _recover_exact_run
+from cli_lane_prompt import build_prompt, git_handoff, workspace_artifacts
+from cli_lane_provider import run_provider
+from cli_lane_records import _persist_candidate, _write_terminal_record
+from cli_lane_recovery import _has_pending_finalization
+from cli_lane_routing import fresh_unavailable_provider, select_route
+
+
+def execute_claim(board: str, task_id: str) -> None:
+    """Execute one already-claimed task and commit its outcome to Kanban."""
+    from hermes_cli import kanban_db
+
+    run_id = None
+    with kanban_db.scoped_current_board(board):
+        conn = kanban_db.connect(board=board)
+        try:
+            task = kanban_db.get_task(conn, task_id)
+            if task is None:
+                return
+            run_id = _task_value(task, "current_run_id", None)
+            assignee = str(_task_value(task, "assignee", "cli-auto") or "cli-auto")
+            state_file = state_path(board, task_id)
+            state = load_json(state_file)
+            state.update(
+                {
+                    "board": board,
+                    "task_id": task_id,
+                    "run_id": run_id,
+                    "assignee": assignee,
+                }
+            )
+            atomic_json(state_file, state)
+            log_path = Path(kanban_db.worker_log_path(task_id, board=board))
+            workspace = _resolve_workspace(kanban_db, conn, task, board)
+            kanban_db.set_workspace_path(conn, task_id, str(workspace))
+            context = _task_context(kanban_db, conn, task_id)
+        except Exception as error:
+            failure_reason = (
+                f"Direct CLI lane preparation failed: {type(error).__name__}: {error}"
+            )
+            conn.close()
+            _board_call(
+                kanban_db,
+                board,
+                lambda fresh: kanban_db.block_task(
+                    fresh,
+                    task_id,
+                    reason=failure_reason,
+                    kind="capability",
+                    expected_run_id=run_id,
+                ),
+            )
+            return
+        finally:
+            conn.close()
+
+        def heartbeat(note: str) -> bool:
+            try:
+                return bool(
+                    _board_call(
+                        kanban_db,
+                        board,
+                        lambda fresh: kanban_db.heartbeat_worker(
+                            fresh,
+                            task_id,
+                            note=note,
+                            expected_run_id=run_id,
+                        ),
+                    )
+                )
+            except (OSError, sqlite3.Error):
+                # The claim TTL is deliberately long. Keep the expensive
+                # provider process alive during a transient volume stall and
+                # retry on the next heartbeat instead of losing its work.
+                return True
+
+        def comment(body: str) -> None:
+            try:
+                _board_call(
+                    kanban_db,
+                    board,
+                    lambda fresh: kanban_db.add_comment(
+                        fresh,
+                        task_id,
+                        "cli-lane-runner",
+                        body,
+                    ),
+                )
+            except (OSError, sqlite3.Error):
+                # Route state is also written to the durable lane-state file;
+                # a later heartbeat or terminal result remains authoritative.
+                pass
+
+        try:
+            previous_route = state.get("current_route")
+            excluded_provider = (
+                fresh_unavailable_provider() if assignee == "cli-auto" else None
+            )
+            unavailable_provider = excluded_provider
+            route = select_route(
+                context,
+                assignee,
+                exclude_provider=excluded_provider,
+                exclude_reason="is unavailable according to fresh native health"
+                if excluded_provider
+                else None,
+            )
+            if excluded_provider:
+                comment(
+                    f"Provider health guard excluded {excluded_provider} before automatic routing.",
+                )
+            comment(
+                f"CLI route: {route.provider}/{route.model} at {route.effort}; classifier={route.classifier}; {route.reason}",
+            )
+            resume_handoff = ""
+            if (
+                isinstance(previous_route, dict)
+                and previous_route.get("provider")
+                and previous_route.get("provider") != route.provider
+            ):
+                try:
+                    previous_output = log_path.read_text(encoding="utf-8")[-10000:]
+                except OSError:
+                    previous_output = "Previous provider log was unavailable after restart."
+                resume_handoff = git_handoff(workspace, previous_output)
+                comment(
+                    f"Restart-time provider change: {previous_route.get('provider')} -> {route.provider}; explicit workspace handoff attached.",
+                )
+            prompt = build_prompt(context, workspace, resume_handoff)
+            max_runtime = int(
+                _task_value(task, "max_runtime_seconds", 0) or DEFAULT_MAX_RUNTIME
+            )
+            goal_mode = bool(_task_value(task, "goal_mode", False))
+            goal_max_turns = max(1, int(_task_value(task, "goal_max_turns", 1) or 1))
+            goal_turn = max(1, int(state.get("goal_turn", 0) or 0) + 1)
+            deadline = time.monotonic() + max_runtime
+            while True:
+                state["goal_turn"] = goal_turn
+                atomic_json(state_file, state)
+                remaining = max(1, int(deadline - time.monotonic()))
+                result = run_provider(
+                    route,
+                    prompt,
+                    workspace,
+                    state,
+                    state_file,
+                    log_path,
+                    heartbeat,
+                    remaining,
+                )
+                candidate_file = None
+                if result.structured:
+                    candidate_file = _persist_candidate(
+                        state,
+                        state_file,
+                        dict(result.structured),
+                        route=route,
+                        returncode=result.returncode,
+                        goal_turn=goal_turn,
+                    )
+                if result.capacity_failure:
+                    unavailable_provider = route.provider
+                    retry_context = (
+                        context
+                        + "\n\nRouting boundary: the first provider failed from capacity/authentication. "
+                        + "Select the alternate hosted provider at an appropriate effort."
+                    )
+                    alternate = "claude" if route.provider == "codex" else "codex"
+                    fallback = select_route(
+                        retry_context,
+                        f"cli-{alternate}-{route.effort}",
+                    )
+                    comment(
+                        f"Provider fallback: {route.provider} -> {fallback.provider}; Jetson reclassified the retry boundary.",
+                    )
+                    result = run_provider(
+                        fallback,
+                        build_prompt(
+                            context,
+                            workspace,
+                            git_handoff(workspace, result.output),
+                        ),
+                        workspace,
+                        state,
+                        state_file,
+                        log_path,
+                        heartbeat,
+                        max(1, int(deadline - time.monotonic())),
+                    )
+                    route = fallback
+                    if result.structured:
+                        candidate_file = _persist_candidate(
+                            state,
+                            state_file,
+                            dict(result.structured),
+                            route=route,
+                            returncode=result.returncode,
+                            goal_turn=goal_turn,
+                        )
+                structured = result.structured
+                if structured:
+                    structured = dict(structured)
+                    structured["artifacts"] = workspace_artifacts(
+                        workspace,
+                        structured.get("artifacts"),
+                    )
+                metadata = {
+                    "executor": "direct-cli-lane",
+                    "provider": route.provider,
+                    "model": route.model,
+                    "effort": route.effort,
+                    "classifier": route.classifier,
+                    "state_file": str(state_file),
+                    "codex_thread_id": state.get("codex_thread_id"),
+                    "claude_session_id": state.get("claude_session_id"),
+                    "goal_mode": goal_mode,
+                    "goal_turn": goal_turn,
+                }
+                if candidate_file is not None:
+                    metadata["candidate_file"] = str(candidate_file)
+                if structured:
+                    for key in (
+                        "changed_files",
+                        "tests_run",
+                        "artifacts",
+                        "findings",
+                        "blockers",
+                    ):
+                        value = structured.get(key)
+                        metadata[key] = value if isinstance(value, list) else []
+
+                completion_problem = None
+                if structured and result.returncode == 0:
+                    completion_problem = cli_lane_goal.unfinished_result_reason(structured)
+                    if (
+                        structured.get("status") == "completed"
+                        and completion_problem is None
+                        and goal_mode
+                    ):
+                        heartbeat("local goal-completion judge active")
+                        rejection_history = state.get("goal_rejections", [])
+                        if not isinstance(rejection_history, list):
+                            rejection_history = []
+                        judge_context = context
+                        if goal_turn > 1 or rejection_history:
+                            judge_context += (
+                                "\n\nAuthoritative Hermes goal-controller evidence: "
+                                f"current turn {goal_turn}/{goal_max_turns}; prior rejected "
+                                f"reports: {json.dumps(rejection_history[-5:])}."
+                            )
+                        accepted, judge_reason = cli_lane_goal.judge_goal_completion(
+                            judge_context,
+                            structured,
+                        )
+                        metadata["goal_judge_reason"] = judge_reason
+                        if not accepted:
+                            completion_problem = f"local goal judge requested continuation: {judge_reason}"
+
+                if (
+                    structured
+                    and structured.get("status") == "completed"
+                    and result.returncode == 0
+                    and completion_problem is None
+                ):
+                    terminal_file = _terminal_path(state_file, run_id, "pending")
+                    metadata["terminal_record"] = str(
+                        _terminal_path(state_file, run_id, "committed")
+                    )
+                    try:
+                        terminal_file, terminal_record = _write_terminal_record(
+                            state_file,
+                            board=board,
+                            task_id=task_id,
+                            run_id=run_id,
+                            structured=structured,
+                            summary=str(structured.get("summary") or "Completed"),
+                            metadata=metadata,
+                        )
+                        outcome = _finalize_terminal_record(
+                            kanban_db,
+                            terminal_file,
+                            terminal_record,
+                        )
+                    except Exception as error:
+                        identity = _terminal_identity(terminal_file)
+                        replayable = bool(
+                            identity is not None
+                            and _has_pending_finalization(
+                                identity.board,
+                                identity.task_id,
+                                identity.run_id,
+                            )
+                        )
+                        if not replayable:
+                            _recover_exact_run(
+                                kanban_db,
+                                identity,
+                                f"persistence raised {type(error).__name__}",
+                            )
+                        raise TerminalFinalizationPending(
+                            "accepted worker result remains outside the generic block path; "
+                            f"terminal replayable={replayable}; persistence/finalization "
+                            f"raised {type(error).__name__}"
+                        ) from error
+                    if outcome != "committed":
+                        raise TerminalFinalizationPending(
+                            "accepted worker result is durably journaled but "
+                            f"Kanban finalization is {outcome}"
+                        )
+                    break
+
+                can_continue = (
+                    goal_mode
+                    and completion_problem is not None
+                    and goal_turn < goal_max_turns
+                    and deadline - time.monotonic() > 30
+                )
+                if can_continue:
+                    rejection_history = state.get("goal_rejections", [])
+                    if not isinstance(rejection_history, list):
+                        rejection_history = []
+                    state["goal_rejections"] = [
+                        *rejection_history[-4:],
+                        completion_problem,
+                    ]
+                    goal_turn += 1
+                    comment(
+                        f"Goal completion rejected; continuing turn {goal_turn}/{goal_max_turns}: {completion_problem}",
+                    )
+                    escalation_context = (
+                        context
+                        + "\n\nThe previous worker turn missed its completion quality mark: "
+                        + completion_problem
+                        + "\nSelect a route that can finish and verify the remaining work."
+                    )
+                    excluded = unavailable_provider
+                    if excluded is None and assignee == "cli-auto":
+                        excluded = fresh_unavailable_provider()
+                    next_route = select_route(
+                        escalation_context,
+                        assignee,
+                        exclude_provider=excluded,
+                        exclude_reason="is unavailable according to fresh native health"
+                        if excluded
+                        else None,
+                    )
+                    comment(
+                        f"Goal route {goal_turn}/{goal_max_turns}: "
+                        f"{next_route.provider}/{next_route.model} at {next_route.effort}; "
+                        f"classifier={next_route.classifier}; {next_route.reason}",
+                    )
+                    handoff = (
+                        git_handoff(workspace, result.output)
+                        if next_route.provider != route.provider
+                        else ""
+                    )
+                    route = next_route
+                    prompt = build_prompt(
+                        context,
+                        workspace,
+                        handoff
+                        + "\nGoal-loop continuation: the previous final report was rejected because "
+                        + completion_problem
+                        + ". Reinspect live state, finish the outstanding work, and return new final evidence.",
+                    )
+                    continue
+
+                reason = completion_problem
+                if reason is None and structured:
+                    blockers = structured.get("blockers", [])
+                    reason = "; ".join(str(item) for item in blockers)
+                    reason = reason or str(structured.get("summary") or "")
+                if reason is None:
+                    reason = result.output[-4000:]
+                _board_call(
+                    kanban_db,
+                    board,
+                    lambda fresh: kanban_db.block_task(
+                        fresh,
+                        task_id,
+                        reason=reason
+                        or f"{route.provider} worker failed with exit {result.returncode}",
+                        kind="transient" if result.capacity_failure else "capability",
+                        expected_run_id=run_id,
+                    ),
+                )
+                break
+        except TerminalFinalizationPending as error:
+            comment(str(error))
+        except Exception as error:
+            failure_reason = f"Direct CLI lane failed: {type(error).__name__}: {error}"
+            _board_call(
+                kanban_db,
+                board,
+                lambda fresh: kanban_db.block_task(
+                    fresh,
+                    task_id,
+                    reason=failure_reason,
+                    kind="capability",
+                    expected_run_id=run_id,
+                ),
+            )

services/hermes/scripts/cli_lane_files.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+"""Durable path and filesystem primitives for Hermes CLI lane journals."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import stat
+import uuid
+from pathlib import Path
+from typing import Any
+
+from cli_lane_config import STATE_ROOT, TerminalIdentity, canonical_run_id
+
+
+def atomic_json(path: Path, value: dict[str, Any], mode: int = 0o600) -> None:
+    """Durably replace a small state document without following temp symlinks."""
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    os.chmod(path.parent, 0o700, follow_symlinks=False)
+    temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    flags |= getattr(os, "O_NOFOLLOW", 0)
+    descriptor = None
+    try:
+        descriptor = os.open(temporary, flags, mode)
+        with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
+            descriptor = None
+            json.dump(value, stream, indent=2, sort_keys=True)
+            stream.write("\n")
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.chmod(temporary, mode, follow_symlinks=False)
+        os.replace(temporary, path)
+        _fsync_directory(path.parent)
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        try:
+            temporary.unlink()
+        except FileNotFoundError:
+            pass
+
+def _fsync_directory(directory: Path) -> None:
+    """Persist directory-entry changes after an atomic rename or quarantine."""
+    flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    flags |= getattr(os, "O_NOFOLLOW", 0)
+    descriptor = os.open(directory, flags)
+    try:
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+def load_json(path: Path) -> dict[str, Any]:
+    try:
+        flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+        descriptor = os.open(path, flags)
+        with os.fdopen(descriptor, "r", encoding="utf-8") as stream:
+            value = json.load(stream)
+    except (OSError, UnicodeError, json.JSONDecodeError):
+        return {}
+    return value if isinstance(value, dict) else {}
+
+def state_path(board: str, task_id: str) -> Path:
+    safe_board = re.sub(r"[^a-zA-Z0-9_.-]+", "-", board)
+    safe_task = re.sub(r"[^a-zA-Z0-9_.-]+", "-", task_id)
+    return STATE_ROOT / safe_board / f"{safe_task}.json"
+
+def _result_path(state_file: Path, run_id: Any, sequence: int) -> Path:
+    """Return a unique provider-result path; completed turns are never reused."""
+    safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown"))
+    return state_file.with_name(
+        f"{state_file.stem}.run-{safe_run}.provider-{sequence}.result.json"
+    )
+
+def _candidate_path(state_file: Path, run_id: Any, sequence: int) -> Path:
+    """Return the durable path for one exact structured worker response."""
+    safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown"))
+    return state_file.with_name(
+        f"{state_file.stem}.run-{safe_run}.candidate-{sequence}.json"
+    )
+
+def _terminal_path(
+    state_file: Path,
+    run_id: Any,
+    state: str = "pending",
+) -> Path:
+    """Return the identity-bound replay path for an accepted terminal response."""
+    if state not in {"pending", "committed"}:
+        raise ValueError(f"invalid terminal journal state: {state}")
+    canonical = canonical_run_id(run_id)
+    if type(run_id) is not int or canonical is None:
+        raise ValueError("terminal run ID must be a positive SQLite int64")
+    return state_file.with_name(
+        f"{state_file.stem}.run-{canonical}.terminal.{state}.json"
+    )
+
+_TERMINAL_NAME = re.compile(
+    r"^(?P<task>[a-zA-Z0-9_.-]+)\.run-(?P<run>[0-9]+)\."
+    r"terminal\.(?P<state>pending|committed)\.json$"
+)
+_TERMINAL_EVIDENCE_NAME = re.compile(
+    r"^(?P<task>[a-zA-Z0-9_.-]+)\.run-(?P<run>[0-9]+)\."
+    r"terminal\.(?P<state>prepared|conflict)-(?P<digest>[a-f0-9]{32})\.json$"
+)
+_RETIRE_NAME = re.compile(
+    r"^\.retire\.(?P<path_digest>[a-f0-9]{16})\."
+    r"(?P<source_digest>[a-f0-9]{16})\.(?P<sequence>[0-9]+)$"
+)
+_LEGACY_RETIRE_NAME = re.compile(r"^\.retire\.[a-f0-9]{32}\.[0-9]+$")
+_SAFE_BOARD = re.compile(r"^[a-zA-Z0-9_.-]+$")
+
+
+def _terminal_identity(path: Path) -> TerminalIdentity | None:
+    """Parse replay authority lexically before opening a journal or board."""
+    try:
+        relative = path.relative_to(STATE_ROOT)
+    except ValueError:
+        return None
+    if len(relative.parts) != 2:
+        return None
+    board, filename = relative.parts
+    match = _TERMINAL_NAME.fullmatch(filename)
+    run_id = canonical_run_id(match.group("run")) if match else None
+    try:
+        board_stat = path.parent.stat(follow_symlinks=False)
+    except OSError:
+        return None
+    if (
+        not match
+        or run_id is None
+        or not stat.S_ISDIR(board_stat.st_mode)
+        or not _SAFE_BOARD.fullmatch(board)
+        or board in {".", ".."}
+        or match.group("task") in {".", ".."}
+    ):
+        return None
+    return TerminalIdentity(
+        board=board,
+        task_id=match.group("task"),
+        run_id=run_id,
+        state=match.group("state"),
+    )
+
+def _terminal_evidence_identity(path: Path) -> TerminalIdentity | None:
+    """Parse immutable prepared/conflict evidence authority from its path."""
+    try:
+        relative = path.relative_to(STATE_ROOT)
+    except ValueError:
+        return None
+    if len(relative.parts) != 2:
+        return None
+    board, filename = relative.parts
+    match = _TERMINAL_EVIDENCE_NAME.fullmatch(filename)
+    run_id = canonical_run_id(match.group("run")) if match else None
+    try:
+        board_stat = path.parent.stat(follow_symlinks=False)
+    except OSError:
+        return None
+    if (
+        not match
+        or run_id is None
+        or not stat.S_ISDIR(board_stat.st_mode)
+        or not _SAFE_BOARD.fullmatch(board)
+        or board in {".", ".."}
+        or match.group("task") in {".", ".."}
+    ):
+        return None
+    return TerminalIdentity(
+        board=board,
+        task_id=match.group("task"),
+        run_id=run_id,
+        state=match.group("state"),
+    )

services/hermes/scripts/cli_lane_finalization.py

@@ -0,0 +1,323 @@
+#!/usr/bin/env python3
+"""Exact-run database completion and terminal evidence publication."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Any
+
+from cli_lane_board import _board_call, _external, _record_board_access_error, _task_value
+from cli_lane_config import (
+    TerminalIdentity,
+    TerminalRecoverySnapshot,
+    TerminalSnapshot,
+    canonical_run_id,
+    utc_now,
+)
+from cli_lane_evidence import (
+    _persist_conflict_evidence,
+    _persist_prepared_evidence,
+    _retire_terminal_entry,
+    _same_terminal_document,
+    _write_json_noreplace,
+)
+from cli_lane_files import _terminal_identity, _terminal_path, state_path
+from cli_lane_records import (
+    _load_terminal_json,
+    _open_small_json_snapshot,
+    _open_terminal_snapshot,
+    _terminal_record_valid,
+)
+
+
+def _recover_exact_run(
+    kanban_db: Any,
+    identity: TerminalIdentity | None,
+    reason: str,
+) -> bool:
+    """Make an exact external run retryable after journal recovery fails."""
+    if identity is None or canonical_run_id(identity.run_id) is None:
+        return False
+
+    def operation(conn: Any) -> bool:
+        task = kanban_db.get_task(conn, identity.task_id)
+        if task is None:
+            return False
+        if (
+            str(_task_value(task, "status", "")) != "running"
+            or _task_value(task, "current_run_id", None) != identity.run_id
+            or not _external(task)
+        ):
+            return False
+        return bool(
+            kanban_db.reclaim_task(
+                conn,
+                identity.task_id,
+                reason=f"terminal journal recovery failed ({reason}); exact run may retry",
+                expected_run_id=identity.run_id,
+            )
+        )
+
+    try:
+        recovered = bool(_board_call(kanban_db, identity.board, operation))
+    except Exception as error:
+        _record_board_access_error(identity.board, error)
+        return False
+    if recovered:
+        print(
+            f"reclaimed {identity.board}/{identity.task_id} run {identity.run_id} "
+            f"after terminal journal recovery: {reason}",
+            file=sys.stderr,
+            flush=True,
+        )
+    return recovered
+
+def _retire_snapshot(
+    path: Path,
+    snapshot: TerminalSnapshot,
+    *,
+    authority_name: str | None = None,
+) -> str:
+    """Retire only the directory entry still naming an opened snapshot inode."""
+    return _retire_terminal_entry(
+        path,
+        snapshot.file_stat,
+        board_descriptor=snapshot.directory_descriptor,
+        quarantine_descriptor=snapshot.directory_descriptor,
+        authority_name=authority_name,
+    )
+
+def _retire_snapshot_after_db(path: Path, snapshot: TerminalSnapshot) -> str:
+    """Keep a committed DB result recoverable across retirement fsync errors."""
+    try:
+        return _retire_snapshot(path, snapshot)
+    except OSError as error:
+        print(
+            f"terminal journal retirement deferred after DB commit: "
+            f"{type(error).__name__}: {error}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return "deferred"
+
+def _discard_evidence(path: Path) -> str:
+    """Identity-safely remove one immutable prepared evidence artifact."""
+    snapshot = _open_small_json_snapshot(path)
+    if snapshot is None:
+        return "missing"
+    try:
+        return _retire_snapshot(path, snapshot)
+    finally:
+        snapshot.close()
+
+def _promote_prepared_evidence(
+    identity: TerminalIdentity,
+    document: dict[str, Any],
+    prepared: Path,
+    *,
+    discard_prepared: bool = True,
+) -> tuple[Path, str]:
+    """Publish the DB-winning result without overwriting first-writer evidence."""
+    committed = _terminal_path(
+        state_path(identity.board, identity.task_id),
+        identity.run_id,
+        "committed",
+    )
+    committed_document = dict(document)
+    committed_document["kanban_state"] = "committed"
+    committed_document["committed_at"] = utc_now()
+    state = "committed"
+    if not _write_json_noreplace(committed, committed_document):
+        committed_identity = TerminalIdentity(
+            identity.board,
+            identity.task_id,
+            identity.run_id,
+            "committed",
+        )
+        existing = _load_terminal_json(committed, committed_identity)
+        if (
+            not _terminal_record_valid(existing, committed_identity)
+            or existing.get("kanban_state") != "committed"
+            or not _same_terminal_document(existing, committed_document)
+        ):
+            _persist_conflict_evidence(
+                identity,
+                document,
+                "canonical committed evidence already contains a different result",
+            )
+            state = "conflict"
+    if discard_prepared:
+        _discard_evidence(prepared)
+    return committed, state
+
+def _finalize_document_db(
+    kanban_db: Any,
+    identity: TerminalIdentity,
+    document: dict[str, Any],
+) -> str:
+    """Apply one validated exact-run result to Kanban under DB run guards."""
+    if canonical_run_id(identity.run_id) is None:
+        return "stale"
+
+    def operation(conn: Any) -> str:
+        task = kanban_db.get_task(conn, identity.task_id)
+        if task is None:
+            return "stale"
+        status = str(_task_value(task, "status", ""))
+        if status == "done":
+            return (
+                "committed"
+                if (
+                    _task_value(task, "completed_run_id", None) == identity.run_id
+                    and str(_task_value(task, "result", "") or "")
+                    == document["result"]
+                )
+                else "stale"
+            )
+        current_run_id = _task_value(task, "current_run_id", None)
+        completion_guard: dict[str, int]
+        if status == "running" and current_run_id == identity.run_id:
+            completion_guard = {"expected_run_id": identity.run_id}
+        elif status in {"ready", "blocked", "triage"} and current_run_id is None:
+            # The journal may have survived an older post-persistence error
+            # path that ended its run. The patched DB verifies atomically that
+            # this is still the latest ended run before allowing completion.
+            completion_guard = {"replay_ended_run_id": identity.run_id}
+        else:
+            return "stale"
+        completed = kanban_db.complete_task(
+            conn,
+            identity.task_id,
+            result=document["result"],
+            summary=document["summary"],
+            metadata=document["metadata"],
+            **completion_guard,
+        )
+        if completed:
+            return "committed"
+        # A duplicate finalizer can lose the guarded UPDATE to an identical
+        # first writer. Re-read rather than reporting a false pending state.
+        latest = kanban_db.get_task(conn, identity.task_id)
+        latest_status = (
+            str(_task_value(latest, "status", "")) if latest is not None else ""
+        )
+        if latest_status == "done":
+            return (
+                "committed"
+                if (
+                    _task_value(latest, "completed_run_id", None) == identity.run_id
+                    and str(_task_value(latest, "result", "") or "")
+                    == document["result"]
+                )
+                else "stale"
+            )
+        # A guarded completion can be retried only while this exact live run
+        # still owns the task. An ended, replaced, or nonexistent run cannot
+        # become authoritative later and must converge to conflict evidence.
+        if (
+            latest_status == "running"
+            and _task_value(latest, "current_run_id", None) == identity.run_id
+        ):
+            return "pending"
+        return "stale"
+
+    return str(_board_call(kanban_db, identity.board, operation))
+
+def _resolve_pending_after_winner(
+    path: Path,
+    identity: TerminalIdentity,
+    winning_document: dict[str, Any],
+) -> bool:
+    """Retire duplicates and preserve differing replacements as conflicts."""
+    for _attempt in range(8):
+        snapshot = _open_terminal_snapshot(path, identity)
+        if snapshot is None:
+            return False
+        document = snapshot.document
+        if not _terminal_record_valid(document, identity):
+            snapshot.close()
+            return False
+        differing = not _same_terminal_document(document, winning_document)
+        try:
+            if differing:
+                _persist_conflict_evidence(
+                    identity,
+                    document,
+                    "different valid result lost the exact-run first-writer race",
+                )
+            retirement = _retire_snapshot_after_db(path, snapshot)
+        finally:
+            snapshot.close()
+        if retirement in {"retired", "missing"}:
+            if differing:
+                print(
+                    f"preserved terminal result conflict for {identity.board}/"
+                    f"{identity.task_id} run {identity.run_id}",
+                    file=sys.stderr,
+                    flush=True,
+                )
+            return differing
+        if retirement not in {"replacement", "replacement-staged", "collision"}:
+            return differing
+    return False
+
+def _finalize_terminal_record(
+    kanban_db: Any,
+    path: Path,
+    _record: dict[str, Any] | None = None,
+    *,
+    snapshot: TerminalRecoverySnapshot | None = None,
+) -> str:
+    """Commit one inode-pinned exact-run journal, or leave it safely pending."""
+    identity = _terminal_identity(path)
+    if identity is None or identity.state != "pending":
+        if snapshot is not None:
+            snapshot.close()
+        return "invalid"
+    pinned: TerminalSnapshot | TerminalRecoverySnapshot | None = snapshot
+    if pinned is None:
+        pinned = _open_terminal_snapshot(path, identity)
+    if pinned is None or pinned.document is None:
+        if pinned is not None:
+            pinned.close()
+        return "invalid"
+    document = pinned.document
+    try:
+        if not _terminal_record_valid(document):
+            return "invalid"
+        if not _terminal_record_valid(document, identity):
+            return "foreign"
+        prepared = _persist_prepared_evidence(identity, document)
+        outcome = _finalize_document_db(kanban_db, identity, document)
+        if outcome == "committed":
+            _committed, evidence_state = _promote_prepared_evidence(
+                identity,
+                document,
+                prepared,
+            )
+            retirement = _retire_snapshot_after_db(path, pinned)
+        elif outcome == "stale":
+            _persist_conflict_evidence(
+                identity,
+                document,
+                "valid terminal result no longer matches the authoritative task run",
+            )
+            _discard_evidence(prepared)
+            retirement = _retire_snapshot_after_db(path, pinned)
+            outcome = "conflict"
+            evidence_state = "conflict"
+        else:
+            return outcome
+    finally:
+        pinned.close()
+    if outcome == "committed" and retirement not in {"retired", "missing"}:
+        _resolve_pending_after_winner(path, identity, document)
+    if evidence_state == "conflict":
+        print(
+            f"terminal first-writer conflict recorded for {identity.board}/"
+            f"{identity.task_id} run {identity.run_id}",
+            file=sys.stderr,
+            flush=True,
+        )
+    return outcome

services/hermes/scripts/cli_lane_prompt.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""Prompt, artifact, and structured-event helpers for CLI workers."""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+from pathlib import Path
+from typing import Any
+
+import cli_lane_goal
+from cli_lane_files import atomic_json
+
+
+def build_prompt(context: str, workspace: Path, handoff: str = "") -> str:
+    """Create a bounded worker contract with an explicit machine-readable result."""
+    return f"""You are a durable coding worker managed by Hermes Kanban.
+
+Work only on this objective and its acceptance criteria:
+
+{context}
+
+Workspace: {workspace}
+{handoff}
+
+Operate autonomously inside the workspace. Inspect before editing, preserve unrelated user changes, run proportionate tests, and do not claim completion without evidence. You have owner-level Kubernetes access in every namespace. Prefer Flux-tracked manifests for durable changes, but use kubectl, Flux, exec, port-forwarding, rollout operations, and existing Vault workflows when the objective or incident requires them. Persist any desired-state mutation back to Git. Do not force-push, hard-reset, clean untracked files, or expose credentials.
+
+Return a final JSON object matching the supplied schema. Use status=incomplete when required work, tests, commands, commits, pushes, or verification are still running or remain to be done. Use status=blocked only when an obstacle prevents completion of the assigned task itself. Never use status=completed for a progress report. For review or diagnostic tasks, put defects and risks in findings; those findings can make the reviewed change unfit to ship without blocking completion of the review. The blockers array must be empty whenever status is completed. List changed files, tests run, durable artifact paths, findings, and task blockers explicitly.
+"""
+
+def git_handoff(workspace: Path, prior_output: str) -> str:
+    """Build an explicit cross-provider handoff without transferring hidden state."""
+    def read_git(*args: str) -> str:
+        completed = subprocess.run(
+            ["git", "-C", str(workspace), *args],
+            text=True,
+            capture_output=True,
+            timeout=30,
+            check=False,
+        )
+        return (completed.stdout or completed.stderr).strip()[-12000:]
+
+    return (
+        "\nCross-provider handoff from a failed or exhausted worker:\n"
+        f"Git status:\n{read_git('status', '--short', '--branch')}\n\n"
+        f"Current diff summary:\n{read_git('diff', '--stat')}\n\n"
+        f"Prior worker tail:\n{prior_output[-10000:]}\n"
+        "Reinspect the workspace and verify all inherited claims before continuing."
+    )
+
+def workspace_artifacts(workspace: Path, values: Any) -> list[str]:
+    """Return existing regular artifacts contained by the task worktree."""
+    if not isinstance(values, list):
+        return []
+    root = workspace.resolve()
+    artifacts: list[str] = []
+    for value in values:
+        if not isinstance(value, str) or not value.strip():
+            continue
+        candidate = Path(value.strip()).expanduser()
+        if not candidate.is_absolute():
+            candidate = root / candidate
+        try:
+            resolved = candidate.resolve(strict=True)
+            resolved.relative_to(root)
+        except (OSError, RuntimeError, ValueError):
+            continue
+        if resolved.is_file():
+            artifacts.append(str(resolved))
+    return list(dict.fromkeys(artifacts))
+
+def _extract_json(value: Any) -> dict[str, Any] | None:
+    if isinstance(value, dict) and value.get("status") in cli_lane_goal.RESULT_STATUSES:
+        return value
+    if not isinstance(value, str):
+        return None
+    candidates = [value]
+    match = re.search(r"\{.*\}", value, re.DOTALL)
+    if match:
+        candidates.append(match.group(0))
+    for candidate in candidates:
+        try:
+            parsed = json.loads(candidate)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(parsed, dict) and parsed.get("status") in cli_lane_goal.RESULT_STATUSES:
+            return parsed
+    return None
+
+def _event_payload(provider: str, line: str, state: dict[str, Any], state_file: Path) -> dict[str, Any] | None:
+    """Persist provider session identifiers before interpreting final output."""
+    try:
+        event = json.loads(line)
+    except json.JSONDecodeError:
+        return None
+    if not isinstance(event, dict):
+        return None
+    if provider == "codex" and event.get("type") == "thread.started":
+        thread_id = event.get("thread_id") or event.get("thread", {}).get("id")
+        if thread_id:
+            state["codex_thread_id"] = str(thread_id)
+            atomic_json(state_file, state)
+    if provider == "claude" and event.get("session_id"):
+        state["claude_session_id"] = str(event["session_id"])
+        atomic_json(state_file, state)
+    for key in ("structured_output", "result", "output", "text"):
+        parsed = _extract_json(event.get(key))
+        if parsed:
+            return parsed
+    item = event.get("item")
+    if isinstance(item, dict):
+        for key in ("text", "content"):
+            parsed = _extract_json(item.get(key))
+            if parsed:
+                return parsed
+    return None

services/hermes/scripts/cli_lane_provider.py

@@ -0,0 +1,406 @@
+#!/usr/bin/env python3
+"""Native Codex and Claude process lifecycle for Hermes CLI workers."""
+
+from __future__ import annotations
+
+import json
+import os
+import selectors
+import signal
+import subprocess
+import time
+import uuid
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any, Callable
+
+import cli_lane_goal
+from cli_lane_config import (
+    CAPACITY_PATTERN,
+    CLAUDE_BIN,
+    CLAUDE_SESSION_COLLISION,
+    CLAUDE_SETTINGS,
+    CODEX_BIN,
+    DATA_ROOT,
+    HEARTBEAT_SECONDS,
+    NO_CLAUDE_SESSION,
+    NO_CODEX_THREAD,
+    ProcessResult,
+    RESULT_SCHEMA,
+    RESULT_SCHEMA_PATH,
+    Route,
+    utc_now,
+)
+from cli_lane_files import _result_path, atomic_json, load_json
+from cli_lane_prompt import _event_payload
+
+
+def stream_process(
+    command: list[str],
+    *,
+    provider: str,
+    cwd: Path,
+    env: dict[str, str],
+    log_path: Path,
+    state: dict[str, Any],
+    state_file: Path,
+    heartbeat: Callable[[str], bool],
+    max_runtime: int,
+) -> ProcessResult:
+    """Stream JSONL to Kanban logs while maintaining the authoritative lease."""
+    process = subprocess.Popen(
+        command,
+        cwd=cwd,
+        env=env,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        bufsize=1,
+        start_new_session=True,
+    )
+    assert process.stdout is not None
+    selector = selectors.DefaultSelector()
+    selector.register(process.stdout, selectors.EVENT_READ)
+    started = time.monotonic()
+    last_heartbeat = 0.0
+    known_descendants: dict[int, tuple[int, int]] = {}
+    lines: list[str] = []
+    structured: dict[str, Any] | None = None
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    forced_failure = ""
+    with log_path.open("a", encoding="utf-8") as log:
+        log.write(f"\n[{utc_now()}] starting {provider} worker\n")
+        log.flush()
+        while process.poll() is None:
+            known_descendants.update(_descendant_processes(process.pid))
+            now = time.monotonic()
+            if now - started > max_runtime:
+                _terminate_worker_process(process, known_descendants)
+                forced_failure = "worker exceeded its maximum runtime"
+                lines.append(forced_failure + "\n")
+                break
+            if now - last_heartbeat >= HEARTBEAT_SECONDS:
+                if not heartbeat(f"{provider} worker active for {round(now - started)}s"):
+                    _terminate_worker_process(process, known_descendants)
+                    forced_failure = "Kanban lease was lost; provider process terminated"
+                    lines.append(forced_failure + "\n")
+                    break
+                last_heartbeat = now
+            for key, _ in selector.select(timeout=1.0):
+                line = key.fileobj.readline()
+                if not line:
+                    continue
+                lines.append(line)
+                if len(lines) > 4000:
+                    lines = lines[-4000:]
+                log.write(line)
+                log.flush()
+                parsed = _event_payload(provider, line, state, state_file)
+                structured = parsed or structured
+        _terminate_worker_process(process, known_descendants)
+        remainder = process.stdout.read()
+        if remainder:
+            lines.append(remainder)
+            log.write(remainder)
+            for line in remainder.splitlines():
+                parsed = _event_payload(provider, line, state, state_file)
+                structured = parsed or structured
+        if forced_failure:
+            log.write(forced_failure + "\n")
+        log.write(f"\n[{utc_now()}] {provider} exit={process.returncode}\n")
+    selector.close()
+    output = "".join(lines)[-100000:]
+    returncode = int(process.returncode if process.returncode is not None else 1)
+    return ProcessResult(
+        returncode=returncode,
+        output=output,
+        structured=structured,
+        capacity_failure=returncode != 0 and bool(CAPACITY_PATTERN.search(output)),
+    )
+
+def _process_record(pid: int) -> tuple[int, int, int] | None:
+    """Return one Linux process's parent, group, and start-time identity."""
+    try:
+        raw = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8")
+        fields = raw[raw.rfind(")") + 2 :].split()
+        return int(fields[1]), int(fields[2]), int(fields[19])
+    except (IndexError, OSError, ValueError):
+        return None
+
+def _descendant_processes(root_pid: int) -> dict[int, tuple[int, int]]:
+    """Snapshot descendants as pid -> (process group, start-time identity)."""
+    records: dict[int, tuple[int, int, int]] = {}
+    for entry in Path("/proc").iterdir():
+        if not entry.name.isdigit():
+            continue
+        pid = int(entry.name)
+        record = _process_record(pid)
+        if record is not None:
+            records[pid] = record
+    family = {root_pid}
+    changed = True
+    while changed:
+        changed = False
+        for pid, (parent, _, _) in records.items():
+            if pid not in family and parent in family:
+                family.add(pid)
+                changed = True
+    return {
+        pid: (records[pid][1], records[pid][2])
+        for pid in family
+        if pid != root_pid and pid in records
+    }
+
+def _process_identity_matches(pid: int, start_time: int) -> bool:
+    record = _process_record(pid)
+    return record is not None and record[2] == start_time
+
+def _signal_worker_tree(
+    root_pid: int,
+    descendants: dict[int, tuple[int, int]],
+    sig: signal.Signals,
+) -> None:
+    """Signal captured descendants across terminal-created process groups."""
+    groups = {root_pid}
+    for pid, (process_group, start_time) in descendants.items():
+        if _process_identity_matches(pid, start_time):
+            groups.add(process_group)
+    for process_group in groups:
+        try:
+            os.killpg(process_group, sig)
+        except ProcessLookupError:
+            pass
+    for pid, (_, start_time) in descendants.items():
+        if not _process_identity_matches(pid, start_time):
+            continue
+        try:
+            os.kill(pid, sig)
+        except ProcessLookupError:
+            pass
+
+def _terminate_worker_process(
+    process: subprocess.Popen[str],
+    known_descendants: dict[int, tuple[int, int]] | None = None,
+) -> None:
+    """Reap a worker and terminal descendants that created their own groups."""
+    descendants = dict(known_descendants or {})
+    descendants.update(_descendant_processes(process.pid))
+    if process.poll() is None:
+        _signal_worker_tree(process.pid, descendants, signal.SIGTERM)
+        try:
+            process.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            pass
+
+    # Provider terminal tools create their own sessions, so killing only the
+    # native CLI's process group leaves test/build subprocesses orphaned. The
+    # identity check prevents a reused PID from targeting an unrelated worker.
+    _signal_worker_tree(process.pid, descendants, signal.SIGKILL)
+    if process.poll() is None:
+        process.wait(timeout=10)
+
+def _base_env() -> dict[str, str]:
+    """Build a worker environment while preserving Vault-backed CLI homes."""
+    env = os.environ.copy()
+    env.update(
+        {
+            "HOME": str(DATA_ROOT / "home"),
+            "CODEX_HOME": os.environ.get("CODEX_HOME", "/runtime-access/codex"),
+            "CLAUDE_CONFIG_DIR": os.environ.get(
+                "CLAUDE_CONFIG_DIR", "/runtime-access/claude"
+            ),
+            "GIT_TERMINAL_PROMPT": "0",
+        }
+    )
+    env["PATH"] = f"{DATA_ROOT / 'tools/bin'}:/opt/coordinator:" + env.get("PATH", "")
+    return env
+
+def _codex_command(route: Route, prompt: str, workspace: Path, state: dict[str, Any], result_file: Path) -> list[str]:
+    base = [
+        str(CODEX_BIN),
+        "--dangerously-bypass-approvals-and-sandbox",
+        "--dangerously-bypass-hook-trust",
+        "exec",
+    ]
+    thread_id = str(state.get("codex_thread_id") or "")
+    common = [
+        "--json",
+        "-m",
+        route.model,
+        "-c",
+        f'model_reasoning_effort="{route.effort}"',
+        "--output-schema",
+        str(RESULT_SCHEMA_PATH),
+        "-o",
+        str(result_file),
+    ]
+    if thread_id:
+        return [*base, "resume", *common, thread_id, prompt]
+    return [
+        *base,
+        *common,
+        "-C",
+        str(workspace),
+        prompt,
+    ]
+
+def _claude_command(route: Route, prompt: str, state: dict[str, Any], resume: bool) -> list[str]:
+    session_id = str(state["claude_session_id"])
+    session = ["--resume", session_id] if resume else ["--session-id", session_id]
+    denied = [
+        "Bash(git push --force *)",
+        "Bash(git reset --hard *)",
+        "Bash(git clean -f *)",
+    ]
+    return [
+        str(CLAUDE_BIN),
+        "--dangerously-skip-permissions",
+        "--autocompact",
+        "auto",
+        "--settings",
+        str(CLAUDE_SETTINGS),
+        "--disallowedTools",
+        *denied,
+        "--model",
+        route.model,
+        "--effort",
+        route.effort,
+        "--output-format",
+        "stream-json",
+        "--verbose",
+        "--json-schema",
+        json.dumps(RESULT_SCHEMA, separators=(",", ":")),
+        *session,
+        "-p",
+        prompt,
+    ]
+
+def run_provider(
+    route: Route,
+    prompt: str,
+    workspace: Path,
+    state: dict[str, Any],
+    state_file: Path,
+    log_path: Path,
+    heartbeat: Callable[[str], bool],
+    max_runtime: int,
+) -> ProcessResult:
+    """Run or resume one provider session using its pinned structured CLI."""
+    state.setdefault("attempts", []).append({"route": asdict(route), "started_at": utc_now()})
+    if route.provider == "claude" and not state.get("claude_session_id"):
+        # The reservation is durable before Claude starts, closing the crash gap.
+        state["claude_session_id"] = str(uuid.uuid4())
+    state["current_route"] = asdict(route)
+    state["updated_at"] = utc_now()
+    env = _base_env()
+    if route.provider == "codex":
+        result_sequence = max(0, int(state.get("result_sequence", 0) or 0)) + 1
+        result_file = _result_path(state_file, state.get("run_id"), result_sequence)
+        while result_file.exists():
+            result_sequence += 1
+            result_file = _result_path(
+                state_file,
+                state.get("run_id"),
+                result_sequence,
+            )
+        result_file.parent.mkdir(parents=True, exist_ok=True)
+        result_file.touch(mode=0o600, exist_ok=False)
+        state["result_sequence"] = result_sequence
+        state["current_result_file"] = str(result_file)
+        atomic_json(state_file, state)
+        command = _codex_command(route, prompt, workspace, state, result_file)
+        result = stream_process(
+            command,
+            provider="codex",
+            cwd=workspace,
+            env=env,
+            log_path=log_path,
+            state=state,
+            state_file=state_file,
+            heartbeat=heartbeat,
+            max_runtime=max_runtime,
+        )
+        if (
+            state.get("codex_thread_id")
+            and result.returncode != 0
+            and NO_CODEX_THREAD in result.output.lower()
+        ):
+            state.pop("codex_thread_id", None)
+            result_sequence += 1
+            result_file = _result_path(
+                state_file,
+                state.get("run_id"),
+                result_sequence,
+            )
+            while result_file.exists():
+                result_sequence += 1
+                result_file = _result_path(
+                    state_file,
+                    state.get("run_id"),
+                    result_sequence,
+                )
+            result_file.touch(mode=0o600, exist_ok=False)
+            state["result_sequence"] = result_sequence
+            state["current_result_file"] = str(result_file)
+            atomic_json(state_file, state)
+            result = stream_process(
+                _codex_command(route, prompt, workspace, state, result_file),
+                provider="codex",
+                cwd=workspace,
+                env=env,
+                log_path=log_path,
+                state=state,
+                state_file=state_file,
+                heartbeat=heartbeat,
+                max_runtime=max_runtime,
+            )
+        file_result = load_json(result_file)
+        if file_result.get("status") in cli_lane_goal.RESULT_STATUSES:
+            result.structured = file_result
+            try:
+                result_file.chmod(0o600)
+            except OSError:
+                pass
+        return result
+
+    atomic_json(state_file, state)
+    resume = bool(state.get("claude_started"))
+    result = stream_process(
+        _claude_command(route, prompt, state, resume),
+        provider="claude",
+        cwd=workspace,
+        env=env,
+        log_path=log_path,
+        state=state,
+        state_file=state_file,
+        heartbeat=heartbeat,
+        max_runtime=max_runtime,
+    )
+    if resume and result.returncode == 1 and NO_CLAUDE_SESSION in result.output:
+        result = stream_process(
+            _claude_command(route, prompt, state, False),
+            provider="claude",
+            cwd=workspace,
+            env=env,
+            log_path=log_path,
+            state=state,
+            state_file=state_file,
+            heartbeat=heartbeat,
+            max_runtime=max_runtime,
+        )
+    elif not resume and result.returncode == 1 and CLAUDE_SESSION_COLLISION in result.output:
+        result = stream_process(
+            _claude_command(route, prompt, state, True),
+            provider="claude",
+            cwd=workspace,
+            env=env,
+            log_path=log_path,
+            state=state,
+            state_file=state_file,
+            heartbeat=heartbeat,
+            max_runtime=max_runtime,
+        )
+    if result.returncode == 0 or NO_CLAUDE_SESSION not in result.output:
+        state["claude_started"] = True
+        atomic_json(state_file, state)
+    return result

services/hermes/scripts/cli_lane_quarantine.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+"""Bounded diagnostic quarantine for malformed terminal journals."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import stat
+import sys
+from pathlib import Path
+
+from cli_lane_config import QUARANTINE_HASH_BYTES, STATE_ROOT, TerminalIdentity, TerminalRecoverySnapshot
+from cli_lane_evidence import _retire_terminal_entry
+from cli_lane_records import _read_bounded
+
+
+def _quarantine_terminal(
+    path: Path,
+    identity: TerminalIdentity | None,
+    reason: str,
+    *,
+    snapshot: TerminalRecoverySnapshot | None = None,
+) -> Path:
+    """Record bounded metadata, then retire only the inspected journal inode."""
+    board_dir = path.parent
+    try:
+        board_stat = board_dir.stat(follow_symlinks=False)
+    except OSError:
+        board_stat = None
+    if board_stat is None or not stat.S_ISDIR(board_stat.st_mode):
+        print(
+            f"refused terminal journal path through unsafe board directory: {path}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return path
+    board_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    board_flags |= getattr(os, "O_NOFOLLOW", 0)
+    file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    create_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    create_flags |= getattr(os, "O_NOFOLLOW", 0)
+    board_descriptor = None
+    quarantine_descriptor = None
+    source_descriptor = None
+    source_stat = None
+    destination = path
+    retirement = "not-attempted"
+    owns_source = snapshot is None
+    try:
+        relative = path.relative_to(STATE_ROOT)
+    except ValueError:
+        relative = Path()
+    already_quarantined = (
+        len(relative.parts) == 3 and relative.parts[1] == "quarantine"
+    )
+    try:
+        if snapshot is None:
+            board_descriptor = os.open(board_dir, board_flags)
+            source_stat = os.stat(
+                path.name,
+                dir_fd=board_descriptor,
+                follow_symlinks=False,
+            )
+        else:
+            board_descriptor = snapshot.directory_descriptor
+            source_descriptor = snapshot.descriptor
+            source_stat = snapshot.file_stat
+            pinned_board = os.fstat(board_descriptor)
+            if (
+                pinned_board.st_dev != board_stat.st_dev
+                or pinned_board.st_ino != board_stat.st_ino
+            ):
+                raise OSError("terminal recovery board identity changed")
+        safe_source = stat.S_ISREG(source_stat.st_mode) and source_stat.st_nlink == 1
+        digest = None
+        hashed_bytes = 0
+        hash_complete = False
+        if safe_source and snapshot is None:
+            source_descriptor = os.open(
+                path.name,
+                file_flags,
+                dir_fd=board_descriptor,
+            )
+            opened_stat = os.fstat(source_descriptor)
+            if (
+                opened_stat.st_dev != source_stat.st_dev
+                or opened_stat.st_ino != source_stat.st_ino
+                or opened_stat.st_nlink != 1
+            ):
+                raise OSError("terminal journal identity changed while opening")
+            # Keep this descriptor open through retirement. The staging entry
+            # must match the exact fstat-verified inode, not merely a path stat
+            # captured before a concurrent replacement.
+            source_stat = opened_stat
+            prefix = _read_bounded(source_descriptor, QUARANTINE_HASH_BYTES)
+            hashed_bytes = len(prefix)
+            hash_complete = opened_stat.st_size <= QUARANTINE_HASH_BYTES
+            digest = hashlib.sha256(prefix).hexdigest()
+        elif safe_source and source_descriptor is not None:
+            prefix = snapshot.prefix
+            hashed_bytes = len(prefix)
+            hash_complete = (
+                source_stat.st_size <= QUARANTINE_HASH_BYTES
+                and hashed_bytes == source_stat.st_size
+            )
+            digest = hashlib.sha256(prefix).hexdigest()
+        source_kind = (
+            "regular"
+            if safe_source
+            else "hardlink"
+            if stat.S_ISREG(source_stat.st_mode) and source_stat.st_nlink != 1
+            else "non-regular"
+        )
+        diagnostic = {
+            "hash_complete": hash_complete,
+            "hashed_bytes": hashed_bytes,
+            "original_name": path.name,
+            "reason": reason,
+            "sha256": digest,
+            "size": source_stat.st_size,
+            "source_kind": source_kind,
+        }
+        payload = (json.dumps(diagnostic, sort_keys=True) + "\n").encode("utf-8")
+        fingerprint = (
+            (digest or hashlib.sha256(path.name.encode("utf-8")).hexdigest())[:16]
+        )
+        safe_reason = (
+            re.sub(r"[^a-zA-Z0-9_.-]+", "-", reason).strip("-") or "invalid"
+        )
+        if already_quarantined:
+            # A staged artifact already lives in the board's one bounded
+            # quarantine directory. Reuse it so adversarial replacements can
+            # never drive recursive quarantine/quarantine/... authority.
+            quarantine_descriptor = os.dup(board_descriptor)
+            quarantine_dir = board_dir
+        else:
+            try:
+                os.mkdir("quarantine", 0o700, dir_fd=board_descriptor)
+            except FileExistsError:
+                pass
+            quarantine_descriptor = os.open(
+                "quarantine",
+                board_flags,
+                dir_fd=board_descriptor,
+            )
+            quarantine_dir = board_dir / "quarantine"
+        for sequence in range(32):
+            name = f"{path.name}.{safe_reason}.{fingerprint}.{sequence}.quarantine"
+            try:
+                destination_descriptor = os.open(
+                    name,
+                    create_flags,
+                    0o600,
+                    dir_fd=quarantine_descriptor,
+                )
+            except FileExistsError:
+                continue
+            try:
+                with os.fdopen(destination_descriptor, "wb") as target:
+                    target.write(payload)
+                    target.flush()
+                    os.fsync(target.fileno())
+                    target_stat = os.fstat(target.fileno())
+                    if (
+                        stat.S_IMODE(target_stat.st_mode) != 0o600
+                        or target_stat.st_nlink != 1
+                    ):
+                        raise OSError("quarantine destination is not private and singly linked")
+            except Exception:
+                os.unlink(name, dir_fd=quarantine_descriptor)
+                raise
+            destination = quarantine_dir / name
+            break
+        else:
+            raise OSError("could not reserve a unique quarantine destination")
+        os.fsync(quarantine_descriptor)
+        retirement = _retire_terminal_entry(
+            path,
+            source_stat,
+            board_descriptor=board_descriptor,
+            quarantine_descriptor=quarantine_descriptor,
+        )
+    except OSError as error:
+        # A bad quarantine target must not make an attacker-controlled pending
+        # path eligible for replay forever. The same identity-safe staging
+        # protocol works in the board directory when quarantine is unusable.
+        if board_descriptor is not None and source_stat is not None:
+            try:
+                retirement = _retire_terminal_entry(
+                    path,
+                    source_stat,
+                    board_descriptor=board_descriptor,
+                    quarantine_descriptor=(
+                        quarantine_descriptor
+                        if quarantine_descriptor is not None
+                        else board_descriptor
+                    ),
+                )
+            except OSError:
+                retirement = "deferred"
+        print(
+            f"terminal journal quarantine degraded safely: {type(error).__name__}: {error}",
+            file=sys.stderr,
+            flush=True,
+        )
+    finally:
+        if owns_source and source_descriptor is not None:
+            os.close(source_descriptor)
+        if quarantine_descriptor is not None:
+            os.close(quarantine_descriptor)
+        if owns_source and board_descriptor is not None:
+            os.close(board_descriptor)
+    print(
+        "quarantined terminal journal "
+        f"{path.name}: {reason}; identity={identity or 'unparseable'}; "
+        f"retirement={retirement}",
+        file=sys.stderr,
+        flush=True,
+    )
+    return destination

services/hermes/scripts/cli_lane_records.py

@@ -0,0 +1,356 @@
+#!/usr/bin/env python3
+"""Bounded inode-pinned reads and validation for Hermes terminal records."""
+
+from __future__ import annotations
+
+import json
+import os
+import stat
+from pathlib import Path
+from typing import Any
+
+import cli_lane_goal
+from cli_lane_config import (
+    MAX_TERMINAL_RECORD_BYTES,
+    QUARANTINE_HASH_BYTES,
+    RESULT_SCHEMA,
+    Route,
+    TerminalIdentity,
+    TerminalRecoverySnapshot,
+    TerminalSnapshot,
+    canonical_run_id,
+    utc_now,
+)
+from cli_lane_files import (
+    _candidate_path,
+    _terminal_identity,
+    _terminal_path,
+    atomic_json,
+)
+
+
+def _read_bounded(descriptor: int, limit: int) -> bytes:
+    """Read at most ``limit`` bytes from a regular file descriptor."""
+    chunks: list[bytes] = []
+    remaining = max(0, limit)
+    while remaining:
+        chunk = os.read(descriptor, min(64 * 1024, remaining))
+        if not chunk:
+            break
+        chunks.append(chunk)
+        remaining -= len(chunk)
+    return b"".join(chunks)
+
+def _open_small_json_snapshot(path: Path) -> TerminalSnapshot | None:
+    """Open and pin one bounded, singly-linked JSON document."""
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
+    file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    directory = None
+    descriptor = None
+    try:
+        directory = os.open(path.parent, directory_flags)
+        descriptor = os.open(path.name, file_flags, dir_fd=directory)
+        opened = os.fstat(descriptor)
+        if (
+            not stat.S_ISREG(opened.st_mode)
+            or opened.st_nlink != 1
+            or opened.st_size > MAX_TERMINAL_RECORD_BYTES
+        ):
+            return None
+        payload = _read_bounded(descriptor, MAX_TERMINAL_RECORD_BYTES + 1)
+        if len(payload) > MAX_TERMINAL_RECORD_BYTES:
+            return None
+        current = os.stat(
+            path.name,
+            dir_fd=directory,
+            follow_symlinks=False,
+        )
+        if (
+            current.st_dev != opened.st_dev
+            or current.st_ino != opened.st_ino
+            or current.st_size != opened.st_size
+            or len(payload) != opened.st_size
+        ):
+            return None
+        value = json.loads(payload.decode("utf-8"))
+        if not isinstance(value, dict):
+            return None
+        snapshot = TerminalSnapshot(value, opened, descriptor, directory)
+        descriptor = None
+        directory = None
+        return snapshot
+    except (OSError, UnicodeError, json.JSONDecodeError):
+        return None
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        if directory is not None:
+            os.close(directory)
+
+def _open_terminal_recovery_snapshot(path: Path) -> TerminalRecoverySnapshot | None:
+    """Pin the entry recovery inspected, even when its payload is malformed."""
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
+    file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    directory = None
+    descriptor = None
+    try:
+        directory = os.open(path.parent, directory_flags)
+        observed = os.stat(
+            path.name,
+            dir_fd=directory,
+            follow_symlinks=False,
+        )
+        if not stat.S_ISREG(observed.st_mode):
+            snapshot = TerminalRecoverySnapshot(
+                None,
+                observed,
+                None,
+                directory,
+                b"",
+                "non-regular",
+            )
+            directory = None
+            return snapshot
+        try:
+            descriptor = os.open(path.name, file_flags, dir_fd=directory)
+        except OSError:
+            snapshot = TerminalRecoverySnapshot(
+                None,
+                observed,
+                None,
+                directory,
+                b"",
+                "open-failed",
+            )
+            directory = None
+            return snapshot
+        opened = os.fstat(descriptor)
+        if opened.st_dev != observed.st_dev or opened.st_ino != observed.st_ino:
+            os.close(descriptor)
+            descriptor = None
+            snapshot = TerminalRecoverySnapshot(
+                None,
+                observed,
+                None,
+                directory,
+                b"",
+                "identity-changed-during-open",
+            )
+            directory = None
+            return snapshot
+        observed = opened
+        if observed.st_nlink != 1:
+            snapshot = TerminalRecoverySnapshot(
+                None,
+                observed,
+                descriptor,
+                directory,
+                b"",
+                "hardlinked",
+            )
+            descriptor = None
+            directory = None
+            return snapshot
+        if observed.st_size > MAX_TERMINAL_RECORD_BYTES:
+            prefix = _read_bounded(descriptor, QUARANTINE_HASH_BYTES)
+            snapshot = TerminalRecoverySnapshot(
+                None,
+                observed,
+                descriptor,
+                directory,
+                prefix,
+                "oversized",
+            )
+            descriptor = None
+            directory = None
+            return snapshot
+        payload = _read_bounded(descriptor, MAX_TERMINAL_RECORD_BYTES + 1)
+        after_read = os.fstat(descriptor)
+        unchanged = (
+            after_read.st_dev == observed.st_dev
+            and after_read.st_ino == observed.st_ino
+            and after_read.st_size == observed.st_size
+            and after_read.st_mtime_ns == observed.st_mtime_ns
+            and after_read.st_ctime_ns == observed.st_ctime_ns
+            and len(payload) == observed.st_size
+        )
+        document = None
+        invalid_reason = "unstable-payload"
+        if unchanged:
+            try:
+                parsed = json.loads(payload.decode("utf-8"))
+                if isinstance(parsed, dict):
+                    document = parsed
+                    invalid_reason = None
+                else:
+                    invalid_reason = "non-object-payload"
+            except (UnicodeError, json.JSONDecodeError):
+                invalid_reason = "malformed-payload"
+        snapshot = TerminalRecoverySnapshot(
+            document,
+            after_read,
+            descriptor,
+            directory,
+            payload[:QUARANTINE_HASH_BYTES],
+            invalid_reason,
+        )
+        descriptor = None
+        directory = None
+        return snapshot
+    except OSError:
+        return None
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        if directory is not None:
+            os.close(directory)
+
+def _open_terminal_snapshot(
+    path: Path,
+    identity: TerminalIdentity,
+) -> TerminalSnapshot | None:
+    """Open a terminal journal only after its lexical identity is verified."""
+    if _terminal_identity(path) != identity:
+        return None
+    return _open_small_json_snapshot(path)
+
+def _load_terminal_json(path: Path, identity: TerminalIdentity) -> dict[str, Any]:
+    """Read one small, singly-linked journal through its validated directory."""
+    snapshot = _open_terminal_snapshot(path, identity)
+    if snapshot is None:
+        return {}
+    try:
+        return snapshot.document
+    finally:
+        snapshot.close()
+
+def _load_small_json(path: Path) -> dict[str, Any]:
+    """Read one bounded immutable evidence document without following links."""
+    snapshot = _open_small_json_snapshot(path)
+    if snapshot is None:
+        return {}
+    try:
+        return snapshot.document
+    finally:
+        snapshot.close()
+
+def _persist_candidate(
+    state: dict[str, Any],
+    state_file: Path,
+    structured: dict[str, Any],
+    *,
+    route: Route,
+    returncode: int,
+    goal_turn: int,
+) -> Path:
+    """Persist every structured response before any judge can supersede it."""
+    sequence = max(0, int(state.get("candidate_sequence", 0) or 0)) + 1
+    path = _candidate_path(state_file, state.get("run_id"), sequence)
+    while path.exists():
+        sequence += 1
+        path = _candidate_path(state_file, state.get("run_id"), sequence)
+    atomic_json(
+        path,
+        {
+            "board": state.get("board"),
+            "task_id": state.get("task_id"),
+            "expected_run_id": state.get("run_id"),
+            "goal_turn": goal_turn,
+            "provider": route.provider,
+            "model": route.model,
+            "effort": route.effort,
+            "returncode": returncode,
+            "structured": structured,
+            "recorded_at": utc_now(),
+        },
+    )
+    state["candidate_sequence"] = sequence
+    state["last_candidate_file"] = str(path)
+    atomic_json(state_file, state)
+    return path
+
+def _write_terminal_record(
+    state_file: Path,
+    *,
+    board: str,
+    task_id: str,
+    run_id: Any,
+    structured: dict[str, Any],
+    summary: str,
+    metadata: dict[str, Any],
+) -> tuple[Path, dict[str, Any]]:
+    """Journal an accepted result before attempting the Kanban transaction."""
+    if type(run_id) is not int or canonical_run_id(run_id) is None:
+        raise ValueError("terminal run ID must be a positive SQLite int64")
+    path = _terminal_path(state_file, run_id)
+    record = {
+        "board": board,
+        "task_id": task_id,
+        "expected_run_id": run_id,
+        "result": json.dumps(structured, sort_keys=True),
+        "summary": summary,
+        "metadata": metadata,
+        "kanban_state": "pending",
+        "recorded_at": utc_now(),
+    }
+    atomic_json(path, record)
+    return path, record
+
+def _terminal_record_valid(
+    record: dict[str, Any],
+    identity: TerminalIdentity | None = None,
+) -> bool:
+    """Reject malformed or non-terminal replay journals without side effects."""
+    if not isinstance(record, dict):
+        return False
+    required_strings = ("board", "task_id", "result", "summary")
+    if not all(
+        isinstance(record.get(key), str) and record[key]
+        for key in required_strings
+    ):
+        return False
+    expected_run_id = record.get("expected_run_id")
+    if (
+        type(expected_run_id) is not int
+        or canonical_run_id(expected_run_id) is None
+    ):
+        return False
+    if not isinstance(record.get("metadata"), dict):
+        return False
+    try:
+        structured = json.loads(record["result"])
+    except (TypeError, json.JSONDecodeError):
+        return False
+    if not isinstance(structured, dict):
+        return False
+    required = set(RESULT_SCHEMA["required"])
+    if set(structured) != required:
+        return False
+    if (
+        type(structured["status"]) is not str
+        or structured["status"] not in cli_lane_goal.RESULT_STATUSES
+        or type(structured["summary"]) is not str
+        or not structured["summary"].strip()
+    ):
+        return False
+    for key in ("changed_files", "tests_run", "artifacts", "findings", "blockers"):
+        value = structured[key]
+        if type(value) is not list or any(type(item) is not str for item in value):
+            return False
+    valid = (
+        structured["status"] == "completed"
+        and structured["blockers"] == []
+        and cli_lane_goal.unfinished_result_reason(structured) is None
+        and record["summary"] == structured["summary"]
+        and record.get("kanban_state") in {"pending", "prepared", "committed"}
+    )
+    if not valid or identity is None:
+        return valid
+    return (
+        record.get("board") == identity.board
+        and record.get("task_id") == identity.task_id
+        and record.get("expected_run_id") == identity.run_id
+        and record.get("kanban_state") == identity.state
+    )

services/hermes/scripts/cli_lane_recovery.py

@@ -0,0 +1,460 @@
+#!/usr/bin/env python3
+"""Bounded replay of pending, prepared, and staged terminal journals."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+from cli_lane_board import _record_board_access_error
+from cli_lane_config import (
+    STATE_ROOT,
+    TerminalIdentity,
+    TerminalRecoverySnapshot,
+    canonical_run_id,
+)
+from cli_lane_evidence import (
+    _persist_conflict_evidence,
+    _persist_prepared_evidence,
+    _same_terminal_document,
+    _terminal_evidence_path,
+    _terminal_evidence_valid,
+    _write_json_noreplace,
+)
+from cli_lane_files import (
+    _LEGACY_RETIRE_NAME,
+    _RETIRE_NAME,
+    _SAFE_BOARD,
+    _terminal_evidence_identity,
+    _terminal_identity,
+    _terminal_path,
+    state_path,
+)
+from cli_lane_finalization import (
+    _finalize_document_db,
+    _finalize_terminal_record,
+    _promote_prepared_evidence,
+    _recover_exact_run,
+    _resolve_pending_after_winner,
+    _retire_snapshot,
+    _retire_snapshot_after_db,
+)
+from cli_lane_quarantine import _quarantine_terminal
+from cli_lane_records import (
+    _load_small_json,
+    _load_terminal_json,
+    _open_terminal_recovery_snapshot,
+    _terminal_record_valid,
+)
+
+
+def _terminal_entry_absent(path: Path) -> bool:
+    """Confirm absence through a nofollow directory descriptor."""
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
+    directory = None
+    try:
+        directory = os.open(path.parent, directory_flags)
+        os.stat(path.name, dir_fd=directory, follow_symlinks=False)
+        return False
+    except FileNotFoundError:
+        return True
+    except OSError:
+        return False
+    finally:
+        if directory is not None:
+            os.close(directory)
+
+def _staged_terminal_authority(
+    path: Path,
+    snapshot: TerminalRecoverySnapshot,
+) -> tuple[TerminalIdentity, Path] | None:
+    """Bind a staged inode to the canonical pending name it was moved from."""
+    try:
+        relative = path.relative_to(STATE_ROOT)
+    except ValueError:
+        return None
+    if len(relative.parts) == 2:
+        board, filename = relative.parts
+    elif len(relative.parts) == 3 and relative.parts[1] == "quarantine":
+        board, _quarantine, filename = relative.parts
+    else:
+        return None
+    match = _RETIRE_NAME.fullmatch(filename)
+    legacy = _LEGACY_RETIRE_NAME.fullmatch(filename)
+    document = snapshot.document
+    if (
+        (match is None and legacy is None)
+        or document is None
+        or not _SAFE_BOARD.fullmatch(board)
+        or board in {".", ".."}
+        or document.get("board") != board
+        or not isinstance(document.get("task_id"), str)
+        or type(document.get("expected_run_id")) is not int
+        or canonical_run_id(document.get("expected_run_id")) is None
+    ):
+        return None
+    state = document.get("kanban_state")
+    if state not in {"pending", "prepared", "committed", "conflict"}:
+        return None
+    identity = TerminalIdentity(
+        board, document["task_id"], document["expected_run_id"], state
+    )
+    if state in {"pending", "committed"}:
+        canonical = _terminal_path(
+            state_path(identity.board, identity.task_id),
+            identity.run_id,
+            state,
+        )
+        valid = _terminal_record_valid(document, identity)
+    else:
+        canonical = _terminal_evidence_path(identity, state, document)
+        valid = _terminal_evidence_valid(document, identity, state)
+    board_dir = STATE_ROOT / board
+    if (
+        canonical.parent != board_dir
+        or path.parent not in {board_dir, board_dir / "quarantine"}
+        or (
+            match is not None
+            and hashlib.sha256(canonical.name.encode("utf-8")).hexdigest()[:16]
+            != match.group("path_digest")
+        )
+        or not valid
+    ):
+        return None
+    return identity, canonical
+
+def _retirement_staging_paths() -> list[Path]:
+    """List every directory where identity-safe retirement can stage an inode."""
+    paths = list(STATE_ROOT.glob("*/.retire.*"))
+    paths.extend(STATE_ROOT.glob("*/quarantine/.retire.*"))
+    return sorted(set(paths))
+
+def _restore_staged_terminal_authority(
+    identity: TerminalIdentity,
+    canonical: Path,
+    document: dict[str, Any],
+) -> None:
+    """Restore one hidden accepted artifact without executing it prematurely."""
+    if identity.state == "pending":
+        _persist_prepared_evidence(identity, document)
+        return
+    if _write_json_noreplace(canonical, document):
+        return
+    existing = _load_small_json(canonical)
+    if identity.state in {"prepared", "conflict"}:
+        valid = _terminal_evidence_valid(existing, identity, identity.state)
+    else:
+        valid = _terminal_record_valid(existing, identity)
+    if valid and _same_terminal_document(existing, document):
+        return
+    if identity.state == "committed":
+        _persist_conflict_evidence(
+            identity,
+            document,
+            "staged committed evidence conflicts with the canonical first writer",
+        )
+        return
+    raise OSError("staged terminal evidence path contains a conflicting result")
+
+def _recover_retirement_staging() -> int:
+    """Promote valid hidden journals to durable prepared evidence."""
+    recovered = 0
+    for path in _retirement_staging_paths():
+        snapshot = _open_terminal_recovery_snapshot(path)
+        if snapshot is None:
+            continue
+        authority = _staged_terminal_authority(path, snapshot)
+        if authority is None:
+            try:
+                _quarantine_terminal(
+                    path,
+                    None,
+                    "untrusted-retirement-staging",
+                    snapshot=snapshot,
+                )
+            finally:
+                snapshot.close()
+            continue
+        identity, canonical = authority
+        try:
+            _restore_staged_terminal_authority(
+                identity,
+                canonical,
+                snapshot.document or {},
+            )
+            retirement = _retire_snapshot(
+                path,
+                snapshot,
+                authority_name=canonical.name,
+            )
+        except Exception as error:
+            _record_board_access_error(identity.board, error)
+            continue
+        finally:
+            snapshot.close()
+        if retirement in {"retired", "missing"}:
+            recovered += 1
+    return recovered
+
+def _drain_retirement_staging() -> int:
+    """Converge bounded replacement churn without spinning on storage failure."""
+    recovered = 0
+    previous: tuple[tuple[str, int, int, int, int], ...] | None = None
+    for _attempt in range(16):
+        surface = []
+        for path in _retirement_staging_paths():
+            try:
+                observed = path.stat(follow_symlinks=False)
+            except OSError:
+                continue
+            surface.append(
+                (
+                    str(path),
+                    observed.st_dev,
+                    observed.st_ino,
+                    observed.st_size,
+                    observed.st_mtime_ns,
+                )
+            )
+        signature = tuple(surface)
+        if not signature or signature == previous:
+            break
+        previous = signature
+        recovered += _recover_retirement_staging()
+    return recovered
+
+def _recover_prepared_finalizations(kanban_db: Any) -> int:
+    """Recover an accepted result staged before an interrupted DB boundary."""
+    recovered = 0
+    recovered_identities: set[tuple[str, str, int]] = set()
+    prepared_paths = list(STATE_ROOT.glob("*/*.terminal.prepared-*.json"))
+
+    def durable_order(path: Path) -> tuple[int, str]:
+        try:
+            return path.stat(follow_symlinks=False).st_mtime_ns, str(path)
+        except OSError:
+            return 2**63 - 1, str(path)
+
+    # When no DB winner exists yet, the first durably prepared result owns the
+    # run. The DB transaction remains the final arbiter across runner processes.
+    for path in sorted(prepared_paths, key=durable_order):
+        for _replacement_attempt in range(16):
+            identity = _terminal_evidence_identity(path)
+            snapshot = _open_terminal_recovery_snapshot(path)
+            if snapshot is None:
+                break
+            document = snapshot.document
+            if identity is None or identity.state != "prepared" or document is None:
+                try:
+                    _quarantine_terminal(
+                        path,
+                        identity,
+                        snapshot.invalid_reason or "malformed-prepared-evidence",
+                        snapshot=snapshot,
+                    )
+                finally:
+                    snapshot.close()
+                continue
+            expected = _terminal_evidence_path(identity, "prepared", document)
+            if (
+                expected.name != path.name
+                or not _terminal_evidence_valid(document, identity, "prepared")
+            ):
+                try:
+                    _quarantine_terminal(
+                        path,
+                        identity,
+                        "foreign-prepared-evidence",
+                        snapshot=snapshot,
+                    )
+                finally:
+                    snapshot.close()
+                continue
+            try:
+                outcome = _finalize_document_db(kanban_db, identity, document)
+            except Exception as error:
+                _record_board_access_error(identity.board, error)
+                snapshot.close()
+                break
+            try:
+                if outcome == "committed":
+                    _promote_prepared_evidence(
+                        identity,
+                        document,
+                        path,
+                        discard_prepared=False,
+                    )
+                    _retire_snapshot_after_db(path, snapshot)
+                    pending = _terminal_path(
+                        state_path(identity.board, identity.task_id),
+                        identity.run_id,
+                        "pending",
+                    )
+                    pending_identity = TerminalIdentity(
+                        identity.board,
+                        identity.task_id,
+                        identity.run_id,
+                        "pending",
+                    )
+                    _resolve_pending_after_winner(
+                        pending,
+                        pending_identity,
+                        document,
+                    )
+                    recovered_identity = (
+                        identity.board,
+                        identity.task_id,
+                        identity.run_id,
+                    )
+                    if recovered_identity not in recovered_identities:
+                        recovered_identities.add(recovered_identity)
+                        recovered += 1
+                elif outcome == "stale":
+                    _persist_conflict_evidence(
+                        identity,
+                        document,
+                        "prepared result no longer matches the authoritative task run",
+                    )
+                    _retire_snapshot_after_db(path, snapshot)
+                else:
+                    snapshot.close()
+                    break
+            except Exception as error:
+                _record_board_access_error(identity.board, error)
+                snapshot.close()
+                break
+            snapshot.close()
+    return recovered
+
+def recover_pending_finalizations() -> int:
+    """Replay accepted exact-run results before scheduling more provider work."""
+    from hermes_cli import kanban_db
+
+    _drain_retirement_staging()
+    recovered = _recover_prepared_finalizations(kanban_db)
+    for path in sorted(STATE_ROOT.glob("*/*.terminal.pending.json")):
+        identity = _terminal_identity(path)
+        last_invalid_reason = "malformed-name" if identity is None else None
+        retired_invalid = False
+        for _replacement_attempt in range(16):
+            snapshot = _open_terminal_recovery_snapshot(path)
+            if snapshot is None:
+                if (
+                    identity is not None
+                    and retired_invalid
+                    and _terminal_entry_absent(path)
+                ):
+                    _recover_exact_run(
+                        kanban_db,
+                        identity,
+                        last_invalid_reason or "malformed-payload",
+                    )
+                break
+            record = snapshot.document or {}
+            reason = None
+            if identity is None:
+                reason = "malformed-name"
+            elif not _terminal_record_valid(record):
+                reason = snapshot.invalid_reason or "malformed-payload"
+            elif not _terminal_record_valid(record, identity):
+                reason = "foreign-identity"
+            if reason is not None:
+                try:
+                    _quarantine_terminal(
+                        path,
+                        identity,
+                        reason,
+                        snapshot=snapshot,
+                    )
+                finally:
+                    snapshot.close()
+                retired_invalid = True
+                last_invalid_reason = reason
+                continue
+            assert identity is not None
+            try:
+                outcome = _finalize_terminal_record(
+                    kanban_db,
+                    path,
+                    record,
+                    snapshot=snapshot,
+                )
+            except Exception as error:
+                _record_board_access_error(identity.board, error)
+                break
+            if outcome == "committed":
+                recovered += 1
+                break
+            if outcome in {"invalid", "foreign"}:
+                # The pathname changed after the recovery snapshot closed.
+                # Reclassify and pin the new inode instead of quarantining it
+                # using authority derived from the older entry.
+                last_invalid_reason = outcome
+                continue
+            break
+        else:
+            print(
+                f"terminal journal replacement churn deferred for {path.name}",
+                file=sys.stderr,
+                flush=True,
+            )
+    # A quarantine or identity-safe retirement performed above can itself
+    # encounter replacement churn. Promote every resulting hidden generation
+    # back to first-class prepared/conflict/committed evidence before this
+    # recovery pass returns; DB replay, if still needed, occurs on the next
+    # bounded dispatcher pass.
+    _drain_retirement_staging()
+    return recovered
+
+def _has_pending_finalization(board: str, task_id: str, run_id: Any) -> bool:
+    """Keep an exact run claimed while its accepted result awaits replay."""
+    if type(run_id) is not int or canonical_run_id(run_id) is None:
+        return False
+    path = _terminal_path(state_path(board, task_id), run_id, "pending")
+    try:
+        path.stat()
+    except FileNotFoundError:
+        pass
+    except OSError:
+        return False
+    else:
+        identity = _terminal_identity(path)
+        if identity is not None:
+            record = _load_terminal_json(path, identity)
+            if _terminal_record_valid(record, identity):
+                return True
+    base = state_path(board, task_id)
+    for prepared in base.parent.glob(
+        f"{base.stem}.run-{run_id}.terminal.prepared-*.json"
+    ):
+        identity = _terminal_evidence_identity(prepared)
+        if identity is None or identity.state != "prepared":
+            continue
+        record = _load_small_json(prepared)
+        if _terminal_evidence_valid(record, identity, "prepared"):
+            return True
+    staged_paths = list(base.parent.glob(".retire.*"))
+    quarantine = base.parent / "quarantine"
+    if quarantine.is_dir() and not quarantine.is_symlink():
+        staged_paths.extend(quarantine.glob(".retire.*"))
+    for staged in staged_paths:
+        snapshot = _open_terminal_recovery_snapshot(staged)
+        if snapshot is None:
+            continue
+        try:
+            authority = _staged_terminal_authority(staged, snapshot)
+            if (
+                authority is not None
+                and authority[0].board == board
+                and authority[0].task_id == task_id
+                and authority[0].run_id == run_id
+                and authority[0].state in {"pending", "prepared", "committed"}
+            ):
+                return True
+        finally:
+            snapshot.close()
+    return False

services/hermes/scripts/cli_lane_retention.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+"""Bounded age, count, and byte retention for non-pending lane artifacts."""
+
+from __future__ import annotations
+
+import os
+import stat
+import sys
+import time
+from pathlib import Path
+
+from cli_lane_config import (
+    ARTIFACT_GC_INTERVAL_SECONDS,
+    ARTIFACT_RETENTION_AGE_SECONDS,
+    ARTIFACT_RETENTION_BYTES,
+    ARTIFACT_RETENTION_COUNT,
+    STATE_ROOT,
+)
+from cli_lane_evidence import _retire_terminal_entry, _terminal_evidence_valid
+from cli_lane_files import (
+    _fsync_directory,
+    _terminal_evidence_identity,
+    _terminal_identity,
+)
+from cli_lane_quarantine import _quarantine_terminal
+from cli_lane_recovery import _staged_terminal_authority
+from cli_lane_records import _open_terminal_recovery_snapshot, _terminal_record_valid
+
+
+def _artifact_gc_candidates(board_dir: Path) -> list[Path]:
+    """List only bounded artifact classes; pending journals are excluded."""
+    candidates: list[Path] = []
+    patterns = (
+        "*.provider-*.result.json",
+        "*.candidate-*.json",
+        "*.terminal.committed.json",
+        "*.terminal.conflict-*.json",
+        ".retire.*",
+    )
+    for pattern in patterns:
+        candidates.extend(board_dir.glob(pattern))
+    quarantine = board_dir / "quarantine"
+    if quarantine.is_dir():
+        candidates.extend(quarantine.glob("*.quarantine"))
+        candidates.extend(quarantine.glob(".retire.*"))
+    return candidates
+
+def _unlink_artifact_if_same(path: Path, observed: os.stat_result) -> bool:
+    """Remove only the exact artifact inode selected by retention."""
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_flags |= getattr(os, "O_NOFOLLOW", 0)
+    descriptor = None
+    try:
+        descriptor = os.open(path.parent, directory_flags)
+        outcome = _retire_terminal_entry(
+            path,
+            observed,
+            board_descriptor=descriptor,
+            quarantine_descriptor=descriptor,
+        )
+        return outcome == "retired"
+    except OSError:
+        return False
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+
+
+def _quarantine_invalid_retained_terminal(path: Path) -> tuple[bool, bool]:
+    """Audit retained committed/conflict evidence on the bounded GC cadence."""
+    committed = path.name.endswith(".terminal.committed.json")
+    conflict = ".terminal.conflict-" in path.name and path.name.endswith(".json")
+    if not committed and not conflict:
+        return False, False
+    identity = (
+        _terminal_identity(path) if committed else _terminal_evidence_identity(path)
+    )
+    snapshot = _open_terminal_recovery_snapshot(path)
+    if snapshot is None:
+        return True, False
+    document = snapshot.document
+    valid = False
+    if identity is not None and document is not None:
+        valid = (
+            _terminal_record_valid(document, identity)
+            if committed
+            else _terminal_evidence_valid(document, identity, "conflict")
+        )
+    if valid:
+        snapshot.close()
+        return False, False
+    try:
+        _quarantine_terminal(
+            path,
+            identity,
+            snapshot.invalid_reason or "invalid-retained-terminal-evidence",
+            snapshot=snapshot,
+        )
+    finally:
+        snapshot.close()
+    try:
+        path.stat(follow_symlinks=False)
+    except FileNotFoundError:
+        return True, True
+    except OSError:
+        return True, False
+    return True, False
+
+def gc_lane_artifacts(
+    *,
+    now: float | None = None,
+    max_age_seconds: int = ARTIFACT_RETENTION_AGE_SECONDS,
+    max_count: int = ARTIFACT_RETENTION_COUNT,
+    max_bytes: int = ARTIFACT_RETENTION_BYTES,
+) -> int:
+    """Bound non-pending lane artifacts by age, count, and bytes per board."""
+    current = time.time() if now is None else now
+    removed = 0
+    for board_dir in STATE_ROOT.iterdir() if STATE_ROOT.is_dir() else ():
+        if not board_dir.is_dir() or board_dir.is_symlink():
+            continue
+        board_removed = 0
+        entries: list[tuple[Path, os.stat_result]] = []
+        for path in _artifact_gc_candidates(board_dir):
+            classified, quarantined = _quarantine_invalid_retained_terminal(path)
+            if classified:
+                if quarantined:
+                    removed += 1
+                    board_removed += 1
+                continue
+            if path.name.startswith(".retire."):
+                snapshot = _open_terminal_recovery_snapshot(path)
+                if snapshot is not None:
+                    try:
+                        if _staged_terminal_authority(path, snapshot) is not None:
+                            # Accepted authority is first promoted/replayed by
+                            # recovery; retention never deletes it directly.
+                            continue
+                    finally:
+                        snapshot.close()
+            try:
+                file_stat = path.stat(follow_symlinks=False)
+            except OSError:
+                continue
+            if not stat.S_ISREG(file_stat.st_mode):
+                if _unlink_artifact_if_same(path, file_stat):
+                    removed += 1
+                    board_removed += 1
+                continue
+            if max_age_seconds >= 0 and current - file_stat.st_mtime > max_age_seconds:
+                if _unlink_artifact_if_same(path, file_stat):
+                    removed += 1
+                    board_removed += 1
+            else:
+                entries.append((path, file_stat))
+        retained_count = 0
+        retained_bytes = 0
+        for path, file_stat in sorted(
+            entries, key=lambda item: item[1].st_mtime, reverse=True
+        ):
+            exceeds_count = max_count >= 0 and retained_count >= max_count
+            exceeds_bytes = (
+                max_bytes >= 0 and retained_bytes + file_stat.st_size > max_bytes
+            )
+            if exceeds_count or exceeds_bytes:
+                if _unlink_artifact_if_same(path, file_stat):
+                    removed += 1
+                    board_removed += 1
+                continue
+            retained_count += 1
+            retained_bytes += file_stat.st_size
+        if board_removed:
+            _fsync_directory(board_dir)
+    return removed
+
+LAST_ARTIFACT_GC = 0.0
+
+def maybe_gc_lane_artifacts(*, now: float | None = None) -> int:
+    """Run artifact retention on a bounded cadence, not every dispatcher tick."""
+    global LAST_ARTIFACT_GC
+    current = time.time() if now is None else now
+    if current - LAST_ARTIFACT_GC < ARTIFACT_GC_INTERVAL_SECONDS:
+        return 0
+    LAST_ARTIFACT_GC = current
+    try:
+        return gc_lane_artifacts(now=current)
+    except OSError as error:
+        print(
+            f"lane artifact retention deferred: {type(error).__name__}: {error}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return 0

services/hermes/scripts/cli_lane_routing.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""Switchyard route selection and provider-health exclusion for CLI workers."""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+import urllib.error
+import urllib.request
+from typing import Any, Callable
+
+from cli_lane_config import (
+    EFFORTS,
+    PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS,
+    PROVIDER_HEALTH_MAX_AGE_SECONDS,
+    PROVIDER_HEALTH_PATHS,
+    SWITCHYARD_URL,
+    Route,
+)
+from cli_lane_files import load_json
+
+
+def parse_assignee(assignee: str) -> tuple[str | None, str | None]:
+    """Parse external lane overrides while leaving cli-auto fully automatic."""
+    value = str(assignee or "").strip().lower()
+    if value == "cli-auto":
+        return None, None
+    match = re.fullmatch(r"cli-(codex|claude)-(low|medium|high|xhigh)", value)
+    if not match:
+        raise ValueError(f"unsupported external lane assignee: {assignee}")
+    return match.group(1), match.group(2)
+
+def fresh_unavailable_provider(now: float | None = None) -> str | None:
+    """Return one recently proven-down provider for automatic route exclusion."""
+    current = time.time() if now is None else now
+    unavailable: list[str] = []
+    for provider, path in PROVIDER_HEALTH_PATHS.items():
+        health = load_json(path)
+        try:
+            age = current - path.stat().st_mtime
+        except OSError:
+            continue
+        max_age = (
+            PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS
+            if health.get("authenticated") is False
+            else PROVIDER_HEALTH_MAX_AGE_SECONDS
+        )
+        if (
+            0 <= age <= max_age
+            and health.get("state") == "unavailable"
+        ):
+            unavailable.append(provider)
+    # If both providers are down, keep the normal attempt/fallback path so the
+    # card records authoritative current errors instead of trusting snapshots.
+    return unavailable[0] if len(unavailable) == 1 else None
+
+def _decode_worker_target(value: str) -> tuple[str, str, str]:
+    """Decode the selected model header emitted by a worker decision route."""
+    parts = value.split("/", 3)
+    if len(parts) != 4 or parts[0] != "worker":
+        raise RuntimeError(f"invalid Switchyard worker target: {value}")
+    provider, model, effort = parts[1:]
+    if provider not in {"codex", "claude"} or effort not in EFFORTS:
+        raise RuntimeError(f"unsupported Switchyard worker target: {value}")
+    return provider, model, effort
+
+def select_route(
+    prompt: str,
+    assignee: str,
+    *,
+    exclude_provider: str | None = None,
+    exclude_reason: str | None = None,
+    switchyard_url: str = SWITCHYARD_URL,
+    open_request: Callable[..., Any] = urllib.request.urlopen,
+) -> Route:
+    """Ask Switchyard to select one native CLI worker at this boundary."""
+    started = time.monotonic()
+    manual_provider, manual_effort = parse_assignee(assignee)
+    if manual_provider and manual_effort:
+        route_id = f"atlas/worker/manual/{manual_provider}/{manual_effort}"
+        source = "switchyard-manual"
+    else:
+        route_id = "atlas/worker/auto/maximum"
+        source = "switchyard-classifier"
+    context = prompt
+    if exclude_provider:
+        reason = exclude_reason or "failed or exhausted capacity at this boundary"
+        context += (
+            f"\n\nRouting constraint: the {exclude_provider} provider {reason}. "
+            "Do not select it."
+        )
+    payload = json.dumps(
+        {
+            "model": route_id,
+            "messages": [{"role": "user", "content": context}],
+            "stream": False,
+            "max_tokens": 1,
+        }
+    ).encode("utf-8")
+    request = urllib.request.Request(
+        switchyard_url,
+        data=payload,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with open_request(request, timeout=60) as response:
+            selected = str(response.headers.get("x-model-router-selected-model") or "")
+            rationale = str(response.headers.get("x-model-router-rationale") or "")
+            response_body = response.read()
+    except (OSError, urllib.error.URLError) as exc:
+        raise RuntimeError(f"Switchyard worker routing failed: {exc}") from exc
+    provider, model, effort = _decode_worker_target(selected)
+    # Switchyard preserves the stable tier target in the selection header and
+    # top-level response model. The worker broker's assistant content contains
+    # the steward-resolved provider model required by the native CLI.
+    try:
+        response_document = json.loads(response_body)
+        resolved_target = str(response_document["choices"][0]["message"]["content"] or "")
+        resolved_provider, resolved_model, resolved_effort = _decode_worker_target(resolved_target)
+        required_prefix = "gpt-" if provider == "codex" else "claude-"
+        if (
+            resolved_provider == provider
+            and resolved_effort == effort
+            and resolved_model.startswith(required_prefix)
+        ):
+            model = resolved_model
+    except (AttributeError, IndexError, KeyError, RuntimeError, TypeError, ValueError, json.JSONDecodeError):
+        pass
+    if exclude_provider and provider == exclude_provider:
+        alternate = "claude" if provider == "codex" else "codex"
+        guarded = select_route(
+            prompt,
+            f"cli-{alternate}-{effort}",
+            switchyard_url=switchyard_url,
+            open_request=open_request,
+        )
+        return Route(
+            provider=guarded.provider,
+            model=guarded.model,
+            effort=guarded.effort,
+            profile=guarded.profile,
+            classifier=f"{source}-health-guard",
+            reason=(
+                f"Switchyard selected excluded {exclude_provider}; preserved "
+                f"its {effort} effort on healthy-provider route. {guarded.reason}"
+            ),
+            latency_ms=int((time.monotonic() - started) * 1000),
+            fallback_chain=(),
+        )
+    return Route(
+        provider=provider,
+        model=model,
+        effort=effort,
+        profile=f"{provider}-{effort}",
+        classifier=source,
+        reason=rationale or f"Switchyard selected {selected}",
+        latency_ms=int((time.monotonic() - started) * 1000),
+        fallback_chain=(),
+    )

services/hermes/scripts/recover_cassandra_workers.py

@@ -24,10 +24,12 @@ def recover_running_tasks(kanban_db: Any) -> list[str]:
                 if str(getattr(task, "status", "")) != "running":
                     continue
                 task_id = str(getattr(task, "id", ""))
-                if not task_id or not kanban_db.reclaim_task(
+                run_id = getattr(task, "current_run_id", None)
+                if not task_id or not isinstance(run_id, int) or not kanban_db.reclaim_task(
                     connection,
                     task_id,
                     reason=REASON,
+                    expected_run_id=run_id,
                 ):
                     continue
                 kanban_db.add_comment(

testing/tests/test_hermes_agent_access.py

@@ -0,0 +1,253 @@
+"""Owner and Switchyard access-boundary contracts."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    HERMES,
+    SCRIPTS,
+    _agent_deployment,
+    yaml,
+)
+
+
+def test_owner_agent_has_pinned_dedicated_node_ssh_access():
+    deployment = _agent_deployment()
+    annotations = deployment["spec"]["template"]["metadata"]["annotations"]
+    assert annotations[
+        "vault.hashicorp.com/agent-inject-secret-node-ssh-private-key"
+    ] == "kv/data/atlas/hermes/developer-ssh"
+    assert annotations[
+        "vault.hashicorp.com/agent-inject-secret-node-ssh-config"
+    ] == "kv/data/atlas/hermes/developer-ssh"
+    assert annotations[
+        "vault.hashicorp.com/agent-inject-secret-node-ssh-known-hosts"
+    ] == "kv/data/atlas/hermes/developer-ssh"
+
+    init = next(
+        item
+        for item in deployment["spec"]["template"]["spec"]["initContainers"]
+        if item["name"] == "init-config"
+    )
+    command = init["command"][2]
+    assert "ln -s /runtime-access/node-ssh-config /opt/data/home/.ssh/config" in command
+    assert (
+        "ln -s /runtime-access/node-ssh-known-hosts /opt/data/home/.ssh/known_hosts"
+        in command
+    )
+    assert "ln -s home/.ssh /opt/data/.ssh" in command
+    assert "chmod 0700 /opt/data/home/.ssh" in command
+    assert (
+        "ln -s /runtime-access/node-ssh-private-key "
+        "/opt/data/home/.ssh/id_ed25519_atlas_nodes"
+    ) in command
+
+    config = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())["data"]
+    assert "ssh_config" not in config
+    assert "ssh_known_hosts" not in config
+    assert "ssh-ed25519" not in (HERMES / "agent-configmap.yaml").read_text()
+
+    resources = yaml.safe_load((HERMES / "kustomization.yaml").read_text())[
+        "resources"
+    ]
+    assert "node-ssh-access.yaml" in resources
+    access = [
+        item
+        for item in yaml.safe_load_all((HERMES / "node-ssh-access.yaml").read_text())
+        if item
+    ]
+    service_account = next(item for item in access if item["kind"] == "ServiceAccount")
+    assert service_account["metadata"]["name"] == "hermes-node-ssh-access"
+    provider = next(item for item in access if item["kind"] == "SecretProviderClass")
+    assert provider["metadata"]["name"] == "hermes-node-ssh-access"
+    assert provider["spec"]["provider"] == "vault"
+    parameters = provider["spec"]["parameters"]
+    assert parameters["roleName"] == "hermes-node-ssh"
+    assert 'secretPath: "kv/data/atlas/hermes/developer-ssh"' in parameters["objects"]
+    assert 'secretKey: "public_key"' in parameters["objects"]
+    daemonset = next(item for item in access if item["kind"] == "DaemonSet")
+    pod = daemonset["spec"]["template"]["spec"]
+    assert pod["serviceAccountName"] == "hermes-node-ssh-access"
+    assert pod["automountServiceAccountToken"] is True
+    host_home = next(item for item in pod["volumes"] if item["name"] == "host-home")
+    assert host_home["hostPath"] == {"path": "/home", "type": "Directory"}
+    vault_secrets = next(
+        item for item in pod["volumes"] if item["name"] == "vault-secrets"
+    )
+    assert vault_secrets["csi"]["driver"] == "secrets-store.csi.k8s.io"
+    assert vault_secrets["csi"]["volumeAttributes"] == {
+        "secretProviderClass": "hermes-node-ssh-access"
+    }
+    reconciler = pod["containers"][0]["args"][0]
+    assert "cat /vault/secrets/node-ssh-public-key" in reconciler
+    assert "grep -qxF" in reconciler
+    assert "for user in atlas oceanus" in reconciler
+    assert "/host-etc/passwd" in reconciler
+    assert "chown \"${uid}:${gid}\"" in reconciler
+    host_passwd = next(
+        item for item in pod["volumes"] if item["name"] == "host-passwd"
+    )
+    assert host_passwd["hostPath"] == {"path": "/etc/passwd", "type": "File"}
+
+
+def test_owner_agent_tracks_no_ssh_identity_or_host_key_material():
+    """Vault references may be tracked; SSH identities and trust data may not."""
+    forbidden = (
+        "BEGIN OPENSSH PRIVATE KEY",
+        "ssh-ed25519 AAAA",
+        "ssh-rsa AAAA",
+        "IdentityFile ",
+        "UserKnownHostsFile ",
+        "StrictHostKeyChecking ",
+        "ssh_config:",
+        "ssh_known_hosts:",
+    )
+    text_suffixes = {
+        ".conf",
+        ".json",
+        ".md",
+        ".py",
+        ".sh",
+        ".toml",
+        ".yaml",
+        ".yml",
+    }
+    tracked = "\n".join(
+        path.read_text(encoding="utf-8")
+        for path in HERMES.rglob("*")
+        if path.is_file() and path.suffix in text_suffixes
+    )
+    for marker in forbidden:
+        assert marker not in tracked
+
+
+def test_switchyard_has_a_dedicated_non_owner_identity_and_read_only_catalog():
+    """Routing must not inherit the owner agent's cluster-admin capability."""
+    service_accounts = [
+        item
+        for item in yaml.safe_load_all(
+            (HERMES / "vault-serviceaccount.yaml").read_text()
+        )
+        if item
+    ]
+    assert any(
+        item["kind"] == "ServiceAccount"
+        and item["metadata"]["name"] == "hermes-switchyard"
+        for item in service_accounts
+    )
+
+    switchyard = yaml.safe_load(
+        (HERMES / "switchyard-deployment.yaml").read_text()
+    )
+    switchyard_pod = switchyard["spec"]["template"]["spec"]
+    assert switchyard_pod["serviceAccountName"] == "hermes-switchyard"
+    agent_pod = _agent_deployment()["spec"]["template"]["spec"]
+    for pod, container_name in (
+        (switchyard_pod, "worker-route-broker"),
+        (agent_pod, "claude-broker"),
+    ):
+        container = next(item for item in pod["containers"] if item["name"] == container_name)
+        catalog = next(
+            item
+            for item in container["volumeMounts"]
+            if item["mountPath"] == "/routing-catalog"
+        )
+        assert catalog["readOnly"] is True
+
+    rbac = [
+        item
+        for item in yaml.safe_load_all((HERMES / "agent-rbac.yaml").read_text())
+        if item
+    ]
+    binding = next(item for item in rbac if item["kind"] == "ClusterRoleBinding")
+    assert binding["subjects"] == [
+        {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"}
+    ]
+
+
+def test_switchyard_active_state_uses_a_relocatable_rwx_claim():
+    """A stale node attachment must not strand the routing authority."""
+    claims = [
+        item
+        for item in yaml.safe_load_all((HERMES / "switchyard-pvc.yaml").read_text())
+        if item
+    ]
+    active_claim = next(
+        item
+        for item in claims
+        if item["metadata"]["name"] == "hermes-switchyard-state-rwx"
+    )
+    assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"]
+
+    deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text())
+    strategy = deployment["spec"]["strategy"]
+    assert strategy == {
+        "type": "RollingUpdate",
+        "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0},
+    }
+    pod = deployment["spec"]["template"]["spec"]
+    state = next(item for item in pod["volumes"] if item["name"] == "state")
+    assert state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"][
+        "name"
+    ]
+
+
+def test_worker_route_broker_accepts_pod_network_health_checks():
+    """Kubelet probes the pod IP, so the broker cannot bind to loopback only."""
+    script = (SCRIPTS / "worker_route_broker.py").read_text()
+    assert 'ThreadingHTTPServer(("0.0.0.0", PORT), Handler)' in script
+
+
+def test_switchyard_network_boundary_allows_vault_bootstrap():
+    """The pre-populate init container must reach Vault before routing starts."""
+    documents = [
+        item
+        for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text())
+        if item
+    ]
+    isolation = next(
+        item
+        for item in documents
+        if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation"
+    )
+    assert any(
+        rule.get("to")
+        == [
+            {
+                "namespaceSelector": {
+                    "matchLabels": {"kubernetes.io/metadata.name": "vault"}
+                },
+                "podSelector": {"matchLabels": {"app": "vault"}},
+            }
+        ]
+        and rule.get("ports") == [{"protocol": "TCP", "port": 8200}]
+        for rule in isolation["spec"]["egress"]
+    )
+
+
+def test_switchyard_network_boundary_allows_metrics_scraping():
+    """VictoriaMetrics may scrape Switchyard without widening its API boundary."""
+    documents = [
+        item
+        for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text())
+        if item
+    ]
+    isolation = next(
+        item
+        for item in documents
+        if item.get("metadata", {}).get("name") == "hermes-switchyard-isolation"
+    )
+    assert any(
+        rule.get("from")
+        == [
+            {
+                "namespaceSelector": {
+                    "matchLabels": {
+                        "kubernetes.io/metadata.name": "monitoring"
+                    }
+                },
+                "podSelector": {"matchLabels": {"app": "server"}},
+            }
+        ]
+        and rule.get("ports") == [{"protocol": "TCP", "port": 9005}]
+        for rule in isolation["spec"]["ingress"]
+    )

testing/tests/test_hermes_agent_layout.py

@@ -0,0 +1,323 @@
+"""Agent deployment layout and runtime policy."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    HERMES,
+    Path,
+    _agent_deployment,
+    _services,
+    client_config,
+    json,
+    migration,
+    policy,
+    pytest,
+    yaml,
+)
+
+
+@pytest.mark.parametrize(
+    "command",
+    [
+        "git push --force origin main",
+        "git reset --hard HEAD~1",
+        "git clean -fd",
+    ],
+)
+def test_claude_pretool_hook_blocks_hard_denies(command: str):
+    assert policy.denial_reason(command)
+
+
+def test_claude_pretool_hook_allows_normal_engineering():
+    assert policy.denial_reason("pytest -q testing/tests") is None
+    assert policy.denial_reason("git push origin feature/hermes") is None
+    assert policy.denial_reason("kubectl delete pod -n cassandra stuck-worker") is None
+    assert policy.denial_reason("flux reconcile kustomization hermes") is None
+    assert policy.denial_reason("vault kv get kv/atlas/hermes") is None
+
+
+def test_claude_settings_preserve_state_and_install_three_guardrail_layers(tmp_path: Path):
+    state = tmp_path / ".claude.json"
+    settings = tmp_path / "settings.json"
+    state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8")
+    settings.write_text(
+        json.dumps(
+            {
+                "theme": "dark",
+                "permissions": {
+                    "deny": [
+                        "Bash(kubectl apply *)",
+                        "Bash(flux reconcile *)",
+                        "Bash(vault kv *)",
+                        "Bash(custom-owner-rule *)",
+                    ]
+                },
+            }
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+    client_config.configure_claude_state(state)
+    client_config.configure_claude_settings(settings)
+
+    state_value = json.loads(state.read_text())
+    settings_value = json.loads(settings.read_text())
+    assert state_value["promptQueueUseCount"] == 4
+    assert state_value["bypassPermissionsModeAccepted"] is True
+    assert settings_value["theme"] == "dark"
+    assert "Bash(git reset --hard *)" in settings_value["permissions"]["deny"]
+    assert "Bash(custom-owner-rule *)" in settings_value["permissions"]["deny"]
+    assert "Bash(kubectl apply *)" not in settings_value["permissions"]["deny"]
+    assert "Bash(flux reconcile *)" not in settings_value["permissions"]["deny"]
+    assert "Bash(vault kv *)" not in settings_value["permissions"]["deny"]
+    hook = settings_value["hooks"]["PreToolUse"][0]["hooks"][0]
+    assert "claude_command_policy.py" in hook["command"]
+
+
+def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path):
+    session = tmp_path / "home/.config/herdr/session.json"
+    session.parent.mkdir(parents=True)
+    session.write_text('{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8")
+    binary = tmp_path / "tools/bin/herdr"
+    binary.parent.mkdir(parents=True)
+    binary.write_text("legacy", encoding="utf-8")
+    (tmp_path / "home/.claude").mkdir()
+    (tmp_path / "home/.codex").mkdir()
+
+    archive = migration.archive_legacy_state(tmp_path)
+
+    value = json.loads(archive.read_text())
+    assert value["legacy_session"]["agents"][0]["session_id"] == "abc"
+    assert (tmp_path / "home/.claude").is_dir()
+    assert (tmp_path / "home/.codex").is_dir()
+    assert not binary.exists()
+    assert not (tmp_path / "home/.config/herdr").exists()
+
+
+def test_agent_uses_one_native_kanban_control_plane():
+    configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())
+    config = yaml.safe_load(configmap["data"]["config.yaml"])
+    assert config["model"] == {
+        "provider": "atlas-switchyard",
+        "default": "atlas/auto/balanced",
+        "model": "atlas/auto/balanced",
+    }
+    assert config["kanban"]["dispatch_in_gateway"] is True
+    assert config["kanban"]["default_assignee"] == "cli-auto"
+    assert config["plugins"]["enabled"] == ["auto-router"]
+
+    deployment = _agent_deployment()
+    pod = deployment["spec"]["template"]["spec"]
+    assert pod["enableServiceLinks"] is False
+    names = {item["name"] for item in pod["containers"]}
+    assert "cli-lane-runner" in names
+    assert "terminal" in names
+    assert not any("herdr" in name for name in names)
+    rendered = (HERMES / "agent-deployment.yaml").read_text()
+    assert "herdr server" not in rendered
+    assert "herdr-dispatch" not in rendered
+
+
+def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth():
+    deployment = _agent_deployment()
+    containers = {
+        item["name"]: item
+        for item in deployment["spec"]["template"]["spec"]["containers"]
+    }
+    lane = containers["cli-lane-runner"]
+    environment = {item["name"]: item["value"] for item in lane["env"]}
+
+    assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2"
+    assert lane["resources"] == {
+        "requests": {"cpu": "100m", "memory": "256Mi"},
+        "limits": {"cpu": "2", "memory": "6Gi"},
+    }
+
+
+def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity():
+    """Placement correction: keep the agent off nodes that cannot hold it.
+
+    titan-04 is cordoned after repeated kernel undervoltage and kubelet
+    failure, and titan-19 was probe/Longhorn unstable under worker load, so
+    both must join the existing hard exclusions. That leaves titan-05 as the
+    healthy candidate, which is tight enough on requested CPU that the main
+    container has to give back 50m to schedule there.
+    """
+    pod = _agent_deployment()["spec"]["template"]["spec"]
+    hostnames = next(
+        item
+        for item in pod["affinity"]["nodeAffinity"][
+            "requiredDuringSchedulingIgnoredDuringExecution"
+        ]["nodeSelectorTerms"][0]["matchExpressions"]
+        if item["key"] == "kubernetes.io/hostname"
+    )
+
+    assert hostnames["operator"] == "NotIn"
+    assert set(hostnames["values"]) >= {"titan-04", "titan-19"}
+
+    hermes = next(
+        item for item in pod["containers"] if item["name"] == "hermes"
+    )
+    assert hermes["resources"]["requests"]["cpu"] == "300m"
+
+
+def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path():
+    deployment = _agent_deployment()
+    pod = deployment["spec"]["template"]["spec"]
+    containers = {item["name"]: item for item in pod["containers"]}
+    assert "webui" not in containers
+
+    assert "dashboard" not in containers
+    hermes = containers["hermes"]
+    hermes_env = {item["name"]: item["value"] for item in hermes["env"]}
+    assert hermes_env["HERMES_STREAM_STALE_TIMEOUT"] == "600"
+    assert hermes_env["HERMES_API_CALL_STALE_TIMEOUT"] == "600"
+    assert hermes["command"] == ["/bin/sh", "-ec"]
+    startup = hermes["args"][0]
+    assert ". /opt/data/.env" in startup
+    assert "exec /init /opt/hermes/docker/main-wrapper.sh gateway run" in startup
+    hermes_env = {item["name"]: item["value"] for item in hermes["env"]}
+    assert hermes_env["HERMES_DASHBOARD"] == "1"
+    assert hermes_env["HERMES_DASHBOARD_HOST"] == "127.0.0.1"
+    assert hermes_env["HERMES_DASHBOARD_PORT"] == "9119"
+    assert hermes_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180"
+    assert hermes["securityContext"]["runAsUser"] == 0
+    assert hermes["securityContext"]["runAsGroup"] == 0
+    for probe_name in ("startupProbe", "readinessProbe", "livenessProbe"):
+        probe = hermes[probe_name]
+        assert probe["exec"]["command"] == [
+            "curl",
+            "-fsS",
+            "http://127.0.0.1:9119/api/status",
+        ]
+
+    terminal = containers["terminal"]
+    command = terminal["args"][0]
+    assert "--base-path /terminal" in command
+    assert "--check-origin" not in command
+    assert "/usr/bin/tmux new-session -A" in command
+    assert "--continue" in command
+    assert "--yolo" in command
+    terminal_env = {item["name"]: item["value"] for item in terminal["env"]}
+    assert terminal_env["HERMES_TUI_AGENT_INIT_TIMEOUT_S"] == "180"
+
+    claude_broker = containers["claude-broker"]
+    claude_env = {item["name"]: item["value"] for item in claude_broker["env"]}
+    assert claude_env["HERMES_CLAUDE_BROKER_CONCURRENCY"] == "2"
+    assert claude_broker["readinessProbe"]["tcpSocket"] == {"port": "claude-broker"}
+    assert claude_broker["livenessProbe"]["tcpSocket"] == {"port": "claude-broker"}
+
+    args = containers["oauth2-proxy"]["args"]
+    terminal_upstream = "--upstream=http://127.0.0.1:7681/terminal/"
+    dashboard_upstream = "--upstream=http://127.0.0.1:9119/"
+    assert terminal_upstream in args
+    assert dashboard_upstream in args
+    assert args.index(terminal_upstream) < args.index(dashboard_upstream)
+    assert "--pass-host-header=false" in args
+    assert "--cookie-refresh=19m" in args
+    assert "--session-store-type=redis" in args
+    assert any(
+        arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.")
+        for arg in args
+    )
+
+    patch_init = next(
+        item for item in pod["initContainers"]
+        if item["name"] == "patch-tui-gateway"
+    )
+    assert patch_init["command"][-1] == "/patched/server.py"
+    for name in ("hermes", "terminal"):
+        mounts = containers[name]["volumeMounts"]
+        assert {
+            "name": "tui-gateway-patch",
+            "mountPath": "/opt/hermes/tui_gateway/server.py",
+            "subPath": "server.py",
+        } in mounts
+
+    ingress_documents = [
+        item
+        for item in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text())
+        if item
+    ]
+    middlewares = {
+        item["metadata"]["name"]: item
+        for item in ingress_documents
+        if item["kind"] == "Middleware"
+    }
+    assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][
+        "replacement"
+    ].endswith("/terminal/")
+    assert middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][
+        "customRequestHeaders"
+    ]["Origin"] == "http://127.0.0.1:9119"
+    ingresses = {
+        item["metadata"]["name"]: item
+        for item in ingress_documents
+        if item["kind"] == "Ingress"
+    }
+    assert ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][
+        "traefik.ingress.kubernetes.io/router.middlewares"
+    ] == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd"
+    assert ingresses["hermes-agent-terminal"]["metadata"]["annotations"][
+        "traefik.ingress.kubernetes.io/router.middlewares"
+    ] == "hermes-hermes-agent-terminal-slash@kubernetescrd"
+
+
+def test_broker_services_survive_sibling_container_readiness_loss():
+    services = _services()
+
+    for name in (
+        "hermes-image-broker",
+        "hermes-codex-broker",
+        "hermes-local-image",
+        "hermes-claude-broker",
+    ):
+        assert services[name]["spec"]["publishNotReadyAddresses"] is True
+
+
+def test_cli_lane_domain_modules_are_all_mounted_with_the_runner():
+    manifest = yaml.safe_load((HERMES / "kustomization.yaml").read_text())
+    coordinator = next(
+        item
+        for item in manifest["configMapGenerator"]
+        if item["name"] == "hermes-coordinator"
+    )
+    mounted = {entry.split("=", 1)[0] for entry in coordinator["files"]}
+    expected = {
+        f"cli_lane_{domain}.py"
+        for domain in (
+            "board",
+            "config",
+            "dispatch",
+            "evidence",
+            "execution",
+            "files",
+            "finalization",
+            "goal",
+            "prompt",
+            "provider",
+            "quarantine",
+            "records",
+            "recovery",
+            "retention",
+            "routing",
+            "runner",
+        )
+    }
+    assert expected <= mounted
+
+
+def test_agent_dashboard_reconnects_all_transient_websockets():
+    dockerfile = (
+        HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent"
+    ).read_text(encoding="utf-8")
+    assert "eventsRetryAttempt.current" in dockerfile
+    assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile
+    assert "events feed rejected (${ev.code}) — reload the page" in dockerfile
+    assert 'url = await api.buildWsUrl("/api/pty", params);' in dockerfile
+    assert 'url = await buildWsUrl("/api/events", { channel });' in dockerfile
+    assert dockerfile.count("'        await api.getSessions(1, 0,") == 2
+    assert ".then(() => gw.connect())" in dockerfile
+    assert 'api.getSessions(1, 0, profile ?? "")' in dockerfile
+    assert "dashboard token rotated by a server restart" in dockerfile

testing/tests/test_hermes_agent_runtime_patches.py

@@ -0,0 +1,329 @@
+"""Pinned operator tooling and runtime patch contracts."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    HERMES,
+    Path,
+    SCRIPTS,
+    _agent_deployment,
+    auth_patch,
+    client_config,
+    codex_runtime_patch,
+    json,
+    pytest,
+    ttyd_patch,
+    tui_gateway_patch,
+    yaml,
+)
+
+
+def test_owner_agent_installs_the_pinned_operator_toolchain():
+    script = (SCRIPTS / "install_agent_tools.sh").read_text()
+    for value in [
+        "flux",
+        "helm",
+        "kustomize",
+        "jq",
+        "yq",
+        "gh",
+        "vault",
+        "sops",
+        "age",
+        "age-keygen",
+        "k9s",
+        "terraform",
+        "go",
+        "gofmt",
+    ]:
+        assert value in script
+    assert "go1.26.5.linux-arm64.tar.gz" in script
+    assert (
+        "fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49"
+        in script
+    )
+    assert script.count("sha256sum -c -") == 1
+
+    deployment = _agent_deployment()
+    installer = next(
+        item
+        for item in deployment["spec"]["template"]["spec"]["initContainers"]
+        if item["name"] == "install-agent-tools"
+    )
+    assert "/bin/sh /opt/coordinator/install_agent_tools.sh" in installer["command"][2]
+    assert any(mount["name"] == "coordinator" for mount in installer["volumeMounts"])
+    init_config = next(
+        item
+        for item in deployment["spec"]["template"]["spec"]["initContainers"]
+        if item["name"] == "init-config"
+    )
+    init_command = init_config["command"][2]
+    assert "# Hermes managed operator PATH." in init_command
+    assert "/opt/coordinator:/opt/data/tools/bin:/opt/hermes/.venv/bin" in init_command
+    assert 'chmod 0644 "${profile_file}"' in init_command
+
+
+def test_owner_agent_uses_only_the_canonical_hostname():
+    paths = [
+        HERMES / "agent-configmap.yaml",
+        HERMES / "agent-deployment.yaml",
+        HERMES / "agent-ingress.yaml",
+        Path(__file__).parents[2] / "scripts/ops/hermes_triage_monitor.py",
+    ]
+    for path in paths:
+        content = path.read_text()
+        assert "agent.bstein.dev" not in content
+        assert "agent.hermes.bstein.dev" in content
+
+
+def test_agent_reconnect_retains_complete_history_and_long_tool_budget():
+    configmap = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())
+    config = yaml.safe_load(configmap["data"]["config.yaml"])
+    display = config["display"]
+    assert display["resume_exchanges"] >= 10000
+    assert display["resume_max_user_chars"] >= 10000000
+    assert display["resume_max_assistant_chars"] >= 10000000
+    assert config["agent"]["max_turns"] == 180
+    assert config["delegation"]["max_iterations"] == 120
+
+
+def test_auth_patch_honors_explicit_shared_store(tmp_path: Path):
+    source = tmp_path / "auth.py"
+    destination = tmp_path / "patched/auth.py"
+    source.write_text(
+        'from pathlib import Path\nimport os\n\ndef _auth_file_path() -> Path:\n    path = get_hermes_home() / "auth.json"\n    return path\n',
+        encoding="utf-8",
+    )
+    auth_patch.patch(source, destination)
+    content = destination.read_text()
+    assert 'os.environ.get("HERMES_AUTH_FILE"' in content
+
+
+def test_auth_patch_fails_closed_on_upstream_drift(tmp_path: Path):
+    source = tmp_path / "auth.py"
+    source.write_text("def changed():\n    pass\n", encoding="utf-8")
+    with pytest.raises(RuntimeError, match="context changed"):
+        auth_patch.patch(source, tmp_path / "patched.py")
+
+
+def test_codex_runtime_patch_uses_cli_and_forwards_route(tmp_path: Path):
+    provider = tmp_path / "runtime_provider.py"
+    provider.write_text(codex_runtime_patch.PROVIDER_BEFORE, encoding="utf-8")
+    provider_out = tmp_path / "patched/runtime_provider.py"
+    codex_runtime_patch.patch_provider(provider, provider_out)
+    assert '"api_mode": "codex_app_server"' in provider_out.read_text()
+
+    session = tmp_path / "codex_app_server_session.py"
+    session.write_text(
+        codex_runtime_patch.SESSION_SIGNATURE_BEFORE
+        + codex_runtime_patch.SESSION_REQUEST_BEFORE,
+        encoding="utf-8",
+    )
+    session_out = tmp_path / "patched/codex_app_server_session.py"
+    codex_runtime_patch.patch_session(session, session_out)
+    session_content = session_out.read_text()
+    assert 'turn_params["model"] = model' in session_content
+    assert 'turn_params["effort"] = effort' in session_content
+    assert '"approvalPolicy": "never"' in session_content
+    assert '"sandboxPolicy": {"type": "dangerFullAccess"}' in session_content
+
+    turn = tmp_path / "codex_runtime.py"
+    turn.write_text(
+        codex_runtime_patch.FALLBACK_CONTEXT_BEFORE
+        + codex_runtime_patch.TURN_BEFORE,
+        encoding="utf-8",
+    )
+    turn_out = tmp_path / "patched/codex_runtime.py"
+    codex_runtime_patch.patch_turn(turn, turn_out)
+    turn_content = turn_out.read_text()
+    assert "model=str(getattr(agent" in turn_content
+    assert "build_cross_provider_codex_prompt" in turn_content
+
+    fallback = tmp_path / "chat_completion_helpers.py"
+    fallback.write_text(
+        codex_runtime_patch.FALLBACK_RESOLUTION_BEFORE,
+        encoding="utf-8",
+    )
+    fallback_out = tmp_path / "patched/chat_completion_helpers.py"
+    codex_runtime_patch.patch_fallback(fallback, fallback_out)
+    fallback_content = fallback_out.read_text()
+    assert 'agent.api_mode = "codex_app_server"' in fallback_content
+    assert "agent._codex_cross_provider_fallback = True" in fallback_content
+
+    loop = tmp_path / "conversation_loop.py"
+    loop.write_text(
+        codex_runtime_patch.FALLBACK_DISPATCH_BEFORE
+        + codex_runtime_patch.RETRY_FALLBACK_DISPATCH_BEFORE
+        + codex_runtime_patch.STREAM_RECOVERY_BEFORE,
+        encoding="utf-8",
+    )
+    loop_out = tmp_path / "patched/conversation_loop.py"
+    codex_runtime_patch.patch_loop(loop, loop_out)
+    loop_content = loop_out.read_text()
+    assert loop_content.count('if agent.api_mode == "codex_app_server"') == 2
+    assert "build_cross_provider_codex_prompt" in loop_content
+    retry_dispatch = loop_content.index(
+        "Fallback activation happens inside this retry loop"
+    )
+    api_kwargs = loop_content.find("agent._build_api_kwargs", retry_dispatch)
+    assert api_kwargs == -1 or retry_dispatch < api_kwargs
+    assert "Provider stream ended before a complete response" in loop_content
+    assert "_is_transport_stub" in loop_content
+    assert "rerouting ({truncated_tool_call_retries}/4)" in loop_content
+
+    auxiliary = tmp_path / "auxiliary_client.py"
+    auxiliary.write_text(
+        codex_runtime_patch.AUXILIARY_TOKEN_BEFORE,
+        encoding="utf-8",
+    )
+    auxiliary_out = tmp_path / "patched/auxiliary_client.py"
+    codex_runtime_patch.patch_auxiliary(auxiliary, auxiliary_out)
+    auxiliary_content = auxiliary_out.read_text()
+    assert 'os.environ.get("CODEX_HOME"' in auxiliary_content
+    assert 'Path(codex_home).expanduser() / "auth.json"' in auxiliary_content
+    assert "never creates a metered API-key lane" in auxiliary_content
+
+
+def test_agent_mounts_codex_auxiliary_runtime_patch():
+    deployment = _agent_deployment()
+    pod = deployment["spec"]["template"]["spec"]
+    patch_init = next(
+        item for item in pod["initContainers"]
+        if item["name"] == "patch-codex-runtime"
+    )
+    assert patch_init["command"][-2:] == [
+        "/opt/hermes/agent/auxiliary_client.py",
+        "/patched/auxiliary_client.py",
+    ]
+    expected_mount = {
+        "name": "codex-runtime-patch",
+        "mountPath": "/opt/hermes/agent/auxiliary_client.py",
+        "subPath": "auxiliary_client.py",
+    }
+    containers = {item["name"]: item for item in pod["containers"]}
+    for name in ("hermes", "terminal"):
+        assert expected_mount in containers[name]["volumeMounts"]
+
+
+def test_codex_auxiliary_patch_reads_cli_token_without_copying_it(
+    tmp_path: Path,
+    monkeypatch,
+):
+    codex_home = tmp_path / ".codex"
+    codex_home.mkdir()
+    (codex_home / "auth.json").write_text(
+        json.dumps({"tokens": {"access_token": "cli-access-token"}}),
+        encoding="utf-8",
+    )
+    monkeypatch.setenv("CODEX_HOME", str(codex_home))
+
+    source = tmp_path / "auxiliary_client.py"
+    source.write_text(
+        "import json, logging, os, time\n"
+        "from pathlib import Path\n"
+        "logger = logging.getLogger(__name__)\n"
+        "def read_token():\n"
+        "    try:\n"
+        "        raise RuntimeError('Hermes provider store intentionally empty')\n"
+        + codex_runtime_patch.AUXILIARY_TOKEN_BEFORE,
+        encoding="utf-8",
+    )
+    destination = tmp_path / "patched/auxiliary_client.py"
+    codex_runtime_patch.patch_auxiliary(source, destination)
+    namespace: dict = {}
+    exec(compile(destination.read_text(), str(destination), "exec"), namespace)
+
+    assert namespace["read_token"]() == "cli-access-token"
+
+
+def test_codex_runtime_patch_fails_closed_on_upstream_drift(tmp_path: Path):
+    source = tmp_path / "runtime_provider.py"
+    source.write_text("def changed():\n    pass\n", encoding="utf-8")
+    with pytest.raises(RuntimeError, match="context changed"):
+        codex_runtime_patch.patch_provider(source, tmp_path / "patched.py")
+
+
+def test_codex_runtime_migration_uses_owner_unsafe_mode(tmp_path: Path):
+    config = tmp_path / "config.yaml"
+    config.write_text("model: {}\n", encoding="utf-8")
+    codex_home = tmp_path / ".codex"
+    calls = []
+
+    class Report:
+        errors = []
+
+        @staticmethod
+        def summary():
+            return "configured"
+
+    def migrate(value, **kwargs):
+        calls.append((value, kwargs))
+        return Report()
+
+    client_config.configure_codex_runtime(config, migrate, codex_home)
+
+    assert calls[0][1]["default_permission_profile"] is None
+    assert calls[0][1]["codex_home"] == codex_home
+    content = (codex_home / "config.toml").read_text(encoding="utf-8")
+    assert 'approval_policy = "never"' in content
+    assert 'sandbox_mode = "danger-full-access"' in content
+    assert "default_permissions" not in content
+
+
+def test_codex_owner_permissions_replace_stale_profile(tmp_path: Path):
+    config = tmp_path / "config.toml"
+    config.write_text(
+        'default_permissions = ":danger-no-sandbox"\n\n[features]\nhooks = true\n',
+        encoding="utf-8",
+    )
+
+    client_config.configure_codex_owner_permissions(config)
+    client_config.configure_codex_owner_permissions(config)
+
+    content = config.read_text(encoding="utf-8")
+    assert content.count(client_config.OWNER_PERMISSIONS_BEGIN) == 1
+    assert content.count('approval_policy = "never"') == 1
+    assert content.count('sandbox_mode = "danger-full-access"') == 1
+    assert "default_permissions" not in content
+    assert "[features]\nhooks = true" in content
+
+
+def test_tui_gateway_patch_extends_and_bounds_agent_startup(tmp_path: Path):
+    source = tmp_path / "server.py"
+    destination = tmp_path / "patched/server.py"
+    source.write_text(
+        "import os\n\n" + tui_gateway_patch.BEFORE + "\ndef unchanged():\n    pass\n",
+        encoding="utf-8",
+    )
+
+    tui_gateway_patch.patch(source, destination)
+
+    content = destination.read_text(encoding="utf-8")
+    assert "HERMES_TUI_AGENT_INIT_TIMEOUT_S" in content
+    assert 'configured = 180.0' in content
+    assert "return max(30.0, min(configured, 900.0))" in content
+    assert "timeout: float | None = None" in content
+    assert "ready.wait(timeout=wait_timeout)" in content
+    assert "def unchanged():" in content
+
+
+def test_tui_gateway_patch_fails_closed_on_upstream_drift(tmp_path: Path):
+    source = tmp_path / "server.py"
+    source.write_text("def changed():\n    pass\n", encoding="utf-8")
+    with pytest.raises(RuntimeError, match="context changed"):
+        tui_gateway_patch.patch(source, tmp_path / "patched.py")
+
+
+def test_ttyd_clipboard_and_reconnect_patch_remain_enabled():
+    source = '<html><body><script>document.execCommand("copy")</script></body></html>'
+    content = ttyd_patch.patch_html(source)
+    assert 'id="atlas-ttyd-clipboard"' in content
+    assert "navigator.clipboard.writeText(text)" in content
+    assert "class AtlasRecoveringWebSocket" in content
+    assert "window.location.reload()" in content
+    assert "event.stopImmediatePropagation()" in content
+
+
+def test_ttyd_patch_fails_closed_on_upstream_drift():
+    with pytest.raises(RuntimeError, match="context changed"):
+        ttyd_patch.patch_html("<html><body>changed</body></html>")

testing/tests/test_hermes_agent_security.py

@@ -0,0 +1,202 @@
+"""Agent authentication, networking, and Flux health contracts."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    FLUX_HERMES,
+    HERMES,
+    KEYCLOAK,
+    _agent_deployment,
+    yaml,
+)
+
+
+def test_agent_image_runs_execution_safety_patch_and_regressions():
+    dockerfiles = HERMES.parents[1] / "dockerfiles"
+    dockerfile = (dockerfiles / "Dockerfile.hermes-agent").read_text(encoding="utf-8")
+    dockerignore = (dockerfiles / "Dockerfile.hermes-agent.dockerignore").read_text(
+        encoding="utf-8"
+    )
+
+    for name in (
+        "patch-hermes-execution-safety.py",
+        "hermes-execution-safety-regression.py",
+        "hermes_execution_patch_support.py",
+        "patch_hermes_run_safety.py",
+        "patch_hermes_decomposition_safety.py",
+        "hermes_execution_regression_support.py",
+        "hermes_run_safety_regression.py",
+        "hermes_decomposition_safety_regression.py",
+    ):
+        assert f"COPY dockerfiles/{name}" in dockerfile
+        assert f"!dockerfiles/{name}" in dockerignore
+    assert "/opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py" in (
+        dockerfile
+    )
+    assert "/opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py" in (
+        dockerfile
+    )
+    assert "COPY services/hermes/scripts/cli_lane_*.py" in dockerfile
+    assert "!services/hermes/scripts/cli_lane_*.py" in dockerignore
+    assert "HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression" in dockerfile
+
+
+def test_agent_refreshes_routes_after_restoring_cli_logins():
+    deployment = _agent_deployment()
+    init_containers = {
+        item["name"]: item
+        for item in deployment["spec"]["template"]["spec"]["initContainers"]
+    }
+    configure = init_containers["configure-agent-clients"]
+    command = configure["command"][-1]
+    assert "configure_agent_clients.py" in command
+    assert command.index("configure_agent_clients.py") < command.index(
+        "hermes_coordinator.py --once"
+    )
+    env = {item["name"]: item["value"] for item in configure["env"]}
+    assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json"
+    assert env["PYTHONPATH"] == "/opt/hermes"
+    for name in ("bootstrap-coordinator", "configure-agent-clients"):
+        route_env = {
+            item["name"]: item["value"]
+            for item in init_containers[name]["env"]
+        }
+        assert route_env["CODEX_HOME"] == "/runtime-access/codex"
+        assert route_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude"
+        assert "/opt/data/tools/bin" in route_env["PATH"]
+
+    assert "patch-web-session-activity" in init_containers
+    web_patch = init_containers["patch-web-session-activity"]
+    assert "/opt/coordinator/patch_web_session_activity.py" in web_patch["command"]
+
+    containers = {
+        item["name"]: item
+        for item in deployment["spec"]["template"]["spec"]["containers"]
+    }
+    steward_env = {
+        item["name"]: item["value"]
+        for item in containers["model-steward"]["env"]
+    }
+    assert steward_env["CODEX_HOME"] == "/runtime-access/codex"
+    assert steward_env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude"
+    assert "/opt/data/tools/bin" in steward_env["PATH"]
+    hermes_mounts = {
+        (item["name"], item["mountPath"], item.get("subPath"))
+        for item in containers["hermes"]["volumeMounts"]
+    }
+    assert (
+        "web-server-patch",
+        "/opt/hermes/hermes_cli/web_server.py",
+        "web_server.py",
+    ) in hermes_mounts
+
+
+def test_flux_health_checks_follow_the_owner_oauth_sidecar():
+    flux = yaml.safe_load(FLUX_HERMES.read_text())
+    checks = {
+        (item["kind"], item["name"])
+        for item in flux["spec"]["healthChecks"]
+    }
+    assert ("Deployment", "hermes-agent") in checks
+    assert ("DaemonSet", "hermes-node-ssh-access") in checks
+    assert ("Deployment", "oauth2-proxy-hermes-agent") not in checks
+
+
+def test_agent_auth_is_bstein_group_and_email_bounded():
+    deployment = _agent_deployment()
+    oauth = next(
+        item for item in deployment["spec"]["template"]["spec"]["containers"]
+        if item["name"] == "oauth2-proxy"
+    )
+    args = oauth["args"]
+    assert "--user-id-claim=sub" in args
+    assert "--oidc-groups-claim=groups" in args
+    assert "--allowed-group=/hermes-owner" in args
+    assert "--authenticated-emails-file=/etc/oauth2-proxy/allowed-emails" in args
+    script = (KEYCLOAK / "scripts/hermes_access_oidc_ensure.sh").read_text()
+    assert 'group_name="hermes-owner"' in script
+    assert "username=bstein&exact=true" in script
+    assert '"full.path":"true"' in script
+
+
+def test_agent_network_boundary_allows_only_authenticated_and_metrics_surfaces():
+    documents = [
+        item
+        for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text())
+        if item
+    ]
+    isolation = next(item for item in documents if item.get("metadata", {}).get("name") == "hermes-agent-isolation")
+    assert isolation["spec"]["ingress"] == [
+        {
+            "from": [
+                {
+                    "namespaceSelector": {
+                        "matchLabels": {
+                            "kubernetes.io/metadata.name": "traefik"
+                        }
+                    },
+                    "podSelector": {
+                        "matchLabels": {"app.kubernetes.io/name": "traefik"}
+                    },
+                }
+            ],
+            "ports": [{"protocol": "TCP", "port": 4180}],
+        },
+        {
+            "from": [
+                {
+                    "podSelector": {
+                        "matchLabels": {"app": "hermes-chat-tenant"}
+                    }
+                }
+            ],
+            "ports": [
+                {"protocol": "TCP", "port": 9002},
+                {"protocol": "TCP", "port": 9003},
+            ],
+        },
+        {
+            "from": [
+                {
+                    "podSelector": {
+                        "matchLabels": {"app": "hermes-switchyard"}
+                    }
+                }
+            ],
+            "ports": [
+                {"protocol": "TCP", "port": 9003},
+                {"protocol": "TCP", "port": 9006},
+            ],
+        },
+        {
+            "from": [
+                {
+                    "namespaceSelector": {
+                        "matchLabels": {
+                            "kubernetes.io/metadata.name": "monitoring"
+                        }
+                    },
+                    "podSelector": {"matchLabels": {"app": "server"}},
+                }
+            ],
+            "ports": [{"protocol": "TCP", "port": 9010}],
+        },
+    ]
+    assert isolation["spec"]["egress"] == [{}]
+
+
+def test_owner_agent_has_cluster_admin_kubernetes_context():
+    config = yaml.safe_load((HERMES / "agent-kubeconfig.yaml").read_text())
+    assert config["current-context"] == "atlas-owner"
+    assert config["contexts"][0]["context"]["namespace"] == "default"
+    rbac_path = HERMES / "agent-rbac.yaml"
+    documents = [item for item in yaml.safe_load_all(rbac_path.read_text()) if item]
+    binding = next(item for item in documents if item["kind"] == "ClusterRoleBinding")
+    assert binding["roleRef"] == {
+        "apiGroup": "rbac.authorization.k8s.io",
+        "kind": "ClusterRole",
+        "name": "cluster-admin",
+    }
+    assert binding["subjects"] == [
+        {"kind": "ServiceAccount", "name": "hermes-agent", "namespace": "hermes"}
+    ]

testing/tests/test_hermes_chat_codex_broker.py

@@ -0,0 +1,75 @@
+"""Hermes chat codex broker contracts."""
+
+from __future__ import annotations
+
+from test_hermes_chat_support import (
+    ModuleType,
+    Path,
+    _load_broker_module,
+    base64,
+    json,
+    sys,
+    time,
+)
+
+
+def test_codex_broker_refreshes_and_persists_first_party_oauth(
+    tmp_path: Path, monkeypatch
+):
+    """Expired ChatGPT OAuth refreshes in the canonical Codex CLI store."""
+    module = _load_broker_module(
+        "hermes_codex_refresh_broker", "codex_broker.py", monkeypatch
+    )
+    auth_dir = tmp_path / ".codex"
+    auth_dir.mkdir()
+
+    def jwt(expires_at: float) -> str:
+        payload = base64.urlsafe_b64encode(
+            json.dumps({"exp": expires_at}).encode()
+        ).decode().rstrip("=")
+        return f"header.{payload}.signature"
+
+    expired = jwt(time.time() - 60)
+    live = jwt(time.time() + 3600)
+    auth_path = auth_dir / "auth.json"
+    auth_path.write_text(
+        json.dumps(
+            {
+                "auth_mode": "chatgpt",
+                "tokens": {
+                    "access_token": expired,
+                    "refresh_token": "refresh-old",
+                },
+            }
+        )
+    )
+    calls = []
+    auth_module = ModuleType("hermes_cli.auth")
+
+    def refresh(access_token, refresh_token, *, timeout_seconds):
+        calls.append((access_token, refresh_token, timeout_seconds))
+        return {
+            "access_token": live,
+            "refresh_token": "refresh-new",
+            "last_refresh": "2026-08-12T20:00:00Z",
+        }
+
+    auth_module.refresh_codex_oauth_pure = refresh
+    package = ModuleType("hermes_cli")
+    package.auth = auth_module
+    monkeypatch.setitem(sys.modules, "hermes_cli", package)
+    monkeypatch.setitem(sys.modules, "hermes_cli.auth", auth_module)
+    monkeypatch.setenv("CODEX_HOME", str(auth_dir))
+
+    assert module._access_token() == live
+    persisted = json.loads(auth_path.read_text())
+    assert persisted["tokens"]["access_token"] == live
+    assert persisted["tokens"]["refresh_token"] == "refresh-new"
+    assert persisted["last_refresh"] == "2026-08-12T20:00:00Z"
+    assert calls == [(expired, "refresh-old", 30.0)]
+    assert auth_path.stat().st_mode & 0o777 == 0o600
+
+    # A healthy token is reused, so repeated routed turns do not spend a
+    # refresh token or create a second billing/authentication path.
+    assert module._access_token() == live
+    assert len(calls) == 1

testing/tests/test_hermes_chat_config.py

@@ -0,0 +1,273 @@
+"""Hermes chat config contracts."""
+
+from __future__ import annotations
+
+from test_hermes_chat_support import (
+    HERMES,
+    ROOT,
+    _documents,
+    yaml,
+)
+
+
+def test_chat_config_enables_real_research_compute_and_delegation():
+    configmap = _documents(HERMES / "chat-configmap.yaml")[0]
+    config = yaml.safe_load(configmap["data"]["config.yaml"])
+
+    # AUTO leaves effort unset so the target chosen by Switchyard owns it.
+    assert "reasoning_effort" not in config["agent"]
+    assert config["model"]["model"] == "atlas/auto/fast"
+    assert config["web"] == {
+        "backend": "ddgs",
+        "search_backend": "ddgs",
+        "extract_backend": "public-extract",
+    }
+    assert config["delegation"]["max_concurrent_children"] == 2
+    assert config["delegation"]["max_iterations"] == 80
+    assert config["agent"]["max_turns"] == 120
+    assert config["tool_loop_guardrails"]["hard_stop_enabled"] is True
+    for platform in ("cli", "api_server"):
+        toolsets = config["platform_toolsets"][platform]
+        assert "delegation" in toolsets
+        assert "browser" in toolsets
+        assert "python_sandbox" in toolsets
+        assert "vision" in toolsets
+        assert "web" in toolsets
+        assert "image_gen" in toolsets
+        assert "terminal" not in toolsets
+        assert "code_execution" not in toolsets
+
+def test_agent_config_keeps_delegated_reviewers_from_owning_task_lifecycle():
+    configmap = _documents(HERMES / "agent-configmap.yaml")[0]
+    instructions = configmap["data"]["AGENTS.md"]
+    soul = configmap["data"]["SOUL.md"]
+    config = yaml.safe_load(configmap["data"]["config.yaml"])
+
+    assert (
+        "Only the foreground durable worker owns its Kanban task lifecycle"
+        in instructions
+    )
+    assert "Never unblock and then complete" in instructions
+    assert "gateway dispatcher can claim the transient" in instructions
+    for platform in ("cli", "api_server"):
+        assert "kanban" in config["platform_toolsets"][platform]
+    assert "must never complete, block, unblock, reclaim" in instructions
+    assert "task's final structured result itself" in instructions
+    assert "Atlas organization has private visibility" in instructions
+    assert "may be public or private" in instructions
+    assert "do not infer a\nrepository's visibility" in instructions
+    assert "already supplied through `GIT_ASKPASS`" in instructions
+    assert "Never call `kanban_show` without a known, non-empty task ID" in instructions
+    assert "bounded ad-hoc inspection and acceptance checks may" in instructions
+    assert "load implementation or TDD skills" in instructions
+    assert "Never call `kanban_show` without\na known, non-empty task ID" in soul
+    assert "must load a skill only when its workflow\nmaterially applies" in soul
+    assert "runtime-only `GIT_ASKPASS`" in soul
+    assert "`scm.bstein.dev` is Gitea, not GitHub" in instructions
+    assert "Never load or follow a GitHub/`gh`" in instructions
+    assert "/opt/coordinator/gitea_api.py METHOD /api/v1/..." in instructions
+    assert "JENKINS_BASE_URL" in instructions
+    assert "Do not\nuse `git reset --hard`" in instructions
+    rendered = (HERMES / "agent-deployment.yaml").read_text()
+    assert 'git config --global user.name "Hermes Agent"' in rendered
+    assert 'git config --global user.email "hermes@bstein.dev"' in rendered
+
+def test_agent_image_completes_parked_kanban_tasks_atomically():
+    dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-agent").read_text()
+
+    assert (
+        "AND status IN ('running', 'ready', 'blocked', 'scheduled')"
+        in dockerfile
+    )
+    assert (
+        "Atomically mark running, ready, blocked, or scheduled tasks done"
+        in dockerfile
+    )
+    assert "kind IN ('created', 'blocked', 'unblocked')" in dockerfile
+    assert "payload.get(\"status\") == \"blocked\"" in dockerfile
+    assert "hermes-kanban-blocked-regression.py" in dockerfile
+
+def test_sandbox_shares_only_the_tenant_workspace_without_credentials():
+    sandbox_docs = _documents(HERMES / "chat-sandbox.yaml")
+    deployments = [doc for doc in sandbox_docs if doc["kind"] == "Deployment"]
+    assert len(deployments) == 8
+    for ordinal, deployment in enumerate(deployments):
+        pod_spec = deployment["spec"]["template"]["spec"]
+        container = pod_spec["containers"][0]
+        assert pod_spec["automountServiceAccountToken"] is False
+        assert container["securityContext"]["readOnlyRootFilesystem"] is True
+        assert container["securityContext"]["runAsNonRoot"] is True
+        assert container["securityContext"]["runAsGroup"] == 10000
+        assert not container.get("env")
+        assert {mount["mountPath"] for mount in container["volumeMounts"]} == {
+            "/tmp",
+            "/workspace",
+            "/opt/data/workspace",
+        }
+        workspace_volume = next(
+            item for item in pod_spec["volumes"] if item["name"] == "workspace"
+        )
+        assert workspace_volume["persistentVolumeClaim"]["claimName"] == (
+            f"workspace-hermes-chat-tenant-{ordinal}"
+        )
+
+    statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
+    templates = statefulset["spec"]["volumeClaimTemplates"]
+    workspace = next(item for item in templates if item["metadata"]["name"] == "workspace")
+    assert workspace["spec"]["resources"]["requests"]["storage"] == "10Gi"
+    assert workspace["spec"]["accessModes"] == ["ReadWriteMany"]
+
+    pod_spec = statefulset["spec"]["template"]["spec"]
+    hermes = next(item for item in pod_spec["containers"] if item["name"] == "hermes")
+    startup = hermes["args"][0]
+    assert "hermes-chat-sandbox-${ordinal}.hermes-chat-sandbox" in startup
+    assert any(
+        mount["name"] == "workspace" and mount["mountPath"] == "/opt/data/workspace"
+        for mount in hermes["volumeMounts"]
+    )
+
+    policies = _documents(HERMES / "networkpolicy.yaml")
+    deny = next(
+        item
+        for item in policies
+        if item["metadata"]["name"] == "hermes-chat-sandbox-deny"
+    )
+    assert deny["spec"]["ingress"] == []
+    assert deny["spec"]["egress"] == []
+    for ordinal in range(4):
+        policy = next(
+            item
+            for item in policies
+            if item["metadata"]["name"] == f"hermes-chat-sandbox-tenant-{ordinal}"
+        )
+        assert policy["spec"]["podSelector"]["matchLabels"][
+            "ai.bstein.dev/tenant-ordinal"
+        ] == str(ordinal)
+        source = policy["spec"]["ingress"][0]["from"][0]["podSelector"][
+            "matchLabels"
+        ]
+        assert source["statefulset.kubernetes.io/pod-name"] == (
+            f"hermes-chat-tenant-{ordinal}"
+        )
+
+def test_gateway_image_honors_ui_model_and_caps_reasoning():
+    dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-agent").read_text()
+    assert (
+        "FROM nousresearch/hermes-agent@sha256:"
+        "47d4bd4cc420b70e40ed75efdade373e45b86b7382d4013a054208982bb6ba08"
+    ) in dockerfile
+    assert "_resolve_request_route" in dockerfile
+    assert 'allowed_providers = {"atlas-switchyard"}' in dockerfile
+    assert 'reasoning_effort=body.get("reasoning_effort")' in dockerfile
+    assert 'reasoning_config = {"enabled": True, "effort": "xhigh"}' in dockerfile
+    assert "ddgs==9.14.4" in dockerfile
+    assert "specific not in _LEGACY_WEB_BACKENDS" in dockerfile
+    assert '"pre_internal_route"' in dockerfile
+    assert "pre_internal_route hook failed" in dockerfile
+    assert '"pre_subagent_route"' in dockerfile
+    assert "pre_subagent_route hook failed" in dockerfile
+    assert "agent._hermes_explicit_model_pick = bool" in dockerfile
+    assert "agent._hermes_oneshot = True" in dockerfile
+    assert 'getattr(parent_agent, "_hermes_oneshot", False)' in dockerfile
+    assert 'var != "CLAUDE_CODE_OAUTH_TOKEN"' in dockerfile
+    assert '"source": "managed_claude_code_subscription"' in dockerfile
+    assert '"name": "Claude Code subscription (Switchyard)"' in dockerfile
+    assert "Anthropic API / direct OAuth (not used by Switchyard)" in dockerfile
+    assert "scheduleTerminalPaint" in dockerfile
+    assert "sessionTree.childrenByParent" in dockerfile
+    assert "SessionActivityPanel" in dockerfile
+    assert "include_children=${includeChildren}" in dockerfile
+    assert "_dashboard_session_is_active" in dockerfile
+    assert 'searchParams.get("lineage_root") || resumeParam' in dockerfile
+    assert "followActiveLineage" in dockerfile
+    assert 'next.delete("lineage_root")' in dockerfile
+    activity = (
+        ROOT / "dockerfiles" / "hermes-session-activity-panel.tsx"
+    ).read_text()
+    assert "Live worker activity" in activity
+    assert "api.getSessionMessages" in activity
+    assert "aria-live=\"polite\"" in activity
+    assert "role=\"log\"" in activity
+    assert "aria-busy={running && messages.length === 0}" in activity
+    assert 'detail.source !== "api_server"' not in activity
+    assert "Poll-backed transcript; terminal rendering is not required." in activity
+    assert "DEFAULT_VISIBLE_MESSAGES = 250" in activity
+    assert "current + DEFAULT_VISIBLE_MESSAGES" in activity
+    assert 'Show up to{" "}' in activity
+    assert "Math.min(" in activity
+    assert "visibleMessages.map" in activity
+    assert "response.total_messages" in activity
+    assert "visibleLimit)," in activity
+    assert "function displayText(value: unknown)" in activity
+    assert "open && (" in activity
+    assert 'document.getElementById("hermes-resume-bootstrap")?.remove()' in activity
+    assert 'id="hermes-resume-bootstrap"' in dockerfile
+    assert 'role="status"' in dockerfile
+    assert 'aria-busy="true"' in dockerfile
+    assert "Opening Hermes worker activity…" in dockerfile
+    assert 'params.has("resume")' in dockerfile
+    assert "Open Telegram setup" in dockerfile
+
+def test_telegram_router_local_build_defaults_to_cluster_architecture():
+    dockerfile = (
+        ROOT / "dockerfiles" / "Dockerfile.hermes-chat-router"
+    ).read_text(encoding="utf-8")
+
+    assert "ARG TARGETOS=linux" in dockerfile
+    assert "ARG TARGETARCH=arm64" in dockerfile
+    assert "GOOS=${TARGETOS} GOARCH=${TARGETARCH}" in dockerfile
+
+def test_chat_oauth_allows_stale_service_worker_retirement():
+    documents = _documents(HERMES / "oauth2-proxy.yaml")
+    deployment = next(
+        document
+        for document in documents
+        if document["kind"] == "Deployment"
+        and document["metadata"]["name"] == "oauth2-proxy-hermes-chat"
+    )
+    args = deployment["spec"]["template"]["spec"]["containers"][0]["args"]
+
+    assert "--skip-auth-route=GET=^/sw[.]js([?].*)?$" in args
+    assert "--custom-templates-dir=/etc/oauth2-proxy/templates" in args
+
+    template = (HERMES / "oauth2-proxy-templates" / "error.html").read_text()
+    assert 'http-equiv="refresh"' not in template
+    assert "Unable to find a valid CSRF token" in template
+    assert "expired or was already used" in template
+    assert "sessionStorage" in template
+    assert "Automatic recovery paused" in template
+    assert "/start?rd=/" in template
+    container = deployment["spec"]["template"]["spec"]["containers"][0]
+    assert "v7.15.3@sha256:10a1165743a192e" in container["image"]
+    assert "--cookie-csrf-per-request=true" in args
+    assert "--cookie-csrf-per-request-limit=8" in args
+    assert "--trusted-proxy-ip=10.42.0.0/16" in args
+    assert "--api-route=^/api/" in args
+    assert "--api-route=^/health$" in args
+    assert "--cookie-expire=168h" in args
+    assert "--cookie-refresh=19m" in args
+    assert "--session-store-type=redis" in args
+    assert any(
+        arg.startswith("--redis-connection-url=redis://hermes-oauth-sessions.")
+        for arg in args
+    )
+
+    agent = _documents(HERMES / "agent-deployment.yaml")[0]
+    agent_pod = agent["spec"]["template"]["spec"]
+    agent_oauth = next(
+        container for container in agent_pod["containers"]
+        if container["name"] == "oauth2-proxy"
+    )
+    assert "--custom-templates-dir=/etc/oauth2-proxy/templates" in agent_oauth["args"]
+    assert {
+        "name": "oauth-templates",
+        "mountPath": "/etc/oauth2-proxy/templates",
+        "readOnly": True,
+    } in agent_oauth["volumeMounts"]
+    agent_template_volume = next(
+        volume for volume in agent_pod["volumes"]
+        if volume["name"] == "oauth-templates"
+    )
+    assert agent_template_volume["configMap"]["name"] == (
+        "hermes-chat-oauth-templates"
+    )

testing/tests/test_hermes_chat_images.py

@@ -0,0 +1,284 @@
+"""Hermes chat images contracts."""
+
+from __future__ import annotations
+
+from test_hermes_chat_support import (
+    HERMES,
+    Path,
+    SimpleNamespace,
+    _documents,
+    importlib,
+    sys,
+    time,
+    yaml,
+)
+
+
+def test_compact_image_edit_resolves_latest_tenant_artifact(tmp_path, monkeypatch):
+    """Follow-up edits resolve the source server-side and keep tool JSON small."""
+    provider_module = SimpleNamespace(
+        DEFAULT_ASPECT_RATIO="square",
+        ImageGenProvider=object,
+        error_response=lambda **value: value,
+        normalize_reference_images=lambda value: value,
+        resolve_aspect_ratio=lambda value: value,
+        save_b64_image=lambda *_args, **_kwargs: tmp_path / "saved.png",
+        success_response=lambda **value: value,
+    )
+    monkeypatch.setitem(sys.modules, "agent", SimpleNamespace())
+    monkeypatch.setitem(sys.modules, "agent.image_gen_provider", provider_module)
+    spec = importlib.util.spec_from_file_location(
+        "hermes_image_plugin",
+        HERMES / "plugins" / "image-gen-broker" / "__init__.py",
+    )
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    image_dir = tmp_path / "cache" / "images"
+    image_dir.mkdir(parents=True)
+    older = image_dir / "atlas_flux-old.png"
+    newest = image_dir / "atlas_gpt-image-new.png"
+    older.write_bytes(b"older")
+    newest.write_bytes(b"newest")
+    older.touch()
+    time.sleep(0.001)
+    newest.touch()
+    monkeypatch.setenv("HERMES_HOME", str(tmp_path))
+
+    calls = []
+    monkeypatch.setattr(
+        module,
+        "_handle_image_generate",
+        lambda args, route: calls.append((args, route)) or "ok",
+    )
+    assert module._handle_hosted_edit({"prompt": "make it a clown"}) == "ok"
+    assert calls == [
+        (
+            {
+                "prompt": "make it a clown",
+                "image_url": str(newest.resolve()),
+            },
+            "hosted",
+        )
+    ]
+
+def test_chat_reasoning_uses_switchyard_without_owner_credentials():
+    """Family pods use AUTO/manual routes without mounting owner credentials."""
+    configmap = _documents(HERMES / "chat-configmap.yaml")[0]
+    config = yaml.safe_load(configmap["data"]["config.yaml"])
+    assert config["model"] == {
+        "provider": "atlas-switchyard",
+        "default": "atlas/auto/fast",
+        "model": "atlas/auto/fast",
+    }
+    assert config["providers"]["atlas-switchyard"] == {
+        "name": "Automatic Router",
+        "api": "http://hermes-switchyard.hermes.svc.cluster.local:9005/v1",
+        "api_key": "atlas-switchyard",
+        "default_model": "atlas/auto/fast",
+        "transport": "chat_completions",
+    }
+    assert config["platforms"]["api_server"]["extra"]["model_routes"] == {
+        route: {"provider": "atlas-switchyard", "model": route}
+        for route in [
+            "atlas/auto/fast",
+            "atlas/auto/balanced",
+            "atlas/auto/deep",
+            "atlas/auto/maximum",
+            "atlas/manual/codex/luna",
+            "atlas/manual/codex/terra",
+            "atlas/manual/codex/sol",
+            "atlas/manual/claude/haiku",
+            "atlas/manual/claude/fable",
+            "atlas/manual/claude/sonnet",
+            "atlas/manual/claude/opus",
+            "atlas/manual/local/qwen-14b",
+        ]
+    }
+
+    agent = _documents(HERMES / "agent-deployment.yaml")[0]
+    containers = agent["spec"]["template"]["spec"]["containers"]
+    broker = next(item for item in containers if item["name"] == "codex-broker")
+    assert broker["ports"] == [
+        {"name": "codex-broker", "containerPort": 9003, "protocol": "TCP"}
+    ]
+    assert broker["securityContext"]["readOnlyRootFilesystem"] is True
+    assert broker["securityContext"]["runAsNonRoot"] is True
+    assert {item["name"]: item["value"] for item in broker["env"]}.items() >= {
+        "PYTHONPATH": "/opt/hermes",
+        "HERMES_CODEX_BROKER_LISTEN_PORT": "9003",
+        "HERMES_ROUTING_CATALOG_PATH": "/routing-catalog/catalog.json",
+    }.items()
+
+    services = _documents(HERMES / "service.yaml")
+    service = next(
+        item for item in services if item["metadata"]["name"] == "hermes-codex-broker"
+    )
+    assert service["spec"]["selector"] == {"app": "hermes-agent"}
+    assert service["spec"]["ports"] == [
+        {
+            "name": "http",
+            "port": 9003,
+            "targetPort": "codex-broker",
+            "protocol": "TCP",
+        }
+    ]
+
+    statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
+    assert statefulset["spec"]["template"]["metadata"]["annotations"][
+        "ai.bstein.dev/config-rev"
+    ] == "20260816-telegram-topics"
+    pod_spec = statefulset["spec"]["template"]["spec"]
+    patch_init = next(
+        item for item in pod_spec["initContainers"]
+        if item["name"] == "patch-stream-recovery"
+    )
+    assert patch_init["command"][-2:] == [
+        "/opt/hermes/agent/conversation_loop.py",
+        "/patched/conversation_loop.py",
+    ]
+    hermes = next(
+        item
+        for item in pod_spec["containers"]
+        if item["name"] == "hermes"
+    )
+    assert {
+        "name": "stream-recovery-patch",
+        "mountPath": "/opt/hermes/agent/conversation_loop.py",
+        "subPath": "conversation_loop.py",
+    } in hermes["volumeMounts"]
+    api_session_init = next(
+        item for item in pod_spec["initContainers"]
+        if item["name"] == "patch-api-server-sessions"
+    )
+    assert "patch_api_server_sessions.py" in api_session_init["args"][0]
+    assert "migrate_telegram_api_sessions.py" in api_session_init["args"][0]
+    assert {
+        "name": "api-server-patch",
+        "mountPath": "/opt/hermes/gateway/platforms/api_server.py",
+        "subPath": "api_server.py",
+    } in hermes["volumeMounts"]
+    assert any(
+        volume["name"] == "api-server-patch" for volume in pod_spec["volumes"]
+    )
+    assert not any(
+        mount["mountPath"].endswith("/.codex")
+        for mount in hermes["volumeMounts"]
+    )
+
+    policies = _documents(HERMES / "networkpolicy.yaml")
+    agent_policy = next(
+        item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation"
+    )
+    broker_ingress = next(
+        rule
+        for rule in agent_policy["spec"]["ingress"]
+        if {port["port"] for port in rule["ports"]} == {9002, 9003}
+    )
+    assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == {
+        "app": "hermes-chat-tenant"
+    }
+
+def test_image_broker_returns_bytes_and_removes_owner_cache(tmp_path: Path, monkeypatch):
+    """The broker must not retain a family user's generated image."""
+    broker_path = HERMES / "scripts" / "image_broker.py"
+    spec = importlib.util.spec_from_file_location("hermes_image_broker", broker_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    generated = tmp_path / "generated.png"
+    generated.write_bytes(b"\x89PNG\r\n\x1a\nprivate-image")
+
+    class Provider:
+        def generate(self, prompt, aspect, **kwargs):
+            assert prompt == "paint a blue sphere"
+            assert aspect == "square"
+            return {
+                "success": True,
+                "image": str(generated),
+                "model": "gpt-image-2-high",
+                "quality": "high",
+            }
+
+    monkeypatch.setattr(module, "_PROVIDER", Provider())
+    result = module._generate(
+        {
+            "prompt": "paint a blue sphere",
+            "aspect_ratio": "square",
+            "model": "gpt-image-2-high",
+        }
+    )
+
+    assert result["success"] is True
+    assert result["image_b64"]
+    assert "image" not in result
+    assert not generated.exists()
+
+def test_image_broker_auto_falls_back_to_local_and_honors_explicit_routes(
+    monkeypatch,
+):
+    """AUTO is hosted-first while explicit local never calls the hosted lane."""
+    broker_path = HERMES / "scripts" / "image_broker.py"
+    spec = importlib.util.spec_from_file_location("hermes_image_router", broker_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    calls = []
+
+    def hosted(payload, model, prompt, aspect):
+        calls.append(("hosted", model, prompt, aspect))
+        return {"success": False, "error": "hosted refusal"}
+
+    def local(payload, timeout=1800.0):
+        calls.append(("local", payload["model"], timeout))
+        return {
+            "success": True,
+            "image_b64": "aW1hZ2U=",
+            "model": "flux-2-klein-4b-local",
+            "route": "local",
+        }
+
+    monkeypatch.setattr(module, "_generate_hosted", hosted)
+    monkeypatch.setattr(module, "_local_request", local)
+    auto = module._generate(
+        {
+            "prompt": "colorize this family photograph",
+            "aspect_ratio": "portrait",
+            "model": "atlas-image-auto-high",
+        }
+    )
+    assert auto["success"] is True
+    assert auto["route"] == "local"
+    assert auto["hosted_fallback_reason"] == "hosted refusal"
+    assert [call[0] for call in calls] == ["hosted", "local"]
+
+    calls.clear()
+    explicit = module._generate(
+        {
+            "prompt": "make a local landscape",
+            "aspect_ratio": "landscape",
+            "model": "flux-2-klein-4b-local",
+        }
+    )
+    assert explicit["route"] == "local"
+    assert [call[0] for call in calls] == ["local"]
+
+def test_image_broker_policy_is_narrow_and_operator_extensible(tmp_path: Path, monkeypatch):
+    """Family-photo restoration stays allowed while the hard boundary remains."""
+    broker_path = HERMES / "scripts" / "image_broker.py"
+    spec = importlib.util.spec_from_file_location("hermes_image_policy", broker_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    policy = tmp_path / "policy.json"
+    policy.write_text('{"additional_blocked_phrases":["site-specific block"]}')
+    monkeypatch.setattr(module, "POLICY_PATH", policy)
+
+    assert module._policy_error(
+        "Colorize my baby photograph with a lighter natural skin tone"
+    ) is None
+    assert "minors" in module._policy_error("Create a sexual image of a child")
+    assert module._policy_error("A site-specific block request") == (
+        "request is blocked by the operator image policy"
+    )

testing/tests/test_hermes_chat_local_compute.py

@@ -0,0 +1,295 @@
+"""Hermes chat local compute contracts."""
+
+from __future__ import annotations
+
+from test_hermes_chat_support import (
+    HERMES,
+    Path,
+    ROOT,
+    SimpleNamespace,
+    _documents,
+    importlib,
+    json,
+    tomllib,
+)
+
+
+def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
+    """FLUX and Wolf share titan-24 while text stays on titan-20."""
+    deployment = _documents(HERMES / "local-image-deployment.yaml")[0]
+    assert deployment["metadata"]["name"] == "hermes-local-image"
+    pod = deployment["spec"]["template"]["spec"]
+    assert pod["serviceAccountName"] == "hermes-gpu-runtime"
+    local = next(item for item in pod["containers"] if item["name"] == "local-image")
+    assert len(pod["containers"]) == 1
+    assert local["resources"]["requests"]["nvidia.com/gpu.shared"] == 1
+    assert local["ports"] == [{"name": "local-image", "containerPort": 9004}]
+    assert any(mount["mountPath"] == "/models" for mount in local["volumeMounts"])
+    model_env = {item["name"]: item["value"] for item in local["env"]}
+    assert model_env["HERMES_LOCAL_IMAGE_LISTEN_PORT"] == "9004"
+    assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == (
+        "e7b7dc27f91deacad38e78976d1f2b499d76a294"
+    )
+    assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24"
+    assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80"
+    assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == (
+        "3221225472"
+    )
+    assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential"
+    assert "nvidia-process-exporter-local.monitoring.svc.cluster.local" in model_env[
+        "HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL"
+    ]
+    models_volume = next(item for item in pod["volumes"] if item["name"] == "models")
+    assert models_volume["persistentVolumeClaim"]["claimName"] == (
+        "hermes-image-models"
+    )
+
+    services = _documents(HERMES / "service.yaml")
+    image_service = next(
+        item for item in services if item["metadata"]["name"] == "hermes-local-image"
+    )
+    assert image_service["spec"]["selector"] == {"app": "hermes-local-image"}
+
+    handoff_services = _documents(HERMES / "model-gate-deployment.yaml")
+    handoff = next(
+        item
+        for item in handoff_services
+        if item["kind"] == "Service"
+        and item["metadata"]["name"] == "hermes-gpu-handoff"
+    )
+    assert handoff["spec"]["ports"][0]["targetPort"] == "handoff"
+
+    ariadne = _documents(
+        Path(__file__).parents[2]
+        / "services/maintenance/apps/ariadne-deployment.yaml"
+    )[0]
+    env = {
+        item["name"]: item["value"]
+        for item in ariadne["spec"]["template"]["spec"]["containers"][0]["env"]
+        if "value" in item
+    }
+    assert env["GAME_MODE_OLLAMA_URL"] == (
+        "http://hermes-gpu-handoff.hermes.svc.cluster.local:11434"
+    )
+    assert env["GAME_MODE_OLLAMA_MODEL"] == "flux-2-klein-4b-local"
+
+    for config_name in ("configmap.yaml", "agent-configmap.yaml", "chat-configmap.yaml"):
+        config = _documents(HERMES / config_name)[0]["data"]["config.yaml"]
+        assert "gpt-oss:20b" not in config
+        assert "atlas-switchyard" in config
+    switchyard = _documents(HERMES / "switchyard-configmap.yaml")[0]["data"][
+        "routes.toml"
+    ]
+    assert 'id = "qwen2.5:14b-instruct-q4_0"' in switchyard
+    assert "qwen2.5:3b-instruct-q4_0" not in switchyard
+    assert "route/local/qwen2.5-14b/medium" in switchyard
+    assert "Anthropic and Claude name the same provider" in switchyard
+    assert "OpenAI and Codex name the same provider" in switchyard
+    assert "Choose across every configured Codex and Claude family" in switchyard
+    assert "Claude Fable" in switchyard
+    assert switchyard.count('Treat "think hard"') == 4
+    assert switchyard.count("Never choose below the") >= 5
+    switchyard_config = tomllib.loads(switchyard)
+    routes = switchyard_config["routes"]
+    configured_targets = switchyard_config["targets"]
+    for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
+        leading_targets = set(routes[route_name]["targets"][:2])
+        assert leading_targets == {"codex_sol_xhigh", "claude_opus_xhigh"}
+        assert "max_output_tokens" not in routes[route_name]
+    for route_name in ("auto_fast", "auto_balanced", "auto_deep", "auto_maximum"):
+        targets = routes[route_name]["targets"]
+        selector_targets = routes[route_name]["response_schema"]
+        assert not any(target.startswith("local_") for target in targets)
+        assert any("fable" in target for target in targets)
+        assert "local_qwen" not in selector_targets
+        assert "not eligible for foreground" in routes[route_name]["prompt"]
+    for route_name in ("auto_deep", "auto_maximum"):
+        targets = routes[route_name]["targets"]
+        selector_targets = routes[route_name]["response_schema"]
+        assert not any(target.endswith("_low") for target in targets)
+        assert "_low" not in selector_targets
+    maximum_targets = routes["auto_maximum"]["targets"]
+    maximum_selector_targets = routes["auto_maximum"]["response_schema"]
+    assert not any(target.endswith("_medium") for target in maximum_targets)
+    assert "_medium" not in maximum_selector_targets
+    assert "absolute high effort floor" in routes["auto_maximum"]["prompt"]
+    assert "quality mark was missed" in routes["auto_balanced"]["prompt"]
+    assert "raises the next boundary to xhigh" in routes["auto_maximum"]["prompt"]
+    assert "Repeated quality misses require xhigh" in routes[
+        "worker_auto_maximum"
+    ]["prompt"]
+    assert any(
+        target.startswith("local_")
+        for target in routes["manual_local_qwen"]["targets"]
+    )
+    for route_name in (
+        "manual_codex_luna",
+        "manual_codex_terra",
+        "manual_codex_sol",
+        "manual_claude_haiku",
+        "manual_claude_fable",
+        "manual_claude_sonnet",
+        "manual_claude_opus",
+    ):
+        assert not any(
+            target.startswith("local_") for target in routes[route_name]["targets"]
+        )
+    for provider, families in {
+        "codex": ("luna", "terra", "sol"),
+        "claude": ("haiku", "fable", "sonnet", "opus"),
+    }.items():
+        for family in families:
+            for effort in ("low", "medium", "high", "xhigh"):
+                route = routes[f"manual_{provider}_{family}_{effort}"]
+                assert route["id"] == f"atlas/manual/{provider}/{family}/{effort}"
+                assert route["targets"][0] == f"{provider}_{family}_{effort}"
+                worker_target = f"worker_{provider}_{family}_{effort}"
+                assert worker_target in routes["worker_auto_maximum"]["targets"]
+                assert configured_targets[worker_target]["id"] == (
+                    f"worker/{provider}/{family}/{effort}"
+                )
+    for route_name in ("auto_fast", "auto_balanced"):
+        prompt = routes[route_name]["prompt"]
+        assert "image tool—not the conversational model" in prompt
+        assert "Do not select a" in prompt
+        assert "local Qwen or Claude target" in prompt
+    assert "max_output_tokens" not in routes["worker_auto_maximum"]
+    model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][
+        "model_gate.py"
+    ]
+    assert "qwen2.5:14b-instruct-q4_0" in model_gate
+
+def test_titan20_serializes_classifier_and_local_chat_model_residency():
+    """Classifier and local chat share one serialized resident Qwen weight."""
+    deployment = _documents(
+        Path(__file__).parents[2] / "services/ai-llm/deployment.yaml"
+    )[0]
+    pod = deployment["spec"]["template"]["spec"]
+    required = pod["affinity"]["nodeAffinity"][
+        "requiredDuringSchedulingIgnoredDuringExecution"
+    ]["nodeSelectorTerms"][0]["matchExpressions"][0]
+    assert required["values"] == ["titan-20"]
+    container = pod["containers"][0]
+    env = {item["name"]: item["value"] for item in container["env"]}
+    assert env["OLLAMA_MAX_LOADED_MODELS"] == "1"
+    assert env["OLLAMA_NUM_PARALLEL"] == "1"
+    assert env["OLLAMA_KEEP_ALIVE"] == "-1"
+    assert env["OLLAMA_CONTEXT_LENGTH"] == "8192"
+    warm_command = " ".join(container["command"])
+    assert "--keepalive=-1" not in warm_command
+    models = next(item for item in pod["volumes"] if item["name"] == "models")
+    assert models["persistentVolumeClaim"]["claimName"] == (
+        "ollama-models-titan20"
+    )
+
+def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch):
+    """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX."""
+    source = ROOT / "dockerfiles" / "hermes-local-image-server.py"
+    spec = importlib.util.spec_from_file_location("hermes_local_image_server", source)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24")
+    monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0)
+    monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30)
+
+    idle = module._parse_gpu_activity(
+        '\n'.join(
+            [
+                'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0',
+                'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000',
+                'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3',
+                'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000',
+            ]
+        )
+    )
+    assert idle["interactive_active"] is False
+    assert idle["external_gpu_memory_bytes"] == 1900000000
+    assert idle["external_gpu_sm_percent"] == 3
+
+    background_spike = module._parse_gpu_activity(
+        'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n'
+    )
+    assert background_spike["interactive_active"] is False
+
+    active = module._parse_gpu_activity(
+        'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n'
+    )
+    assert active["interactive_active"] is True
+    assert "91%" in active["gpu_guard_reason"]
+
+def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch):
+    """A completed render must not leave its CUDA context in the API process."""
+    source = ROOT / "dockerfiles" / "hermes-local-image-server.py"
+    spec = importlib.util.spec_from_file_location("hermes_local_image_worker", source)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    calls = []
+
+    def run(command, **kwargs):
+        calls.append((command, kwargs))
+        return SimpleNamespace(
+            returncode=0,
+            stdout=b'{"success":true,"route":"local","image_b64":"cG5n"}',
+            stderr=b"",
+        )
+
+    monkeypatch.setattr(module.subprocess, "run", run)
+    result = module._render({"prompt": "black cat", "aspect_ratio": "square"})
+
+    assert result["route"] == "local"
+    command, options = calls[0]
+    assert command[-1] == "--render-worker"
+    assert json.loads(options["input"]) == {
+        "prompt": "black cat",
+        "aspect_ratio": "square",
+    }
+    assert options["timeout"] == module.RENDER_TIMEOUT_SECONDS
+    assert options["check"] is False
+
+def test_local_flux_uses_low_vram_offload_without_reducing_resolution():
+    """The shared 3080 lane must trade time, not image size, for headroom."""
+    source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text()
+    assert 'OFFLOAD_MODE = os.environ.get(' in source
+    assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source
+    assert "pipe.enable_sequential_cpu_offload()" in source
+    assert '"square": (1024, 1024)' in source
+
+def test_chat_auth_and_relay_are_pod_lifetime_only():
+    statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
+    pod = statefulset["spec"]["template"]["spec"]
+    containers = statefulset["spec"]["template"]["spec"]["containers"]
+
+    for name in ("hermes", "webui"):
+        container = next(item for item in containers if item["name"] == name)
+        env = {item["name"]: item["value"] for item in container["env"]}
+        assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json"
+        mount = next(
+            item for item in container["volumeMounts"] if item["name"] == "runtime-access"
+        )
+        assert mount["mountPath"] == "/runtime-access"
+        assert "subPath" not in mount
+
+    hermes_env = {
+        item["name"]: item["value"]
+        for item in next(item for item in containers if item["name"] == "hermes")["env"]
+    }
+    runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access")
+    assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"}
+    assert not any(item["name"] == "provider-auth" for item in pod["volumes"])
+    init_command = next(
+        item for item in pod["initContainers"] if item["name"] == "init-config"
+    )["command"][2]
+    for key in (
+        "ANTHROPIC_API_KEY",
+        "API_SERVER_KEY",
+        "CLAUDE_CODE_OAUTH_TOKEN",
+        "GITEA_TOKEN",
+        "HERMES_IMAGE_BROKER_KEY",
+        "OPENAI_API_KEY",
+    ):
+        assert key in init_command
+    assert "printf 'API_SERVER_KEY=%s" not in init_command
+    assert hermes_env["AGENT_BROWSER_EXECUTABLE_PATH"].endswith("/chrome-linux/headless_shell")
+    assert "--no-sandbox" in hermes_env["AGENT_BROWSER_ARGS"]

testing/tests/test_hermes_chat_provider_auth.py

@@ -0,0 +1,192 @@
+"""Hermes chat provider auth contracts."""
+
+from __future__ import annotations
+
+from test_hermes_chat_support import (
+    HERMES,
+    Path,
+    ROOT,
+    _documents,
+    _load_broker_module,
+    importlib,
+    json,
+)
+
+
+def test_claude_broker_uses_native_subscription_without_api_billing(monkeypatch):
+    """Claude traffic must use the native first-party CLI subscription lane."""
+    module = _load_broker_module(
+        "hermes_claude_broker", "claude_oauth_broker.py", monkeypatch
+    )
+    monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-leak")
+    monkeypatch.setenv("CLAUDE_API_KEY", "must-not-leak")
+    monkeypatch.setattr(
+        module,
+        "resolve_route",
+        lambda route: "claude-fable-5" if "/fable/" in route else route,
+    )
+
+    model, effort = module._route(
+        "route/claude/fable/xhigh", {"output_config": {"effort": "xhigh"}}
+    )
+
+    assert (model, effort) == ("claude-fable-5", "xhigh")
+    assert "ANTHROPIC_API_KEY" not in module._claude_environment()
+    assert "CLAUDE_API_KEY" not in module._claude_environment()
+    assert module.CAPACITY_PATTERN.search("weekly usage limit exhausted")
+
+def test_codex_native_health_overrides_historical_router_errors(
+    tmp_path: Path, monkeypatch
+):
+    """Fresh first-party health is authoritative over old Switchyard probes."""
+    plugin_path = HERMES / "plugins" / "auto-router" / "provider_status.py"
+    spec = importlib.util.spec_from_file_location("hermes_provider_status", plugin_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    health_path = tmp_path / "codex.json"
+    health_path.write_text(
+        json.dumps(
+            {
+                "state": "available",
+                "authenticated": True,
+                "transport": "codex-chatgpt-subscription",
+            }
+        )
+    )
+    monkeypatch.setattr(module, "CODEX_HEALTH_PATH", health_path)
+    monkeypatch.setattr(module, "CLAUDE_HEALTH_PATH", tmp_path / "missing.json")
+    monkeypatch.setattr(
+        module,
+        "_get_json",
+        lambda url: {"status": "ok"}
+        if url.endswith("/health")
+        else {
+            "models": {
+                "route/codex/terra/medium": {
+                    "calls": 1,
+                    "errors": 99,
+                    "total_tokens": 12,
+                }
+            }
+        },
+    )
+    monkeypatch.setattr(module, "_codex_account", lambda: {})
+    monkeypatch.setattr(module, "_claude_account", lambda: {})
+
+    codex = module.provider_status_payload()["providers"]["codex"]
+
+    assert codex["errors"] == 99
+    assert codex["state"] == "available"
+    assert codex["native_health"]["transport"] == "codex-chatgpt-subscription"
+
+def test_api_session_patch_accepts_parent_lineage(tmp_path: Path):
+    """API-created workers must persist the originating Hermes session."""
+    module_path = HERMES / "scripts" / "patch_api_server_sessions.py"
+    spec = importlib.util.spec_from_file_location("patch_api_sessions", module_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    source = tmp_path / "api_server.py"
+    destination = tmp_path / "patched.py"
+    source.write_text(
+        "prefix\n"
+        + module.BEFORE
+        + "middle\n"
+        + module.RUNS_BEFORE
+        + "run body\n"
+        + module.RUN_CLOSE_BEFORE
+        + module.RESPONSES_SESSION_BEFORE
+        + module.EVENT_CALLBACK_SIGNATURE_BEFORE
+        + "callback docstring and push helper\n"
+        + module.EVENT_CALLBACK_BODY_BEFORE
+        + "tool start body\n"
+        + module.EVENT_CALLBACK_END_BEFORE
+        + module.EVENT_CALLBACK_CALL_BEFORE
+        + module.RUN_SWEEP_BEFORE
+        + "suffix\n",
+        encoding="utf-8",
+    )
+
+    module.patch(source, destination)
+    patched = destination.read_text(encoding="utf-8")
+
+    assert "X-Hermes-Parent-Session-Id" in patched
+    assert "parent_session_id=parent_session_id" in patched
+    assert "Parent session not found" in patched
+    assert "HERMES_API_DEFAULT_PARENT_MATCH_PREFIXES" in patched
+    assert "user_message.startswith(default_prefixes)" in patched
+    assert "session_parent_conflict" in patched
+    assert "X-Hermes-Conversation-Platform" in patched
+    assert "X-Hermes-Conversation-Title" in patched
+    assert 'conversation_platform != "telegram"' in patched
+    assert "db.record_gateway_session_peer(" in patched
+    assert 'display_name="Telegram"' in patched
+    assert "db.reopen_session(session_id)" in patched
+    assert 'db.end_session(session_id, f"api_run_{terminal_status}")' in patched
+    assert "def _record_run_activity(" in patched
+    assert '"_thinking": "Hermes is reasoning"' in patched
+    assert '"run.started": "Worker started"' in patched
+    assert '"run.completed": "Worker completed"' in patched
+    assert '"reasoning.available": "Hermes finished a reasoning step"' in patched
+    assert '"subagent.progress": "Nested worker progress"' in patched
+    assert "redact_sensitive_text" in patched
+    assert 'getattr(os, "O_NOFOLLOW", 0)' in patched
+    assert "os.fchmod(fd, 0o600)" in patched
+    assert "session_id=session_id" in patched
+    assert 'self._record_run_activity(session_id, "run.started")' in patched
+    assert 'detail = tool_name if event_type in {' in patched
+    assert 'if event_type == "subagent.tool"' in patched
+    assert "_RUN_ACTIVITY_HEARTBEAT_SECONDS = 15.0" in patched
+    assert 'heartbeats.get(session_id, 0.0)' in patched
+    assert '"subagent.thinking",' in patched
+    assert "Stream retention and run lifetime are separate" in patched
+    assert 'terminal_status in {"completed", "failed", "cancelled"}' in patched
+    assert patched.index("terminal_status = self._run_statuses") < patched.index(
+        "self._active_run_tasks.pop(run_id, None)"
+    )
+
+def test_switchyard_brokers_and_native_claude_lane_use_the_right_images():
+    """Thin brokers stay small while native Claude runs beside owner auth."""
+    dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-switchyard-brokers").read_text()
+    assert "httpx==0.28.1" in dockerfile
+    assert "worker_route_broker.py" in dockerfile
+    assert "routing_catalog.py" in dockerfile
+
+    deployment = _documents(HERMES / "switchyard-deployment.yaml")[0]
+    containers = {
+        container["name"]: container
+        for container in deployment["spec"]["template"]["spec"]["containers"]
+    }
+    expected = (
+        "registry.bstein.dev/bstein/hermes-switchyard-brokers@"
+        "sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083"
+    )
+    assert containers["worker-route-broker"]["image"] == expected
+    assert containers["classifier-broker"]["image"] == expected
+    assert "claude-oauth-broker" not in containers
+
+    agent = _documents(HERMES / "agent-deployment.yaml")[0]
+    agent_containers = {
+        container["name"]: container
+        for container in agent["spec"]["template"]["spec"]["containers"]
+    }
+    for container_name in ("hermes", "terminal"):
+        container = agent_containers[container_name]
+        environment = {item["name"]: item["value"] for item in container["env"]}
+        mounts = {item["name"]: item for item in container["volumeMounts"]}
+        assert environment["HERMES_ROUTING_CATALOG_PATH"] == "/routing-catalog/catalog.json"
+        assert environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json"
+        assert environment["HERMES_CLAUDE_HEALTH_PATH"] == "/opt/data/provider-health/claude.json"
+        assert mounts["routing-catalog"]["mountPath"] == "/routing-catalog"
+        assert mounts["routing-catalog"]["readOnly"] is True
+    codex = agent_containers["codex-broker"]
+    codex_environment = {item["name"]: item["value"] for item in codex["env"]}
+    assert codex_environment["HERMES_CODEX_HEALTH_PATH"] == "/opt/data/provider-health/codex.json"
+    claude = agent_containers["claude-broker"]
+    assert claude["image"].startswith("registry.bstein.dev/bstein/hermes-agent@")
+    assert "unset ANTHROPIC_API_KEY CLAUDE_API_KEY" in claude["args"][0]
+    assert any(
+        mount["name"] == "home" and mount["mountPath"] == "/opt/data"
+        for mount in claude["volumeMounts"]
+    )

testing/tests/test_hermes_chat_sandbox.py

@@ -0,0 +1,24 @@
+"""Hermes chat sandbox contracts."""
+
+from __future__ import annotations
+
+from test_hermes_chat_support import (
+    Path,
+    ROOT,
+    importlib,
+)
+
+
+def test_sandbox_executes_python_with_bounded_output(tmp_path: Path, monkeypatch):
+    source = ROOT / "dockerfiles" / "hermes-chat-sandbox-server.py"
+    spec = importlib.util.spec_from_file_location("hermes_chat_sandbox_server", source)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    monkeypatch.setattr(module, "WORKSPACE", tmp_path)
+
+    result = module._execute("import math\nprint(math.comb(10, 3))")
+
+    assert result["success"] is True
+    assert result["stdout"] == "120\n"
+    assert result["stderr"] == ""

testing/tests/test_hermes_chat_sessions.py

@@ -0,0 +1,392 @@
+"""Hermes chat sessions contracts."""
+
+from __future__ import annotations
+
+from test_hermes_chat_support import (
+    HERMES,
+    Path,
+    importlib,
+    json,
+    pytest,
+    sqlite3,
+)
+
+
+def test_telegram_api_session_migration_is_bounded_and_idempotent(tmp_path: Path):
+    """Only named Telegram conversations receive presentation metadata."""
+    module_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py"
+    spec = importlib.util.spec_from_file_location("migrate_telegram_sessions", module_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    state = tmp_path / "state.db"
+    responses = tmp_path / "response_store.db"
+    with sqlite3.connect(state) as connection:
+        connection.execute(
+            """CREATE TABLE sessions (
+                id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT,
+                display_name TEXT, origin_json TEXT, title TEXT
+            )"""
+        )
+        connection.executemany(
+            "INSERT INTO sessions (id, source) VALUES (?, ?)",
+            (("telegram-session", "api_server"), ("other-session", "api_server")),
+        )
+    with sqlite3.connect(responses) as connection:
+        connection.execute(
+            "CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)"
+        )
+        connection.execute(
+            "CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)"
+        )
+        connection.executemany(
+            "INSERT INTO responses VALUES (?, ?, 0)",
+            (
+                ("telegram-response", json.dumps({"session_id": "telegram-session"})),
+                ("other-response", json.dumps({"session_id": "other-session"})),
+            ),
+        )
+        connection.executemany(
+            "INSERT INTO conversations VALUES (?, ?)",
+            (("telegram", "telegram-response"), ("unrelated", "other-response")),
+        )
+
+    assert module.migrate(state, responses) == 1
+    assert module.migrate(state, responses) == 0
+    with sqlite3.connect(state) as connection:
+        telegram = connection.execute(
+            "SELECT session_key, chat_type, display_name, origin_json, title "
+            "FROM sessions WHERE id = 'telegram-session'"
+        ).fetchone()
+        other = connection.execute(
+            "SELECT session_key, title FROM sessions WHERE id = 'other-session'"
+        ).fetchone()
+    assert telegram[:3] == ("telegram", "private", "Telegram")
+    assert json.loads(telegram[3]) == {
+        "platform": "telegram",
+        "session_key": "telegram",
+    }
+    assert telegram[4] == "Telegram · General"
+    assert other == (None, None)
+
+def test_web_session_activity_patch_projects_bounded_events(tmp_path: Path):
+    """The DOM transcript includes events without polluting agent history."""
+    module_path = HERMES / "scripts" / "patch_web_session_activity.py"
+    spec = importlib.util.spec_from_file_location("patch_web_activity", module_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    source = tmp_path / "web_server.py"
+    destination = tmp_path / "patched.py"
+    source.write_text(
+        "prefix\n"
+        + module.LATEST_ROWS_BEFORE
+        + module.LATEST_SELECTION_BEFORE
+        + module.HELPER_MARKER
+        + "    db = object()\n"
+        + module.MESSAGES_BEFORE
+        + "suffix\n",
+        encoding="utf-8",
+    )
+
+    module.patch(source, destination)
+    patched = destination.read_text(encoding="utf-8")
+
+    assert "def _run_activity_messages(" in patched
+    assert 'root = get_hermes_home() / "run-activity"' in patched
+    assert "path.stat().st_size > 600_000" in patched
+    assert "entries[-1_000:]" in patched
+    assert "*_run_activity_messages(sid)" in patched
+    assert "messages.sort(" in patched
+    assert "limit: Optional[int] = None" in patched
+    assert "total_messages = len(messages)" in patched
+    assert "min(int(limit), 10_000)" in patched
+    assert '"total_messages": total_messages' in patched
+    assert "SELECT id, parent_session_id, started_at, ended_at" in patched
+    assert "newest still-open member of the lineage" in patched
+    assert "immediate objective parent" in patched
+    assert "mixes unrelated workstreams" in patched
+    assert "orphaned children" in patched
+    assert 'item[0].get("ended_at") is None' in patched
+
+def test_web_session_lineage_returns_to_resumed_parent():
+    """An ended reviewer must not strand the live view away from its parent."""
+    module_path = HERMES / "scripts" / "patch_web_session_activity.py"
+    spec = importlib.util.spec_from_file_location("patch_web_lineage", module_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    namespace: dict[str, object] = {}
+    exec(
+        "def select_active(sid, db, rows):\n" + module.LATEST_SELECTION_AFTER,
+        namespace,
+    )
+
+    class FakeDB:
+        def get_session(self, session_id):
+            return next((row for row in rows if row["id"] == session_id), None)
+
+    rows = [
+        {
+            "id": "umbrella",
+            "parent_session_id": None,
+            "started_at": 0.0,
+            "ended_at": None,
+        },
+        {
+            "id": "root",
+            "parent_session_id": "umbrella",
+            "started_at": 1.0,
+            "ended_at": None,
+        },
+        {
+            "id": "review-1",
+            "parent_session_id": "root",
+            "started_at": 2.0,
+            "ended_at": 3.0,
+        },
+        {
+            "id": "review-2",
+            "parent_session_id": "root",
+            "started_at": 4.0,
+            "ended_at": None,
+        },
+    ]
+    select_active = namespace["select_active"]
+
+    assert select_active("root", FakeDB(), rows) == (
+        "review-2",
+        ["root", "review-2"],
+    )
+    assert select_active("review-1", FakeDB(), rows) == (
+        "review-2",
+        ["root", "review-2"],
+    )
+    rows[2]["ended_at"] = 5.0
+    rows[3]["ended_at"] = 5.0
+    assert select_active("root", FakeDB(), rows) == ("root", ["root"])
+
+    rows[1]["ended_at"] = 9.0
+    rows.append(
+        {
+            "id": "orphaned-review",
+            "parent_session_id": "root",
+            "started_at": 6.0,
+            "ended_at": None,
+        }
+    )
+    rows.append(
+        {
+            "id": "accepted-review",
+            "parent_session_id": "root",
+            "started_at": 7.0,
+            "ended_at": 8.0,
+        }
+    )
+    assert select_active("review-1", FakeDB(), rows) == (
+        "accepted-review",
+        ["root", "accepted-review"],
+    )
+
+def test_api_activity_patch_coalesces_streaming_heartbeats(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+):
+    """Token callbacks stay bounded while every tool transition is retained."""
+    module_path = HERMES / "scripts" / "patch_api_server_sessions.py"
+    spec = importlib.util.spec_from_file_location("patch_api_activity", module_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    activity_body = module.EVENT_CALLBACK_SIGNATURE_AFTER.split(
+        "    def _make_run_event_callback(", 1
+    )[0]
+    namespace: dict[str, object] = {}
+    exec(
+        "import hashlib, json, logging, os, time\n"
+        "from pathlib import Path\n"
+        "logger = logging.getLogger(__name__)\n"
+        "def redact_sensitive_text(value): return value\n"
+        "class ActivityRecorder:\n"
+        + activity_body,
+        namespace,
+    )
+    recorder = namespace["ActivityRecorder"]()
+    monkeypatch.setenv("HERMES_HOME", str(tmp_path))
+
+    recorder._record_run_activity("session", "_thinking")
+    recorder._record_run_activity("session", "reasoning.available")
+    recorder._record_run_activity("session", "subagent.thinking")
+    recorder._record_run_activity("session", "tool.started", tool_name="terminal")
+    recorder._record_run_activity("session", "tool.completed", tool_name="terminal")
+
+    journals = list((tmp_path / "run-activity").glob("*.jsonl"))
+    assert len(journals) == 1
+    entries = [json.loads(line) for line in journals[0].read_text().splitlines()]
+    assert [entry["activity_event"] for entry in entries] == [
+        "_thinking",
+        "tool.started",
+        "tool.completed",
+    ]
+
+def test_legacy_api_sessions_are_nested_idempotently(tmp_path: Path):
+    """Known standalone API workers move under Cassandra without data loss."""
+    module_path = HERMES / "scripts" / "migrate_api_session_lineage.py"
+    spec = importlib.util.spec_from_file_location("migrate_api_sessions", module_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    database = tmp_path / "state.db"
+    with sqlite3.connect(database) as connection:
+        connection.execute(
+            "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, "
+            "parent_session_id TEXT, title TEXT, transcript TEXT, archived INTEGER DEFAULT 0)"
+        )
+        connection.execute(
+            "INSERT INTO sessions (id, source, parent_session_id, title, transcript) "
+            "VALUES (?, 'tui', NULL, 'Cassandra', 'parent-data')",
+            (module.LEGACY_CASSANDRA_PARENT,),
+        )
+        worker_id = next(iter(module.LEGACY_CASSANDRA_WORKERS))
+        connection.execute(
+            "INSERT INTO sessions (id, source, parent_session_id, title, transcript) "
+            "VALUES (?, 'api_server', NULL, 'old', 'keep-me')",
+            (worker_id,),
+        )
+        connection.executemany(
+            "INSERT INTO sessions (id, source, parent_session_id, title, transcript) "
+            "VALUES (?, 'api_server', NULL, 'old smoke', 'keep-smoke')",
+            ((orphan_id,) for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS),
+        )
+
+    assert module.migrate(database) == 1 + len(module.LEGACY_ORPHANED_SMOKE_SESSIONS)
+    assert module.migrate(database) == 0
+    with sqlite3.connect(database) as connection:
+        row = connection.execute(
+            "SELECT parent_session_id, title, transcript FROM sessions WHERE id = ?",
+            (worker_id,),
+        ).fetchone()
+    assert row == (
+        module.LEGACY_CASSANDRA_PARENT,
+        module.LEGACY_CASSANDRA_WORKERS[worker_id],
+        "keep-me",
+    )
+    with sqlite3.connect(database) as connection:
+        orphans = connection.execute(
+            "SELECT id, archived, title, transcript FROM sessions "
+            "WHERE id IN ({}) ORDER BY id".format(
+                ",".join("?" for _ in module.LEGACY_ORPHANED_SMOKE_SESSIONS)
+            ),
+            tuple(module.LEGACY_ORPHANED_SMOKE_SESSIONS),
+        ).fetchall()
+    assert orphans == sorted(
+        (
+            orphan_id,
+            1,
+            module.LEGACY_ORPHANED_SMOKE_SESSIONS[orphan_id],
+            "keep-smoke",
+        )
+        for orphan_id in module.LEGACY_ORPHANED_SMOKE_SESSIONS
+    )
+
+def test_automated_triage_sessions_are_grouped_without_touching_interactive_runs(tmp_path: Path):
+    """Only the stable Ariadne contract moves below the triage parent."""
+    module_path = HERMES / "scripts" / "migrate_api_session_lineage.py"
+    spec = importlib.util.spec_from_file_location("migrate_triage_sessions", module_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    database = tmp_path / "state.db"
+    with sqlite3.connect(database) as connection:
+        connection.execute(
+            "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, "
+            "parent_session_id TEXT, title TEXT, started_at REAL, archived INTEGER DEFAULT 0)"
+        )
+        connection.execute(
+            "CREATE TABLE messages (session_id TEXT, role TEXT, content TEXT)"
+        )
+        connection.executemany(
+            "INSERT INTO sessions (id, source, started_at) VALUES (?, 'api_server', ?)",
+            (("triage-run", 1.0), ("jenkins-run", 1.5), ("interactive-run", 2.0)),
+        )
+        connection.executemany(
+            "INSERT INTO messages (session_id, role, content) VALUES (?, 'user', ?)",
+            (
+                (
+                    "triage-run",
+                    module.TRIAGE_MESSAGE_PREFIXES[0]
+                    + " Fix for incident sonar/bstein_home/python:S2208/finding-key.",
+                ),
+                (
+                    "jenkins-run",
+                    module.TRIAGE_MESSAGE_PREFIXES[1]
+                    + "\nAnalyze incident soteria/291 for the Jenkins job soteria.",
+                ),
+                ("interactive-run", "Please explain this alert to me."),
+            ),
+        )
+
+    assert module.migrate(database, group_triage=True) == 2
+    assert module.migrate(database, group_triage=True) == 0
+    with sqlite3.connect(database) as connection:
+        parent = connection.execute(
+            "SELECT title FROM sessions WHERE id = ?", (module.TRIAGE_PARENT,)
+        ).fetchone()
+        triage = connection.execute(
+            "SELECT parent_session_id, title FROM sessions WHERE id = 'triage-run'"
+        ).fetchone()
+        interactive = connection.execute(
+            "SELECT parent_session_id FROM sessions WHERE id = 'interactive-run'"
+        ).fetchone()
+        jenkins = connection.execute(
+            "SELECT parent_session_id, title FROM sessions WHERE id = 'jenkins-run'"
+        ).fetchone()
+    assert parent == (module.TRIAGE_PARENT_TITLE,)
+    assert triage == (
+        module.TRIAGE_PARENT,
+        "Sonar · bstein_home · python:S2208 · iage-run",
+    )
+    assert interactive == (None,)
+    assert jenkins == (
+        module.TRIAGE_PARENT,
+        "Sonar · soteria/291 · kins-run",
+    )
+
+def test_stale_parent_linked_api_workers_close_on_startup(tmp_path: Path):
+    """A previous gateway lifetime cannot leave phantom active workers."""
+    module_path = HERMES / "scripts" / "migrate_api_session_lineage.py"
+    spec = importlib.util.spec_from_file_location("close_stale_api_workers", module_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    database = tmp_path / "state.db"
+    with sqlite3.connect(database) as connection:
+        connection.execute(
+            "CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, "
+            "parent_session_id TEXT, title TEXT, started_at REAL, ended_at REAL, "
+            "end_reason TEXT, archived INTEGER DEFAULT 0)"
+        )
+        connection.executemany(
+            "INSERT INTO sessions "
+            "(id, source, parent_session_id, started_at, ended_at, end_reason) "
+            "VALUES (?, ?, ?, 1, ?, ?)",
+            (
+                ("stale", "api_server", "parent", None, None),
+                ("root", "api_server", None, None, None),
+                ("finished", "api_server", "parent", 2.0, "api_run_completed"),
+                ("interactive", "tui", "parent", None, None),
+            ),
+        )
+
+    assert module.migrate(database) == 1
+    assert module.migrate(database) == 0
+    with sqlite3.connect(database) as connection:
+        rows = connection.execute(
+            "SELECT id, ended_at, end_reason FROM sessions ORDER BY id"
+        ).fetchall()
+    by_id = {row[0]: row[1:] for row in rows}
+    assert by_id["stale"][0] is not None
+    assert by_id["stale"][1] == "api_run_recovered_stale"
+    assert by_id["root"] == (None, None)
+    assert by_id["finished"] == (2.0, "api_run_completed")
+    assert by_id["interactive"] == (None, None)

testing/tests/test_hermes_chat_support.py

@@ -0,0 +1,73 @@
+"""Contracts for isolated high-quality Hermes chat capabilities."""
+
+from __future__ import annotations
+
+import base64
+
+import importlib.util
+
+import json
+
+import sqlite3
+
+import sys
+
+import time
+
+import tomllib
+
+from pathlib import Path
+
+from types import ModuleType, SimpleNamespace
+
+import pytest
+
+import yaml
+
+ROOT = Path(__file__).parents[2]
+
+HERMES = ROOT / "services" / "hermes"
+
+VAULT = ROOT / "services" / "vault"
+
+def _documents(path: Path) -> list[dict]:
+    return [doc for doc in yaml.safe_load_all(path.read_text()) if doc]
+
+def _load_broker_module(name: str, filename: str, monkeypatch):
+    """Load one broker with its mounted routing-catalog dependency."""
+    catalog_path = HERMES / "scripts" / "routing_catalog.py"
+    catalog_spec = importlib.util.spec_from_file_location(
+        "routing_catalog", catalog_path
+    )
+    assert catalog_spec and catalog_spec.loader
+    catalog = importlib.util.module_from_spec(catalog_spec)
+    catalog_spec.loader.exec_module(catalog)
+    monkeypatch.setitem(sys.modules, "routing_catalog", catalog)
+    monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace())
+
+    broker_path = HERMES / "scripts" / filename
+    spec = importlib.util.spec_from_file_location(name, broker_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+__all__ = (
+    "HERMES",
+    "ModuleType",
+    "Path",
+    "ROOT",
+    "SimpleNamespace",
+    "VAULT",
+    "_documents",
+    "_load_broker_module",
+    "base64",
+    "importlib",
+    "json",
+    "pytest",
+    "sqlite3",
+    "sys",
+    "time",
+    "tomllib",
+    "yaml",
+)

testing/tests/test_hermes_chat_switchyard.py

@@ -0,0 +1,483 @@
+"""Hermes chat switchyard contracts."""
+
+from __future__ import annotations
+
+from test_hermes_chat_support import (
+    HERMES,
+    Path,
+    SimpleNamespace,
+    _documents,
+    _load_broker_module,
+    base64,
+    importlib,
+    json,
+    pytest,
+    sys,
+    time,
+    tomllib,
+)
+
+
+def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch):
+    """The relay is bounded, stateless, and rejects unapproved models."""
+    module = _load_broker_module(
+        "hermes_codex_broker", "codex_broker.py", monkeypatch
+    )
+    monkeypatch.setattr(module, "TOKEN", "relay-secret")
+
+    assert module._authorized("Bearer relay-secret") is True
+    assert module._authorized("Bearer wrong") is False
+    assert module._real_model("route/codex/gpt-5.6-sol/xhigh") == "gpt-5.6-sol"
+    payload = module._validate_payload(
+        {
+            "model": "gpt-5.6-terra",
+            "input": "route this chat turn",
+            "store": True,
+            "stream": False,
+            "max_output_tokens": 96,
+            "max_completion_tokens": 96,
+            "max_tokens": 96,
+            "temperature": 0.7,
+            "top_p": 0.9,
+        }
+    )
+    assert payload["store"] is False
+    assert payload["stream"] is True
+    assert "max_output_tokens" not in payload
+    assert "max_completion_tokens" not in payload
+    assert "max_tokens" not in payload
+    assert "temperature" not in payload
+    assert "top_p" not in payload
+    assert payload["input"] == [
+        {
+            "type": "message",
+            "role": "user",
+            "content": [{"type": "input_text", "text": "route this chat turn"}],
+        }
+    ]
+    response_item = {
+        "type": "message",
+        "role": "user",
+        "content": [{"type": "input_text", "text": "keep this item"}],
+    }
+    assert module._validate_payload(
+        {"model": "gpt-5.6-terra", "input": response_item}
+    )["input"] == [response_item]
+    response_items = [response_item]
+    assert module._validate_payload(
+        {"model": "gpt-5.6-terra", "input": response_items}
+    )["input"] is response_items
+    image_items = [
+        {
+            "type": "message",
+            "role": "user",
+            "content": [
+                {"type": "input_text", "text": "What color is this?"},
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": "data:image/png;base64,cHJpdmF0ZQ==",
+                        "detail": "high",
+                    },
+                },
+            ],
+        }
+    ]
+    assert module._validate_payload(
+        {"model": "gpt-5.6-terra", "input": image_items}
+    )["input"][0]["content"][1] == {
+        "type": "input_image",
+        "image_url": "data:image/png;base64,cHJpdmF0ZQ==",
+        "detail": "high",
+    }
+    switchyard_image_items = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "input_text", "text": "What color is this?"},
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": "data:image/png;base64,cHJpdmF0ZQ==",
+                        "detail": "auto",
+                    },
+                },
+            ],
+        }
+    ]
+    assert module._validate_payload(
+        {"model": "gpt-5.6-terra", "input": switchyard_image_items}
+    )["input"][0]["content"][1] == {
+        "type": "input_image",
+        "image_url": "data:image/png;base64,cHJpdmF0ZQ==",
+        "detail": "auto",
+    }
+    switchyard_base64_items = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "input_text", "text": "What color is this?"},
+                {
+                    "type": "image",
+                    "source": {
+                        "type": "base64",
+                        "media_type": "image/png",
+                        "data": "cHJpdmF0ZQ==",
+                    },
+                },
+            ],
+        }
+    ]
+    normalized_base64 = module._validate_payload(
+        {"model": "gpt-5.6-terra", "input": switchyard_base64_items}
+    )["input"][0]["content"][1]
+    assert normalized_base64 == {
+        "type": "input_image",
+        "image_url": "data:image/png;base64,cHJpdmF0ZQ==",
+    }
+    switchyard_enum_items = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "input_image",
+                    "image_url": {
+                        "type": "url",
+                        "data": {
+                            "url": "data:image/png;base64,cHJpdmF0ZQ==",
+                            "detail": "high",
+                        },
+                    },
+                }
+            ],
+        }
+    ]
+    nested_image = module._validate_payload(
+        {"model": "gpt-5.6-terra", "input": switchyard_enum_items}
+    )["input"][0]["content"][0]
+    assert nested_image["image_url"] == "data:image/png;base64,cHJpdmF0ZQ=="
+    assert nested_image["detail"] == "high"
+    with pytest.raises(ValueError, match=r"non-empty Responses image URL.*str\[4\]"):
+        module._validate_payload(
+            {
+                "model": "gpt-5.6-terra",
+                "input": [
+                    {
+                        "type": "message",
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "input_image",
+                                "image_url": {"detail": "high"},
+                            }
+                        ],
+                    }
+                ],
+            }
+        )
+    routed = module._validate_payload(
+        {
+            "model": "route/codex/gpt-5.6-luna/low",
+            "input": "use the low route",
+            "stream": False,
+        }
+    )
+    assert routed["model"] == "gpt-5.6-luna"
+    with pytest.raises(ValueError, match="unsupported Codex model"):
+        module._validate_payload({"model": "unapproved-model", "input": "hello"})
+    with pytest.raises(ValueError, match="non-empty Responses input"):
+        module._validate_payload({"model": "gpt-5.6-terra", "input": ""})
+    with pytest.raises(ValueError, match="non-empty Responses input list"):
+        module._validate_payload({"model": "gpt-5.6-terra", "input": []})
+
+    completed = {
+        "id": "resp_test",
+        "object": "response",
+        "status": "completed",
+        "output": [],
+    }
+    completed_item = {
+        "type": "message",
+        "role": "assistant",
+        "status": "completed",
+        "content": [{"type": "output_text", "text": "done"}],
+    }
+    assert module._completed_response(
+        [
+            "event: response.created",
+            'data: {"type":"response.created","response":{}}',
+            "event: response.output_item.done",
+            "data: "
+            + json.dumps(
+                {
+                    "type": "response.output_item.done",
+                    "output_index": 0,
+                    "item": completed_item,
+                }
+            ),
+            "event: response.completed",
+            "data: "
+            + json.dumps({"type": "response.completed", "response": completed}),
+            "data: [DONE]",
+        ]
+    )["output"] == [completed_item]
+    raw_stream = (
+        "event: response.output_item.done\n"
+        "data: "
+        + json.dumps(
+            {
+                "type": "response.output_item.done",
+                "output_index": 0,
+                "item": completed_item,
+            }
+        )
+        + "\n\nevent: response.completed\ndata: "
+        + json.dumps({"type": "response.completed", "response": completed})
+        + "\n\n"
+    ).encode()
+    normalized = module._normalized_stream(
+        raw_stream, {**completed, "output": [completed_item]}
+    ).decode()
+    terminal_data = next(
+        line for line in normalized.splitlines() if '"response.completed"' in line
+    )
+    assert json.loads(terminal_data.removeprefix("data: "))["response"][
+        "output"
+    ] == [completed_item]
+    assert normalized.endswith("\n\n")
+    streamed_function_item = {
+        "type": "function_call",
+        "name": "read_file",
+        "status": "completed",
+        "arguments": '{"path":"/tmp"}',
+    }
+    streamed_function_body = (
+        "event: response.function_call_arguments.delta\n"
+        'data: {"type":"response.function_call_arguments.delta",'
+        '"item_id":"call_1","delta":"{\\"path\\":\\"/tmp\\"}"}\n\n'
+        "event: response.function_call_arguments.done\n"
+        'data: {"type":"response.function_call_arguments.done",'
+        '"item_id":"call_1","arguments":"{\\"path\\":\\"/tmp\\"}"}\n\n'
+        "event: response.output_item.done\n"
+        'data: {"type":"response.output_item.done","output_index":0,'
+        '"item":{"type":"function_call","name":"read_file",'
+        '"arguments":"{\\"path\\":\\"/tmp\\"}"}}\n\n'
+        "event: response.completed\n"
+        "data: "
+        + json.dumps(
+            {
+                "type": "response.completed",
+                "response": {**completed, "output": [streamed_function_item]},
+            }
+        )
+        + "\n\n"
+    ).encode()
+    normalized_function_stream = module._normalized_stream(
+        streamed_function_body, {**completed, "output": [streamed_function_item]}
+    ).decode()
+    assert "response.function_call_arguments.delta" in normalized_function_stream
+    assert "response.function_call_arguments.done" not in normalized_function_stream
+    assert "response.output_item.done" not in normalized_function_stream
+    normalized_terminal = next(
+        line
+        for line in normalized_function_stream.splitlines()
+        if '"response.completed"' in line
+    )
+    assert json.loads(normalized_terminal.removeprefix("data: "))["response"][
+        "output"
+    ] == []
+    with pytest.raises(RuntimeError, match="retryable incomplete response"):
+        module._completed_response(
+            [
+                "event: response.incomplete",
+                'data: {"type":"response.incomplete","response":'
+                '{"status":"incomplete","incomplete_details":'
+                '{"reason":"max_output_tokens"}}}',
+            ]
+        )
+    with pytest.raises(RuntimeError, match="provider unavailable"):
+        module._completed_response(
+            [
+                "event: error",
+                'data: {"type":"error","error":{"message":"provider unavailable"}}',
+            ]
+        )
+    malformed_tool_item = {
+        "type": "function_call",
+        "name": "search_files",
+        "status": "completed",
+        "arguments": '{"path":"","offset":',
+    }
+    with pytest.raises(RuntimeError, match="malformed function arguments"):
+        module._completed_response(
+            [
+                "event: response.output_item.done",
+                "data: "
+                + json.dumps(
+                    {
+                        "type": "response.output_item.done",
+                        "output_index": 0,
+                        "item": malformed_tool_item,
+                    }
+                ),
+                "event: response.completed",
+                "data: "
+                + json.dumps({"type": "response.completed", "response": completed}),
+            ]
+        )
+    valid_tool_item = {
+        **malformed_tool_item,
+        "arguments": '{"path":"","offset":0}',
+    }
+    assert module._completed_response(
+        [
+            "event: response.output_item.done",
+            "data: "
+            + json.dumps(
+                {
+                    "type": "response.output_item.done",
+                    "output_index": 0,
+                    "item": valid_tool_item,
+                }
+            ),
+            "event: response.completed",
+            "data: "
+            + json.dumps({"type": "response.completed", "response": completed}),
+        ]
+    )["output"] == [valid_tool_item]
+    with pytest.raises(RuntimeError, match="malformed function arguments"):
+        module._completed_response(
+            [
+                "event: response.function_call_arguments.delta",
+                'data: {"type":"response.function_call_arguments.delta",'
+                '"item_id":"call_1","output_index":0,'
+                '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}',
+                "event: response.completed",
+                "data: "
+                + json.dumps({"type": "response.completed", "response": completed}),
+            ]
+        )
+    streamed_tool = module._completed_response(
+        [
+            "event: response.function_call_arguments.delta",
+            'data: {"type":"response.function_call_arguments.delta",'
+            '"item_id":"call_2","output_index":0,'
+            '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}',
+            "event: response.function_call_arguments.done",
+            'data: {"type":"response.function_call_arguments.done",'
+            '"item_id":"call_2","output_index":0,'
+            '"arguments":"{\\"path\\":\\"/tmp\\",\\"offset\\":0}"}',
+            "event: response.completed",
+            "data: "
+            + json.dumps({"type": "response.completed", "response": completed}),
+        ]
+    )
+    assert streamed_tool["status"] == "completed"
+
+    auth_dir = tmp_path / ".codex"
+    auth_dir.mkdir()
+    # The token payload need only prove the broker reads CODEX_HOME directly.
+    encoded = base64.urlsafe_b64encode(
+        json.dumps({"exp": time.time() + 3600}).encode()
+    ).decode().rstrip("=")
+    (auth_dir / "auth.json").write_text(
+        json.dumps({"tokens": {"access_token": f"header.{encoded}.signature"}})
+    )
+    monkeypatch.setenv("CODEX_HOME", str(auth_dir))
+    assert module._access_token().startswith("header.")
+
+def test_classifier_broker_bounds_history_without_losing_routing_intent(monkeypatch):
+    """AUTO classification must fit the Jetson context without losing intent."""
+    broker_path = HERMES / "scripts" / "classifier_broker.py"
+    monkeypatch.setitem(sys.modules, "httpx", SimpleNamespace())
+    spec = importlib.util.spec_from_file_location("hermes_classifier_broker", broker_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    payload = {
+        "model": "qwen2.5:14b-instruct-q4_0",
+        "messages": [
+            {
+                "role": "system",
+                "content": "routing contract\n" + ("candidate policy " * 1000),
+            },
+            {
+                "role": "user",
+                "content": "Build and verify the Cassandra release safely.",
+            },
+            {
+                "role": "assistant",
+                "content": None,
+                "tool_calls": [
+                    {
+                        "id": "call_large",
+                        "type": "function",
+                        "function": {
+                            "name": "large_tool",
+                            "arguments": '{"command":"' + ("x" * 5000) + '"}',
+                        },
+                    }
+                ],
+            },
+            {
+                "role": "tool",
+                "content": "unbounded test output " * 10000,
+                "tool_call_id": "call_large",
+            },
+            {
+                "role": "user",
+                "content": [
+                    {"type": "image_url", "image_url": {"url": "data:image/png;base64,private"}},
+                    {"type": "text", "text": "Turn this cat into a cute clown."},
+                ],
+            },
+        ],
+        "tools": [{"type": "function", "function": {"name": "large_tool"}}],
+        "tool_choice": "auto",
+        "parallel_tool_calls": True,
+        "response_format": {"type": "json_object"},
+    }
+
+    compacted = module.compact_payload(payload)
+    encoded = json.dumps(compacted)
+
+    assert compacted["model"] == payload["model"]
+    assert compacted["response_format"] == payload["response_format"]
+    assert "tools" not in compacted
+    assert "tool_choice" not in compacted
+    assert "parallel_tool_calls" not in compacted
+    assert all(message.get("role") != "tool" for message in compacted["messages"])
+    assert all("tool_call_id" not in message for message in compacted["messages"])
+    assert all("tool_calls" not in message for message in compacted["messages"])
+    assert "[tool evidence]" in encoded
+    assert "[assistant requested an external tool]" in encoded
+    assert "Build and verify the Cassandra release safely." in encoded
+    assert "Turn this cat into a cute clown." in encoded
+    assert "image attachment available to the selected worker" in encoded
+    assert "data:image/png;base64" not in encoded
+    assert "unbounded test output " * 100 not in encoded
+    assert len(encoded) < 18_000
+
+def test_switchyard_classifier_is_bounded_and_fails_open_once():
+    """A sick local judge must not hold chat through repeated long retries."""
+    config = tomllib.loads(
+        _documents(HERMES / "switchyard-configmap.yaml")[0]["data"]["routes.toml"]
+    )
+    classifier = config["llm_clients"]["classifier"]
+    assert classifier["base_url"] == "http://127.0.0.1:9008/v1"
+    assert classifier["max_retries"] == 0
+    for route in ("auto_fast", "auto_balanced"):
+        assert config["routes"][route]["recent_turn_window"] == 4
+    for route in ("auto_deep", "auto_maximum", "worker_auto_maximum"):
+        assert config["routes"][route]["recent_turn_window"] == 6
+
+    deployment = _documents(HERMES / "switchyard-deployment.yaml")[0]
+    containers = {
+        item["name"]: item
+        for item in deployment["spec"]["template"]["spec"]["containers"]
+    }
+    classifier_container = containers["classifier-broker"]
+    env = {item["name"]: item["value"] for item in classifier_container["env"]}
+    assert env["HERMES_CLASSIFIER_BROKER_READ_TIMEOUT"] == "60"
+    assert classifier_container["readinessProbe"]["httpGet"]["port"] == "classifier"

testing/tests/test_hermes_chat_voice.py

@@ -0,0 +1,265 @@
+"""Hermes chat voice contracts."""
+
+from __future__ import annotations
+
+from test_hermes_chat_support import (
+    HERMES,
+    ROOT,
+    SimpleNamespace,
+    VAULT,
+    _documents,
+    importlib,
+    sys,
+    yaml,
+)
+
+
+def test_webui_recovers_auth_and_labels_session_scoped_controls():
+    dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-webui").read_text()
+
+    assert "res.status===401||res.status===403" in dockerfile
+    assert "window.location.assign('/oauth2/start?rd='" in dockerfile
+    assert "childrenExpanded?'▾ ':'▸ '" in dockerfile
+    assert "'atlas/auto/maximum': 'Automatic · Maximum'" in dockerfile
+    assert "hermes-webui-telegram-project-patch.py" in dockerfile
+    telegram_project_patch = (
+        ROOT / "dockerfiles" / "hermes-webui-telegram-project-patch.py"
+    ).read_text()
+    assert "TELEGRAM_PROJECT_NAME = 'Telegram'" in telegram_project_patch
+    assert "session_key.startswith('telegram-topic-')" in telegram_project_patch
+
+    router = (ROOT / "dockerfiles" / "hermes-webui-router.js").read_text()
+    assert "'atlas/auto/fast':'AUTO · Fast'" in router
+    assert "'atlas/manual/codex/sol':'Codex · SOL'" in router
+    assert "'atlas/manual/claude/opus':'Claude · Opus'" in router
+    assert "watchModelOptions('modelSelect')" in router
+    assert "watchModelOptions('settingsModel')" in router
+
+def test_chat_voice_uses_private_jetson_services_and_shared_auto_route():
+    statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
+    containers = statefulset["spec"]["template"]["spec"]["containers"]
+    webui = next(item for item in containers if item["name"] == "webui")
+    env = {item["name"]: item["value"] for item in webui["env"]}
+
+    assert env["HERMES_STT_URL"] == (
+        "http://hermes-stt.hermes.svc.cluster.local:9000/v1/audio/transcriptions"
+    )
+    assert env["HERMES_WEBUI_ATLAS_TTS_URL"] == (
+        "http://hermes-tts.hermes.svc.cluster.local:9001/v1/audio/speech"
+    )
+    assert "hermes_stt_client.py" in env["HERMES_LOCAL_STT_COMMAND"]
+
+    configmap = _documents(HERMES / "chat-configmap.yaml")[0]
+    config = yaml.safe_load(configmap["data"]["config.yaml"])
+    assert config["stt"] == {
+        "enabled": True,
+        "provider": "local_command",
+        "local": {"model": "small", "language": "auto"},
+    }
+
+    dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-webui").read_text()
+    assert "hermes-webui-atlas-patch.py" in dockerfile
+    assert "hermes-webui-atlas-voice.js" in dockerfile
+    voice_script = (ROOT / "dockerfiles" / "hermes-webui-atlas-voice.js").read_text()
+    assert "/api/transcribe/capability" in voice_script
+    assert "/api/transcribe" in voice_script
+    assert "/api/tts" in voice_script
+    assert "speakResponse(generation)" in voice_script
+    assert "window._splitForTTS(text,280)" in voice_script
+    assert "pending=fetchSpeech(chunks[index+1])" in voice_script
+    assert "restartSoon(token,450)" in voice_script
+    assert "constraints.voiceIsolation=true" in voice_script
+    assert "highpass.frequency.value=140" in voice_script
+    assert "Math.max(0.04,noiseFloor*2.4+0.006)" in voice_script
+    assert "while(preRoll.length>3) preRoll.shift()" in voice_script
+
+    stt_server = (ROOT / "dockerfiles" / "hermes-jetson-stt-server.py").read_text()
+    assert "def _repetitive_token" in stt_server
+    assert "compression_ratio_threshold=2.0" in stt_server
+    assert "no_speech_threshold=0.5" in stt_server
+
+def test_voice_transcript_filter_removes_fan_hallucinations(monkeypatch):
+    server_path = ROOT / "dockerfiles" / "hermes-jetson-stt-server.py"
+    spec = importlib.util.spec_from_file_location("hermes_jetson_stt_server", server_path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    monkeypatch.setitem(sys.modules, "cgi", SimpleNamespace())
+    monkeypatch.setitem(
+        sys.modules,
+        "torch",
+        SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: False)),
+    )
+    monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace())
+    spec.loader.exec_module(module)
+
+    result = {
+        "segments": [
+            {
+                "text": " ththththththththth Testing.",
+                "no_speech_prob": 0.12,
+                "avg_logprob": -0.2,
+            },
+            {
+                "text": " background hum",
+                "no_speech_prob": 0.82,
+                "avg_logprob": -0.9,
+            },
+        ]
+    }
+
+    assert module._clean_transcript(result) == "Testing."
+
+def test_voice_models_are_baked_and_runtime_has_no_public_egress():
+    stt_dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-jetson-stt").read_text()
+    tts_dockerfile = (ROOT / "dockerfiles" / "Dockerfile.hermes-jetson-tts").read_text()
+    assert "ADD --checksum=sha256:aff26ae4" in stt_dockerfile
+    assert "ADD --checksum=sha256:9ecf7799" in stt_dockerfile
+    assert "--chmod=0444" in stt_dockerfile
+    assert "chmod 0555 /opt/models /opt/models/whisper" in stt_dockerfile
+    assert "HERMES_STT_CACHE=/opt/models/whisper" in stt_dockerfile
+    assert "ADD --checksum=sha256:4cabf7c3" in tts_dockerfile
+    assert "ADD --checksum=sha256:db42b97d" in tts_dockerfile
+    assert tts_dockerfile.count("--chmod=0444") == 6
+    assert "chmod 0555 /opt/models /opt/models/piper" in tts_dockerfile
+    assert "HERMES_TTS_CACHE=/opt/models/piper" in tts_dockerfile
+    tts_server = (ROOT / "dockerfiles" / "hermes-jetson-tts-server.py").read_text()
+    assert "download_voice" not in tts_server
+    assert "baked Piper voice is missing" in tts_server
+    assert "session_options.intra_op_num_threads = ONNX_THREADS" in tts_server
+
+    policies = _documents(HERMES / "networkpolicy.yaml")
+    voice_policy = next(
+        item for item in policies if item["metadata"]["name"] == "hermes-private-voice"
+    )
+    assert voice_policy["spec"]["policyTypes"] == ["Ingress", "Egress"]
+    assert not any(
+        "ipBlock" in destination
+        for rule in voice_policy["spec"]["egress"]
+        for destination in rule.get("to", [])
+    )
+
+def test_voice_workloads_have_deliberate_xavier_placement():
+    documents = _documents(HERMES / "voice-deployment.yaml")
+    deployments = {
+        item["metadata"]["name"]: item
+        for item in documents
+        if item["kind"] == "Deployment"
+    }
+    stt = deployments["hermes-stt"]["spec"]["template"]["spec"]
+    tts = deployments["hermes-tts"]["spec"]["template"]["spec"]
+
+    assert "@sha256:" in stt["containers"][0]["image"]
+    assert "@sha256:" in tts["containers"][0]["image"]
+    assert stt["nodeSelector"] == {"kubernetes.io/hostname": "titan-21"}
+    assert tts["nodeSelector"] == {"kubernetes.io/hostname": "titan-21"}
+    assert stt["automountServiceAccountToken"] is False
+    assert tts["automountServiceAccountToken"] is False
+    assert stt["enableServiceLinks"] is False
+    assert tts["enableServiceLinks"] is False
+    assert stt["runtimeClassName"] == "nvidia"
+    assert stt["securityContext"]["supplementalGroups"] == [44]
+    stt_resources = stt["containers"][0]["resources"]
+    stt_env = {
+        item["name"]: item["value"] for item in stt["containers"][0]["env"]
+    }
+    assert stt_env["NVIDIA_DRIVER_CAPABILITIES"] == "compute,utility"
+    assert stt_resources["requests"]["nvidia.com/gpu.shared"] == 1
+    assert stt_resources["limits"]["nvidia.com/gpu.shared"] == 1
+    assert "nvidia.com/gpu.shared" not in tts["containers"][0]["resources"]["requests"]
+    tts_env = {
+        item["name"]: item["value"] for item in tts["containers"][0]["env"]
+    }
+    assert tts_env["HERMES_TTS_VOICE"] == "en_US-lessac-medium"
+    assert tts_env["HERMES_TTS_ONNX_THREADS"] == "2"
+    assert tts["containers"][0]["resources"]["limits"]["cpu"] == "4"
+    assert all("hostPath" not in volume for volume in stt["volumes"])
+    assert all("hostPath" not in volume for volume in tts["volumes"])
+
+def test_chat_image_generation_uses_private_owner_broker():
+    """Family pods get image bytes without receiving the owner's OAuth file."""
+    configmap = _documents(HERMES / "chat-configmap.yaml")[0]
+    assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"]
+    assert "local FLUX waits" in configmap["data"]["SOUL.md"]
+    assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"]
+    assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"]
+    assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"]
+    assert "`MEDIA:` path" in configmap["data"]["SOUL.md"]
+    assert "`image_edit_latest`" in configmap["data"]["SOUL.md"]
+    assert "Preserve the most recently selected image lane" in configmap["data"]["SOUL.md"]
+    config = yaml.safe_load(configmap["data"]["config.yaml"])
+    assert config["image_gen"] == {
+        "provider": "atlas-broker",
+        "model": "atlas-image-auto-high",
+    }
+    assert config["plugins"]["enabled"] == ["atlas-broker", "auto-router"]
+
+    statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
+    pod = statefulset["spec"]["template"]["spec"]
+    hermes = next(item for item in pod["containers"] if item["name"] == "hermes")
+    mounts = {item["name"]: item for item in hermes["volumeMounts"]}
+    assert mounts["image-plugin"]["mountPath"] == (
+        "/opt/hermes/plugins/image_gen/atlas-broker"
+    )
+    assert mounts["runtime-access"]["mountPath"] == "/runtime-access"
+    assert "provider-auth" not in mounts
+    assert not any(mount["name"] == "home" and "agent" in str(mount) for mount in hermes["volumeMounts"])
+    env = {item["name"]: item["value"] for item in hermes["env"]}
+    assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.")
+
+    plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text()
+    assert '"local": "flux-2-klein-4b-local"' in plugin
+    assert '"hosted": "gpt-image-2-high"' in plugin
+    assert 'name="image_generate_local"' in plugin
+    assert 'name="image_generate_hosted"' in plugin
+    assert '"name": "image_edit_latest"' in plugin
+    assert '"name": "image_edit_latest_local"' in plugin
+    assert '"name": "image_edit_latest_hosted"' in plugin
+    assert "def _latest_generated_image" in plugin
+    assert "newest MEDIA: path from the conversation" in plugin
+    assert 'candidate.upper().startswith("MEDIA:")' in plugin
+    assert "override=True" not in plugin
+
+    agent = _documents(HERMES / "agent-deployment.yaml")[0]
+    containers = agent["spec"]["template"]["spec"]["containers"]
+    broker = next(item for item in containers if item["name"] == "image-broker")
+    assert broker["ports"] == [
+        {"name": "image-broker", "containerPort": 9002, "protocol": "TCP"}
+    ]
+    assert broker["securityContext"]["readOnlyRootFilesystem"] is True
+    assert broker["securityContext"]["runAsNonRoot"] is True
+
+    services = _documents(HERMES / "service.yaml")
+    service = next(
+        item for item in services if item["metadata"]["name"] == "hermes-image-broker"
+    )
+    assert service["spec"]["selector"] == {"app": "hermes-agent"}
+
+    oauth_store = _documents(HERMES / "oauth-session-store.yaml")
+    redis = next(item for item in oauth_store if item["kind"] == "Deployment")
+    assert redis["spec"]["strategy"]["type"] == "Recreate"
+    assert "--appendonly" in redis["spec"]["template"]["spec"]["containers"][0]["args"]
+
+    policies = _documents(HERMES / "networkpolicy.yaml")
+    agent_policy = next(
+        item for item in policies if item["metadata"]["name"] == "hermes-agent-isolation"
+    )
+    broker_ingress = next(
+        rule
+        for rule in agent_policy["spec"]["ingress"]
+        if {port["port"] for port in rule["ports"]} == {9002, 9003}
+    )
+    assert broker_ingress["from"][0]["podSelector"]["matchLabels"] == {
+        "app": "hermes-chat-tenant"
+    }
+    vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text()
+    assert (
+        '"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram '
+        'hermes/developer-keycloak hermes/developer-gitea '
+        'hermes/developer-harbor hermes/developer-jenkins '
+        'hermes/developer-ssh"'
+        in vault_policy
+    )
+    assert (
+        'write_policy_and_role "hermes-node-ssh" "hermes" '
+        '"hermes-node-ssh-access"' in vault_policy
+    )

testing/tests/test_hermes_cli_dispatch.py

@@ -0,0 +1,420 @@
+"""Board scanning, claiming, orphan recovery, and replay guards."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    Path,
+    SimpleNamespace,
+    _completed_result,
+    json,
+    lanes,
+    nullcontext,
+    pytest,
+    sys,
+)
+
+
+def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch):
+    task = SimpleNamespace(id="t_auto", assignee=None, status="ready")
+    assigned = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    def assign_task(_conn, task_id, profile):
+        assigned.append((task_id, profile))
+        task.assignee = profile
+        return True
+
+    fake_db = SimpleNamespace(
+        list_boards=lambda include_archived=False: [{"slug": "cassandra"}],
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        recompute_ready=lambda _conn: None,
+        list_tasks=lambda _conn: [task],
+        assign_task=assign_task,
+        get_task=lambda _conn, _task_id: task,
+        claim_task=lambda _conn, _task_id, **_kwargs: task,
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+
+    assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")]
+    assert assigned == [("t_auto", "cli-auto")]
+
+
+def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes(monkeypatch, capsys):
+    class CorruptBoardError(Exception):
+        pass
+
+    task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready")
+
+    class Connection:
+        def close(self):
+            return None
+
+    def connect(*, board):
+        if board == "cassandra":
+            raise CorruptBoardError("integrity_check failed")
+        return Connection()
+
+    fake_db = SimpleNamespace(
+        KanbanDbCorruptError=CorruptBoardError,
+        list_boards=lambda include_archived=False: [
+            {"slug": "cassandra"},
+            {"slug": "healthy"},
+        ],
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=connect,
+        recompute_ready=lambda _conn: None,
+        list_tasks=lambda _conn: [task],
+        claim_task=lambda _conn, _task_id, **_kwargs: task,
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+    lanes.BOARD_CORRUPTION_ERRORS.clear()
+
+    assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")]
+    assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err
+
+
+def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys):
+    task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready")
+
+    class Connection:
+        def __init__(self, board):
+            self.board = board
+
+        def close(self):
+            return None
+
+    def recompute_ready(connection):
+        if connection.board == "cassandra":
+            raise lanes.sqlite3.OperationalError("disk I/O error")
+
+    fake_db = SimpleNamespace(
+        list_boards=lambda include_archived=False: [
+            {"slug": "cassandra"},
+            {"slug": "healthy"},
+        ],
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(board),
+        recompute_ready=recompute_ready,
+        list_tasks=lambda _conn: [task],
+        claim_task=lambda _conn, _task_id, **_kwargs: task,
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+    lanes.BOARD_CORRUPTION_ERRORS.clear()
+
+    assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")]
+    error = capsys.readouterr().err
+    assert "temporarily skipping Kanban board 'cassandra'" in error
+    assert "storage OperationalError: disk I/O error" in error
+
+
+def test_board_call_retries_storage_faults_on_fresh_connections():
+    connections = []
+
+    class Connection:
+        def __init__(self):
+            self.closed = False
+
+        def close(self):
+            self.closed = True
+
+    def connect(*, board):
+        assert board == "cassandra"
+        connection = Connection()
+        connections.append(connection)
+        return connection
+
+    attempts = []
+
+    def operation(_connection):
+        attempts.append(1)
+        if len(attempts) < 3:
+            raise lanes.sqlite3.OperationalError("disk I/O error")
+        return "healthy"
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=connect,
+    )
+    lanes.BOARD_CORRUPTION_ERRORS.clear()
+
+    assert lanes._board_call(fake_db, "cassandra", operation) == "healthy"
+    assert len(connections) == 3
+    assert all(connection.closed for connection in connections)
+
+
+def test_accepted_result_survives_failed_finalization_and_replays_exact_run(
+    tmp_path: Path,
+    monkeypatch,
+):
+    state_root = tmp_path / "cli-lanes"
+    monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
+    task = SimpleNamespace(
+        id="t_terminal",
+        status="running",
+        result=None,
+        current_run_id=41,
+        assignee="cli-auto",
+        max_runtime_seconds=60,
+    )
+    completions = []
+    comments = []
+    permit_completion = {"value": False}
+
+    class Connection:
+        def close(self):
+            return None
+
+    def complete_task(_conn, _task_id, **kwargs):
+        completions.append(kwargs)
+        if not permit_completion["value"]:
+            return False
+        task.status = "done"
+        task.result = kwargs["result"]
+        task.completed_run_id = kwargs["expected_run_id"]
+        task.current_run_id = None
+        return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
+        _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_terminal"),
+        set_branch_name=lambda *_args: None,
+        set_workspace_path=lambda *_args: None,
+        build_worker_context=lambda *_args: "Finish and verify the objective.",
+        heartbeat_worker=lambda *_args, **_kwargs: True,
+        add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
+        complete_task=complete_task,
+        block_task=lambda *_args, **_kwargs: pytest.fail(
+            "an accepted journaled result must not be converted into a block"
+        ),
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+    monkeypatch.setattr(
+        lanes,
+        "select_route",
+        lambda *_args, **_kwargs: lanes.Route(
+            "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
+        ),
+    )
+    structured = {
+        "status": "completed",
+        "summary": "Verified exact terminal result.",
+        "changed_files": [],
+        "tests_run": ["pytest: passed"],
+        "artifacts": [],
+        "findings": [],
+        "blockers": [],
+    }
+    monkeypatch.setattr(
+        lanes,
+        "run_provider",
+        lambda *_args, **_kwargs: lanes.ProcessResult(0, "", structured, False),
+    )
+
+    lanes.execute_claim("cassandra", "t_terminal")
+
+    candidates = list((state_root / "cassandra").glob("*.candidate-*.json"))
+    terminals = list((state_root / "cassandra").glob("*.terminal.pending.json"))
+    assert len(candidates) == 1
+    assert json.loads(candidates[0].read_text())["structured"] == structured
+    assert len(terminals) == 1
+    assert json.loads(terminals[0].read_text())["kanban_state"] == "pending"
+    assert task.status == "running"
+    assert any("durably journaled" in body for body in comments)
+
+    permit_completion["value"] = True
+    assert lanes.recover_pending_finalizations() == 1
+    assert task.status == "done"
+    assert len(completions) == 2
+    assert not terminals[0].exists()
+    committed = list(
+        (state_root / "cassandra").glob("*.terminal.committed.json")
+    )
+    assert len(committed) == 1
+    terminal = json.loads(committed[0].read_text())
+    assert terminal["kanban_state"] == "committed"
+    assert completions[-1]["result"] == terminal["result"]
+
+
+def test_restart_does_not_reclaim_an_exact_run_awaiting_finalization(
+    tmp_path: Path,
+    monkeypatch,
+):
+    state_root = tmp_path / "cli-lanes"
+    monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
+    state_file = lanes.state_path("cassandra", "t_terminal")
+    terminal_path, _record = lanes._write_terminal_record(
+        state_file,
+        board="cassandra",
+        task_id="t_terminal",
+        run_id=8,
+        structured={
+            "status": "completed",
+            "summary": "done",
+            "changed_files": [],
+            "tests_run": [],
+            "artifacts": [],
+            "findings": [],
+            "blockers": [],
+        },
+        summary="done",
+        metadata={},
+    )
+    assert terminal_path.exists()
+    task = SimpleNamespace(
+        id="t_terminal",
+        status="running",
+        assignee="cli-auto",
+        current_run_id=8,
+    )
+    reclaimed = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    fake_db = SimpleNamespace(
+        list_boards=lambda include_archived=False: [{"slug": "cassandra"}],
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        list_tasks=lambda _conn: [task],
+        reclaim_task=lambda *_args, **_kwargs: reclaimed.append(True),
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+    monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
+
+    lanes.recover_orphans()
+
+    assert reclaimed == []
+
+
+def test_orphan_recovery_passes_the_scanned_run_as_atomic_reclaim_guard(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    tasks = [
+        SimpleNamespace(
+            id="t_guarded",
+            status="running",
+            assignee="cli-auto",
+            current_run_id=91,
+        ),
+        SimpleNamespace(
+            id="t_no_run",
+            status="running",
+            assignee="cli-auto",
+            current_run_id=None,
+        ),
+    ]
+    reclaimed = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    fake_db = SimpleNamespace(
+        list_boards=lambda include_archived=False: [{"slug": "cassandra"}],
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        list_tasks=lambda _conn: tasks,
+        reclaim_task=lambda _conn, task_id, **kwargs: (
+            reclaimed.append((task_id, kwargs)) or True
+        ),
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+    monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
+
+    lanes.recover_orphans()
+
+    assert reclaimed == [
+        (
+            "t_guarded",
+            {
+                "reason": "direct CLI lane restarted; provider session will resume",
+                "expected_run_id": 91,
+            },
+        )
+    ]
+
+
+def test_prepared_evidence_without_pending_still_pins_the_exact_run(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    pending, record = lanes._write_terminal_record(
+        lanes.state_path("cassandra", "t_prepared_pin"),
+        board="cassandra",
+        task_id="t_prepared_pin",
+        run_id=14,
+        structured=_completed_result("accepted and prepared"),
+        summary="accepted and prepared",
+        metadata={},
+    )
+    identity = lanes._terminal_identity(pending)
+    assert identity is not None
+    lanes._persist_prepared_evidence(identity, record)
+    pending.unlink()
+
+    assert lanes._has_pending_finalization("cassandra", "t_prepared_pin", 14) is True
+
+
+def test_terminal_replay_never_crosses_into_a_replacement_run(
+    tmp_path: Path,
+    monkeypatch,
+):
+    state_root = tmp_path / "cli-lanes"
+    monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
+    state_file = lanes.state_path("cassandra", "t_terminal")
+    terminal_path, _record = lanes._write_terminal_record(
+        state_file,
+        board="cassandra",
+        task_id="t_terminal",
+        run_id=8,
+        structured={
+            "status": "completed",
+            "summary": "old run",
+            "changed_files": [],
+            "tests_run": [],
+            "artifacts": [],
+            "findings": [],
+            "blockers": [],
+        },
+        summary="old run",
+        metadata={},
+    )
+    replacement = SimpleNamespace(
+        id="t_terminal",
+        status="running",
+        result=None,
+        current_run_id=9,
+    )
+    completions = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: replacement,
+        complete_task=lambda *_args, **_kwargs: completions.append(True),
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+
+    assert lanes.recover_pending_finalizations() == 0
+    assert completions == []
+    assert not terminal_path.exists()
+    conflicts = list((state_root / "cassandra").glob("*.terminal.conflict-*.json"))
+    assert len(conflicts) == 1
+    conflict = json.loads(conflicts[0].read_text(encoding="utf-8"))
+    assert conflict["result"] == _record["result"]
+    assert conflict["kanban_state"] == "conflict"

testing/tests/test_hermes_cli_durability.py

@@ -0,0 +1,361 @@
+"""Prepared evidence and filesystem durability boundaries."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    Path,
+    SimpleNamespace,
+    _completed_result,
+    errno,
+    json,
+    lanes,
+    nullcontext,
+    os,
+    pytest,
+    stat,
+    sys,
+    threading,
+)
+
+
+def test_unknown_db_completion_outcome_replays_from_prepared_evidence(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path, record = lanes._write_terminal_record(
+        lanes.state_path("cassandra", "t_unknown_db_outcome"),
+        board="cassandra",
+        task_id="t_unknown_db_outcome",
+        run_id=35,
+        structured=_completed_result("DB committed before transport error"),
+        summary="DB committed before transport error",
+        metadata={},
+    )
+    task = SimpleNamespace(
+        id="t_unknown_db_outcome",
+        status="running",
+        result=None,
+        current_run_id=35,
+        completed_run_id=None,
+        assignee="cli-auto",
+    )
+    fail_once = {"value": True}
+
+    class Connection:
+        def close(self):
+            return None
+
+    def complete_task(_conn, _task_id, **kwargs):
+        task.status = "done"
+        task.result = kwargs["result"]
+        task.completed_run_id = kwargs["expected_run_id"]
+        task.current_run_id = None
+        if fail_once["value"]:
+            fail_once["value"] = False
+            raise RuntimeError("transport failed after DB commit")
+        return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        complete_task=complete_task,
+    )
+
+    with pytest.raises(RuntimeError, match="after DB commit"):
+        lanes._finalize_terminal_record(fake_db, path, record)
+
+    assert task.status == "done"
+    assert path.exists()
+    assert len(list(path.parent.glob("*.terminal.prepared-*.json"))) == 1
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+    assert lanes.recover_pending_finalizations() == 1
+    assert not path.exists()
+    assert list(path.parent.glob("*.terminal.prepared-*.json")) == []
+    committed = list(path.parent.glob("*.terminal.committed.json"))
+    assert len(committed) == 1
+    assert json.loads(committed[0].read_text())["result"] == record["result"]
+
+
+def test_pending_retirement_fsync_failure_keeps_db_winner_committed(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path, record = lanes._write_terminal_record(
+        lanes.state_path("cassandra", "t_retire_fsync"),
+        board="cassandra",
+        task_id="t_retire_fsync",
+        run_id=33,
+        structured=_completed_result("retirement fsync winner"),
+        summary="retirement fsync winner",
+        metadata={},
+    )
+    task = SimpleNamespace(
+        id="t_retire_fsync",
+        status="running",
+        result=None,
+        current_run_id=33,
+        assignee="cli-auto",
+    )
+
+    class Connection:
+        def close(self):
+            return None
+
+    def complete_task(_conn, _task_id, **kwargs):
+        task.status = "done"
+        task.result = kwargs["result"]
+        task.current_run_id = None
+        return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        complete_task=complete_task,
+    )
+    real_fsync = lanes.os.fsync
+    failed = {"value": False}
+
+    def fail_pending_retirement_fsync(descriptor):
+        if (
+            stat.S_ISDIR(os.fstat(descriptor).st_mode)
+            and not path.exists()
+            and list(path.parent.glob("*.terminal.committed.json"))
+            and not failed["value"]
+        ):
+            failed["value"] = True
+            raise OSError(errno.ENOSPC, "pending retirement fsync failed")
+        real_fsync(descriptor)
+
+    monkeypatch.setattr(lanes.os, "fsync", fail_pending_retirement_fsync)
+
+    assert lanes._finalize_terminal_record(fake_db, path, record) == "committed"
+    assert failed["value"] is True
+    assert task.status == "done"
+    assert task.result == record["result"]
+    assert not path.exists()
+    assert len(list(path.parent.glob("*.terminal.committed.json"))) == 1
+    assert list(path.parent.glob("*.terminal.prepared-*.json")) == []
+
+
+def test_committed_first_writer_is_never_overwritten_by_a_db_winner(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    state_file = lanes.state_path("cassandra", "t_committed_collision")
+    pending, winner = lanes._write_terminal_record(
+        state_file,
+        board="cassandra",
+        task_id="t_committed_collision",
+        run_id=34,
+        structured=_completed_result("DB winner"),
+        summary="DB winner",
+        metadata={"writer": "db"},
+    )
+    committed_path = lanes._terminal_path(state_file, 34, "committed")
+    first_writer = {
+        "board": "cassandra",
+        "task_id": "t_committed_collision",
+        "expected_run_id": 34,
+        "result": json.dumps(_completed_result("evidence first writer"), sort_keys=True),
+        "summary": "evidence first writer",
+        "metadata": {"writer": "evidence"},
+        "kanban_state": "committed",
+        "recorded_at": lanes.utc_now(),
+    }
+    lanes.atomic_json(committed_path, first_writer)
+    task = SimpleNamespace(
+        id="t_committed_collision",
+        status="running",
+        result=None,
+        current_run_id=34,
+        assignee="cli-auto",
+    )
+
+    class Connection:
+        def close(self):
+            return None
+
+    def complete_task(_conn, _task_id, **kwargs):
+        task.status = "done"
+        task.result = kwargs["result"]
+        task.current_run_id = None
+        return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        complete_task=complete_task,
+    )
+
+    assert lanes._finalize_terminal_record(fake_db, pending, winner) == "committed"
+    assert task.result == winner["result"]
+    assert json.loads(committed_path.read_text())["result"] == first_writer["result"]
+    conflicts = list(pending.parent.glob("*.terminal.conflict-*.json"))
+    assert len(conflicts) == 1
+    assert json.loads(conflicts[0].read_text())["result"] == winner["result"]
+    assert not pending.exists()
+
+
+def test_concurrent_duplicate_terminal_finalizers_are_idempotent(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path, record = lanes._write_terminal_record(
+        lanes.state_path("cassandra", "t_duplicate"),
+        board="cassandra",
+        task_id="t_duplicate",
+        run_id=29,
+        structured=_completed_result("same accepted result"),
+        summary="same accepted result",
+        metadata={},
+    )
+    task_state = {
+        "status": "running",
+        "result": None,
+        "current_run_id": 29,
+        "completed_run_id": None,
+    }
+    state_lock = threading.Lock()
+    readers = threading.Barrier(2)
+    completions = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    def get_task(_conn, _task_id):
+        with state_lock:
+            snapshot = SimpleNamespace(
+                id="t_duplicate",
+                assignee="cli-auto",
+                **task_state,
+            )
+        if snapshot.status == "running":
+            readers.wait(timeout=5)
+        return snapshot
+
+    def complete_task(_conn, _task_id, **kwargs):
+        with state_lock:
+            if task_state["status"] != "running":
+                return False
+            task_state.update(
+                    status="done",
+                    result=kwargs["result"],
+                    current_run_id=None,
+                    completed_run_id=kwargs["expected_run_id"],
+                )
+            completions.append(kwargs["result"])
+            return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=get_task,
+        complete_task=complete_task,
+    )
+    outcomes = []
+    errors = []
+
+    def finalize():
+        try:
+            outcomes.append(lanes._finalize_terminal_record(fake_db, path, record))
+        except Exception as error:  # pragma: no cover - assertion reports detail
+            errors.append(error)
+
+    workers = [threading.Thread(target=finalize) for _index in range(2)]
+    for worker in workers:
+        worker.start()
+    for worker in workers:
+        worker.join(timeout=10)
+
+    assert all(not worker.is_alive() for worker in workers)
+    assert errors == []
+    assert outcomes == ["committed", "committed"]
+    assert len(completions) == 1
+    assert not path.exists()
+    assert len(list(path.parent.glob("*.terminal.committed.json"))) == 1
+    assert list(path.parent.glob("*.terminal.conflict-*.json")) == []
+    assert list(path.parent.glob("*.terminal.prepared-*.json")) == []
+
+
+def test_recovery_uses_first_durable_prepared_result_as_db_writer(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    state_file = lanes.state_path("cassandra", "t_prepared_order")
+    pending, first = lanes._write_terminal_record(
+        state_file,
+        board="cassandra",
+        task_id="t_prepared_order",
+        run_id=30,
+        structured=_completed_result("first durable result"),
+        summary="first durable result",
+        metadata={"writer": "first"},
+    )
+    identity = lanes._terminal_identity(pending)
+    assert identity is not None
+    first_prepared = lanes._persist_prepared_evidence(identity, first)
+    os.utime(first_prepared, ns=(100, 100))
+    second_structured = _completed_result("second durable result")
+    second = {
+        "board": "cassandra",
+        "task_id": "t_prepared_order",
+        "expected_run_id": 30,
+        "result": json.dumps(second_structured, sort_keys=True),
+        "summary": "second durable result",
+        "metadata": {"writer": "second"},
+        "kanban_state": "pending",
+        "recorded_at": lanes.utc_now(),
+    }
+    second_prepared = lanes._persist_prepared_evidence(identity, second)
+    os.utime(second_prepared, ns=(200, 200))
+    lanes.atomic_json(pending, second)
+    task = SimpleNamespace(
+        id="t_prepared_order",
+        status="running",
+        result=None,
+        current_run_id=30,
+        assignee="cli-auto",
+    )
+    completions = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    def complete_task(_conn, _task_id, **kwargs):
+        if task.status != "running":
+            return False
+        task.status = "done"
+        task.result = kwargs["result"]
+        task.current_run_id = None
+        completions.append(kwargs["result"])
+        return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        complete_task=complete_task,
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+
+    assert lanes.recover_pending_finalizations() == 1
+    assert completions == [first["result"]]
+    assert task.result == first["result"]
+    assert not pending.exists()
+    assert list(pending.parent.glob("*.terminal.prepared-*.json")) == []
+    committed = list(pending.parent.glob("*.terminal.committed.json"))
+    conflicts = list(pending.parent.glob("*.terminal.conflict-*.json"))
+    assert len(committed) == 1
+    assert len(conflicts) == 1
+    assert json.loads(committed[0].read_text())["result"] == first["result"]
+    assert json.loads(conflicts[0].read_text())["result"] == second["result"]

testing/tests/test_hermes_cli_fallback.py

@@ -0,0 +1,381 @@
+"""Provider fallback, workspace, and command contracts."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    Path,
+    SimpleNamespace,
+    _completed_result,
+    json,
+    lanes,
+    nullcontext,
+    pytest,
+    sys,
+)
+
+
+def test_goal_card_continues_after_local_judge_rejects_progress(
+    tmp_path: Path,
+    monkeypatch,
+):
+    task = SimpleNamespace(
+        id="t_goal",
+        status="running",
+        result=None,
+        current_run_id=12,
+        assignee="cli-auto",
+        max_runtime_seconds=300,
+        goal_mode=True,
+        goal_max_turns=3,
+    )
+    calls = []
+    comments = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
+        _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"),
+        set_branch_name=lambda *_args: None,
+        set_workspace_path=lambda *_args: None,
+        build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.",
+        heartbeat_worker=lambda *_args, **_kwargs: True,
+        add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
+        complete_task=lambda *_args, **kwargs: (
+            calls.append(("complete", kwargs)) or True
+        ),
+        block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)),
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    claude_low = lanes.Route(
+        "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, ()
+    )
+    codex_low = lanes.Route(
+        "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, ()
+    )
+    codex_xhigh = lanes.Route(
+        "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, ()
+    )
+    route_calls = []
+
+    def select_route(_prompt, assignee, **kwargs):
+        route_calls.append((assignee, kwargs))
+        if assignee == "cli-codex-low":
+            return codex_low
+        if len(route_calls) == 1:
+            return claude_low
+        return codex_xhigh
+
+    monkeypatch.setattr(lanes, "select_route", select_route)
+    monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None)
+    reports = [
+        lanes.ProcessResult(1, "authentication expired", None, True),
+        lanes.ProcessResult(
+            0,
+            "first turn",
+            {
+                "status": "completed",
+                "summary": "Focused tests passed.",
+                "changed_files": ["src/a.py"],
+                "tests_run": ["pytest focused: passed"],
+                "artifacts": [],
+                "findings": [],
+                "blockers": [],
+            },
+            False,
+        ),
+        lanes.ProcessResult(
+            0,
+            "second turn",
+            {
+                "status": "completed",
+                "summary": "Full tests passed; commit pushed and remote HEAD verified.",
+                "changed_files": ["src/a.py"],
+                "tests_run": ["pytest full: passed"],
+                "artifacts": [],
+                "findings": [],
+                "blockers": [],
+            },
+            False,
+        ),
+    ]
+    monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
+    verdicts = iter(
+        [
+            (False, "commit, push, and remote verification are missing"),
+            (True, "all explicit acceptance criteria have evidence"),
+        ]
+    )
+    judge_contexts = []
+
+    def judge_goal_completion(objective, *_args, **_kwargs):
+        judge_contexts.append(objective)
+        return next(verdicts)
+
+    monkeypatch.setattr(
+        lanes.cli_lane_goal,
+        "judge_goal_completion",
+        judge_goal_completion,
+    )
+
+    lanes.execute_claim("cassandra", "t_goal")
+
+    assert calls[0][0] == "complete"
+    assert calls[0][1]["metadata"]["goal_turn"] == 2
+    assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments)
+    assert any("Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments)
+    assert route_calls[2][1]["exclude_provider"] == "claude"
+    assert "prior rejected reports" in judge_contexts[1]
+    assert "commit, push, and remote verification are missing" in judge_contexts[1]
+    candidates = sorted(
+        (lanes.STATE_ROOT / "cassandra").glob("t_goal.run-12.candidate-*.json")
+    )
+    assert len(candidates) == 2
+    assert json.loads(candidates[0].read_text())["structured"]["summary"] == (
+        "Focused tests passed."
+    )
+    assert json.loads(candidates[1].read_text())["structured"]["summary"].startswith(
+        "Full tests passed"
+    )
+    assert reports == []
+
+
+def test_capacity_fallback_preserves_first_claude_structured_response(
+    tmp_path: Path,
+    monkeypatch,
+):
+    task = SimpleNamespace(
+        id="t_fallback",
+        status="running",
+        result=None,
+        current_run_id=14,
+        assignee="cli-auto",
+        max_runtime_seconds=60,
+    )
+
+    class Connection:
+        def close(self):
+            return None
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
+        _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_fallback"),
+        set_branch_name=lambda *_args: None,
+        set_workspace_path=lambda *_args: None,
+        build_worker_context=lambda *_args: "Complete with a fallback if needed.",
+        heartbeat_worker=lambda *_args, **_kwargs: True,
+        add_comment=lambda *_args: None,
+        complete_task=lambda _conn, _task_id, **kwargs: (
+            setattr(task, "status", "done") or setattr(task, "result", kwargs["result"]) or True
+        ),
+        block_task=lambda *_args, **_kwargs: pytest.fail("fallback should complete"),
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    claude = lanes.Route(
+        "claude", "claude-fable-5", "high", "claude-high", "test", "test", 1, ()
+    )
+    codex = lanes.Route(
+        "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, ()
+    )
+    monkeypatch.setattr(
+        lanes,
+        "select_route",
+        lambda _prompt, assignee, **_kwargs: codex
+        if assignee == "cli-codex-high"
+        else claude,
+    )
+    reports = [
+        lanes.ProcessResult(
+            1,
+            "subscription capacity exhausted",
+            {
+                **_completed_result("Claude preserved evidence"),
+                "status": "incomplete",
+                "blockers": ["subscription capacity exhausted"],
+            },
+            True,
+        ),
+        lanes.ProcessResult(0, "", _completed_result("Codex completed"), False),
+    ]
+    monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
+
+    lanes.execute_claim("cassandra", "t_fallback")
+
+    candidates = sorted(
+        (lanes.STATE_ROOT / "cassandra").glob("t_fallback.run-14.candidate-*.json")
+    )
+    assert len(candidates) == 2
+    first, second = [json.loads(path.read_text()) for path in candidates]
+    assert (first["provider"], first["structured"]["summary"]) == (
+        "claude",
+        "Claude preserved evidence",
+    )
+    assert (second["provider"], second["structured"]["summary"]) == (
+        "codex",
+        "Codex completed",
+    )
+    assert task.status == "done"
+
+
+def test_workspace_preparation_failure_durably_blocks_the_claim(tmp_path: Path, monkeypatch):
+    task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto")
+    calls = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
+        _resolve_worktree_workspace=lambda _task, board: (_ for _ in ()).throw(
+            ValueError("not a Git repository")
+        ),
+        block_task=lambda *_args, **kwargs: calls.append(kwargs),
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+    monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json")
+
+    lanes.execute_claim("cassandra", "t_bad_worktree")
+
+    assert calls[0]["kind"] == "capability"
+    assert calls[0]["expected_run_id"] == 7
+    assert "not a Git repository" in calls[0]["reason"]
+
+
+def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path):
+    workspace = tmp_path / "workspace"
+    workspace.mkdir()
+    inside = workspace / "report.json"
+    outside = tmp_path / "auth.json"
+    inside.write_text("{}\n", encoding="utf-8")
+    outside.write_text("secret\n", encoding="utf-8")
+
+    assert lanes.workspace_artifacts(
+        workspace,
+        ["report.json", str(outside), "missing.json"],
+    ) == [str(inside)]
+
+
+def test_restart_provider_change_includes_explicit_workspace_handoff(tmp_path: Path, monkeypatch):
+    task = SimpleNamespace(
+        id="t_resume",
+        status="running",
+        result=None,
+        current_run_id=9,
+        assignee="cli-auto",
+        max_runtime_seconds=60,
+    )
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    state_file = lanes.state_path("cassandra", "t_resume")
+    state_file.parent.mkdir(parents=True)
+    state_file.write_text(
+        json.dumps({"current_route": {"provider": "claude"}}),
+        encoding="utf-8",
+    )
+    (tmp_path / "worker.log").write_text("prior provider evidence", encoding="utf-8")
+    prompts = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
+        _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_resume"),
+        set_branch_name=lambda *_args: None,
+        set_workspace_path=lambda *_args: None,
+        build_worker_context=lambda *_args: "resume objective",
+        add_comment=lambda *_args: None,
+        complete_task=lambda *_args, **_kwargs: True,
+        block_task=lambda *_args, **_kwargs: None,
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+    monkeypatch.setattr(
+        lanes,
+        "select_route",
+        lambda *_args, **_kwargs: lanes.Route(
+            "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
+        ),
+    )
+    monkeypatch.setattr(
+        lanes,
+        "git_handoff",
+        lambda _workspace, output: f"HANDOFF:{output}",
+    )
+    monkeypatch.setattr(
+        lanes,
+        "run_provider",
+        lambda _route, prompt, *_args, **_kwargs: (
+            prompts.append(prompt)
+            or lanes.ProcessResult(
+                0,
+                "",
+                {
+                    "status": "completed",
+                    "summary": "done",
+                    "changed_files": [],
+                    "tests_run": [],
+                    "artifacts": [],
+                    "findings": [],
+                    "blockers": [],
+                },
+                False,
+            )
+        ),
+    )
+
+    lanes.execute_claim("cassandra", "t_resume")
+
+    assert "HANDOFF:prior provider evidence" in prompts[0]
+
+
+def test_provider_commands_are_structured_unattended_and_capped(tmp_path: Path):
+    route = lanes.Route("codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, ())
+    command = lanes._codex_command(route, "Work.", tmp_path, {}, tmp_path / "result.json")
+    assert "--dangerously-bypass-approvals-and-sandbox" in command
+    assert "--json" in command
+    assert "--output-schema" in command
+    assert 'model_reasoning_effort="xhigh"' in command
+
+    claude_state = {"claude_session_id": "13864642-2985-4f91-bef5-53f145f878e8"}
+    claude = lanes._claude_command(
+        lanes.Route("claude", "claude-opus-5", "xhigh", "claude-xhigh", "jetson", "vote", 1, ()),
+        "Review.",
+        claude_state,
+        False,
+    )
+    assert "--dangerously-skip-permissions" in claude
+    assert "--output-format" in claude and "stream-json" in claude
+    assert "--json-schema" in claude
+    assert "--disallowedTools" in claude
+    assert "Bash(kubectl apply *)" not in claude
+    assert "Bash(flux reconcile *)" not in claude
+    assert "max" not in claude
+
+
+def test_worker_contract_separates_review_findings_from_task_blockers(tmp_path: Path):
+    prompt = lanes.build_prompt("Review the change.", tmp_path)
+
+    assert "put defects and risks in findings" in prompt
+    assert "blockers array must be empty whenever status is completed" in prompt
+    assert "findings" in lanes.RESULT_SCHEMA["properties"]
+    assert set(lanes.RESULT_SCHEMA["required"]) == set(
+        lanes.RESULT_SCHEMA["properties"]
+    )
+    assert "assigned task itself" in lanes.RESULT_SCHEMA["properties"]["blockers"]["description"]

testing/tests/test_hermes_cli_finalization.py

@@ -0,0 +1,411 @@
+"""Terminal-result validation and exact-run finalization."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    Path,
+    SimpleNamespace,
+    _completed_result,
+    errno,
+    json,
+    lanes,
+    nullcontext,
+    pytest,
+    sys,
+)
+
+
+def test_terminal_recovery_can_complete_the_exact_latest_ended_run(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    state_file = lanes.state_path("cassandra", "t_ended")
+    pending, _record = lanes._write_terminal_record(
+        state_file,
+        board="cassandra",
+        task_id="t_ended",
+        run_id=17,
+        structured=_completed_result("accepted before legacy block"),
+        summary="accepted before legacy block",
+        metadata={},
+    )
+    task = SimpleNamespace(
+        id="t_ended",
+        status="blocked",
+        result=None,
+        current_run_id=None,
+        assignee="cli-auto",
+    )
+    guards = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    def complete_task(_conn, _task_id, **kwargs):
+        guards.append(kwargs)
+        task.status = "done"
+        task.result = kwargs["result"]
+        return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        complete_task=complete_task,
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+
+    assert lanes.recover_pending_finalizations() == 1
+    assert guards[0]["replay_ended_run_id"] == 17
+    assert "expected_run_id" not in guards[0]
+    assert task.status == "done"
+    assert not pending.exists()
+
+
+def test_terminal_recovery_completes_an_unchanged_triage_exact_run(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    pending, record = lanes._write_terminal_record(
+        lanes.state_path("cassandra", "t_triage_ended"),
+        board="cassandra",
+        task_id="t_triage_ended",
+        run_id=18,
+        structured=_completed_result("accepted before loop-breaker triage"),
+        summary="accepted before loop-breaker triage",
+        metadata={},
+    )
+    task = SimpleNamespace(
+        id="t_triage_ended",
+        status="triage",
+        result=None,
+        current_run_id=None,
+        completed_run_id=None,
+        assignee="cli-auto",
+    )
+    guards = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    def complete_task(_conn, _task_id, **kwargs):
+        guards.append(kwargs)
+        assert kwargs == {
+            "result": record["result"],
+            "summary": record["summary"],
+            "metadata": {},
+            "replay_ended_run_id": 18,
+        }
+        task.status = "done"
+        task.result = kwargs["result"]
+        task.completed_run_id = 18
+        return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        complete_task=complete_task,
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+
+    assert lanes.recover_pending_finalizations() == 1
+    assert len(guards) == 1
+    assert task.status == "done"
+    assert task.completed_run_id == 18
+    assert not pending.exists()
+    assert lanes.recover_pending_finalizations() == 0
+    assert len(guards) == 1
+
+
+def test_complete_exception_after_journal_is_replayable_and_never_blocks(
+    tmp_path: Path,
+    monkeypatch,
+):
+    state_root = tmp_path / "cli-lanes"
+    monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
+    task = SimpleNamespace(
+        id="t_crash",
+        status="running",
+        result=None,
+        current_run_id=51,
+        assignee="cli-auto",
+        max_runtime_seconds=60,
+    )
+    completion_raises = {"value": True}
+    blocks = []
+    comments = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    def complete_task(_conn, _task_id, **kwargs):
+        if completion_raises["value"]:
+            raise RuntimeError("crash between journal and commit")
+        task.status = "done"
+        task.result = kwargs["result"]
+        task.current_run_id = None
+        return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
+        _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_crash"),
+        set_branch_name=lambda *_args: None,
+        set_workspace_path=lambda *_args: None,
+        build_worker_context=lambda *_args: "Finish safely.",
+        heartbeat_worker=lambda *_args, **_kwargs: True,
+        add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
+        complete_task=complete_task,
+        block_task=lambda *_args, **kwargs: blocks.append(kwargs),
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+    monkeypatch.setattr(
+        lanes,
+        "select_route",
+        lambda *_args, **_kwargs: lanes.Route(
+            "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, ()
+        ),
+    )
+    monkeypatch.setattr(
+        lanes,
+        "run_provider",
+        lambda *_args, **_kwargs: lanes.ProcessResult(
+            0, "", _completed_result("crash-safe result"), False
+        ),
+    )
+
+    lanes.execute_claim("cassandra", "t_crash")
+
+    pending = list((state_root / "cassandra").glob("*.terminal.pending.json"))
+    assert len(pending) == 1
+    assert blocks == []
+    assert any("RuntimeError" in body for body in comments)
+
+    completion_raises["value"] = False
+    assert lanes.recover_pending_finalizations() == 1
+    assert task.status == "done"
+    assert not pending[0].exists()
+
+
+def test_terminal_replace_then_directory_fsync_enospc_replays_after_restart(
+    tmp_path: Path,
+    monkeypatch,
+):
+    state_root = tmp_path / "cli-lanes"
+    monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
+    task = SimpleNamespace(
+        id="t_enospc",
+        status="running",
+        result=None,
+        current_run_id=52,
+        assignee="cli-auto",
+        max_runtime_seconds=60,
+    )
+    blocks = []
+    comments = []
+    completions = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    def complete_task(_conn, _task_id, **kwargs):
+        completions.append(kwargs)
+        task.status = "done"
+        task.result = kwargs["result"]
+        task.current_run_id = None
+        return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
+        _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_enospc"),
+        set_branch_name=lambda *_args: None,
+        set_workspace_path=lambda *_args: None,
+        build_worker_context=lambda *_args: "Finish without losing the result.",
+        heartbeat_worker=lambda *_args, **_kwargs: True,
+        add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
+        complete_task=complete_task,
+        block_task=lambda *_args, **kwargs: blocks.append(kwargs),
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+    monkeypatch.setattr(
+        lanes,
+        "select_route",
+        lambda *_args, **_kwargs: lanes.Route(
+            "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, ()
+        ),
+    )
+    monkeypatch.setattr(
+        lanes,
+        "run_provider",
+        lambda *_args, **_kwargs: lanes.ProcessResult(
+            0, "", _completed_result("persisted before ENOSPC"), False
+        ),
+    )
+    real_replace = lanes.os.replace
+    real_fsync_directory = lanes._fsync_directory
+    terminal_replaced = {"value": False}
+    fail_once = {"value": True}
+
+    def replace_then_mark(source, destination):
+        real_replace(source, destination)
+        if str(destination).endswith(".terminal.pending.json"):
+            terminal_replaced["value"] = True
+
+    def fail_after_terminal_replace(directory):
+        if terminal_replaced["value"] and fail_once["value"]:
+            fail_once["value"] = False
+            raise OSError(errno.ENOSPC, "no space after terminal rename")
+        real_fsync_directory(directory)
+
+    monkeypatch.setattr(lanes.os, "replace", replace_then_mark)
+    monkeypatch.setattr(lanes, "_fsync_directory", fail_after_terminal_replace)
+
+    lanes.execute_claim("cassandra", "t_enospc")
+
+    pending = list((state_root / "cassandra").glob("*.terminal.pending.json"))
+    assert terminal_replaced["value"] is True
+    assert fail_once["value"] is False
+    assert len(pending) == 1
+    assert blocks == []
+    assert completions == []
+    assert task.status == "running"
+    assert any("terminal replayable=True" in body for body in comments)
+
+    # A restarted runner sees the exact pending journal and completes the run.
+    assert lanes.recover_pending_finalizations() == 1
+    assert len(completions) == 1
+    assert task.status == "done"
+    assert not pending[0].exists()
+    committed = list((state_root / "cassandra").glob("*.terminal.committed.json"))
+    assert len(committed) == 1
+
+
+def test_terminal_payload_cannot_select_a_different_board(
+    tmp_path: Path,
+    monkeypatch,
+):
+    state_root = tmp_path / "cli-lanes"
+    monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
+    path = lanes._terminal_path(lanes.state_path("alpha", "t_alpha"), 3)
+    lanes.atomic_json(
+        path,
+        {
+            "board": "beta",
+            "task_id": "t_beta",
+            "expected_run_id": 9,
+            "result": json.dumps(_completed_result()),
+            "summary": "forged",
+            "metadata": {},
+            "kanban_state": "pending",
+        },
+    )
+    alpha_task = SimpleNamespace(
+        id="t_alpha", status="running", current_run_id=3, assignee="cli-auto"
+    )
+    opened = []
+    reclaimed = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: (opened.append(board) or Connection()),
+        get_task=lambda _conn, task_id: alpha_task if task_id == "t_alpha" else None,
+        reclaim_task=lambda _conn, task_id, **_kwargs: (reclaimed.append(task_id) or True),
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+
+    assert lanes.recover_pending_finalizations() == 0
+
+    assert opened == ["alpha"]
+    assert reclaimed == ["t_alpha"]
+    assert not path.exists()
+    quarantined = list((state_root / "alpha/quarantine").glob("*.quarantine"))
+    assert len(quarantined) == 1
+    assert quarantined[0].stat().st_mode & 0o777 == 0o600
+
+
+def test_terminal_finalize_rereads_journal_after_a_path_swap(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    state_file = lanes.state_path("alpha", "t_alpha")
+    path, safe_record = lanes._write_terminal_record(
+        state_file,
+        board="alpha",
+        task_id="t_alpha",
+        run_id=4,
+        structured=_completed_result("safe"),
+        summary="safe",
+        metadata={},
+    )
+    forged = dict(safe_record)
+    forged.update({"board": "beta", "task_id": "t_beta", "expected_run_id": 7})
+    lanes.atomic_json(path, forged)
+    opened = []
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: (opened.append(board) or pytest.fail("must not open DB")),
+    )
+
+    assert lanes._finalize_terminal_record(fake_db, path, safe_record) == "foreign"
+    assert opened == []
+
+
+@pytest.mark.parametrize("payload", [b"", b'{"board":"cassandra"'])
+def test_malformed_exact_run_journal_is_quarantined_and_reclaimed(
+    tmp_path: Path,
+    monkeypatch,
+    payload: bytes,
+):
+    state_root = tmp_path / "cli-lanes"
+    monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
+    path = lanes._terminal_path(lanes.state_path("cassandra", "t_partial"), 12)
+    path.parent.mkdir(parents=True)
+    path.write_bytes(payload)
+    task = SimpleNamespace(
+        id="t_partial", status="running", current_run_id=12, assignee="cli-auto"
+    )
+    reclaimed = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        reclaim_task=lambda *_args, **kwargs: (reclaimed.append(kwargs) or True),
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+
+    assert lanes.recover_pending_finalizations() == 0
+
+    assert reclaimed == [{
+        "reason": (
+            "terminal journal recovery failed (malformed-payload); "
+            "exact run may retry"
+        ),
+        "expected_run_id": 12,
+    }]
+    assert not path.exists()
+    assert lanes._has_pending_finalization("cassandra", "t_partial", 12) is False
+    quarantined = list((state_root / "cassandra/quarantine").glob("*.quarantine"))
+    assert len(quarantined) == 1
+    assert quarantined[0].stat().st_mode & 0o777 == 0o600

testing/tests/test_hermes_cli_finalization_authority.py

@@ -0,0 +1,53 @@
+"""Post-transaction authority classification for exact terminal results."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    SimpleNamespace,
+    lanes,
+    nullcontext,
+    pytest,
+)
+
+
+@pytest.mark.parametrize(
+    ("status", "current_run_id", "expected"),
+    [
+        ("running", 17, "pending"),
+        ("running", 18, "stale"),
+        ("ready", None, "stale"),
+        ("blocked", None, "stale"),
+        ("triage", None, "stale"),
+    ],
+)
+def test_failed_completion_is_pending_only_for_the_exact_live_run(
+    status: str,
+    current_run_id: int | None,
+    expected: str,
+):
+    task = SimpleNamespace(
+        id="t_authority",
+        status=status,
+        current_run_id=current_run_id,
+        completed_run_id=None,
+        result=None,
+    )
+
+    class Connection:
+        def close(self):
+            return None
+
+    database = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        complete_task=lambda *_args, **_kwargs: False,
+    )
+
+    outcome = lanes._finalize_document_db(
+        database,
+        lanes.TerminalIdentity("cassandra", "t_authority", 17, "pending"),
+        {"result": "accepted", "summary": "accepted", "metadata": {}},
+    )
+
+    assert outcome == expected

testing/tests/test_hermes_cli_goal_execution.py

@@ -0,0 +1,151 @@
+"""Structured goal execution and local completion judging."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    Path,
+    SimpleNamespace,
+    lanes,
+    nullcontext,
+    pytest,
+    sys,
+)
+
+
+def test_artifact_gc_is_interval_gated(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    monkeypatch.setattr(lanes, "LAST_ARTIFACT_GC", 0.0)
+    calls = []
+    monkeypatch.setattr(
+        lanes, "gc_lane_artifacts", lambda **kwargs: (calls.append(kwargs) or 0)
+    )
+
+    assert lanes.maybe_gc_lane_artifacts(now=1000.0) == 0
+    assert lanes.maybe_gc_lane_artifacts(now=1001.0) == 0
+    assert len(calls) == 1
+
+
+@pytest.mark.parametrize(
+    ("result", "expected_action"),
+    [
+        (lanes.ProcessResult(0, "plain text only", None, False), "block"),
+        (
+            lanes.ProcessResult(
+                0,
+                "",
+                {
+                    "status": "completed",
+                    "summary": "done",
+                    "changed_files": ["src/a.py"],
+                        "tests_run": ["pytest -q"],
+                        "artifacts": ["reports/result.json"],
+                        "findings": [],
+                        "blockers": [],
+                },
+                False,
+            ),
+            "complete",
+        ),
+        (
+            lanes.ProcessResult(
+                0,
+                "",
+                {
+                    "status": "completed",
+                    "summary": "The full test suite is still running.",
+                    "changed_files": ["src/a.py"],
+                        "tests_run": ["pytest -q — in progress"],
+                        "artifacts": [],
+                        "findings": [],
+                        "blockers": [],
+                },
+                False,
+            ),
+            "block",
+        ),
+    ],
+)
+def test_claim_requires_structured_evidence_and_surfaces_artifacts(
+    tmp_path: Path,
+    monkeypatch,
+    result,
+    expected_action,
+):
+    task = SimpleNamespace(
+        id="t_worker",
+        status="running",
+        result=None,
+        current_run_id=4,
+        assignee="cli-auto",
+        max_runtime_seconds=60,
+    )
+    calls = []
+    heartbeats = []
+    connections = []
+    artifact = tmp_path / "reports/result.json"
+    artifact.parent.mkdir()
+    artifact.write_text("{}\n", encoding="utf-8")
+
+    class Connection:
+        def __init__(self):
+            self.closed = False
+
+        def close(self):
+            self.closed = True
+
+    def connect(*, board):
+        assert board == "cassandra"
+        connection = Connection()
+        connections.append(connection)
+        return connection
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=connect,
+        get_task=lambda _conn, _task_id: task,
+        worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
+        _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"),
+        set_branch_name=lambda *_args: None,
+        set_workspace_path=lambda *_args: None,
+        build_worker_context=lambda *_args: "bounded objective",
+        heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: (
+            heartbeats.append((note, expected_run_id)) or True
+        ),
+        add_comment=lambda *_args: None,
+        complete_task=lambda *_args, **kwargs: (
+            calls.append(("complete", kwargs)) or True
+        ),
+        block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)),
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    monkeypatch.setattr(
+        lanes,
+        "select_route",
+        lambda *_args, **_kwargs: lanes.Route(
+            "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
+        ),
+    )
+    def run_provider(*args, **_kwargs):
+        assert connections[0].closed
+        before_heartbeat = len(connections)
+        assert args[6]("working") is True
+        assert len(connections) == before_heartbeat + 1
+        assert connections[-1].closed
+        return result
+
+    monkeypatch.setattr(lanes, "run_provider", run_provider)
+
+    lanes.execute_claim("cassandra", "t_worker")
+
+    assert calls[0][0] == expected_action
+    assert heartbeats == [("working", 4)]
+    if expected_action == "complete":
+        assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)]
+        assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"]
+    else:
+        assert calls[0][1]["kind"] == "capability"
+    assert all(connection.closed for connection in connections)

testing/tests/test_hermes_cli_lane_routing.py

@@ -11,7 +11,7 @@ from pathlib import Path
 SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts"
 sys.path.insert(0, str(SCRIPTS))
 SCRIPT = SCRIPTS / "cli_lane_runner.py"
-SPEC = importlib.util.spec_from_file_location("cli_lane_routing", SCRIPT)
+SPEC = importlib.util.spec_from_file_location("cli_lane_runner_routing_test", SCRIPT)
 assert SPEC and SPEC.loader
 lanes = importlib.util.module_from_spec(SPEC)
 sys.modules[SPEC.name] = lanes

testing/tests/test_hermes_cli_provenance.py

@@ -0,0 +1,313 @@
+"""Run provenance and first-writer conflict behavior."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    Path,
+    SimpleNamespace,
+    _completed_result,
+    _install_terminal_recovery_db,
+    _pending_terminal_record,
+    errno,
+    json,
+    lanes,
+    nullcontext,
+    os,
+    pytest,
+    stat,
+    sys,
+)
+
+
+def test_legacy_valid_retirement_staging_is_promoted_without_reexecution(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    board = lanes.STATE_ROOT / "cassandra"
+    board.mkdir(parents=True)
+    staged = board / f".retire.{'d' * 32}.0"
+    record = _pending_terminal_record(
+        "cassandra", "t_legacy_staged", 56, "legacy staged result"
+    )
+    lanes.atomic_json(staged, record)
+    task = SimpleNamespace(
+        id="t_legacy_staged",
+        status="running",
+        current_run_id=56,
+        completed_run_id=None,
+        result=None,
+        assignee="cli-auto",
+    )
+    completions = []
+    reclaims = []
+    _install_terminal_recovery_db(monkeypatch, task, completions, reclaims)
+
+    assert lanes._has_pending_finalization(
+        "cassandra", "t_legacy_staged", 56
+    ) is True
+    assert lanes.recover_pending_finalizations() == 1
+    assert completions == [record["result"]]
+    assert reclaims == []
+    assert not staged.exists()
+    assert lanes.recover_pending_finalizations() == 0
+    assert completions == [record["result"]]
+
+
+def test_done_result_bytes_do_not_authorize_a_different_run_journal(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path, stale = lanes._write_terminal_record(
+        lanes.state_path("cassandra", "t_provenance"),
+        board="cassandra",
+        task_id="t_provenance",
+        run_id=1,
+        structured=_completed_result("same bytes"),
+        summary="same bytes",
+        metadata={},
+    )
+    task = SimpleNamespace(
+        id="t_provenance",
+        status="done",
+        current_run_id=None,
+        completed_run_id=2,
+        result=stale["result"],
+        assignee="cli-auto",
+    )
+
+    class Connection:
+        def close(self):
+            return None
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        complete_task=lambda *_args, **_kwargs: pytest.fail(
+            "a done task with different run provenance must not be completed"
+        ),
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+
+    assert lanes.recover_pending_finalizations() == 0
+    assert not path.exists()
+    assert not list(path.parent.glob("*.terminal.committed.json"))
+    conflicts = list(path.parent.glob("*.terminal.conflict-*.json"))
+    assert len(conflicts) == 1
+    conflict = json.loads(conflicts[0].read_text())
+    assert conflict["expected_run_id"] == 1
+    assert conflict["result"] == task.result
+
+
+@pytest.mark.parametrize(
+    "swap_point",
+    [
+        "during-complete",
+        "after-db-before-promote",
+        "before-committed-create",
+        "before-pending-retire",
+    ],
+)
+def test_terminal_first_writer_preserves_valid_replacement_conflicts(
+    tmp_path: Path,
+    monkeypatch,
+    swap_point: str,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path, old_record = lanes._write_terminal_record(
+        lanes.state_path("cassandra", "t_first_writer"),
+        board="cassandra",
+        task_id="t_first_writer",
+        run_id=27,
+        structured=_completed_result("first result"),
+        summary="first result",
+        metadata={"writer": "first"},
+    )
+    replacement_structured = _completed_result("replacement result")
+    replacement_record = {
+        "board": "cassandra",
+        "task_id": "t_first_writer",
+        "expected_run_id": 27,
+        "result": json.dumps(replacement_structured, sort_keys=True),
+        "summary": "replacement result",
+        "metadata": {"writer": "replacement"},
+        "kanban_state": "pending",
+        "recorded_at": lanes.utc_now(),
+    }
+    replacement = path.with_name(f"replacement-{swap_point}.tmp")
+    lanes.atomic_json(replacement, replacement_record)
+    swapped = {"value": False}
+
+    def swap_pending():
+        if not swapped["value"]:
+            os.replace(replacement, path)
+            swapped["value"] = True
+
+    task = SimpleNamespace(
+        id="t_first_writer",
+        status="running",
+        result=None,
+        current_run_id=27,
+        assignee="cli-auto",
+    )
+
+    class Connection:
+        def close(self):
+            return None
+
+    def complete_task(_conn, _task_id, **kwargs):
+        task.status = "done"
+        task.result = kwargs["result"]
+        task.current_run_id = None
+        if swap_point == "during-complete":
+            swap_pending()
+        return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        complete_task=complete_task,
+    )
+    if swap_point == "after-db-before-promote":
+        real_promote = lanes._promote_prepared_evidence
+
+        def swap_then_promote(*args, **kwargs):
+            swap_pending()
+            return real_promote(*args, **kwargs)
+
+        monkeypatch.setattr(lanes, "_promote_prepared_evidence", swap_then_promote)
+    elif swap_point == "before-committed-create":
+        real_create = lanes._write_json_noreplace
+
+        def swap_before_committed(path_arg, value):
+            if path_arg.name.endswith(".terminal.committed.json"):
+                swap_pending()
+            return real_create(path_arg, value)
+
+        monkeypatch.setattr(lanes, "_write_json_noreplace", swap_before_committed)
+    elif swap_point == "before-pending-retire":
+        real_retire = lanes._retire_snapshot
+
+        def swap_before_retire(path_arg, snapshot):
+            if path_arg == path:
+                swap_pending()
+            return real_retire(path_arg, snapshot)
+
+        monkeypatch.setattr(lanes, "_retire_snapshot", swap_before_retire)
+
+    assert lanes._finalize_terminal_record(fake_db, path, old_record) == "committed"
+
+    assert swapped["value"] is True
+    assert task.result == old_record["result"]
+    assert not path.exists()
+    committed = list(path.parent.glob("*.terminal.committed.json"))
+    conflicts = list(path.parent.glob("*.terminal.conflict-*.json"))
+    prepared = list(path.parent.glob("*.terminal.prepared-*.json"))
+    assert len(committed) == 1
+    assert len(conflicts) == 1
+    assert prepared == []
+    assert committed[0].stat().st_mode & 0o777 == 0o600
+    assert conflicts[0].stat().st_mode & 0o777 == 0o600
+    committed_document = json.loads(committed[0].read_text(encoding="utf-8"))
+    conflict_document = json.loads(conflicts[0].read_text(encoding="utf-8"))
+    assert committed_document["result"] == old_record["result"]
+    assert committed_document["kanban_state"] == "committed"
+    assert conflict_document["result"] == replacement_record["result"]
+    assert conflict_document["metadata"] == {"writer": "replacement"}
+    assert conflict_document["kanban_state"] == "conflict"
+
+
+def test_terminal_commit_directory_fsync_failure_recovers_from_prepared_evidence(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path, record = lanes._write_terminal_record(
+        lanes.state_path("cassandra", "t_post_db_fsync"),
+        board="cassandra",
+        task_id="t_post_db_fsync",
+        run_id=28,
+        structured=_completed_result("durable DB winner"),
+        summary="durable DB winner",
+        metadata={},
+    )
+    task = SimpleNamespace(
+        id="t_post_db_fsync",
+        status="running",
+        result=None,
+        current_run_id=28,
+        completed_run_id=None,
+        assignee="cli-auto",
+    )
+    replacement_structured = _completed_result("replacement after DB commit")
+    replacement_record = {
+        "board": "cassandra",
+        "task_id": "t_post_db_fsync",
+        "expected_run_id": 28,
+        "result": json.dumps(replacement_structured, sort_keys=True),
+        "summary": "replacement after DB commit",
+        "metadata": {"writer": "replacement"},
+        "kanban_state": "pending",
+        "recorded_at": lanes.utc_now(),
+    }
+    replacement = path.with_name("post-db-fsync-replacement.tmp")
+    lanes.atomic_json(replacement, replacement_record)
+
+    class Connection:
+        def close(self):
+            return None
+
+    def complete_task(_conn, _task_id, **kwargs):
+        task.status = "done"
+        task.result = kwargs["result"]
+        task.completed_run_id = kwargs["expected_run_id"]
+        task.current_run_id = None
+        os.replace(replacement, path)
+        return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        complete_task=complete_task,
+    )
+    real_fsync = lanes.os.fsync
+    failed = {"value": False}
+
+    def fail_committed_directory_fsync(descriptor):
+        if (
+            stat.S_ISDIR(os.fstat(descriptor).st_mode)
+            and list(path.parent.glob("*.terminal.committed.json"))
+            and not failed["value"]
+        ):
+            failed["value"] = True
+            raise OSError(errno.ENOSPC, "post-DB committed directory fsync failed")
+        real_fsync(descriptor)
+
+    monkeypatch.setattr(lanes.os, "fsync", fail_committed_directory_fsync)
+
+    with pytest.raises(OSError, match="post-DB committed"):
+        lanes._finalize_terminal_record(fake_db, path, record)
+
+    assert task.status == "done"
+    assert task.result == record["result"]
+    assert path.exists()
+    assert json.loads(path.read_text(encoding="utf-8"))["result"] == (
+        replacement_record["result"]
+    )
+    assert len(list(path.parent.glob("*.terminal.prepared-*.json"))) == 1
+
+    monkeypatch.setattr(lanes.os, "fsync", real_fsync)
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+    assert lanes.recover_pending_finalizations() == 1
+    assert not path.exists()
+    assert list(path.parent.glob("*.terminal.prepared-*.json")) == []
+    committed = list(path.parent.glob("*.terminal.committed.json"))
+    conflicts = list(path.parent.glob("*.terminal.conflict-*.json"))
+    assert len(committed) == 1
+    assert len(conflicts) == 1
+    assert json.loads(committed[0].read_text())["result"] == record["result"]
+    assert json.loads(conflicts[0].read_text())["result"] == replacement_record["result"]

testing/tests/test_hermes_cli_provider_sessions.py

@@ -0,0 +1,301 @@
+"""Codex, Claude, and worker-process session behavior."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    Path,
+    json,
+    lanes,
+    os,
+    signal,
+    sys,
+)
+
+
+def test_claude_session_is_reserved_before_first_process(tmp_path: Path, monkeypatch):
+    state: dict = {}
+    state_file = tmp_path / "state.json"
+    observed = {}
+
+    def fake_stream(command, **kwargs):
+        observed["command"] = command
+        observed["state"] = json.loads(state_file.read_text())
+        return lanes.ProcessResult(0, "", {"status": "completed"}, False)
+
+    monkeypatch.setattr(lanes, "stream_process", fake_stream)
+    route = lanes.Route("claude", "claude-opus-5", "high", "claude-high", "jetson", "vote", 1, ())
+
+    lanes.run_provider(
+        route,
+        "Do the work.",
+        tmp_path,
+        state,
+        state_file,
+        tmp_path / "worker.log",
+        lambda _note: True,
+        60,
+    )
+
+    reserved = observed["state"]["claude_session_id"]
+    assert reserved
+    assert ["--session-id", reserved] == observed["command"][-4:-2]
+
+
+def test_claude_recovery_uses_verified_error_state_machine(tmp_path: Path, monkeypatch):
+    state = {"claude_session_id": "d4f91f59-88cb-43d0-a64f-f3284688fe9e", "claude_started": True}
+    state_file = tmp_path / "state.json"
+    calls = []
+
+    def fake_stream(command, **kwargs):
+        calls.append(command)
+        if len(calls) == 1:
+            return lanes.ProcessResult(
+                1,
+                "No conversation found with session ID: d4f91f59-88cb-43d0-a64f-f3284688fe9e",
+                None,
+                False,
+            )
+        return lanes.ProcessResult(0, "", {"status": "completed"}, False)
+
+    monkeypatch.setattr(lanes, "stream_process", fake_stream)
+    route = lanes.Route("claude", "claude-opus-5", "high", "claude-high", "jetson", "vote", 1, ())
+    lanes.run_provider(route, "Continue.", tmp_path, state, state_file, tmp_path / "log", lambda _: True, 60)
+
+    assert "--resume" in calls[0]
+    assert "--session-id" in calls[1]
+    assert calls[0][calls[0].index("--resume") + 1] == calls[1][calls[1].index("--session-id") + 1]
+
+
+def test_claude_collision_never_double_starts_a_reserved_session(tmp_path: Path, monkeypatch):
+    state = {"claude_session_id": "24927cee-f3da-4bcf-b624-602c22860155"}
+    state_file = tmp_path / "state.json"
+    calls = []
+
+    def fake_stream(command, **kwargs):
+        calls.append(command)
+        if len(calls) == 1:
+            return lanes.ProcessResult(1, "Session ID already in use", None, False)
+        return lanes.ProcessResult(0, "", {"status": "completed"}, False)
+
+    monkeypatch.setattr(lanes, "stream_process", fake_stream)
+    route = lanes.Route("claude", "claude-opus-5", "high", "claude-high", "jetson", "vote", 1, ())
+    lanes.run_provider(route, "Continue.", tmp_path, state, state_file, tmp_path / "log", lambda _: True, 60)
+
+    assert "--session-id" in calls[0]
+    assert "--resume" in calls[1]
+    assert calls[0][calls[0].index("--session-id") + 1] == calls[1][calls[1].index("--resume") + 1]
+
+
+def test_codex_thread_started_is_persisted_before_completion(tmp_path: Path):
+    state: dict = {}
+    state_file = tmp_path / "state.json"
+
+    lanes._event_payload(
+        "codex",
+        json.dumps({"type": "thread.started", "thread_id": "thread-123"}),
+        state,
+        state_file,
+    )
+
+    assert json.loads(state_file.read_text())["codex_thread_id"] == "thread-123"
+
+
+def test_codex_missing_server_thread_restarts_fresh(tmp_path: Path, monkeypatch):
+    state = {"codex_thread_id": "failed-thread"}
+    state_file = tmp_path / "state.json"
+    calls = []
+
+    def fake_stream(command, **kwargs):
+        calls.append(command)
+        if len(calls) == 1:
+            return lanes.ProcessResult(
+                1,
+                "thread/resume failed: no rollout found for thread id failed-thread",
+                None,
+                False,
+            )
+        return lanes.ProcessResult(0, "", {"status": "completed"}, False)
+
+    monkeypatch.setattr(lanes, "stream_process", fake_stream)
+    route = lanes.Route(
+        "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
+    )
+
+    result = lanes.run_provider(
+        route,
+        "Continue.",
+        tmp_path,
+        state,
+        state_file,
+        tmp_path / "log",
+        lambda _: True,
+        60,
+    )
+
+    assert result.returncode == 0
+    assert "resume" in calls[0]
+    assert "resume" not in calls[1]
+    assert "codex_thread_id" not in json.loads(state_file.read_text())
+
+
+def test_codex_result_files_are_unique_and_prior_turns_are_retained(
+    tmp_path: Path,
+    monkeypatch,
+):
+    state = {"run_id": 17}
+    state_file = tmp_path / "task.json"
+    result_paths = []
+
+    def fake_stream(command, **_kwargs):
+        result_path = Path(command[command.index("-o") + 1])
+        result_path.write_text(
+            json.dumps(
+                {
+                    "status": "completed",
+                    "summary": f"turn {len(result_paths) + 1}",
+                    "changed_files": [],
+                    "tests_run": [],
+                    "artifacts": [],
+                    "findings": [],
+                    "blockers": [],
+                }
+            ),
+            encoding="utf-8",
+        )
+        result_paths.append(result_path)
+        return lanes.ProcessResult(0, "", None, False)
+
+    monkeypatch.setattr(lanes, "stream_process", fake_stream)
+    route = lanes.Route(
+        "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
+    )
+
+    first = lanes.run_provider(
+        route,
+        "First turn.",
+        tmp_path,
+        state,
+        state_file,
+        tmp_path / "worker.log",
+        lambda _note: True,
+        60,
+    )
+    second = lanes.run_provider(
+        route,
+        "Second turn.",
+        tmp_path,
+        state,
+        state_file,
+        tmp_path / "worker.log",
+        lambda _note: True,
+        60,
+    )
+
+    assert first.structured["summary"] == "turn 1"
+    assert second.structured["summary"] == "turn 2"
+    assert len(set(result_paths)) == 2
+    assert all(path.exists() for path in result_paths)
+    assert all(path.stat().st_mode & 0o777 == 0o600 for path in result_paths)
+
+
+def test_successful_process_text_cannot_masquerade_as_capacity_failure(tmp_path: Path):
+    result = lanes.stream_process(
+        [sys.executable, "-c", "print('authentication work completed')"],
+        provider="codex",
+        cwd=tmp_path,
+        env=dict(os.environ),
+        log_path=tmp_path / "worker.log",
+        state={},
+        state_file=tmp_path / "state.json",
+        heartbeat=lambda _note: True,
+        max_runtime=60,
+    )
+
+    assert result.returncode == 0
+    assert result.capacity_failure is False
+
+
+def test_worker_process_uses_isolated_process_session(monkeypatch, tmp_path: Path):
+    original_popen = lanes.subprocess.Popen
+    calls = []
+
+    def recording_popen(*args, **kwargs):
+        calls.append(kwargs)
+        return original_popen(*args, **kwargs)
+
+    monkeypatch.setattr(lanes.subprocess, "Popen", recording_popen)
+    result = lanes.stream_process(
+        [sys.executable, "-c", "print('done')"],
+        provider="codex",
+        cwd=tmp_path,
+        env=dict(os.environ),
+        log_path=tmp_path / "worker.log",
+        state={},
+        state_file=tmp_path / "state.json",
+        heartbeat=lambda _note: True,
+        max_runtime=60,
+    )
+
+    assert result.returncode == 0
+    assert calls[0]["start_new_session"] is True
+
+
+def test_worker_process_group_is_killed_after_leader_exits(monkeypatch):
+    class FinishedProcess:
+        pid = 4321
+
+        @staticmethod
+        def poll():
+            return 0
+
+    signals = []
+    monkeypatch.setattr(lanes.os, "killpg", lambda pid, sig: signals.append((pid, sig)))
+
+    lanes._terminate_worker_process(FinishedProcess())
+
+    assert signals == [(4321, signal.SIGKILL)]
+
+
+def test_worker_terminal_descendants_in_separate_groups_are_killed(monkeypatch):
+    class RunningProcess:
+        pid = 4321
+        running = True
+
+        @classmethod
+        def poll(cls):
+            return None if cls.running else 0
+
+        @classmethod
+        def wait(cls, timeout):
+            assert timeout == 10
+            cls.running = False
+            return 0
+
+    group_signals = []
+    process_signals = []
+    monkeypatch.setattr(
+        lanes,
+        "_descendant_processes",
+        lambda _pid: {5000: (5000, 12345)},
+    )
+    monkeypatch.setattr(lanes, "_process_identity_matches", lambda _pid, _start: True)
+    monkeypatch.setattr(
+        lanes.os,
+        "killpg",
+        lambda process_group, sig: group_signals.append((process_group, sig)),
+    )
+    monkeypatch.setattr(
+        lanes.os,
+        "kill",
+        lambda pid, sig: process_signals.append((pid, sig)),
+    )
+
+    lanes._terminate_worker_process(RunningProcess())
+
+    assert set(group_signals) == {
+        (4321, signal.SIGTERM),
+        (5000, signal.SIGTERM),
+        (4321, signal.SIGKILL),
+        (5000, signal.SIGKILL),
+    }
+    assert process_signals == [(5000, signal.SIGTERM), (5000, signal.SIGKILL)]

testing/tests/test_hermes_cli_quarantine.py

@@ -0,0 +1,375 @@
+"""Identity-safe quarantine and replacement races."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    Path,
+    SimpleNamespace,
+    _completed_result,
+    _install_terminal_recovery_db,
+    _pending_terminal_record,
+    hashlib,
+    json,
+    lanes,
+    nullcontext,
+    os,
+    pytest,
+    stat,
+    sys,
+)
+
+
+@pytest.mark.parametrize(
+    "structured",
+    [
+        {"status": "completed", "blockers": {}},
+        {**_completed_result(), "changed_files": "src/a.py"},
+        {**_completed_result(), "unexpected": True},
+        {**_completed_result(), "blockers": ["work remains"]},
+        {**_completed_result("tests are still running")},
+    ],
+)
+def test_terminal_record_requires_exact_completed_result_contract(structured):
+    record = {
+        "board": "cassandra",
+        "task_id": "t_schema",
+        "expected_run_id": 1,
+        "result": json.dumps(structured),
+        "summary": str(structured.get("summary") or "done"),
+        "metadata": {},
+        "kanban_state": "pending",
+    }
+
+    assert lanes._terminal_record_valid(record) is False
+
+
+def test_minimal_completed_result_with_mapping_blockers_quarantines_without_completion(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path = lanes._terminal_path(lanes.state_path("cassandra", "t_schema"), 6)
+    lanes.atomic_json(
+        path,
+        {
+            "board": "cassandra",
+            "task_id": "t_schema",
+            "expected_run_id": 6,
+            "result": json.dumps({"status": "completed", "blockers": {}}),
+            "summary": "done",
+            "metadata": {},
+            "kanban_state": "pending",
+        },
+    )
+    task = SimpleNamespace(
+        id="t_schema", status="running", current_run_id=6, assignee="cli-auto"
+    )
+    completed = []
+    reclaimed = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        complete_task=lambda *_args, **_kwargs: completed.append(True),
+        reclaim_task=lambda *_args, **_kwargs: (reclaimed.append(True) or True),
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+
+    assert lanes.recover_pending_finalizations() == 0
+    assert completed == []
+    assert reclaimed == [True]
+    assert not path.exists()
+    quarantined = list((path.parent / "quarantine").glob("*.quarantine"))
+    assert len(quarantined) == 1
+    assert quarantined[0].stat().st_mode & 0o777 == 0o600
+
+
+def test_quarantine_avoids_symlink_and_mode_collision_destinations(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path = lanes._terminal_path(lanes.state_path("cassandra", "t_collision"), 2)
+    path.parent.mkdir(parents=True)
+    path.write_bytes(b"invalid")
+    quarantine = path.parent / "quarantine"
+    quarantine.mkdir()
+    fingerprint = hashlib.sha256(b"invalid").hexdigest()[:16]
+    base = f"{path.name}.malformed-payload.{fingerprint}"
+    victim = tmp_path / "victim"
+    victim.write_text("unchanged", encoding="utf-8")
+    (quarantine / f"{base}.0.quarantine").symlink_to(victim)
+    collision = quarantine / f"{base}.1.quarantine"
+    collision.write_text("attacker collision", encoding="utf-8")
+    collision.chmod(0o644)
+    monkeypatch.setattr(
+        lanes.os,
+        "chmod",
+        lambda *_args, **_kwargs: pytest.fail("quarantine must not chmod foreign inodes"),
+    )
+
+    destination = lanes._quarantine_terminal(
+        path,
+        lanes._terminal_identity(path),
+        "malformed-payload",
+    )
+
+    assert not path.exists()
+    assert destination.name == f"{base}.2.quarantine"
+    assert destination.stat().st_mode & 0o777 == 0o600
+    diagnostic = json.loads(destination.read_text(encoding="utf-8"))
+    assert diagnostic["size"] == len(b"invalid")
+    assert diagnostic["sha256"] == hashlib.sha256(b"invalid").hexdigest()
+    assert diagnostic["source_kind"] == "regular"
+    assert victim.read_text(encoding="utf-8") == "unchanged"
+    assert collision.read_text(encoding="utf-8") == "attacker collision"
+    assert collision.stat().st_mode & 0o777 == 0o644
+
+
+def test_hardlinked_terminal_source_is_never_chmodded_or_copied_as_data(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path = lanes._terminal_path(lanes.state_path("cassandra", "t_hardlink"), 5)
+    path.parent.mkdir(parents=True)
+    foreign = tmp_path / "foreign"
+    foreign.write_text("foreign inode contents", encoding="utf-8")
+    foreign.chmod(0o644)
+    os.link(foreign, path)
+    monkeypatch.setattr(
+        lanes.os,
+        "chmod",
+        lambda *_args, **_kwargs: pytest.fail("hardlinked source must not be chmodded"),
+    )
+
+    destination = lanes._quarantine_terminal(
+        path,
+        lanes._terminal_identity(path),
+        "malformed-payload",
+    )
+
+    assert not path.exists()
+    assert foreign.read_text(encoding="utf-8") == "foreign inode contents"
+    assert foreign.stat().st_mode & 0o777 == 0o644
+    assert destination.stat().st_mode & 0o777 == 0o600
+    diagnostic = json.loads(destination.read_text(encoding="utf-8"))
+    assert diagnostic["source_kind"] == "hardlink"
+    assert "foreign inode contents" not in destination.read_text(encoding="utf-8")
+
+
+def test_quarantine_preserves_and_replays_an_atomic_replacement(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path = lanes._terminal_path(lanes.state_path("cassandra", "t_swap"), 21)
+    path.parent.mkdir(parents=True)
+    path.write_bytes(b"malformed")
+    identity = lanes._terminal_identity(path)
+    assert identity is not None
+    structured = _completed_result("replacement result")
+    replacement_record = {
+        "board": "cassandra",
+        "task_id": "t_swap",
+        "expected_run_id": 21,
+        "result": json.dumps(structured, sort_keys=True),
+        "summary": structured["summary"],
+        "metadata": {},
+        "kanban_state": "pending",
+        "recorded_at": lanes.utc_now(),
+    }
+    replacement = path.with_name("replacement.tmp")
+    replacement.write_text(
+        json.dumps(replacement_record, sort_keys=True),
+        encoding="utf-8",
+    )
+    replacement.chmod(0o600)
+    real_fsync = lanes.os.fsync
+    swapped = {"value": False}
+
+    def swap_on_quarantine_directory_fsync(descriptor):
+        descriptor_stat = os.fstat(descriptor)
+        if stat.S_ISDIR(descriptor_stat.st_mode) and not swapped["value"]:
+            os.replace(replacement, path)
+            swapped["value"] = True
+        real_fsync(descriptor)
+
+    monkeypatch.setattr(lanes.os, "fsync", swap_on_quarantine_directory_fsync)
+
+    lanes._quarantine_terminal(path, identity, "malformed-payload")
+
+    assert swapped["value"] is True
+    assert path.exists()
+    assert lanes._terminal_record_valid(
+        lanes._load_terminal_json(path, identity), identity
+    )
+
+    task = SimpleNamespace(
+        id="t_swap", status="running", current_run_id=21, result=None, assignee="cli-auto"
+    )
+
+    class Connection:
+        def close(self):
+            return None
+
+    def complete_task(_conn, _task_id, **kwargs):
+        task.status = "done"
+        task.result = kwargs["result"]
+        task.current_run_id = None
+        return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        complete_task=complete_task,
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+
+    assert lanes.recover_pending_finalizations() == 1
+    assert task.status == "done"
+    assert not path.exists()
+
+
+def test_recovery_quarantines_the_loaded_inode_not_a_before_open_replacement(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path = lanes._terminal_path(lanes.state_path("cassandra", "t_gap"), 51)
+    path.parent.mkdir(parents=True)
+    path.write_bytes(b"malformed-original")
+    replacement = path.with_name("replacement.valid")
+    lanes.atomic_json(
+        replacement,
+        _pending_terminal_record("cassandra", "t_gap", 51, "preserved replacement"),
+    )
+    task = SimpleNamespace(
+        id="t_gap",
+        status="running",
+        current_run_id=51,
+        result=None,
+        assignee="cli-auto",
+    )
+    completions = []
+    reclaims = []
+    _install_terminal_recovery_db(monkeypatch, task, completions, reclaims)
+    real_quarantine = lanes._quarantine_terminal
+    swapped = {"value": False}
+
+    def swap_before_quarantine_reopens(*args, **kwargs):
+        assert kwargs["snapshot"].prefix == b"malformed-original"
+        if not swapped["value"]:
+            os.replace(replacement, path)
+            swapped["value"] = True
+        return real_quarantine(*args, **kwargs)
+
+    monkeypatch.setattr(lanes, "_quarantine_terminal", swap_before_quarantine_reopens)
+
+    assert lanes.recover_pending_finalizations() == 1
+    assert swapped["value"] is True
+    assert task.status == "done"
+    assert len(completions) == 1
+    assert reclaims == []
+    assert not path.exists()
+    committed = next(path.parent.glob("*.terminal.committed.json"))
+    assert json.loads(committed.read_text())["result"] == completions[0]
+
+
+def test_recovery_replays_replacement_installed_after_quarantine_pins_source(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path = lanes._terminal_path(lanes.state_path("cassandra", "t_after_open"), 52)
+    path.parent.mkdir(parents=True)
+    path.write_bytes(b"malformed-original")
+    replacement = path.with_name("replacement.valid")
+    lanes.atomic_json(
+        replacement,
+        _pending_terminal_record(
+            "cassandra", "t_after_open", 52, "replacement after open"
+        ),
+    )
+    task = SimpleNamespace(
+        id="t_after_open",
+        status="running",
+        current_run_id=52,
+        result=None,
+        assignee="cli-auto",
+    )
+    completions = []
+    reclaims = []
+    _install_terminal_recovery_db(monkeypatch, task, completions, reclaims)
+    real_fsync = lanes.os.fsync
+    swapped = {"value": False}
+
+    def swap_after_quarantine_open(descriptor):
+        descriptor_stat = os.fstat(descriptor)
+        if stat.S_ISDIR(descriptor_stat.st_mode) and not swapped["value"]:
+            os.replace(replacement, path)
+            swapped["value"] = True
+        real_fsync(descriptor)
+
+    monkeypatch.setattr(lanes.os, "fsync", swap_after_quarantine_open)
+
+    assert lanes.recover_pending_finalizations() == 1
+    assert swapped["value"] is True
+    assert task.status == "done"
+    assert len(completions) == 1
+    assert reclaims == []
+    assert not path.exists()
+
+
+def test_recovery_survives_repeated_invalid_then_valid_replacements(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path = lanes._terminal_path(lanes.state_path("cassandra", "t_churn"), 53)
+    path.parent.mkdir(parents=True)
+    path.write_bytes(b"malformed-zero")
+    second = path.with_name("replacement.invalid")
+    second.write_bytes(b"malformed-one")
+    valid = path.with_name("replacement.valid")
+    lanes.atomic_json(
+        valid,
+        _pending_terminal_record("cassandra", "t_churn", 53, "valid after churn"),
+    )
+    task = SimpleNamespace(
+        id="t_churn",
+        status="running",
+        current_run_id=53,
+        result=None,
+        assignee="cli-auto",
+    )
+    completions = []
+    reclaims = []
+    _install_terminal_recovery_db(monkeypatch, task, completions, reclaims)
+    real_quarantine = lanes._quarantine_terminal
+    replacements = [second, valid]
+    pinned = []
+
+    def churn_before_quarantine(*args, **kwargs):
+        pinned.append(kwargs["snapshot"].prefix)
+        if replacements:
+            os.replace(replacements.pop(0), path)
+        return real_quarantine(*args, **kwargs)
+
+    monkeypatch.setattr(lanes, "_quarantine_terminal", churn_before_quarantine)
+
+    assert lanes.recover_pending_finalizations() == 1
+    assert pinned == [b"malformed-zero", b"malformed-one"]
+    assert replacements == []
+    assert task.status == "done"
+    assert len(completions) == 1
+    assert reclaims == []
+    assert not path.exists()
+    diagnostics = list((path.parent / "quarantine").glob("*.quarantine"))
+    assert len(diagnostics) == 2

testing/tests/test_hermes_cli_retention.py

@@ -0,0 +1,325 @@
+"""Bounded reads, artifact retention, and recovery cadence."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    Path,
+    SimpleNamespace,
+    _completed_result,
+    json,
+    lanes,
+    nullcontext,
+    os,
+    pytest,
+    sys,
+)
+
+
+def test_invalid_utf8_journal_quarantines_and_does_not_stop_later_replay(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    bad = lanes._terminal_path(lanes.state_path("cassandra", "a_bad_utf8"), 31)
+    bad.parent.mkdir(parents=True)
+    bad.write_bytes(b"\xff\xfe\x80not-json")
+    good, _record = lanes._write_terminal_record(
+        lanes.state_path("cassandra", "z_good"),
+        board="cassandra",
+        task_id="z_good",
+        run_id=32,
+        structured=_completed_result("valid after invalid UTF-8"),
+        summary="valid after invalid UTF-8",
+        metadata={},
+    )
+    tasks = {
+        "a_bad_utf8": SimpleNamespace(
+            id="a_bad_utf8",
+            status="running",
+            current_run_id=31,
+            result=None,
+            assignee="cli-auto",
+        ),
+        "z_good": SimpleNamespace(
+            id="z_good",
+            status="running",
+            current_run_id=32,
+            result=None,
+            assignee="cli-auto",
+        ),
+    }
+
+    class Connection:
+        def close(self):
+            return None
+
+    def reclaim_task(_conn, task_id, **_kwargs):
+        tasks[task_id].status = "ready"
+        tasks[task_id].current_run_id = None
+        return True
+
+    def complete_task(_conn, task_id, **kwargs):
+        tasks[task_id].status = "done"
+        tasks[task_id].result = kwargs["result"]
+        tasks[task_id].current_run_id = None
+        return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, task_id: tasks[task_id],
+        reclaim_task=reclaim_task,
+        complete_task=complete_task,
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+
+    assert lanes.recover_pending_finalizations() == 1
+    assert not bad.exists()
+    assert not good.exists()
+    assert tasks["a_bad_utf8"].status == "ready"
+    assert tasks["z_good"].status == "done"
+    diagnostic_path = next((bad.parent / "quarantine").glob("*.quarantine"))
+    diagnostic = json.loads(diagnostic_path.read_text(encoding="utf-8"))
+    assert diagnostic["size"] == len(b"\xff\xfe\x80not-json")
+    assert diagnostic_path.stat().st_size < 4096
+
+
+@pytest.mark.parametrize(
+    "size",
+    [lanes.MAX_TERMINAL_RECORD_BYTES + 1, 16 * 1024 * 1024],
+)
+def test_oversized_sparse_journal_has_bounded_read_and_quarantine(
+    tmp_path: Path,
+    monkeypatch,
+    size: int,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    huge = lanes._terminal_path(lanes.state_path("cassandra", "a_huge"), 41)
+    huge.parent.mkdir(parents=True)
+    with huge.open("wb") as stream:
+        stream.seek(size - 1)
+        stream.write(b"\0")
+    good, _record = lanes._write_terminal_record(
+        lanes.state_path("cassandra", "z_after_huge"),
+        board="cassandra",
+        task_id="z_after_huge",
+        run_id=42,
+        structured=_completed_result("valid after oversized journal"),
+        summary="valid after oversized journal",
+        metadata={},
+    )
+    tasks = {
+        "a_huge": SimpleNamespace(
+            id="a_huge",
+            status="running",
+            current_run_id=41,
+            result=None,
+            assignee="cli-auto",
+        ),
+        "z_after_huge": SimpleNamespace(
+            id="z_after_huge",
+            status="running",
+            current_run_id=42,
+            result=None,
+            assignee="cli-auto",
+        ),
+    }
+    reads = []
+    real_read_bounded = lanes._read_bounded
+
+    def observe_read_limit(descriptor, limit):
+        reads.append((os.fstat(descriptor).st_size, limit))
+        return real_read_bounded(descriptor, limit)
+
+    monkeypatch.setattr(lanes, "_read_bounded", observe_read_limit)
+
+    class Connection:
+        def close(self):
+            return None
+
+    def reclaim_task(_conn, task_id, **_kwargs):
+        tasks[task_id].status = "ready"
+        tasks[task_id].current_run_id = None
+        return True
+
+    def complete_task(_conn, task_id, **kwargs):
+        tasks[task_id].status = "done"
+        tasks[task_id].result = kwargs["result"]
+        tasks[task_id].current_run_id = None
+        return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, task_id: tasks[task_id],
+        reclaim_task=reclaim_task,
+        complete_task=complete_task,
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+
+    assert lanes.recover_pending_finalizations() == 1
+    assert not huge.exists()
+    assert not good.exists()
+    assert tasks["a_huge"].status == "ready"
+    assert tasks["z_after_huge"].status == "done"
+    diagnostics = list((huge.parent / "quarantine").glob("*.quarantine"))
+    assert len(diagnostics) == 1
+    assert diagnostics[0].stat().st_size < 4096
+    diagnostic = json.loads(diagnostics[0].read_text(encoding="utf-8"))
+    assert diagnostic["size"] == size
+    assert diagnostic["hashed_bytes"] == lanes.QUARANTINE_HASH_BYTES
+    assert diagnostic["hash_complete"] is False
+    assert (size, lanes.QUARANTINE_HASH_BYTES) in reads
+    assert (size, lanes.MAX_TERMINAL_RECORD_BYTES + 1) not in reads
+    assert max(limit for _file_size, limit in reads) <= (
+        lanes.MAX_TERMINAL_RECORD_BYTES + 1
+    )
+
+
+def test_recovery_does_not_parse_committed_journals_every_tick(
+    tmp_path: Path,
+    monkeypatch,
+):
+    state_root = tmp_path / "cli-lanes"
+    monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
+    committed = lanes._terminal_path(
+        lanes.state_path("cassandra", "t_done"), 1, "committed"
+    )
+    lanes.atomic_json(committed, {"kanban_state": "committed"})
+    reads = []
+    monkeypatch.setattr(lanes, "load_json", lambda path: (reads.append(path) or {}))
+    monkeypatch.setitem(
+        sys.modules,
+        "hermes_cli",
+        SimpleNamespace(kanban_db=SimpleNamespace()),
+    )
+
+    assert lanes.recover_pending_finalizations() == 0
+    assert reads == []
+
+
+def test_atomic_json_fsyncs_file_and_directory_and_uses_private_mode(
+    tmp_path: Path,
+    monkeypatch,
+):
+    calls = []
+    real_fsync = lanes.os.fsync
+    monkeypatch.setattr(
+        lanes.os,
+        "fsync",
+        lambda descriptor: (calls.append(descriptor), real_fsync(descriptor))[1],
+    )
+    path = tmp_path / "state.json"
+
+    lanes.atomic_json(path, {"safe": True})
+
+    assert json.loads(path.read_text()) == {"safe": True}
+    assert path.stat().st_mode & 0o777 == 0o600
+    assert len(calls) >= 2
+
+
+def test_atomic_json_refuses_a_precreated_temp_symlink(
+    tmp_path: Path,
+    monkeypatch,
+):
+    path = tmp_path / "state.json"
+    victim = tmp_path / "credential"
+    victim.write_text("do not overwrite", encoding="utf-8")
+    temporary = tmp_path / ".state.json.fixed.tmp"
+    temporary.symlink_to(victim)
+    monkeypatch.setattr(lanes.uuid, "uuid4", lambda: SimpleNamespace(hex="fixed"))
+
+    with pytest.raises(FileExistsError):
+        lanes.atomic_json(path, {"unsafe": True})
+
+    assert victim.read_text(encoding="utf-8") == "do not overwrite"
+    assert not path.exists()
+
+
+def test_artifact_gc_prunes_by_age_without_touching_pending_journals(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    board = lanes.STATE_ROOT / "cassandra"
+    board.mkdir(parents=True)
+    quarantine = board / "quarantine"
+    quarantine.mkdir()
+    old_artifacts = [
+        board / "t.run-1.candidate-1.json",
+        board / "t.run-1.provider-1.result.json",
+        board / "t.run-1.terminal.committed.json",
+        board / f"t.run-1.terminal.conflict-{'a' * 32}.json",
+        quarantine / "t.invalid.1234.quarantine",
+    ]
+    for artifact in old_artifacts:
+        artifact.write_text("{}", encoding="utf-8")
+    quarantine_symlink = quarantine / "attacker.quarantine"
+    quarantine_symlink.symlink_to(tmp_path / "missing-target")
+    pending = board / "t.run-1.terminal.pending.json"
+    pending.write_text("{}", encoding="utf-8")
+    prepared = board / f"t.run-1.terminal.prepared-{'b' * 32}.json"
+    prepared.write_text("{}", encoding="utf-8")
+    old_time = 100.0
+    for artifact in old_artifacts:
+        os.utime(artifact, (old_time, old_time))
+    os.utime(pending, (old_time, old_time))
+    os.utime(prepared, (old_time, old_time))
+
+    assert lanes.gc_lane_artifacts(
+        now=1000.0, max_age_seconds=10, max_count=100, max_bytes=10000
+    ) == len(old_artifacts) + 1
+    assert not any(artifact.exists() for artifact in old_artifacts)
+    assert not quarantine_symlink.is_symlink()
+    assert pending.exists()
+    assert prepared.exists()
+
+
+def test_artifact_gc_prunes_oldest_by_count_and_total_bytes(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    board = lanes.STATE_ROOT / "cassandra"
+    board.mkdir(parents=True)
+    files = []
+    for sequence in range(3):
+        path = board / f"t.run-1.provider-{sequence}.result.json"
+        path.write_text("x" * 40, encoding="utf-8")
+        os.utime(path, (100 + sequence, 100 + sequence))
+        files.append(path)
+
+    assert lanes.gc_lane_artifacts(
+        now=200.0, max_age_seconds=1000, max_count=2, max_bytes=45
+    ) == 2
+    assert [path.exists() for path in files] == [False, False, True]
+
+
+def test_artifact_gc_preserves_a_concurrent_replacement(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    board = lanes.STATE_ROOT / "cassandra"
+    board.mkdir(parents=True)
+    path = board / "t.run-1.provider-1.result.json"
+    path.write_text("old", encoding="utf-8")
+    os.utime(path, (100, 100))
+    replacement = board / "replacement.tmp"
+    replacement.write_text("new", encoding="utf-8")
+    real_unlink = lanes._unlink_artifact_if_same
+    swapped = {"value": False}
+
+    def swap_before_identity_checked(candidate, observed):
+        if not swapped["value"]:
+            os.replace(replacement, candidate)
+            swapped["value"] = True
+        return real_unlink(candidate, observed)
+
+    monkeypatch.setattr(lanes, "_unlink_artifact_if_same", swap_before_identity_checked)
+
+    assert lanes.gc_lane_artifacts(
+        now=1000.0, max_age_seconds=10, max_count=100, max_bytes=10000
+    ) == 0
+    assert swapped["value"] is True
+    assert path.read_text(encoding="utf-8") == "new"

testing/tests/test_hermes_cli_routing.py

@@ -0,0 +1,155 @@
+"""Switchyard routing and provider-health behavior."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    Path,
+    _SwitchyardResponse,
+    json,
+    lanes,
+)
+
+
+def test_auto_lane_uses_switchyard_worker_decision():
+    observed = {}
+
+    def route(request, timeout):
+        observed["payload"] = json.loads(request.data)
+        observed["timeout"] = timeout
+        return _SwitchyardResponse("worker/claude/claude-opus-5/xhigh")
+
+    route = lanes.select_route(
+        "Perform the security review.",
+        "cli-auto",
+        open_request=route,
+    )
+
+    assert route.provider == "claude"
+    assert route.model == "claude-opus-5"
+    assert route.effort == "xhigh"
+    assert route.classifier == "switchyard-classifier"
+    assert observed["payload"]["model"] == "atlas/worker/auto/maximum"
+    assert observed["timeout"] == 60
+
+
+def test_manual_lane_is_still_enforced_by_switchyard():
+    observed = {}
+
+    def route(request, timeout):
+        observed["payload"] = json.loads(request.data)
+        return _SwitchyardResponse(
+            "worker/claude/claude-sonnet-5/high", "manual route"
+        )
+
+    route = lanes.select_route(
+        "Implement it.",
+        "cli-claude-high",
+        open_request=route,
+    )
+
+    assert route.provider == "claude"
+    assert route.effort == "high"
+    assert route.classifier == "switchyard-manual"
+    assert route.reason == "manual route"
+    assert observed["payload"]["model"] == "atlas/worker/manual/claude/high"
+
+
+def test_cross_provider_retry_passes_failed_provider_to_switchyard():
+    observed = {}
+
+    def route(request, timeout):
+        observed["payload"] = json.loads(request.data)
+        return _SwitchyardResponse("worker/claude/claude-sonnet-5/high")
+
+    route = lanes.select_route(
+        "Retry after capacity exhaustion.",
+        "cli-auto",
+        exclude_provider="codex",
+        open_request=route,
+    )
+
+    assert route.provider == "claude"
+    assert route.classifier == "switchyard-classifier"
+    content = observed["payload"]["messages"][0]["content"]
+    assert "codex provider failed or exhausted capacity" in content
+
+
+def test_classifier_cannot_select_a_freshly_excluded_provider():
+    payloads = []
+
+    def route(request, timeout):
+        payload = json.loads(request.data)
+        payloads.append(payload)
+        if len(payloads) == 1:
+            return _SwitchyardResponse("worker/claude/opus/xhigh")
+        return _SwitchyardResponse("worker/codex/sol/xhigh", "healthy route")
+
+    selected = lanes.select_route(
+        "Perform a consequential review.",
+        "cli-auto",
+        exclude_provider="claude",
+        exclude_reason="is unavailable according to fresh native health",
+        open_request=route,
+    )
+
+    assert selected.provider == "codex"
+    assert selected.effort == "xhigh"
+    assert selected.classifier == "switchyard-classifier-health-guard"
+    assert payloads[0]["model"] == "atlas/worker/auto/maximum"
+    assert payloads[1]["model"] == "atlas/worker/manual/codex/xhigh"
+    assert "claude provider is unavailable" in payloads[0]["messages"][0]["content"]
+
+
+def test_fresh_native_health_excludes_only_one_proven_down_provider(
+    tmp_path: Path, monkeypatch
+):
+    paths = {
+        "codex": tmp_path / "codex.json",
+        "claude": tmp_path / "claude.json",
+    }
+    paths["codex"].write_text('{"state":"available"}\n', encoding="utf-8")
+    paths["claude"].write_text('{"state":"unavailable"}\n', encoding="utf-8")
+    monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", paths)
+
+    now = max(path.stat().st_mtime for path in paths.values())
+    assert lanes.fresh_unavailable_provider(now=now) == "claude"
+
+    paths["codex"].write_text('{"state":"unavailable"}\n', encoding="utf-8")
+    now = max(path.stat().st_mtime for path in paths.values())
+    assert lanes.fresh_unavailable_provider(now=now) is None
+
+
+def test_explicit_auth_failure_survives_restart_health_gap(tmp_path: Path, monkeypatch):
+    paths = {
+        "codex": tmp_path / "codex.json",
+        "claude": tmp_path / "claude.json",
+    }
+    paths["codex"].write_text(
+        '{"state":"available","authenticated":true}\n', encoding="utf-8"
+    )
+    paths["claude"].write_text(
+        '{"state":"unavailable","authenticated":false}\n', encoding="utf-8"
+    )
+    monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", paths)
+
+    stale_during_rollout = paths["claude"].stat().st_mtime + 10 * 60
+    assert lanes.fresh_unavailable_provider(now=stale_during_rollout) == "claude"
+
+    paths["claude"].write_text(
+        '{"state":"unavailable","authenticated":true}\n', encoding="utf-8"
+    )
+    transient_stale = paths["claude"].stat().st_mtime + 10 * 60
+    assert lanes.fresh_unavailable_provider(now=transient_stale) is None
+
+
+def test_worker_environment_preserves_vault_backed_cli_homes(monkeypatch):
+    """Kanban workers must not fall back to credential-free persistent homes."""
+    monkeypatch.setenv("CODEX_HOME", "/runtime-access/codex")
+    monkeypatch.setenv("CLAUDE_CONFIG_DIR", "/runtime-access/claude")
+
+    env = lanes._base_env()
+
+    assert env["HOME"] == str(lanes.DATA_ROOT / "home")
+    assert env["CODEX_HOME"] == "/runtime-access/codex"
+    assert env["CLAUDE_CONFIG_DIR"] == "/runtime-access/claude"
+    assert env["GIT_TERMINAL_PROMPT"] == "0"

testing/tests/test_hermes_cli_run_identity.py

@@ -0,0 +1,217 @@
+"""Canonical SQLite run-identity contracts for every terminal artifact class."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    Path,
+    _completed_result,
+    _pending_terminal_record,
+    lanes,
+    pytest,
+    sys,
+)
+
+
+class _NoDatabase:
+    """Fail if malformed run authority reaches a Kanban transaction."""
+
+    def __init__(self) -> None:
+        self.accesses = []
+
+    def scoped_current_board(self, *_args, **_kwargs):
+        self.accesses.append("scope")
+        raise AssertionError("noncanonical authority reached the database")
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        False,
+        True,
+        0,
+        -1,
+        2**63,
+        "0",
+        "01",
+        "9223372036854775808",
+        "9" * 150,
+        "+1",
+        " 1",
+    ],
+)
+def test_run_identity_rejects_noncanonical_or_non_sqlite_values(value):
+    assert lanes.canonical_run_id(value) is None
+
+
+@pytest.mark.parametrize("value", [1, 2**63 - 1, "1", "9223372036854775807"])
+def test_run_identity_accepts_the_positive_sqlite_int64_domain(value):
+    assert lanes.canonical_run_id(value) == int(value)
+
+
+@pytest.mark.parametrize("run_id", [False, True, 0, -1, 2**63, 10**149])
+def test_terminal_writer_refuses_invalid_runtime_run_ids(tmp_path: Path, run_id):
+    with pytest.raises(ValueError, match="SQLite int64"):
+        lanes._write_terminal_record(
+            lanes.state_path("cassandra", "t_invalid_writer"),
+            board="cassandra",
+            task_id="t_invalid_writer",
+            run_id=run_id,
+            structured=_completed_result(),
+            summary="completed",
+            metadata={},
+        )
+
+
+@pytest.mark.parametrize(
+    ("state", "suffix", "parser"),
+    [
+        ("pending", "terminal.pending.json", "_terminal_identity"),
+        ("committed", "terminal.committed.json", "_terminal_identity"),
+        (
+            "prepared",
+            f"terminal.prepared-{'a' * 32}.json",
+            "_terminal_evidence_identity",
+        ),
+        (
+            "conflict",
+            f"terminal.conflict-{'a' * 32}.json",
+            "_terminal_evidence_identity",
+        ),
+    ],
+)
+@pytest.mark.parametrize("encoded", ["0", "01", "9223372036854775808", "9" * 150])
+def test_every_terminal_filename_class_rejects_noncanonical_run_ids(
+    tmp_path: Path,
+    monkeypatch,
+    state: str,
+    suffix: str,
+    parser: str,
+    encoded: str,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    board = lanes.STATE_ROOT / "cassandra"
+    board.mkdir(parents=True)
+    path = board / f"t_filename.run-{encoded}.{suffix}"
+
+    assert getattr(lanes, parser)(path) is None, state
+
+
+@pytest.mark.parametrize("run_id", [False, True, 0, -1, 2**63, 10**149])
+@pytest.mark.parametrize("state", ["pending", "prepared", "committed", "conflict"])
+def test_every_terminal_record_class_rejects_invalid_run_ids(run_id, state):
+    record = _pending_terminal_record(
+        "cassandra", "t_record", run_id, "invalid identity"
+    )
+    record["kanban_state"] = state
+
+    assert lanes._terminal_record_valid(record) is False
+
+
+@pytest.mark.parametrize("run_id", [False, 0, 2**63, 10**149])
+def test_retirement_staging_rejects_invalid_document_run_ids(
+    tmp_path: Path,
+    monkeypatch,
+    run_id,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    board = lanes.STATE_ROOT / "cassandra"
+    board.mkdir(parents=True)
+    staged = board / f".retire.{'a' * 16}.{'b' * 16}.0"
+    lanes.atomic_json(
+        staged,
+        _pending_terminal_record(
+            "cassandra", "t_staged", run_id, "invalid staged identity"
+        ),
+    )
+    snapshot = lanes._open_terminal_recovery_snapshot(staged)
+    assert snapshot is not None
+    try:
+        assert lanes._staged_terminal_authority(staged, snapshot) is None
+    finally:
+        snapshot.close()
+    assert lanes._recover_retirement_staging() == 0
+    assert not staged.exists()
+    assert len(list((board / "quarantine").glob("*.quarantine"))) == 1
+
+
+def test_noncanonical_pending_filename_is_quarantined_before_db_access(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    board = lanes.STATE_ROOT / "cassandra"
+    board.mkdir(parents=True)
+    pending = board / "t_noncanonical.run-01.terminal.pending.json"
+    lanes.atomic_json(
+        pending,
+        _pending_terminal_record(
+            "cassandra", "t_noncanonical", 1, "noncanonical filename"
+        ),
+    )
+    database = _NoDatabase()
+    monkeypatch.setitem(
+        sys.modules,
+        "hermes_cli",
+        type("HermesCli", (), {"kanban_db": database})(),
+    )
+
+    assert lanes.recover_pending_finalizations() == 0
+    assert database.accesses == []
+    assert not pending.exists()
+    assert len(list((board / "quarantine").glob("*.quarantine"))) == 1
+
+
+def test_noncanonical_prepared_filename_is_quarantined_before_db_access(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    board = lanes.STATE_ROOT / "cassandra"
+    board.mkdir(parents=True)
+    prepared = (
+        board / f"t_prepared.run-01.terminal.prepared-{'a' * 32}.json"
+    )
+    record = _pending_terminal_record(
+        "cassandra", "t_prepared", 1, "noncanonical prepared filename"
+    )
+    record["kanban_state"] = "prepared"
+    lanes.atomic_json(prepared, record)
+    database = _NoDatabase()
+    monkeypatch.setitem(
+        sys.modules,
+        "hermes_cli",
+        type("HermesCli", (), {"kanban_db": database})(),
+    )
+
+    assert lanes.recover_pending_finalizations() == 0
+    assert database.accesses == []
+    assert not prepared.exists()
+    assert len(list((board / "quarantine").glob("*.quarantine"))) == 1
+
+
+@pytest.mark.parametrize(
+    ("state", "suffix"),
+    [
+        ("committed", "terminal.committed.json"),
+        ("conflict", f"terminal.conflict-{'a' * 32}.json"),
+    ],
+)
+def test_gc_quarantines_noncanonical_retained_evidence(
+    tmp_path: Path,
+    monkeypatch,
+    state: str,
+    suffix: str,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    board = lanes.STATE_ROOT / "cassandra"
+    board.mkdir(parents=True)
+    artifact = board / f"t_retained.run-01.{suffix}"
+    record = _pending_terminal_record(
+        "cassandra", "t_retained", 1, "noncanonical retained filename"
+    )
+    record["kanban_state"] = state
+    lanes.atomic_json(artifact, record)
+
+    assert lanes.gc_lane_artifacts() == 1
+    assert not artifact.exists()
+    assert len(list((board / "quarantine").glob("*.quarantine"))) == 1

testing/tests/test_hermes_cli_staging.py

@@ -0,0 +1,441 @@
+"""Retirement staging, recovery, and bounded quarantine."""
+
+from __future__ import annotations
+
+from testing.tests.test_hermes_cli_support import (
+    Path,
+    SimpleNamespace,
+    _completed_result,
+    _install_terminal_recovery_db,
+    _pending_terminal_record,
+    hashlib,
+    json,
+    lanes,
+    nullcontext,
+    os,
+    sys,
+    time,
+)
+
+
+def test_recovery_finalizes_the_valid_snapshot_it_classified_before_replacement(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path, first = lanes._write_terminal_record(
+        lanes.state_path("cassandra", "t_valid_gap"),
+        board="cassandra",
+        task_id="t_valid_gap",
+        run_id=54,
+        structured=_completed_result("first valid snapshot"),
+        summary="first valid snapshot",
+        metadata={},
+    )
+    replacement = path.with_name("replacement.valid")
+    lanes.atomic_json(
+        replacement,
+        _pending_terminal_record(
+            "cassandra", "t_valid_gap", 54, "second valid replacement"
+        ),
+    )
+    task = SimpleNamespace(
+        id="t_valid_gap",
+        status="running",
+        current_run_id=54,
+        result=None,
+        assignee="cli-auto",
+    )
+    completions = []
+    reclaims = []
+    _install_terminal_recovery_db(monkeypatch, task, completions, reclaims)
+    real_finalize = lanes._finalize_terminal_record
+    swapped = {"value": False}
+
+    def replace_before_finalize(*args, **kwargs):
+        assert kwargs["snapshot"].document == first
+        if not swapped["value"]:
+            os.replace(replacement, path)
+            swapped["value"] = True
+        return real_finalize(*args, **kwargs)
+
+    monkeypatch.setattr(lanes, "_finalize_terminal_record", replace_before_finalize)
+
+    assert lanes.recover_pending_finalizations() == 1
+    assert task.result == first["result"]
+    assert completions == [first["result"]]
+    assert reclaims == []
+    assert not path.exists()
+    conflicts = list(path.parent.glob("*.terminal.conflict-*.json"))
+    assert len(conflicts) == 1
+    conflict = json.loads(conflicts[0].read_text())
+    assert conflict["summary"] == "second valid replacement"
+
+
+def test_replacement_staged_journal_promotes_replays_and_never_reexecutes(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path = lanes._terminal_path(lanes.state_path("cassandra", "t_staged"), 55)
+    path.parent.mkdir(parents=True)
+    path.write_bytes(b"malformed source")
+    source_stat = path.stat(follow_symlinks=False)
+    staged_winner = path.with_name("staged-winner.tmp")
+    canonical_replacement = path.with_name("canonical-replacement.tmp")
+    first = _pending_terminal_record(
+        "cassandra", "t_staged", 55, "staged accepted result"
+    )
+    second = _pending_terminal_record(
+        "cassandra", "t_staged", 55, "later canonical replacement"
+    )
+    lanes.atomic_json(staged_winner, first)
+    lanes.atomic_json(canonical_replacement, second)
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    descriptor = os.open(path.parent, directory_flags)
+    real_rename = lanes._rename_noreplace
+    rename_steps = []
+
+    def force_replacement_staging(source, destination, **kwargs):
+        rename_steps.append((source, destination))
+        if source == path.name:
+            os.replace(staged_winner, path)
+        elif source.startswith(".retire."):
+            os.replace(canonical_replacement, path)
+        return real_rename(source, destination, **kwargs)
+
+    try:
+        with monkeypatch.context() as race:
+            race.setattr(lanes, "_rename_noreplace", force_replacement_staging)
+            outcome = lanes._retire_terminal_entry(
+                path,
+                source_stat,
+                board_descriptor=descriptor,
+                quarantine_descriptor=descriptor,
+            )
+    finally:
+        os.close(descriptor)
+
+    assert outcome == "replacement-staged"
+    assert len(rename_steps) == 2
+    staged = list(path.parent.glob(".retire.*"))
+    assert len(staged) == 1
+    staged_snapshot = lanes._open_terminal_recovery_snapshot(staged[0])
+    assert staged_snapshot is not None
+    try:
+        authority = lanes._staged_terminal_authority(staged[0], staged_snapshot)
+        assert authority is not None
+        assert authority[0].run_id == 55
+        assert authority[1] == path
+    finally:
+        staged_snapshot.close()
+    assert lanes._has_pending_finalization("cassandra", "t_staged", 55) is True
+    assert lanes.gc_lane_artifacts(
+        now=staged[0].stat().st_mtime + 3600,
+        max_age_seconds=0,
+        max_count=0,
+        max_bytes=0,
+    ) == 0
+    assert staged[0].exists()
+
+    task = SimpleNamespace(
+        id="t_staged",
+        status="running",
+        current_run_id=55,
+        completed_run_id=None,
+        result=None,
+        assignee="cli-auto",
+    )
+    completions = []
+
+    class Connection:
+        def close(self):
+            return None
+
+    def complete_task(_conn, _task_id, **kwargs):
+        task.status = "done"
+        task.result = kwargs["result"]
+        task.completed_run_id = kwargs["expected_run_id"]
+        task.current_run_id = None
+        completions.append(kwargs["result"])
+        return True
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        complete_task=complete_task,
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+
+    assert lanes.recover_pending_finalizations() == 1
+    assert completions == [first["result"]]
+    assert not list(path.parent.glob(".retire.*"))
+    assert not path.exists()
+    conflicts = list(path.parent.glob("*.terminal.conflict-*.json"))
+    assert len(conflicts) == 1
+    assert json.loads(conflicts[0].read_text())["result"] == second["result"]
+    assert lanes.recover_pending_finalizations() == 0
+    assert completions == [first["result"]]
+
+
+def test_quarantine_staged_replacement_is_recovered_bounded_and_preserved(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    path = lanes._terminal_path(lanes.state_path("cassandra", "t_qstage"), 58)
+    path.parent.mkdir(parents=True)
+    path.write_bytes(b"malformed original")
+    identity = lanes._terminal_identity(path)
+    assert identity is not None
+    records = [
+        _pending_terminal_record(
+            "cassandra", "t_qstage", 58, f"quarantine replacement {number}"
+        )
+        for number in range(2)
+    ]
+    replacements = [
+        path.with_name(f"quarantine-replacement-{number}.tmp")
+        for number in range(2)
+    ]
+    for replacement, record in zip(replacements, records, strict=True):
+        lanes.atomic_json(replacement, record)
+    real_rename = lanes._rename_noreplace
+    retirement_steps = []
+
+    def replace_during_cross_directory_retirement(source, destination, **kwargs):
+        if source == path.name and destination.startswith(".retire."):
+            retirement_steps.append("move")
+            os.replace(replacements[0], path)
+        elif source.startswith(".retire.") and destination == path.name:
+            retirement_steps.append("restore")
+            os.replace(replacements[1], path)
+        return real_rename(source, destination, **kwargs)
+
+    with monkeypatch.context() as race:
+        race.setattr(
+            lanes,
+            "_rename_noreplace",
+            replace_during_cross_directory_retirement,
+        )
+        lanes._quarantine_terminal(path, identity, "malformed-payload")
+
+    assert retirement_steps == ["move", "restore"]
+    quarantine_stages = list((path.parent / "quarantine").glob(".retire.*"))
+    assert len(quarantine_stages) == 1
+    snapshot = lanes._open_terminal_recovery_snapshot(quarantine_stages[0])
+    assert snapshot is not None
+    try:
+        authority = lanes._staged_terminal_authority(quarantine_stages[0], snapshot)
+        assert authority is not None
+        assert authority[0].run_id == 58
+        assert authority[1] == path
+    finally:
+        snapshot.close()
+    assert lanes._has_pending_finalization("cassandra", "t_qstage", 58) is True
+    assert lanes.gc_lane_artifacts(
+        now=quarantine_stages[0].stat().st_mtime + 3600,
+        max_age_seconds=0,
+        max_count=0,
+        max_bytes=0,
+    ) == 1
+    assert quarantine_stages[0].exists()
+
+    task = SimpleNamespace(
+        id="t_qstage",
+        status="running",
+        current_run_id=58,
+        completed_run_id=None,
+        result=None,
+        assignee="cli-auto",
+    )
+    completions = []
+    reclaims = []
+    _install_terminal_recovery_db(monkeypatch, task, completions, reclaims)
+
+    assert lanes.recover_pending_finalizations() == 1
+    assert len(completions) == 1
+    assert reclaims == []
+    assert not list(path.parent.glob(".retire.*"))
+    assert not list((path.parent / "quarantine").glob(".retire.*"))
+    conflicts = list(path.parent.glob("*.terminal.conflict-*.json"))
+    assert len(conflicts) == 1
+    preserved = {task.result, json.loads(conflicts[0].read_text())["result"]}
+    assert preserved == {record["result"] for record in records}
+    assert lanes.recover_pending_finalizations() == 0
+    assert len(completions) == 1
+
+
+def test_repeated_retirement_staging_preserves_every_valid_result_once(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    canonical = lanes._terminal_path(
+        lanes.state_path("cassandra", "t_staged_repeat"), 57
+    )
+    canonical.parent.mkdir(parents=True)
+    path_digest = hashlib.sha256(canonical.name.encode("utf-8")).hexdigest()[:16]
+    staged = canonical.parent / f".retire.{path_digest}.{'a' * 16}.0"
+    records = [
+        _pending_terminal_record(
+            "cassandra", "t_staged_repeat", 57, f"accepted result {number}"
+        )
+        for number in range(3)
+    ]
+    lanes.atomic_json(staged, records[0])
+    second = canonical.with_name("second-staged-replacement.tmp")
+    third = canonical.with_name("third-staged-replacement.tmp")
+    lanes.atomic_json(second, records[1])
+    lanes.atomic_json(third, records[2])
+    real_rename = lanes._rename_noreplace
+    retirement_steps = []
+
+    def replace_each_retirement_generation(source, destination, **kwargs):
+        if source == staged.name and destination.startswith(".retire."):
+            retirement_steps.append("move")
+            os.replace(second, staged)
+        elif source.startswith(".retire.") and destination == staged.name:
+            retirement_steps.append("restore")
+            os.replace(third, staged)
+        return real_rename(source, destination, **kwargs)
+
+    with monkeypatch.context() as race:
+        race.setattr(lanes, "_rename_noreplace", replace_each_retirement_generation)
+        assert lanes._recover_retirement_staging() == 0
+
+    assert retirement_steps == ["move", "restore"]
+    staged_generations = list(canonical.parent.glob(".retire.*"))
+    assert len(staged_generations) == 2
+    for generation in staged_generations:
+        snapshot = lanes._open_terminal_recovery_snapshot(generation)
+        assert snapshot is not None
+        try:
+            authority = lanes._staged_terminal_authority(generation, snapshot)
+            assert authority is not None
+            assert authority[1] == canonical
+        finally:
+            snapshot.close()
+
+    task = SimpleNamespace(
+        id="t_staged_repeat",
+        status="running",
+        current_run_id=57,
+        completed_run_id=None,
+        result=None,
+        assignee="cli-auto",
+    )
+    completions = []
+    reclaims = []
+    _install_terminal_recovery_db(monkeypatch, task, completions, reclaims)
+
+    assert lanes.recover_pending_finalizations() == 1
+    assert len(completions) == 1
+    assert reclaims == []
+    assert not list(canonical.parent.glob(".retire.*"))
+    conflicts = list(canonical.parent.glob("*.terminal.conflict-*.json"))
+    assert len(conflicts) == 2
+    preserved_results = {task.result}
+    preserved_results.update(json.loads(path.read_text())["result"] for path in conflicts)
+    assert preserved_results == {record["result"] for record in records}
+    assert lanes.recover_pending_finalizations() == 0
+    assert len(completions) == 1
+
+
+def test_prepared_recovery_quarantines_pinned_inode_then_replays_replacement(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    pending_identity = lanes.TerminalIdentity(
+        "cassandra", "t_prepared_swap", 59, "pending"
+    )
+    pending = _pending_terminal_record(
+        "cassandra", "t_prepared_swap", 59, "valid prepared replacement"
+    )
+    prepared_path = lanes._persist_prepared_evidence(pending_identity, pending)
+    valid_prepared = json.loads(prepared_path.read_text())
+    replacement = prepared_path.with_name("prepared-valid-replacement.tmp")
+    lanes.atomic_json(replacement, valid_prepared)
+    invalid_prepared = dict(valid_prepared)
+    invalid_prepared["summary"] = "does not match the structured result"
+    lanes.atomic_json(prepared_path, invalid_prepared)
+    task = SimpleNamespace(
+        id="t_prepared_swap",
+        status="running",
+        current_run_id=59,
+        completed_run_id=None,
+        result=None,
+        assignee="cli-auto",
+    )
+    completions = []
+    reclaims = []
+    _install_terminal_recovery_db(monkeypatch, task, completions, reclaims)
+    real_quarantine = lanes._quarantine_terminal
+    swapped = {"value": False}
+
+    def swap_before_prepared_quarantine(*args, **kwargs):
+        assert kwargs["snapshot"].document == invalid_prepared
+        if not swapped["value"]:
+            os.replace(replacement, prepared_path)
+            swapped["value"] = True
+        return real_quarantine(*args, **kwargs)
+
+    monkeypatch.setattr(
+        lanes,
+        "_quarantine_terminal",
+        swap_before_prepared_quarantine,
+    )
+
+    assert lanes.recover_pending_finalizations() == 1
+    assert swapped["value"] is True
+    assert completions == [valid_prepared["result"]]
+    assert reclaims == []
+    assert not prepared_path.exists()
+    committed = list(prepared_path.parent.glob("*.terminal.committed.json"))
+    assert len(committed) == 1
+    assert json.loads(committed[0].read_text())["result"] == valid_prepared["result"]
+
+
+def test_untrusted_retirement_staging_is_bounded_by_artifact_gc(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    board = lanes.STATE_ROOT / "cassandra"
+    board.mkdir(parents=True)
+    for sequence in range(10):
+        (board / f".retire.{'a' * 16}.{'b' * 16}.{sequence}").write_bytes(
+            b"not terminal authority"
+        )
+    (board / f".retire.{'c' * 32}.0").write_bytes(b"legacy hidden artifact")
+
+    removed = lanes.gc_lane_artifacts(
+        now=time.time(),
+        max_age_seconds=3600,
+        max_count=2,
+        max_bytes=1024 * 1024,
+    )
+
+    assert removed == 9
+    assert len(list(board.glob(".retire.*"))) == 2
+
+
+def test_untrusted_quarantine_stage_never_creates_recursive_quarantine(
+    tmp_path: Path,
+    monkeypatch,
+):
+    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
+    quarantine = lanes.STATE_ROOT / "cassandra" / "quarantine"
+    quarantine.mkdir(parents=True)
+    staged = quarantine / f".retire.{'a' * 16}.{'b' * 16}.0"
+    staged.write_bytes(b"untrusted staged bytes")
+
+    assert lanes._recover_retirement_staging() == 0
+
+    assert not staged.exists()
+    assert not (quarantine / "quarantine").exists()
+    diagnostics = list(quarantine.glob("*.quarantine"))
+    assert len(diagnostics) == 1
+    assert diagnostics[0].stat().st_mode & 0o777 == 0o600

testing/tests/test_hermes_cli_support.py

@@ -0,0 +1,236 @@
+"""Focused tests for Agent Hermes' direct Codex and Claude Kanban lanes."""
+
+from __future__ import annotations
+
+import errno
+
+import importlib.util
+
+import hashlib
+
+import json
+
+import os
+
+import signal
+
+import stat
+
+import sys
+
+import threading
+
+import time
+
+from contextlib import nullcontext
+
+from pathlib import Path
+
+from types import SimpleNamespace
+
+import pytest
+
+import yaml
+
+SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts"
+
+sys.path.insert(0, str(SCRIPTS))
+
+HERMES = Path(__file__).parents[2] / "services/hermes"
+
+KEYCLOAK = Path(__file__).parents[2] / "services/keycloak"
+
+FLUX_HERMES = (
+    Path(__file__).parents[2]
+    / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml"
+)
+
+def _load(name: str):
+    spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py")
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+_lane_facade = _load("cli_lane_runner")
+
+
+class _LaneSurface:
+    """Patch every module-local import behind the compatibility facade."""
+
+    def __init__(self, facade):
+        object.__setattr__(self, "_facade", facade)
+        object.__setattr__(
+            self,
+            "_modules",
+            tuple(
+                module
+                for name, module in sys.modules.items()
+                if name == "cli_lane_runner" or name.startswith("cli_lane_")
+            ),
+        )
+
+    def __getattr__(self, name):
+        facade = object.__getattribute__(self, "_facade")
+        if hasattr(facade, name):
+            return getattr(facade, name)
+        for module in object.__getattribute__(self, "_modules"):
+            if hasattr(module, name):
+                return getattr(module, name)
+        raise AttributeError(name)
+
+    def __setattr__(self, name, value):
+        matched = False
+        for module in object.__getattribute__(self, "_modules"):
+            if hasattr(module, name):
+                setattr(module, name, value)
+                matched = True
+        if not matched:
+            setattr(object.__getattribute__(self, "_facade"), name, value)
+
+
+lanes = _LaneSurface(_lane_facade)
+
+policy = _load("claude_command_policy")
+
+migration = _load("migrate_herdr_state")
+
+auth_patch = _load("patch_hermes_auth")
+
+tui_gateway_patch = _load("patch_tui_gateway")
+
+codex_runtime_patch = _load("patch_codex_runtime")
+
+ttyd_patch = _load("patch_ttyd_index")
+
+client_config = _load("configure_agent_clients")
+
+def _agent_deployment() -> dict:
+    return yaml.safe_load((HERMES / "agent-deployment.yaml").read_text())
+
+def _services() -> dict[str, dict]:
+    return {
+        item["metadata"]["name"]: item
+        for item in yaml.safe_load_all((HERMES / "service.yaml").read_text())
+        if item
+    }
+
+def _oauth_deployment(name: str) -> dict:
+    documents = [
+        item
+        for item in yaml.safe_load_all((HERMES / "oauth2-proxy.yaml").read_text())
+        if item
+    ]
+    return next(
+        item
+        for item in documents
+        if item["kind"] == "Deployment" and item["metadata"]["name"] == name
+    )
+
+class _SwitchyardResponse:
+    """Minimal context-managed response used by routing contract tests."""
+
+    def __init__(self, selected: str, rationale: str = "local classifier vote"):
+        self.headers = {
+            "x-model-router-selected-model": selected,
+            "x-model-router-rationale": rationale,
+        }
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_args):
+        return False
+
+    def read(self):
+        return b"{}"
+
+def _completed_result(summary: str = "done") -> dict:
+    return {
+        "status": "completed",
+        "summary": summary,
+        "changed_files": [],
+        "tests_run": [],
+        "artifacts": [],
+        "findings": [],
+        "blockers": [],
+    }
+
+def _pending_terminal_record(board: str, task_id: str, run_id: int, summary: str) -> dict:
+    structured = _completed_result(summary)
+    return {
+        "board": board,
+        "task_id": task_id,
+        "expected_run_id": run_id,
+        "result": json.dumps(structured, sort_keys=True),
+        "summary": summary,
+        "metadata": {},
+        "kanban_state": "pending",
+        "recorded_at": lanes.utc_now(),
+    }
+
+def _install_terminal_recovery_db(monkeypatch, task, completions, reclaims) -> None:
+    class Connection:
+        def close(self):
+            return None
+
+    def complete_task(_conn, _task_id, **kwargs):
+        if task.status != "running" or task.current_run_id != kwargs["expected_run_id"]:
+            return False
+        task.status = "done"
+        task.result = kwargs["result"]
+        task.current_run_id = None
+        completions.append(kwargs["result"])
+        return True
+
+    def reclaim_task(_conn, _task_id, **kwargs):
+        reclaims.append(kwargs)
+        return False
+
+    fake_db = SimpleNamespace(
+        scoped_current_board=lambda _board: nullcontext(),
+        connect=lambda board: Connection(),
+        get_task=lambda _conn, _task_id: task,
+        complete_task=complete_task,
+        reclaim_task=reclaim_task,
+    )
+    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
+
+__all__ = [
+    'FLUX_HERMES',
+    'HERMES',
+    'KEYCLOAK',
+    'Path',
+    'SCRIPTS',
+    'SimpleNamespace',
+    '_SwitchyardResponse',
+    '_agent_deployment',
+    '_completed_result',
+    '_install_terminal_recovery_db',
+    '_load',
+    '_oauth_deployment',
+    '_pending_terminal_record',
+    '_services',
+    'auth_patch',
+    'client_config',
+    'codex_runtime_patch',
+    'errno',
+    'hashlib',
+    'importlib',
+    'json',
+    'lanes',
+    'migration',
+    'nullcontext',
+    'os',
+    'policy',
+    'pytest',
+    'signal',
+    'stat',
+    'sys',
+    'threading',
+    'time',
+    'ttyd_patch',
+    'tui_gateway_patch',
+    'yaml',
+]

testing/tests/test_hermes_coordinator_boards.py

@@ -0,0 +1,204 @@
+"""Hermes coordinator board recovery contracts."""
+
+from __future__ import annotations
+
+from test_hermes_coordinator_support import (
+    Path,
+    _base_config,
+    coordinator,
+    pytest,
+    routing,
+    subprocess,
+    yaml,
+)
+
+
+def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch):
+    (tmp_path / "config.yaml").write_text(
+        yaml.safe_dump(_base_config()), encoding="utf-8"
+    )
+    (tmp_path / ".env").write_text("GITEA_TOKEN=do-not-report\n", encoding="utf-8")
+    codex = routing.Catalog("openai-codex", ["gpt-5.6-terra"], True, True, "connected")
+    claude = routing.Catalog("anthropic", ["claude-opus-5"], True, True, "connected")
+    monkeypatch.setattr(coordinator, "discover_codex_models", lambda: codex)
+    monkeypatch.setattr(coordinator, "discover_claude_models", lambda: claude)
+    monkeypatch.setattr(
+        coordinator, "bootstrap_cassandra_state", lambda root: {"state": "ready"}
+    )
+    monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda env: "ready")
+
+    status = coordinator.refresh_once(tmp_path)
+
+    status_path = tmp_path / "workspace/coordinator/model-routing.json"
+    assert status_path.is_file()
+    assert status["projects"]["cassandra"]["state"] == "ready"
+    assert status["projects"]["cassandra"]["board_state"] == {"state": "ready"}
+    assert "do-not-report" not in status_path.read_text(encoding="utf-8")
+
+def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh(
+    tmp_path: Path, monkeypatch
+):
+    """A damaged project board degrades Kanban instead of stopping all services."""
+
+    class FakeKanbanDbCorruptError(Exception):
+        def __init__(self):
+            super().__init__("integrity check failed")
+            self.db_path = tmp_path / "kanban.db"
+            self.backup_path = tmp_path / "kanban.db.corrupt.backup"
+            self.reason = "row out of order"
+
+    def raise_corruption(_root: Path) -> None:
+        raise FakeKanbanDbCorruptError()
+
+    monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption)
+    monkeypatch.setattr(
+        coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError
+    )
+
+    status = coordinator.bootstrap_cassandra_state(tmp_path)
+
+    assert status == {
+        "state": "corrupt-preserved",
+        "database": str(tmp_path / "kanban.db"),
+        "backup": str(tmp_path / "kanban.db.corrupt.backup"),
+        "reason": "row out of order",
+    }
+
+def test_unrelated_cassandra_bootstrap_failure_remains_fatal(
+    tmp_path: Path, monkeypatch
+):
+    """Only the known fail-closed corruption state may be degraded."""
+
+    class FakeKanbanDbCorruptError(Exception):
+        pass
+
+    def raise_permission_error(_root: Path) -> None:
+        raise PermissionError("cannot access board")
+
+    monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error)
+    monkeypatch.setattr(
+        coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError
+    )
+
+    try:
+        coordinator.bootstrap_cassandra_state(tmp_path)
+    except PermissionError as error:
+        assert str(error) == "cannot access board"
+    else:
+        raise AssertionError("unrelated board errors must fail the coordinator refresh")
+
+def test_cassandra_workspace_uses_valid_configured_worktree(
+    tmp_path: Path, monkeypatch
+):
+    """The coordinator selects an explicit Git worktree but rejects stale paths."""
+    active = tmp_path / "cassandra-v69"
+    active.mkdir()
+    (active / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8")
+    monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(active))
+
+    assert coordinator.cassandra_workspace() == active
+
+    (active / ".git").unlink()
+    assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH
+
+@pytest.mark.parametrize(
+    ("current_remote", "repair_action"),
+    [
+        ("https://scm.bstein.dev/bstein/cassandra.git", "set-url"),
+        (None, "add"),
+        (coordinator.CASSANDRA_REMOTE, None),
+    ],
+    ids=["old-origin", "missing-origin", "canonical-origin"],
+)
+def test_cassandra_sync_repairs_existing_worktree(
+    tmp_path: Path, monkeypatch, current_remote: str | None, repair_action: str | None
+):
+    """Existing worktrees use the canonical Atlas origin before fetching."""
+    workspace = tmp_path / "cassandra"
+    (workspace / ".git").mkdir(parents=True)
+    monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace)
+    monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git")
+    token_path = tmp_path / "gitea-token"
+    token_path.write_text("configured\n", encoding="utf-8")
+    monkeypatch.setenv("HERMES_GITEA_TOKEN_FILE", str(token_path))
+    commands: list[list[str]] = []
+    environments: list[dict[str, str]] = []
+
+    def run(command, **kwargs):
+        commands.append(command)
+        environments.append(kwargs.get("env", {}))
+        if command[-2:] == ["get-url", "origin"]:
+            return subprocess.CompletedProcess(
+                command, 0 if current_remote else 2, stdout=f"{current_remote or ''}\n"
+            )
+        return subprocess.CompletedProcess(command, 0)
+
+    monkeypatch.setattr(coordinator.subprocess, "run", run)
+    assert coordinator.sync_cassandra_repo({"GITEA_TOKEN": "must-not-pass"}) == "ready"
+    repair_commands = [command for command in commands if repair_action in command]
+    assert bool(repair_commands) is bool(repair_action)
+    assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"]
+    assert all("GITEA_TOKEN" not in environment for environment in environments)
+
+def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch):
+    """Missing credentials skip only the fetch, not the local origin repair."""
+    workspace = tmp_path / "cassandra"
+    (workspace / ".git").mkdir(parents=True)
+    monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace)
+    monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git")
+    commands: list[list[str]] = []
+
+    def run(command, **_kwargs):
+        commands.append(command)
+        stdout = "old-origin\n" if command[-2:] == ["get-url", "origin"] else None
+        return subprocess.CompletedProcess(command, 0, stdout=stdout)
+
+    monkeypatch.setattr(coordinator.subprocess, "run", run)
+    state = coordinator.sync_cassandra_repo({})
+
+    assert state == "ready; fetch skipped until Gitea token is configured"
+    assert [command[-3] for command in commands] == ["remote", "set-url"]
+
+def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks():
+    """A project switch moves queued work without relocating active evidence."""
+
+    class Task:
+        def __init__(self, task_id: str, status: str):
+            self.id = task_id
+            self.status = status
+            self.workspace_kind = "dir"
+            self.workspace_path = str(coordinator.CASSANDRA_BASE_PATH)
+
+    class Closing:
+        def __enter__(self):
+            return object()
+
+        def __exit__(self, *_args):
+            return None
+
+    class FakeKanban:
+        tasks = [
+            Task("queued", "todo"),
+            Task("active", "running"),
+            Task("done", "done"),
+        ]
+        moved: list[tuple[str, str]] = []
+
+        @staticmethod
+        def connect_closing(*, board: str):
+            assert board == "cassandra"
+            return Closing()
+
+        @classmethod
+        def list_tasks(cls, _connection, *, include_archived: bool):
+            assert include_archived is False
+            return cls.tasks
+
+        @classmethod
+        def set_workspace_path(cls, _connection, task_id: str, path: Path):
+            cls.moved.append((task_id, str(path)))
+
+    active = Path("/opt/data/workspace/projects/cassandra-hermes-v69")
+    coordinator._migrate_open_cassandra_tasks(FakeKanban, active)
+
+    assert FakeKanban.moved == [("queued", str(active))]

testing/tests/test_hermes_coordinator_routing.py

@@ -1,44 +1,18 @@
-"""Unit tests for Hermes coordinator model routing and profile generation."""
+"""Hermes coordinator model routing contracts."""
 
 from __future__ import annotations
 
-import importlib.util
-import json
-import subprocess
-import sys
-import tomllib
-from pathlib import Path
-
-import pytest
-import yaml
-
-
-SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py"
-sys.path.insert(0, str(SCRIPT.parent))
-routing = importlib.import_module("hermes_model_routing")
-catalog_resolver = importlib.import_module("routing_catalog")
-SPEC = importlib.util.spec_from_file_location("hermes_coordinator", SCRIPT)
-assert SPEC and SPEC.loader
-coordinator = importlib.util.module_from_spec(SPEC)
-sys.modules[SPEC.name] = coordinator
-SPEC.loader.exec_module(coordinator)
-
-
-def _base_config() -> dict:
-    """Return the minimal coordinator configuration used by routing tests."""
-    return {
-        "model": {
-            "provider": "openai-codex",
-            "default": "gpt-5.6-terra",
-            "model": "gpt-5.6-terra",
-        },
-        "fallback_providers": [
-            {"provider": "anthropic", "model": "claude-opus-5"},
-            dict(routing.LOCAL_FALLBACK),
-        ],
-        "toolsets": ["kanban"],
-        "terminal": {"cwd": "/opt/data/workspace"},
-    }
+from test_hermes_coordinator_support import (
+    Path,
+    SCRIPT,
+    _base_config,
+    catalog_resolver,
+    json,
+    routing,
+    subprocess,
+    tomllib,
+    yaml,
+)
 
 
 def test_model_version_and_quality_selection_handle_new_and_small_models():
@@ -60,12 +34,10 @@ def test_model_version_and_quality_selection_handle_new_and_small_models():
     assert routing.choose_claude_for_effort(claude, "medium") == "claude-sonnet-5"
     assert routing.choose_claude_for_effort(claude, "xhigh") == "claude-opus-4.8"
 
-
 def test_empty_catalog_retains_current_models():
     assert routing.choose_codex_model([], "gpt-5.6-terra") == "gpt-5.6-terra"
     assert routing.choose_claude_model([], "claude-opus-5") == "claude-opus-5"
 
-
 def test_dynamic_catalog_resolves_new_models_and_preserves_last_known_good():
     """Stable Switchyard aliases follow live releases and survive catalog outages."""
     codex = routing.Catalog(
@@ -112,7 +84,6 @@ def test_dynamic_catalog_resolves_new_models_and_preserves_last_known_good():
         == current["providers"]["claude"]["tiers"]
     )
 
-
 def test_live_catalog_never_routes_to_a_removed_model_tier():
     """A live provider catalog must replace a retired tier with a live model."""
     previous = {
@@ -144,7 +115,6 @@ def test_live_catalog_never_routes_to_a_removed_model_tier():
     assert current["providers"]["codex"]["tiers"]["luna"] in codex.models
     assert current["providers"]["claude"]["tiers"]["opus"] == "claude-sonnet-6"
 
-
 def test_switchyard_targets_have_unique_upstream_identities():
     """Switchyard drops duplicate client/model pairs, so reject them in Git."""
     manifest = yaml.safe_load(
@@ -163,7 +133,6 @@ def test_switchyard_targets_have_unique_upstream_identities():
         if route.get("target"):
             assert route["target"] in target_names
 
-
 def test_worker_alias_resolves_before_cli_launch():
     document = {
         "providers": {
@@ -178,7 +147,6 @@ def test_worker_alias_resolves_before_cli_launch():
         == "worker/codex/gpt-5.8-sol/high"
     )
 
-
 def test_codex_cli_login_counts_as_connected_runtime(monkeypatch):
     """AUTO routing must recognize the authenticated app-server CLI lane."""
     monkeypatch.setattr(routing.shutil, "which", lambda name: "/usr/bin/codex")
@@ -195,7 +163,6 @@ def test_codex_cli_login_counts_as_connected_runtime(monkeypatch):
 
     assert routing.codex_cli_authenticated() is True
 
-
 def test_configure_routes_keeps_every_profile_on_switchyard(tmp_path: Path):
     (tmp_path / "config.yaml").write_text(
         yaml.safe_dump(_base_config()), encoding="utf-8"
@@ -270,7 +237,6 @@ def test_configure_routes_keeps_every_profile_on_switchyard(tmp_path: Path):
     assert "claude-secret" not in json.dumps(routes)
     assert (tmp_path / "profiles/codex-high/.env").stat().st_mode & 0o777 == 0o600
 
-
 def test_degraded_catalog_does_not_replace_switchyard_authority(tmp_path: Path):
     base = _base_config()
     base["model"]["model"] = base["model"]["default"] = "gpt-5.6-sol"
@@ -286,7 +252,6 @@ def test_degraded_catalog_does_not_replace_switchyard_authority(tmp_path: Path):
     assert current["model"]["provider"] == "atlas-switchyard"
     assert current["fallback_providers"] == []
 
-
 def test_degraded_refresh_preserves_switchyard_worker_preference(tmp_path: Path):
     """A catalog outage must not bypass a managed worker's Switchyard route."""
     (tmp_path / "config.yaml").write_text(
@@ -311,200 +276,3 @@ def test_degraded_refresh_preserves_switchyard_worker_preference(tmp_path: Path)
     )
     assert worker["model"]["model"] == "atlas/manual/codex/sol"
     assert worker["model"]["provider"] == "atlas-switchyard"
-
-
-def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch):
-    (tmp_path / "config.yaml").write_text(
-        yaml.safe_dump(_base_config()), encoding="utf-8"
-    )
-    (tmp_path / ".env").write_text("GITEA_TOKEN=do-not-report\n", encoding="utf-8")
-    codex = routing.Catalog("openai-codex", ["gpt-5.6-terra"], True, True, "connected")
-    claude = routing.Catalog("anthropic", ["claude-opus-5"], True, True, "connected")
-    monkeypatch.setattr(coordinator, "discover_codex_models", lambda: codex)
-    monkeypatch.setattr(coordinator, "discover_claude_models", lambda: claude)
-    monkeypatch.setattr(
-        coordinator, "bootstrap_cassandra_state", lambda root: {"state": "ready"}
-    )
-    monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda env: "ready")
-
-    status = coordinator.refresh_once(tmp_path)
-
-    status_path = tmp_path / "workspace/coordinator/model-routing.json"
-    assert status_path.is_file()
-    assert status["projects"]["cassandra"]["state"] == "ready"
-    assert status["projects"]["cassandra"]["board_state"] == {"state": "ready"}
-    assert "do-not-report" not in status_path.read_text(encoding="utf-8")
-
-
-def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh(
-    tmp_path: Path, monkeypatch
-):
-    """A damaged project board degrades Kanban instead of stopping all services."""
-
-    class FakeKanbanDbCorruptError(Exception):
-        def __init__(self):
-            super().__init__("integrity check failed")
-            self.db_path = tmp_path / "kanban.db"
-            self.backup_path = tmp_path / "kanban.db.corrupt.backup"
-            self.reason = "row out of order"
-
-    def raise_corruption(_root: Path) -> None:
-        raise FakeKanbanDbCorruptError()
-
-    monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_corruption)
-    monkeypatch.setattr(
-        coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError
-    )
-
-    status = coordinator.bootstrap_cassandra_state(tmp_path)
-
-    assert status == {
-        "state": "corrupt-preserved",
-        "database": str(tmp_path / "kanban.db"),
-        "backup": str(tmp_path / "kanban.db.corrupt.backup"),
-        "reason": "row out of order",
-    }
-
-
-def test_unrelated_cassandra_bootstrap_failure_remains_fatal(
-    tmp_path: Path, monkeypatch
-):
-    """Only the known fail-closed corruption state may be degraded."""
-
-    class FakeKanbanDbCorruptError(Exception):
-        pass
-
-    def raise_permission_error(_root: Path) -> None:
-        raise PermissionError("cannot access board")
-
-    monkeypatch.setattr(coordinator, "bootstrap_cassandra", raise_permission_error)
-    monkeypatch.setattr(
-        coordinator, "_kanban_corruption_type", lambda: FakeKanbanDbCorruptError
-    )
-
-    try:
-        coordinator.bootstrap_cassandra_state(tmp_path)
-    except PermissionError as error:
-        assert str(error) == "cannot access board"
-    else:
-        raise AssertionError("unrelated board errors must fail the coordinator refresh")
-
-
-def test_cassandra_workspace_uses_valid_configured_worktree(
-    tmp_path: Path, monkeypatch
-):
-    """The coordinator selects an explicit Git worktree but rejects stale paths."""
-    active = tmp_path / "cassandra-v69"
-    active.mkdir()
-    (active / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8")
-    monkeypatch.setenv("HERMES_CASSANDRA_ACTIVE_WORKTREE", str(active))
-
-    assert coordinator.cassandra_workspace() == active
-
-    (active / ".git").unlink()
-    assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH
-
-
-@pytest.mark.parametrize(
-    ("current_remote", "repair_action"),
-    [
-        ("https://scm.bstein.dev/bstein/cassandra.git", "set-url"),
-        (None, "add"),
-        (coordinator.CASSANDRA_REMOTE, None),
-    ],
-    ids=["old-origin", "missing-origin", "canonical-origin"],
-)
-def test_cassandra_sync_repairs_existing_worktree(
-    tmp_path: Path, monkeypatch, current_remote: str | None, repair_action: str | None
-):
-    """Existing worktrees use the canonical Atlas origin before fetching."""
-    workspace = tmp_path / "cassandra"
-    (workspace / ".git").mkdir(parents=True)
-    monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace)
-    monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git")
-    token_path = tmp_path / "gitea-token"
-    token_path.write_text("configured\n", encoding="utf-8")
-    monkeypatch.setenv("HERMES_GITEA_TOKEN_FILE", str(token_path))
-    commands: list[list[str]] = []
-    environments: list[dict[str, str]] = []
-
-    def run(command, **kwargs):
-        commands.append(command)
-        environments.append(kwargs.get("env", {}))
-        if command[-2:] == ["get-url", "origin"]:
-            return subprocess.CompletedProcess(
-                command, 0 if current_remote else 2, stdout=f"{current_remote or ''}\n"
-            )
-        return subprocess.CompletedProcess(command, 0)
-
-    monkeypatch.setattr(coordinator.subprocess, "run", run)
-    assert coordinator.sync_cassandra_repo({"GITEA_TOKEN": "must-not-pass"}) == "ready"
-    repair_commands = [command for command in commands if repair_action in command]
-    assert bool(repair_commands) is bool(repair_action)
-    assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"]
-    assert all("GITEA_TOKEN" not in environment for environment in environments)
-
-
-def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch):
-    """Missing credentials skip only the fetch, not the local origin repair."""
-    workspace = tmp_path / "cassandra"
-    (workspace / ".git").mkdir(parents=True)
-    monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace)
-    monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git")
-    commands: list[list[str]] = []
-
-    def run(command, **_kwargs):
-        commands.append(command)
-        stdout = "old-origin\n" if command[-2:] == ["get-url", "origin"] else None
-        return subprocess.CompletedProcess(command, 0, stdout=stdout)
-
-    monkeypatch.setattr(coordinator.subprocess, "run", run)
-    state = coordinator.sync_cassandra_repo({})
-
-    assert state == "ready; fetch skipped until Gitea token is configured"
-    assert [command[-3] for command in commands] == ["remote", "set-url"]
-
-
-def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks():
-    """A project switch moves queued work without relocating active evidence."""
-
-    class Task:
-        def __init__(self, task_id: str, status: str):
-            self.id = task_id
-            self.status = status
-            self.workspace_kind = "dir"
-            self.workspace_path = str(coordinator.CASSANDRA_BASE_PATH)
-
-    class Closing:
-        def __enter__(self):
-            return object()
-
-        def __exit__(self, *_args):
-            return None
-
-    class FakeKanban:
-        tasks = [
-            Task("queued", "todo"),
-            Task("active", "running"),
-            Task("done", "done"),
-        ]
-        moved: list[tuple[str, str]] = []
-
-        @staticmethod
-        def connect_closing(*, board: str):
-            assert board == "cassandra"
-            return Closing()
-
-        @classmethod
-        def list_tasks(cls, _connection, *, include_archived: bool):
-            assert include_archived is False
-            return cls.tasks
-
-        @classmethod
-        def set_workspace_path(cls, _connection, task_id: str, path: Path):
-            cls.moved.append((task_id, str(path)))
-
-    active = Path("/opt/data/workspace/projects/cassandra-hermes-v69")
-    coordinator._migrate_open_cassandra_tasks(FakeKanban, active)
-
-    assert FakeKanban.moved == [("queued", str(active))]

testing/tests/test_hermes_coordinator_support.py

@@ -0,0 +1,70 @@
+"""Unit tests for Hermes coordinator model routing and profile generation."""
+
+from __future__ import annotations
+
+import importlib.util
+
+import json
+
+import subprocess
+
+import sys
+
+import tomllib
+
+from pathlib import Path
+
+import pytest
+
+import yaml
+
+SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py"
+
+sys.path.insert(0, str(SCRIPT.parent))
+
+routing = importlib.import_module("hermes_model_routing")
+
+catalog_resolver = importlib.import_module("routing_catalog")
+
+SPEC = importlib.util.spec_from_file_location("hermes_coordinator", SCRIPT)
+
+assert SPEC and SPEC.loader
+
+coordinator = importlib.util.module_from_spec(SPEC)
+
+sys.modules[SPEC.name] = coordinator
+
+SPEC.loader.exec_module(coordinator)
+
+def _base_config() -> dict:
+    """Return the minimal coordinator configuration used by routing tests."""
+    return {
+        "model": {
+            "provider": "openai-codex",
+            "default": "gpt-5.6-terra",
+            "model": "gpt-5.6-terra",
+        },
+        "fallback_providers": [
+            {"provider": "anthropic", "model": "claude-opus-5"},
+            dict(routing.LOCAL_FALLBACK),
+        ],
+        "toolsets": ["kanban"],
+        "terminal": {"cwd": "/opt/data/workspace"},
+    }
+
+__all__ = (
+    "Path",
+    "SCRIPT",
+    "SPEC",
+    "_base_config",
+    "catalog_resolver",
+    "coordinator",
+    "importlib",
+    "json",
+    "pytest",
+    "routing",
+    "subprocess",
+    "sys",
+    "tomllib",
+    "yaml",
+)

testing/tests/test_hermes_worker_recovery.py

@@ -42,7 +42,7 @@ class FakeKanban:
         self.tasks = tasks
         self.has_board = board_exists
         self.connection = FakeConnection()
-        self.reclaimed: list[tuple[str, str]] = []
+        self.reclaimed: list[tuple[str, str, int]] = []
         self.comments: list[tuple[str, str, str]] = []
 
     def board_exists(self, board: str) -> bool:
@@ -61,9 +61,16 @@ class FakeKanban:
         assert connection is self.connection
         return self.tasks
 
-    def reclaim_task(self, connection, task_id: str, *, reason: str) -> bool:
+    def reclaim_task(
+        self,
+        connection,
+        task_id: str,
+        *,
+        reason: str,
+        expected_run_id: int,
+    ) -> bool:
         assert connection is self.connection
-        self.reclaimed.append((task_id, reason))
+        self.reclaimed.append((task_id, reason, expected_run_id))
         return task_id != "t_race"
 
     def add_comment(self, connection, task_id: str, author: str, body: str) -> None:
@@ -75,16 +82,17 @@ def test_recover_running_tasks_requeues_only_claimed_workers() -> None:
     module = _load_module()
     kanban = FakeKanban(
         [
-            SimpleNamespace(id="t_running", status="running"),
+            SimpleNamespace(id="t_running", status="running", current_run_id=10),
             SimpleNamespace(id="t_done", status="done"),
-            SimpleNamespace(id="t_race", status="running"),
+            SimpleNamespace(id="t_race", status="running", current_run_id=11),
+            SimpleNamespace(id="t_missing_run", status="running", current_run_id=None),
         ]
     )
 
     assert module.recover_running_tasks(kanban) == ["t_running"]
     assert kanban.reclaimed == [
-        ("t_running", module.REASON),
-        ("t_race", module.REASON),
+        ("t_running", module.REASON, 10),
+        ("t_race", module.REASON, 11),
     ]
     assert kanban.comments == [
         (