"""Build-time regressions for Hermes automatic decomposition safety.""" from __future__ import annotations import json import os import tempfile import threading import unittest from types import SimpleNamespace from unittest import mock _HERMES_HOME = tempfile.TemporaryDirectory(prefix="hermes-execution-test-home-") os.environ["HERMES_HOME"] = _HERMES_HOME.name from agent import auxiliary_client # noqa: E402 from hermes_cli import kanban_db # noqa: E402 from hermes_cli import kanban_decompose # noqa: E402 class _FakeCompletions: def __init__(self, payload: dict, before_response=None) -> None: self.payload = payload self.before_response = before_response self.calls = 0 def create(self, **_kwargs): self.calls += 1 if self.before_response is not None: self.before_response() return SimpleNamespace( choices=[ SimpleNamespace( message=SimpleNamespace(content=json.dumps(self.payload)) ) ] ) class _FakeClient: def __init__(self, completions: _FakeCompletions) -> None: self.chat = SimpleNamespace(completions=completions) class AutomaticDecompositionSafetyTests(unittest.TestCase): """Exercise the patched upstream APIs against an isolated real database.""" def setUp(self) -> None: self.connection = kanban_db.connect() self.patches = [ mock.patch.object( kanban_decompose, "_build_roster", return_value=([], {"default"}), ), mock.patch.object( kanban_decompose, "_load_config", return_value={"kanban": {"auto_promote_children": True}}, ), mock.patch.object( auxiliary_client, "get_auxiliary_extra_body", return_value={}, ), ] for patch in self.patches: patch.start() def tearDown(self) -> None: for patch in reversed(self.patches): patch.stop() self.connection.close() def _client(self, payload: dict, before_response=None) -> _FakeCompletions: completions = _FakeCompletions(payload, before_response) client = _FakeClient(completions) patch = mock.patch.object( auxiliary_client, "get_text_auxiliary_client", return_value=(client, "test-decomposer"), ) patch.start() self.addCleanup(patch.stop) return completions def _status(self, task_id: str) -> str: task = kanban_db.get_task(self.connection, task_id) self.assertIsNotNone(task) return task.status def _route_executed_task_to_triage(self) -> str: task_id = kanban_db.create_task( self.connection, title="already implemented parent", ) for attempt in range(kanban_db.BLOCK_RECURRENCE_LIMIT): if attempt: self.assertTrue(kanban_db.unblock_task(self.connection, task_id)) self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) self.assertTrue( kanban_db.block_task( self.connection, task_id, reason="same unavailable capability", kind="capability", ) ) self.assertEqual(self._status(task_id), "triage") self.assertEqual( len(kanban_db.list_runs(self.connection, task_id)), kanban_db.BLOCK_RECURRENCE_LIMIT, ) return task_id def test_automatic_decomposer_skips_executed_triage_task(self) -> None: task_id = self._route_executed_task_to_triage() completions = self._client( {"fanout": False, "title": "must not run", "body": "must not run"} ) before = len(kanban_db.list_tasks(self.connection)) outcome = kanban_decompose.decompose_task(task_id, automatic=True) self.assertFalse(outcome.ok) self.assertIn("execution history", outcome.reason) self.assertEqual(completions.calls, 0) self.assertEqual(self._status(task_id), "triage") self.assertEqual(len(kanban_db.list_tasks(self.connection)), before) def test_manual_decomposition_remains_available_after_execution(self) -> None: task_id = self._route_executed_task_to_triage() completions = self._client( { "fanout": False, "title": "operator-approved retry", "body": "Retain one bounded task.", "assignee": "default", } ) outcome = kanban_decompose.decompose_task(task_id, author="operator") self.assertTrue(outcome.ok) self.assertFalse(outcome.fanout) self.assertEqual(completions.calls, 1) self.assertEqual(self._status(task_id), "ready") def test_latest_ended_run_can_replay_a_durable_completion(self) -> None: task_id = kanban_db.create_task(self.connection, title="journaled result") self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) run_id = kanban_db.get_task(self.connection, task_id).current_run_id self.assertTrue( kanban_db.block_task( self.connection, task_id, reason="legacy post-journal failure", kind="capability", expected_run_id=run_id, ) ) self.assertTrue( kanban_db.complete_task( self.connection, task_id, result="durable terminal result", summary="durable terminal result", replay_ended_run_id=run_id, ) ) self.assertEqual(self._status(task_id), "done") def test_older_ended_run_cannot_complete_over_a_replacement(self) -> None: task_id = kanban_db.create_task(self.connection, title="replacement guard") self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) old_run = kanban_db.get_task(self.connection, task_id).current_run_id self.assertTrue( kanban_db.block_task( self.connection, task_id, reason="first run", kind="capability", expected_run_id=old_run, ) ) self.assertTrue(kanban_db.unblock_task(self.connection, task_id)) self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) replacement_run = kanban_db.get_task(self.connection, task_id).current_run_id self.assertNotEqual(old_run, replacement_run) self.assertTrue( kanban_db.block_task( self.connection, task_id, reason="replacement run", kind="capability", expected_run_id=replacement_run, ) ) self.assertFalse( kanban_db.complete_task( self.connection, task_id, result="stale result", replay_ended_run_id=old_run, ) ) self.assertIn(self._status(task_id), {"blocked", "triage"}) def test_exact_run_reclaim_succeeds_for_the_authoritative_run(self) -> None: task_id = kanban_db.create_task(self.connection, title="recover exact run") self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) run_id = kanban_db.get_task(self.connection, task_id).current_run_id self.assertTrue( kanban_db.reclaim_task( self.connection, task_id, reason="invalid exact journal", expected_run_id=run_id, ) ) task = kanban_db.get_task(self.connection, task_id) self.assertEqual(task.status, "ready") self.assertIsNone(task.current_run_id) def test_stale_reclaim_before_transaction_preserves_replacement_run(self) -> None: task_id = kanban_db.create_task(self.connection, title="replacement before txn") self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) old_run = kanban_db.get_task(self.connection, task_id).current_run_id self.assertTrue( kanban_db.block_task( self.connection, task_id, reason="old recovery run", expected_run_id=old_run, ) ) self.assertTrue(kanban_db.unblock_task(self.connection, task_id)) self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) replacement = kanban_db.get_task(self.connection, task_id) replacement_run = replacement.current_run_id self.assertNotEqual(old_run, replacement_run) signals = [] self.assertFalse( kanban_db.reclaim_task( self.connection, task_id, reason="stale journal", expected_run_id=old_run, signal_fn=lambda *args: signals.append(args), ) ) latest = kanban_db.get_task(self.connection, task_id) self.assertEqual(latest.status, "running") self.assertEqual(latest.current_run_id, replacement_run) self.assertEqual(latest.claim_lock, replacement.claim_lock) self.assertEqual(signals, []) def test_reclaim_update_guard_preserves_run_changed_inside_transaction(self) -> None: task_id = kanban_db.create_task(self.connection, title="replacement in txn") self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) old_run = kanban_db.get_task(self.connection, task_id).current_run_id replacement = {} def install_replacement(*_args, **_kwargs): cursor = self.connection.execute( "INSERT INTO task_runs (task_id, status, claim_lock, started_at) " "VALUES (?, 'running', ?, strftime('%s','now'))", (task_id, "replacement-lock"), ) replacement["run_id"] = int(cursor.lastrowid) self.connection.execute( "UPDATE tasks SET current_run_id = ?, claim_lock = ? WHERE id = ?", (replacement["run_id"], "replacement-lock", task_id), ) return {} with mock.patch.object( kanban_db, "_terminate_reclaimed_worker", side_effect=install_replacement, ): self.assertFalse( kanban_db.reclaim_task( self.connection, task_id, reason="journal for old run", expected_run_id=old_run, ) ) latest = kanban_db.get_task(self.connection, task_id) self.assertEqual(latest.status, "running") self.assertEqual(latest.current_run_id, replacement["run_id"]) self.assertEqual(latest.claim_lock, "replacement-lock") def test_concurrent_exact_reclaim_and_completion_have_one_winner(self) -> None: task_id = kanban_db.create_task(self.connection, title="concurrent finalizer") self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id)) run_id = kanban_db.get_task(self.connection, task_id).current_run_id barrier = threading.Barrier(2) outcomes = {} errors = [] def reclaim() -> None: try: with kanban_db.connect_closing() as connection: barrier.wait() outcomes["reclaim"] = kanban_db.reclaim_task( connection, task_id, reason="concurrent invalid journal", expected_run_id=run_id, ) except BaseException as error: # pragma: no cover - assertion relay errors.append(error) def finalize() -> None: try: with kanban_db.connect_closing() as connection: barrier.wait() outcomes["complete"] = kanban_db.complete_task( connection, task_id, result="durable winner", summary="durable winner", expected_run_id=run_id, ) except BaseException as error: # pragma: no cover - assertion relay errors.append(error) threads = [threading.Thread(target=reclaim), threading.Thread(target=finalize)] for thread in threads: thread.start() for thread in threads: thread.join(timeout=10) self.assertFalse(thread.is_alive()) self.assertEqual(errors, []) self.assertEqual(set(outcomes), {"reclaim", "complete"}) self.assertEqual(sum(bool(value) for value in outcomes.values()), 1) latest = kanban_db.get_task(self.connection, task_id) self.assertIn(latest.status, {"done", "ready"}) self.assertIsNone(latest.current_run_id) runs = kanban_db.list_runs(self.connection, task_id) self.assertEqual(len(runs), 1) self.assertIsNotNone(runs[0].ended_at) def test_fresh_triage_task_still_auto_promotes(self) -> None: task_id = kanban_db.create_task( self.connection, title="fresh objective", triage=True, ) completions = self._client( { "fanout": False, "title": "specified objective", "body": "One bounded task.", "assignee": "default", } ) outcome = kanban_decompose.decompose_task(task_id, automatic=True) self.assertTrue(outcome.ok) self.assertEqual(completions.calls, 1) self.assertEqual(self._status(task_id), "ready") def _insert_run(self, task_id: str) -> None: with kanban_db.connect_closing() as connection: kanban_db._synthesize_ended_run( connection, task_id, outcome="reclaimed", summary="concurrent execution evidence", ) def test_execution_history_gained_during_single_llm_call_blocks_commit(self) -> None: task_id = kanban_db.create_task( self.connection, title="concurrent single objective", triage=True, ) completions = self._client( { "fanout": False, "title": "redundant rewrite", "body": "must not be committed", "assignee": "default", }, before_response=lambda: self._insert_run(task_id), ) outcome = kanban_decompose.decompose_task(task_id, automatic=True) self.assertFalse(outcome.ok) self.assertIn("gained execution history", outcome.reason) self.assertEqual(completions.calls, 1) self.assertEqual(self._status(task_id), "triage") def test_execution_history_gained_during_fanout_llm_call_blocks_commit(self) -> None: task_id = kanban_db.create_task( self.connection, title="concurrent triage objective", triage=True, ) completions = self._client( { "fanout": True, "tasks": [ { "title": "redundant child", "body": "must not be created", "assignee": "default", "parents": [], } ], }, before_response=lambda: self._insert_run(task_id), ) before = len(kanban_db.list_tasks(self.connection)) outcome = kanban_decompose.decompose_task(task_id, automatic=True) self.assertFalse(outcome.ok) self.assertIn("gained execution history", outcome.reason) self.assertEqual(completions.calls, 1) self.assertEqual(self._status(task_id), "triage") self.assertEqual(len(kanban_db.list_tasks(self.connection)), before) if __name__ == "__main__": unittest.main(verbosity=2)