440 lines
16 KiB
Python
440 lines
16 KiB
Python
"""Build-time regressions for Hermes automatic decomposition safety."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
from unittest import mock
|
|
|
|
|
|
_HERMES_HOME = tempfile.TemporaryDirectory(prefix="hermes-execution-test-home-")
|
|
os.environ["HERMES_HOME"] = _HERMES_HOME.name
|
|
|
|
from agent import auxiliary_client # noqa: E402
|
|
from hermes_cli import kanban_db # noqa: E402
|
|
from hermes_cli import kanban_decompose # noqa: E402
|
|
|
|
|
|
class _FakeCompletions:
|
|
def __init__(self, payload: dict, before_response=None) -> None:
|
|
self.payload = payload
|
|
self.before_response = before_response
|
|
self.calls = 0
|
|
|
|
def create(self, **_kwargs):
|
|
self.calls += 1
|
|
if self.before_response is not None:
|
|
self.before_response()
|
|
return SimpleNamespace(
|
|
choices=[
|
|
SimpleNamespace(
|
|
message=SimpleNamespace(content=json.dumps(self.payload))
|
|
)
|
|
]
|
|
)
|
|
|
|
|
|
class _FakeClient:
|
|
def __init__(self, completions: _FakeCompletions) -> None:
|
|
self.chat = SimpleNamespace(completions=completions)
|
|
|
|
|
|
class AutomaticDecompositionSafetyTests(unittest.TestCase):
|
|
"""Exercise the patched upstream APIs against an isolated real database."""
|
|
|
|
def setUp(self) -> None:
|
|
self.connection = kanban_db.connect()
|
|
self.patches = [
|
|
mock.patch.object(
|
|
kanban_decompose,
|
|
"_build_roster",
|
|
return_value=([], {"default"}),
|
|
),
|
|
mock.patch.object(
|
|
kanban_decompose,
|
|
"_load_config",
|
|
return_value={"kanban": {"auto_promote_children": True}},
|
|
),
|
|
mock.patch.object(
|
|
auxiliary_client,
|
|
"get_auxiliary_extra_body",
|
|
return_value={},
|
|
),
|
|
]
|
|
for patch in self.patches:
|
|
patch.start()
|
|
|
|
def tearDown(self) -> None:
|
|
for patch in reversed(self.patches):
|
|
patch.stop()
|
|
self.connection.close()
|
|
|
|
def _client(self, payload: dict, before_response=None) -> _FakeCompletions:
|
|
completions = _FakeCompletions(payload, before_response)
|
|
client = _FakeClient(completions)
|
|
patch = mock.patch.object(
|
|
auxiliary_client,
|
|
"get_text_auxiliary_client",
|
|
return_value=(client, "test-decomposer"),
|
|
)
|
|
patch.start()
|
|
self.addCleanup(patch.stop)
|
|
return completions
|
|
|
|
def _status(self, task_id: str) -> str:
|
|
task = kanban_db.get_task(self.connection, task_id)
|
|
self.assertIsNotNone(task)
|
|
return task.status
|
|
|
|
def _route_executed_task_to_triage(self) -> str:
|
|
task_id = kanban_db.create_task(
|
|
self.connection,
|
|
title="already implemented parent",
|
|
)
|
|
for attempt in range(kanban_db.BLOCK_RECURRENCE_LIMIT):
|
|
if attempt:
|
|
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
|
|
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
|
self.assertTrue(
|
|
kanban_db.block_task(
|
|
self.connection,
|
|
task_id,
|
|
reason="same unavailable capability",
|
|
kind="capability",
|
|
)
|
|
)
|
|
self.assertEqual(self._status(task_id), "triage")
|
|
self.assertEqual(
|
|
len(kanban_db.list_runs(self.connection, task_id)),
|
|
kanban_db.BLOCK_RECURRENCE_LIMIT,
|
|
)
|
|
return task_id
|
|
|
|
def test_automatic_decomposer_skips_executed_triage_task(self) -> None:
|
|
task_id = self._route_executed_task_to_triage()
|
|
completions = self._client(
|
|
{"fanout": False, "title": "must not run", "body": "must not run"}
|
|
)
|
|
before = len(kanban_db.list_tasks(self.connection))
|
|
|
|
outcome = kanban_decompose.decompose_task(task_id, automatic=True)
|
|
|
|
self.assertFalse(outcome.ok)
|
|
self.assertIn("execution history", outcome.reason)
|
|
self.assertEqual(completions.calls, 0)
|
|
self.assertEqual(self._status(task_id), "triage")
|
|
self.assertEqual(len(kanban_db.list_tasks(self.connection)), before)
|
|
|
|
def test_manual_decomposition_remains_available_after_execution(self) -> None:
|
|
task_id = self._route_executed_task_to_triage()
|
|
completions = self._client(
|
|
{
|
|
"fanout": False,
|
|
"title": "operator-approved retry",
|
|
"body": "Retain one bounded task.",
|
|
"assignee": "default",
|
|
}
|
|
)
|
|
|
|
outcome = kanban_decompose.decompose_task(task_id, author="operator")
|
|
|
|
self.assertTrue(outcome.ok)
|
|
self.assertFalse(outcome.fanout)
|
|
self.assertEqual(completions.calls, 1)
|
|
self.assertEqual(self._status(task_id), "ready")
|
|
|
|
def test_latest_ended_run_can_replay_a_durable_completion(self) -> None:
|
|
task_id = kanban_db.create_task(self.connection, title="journaled result")
|
|
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
|
run_id = kanban_db.get_task(self.connection, task_id).current_run_id
|
|
self.assertTrue(
|
|
kanban_db.block_task(
|
|
self.connection,
|
|
task_id,
|
|
reason="legacy post-journal failure",
|
|
kind="capability",
|
|
expected_run_id=run_id,
|
|
)
|
|
)
|
|
|
|
self.assertTrue(
|
|
kanban_db.complete_task(
|
|
self.connection,
|
|
task_id,
|
|
result="durable terminal result",
|
|
summary="durable terminal result",
|
|
replay_ended_run_id=run_id,
|
|
)
|
|
)
|
|
self.assertEqual(self._status(task_id), "done")
|
|
|
|
def test_older_ended_run_cannot_complete_over_a_replacement(self) -> None:
|
|
task_id = kanban_db.create_task(self.connection, title="replacement guard")
|
|
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
|
old_run = kanban_db.get_task(self.connection, task_id).current_run_id
|
|
self.assertTrue(
|
|
kanban_db.block_task(
|
|
self.connection,
|
|
task_id,
|
|
reason="first run",
|
|
kind="capability",
|
|
expected_run_id=old_run,
|
|
)
|
|
)
|
|
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
|
|
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
|
replacement_run = kanban_db.get_task(self.connection, task_id).current_run_id
|
|
self.assertNotEqual(old_run, replacement_run)
|
|
self.assertTrue(
|
|
kanban_db.block_task(
|
|
self.connection,
|
|
task_id,
|
|
reason="replacement run",
|
|
kind="capability",
|
|
expected_run_id=replacement_run,
|
|
)
|
|
)
|
|
|
|
self.assertFalse(
|
|
kanban_db.complete_task(
|
|
self.connection,
|
|
task_id,
|
|
result="stale result",
|
|
replay_ended_run_id=old_run,
|
|
)
|
|
)
|
|
self.assertIn(self._status(task_id), {"blocked", "triage"})
|
|
|
|
def test_exact_run_reclaim_succeeds_for_the_authoritative_run(self) -> None:
|
|
task_id = kanban_db.create_task(self.connection, title="recover exact run")
|
|
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
|
run_id = kanban_db.get_task(self.connection, task_id).current_run_id
|
|
|
|
self.assertTrue(
|
|
kanban_db.reclaim_task(
|
|
self.connection,
|
|
task_id,
|
|
reason="invalid exact journal",
|
|
expected_run_id=run_id,
|
|
)
|
|
)
|
|
task = kanban_db.get_task(self.connection, task_id)
|
|
self.assertEqual(task.status, "ready")
|
|
self.assertIsNone(task.current_run_id)
|
|
|
|
def test_stale_reclaim_before_transaction_preserves_replacement_run(self) -> None:
|
|
task_id = kanban_db.create_task(self.connection, title="replacement before txn")
|
|
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
|
old_run = kanban_db.get_task(self.connection, task_id).current_run_id
|
|
self.assertTrue(
|
|
kanban_db.block_task(
|
|
self.connection,
|
|
task_id,
|
|
reason="old recovery run",
|
|
expected_run_id=old_run,
|
|
)
|
|
)
|
|
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
|
|
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
|
replacement = kanban_db.get_task(self.connection, task_id)
|
|
replacement_run = replacement.current_run_id
|
|
self.assertNotEqual(old_run, replacement_run)
|
|
signals = []
|
|
|
|
self.assertFalse(
|
|
kanban_db.reclaim_task(
|
|
self.connection,
|
|
task_id,
|
|
reason="stale journal",
|
|
expected_run_id=old_run,
|
|
signal_fn=lambda *args: signals.append(args),
|
|
)
|
|
)
|
|
latest = kanban_db.get_task(self.connection, task_id)
|
|
self.assertEqual(latest.status, "running")
|
|
self.assertEqual(latest.current_run_id, replacement_run)
|
|
self.assertEqual(latest.claim_lock, replacement.claim_lock)
|
|
self.assertEqual(signals, [])
|
|
|
|
def test_reclaim_update_guard_preserves_run_changed_inside_transaction(self) -> None:
|
|
task_id = kanban_db.create_task(self.connection, title="replacement in txn")
|
|
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
|
old_run = kanban_db.get_task(self.connection, task_id).current_run_id
|
|
replacement = {}
|
|
|
|
def install_replacement(*_args, **_kwargs):
|
|
cursor = self.connection.execute(
|
|
"INSERT INTO task_runs (task_id, status, claim_lock, started_at) "
|
|
"VALUES (?, 'running', ?, strftime('%s','now'))",
|
|
(task_id, "replacement-lock"),
|
|
)
|
|
replacement["run_id"] = int(cursor.lastrowid)
|
|
self.connection.execute(
|
|
"UPDATE tasks SET current_run_id = ?, claim_lock = ? WHERE id = ?",
|
|
(replacement["run_id"], "replacement-lock", task_id),
|
|
)
|
|
return {}
|
|
|
|
with mock.patch.object(
|
|
kanban_db,
|
|
"_terminate_reclaimed_worker",
|
|
side_effect=install_replacement,
|
|
):
|
|
self.assertFalse(
|
|
kanban_db.reclaim_task(
|
|
self.connection,
|
|
task_id,
|
|
reason="journal for old run",
|
|
expected_run_id=old_run,
|
|
)
|
|
)
|
|
|
|
latest = kanban_db.get_task(self.connection, task_id)
|
|
self.assertEqual(latest.status, "running")
|
|
self.assertEqual(latest.current_run_id, replacement["run_id"])
|
|
self.assertEqual(latest.claim_lock, "replacement-lock")
|
|
|
|
def test_concurrent_exact_reclaim_and_completion_have_one_winner(self) -> None:
|
|
task_id = kanban_db.create_task(self.connection, title="concurrent finalizer")
|
|
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
|
|
run_id = kanban_db.get_task(self.connection, task_id).current_run_id
|
|
barrier = threading.Barrier(2)
|
|
outcomes = {}
|
|
errors = []
|
|
|
|
def reclaim() -> None:
|
|
try:
|
|
with kanban_db.connect_closing() as connection:
|
|
barrier.wait()
|
|
outcomes["reclaim"] = kanban_db.reclaim_task(
|
|
connection,
|
|
task_id,
|
|
reason="concurrent invalid journal",
|
|
expected_run_id=run_id,
|
|
)
|
|
except BaseException as error: # pragma: no cover - assertion relay
|
|
errors.append(error)
|
|
|
|
def finalize() -> None:
|
|
try:
|
|
with kanban_db.connect_closing() as connection:
|
|
barrier.wait()
|
|
outcomes["complete"] = kanban_db.complete_task(
|
|
connection,
|
|
task_id,
|
|
result="durable winner",
|
|
summary="durable winner",
|
|
expected_run_id=run_id,
|
|
)
|
|
except BaseException as error: # pragma: no cover - assertion relay
|
|
errors.append(error)
|
|
|
|
threads = [threading.Thread(target=reclaim), threading.Thread(target=finalize)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join(timeout=10)
|
|
self.assertFalse(thread.is_alive())
|
|
|
|
self.assertEqual(errors, [])
|
|
self.assertEqual(set(outcomes), {"reclaim", "complete"})
|
|
self.assertEqual(sum(bool(value) for value in outcomes.values()), 1)
|
|
latest = kanban_db.get_task(self.connection, task_id)
|
|
self.assertIn(latest.status, {"done", "ready"})
|
|
self.assertIsNone(latest.current_run_id)
|
|
runs = kanban_db.list_runs(self.connection, task_id)
|
|
self.assertEqual(len(runs), 1)
|
|
self.assertIsNotNone(runs[0].ended_at)
|
|
|
|
def test_fresh_triage_task_still_auto_promotes(self) -> None:
|
|
task_id = kanban_db.create_task(
|
|
self.connection,
|
|
title="fresh objective",
|
|
triage=True,
|
|
)
|
|
completions = self._client(
|
|
{
|
|
"fanout": False,
|
|
"title": "specified objective",
|
|
"body": "One bounded task.",
|
|
"assignee": "default",
|
|
}
|
|
)
|
|
|
|
outcome = kanban_decompose.decompose_task(task_id, automatic=True)
|
|
|
|
self.assertTrue(outcome.ok)
|
|
self.assertEqual(completions.calls, 1)
|
|
self.assertEqual(self._status(task_id), "ready")
|
|
|
|
def _insert_run(self, task_id: str) -> None:
|
|
with kanban_db.connect_closing() as connection:
|
|
kanban_db._synthesize_ended_run(
|
|
connection,
|
|
task_id,
|
|
outcome="reclaimed",
|
|
summary="concurrent execution evidence",
|
|
)
|
|
|
|
def test_execution_history_gained_during_single_llm_call_blocks_commit(self) -> None:
|
|
task_id = kanban_db.create_task(
|
|
self.connection,
|
|
title="concurrent single objective",
|
|
triage=True,
|
|
)
|
|
completions = self._client(
|
|
{
|
|
"fanout": False,
|
|
"title": "redundant rewrite",
|
|
"body": "must not be committed",
|
|
"assignee": "default",
|
|
},
|
|
before_response=lambda: self._insert_run(task_id),
|
|
)
|
|
|
|
outcome = kanban_decompose.decompose_task(task_id, automatic=True)
|
|
|
|
self.assertFalse(outcome.ok)
|
|
self.assertIn("gained execution history", outcome.reason)
|
|
self.assertEqual(completions.calls, 1)
|
|
self.assertEqual(self._status(task_id), "triage")
|
|
|
|
def test_execution_history_gained_during_fanout_llm_call_blocks_commit(self) -> None:
|
|
task_id = kanban_db.create_task(
|
|
self.connection,
|
|
title="concurrent triage objective",
|
|
triage=True,
|
|
)
|
|
|
|
completions = self._client(
|
|
{
|
|
"fanout": True,
|
|
"tasks": [
|
|
{
|
|
"title": "redundant child",
|
|
"body": "must not be created",
|
|
"assignee": "default",
|
|
"parents": [],
|
|
}
|
|
],
|
|
},
|
|
before_response=lambda: self._insert_run(task_id),
|
|
)
|
|
before = len(kanban_db.list_tasks(self.connection))
|
|
|
|
outcome = kanban_decompose.decompose_task(task_id, automatic=True)
|
|
|
|
self.assertFalse(outcome.ok)
|
|
self.assertIn("gained execution history", outcome.reason)
|
|
self.assertEqual(completions.calls, 1)
|
|
self.assertEqual(self._status(task_id), "triage")
|
|
self.assertEqual(len(kanban_db.list_tasks(self.connection)), before)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|