atlas-iac/dockerfiles/hermes-execution-safety-regression.py

"""Build-time regressions for Hermes automatic decomposition safety."""

from __future__ import annotations

import json
import os
import tempfile
import unittest
from types import SimpleNamespace
from unittest import mock


_HERMES_HOME = tempfile.TemporaryDirectory(prefix="hermes-execution-test-home-")
os.environ["HERMES_HOME"] = _HERMES_HOME.name

from agent import auxiliary_client  # noqa: E402
from hermes_cli import kanban_db  # noqa: E402
from hermes_cli import kanban_decompose  # noqa: E402


class _FakeCompletions:
    def __init__(self, payload: dict, before_response=None) -> None:
        self.payload = payload
        self.before_response = before_response
        self.calls = 0

    def create(self, **_kwargs):
        self.calls += 1
        if self.before_response is not None:
            self.before_response()
        return SimpleNamespace(
            choices=[
                SimpleNamespace(
                    message=SimpleNamespace(content=json.dumps(self.payload))
                )
            ]
        )


class _FakeClient:
    def __init__(self, completions: _FakeCompletions) -> None:
        self.chat = SimpleNamespace(completions=completions)


class AutomaticDecompositionSafetyTests(unittest.TestCase):
    """Exercise the patched upstream APIs against an isolated real database."""

    def setUp(self) -> None:
        self.connection = kanban_db.connect()
        self.patches = [
            mock.patch.object(
                kanban_decompose,
                "_build_roster",
                return_value=([], {"default"}),
            ),
            mock.patch.object(
                kanban_decompose,
                "_load_config",
                return_value={"kanban": {"auto_promote_children": True}},
            ),
            mock.patch.object(
                auxiliary_client,
                "get_auxiliary_extra_body",
                return_value={},
            ),
        ]
        for patch in self.patches:
            patch.start()

    def tearDown(self) -> None:
        for patch in reversed(self.patches):
            patch.stop()
        self.connection.close()

    def _client(self, payload: dict, before_response=None) -> _FakeCompletions:
        completions = _FakeCompletions(payload, before_response)
        client = _FakeClient(completions)
        patch = mock.patch.object(
            auxiliary_client,
            "get_text_auxiliary_client",
            return_value=(client, "test-decomposer"),
        )
        patch.start()
        self.addCleanup(patch.stop)
        return completions

    def _status(self, task_id: str) -> str:
        task = kanban_db.get_task(self.connection, task_id)
        self.assertIsNotNone(task)
        return task.status

    def _route_executed_task_to_triage(self) -> str:
        task_id = kanban_db.create_task(
            self.connection,
            title="already implemented parent",
        )
        for attempt in range(kanban_db.BLOCK_RECURRENCE_LIMIT):
            if attempt:
                self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
            self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
            self.assertTrue(
                kanban_db.block_task(
                    self.connection,
                    task_id,
                    reason="same unavailable capability",
                    kind="capability",
                )
            )
        self.assertEqual(self._status(task_id), "triage")
        self.assertEqual(
            len(kanban_db.list_runs(self.connection, task_id)),
            kanban_db.BLOCK_RECURRENCE_LIMIT,
        )
        return task_id

    def test_automatic_decomposer_skips_executed_triage_task(self) -> None:
        task_id = self._route_executed_task_to_triage()
        completions = self._client(
            {"fanout": False, "title": "must not run", "body": "must not run"}
        )
        before = len(kanban_db.list_tasks(self.connection))

        outcome = kanban_decompose.decompose_task(task_id, automatic=True)

        self.assertFalse(outcome.ok)
        self.assertIn("execution history", outcome.reason)
        self.assertEqual(completions.calls, 0)
        self.assertEqual(self._status(task_id), "triage")
        self.assertEqual(len(kanban_db.list_tasks(self.connection)), before)

    def test_manual_decomposition_remains_available_after_execution(self) -> None:
        task_id = self._route_executed_task_to_triage()
        completions = self._client(
            {
                "fanout": False,
                "title": "operator-approved retry",
                "body": "Retain one bounded task.",
                "assignee": "default",
            }
        )

        outcome = kanban_decompose.decompose_task(task_id, author="operator")

        self.assertTrue(outcome.ok)
        self.assertFalse(outcome.fanout)
        self.assertEqual(completions.calls, 1)
        self.assertEqual(self._status(task_id), "ready")

    def test_fresh_triage_task_still_auto_promotes(self) -> None:
        task_id = kanban_db.create_task(
            self.connection,
            title="fresh objective",
            triage=True,
        )
        completions = self._client(
            {
                "fanout": False,
                "title": "specified objective",
                "body": "One bounded task.",
                "assignee": "default",
            }
        )

        outcome = kanban_decompose.decompose_task(task_id, automatic=True)

        self.assertTrue(outcome.ok)
        self.assertEqual(completions.calls, 1)
        self.assertEqual(self._status(task_id), "ready")

    def test_execution_history_gained_during_llm_call_blocks_commit(self) -> None:
        task_id = kanban_db.create_task(
            self.connection,
            title="concurrent triage objective",
            triage=True,
        )

        def add_run() -> None:
            with kanban_db.connect_closing() as connection:
                kanban_db._synthesize_ended_run(
                    connection,
                    task_id,
                    outcome="reclaimed",
                    summary="concurrent execution evidence",
                )

        completions = self._client(
            {
                "fanout": True,
                "tasks": [
                    {
                        "title": "redundant child",
                        "body": "must not be created",
                        "assignee": "default",
                        "parents": [],
                    }
                ],
            },
            before_response=add_run,
        )
        before = len(kanban_db.list_tasks(self.connection))

        outcome = kanban_decompose.decompose_task(task_id, automatic=True)

        self.assertFalse(outcome.ok)
        self.assertIn("gained execution history", outcome.reason)
        self.assertEqual(completions.calls, 1)
        self.assertEqual(self._status(task_id), "triage")
        self.assertEqual(len(kanban_db.list_tasks(self.connection)), before)


if __name__ == "__main__":
    unittest.main(verbosity=2)