atlas-iac/testing/tests/test_hermes_cli_finalization.py

"""Terminal-result validation and exact-run finalization."""

from __future__ import annotations

from testing.tests.test_hermes_cli_support import (
    Path,
    SimpleNamespace,
    _completed_result,
    errno,
    json,
    lanes,
    nullcontext,
    pytest,
    sys,
)


def test_terminal_recovery_can_complete_the_exact_latest_ended_run(
    tmp_path: Path,
    monkeypatch,
):
    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
    state_file = lanes.state_path("cassandra", "t_ended")
    pending, _record = lanes._write_terminal_record(
        state_file,
        board="cassandra",
        task_id="t_ended",
        run_id=17,
        structured=_completed_result("accepted before legacy block"),
        summary="accepted before legacy block",
        metadata={},
    )
    task = SimpleNamespace(
        id="t_ended",
        status="blocked",
        result=None,
        current_run_id=None,
        assignee="cli-auto",
    )
    guards = []

    class Connection:
        def close(self):
            return None

    def complete_task(_conn, _task_id, **kwargs):
        guards.append(kwargs)
        task.status = "done"
        task.result = kwargs["result"]
        return True

    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        get_task=lambda _conn, _task_id: task,
        complete_task=complete_task,
    )
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))

    assert lanes.recover_pending_finalizations() == 1
    assert guards[0]["replay_ended_run_id"] == 17
    assert "expected_run_id" not in guards[0]
    assert task.status == "done"
    assert not pending.exists()


def test_terminal_recovery_completes_an_unchanged_triage_exact_run(
    tmp_path: Path,
    monkeypatch,
):
    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
    pending, record = lanes._write_terminal_record(
        lanes.state_path("cassandra", "t_triage_ended"),
        board="cassandra",
        task_id="t_triage_ended",
        run_id=18,
        structured=_completed_result("accepted before loop-breaker triage"),
        summary="accepted before loop-breaker triage",
        metadata={},
    )
    task = SimpleNamespace(
        id="t_triage_ended",
        status="triage",
        result=None,
        current_run_id=None,
        completed_run_id=None,
        assignee="cli-auto",
    )
    guards = []

    class Connection:
        def close(self):
            return None

    def complete_task(_conn, _task_id, **kwargs):
        guards.append(kwargs)
        assert kwargs == {
            "result": record["result"],
            "summary": record["summary"],
            "metadata": {},
            "replay_ended_run_id": 18,
        }
        task.status = "done"
        task.result = kwargs["result"]
        task.completed_run_id = 18
        return True

    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        get_task=lambda _conn, _task_id: task,
        complete_task=complete_task,
    )
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))

    assert lanes.recover_pending_finalizations() == 1
    assert len(guards) == 1
    assert task.status == "done"
    assert task.completed_run_id == 18
    assert not pending.exists()
    assert lanes.recover_pending_finalizations() == 0
    assert len(guards) == 1


def test_complete_exception_after_journal_is_replayable_and_never_blocks(
    tmp_path: Path,
    monkeypatch,
):
    state_root = tmp_path / "cli-lanes"
    monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
    task = SimpleNamespace(
        id="t_crash",
        status="running",
        result=None,
        current_run_id=51,
        assignee="cli-auto",
        max_runtime_seconds=60,
    )
    completion_raises = {"value": True}
    blocks = []
    comments = []

    class Connection:
        def close(self):
            return None

    def complete_task(_conn, _task_id, **kwargs):
        if completion_raises["value"]:
            raise RuntimeError("crash between journal and commit")
        task.status = "done"
        task.result = kwargs["result"]
        task.current_run_id = None
        return True

    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        get_task=lambda _conn, _task_id: task,
        worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
        _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_crash"),
        set_branch_name=lambda *_args: None,
        set_workspace_path=lambda *_args: None,
        build_worker_context=lambda *_args: "Finish safely.",
        heartbeat_worker=lambda *_args, **_kwargs: True,
        add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
        complete_task=complete_task,
        block_task=lambda *_args, **kwargs: blocks.append(kwargs),
    )
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
    monkeypatch.setattr(
        lanes,
        "select_route",
        lambda *_args, **_kwargs: lanes.Route(
            "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, ()
        ),
    )
    monkeypatch.setattr(
        lanes,
        "run_provider",
        lambda *_args, **_kwargs: lanes.ProcessResult(
            0, "", _completed_result("crash-safe result"), False
        ),
    )

    lanes.execute_claim("cassandra", "t_crash")

    pending = list((state_root / "cassandra").glob("*.terminal.pending.json"))
    assert len(pending) == 1
    assert blocks == []
    assert any("RuntimeError" in body for body in comments)

    completion_raises["value"] = False
    assert lanes.recover_pending_finalizations() == 1
    assert task.status == "done"
    assert not pending[0].exists()


def test_terminal_replace_then_directory_fsync_enospc_replays_after_restart(
    tmp_path: Path,
    monkeypatch,
):
    state_root = tmp_path / "cli-lanes"
    monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
    task = SimpleNamespace(
        id="t_enospc",
        status="running",
        result=None,
        current_run_id=52,
        assignee="cli-auto",
        max_runtime_seconds=60,
    )
    blocks = []
    comments = []
    completions = []

    class Connection:
        def close(self):
            return None

    def complete_task(_conn, _task_id, **kwargs):
        completions.append(kwargs)
        task.status = "done"
        task.result = kwargs["result"]
        task.current_run_id = None
        return True

    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        get_task=lambda _conn, _task_id: task,
        worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
        _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_enospc"),
        set_branch_name=lambda *_args: None,
        set_workspace_path=lambda *_args: None,
        build_worker_context=lambda *_args: "Finish without losing the result.",
        heartbeat_worker=lambda *_args, **_kwargs: True,
        add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
        complete_task=complete_task,
        block_task=lambda *_args, **kwargs: blocks.append(kwargs),
    )
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
    monkeypatch.setattr(
        lanes,
        "select_route",
        lambda *_args, **_kwargs: lanes.Route(
            "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, ()
        ),
    )
    monkeypatch.setattr(
        lanes,
        "run_provider",
        lambda *_args, **_kwargs: lanes.ProcessResult(
            0, "", _completed_result("persisted before ENOSPC"), False
        ),
    )
    real_replace = lanes.os.replace
    real_fsync_directory = lanes._fsync_directory
    terminal_replaced = {"value": False}
    fail_once = {"value": True}

    def replace_then_mark(source, destination):
        real_replace(source, destination)
        if str(destination).endswith(".terminal.pending.json"):
            terminal_replaced["value"] = True

    def fail_after_terminal_replace(directory):
        if terminal_replaced["value"] and fail_once["value"]:
            fail_once["value"] = False
            raise OSError(errno.ENOSPC, "no space after terminal rename")
        real_fsync_directory(directory)

    monkeypatch.setattr(lanes.os, "replace", replace_then_mark)
    monkeypatch.setattr(lanes, "_fsync_directory", fail_after_terminal_replace)

    lanes.execute_claim("cassandra", "t_enospc")

    pending = list((state_root / "cassandra").glob("*.terminal.pending.json"))
    assert terminal_replaced["value"] is True
    assert fail_once["value"] is False
    assert len(pending) == 1
    assert blocks == []
    assert completions == []
    assert task.status == "running"
    assert any("terminal replayable=True" in body for body in comments)

    # A restarted runner sees the exact pending journal and completes the run.
    assert lanes.recover_pending_finalizations() == 1
    assert len(completions) == 1
    assert task.status == "done"
    assert not pending[0].exists()
    committed = list((state_root / "cassandra").glob("*.terminal.committed.json"))
    assert len(committed) == 1


def test_terminal_payload_cannot_select_a_different_board(
    tmp_path: Path,
    monkeypatch,
):
    state_root = tmp_path / "cli-lanes"
    monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
    path = lanes._terminal_path(lanes.state_path("alpha", "t_alpha"), 3)
    lanes.atomic_json(
        path,
        {
            "board": "beta",
            "task_id": "t_beta",
            "expected_run_id": 9,
            "result": json.dumps(_completed_result()),
            "summary": "forged",
            "metadata": {},
            "kanban_state": "pending",
        },
    )
    alpha_task = SimpleNamespace(
        id="t_alpha", status="running", current_run_id=3, assignee="cli-auto"
    )
    opened = []
    reclaimed = []

    class Connection:
        def close(self):
            return None

    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: (opened.append(board) or Connection()),
        get_task=lambda _conn, task_id: alpha_task if task_id == "t_alpha" else None,
        reclaim_task=lambda _conn, task_id, **_kwargs: (reclaimed.append(task_id) or True),
    )
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))

    assert lanes.recover_pending_finalizations() == 0

    assert opened == ["alpha"]
    assert reclaimed == ["t_alpha"]
    assert not path.exists()
    quarantined = list((state_root / "alpha/quarantine").glob("*.quarantine"))
    assert len(quarantined) == 1
    assert quarantined[0].stat().st_mode & 0o777 == 0o600


def test_terminal_finalize_rereads_journal_after_a_path_swap(
    tmp_path: Path,
    monkeypatch,
):
    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
    state_file = lanes.state_path("alpha", "t_alpha")
    path, safe_record = lanes._write_terminal_record(
        state_file,
        board="alpha",
        task_id="t_alpha",
        run_id=4,
        structured=_completed_result("safe"),
        summary="safe",
        metadata={},
    )
    forged = dict(safe_record)
    forged.update({"board": "beta", "task_id": "t_beta", "expected_run_id": 7})
    lanes.atomic_json(path, forged)
    opened = []
    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: (opened.append(board) or pytest.fail("must not open DB")),
    )

    assert lanes._finalize_terminal_record(fake_db, path, safe_record) == "foreign"
    assert opened == []


@pytest.mark.parametrize("payload", [b"", b'{"board":"cassandra"'])
def test_malformed_exact_run_journal_is_quarantined_and_reclaimed(
    tmp_path: Path,
    monkeypatch,
    payload: bytes,
):
    state_root = tmp_path / "cli-lanes"
    monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
    path = lanes._terminal_path(lanes.state_path("cassandra", "t_partial"), 12)
    path.parent.mkdir(parents=True)
    path.write_bytes(payload)
    task = SimpleNamespace(
        id="t_partial", status="running", current_run_id=12, assignee="cli-auto"
    )
    reclaimed = []

    class Connection:
        def close(self):
            return None

    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        get_task=lambda _conn, _task_id: task,
        reclaim_task=lambda *_args, **kwargs: (reclaimed.append(kwargs) or True),
    )
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))

    assert lanes.recover_pending_finalizations() == 0

    assert reclaimed == [{
        "reason": (
            "terminal journal recovery failed (malformed-payload); "
            "exact run may retry"
        ),
        "expected_run_id": 12,
    }]
    assert not path.exists()
    assert lanes._has_pending_finalization("cassandra", "t_partial", 12) is False
    quarantined = list((state_root / "cassandra/quarantine").glob("*.quarantine"))
    assert len(quarantined) == 1
    assert quarantined[0].stat().st_mode & 0o777 == 0o600