atlas-iac/testing/tests/test_hermes_cli_durability.py

"""Prepared evidence and filesystem durability boundaries."""

from __future__ import annotations

from testing.tests.test_hermes_cli_support import (
    Path,
    SimpleNamespace,
    _completed_result,
    errno,
    json,
    lanes,
    nullcontext,
    os,
    pytest,
    stat,
    sys,
    threading,
)


def test_unknown_db_completion_outcome_replays_from_prepared_evidence(
    tmp_path: Path,
    monkeypatch,
):
    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
    path, record = lanes._write_terminal_record(
        lanes.state_path("cassandra", "t_unknown_db_outcome"),
        board="cassandra",
        task_id="t_unknown_db_outcome",
        run_id=35,
        structured=_completed_result("DB committed before transport error"),
        summary="DB committed before transport error",
        metadata={},
    )
    task = SimpleNamespace(
        id="t_unknown_db_outcome",
        status="running",
        result=None,
        current_run_id=35,
        completed_run_id=None,
        assignee="cli-auto",
    )
    fail_once = {"value": True}

    class Connection:
        def close(self):
            return None

    def complete_task(_conn, _task_id, **kwargs):
        task.status = "done"
        task.result = kwargs["result"]
        task.completed_run_id = kwargs["expected_run_id"]
        task.current_run_id = None
        if fail_once["value"]:
            fail_once["value"] = False
            raise RuntimeError("transport failed after DB commit")
        return True

    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        get_task=lambda _conn, _task_id: task,
        complete_task=complete_task,
    )

    with pytest.raises(RuntimeError, match="after DB commit"):
        lanes._finalize_terminal_record(fake_db, path, record)

    assert task.status == "done"
    assert path.exists()
    assert len(list(path.parent.glob("*.terminal.prepared-*.json"))) == 1
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
    assert lanes.recover_pending_finalizations() == 1
    assert not path.exists()
    assert list(path.parent.glob("*.terminal.prepared-*.json")) == []
    committed = list(path.parent.glob("*.terminal.committed.json"))
    assert len(committed) == 1
    assert json.loads(committed[0].read_text())["result"] == record["result"]


def test_pending_retirement_fsync_failure_keeps_db_winner_committed(
    tmp_path: Path,
    monkeypatch,
):
    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
    path, record = lanes._write_terminal_record(
        lanes.state_path("cassandra", "t_retire_fsync"),
        board="cassandra",
        task_id="t_retire_fsync",
        run_id=33,
        structured=_completed_result("retirement fsync winner"),
        summary="retirement fsync winner",
        metadata={},
    )
    task = SimpleNamespace(
        id="t_retire_fsync",
        status="running",
        result=None,
        current_run_id=33,
        assignee="cli-auto",
    )

    class Connection:
        def close(self):
            return None

    def complete_task(_conn, _task_id, **kwargs):
        task.status = "done"
        task.result = kwargs["result"]
        task.current_run_id = None
        return True

    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        get_task=lambda _conn, _task_id: task,
        complete_task=complete_task,
    )
    real_fsync = lanes.os.fsync
    failed = {"value": False}

    def fail_pending_retirement_fsync(descriptor):
        if (
            stat.S_ISDIR(os.fstat(descriptor).st_mode)
            and not path.exists()
            and list(path.parent.glob("*.terminal.committed.json"))
            and not failed["value"]
        ):
            failed["value"] = True
            raise OSError(errno.ENOSPC, "pending retirement fsync failed")
        real_fsync(descriptor)

    monkeypatch.setattr(lanes.os, "fsync", fail_pending_retirement_fsync)

    assert lanes._finalize_terminal_record(fake_db, path, record) == "committed"
    assert failed["value"] is True
    assert task.status == "done"
    assert task.result == record["result"]
    assert not path.exists()
    assert len(list(path.parent.glob("*.terminal.committed.json"))) == 1
    assert list(path.parent.glob("*.terminal.prepared-*.json")) == []


def test_committed_first_writer_is_never_overwritten_by_a_db_winner(
    tmp_path: Path,
    monkeypatch,
):
    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
    state_file = lanes.state_path("cassandra", "t_committed_collision")
    pending, winner = lanes._write_terminal_record(
        state_file,
        board="cassandra",
        task_id="t_committed_collision",
        run_id=34,
        structured=_completed_result("DB winner"),
        summary="DB winner",
        metadata={"writer": "db"},
    )
    committed_path = lanes._terminal_path(state_file, 34, "committed")
    first_writer = {
        "board": "cassandra",
        "task_id": "t_committed_collision",
        "expected_run_id": 34,
        "result": json.dumps(_completed_result("evidence first writer"), sort_keys=True),
        "summary": "evidence first writer",
        "metadata": {"writer": "evidence"},
        "kanban_state": "committed",
        "recorded_at": lanes.utc_now(),
    }
    lanes.atomic_json(committed_path, first_writer)
    task = SimpleNamespace(
        id="t_committed_collision",
        status="running",
        result=None,
        current_run_id=34,
        assignee="cli-auto",
    )

    class Connection:
        def close(self):
            return None

    def complete_task(_conn, _task_id, **kwargs):
        task.status = "done"
        task.result = kwargs["result"]
        task.current_run_id = None
        return True

    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        get_task=lambda _conn, _task_id: task,
        complete_task=complete_task,
    )

    assert lanes._finalize_terminal_record(fake_db, pending, winner) == "committed"
    assert task.result == winner["result"]
    assert json.loads(committed_path.read_text())["result"] == first_writer["result"]
    conflicts = list(pending.parent.glob("*.terminal.conflict-*.json"))
    assert len(conflicts) == 1
    assert json.loads(conflicts[0].read_text())["result"] == winner["result"]
    assert not pending.exists()


def test_concurrent_duplicate_terminal_finalizers_are_idempotent(
    tmp_path: Path,
    monkeypatch,
):
    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
    path, record = lanes._write_terminal_record(
        lanes.state_path("cassandra", "t_duplicate"),
        board="cassandra",
        task_id="t_duplicate",
        run_id=29,
        structured=_completed_result("same accepted result"),
        summary="same accepted result",
        metadata={},
    )
    task_state = {
        "status": "running",
        "result": None,
        "current_run_id": 29,
        "completed_run_id": None,
    }
    state_lock = threading.Lock()
    readers = threading.Barrier(2)
    completions = []

    class Connection:
        def close(self):
            return None

    def get_task(_conn, _task_id):
        with state_lock:
            snapshot = SimpleNamespace(
                id="t_duplicate",
                assignee="cli-auto",
                **task_state,
            )
        if snapshot.status == "running":
            readers.wait(timeout=5)
        return snapshot

    def complete_task(_conn, _task_id, **kwargs):
        with state_lock:
            if task_state["status"] != "running":
                return False
            task_state.update(
                    status="done",
                    result=kwargs["result"],
                    current_run_id=None,
                    completed_run_id=kwargs["expected_run_id"],
                )
            completions.append(kwargs["result"])
            return True

    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        get_task=get_task,
        complete_task=complete_task,
    )
    outcomes = []
    errors = []

    def finalize():
        try:
            outcomes.append(lanes._finalize_terminal_record(fake_db, path, record))
        except Exception as error:  # pragma: no cover - assertion reports detail
            errors.append(error)

    workers = [threading.Thread(target=finalize) for _index in range(2)]
    for worker in workers:
        worker.start()
    for worker in workers:
        worker.join(timeout=10)

    assert all(not worker.is_alive() for worker in workers)
    assert errors == []
    assert outcomes == ["committed", "committed"]
    assert len(completions) == 1
    assert not path.exists()
    assert len(list(path.parent.glob("*.terminal.committed.json"))) == 1
    assert list(path.parent.glob("*.terminal.conflict-*.json")) == []
    assert list(path.parent.glob("*.terminal.prepared-*.json")) == []


def test_recovery_uses_first_durable_prepared_result_as_db_writer(
    tmp_path: Path,
    monkeypatch,
):
    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
    state_file = lanes.state_path("cassandra", "t_prepared_order")
    pending, first = lanes._write_terminal_record(
        state_file,
        board="cassandra",
        task_id="t_prepared_order",
        run_id=30,
        structured=_completed_result("first durable result"),
        summary="first durable result",
        metadata={"writer": "first"},
    )
    identity = lanes._terminal_identity(pending)
    assert identity is not None
    first_prepared = lanes._persist_prepared_evidence(identity, first)
    os.utime(first_prepared, ns=(100, 100))
    second_structured = _completed_result("second durable result")
    second = {
        "board": "cassandra",
        "task_id": "t_prepared_order",
        "expected_run_id": 30,
        "result": json.dumps(second_structured, sort_keys=True),
        "summary": "second durable result",
        "metadata": {"writer": "second"},
        "kanban_state": "pending",
        "recorded_at": lanes.utc_now(),
    }
    second_prepared = lanes._persist_prepared_evidence(identity, second)
    os.utime(second_prepared, ns=(200, 200))
    lanes.atomic_json(pending, second)
    task = SimpleNamespace(
        id="t_prepared_order",
        status="running",
        result=None,
        current_run_id=30,
        assignee="cli-auto",
    )
    completions = []

    class Connection:
        def close(self):
            return None

    def complete_task(_conn, _task_id, **kwargs):
        if task.status != "running":
            return False
        task.status = "done"
        task.result = kwargs["result"]
        task.current_run_id = None
        completions.append(kwargs["result"])
        return True

    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        get_task=lambda _conn, _task_id: task,
        complete_task=complete_task,
    )
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))

    assert lanes.recover_pending_finalizations() == 1
    assert completions == [first["result"]]
    assert task.result == first["result"]
    assert not pending.exists()
    assert list(pending.parent.glob("*.terminal.prepared-*.json")) == []
    committed = list(pending.parent.glob("*.terminal.committed.json"))
    conflicts = list(pending.parent.glob("*.terminal.conflict-*.json"))
    assert len(committed) == 1
    assert len(conflicts) == 1
    assert json.loads(committed[0].read_text())["result"] == first["result"]
    assert json.loads(conflicts[0].read_text())["result"] == second["result"]