"""Prepared evidence and filesystem durability boundaries.""" from __future__ import annotations from testing.tests.test_hermes_cli_support import ( Path, SimpleNamespace, _completed_result, errno, json, lanes, nullcontext, os, pytest, stat, sys, threading, ) def test_unknown_db_completion_outcome_replays_from_prepared_evidence( tmp_path: Path, monkeypatch, ): monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") path, record = lanes._write_terminal_record( lanes.state_path("cassandra", "t_unknown_db_outcome"), board="cassandra", task_id="t_unknown_db_outcome", run_id=35, structured=_completed_result("DB committed before transport error"), summary="DB committed before transport error", metadata={}, ) task = SimpleNamespace( id="t_unknown_db_outcome", status="running", result=None, current_run_id=35, completed_run_id=None, assignee="cli-auto", ) fail_once = {"value": True} class Connection: def close(self): return None def complete_task(_conn, _task_id, **kwargs): task.status = "done" task.result = kwargs["result"] task.completed_run_id = kwargs["expected_run_id"] task.current_run_id = None if fail_once["value"]: fail_once["value"] = False raise RuntimeError("transport failed after DB commit") return True fake_db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), get_task=lambda _conn, _task_id: task, complete_task=complete_task, ) with pytest.raises(RuntimeError, match="after DB commit"): lanes._finalize_terminal_record(fake_db, path, record) assert task.status == "done" assert path.exists() assert len(list(path.parent.glob("*.terminal.prepared-*.json"))) == 1 monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) assert lanes.recover_pending_finalizations() == 1 assert not path.exists() assert list(path.parent.glob("*.terminal.prepared-*.json")) == [] committed = list(path.parent.glob("*.terminal.committed.json")) assert len(committed) == 1 assert json.loads(committed[0].read_text())["result"] == record["result"] def test_pending_retirement_fsync_failure_keeps_db_winner_committed( tmp_path: Path, monkeypatch, ): monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") path, record = lanes._write_terminal_record( lanes.state_path("cassandra", "t_retire_fsync"), board="cassandra", task_id="t_retire_fsync", run_id=33, structured=_completed_result("retirement fsync winner"), summary="retirement fsync winner", metadata={}, ) task = SimpleNamespace( id="t_retire_fsync", status="running", result=None, current_run_id=33, assignee="cli-auto", ) class Connection: def close(self): return None def complete_task(_conn, _task_id, **kwargs): task.status = "done" task.result = kwargs["result"] task.current_run_id = None return True fake_db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), get_task=lambda _conn, _task_id: task, complete_task=complete_task, ) real_fsync = lanes.os.fsync failed = {"value": False} def fail_pending_retirement_fsync(descriptor): if ( stat.S_ISDIR(os.fstat(descriptor).st_mode) and not path.exists() and list(path.parent.glob("*.terminal.committed.json")) and not failed["value"] ): failed["value"] = True raise OSError(errno.ENOSPC, "pending retirement fsync failed") real_fsync(descriptor) monkeypatch.setattr(lanes.os, "fsync", fail_pending_retirement_fsync) assert lanes._finalize_terminal_record(fake_db, path, record) == "committed" assert failed["value"] is True assert task.status == "done" assert task.result == record["result"] assert not path.exists() assert len(list(path.parent.glob("*.terminal.committed.json"))) == 1 assert list(path.parent.glob("*.terminal.prepared-*.json")) == [] def test_committed_first_writer_is_never_overwritten_by_a_db_winner( tmp_path: Path, monkeypatch, ): monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") state_file = lanes.state_path("cassandra", "t_committed_collision") pending, winner = lanes._write_terminal_record( state_file, board="cassandra", task_id="t_committed_collision", run_id=34, structured=_completed_result("DB winner"), summary="DB winner", metadata={"writer": "db"}, ) committed_path = lanes._terminal_path(state_file, 34, "committed") first_writer = { "board": "cassandra", "task_id": "t_committed_collision", "expected_run_id": 34, "result": json.dumps(_completed_result("evidence first writer"), sort_keys=True), "summary": "evidence first writer", "metadata": {"writer": "evidence"}, "kanban_state": "committed", "recorded_at": lanes.utc_now(), } lanes.atomic_json(committed_path, first_writer) task = SimpleNamespace( id="t_committed_collision", status="running", result=None, current_run_id=34, assignee="cli-auto", ) class Connection: def close(self): return None def complete_task(_conn, _task_id, **kwargs): task.status = "done" task.result = kwargs["result"] task.current_run_id = None return True fake_db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), get_task=lambda _conn, _task_id: task, complete_task=complete_task, ) assert lanes._finalize_terminal_record(fake_db, pending, winner) == "committed" assert task.result == winner["result"] assert json.loads(committed_path.read_text())["result"] == first_writer["result"] conflicts = list(pending.parent.glob("*.terminal.conflict-*.json")) assert len(conflicts) == 1 assert json.loads(conflicts[0].read_text())["result"] == winner["result"] assert not pending.exists() def test_concurrent_duplicate_terminal_finalizers_are_idempotent( tmp_path: Path, monkeypatch, ): monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") path, record = lanes._write_terminal_record( lanes.state_path("cassandra", "t_duplicate"), board="cassandra", task_id="t_duplicate", run_id=29, structured=_completed_result("same accepted result"), summary="same accepted result", metadata={}, ) task_state = { "status": "running", "result": None, "current_run_id": 29, "completed_run_id": None, } state_lock = threading.Lock() readers = threading.Barrier(2) completions = [] class Connection: def close(self): return None def get_task(_conn, _task_id): with state_lock: snapshot = SimpleNamespace( id="t_duplicate", assignee="cli-auto", **task_state, ) if snapshot.status == "running": readers.wait(timeout=5) return snapshot def complete_task(_conn, _task_id, **kwargs): with state_lock: if task_state["status"] != "running": return False task_state.update( status="done", result=kwargs["result"], current_run_id=None, completed_run_id=kwargs["expected_run_id"], ) completions.append(kwargs["result"]) return True fake_db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), get_task=get_task, complete_task=complete_task, ) outcomes = [] errors = [] def finalize(): try: outcomes.append(lanes._finalize_terminal_record(fake_db, path, record)) except Exception as error: # pragma: no cover - assertion reports detail errors.append(error) workers = [threading.Thread(target=finalize) for _index in range(2)] for worker in workers: worker.start() for worker in workers: worker.join(timeout=10) assert all(not worker.is_alive() for worker in workers) assert errors == [] assert outcomes == ["committed", "committed"] assert len(completions) == 1 assert not path.exists() assert len(list(path.parent.glob("*.terminal.committed.json"))) == 1 assert list(path.parent.glob("*.terminal.conflict-*.json")) == [] assert list(path.parent.glob("*.terminal.prepared-*.json")) == [] def test_recovery_uses_first_durable_prepared_result_as_db_writer( tmp_path: Path, monkeypatch, ): monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes") state_file = lanes.state_path("cassandra", "t_prepared_order") pending, first = lanes._write_terminal_record( state_file, board="cassandra", task_id="t_prepared_order", run_id=30, structured=_completed_result("first durable result"), summary="first durable result", metadata={"writer": "first"}, ) identity = lanes._terminal_identity(pending) assert identity is not None first_prepared = lanes._persist_prepared_evidence(identity, first) os.utime(first_prepared, ns=(100, 100)) second_structured = _completed_result("second durable result") second = { "board": "cassandra", "task_id": "t_prepared_order", "expected_run_id": 30, "result": json.dumps(second_structured, sort_keys=True), "summary": "second durable result", "metadata": {"writer": "second"}, "kanban_state": "pending", "recorded_at": lanes.utc_now(), } second_prepared = lanes._persist_prepared_evidence(identity, second) os.utime(second_prepared, ns=(200, 200)) lanes.atomic_json(pending, second) task = SimpleNamespace( id="t_prepared_order", status="running", result=None, current_run_id=30, assignee="cli-auto", ) completions = [] class Connection: def close(self): return None def complete_task(_conn, _task_id, **kwargs): if task.status != "running": return False task.status = "done" task.result = kwargs["result"] task.current_run_id = None completions.append(kwargs["result"]) return True fake_db = SimpleNamespace( scoped_current_board=lambda _board: nullcontext(), connect=lambda board: Connection(), get_task=lambda _conn, _task_id: task, complete_task=complete_task, ) monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db)) assert lanes.recover_pending_finalizations() == 1 assert completions == [first["result"]] assert task.result == first["result"] assert not pending.exists() assert list(pending.parent.glob("*.terminal.prepared-*.json")) == [] committed = list(pending.parent.glob("*.terminal.committed.json")) conflicts = list(pending.parent.glob("*.terminal.conflict-*.json")) assert len(committed) == 1 assert len(conflicts) == 1 assert json.loads(committed[0].read_text())["result"] == first["result"] assert json.loads(conflicts[0].read_text())["result"] == second["result"]