362 lines
12 KiB
Python
362 lines
12 KiB
Python
"""Prepared evidence and filesystem durability boundaries."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from testing.tests.test_hermes_cli_support import (
|
|
Path,
|
|
SimpleNamespace,
|
|
_completed_result,
|
|
errno,
|
|
json,
|
|
lanes,
|
|
nullcontext,
|
|
os,
|
|
pytest,
|
|
stat,
|
|
sys,
|
|
threading,
|
|
)
|
|
|
|
|
|
def test_unknown_db_completion_outcome_replays_from_prepared_evidence(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
|
path, record = lanes._write_terminal_record(
|
|
lanes.state_path("cassandra", "t_unknown_db_outcome"),
|
|
board="cassandra",
|
|
task_id="t_unknown_db_outcome",
|
|
run_id=35,
|
|
structured=_completed_result("DB committed before transport error"),
|
|
summary="DB committed before transport error",
|
|
metadata={},
|
|
)
|
|
task = SimpleNamespace(
|
|
id="t_unknown_db_outcome",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=35,
|
|
completed_run_id=None,
|
|
assignee="cli-auto",
|
|
)
|
|
fail_once = {"value": True}
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
def complete_task(_conn, _task_id, **kwargs):
|
|
task.status = "done"
|
|
task.result = kwargs["result"]
|
|
task.completed_run_id = kwargs["expected_run_id"]
|
|
task.current_run_id = None
|
|
if fail_once["value"]:
|
|
fail_once["value"] = False
|
|
raise RuntimeError("transport failed after DB commit")
|
|
return True
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
complete_task=complete_task,
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="after DB commit"):
|
|
lanes._finalize_terminal_record(fake_db, path, record)
|
|
|
|
assert task.status == "done"
|
|
assert path.exists()
|
|
assert len(list(path.parent.glob("*.terminal.prepared-*.json"))) == 1
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
assert lanes.recover_pending_finalizations() == 1
|
|
assert not path.exists()
|
|
assert list(path.parent.glob("*.terminal.prepared-*.json")) == []
|
|
committed = list(path.parent.glob("*.terminal.committed.json"))
|
|
assert len(committed) == 1
|
|
assert json.loads(committed[0].read_text())["result"] == record["result"]
|
|
|
|
|
|
def test_pending_retirement_fsync_failure_keeps_db_winner_committed(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
|
path, record = lanes._write_terminal_record(
|
|
lanes.state_path("cassandra", "t_retire_fsync"),
|
|
board="cassandra",
|
|
task_id="t_retire_fsync",
|
|
run_id=33,
|
|
structured=_completed_result("retirement fsync winner"),
|
|
summary="retirement fsync winner",
|
|
metadata={},
|
|
)
|
|
task = SimpleNamespace(
|
|
id="t_retire_fsync",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=33,
|
|
assignee="cli-auto",
|
|
)
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
def complete_task(_conn, _task_id, **kwargs):
|
|
task.status = "done"
|
|
task.result = kwargs["result"]
|
|
task.current_run_id = None
|
|
return True
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
complete_task=complete_task,
|
|
)
|
|
real_fsync = lanes.os.fsync
|
|
failed = {"value": False}
|
|
|
|
def fail_pending_retirement_fsync(descriptor):
|
|
if (
|
|
stat.S_ISDIR(os.fstat(descriptor).st_mode)
|
|
and not path.exists()
|
|
and list(path.parent.glob("*.terminal.committed.json"))
|
|
and not failed["value"]
|
|
):
|
|
failed["value"] = True
|
|
raise OSError(errno.ENOSPC, "pending retirement fsync failed")
|
|
real_fsync(descriptor)
|
|
|
|
monkeypatch.setattr(lanes.os, "fsync", fail_pending_retirement_fsync)
|
|
|
|
assert lanes._finalize_terminal_record(fake_db, path, record) == "committed"
|
|
assert failed["value"] is True
|
|
assert task.status == "done"
|
|
assert task.result == record["result"]
|
|
assert not path.exists()
|
|
assert len(list(path.parent.glob("*.terminal.committed.json"))) == 1
|
|
assert list(path.parent.glob("*.terminal.prepared-*.json")) == []
|
|
|
|
|
|
def test_committed_first_writer_is_never_overwritten_by_a_db_winner(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
|
state_file = lanes.state_path("cassandra", "t_committed_collision")
|
|
pending, winner = lanes._write_terminal_record(
|
|
state_file,
|
|
board="cassandra",
|
|
task_id="t_committed_collision",
|
|
run_id=34,
|
|
structured=_completed_result("DB winner"),
|
|
summary="DB winner",
|
|
metadata={"writer": "db"},
|
|
)
|
|
committed_path = lanes._terminal_path(state_file, 34, "committed")
|
|
first_writer = {
|
|
"board": "cassandra",
|
|
"task_id": "t_committed_collision",
|
|
"expected_run_id": 34,
|
|
"result": json.dumps(_completed_result("evidence first writer"), sort_keys=True),
|
|
"summary": "evidence first writer",
|
|
"metadata": {"writer": "evidence"},
|
|
"kanban_state": "committed",
|
|
"recorded_at": lanes.utc_now(),
|
|
}
|
|
lanes.atomic_json(committed_path, first_writer)
|
|
task = SimpleNamespace(
|
|
id="t_committed_collision",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=34,
|
|
assignee="cli-auto",
|
|
)
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
def complete_task(_conn, _task_id, **kwargs):
|
|
task.status = "done"
|
|
task.result = kwargs["result"]
|
|
task.current_run_id = None
|
|
return True
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
complete_task=complete_task,
|
|
)
|
|
|
|
assert lanes._finalize_terminal_record(fake_db, pending, winner) == "committed"
|
|
assert task.result == winner["result"]
|
|
assert json.loads(committed_path.read_text())["result"] == first_writer["result"]
|
|
conflicts = list(pending.parent.glob("*.terminal.conflict-*.json"))
|
|
assert len(conflicts) == 1
|
|
assert json.loads(conflicts[0].read_text())["result"] == winner["result"]
|
|
assert not pending.exists()
|
|
|
|
|
|
def test_concurrent_duplicate_terminal_finalizers_are_idempotent(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
|
path, record = lanes._write_terminal_record(
|
|
lanes.state_path("cassandra", "t_duplicate"),
|
|
board="cassandra",
|
|
task_id="t_duplicate",
|
|
run_id=29,
|
|
structured=_completed_result("same accepted result"),
|
|
summary="same accepted result",
|
|
metadata={},
|
|
)
|
|
task_state = {
|
|
"status": "running",
|
|
"result": None,
|
|
"current_run_id": 29,
|
|
"completed_run_id": None,
|
|
}
|
|
state_lock = threading.Lock()
|
|
readers = threading.Barrier(2)
|
|
completions = []
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
def get_task(_conn, _task_id):
|
|
with state_lock:
|
|
snapshot = SimpleNamespace(
|
|
id="t_duplicate",
|
|
assignee="cli-auto",
|
|
**task_state,
|
|
)
|
|
if snapshot.status == "running":
|
|
readers.wait(timeout=5)
|
|
return snapshot
|
|
|
|
def complete_task(_conn, _task_id, **kwargs):
|
|
with state_lock:
|
|
if task_state["status"] != "running":
|
|
return False
|
|
task_state.update(
|
|
status="done",
|
|
result=kwargs["result"],
|
|
current_run_id=None,
|
|
completed_run_id=kwargs["expected_run_id"],
|
|
)
|
|
completions.append(kwargs["result"])
|
|
return True
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=get_task,
|
|
complete_task=complete_task,
|
|
)
|
|
outcomes = []
|
|
errors = []
|
|
|
|
def finalize():
|
|
try:
|
|
outcomes.append(lanes._finalize_terminal_record(fake_db, path, record))
|
|
except Exception as error: # pragma: no cover - assertion reports detail
|
|
errors.append(error)
|
|
|
|
workers = [threading.Thread(target=finalize) for _index in range(2)]
|
|
for worker in workers:
|
|
worker.start()
|
|
for worker in workers:
|
|
worker.join(timeout=10)
|
|
|
|
assert all(not worker.is_alive() for worker in workers)
|
|
assert errors == []
|
|
assert outcomes == ["committed", "committed"]
|
|
assert len(completions) == 1
|
|
assert not path.exists()
|
|
assert len(list(path.parent.glob("*.terminal.committed.json"))) == 1
|
|
assert list(path.parent.glob("*.terminal.conflict-*.json")) == []
|
|
assert list(path.parent.glob("*.terminal.prepared-*.json")) == []
|
|
|
|
|
|
def test_recovery_uses_first_durable_prepared_result_as_db_writer(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
|
state_file = lanes.state_path("cassandra", "t_prepared_order")
|
|
pending, first = lanes._write_terminal_record(
|
|
state_file,
|
|
board="cassandra",
|
|
task_id="t_prepared_order",
|
|
run_id=30,
|
|
structured=_completed_result("first durable result"),
|
|
summary="first durable result",
|
|
metadata={"writer": "first"},
|
|
)
|
|
identity = lanes._terminal_identity(pending)
|
|
assert identity is not None
|
|
first_prepared = lanes._persist_prepared_evidence(identity, first)
|
|
os.utime(first_prepared, ns=(100, 100))
|
|
second_structured = _completed_result("second durable result")
|
|
second = {
|
|
"board": "cassandra",
|
|
"task_id": "t_prepared_order",
|
|
"expected_run_id": 30,
|
|
"result": json.dumps(second_structured, sort_keys=True),
|
|
"summary": "second durable result",
|
|
"metadata": {"writer": "second"},
|
|
"kanban_state": "pending",
|
|
"recorded_at": lanes.utc_now(),
|
|
}
|
|
second_prepared = lanes._persist_prepared_evidence(identity, second)
|
|
os.utime(second_prepared, ns=(200, 200))
|
|
lanes.atomic_json(pending, second)
|
|
task = SimpleNamespace(
|
|
id="t_prepared_order",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=30,
|
|
assignee="cli-auto",
|
|
)
|
|
completions = []
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
def complete_task(_conn, _task_id, **kwargs):
|
|
if task.status != "running":
|
|
return False
|
|
task.status = "done"
|
|
task.result = kwargs["result"]
|
|
task.current_run_id = None
|
|
completions.append(kwargs["result"])
|
|
return True
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
complete_task=complete_task,
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
|
|
assert lanes.recover_pending_finalizations() == 1
|
|
assert completions == [first["result"]]
|
|
assert task.result == first["result"]
|
|
assert not pending.exists()
|
|
assert list(pending.parent.glob("*.terminal.prepared-*.json")) == []
|
|
committed = list(pending.parent.glob("*.terminal.committed.json"))
|
|
conflicts = list(pending.parent.glob("*.terminal.conflict-*.json"))
|
|
assert len(committed) == 1
|
|
assert len(conflicts) == 1
|
|
assert json.loads(committed[0].read_text())["result"] == first["result"]
|
|
assert json.loads(conflicts[0].read_text())["result"] == second["result"]
|