412 lines
13 KiB
Python
412 lines
13 KiB
Python
"""Terminal-result validation and exact-run finalization."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from testing.tests.test_hermes_cli_support import (
|
|
Path,
|
|
SimpleNamespace,
|
|
_completed_result,
|
|
errno,
|
|
json,
|
|
lanes,
|
|
nullcontext,
|
|
pytest,
|
|
sys,
|
|
)
|
|
|
|
|
|
def test_terminal_recovery_can_complete_the_exact_latest_ended_run(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
|
state_file = lanes.state_path("cassandra", "t_ended")
|
|
pending, _record = lanes._write_terminal_record(
|
|
state_file,
|
|
board="cassandra",
|
|
task_id="t_ended",
|
|
run_id=17,
|
|
structured=_completed_result("accepted before legacy block"),
|
|
summary="accepted before legacy block",
|
|
metadata={},
|
|
)
|
|
task = SimpleNamespace(
|
|
id="t_ended",
|
|
status="blocked",
|
|
result=None,
|
|
current_run_id=None,
|
|
assignee="cli-auto",
|
|
)
|
|
guards = []
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
def complete_task(_conn, _task_id, **kwargs):
|
|
guards.append(kwargs)
|
|
task.status = "done"
|
|
task.result = kwargs["result"]
|
|
return True
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
complete_task=complete_task,
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
|
|
assert lanes.recover_pending_finalizations() == 1
|
|
assert guards[0]["replay_ended_run_id"] == 17
|
|
assert "expected_run_id" not in guards[0]
|
|
assert task.status == "done"
|
|
assert not pending.exists()
|
|
|
|
|
|
def test_terminal_recovery_completes_an_unchanged_triage_exact_run(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
|
pending, record = lanes._write_terminal_record(
|
|
lanes.state_path("cassandra", "t_triage_ended"),
|
|
board="cassandra",
|
|
task_id="t_triage_ended",
|
|
run_id=18,
|
|
structured=_completed_result("accepted before loop-breaker triage"),
|
|
summary="accepted before loop-breaker triage",
|
|
metadata={},
|
|
)
|
|
task = SimpleNamespace(
|
|
id="t_triage_ended",
|
|
status="triage",
|
|
result=None,
|
|
current_run_id=None,
|
|
completed_run_id=None,
|
|
assignee="cli-auto",
|
|
)
|
|
guards = []
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
def complete_task(_conn, _task_id, **kwargs):
|
|
guards.append(kwargs)
|
|
assert kwargs == {
|
|
"result": record["result"],
|
|
"summary": record["summary"],
|
|
"metadata": {},
|
|
"replay_ended_run_id": 18,
|
|
}
|
|
task.status = "done"
|
|
task.result = kwargs["result"]
|
|
task.completed_run_id = 18
|
|
return True
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
complete_task=complete_task,
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
|
|
assert lanes.recover_pending_finalizations() == 1
|
|
assert len(guards) == 1
|
|
assert task.status == "done"
|
|
assert task.completed_run_id == 18
|
|
assert not pending.exists()
|
|
assert lanes.recover_pending_finalizations() == 0
|
|
assert len(guards) == 1
|
|
|
|
|
|
def test_complete_exception_after_journal_is_replayable_and_never_blocks(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
state_root = tmp_path / "cli-lanes"
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
|
|
task = SimpleNamespace(
|
|
id="t_crash",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=51,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
completion_raises = {"value": True}
|
|
blocks = []
|
|
comments = []
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
def complete_task(_conn, _task_id, **kwargs):
|
|
if completion_raises["value"]:
|
|
raise RuntimeError("crash between journal and commit")
|
|
task.status = "done"
|
|
task.result = kwargs["result"]
|
|
task.current_run_id = None
|
|
return True
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_crash"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "Finish safely.",
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
|
add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
|
|
complete_task=complete_task,
|
|
block_task=lambda *_args, **kwargs: blocks.append(kwargs),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"select_route",
|
|
lambda *_args, **_kwargs: lanes.Route(
|
|
"codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, ()
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"run_provider",
|
|
lambda *_args, **_kwargs: lanes.ProcessResult(
|
|
0, "", _completed_result("crash-safe result"), False
|
|
),
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_crash")
|
|
|
|
pending = list((state_root / "cassandra").glob("*.terminal.pending.json"))
|
|
assert len(pending) == 1
|
|
assert blocks == []
|
|
assert any("RuntimeError" in body for body in comments)
|
|
|
|
completion_raises["value"] = False
|
|
assert lanes.recover_pending_finalizations() == 1
|
|
assert task.status == "done"
|
|
assert not pending[0].exists()
|
|
|
|
|
|
def test_terminal_replace_then_directory_fsync_enospc_replays_after_restart(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
state_root = tmp_path / "cli-lanes"
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
|
|
task = SimpleNamespace(
|
|
id="t_enospc",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=52,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
blocks = []
|
|
comments = []
|
|
completions = []
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
def complete_task(_conn, _task_id, **kwargs):
|
|
completions.append(kwargs)
|
|
task.status = "done"
|
|
task.result = kwargs["result"]
|
|
task.current_run_id = None
|
|
return True
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_enospc"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "Finish without losing the result.",
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
|
add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
|
|
complete_task=complete_task,
|
|
block_task=lambda *_args, **kwargs: blocks.append(kwargs),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"select_route",
|
|
lambda *_args, **_kwargs: lanes.Route(
|
|
"codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, ()
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"run_provider",
|
|
lambda *_args, **_kwargs: lanes.ProcessResult(
|
|
0, "", _completed_result("persisted before ENOSPC"), False
|
|
),
|
|
)
|
|
real_replace = lanes.os.replace
|
|
real_fsync_directory = lanes._fsync_directory
|
|
terminal_replaced = {"value": False}
|
|
fail_once = {"value": True}
|
|
|
|
def replace_then_mark(source, destination):
|
|
real_replace(source, destination)
|
|
if str(destination).endswith(".terminal.pending.json"):
|
|
terminal_replaced["value"] = True
|
|
|
|
def fail_after_terminal_replace(directory):
|
|
if terminal_replaced["value"] and fail_once["value"]:
|
|
fail_once["value"] = False
|
|
raise OSError(errno.ENOSPC, "no space after terminal rename")
|
|
real_fsync_directory(directory)
|
|
|
|
monkeypatch.setattr(lanes.os, "replace", replace_then_mark)
|
|
monkeypatch.setattr(lanes, "_fsync_directory", fail_after_terminal_replace)
|
|
|
|
lanes.execute_claim("cassandra", "t_enospc")
|
|
|
|
pending = list((state_root / "cassandra").glob("*.terminal.pending.json"))
|
|
assert terminal_replaced["value"] is True
|
|
assert fail_once["value"] is False
|
|
assert len(pending) == 1
|
|
assert blocks == []
|
|
assert completions == []
|
|
assert task.status == "running"
|
|
assert any("terminal replayable=True" in body for body in comments)
|
|
|
|
# A restarted runner sees the exact pending journal and completes the run.
|
|
assert lanes.recover_pending_finalizations() == 1
|
|
assert len(completions) == 1
|
|
assert task.status == "done"
|
|
assert not pending[0].exists()
|
|
committed = list((state_root / "cassandra").glob("*.terminal.committed.json"))
|
|
assert len(committed) == 1
|
|
|
|
|
|
def test_terminal_payload_cannot_select_a_different_board(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
state_root = tmp_path / "cli-lanes"
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
|
|
path = lanes._terminal_path(lanes.state_path("alpha", "t_alpha"), 3)
|
|
lanes.atomic_json(
|
|
path,
|
|
{
|
|
"board": "beta",
|
|
"task_id": "t_beta",
|
|
"expected_run_id": 9,
|
|
"result": json.dumps(_completed_result()),
|
|
"summary": "forged",
|
|
"metadata": {},
|
|
"kanban_state": "pending",
|
|
},
|
|
)
|
|
alpha_task = SimpleNamespace(
|
|
id="t_alpha", status="running", current_run_id=3, assignee="cli-auto"
|
|
)
|
|
opened = []
|
|
reclaimed = []
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: (opened.append(board) or Connection()),
|
|
get_task=lambda _conn, task_id: alpha_task if task_id == "t_alpha" else None,
|
|
reclaim_task=lambda _conn, task_id, **_kwargs: (reclaimed.append(task_id) or True),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
|
|
assert lanes.recover_pending_finalizations() == 0
|
|
|
|
assert opened == ["alpha"]
|
|
assert reclaimed == ["t_alpha"]
|
|
assert not path.exists()
|
|
quarantined = list((state_root / "alpha/quarantine").glob("*.quarantine"))
|
|
assert len(quarantined) == 1
|
|
assert quarantined[0].stat().st_mode & 0o777 == 0o600
|
|
|
|
|
|
def test_terminal_finalize_rereads_journal_after_a_path_swap(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
|
state_file = lanes.state_path("alpha", "t_alpha")
|
|
path, safe_record = lanes._write_terminal_record(
|
|
state_file,
|
|
board="alpha",
|
|
task_id="t_alpha",
|
|
run_id=4,
|
|
structured=_completed_result("safe"),
|
|
summary="safe",
|
|
metadata={},
|
|
)
|
|
forged = dict(safe_record)
|
|
forged.update({"board": "beta", "task_id": "t_beta", "expected_run_id": 7})
|
|
lanes.atomic_json(path, forged)
|
|
opened = []
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: (opened.append(board) or pytest.fail("must not open DB")),
|
|
)
|
|
|
|
assert lanes._finalize_terminal_record(fake_db, path, safe_record) == "foreign"
|
|
assert opened == []
|
|
|
|
|
|
@pytest.mark.parametrize("payload", [b"", b'{"board":"cassandra"'])
|
|
def test_malformed_exact_run_journal_is_quarantined_and_reclaimed(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
payload: bytes,
|
|
):
|
|
state_root = tmp_path / "cli-lanes"
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
|
|
path = lanes._terminal_path(lanes.state_path("cassandra", "t_partial"), 12)
|
|
path.parent.mkdir(parents=True)
|
|
path.write_bytes(payload)
|
|
task = SimpleNamespace(
|
|
id="t_partial", status="running", current_run_id=12, assignee="cli-auto"
|
|
)
|
|
reclaimed = []
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
reclaim_task=lambda *_args, **kwargs: (reclaimed.append(kwargs) or True),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
|
|
assert lanes.recover_pending_finalizations() == 0
|
|
|
|
assert reclaimed == [{
|
|
"reason": (
|
|
"terminal journal recovery failed (malformed-payload); "
|
|
"exact run may retry"
|
|
),
|
|
"expected_run_id": 12,
|
|
}]
|
|
assert not path.exists()
|
|
assert lanes._has_pending_finalization("cassandra", "t_partial", 12) is False
|
|
quarantined = list((state_root / "cassandra/quarantine").glob("*.quarantine"))
|
|
assert len(quarantined) == 1
|
|
assert quarantined[0].stat().st_mode & 0o777 == 0o600
|