421 lines
13 KiB
Python
421 lines
13 KiB
Python
|
"""Board scanning, claiming, orphan recovery, and replay guards."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
from testing.tests.test_hermes_cli_support import (
|
||
|
|
Path,
|
||
|
|
SimpleNamespace,
|
||
|
|
_completed_result,
|
||
|
|
json,
|
||
|
|
lanes,
|
||
|
|
nullcontext,
|
||
|
|
pytest,
|
||
|
|
sys,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_unassigned_ready_task_is_persistently_routed_to_auto_lane(monkeypatch):
|
||
|
|
task = SimpleNamespace(id="t_auto", assignee=None, status="ready")
|
||
|
|
assigned = []
|
||
|
|
|
||
|
|
class Connection:
|
||
|
|
def close(self):
|
||
|
|
return None
|
||
|
|
|
||
|
|
def assign_task(_conn, task_id, profile):
|
||
|
|
assigned.append((task_id, profile))
|
||
|
|
task.assignee = profile
|
||
|
|
return True
|
||
|
|
|
||
|
|
fake_db = SimpleNamespace(
|
||
|
|
list_boards=lambda include_archived=False: [{"slug": "cassandra"}],
|
||
|
|
scoped_current_board=lambda _board: nullcontext(),
|
||
|
|
connect=lambda board: Connection(),
|
||
|
|
recompute_ready=lambda _conn: None,
|
||
|
|
list_tasks=lambda _conn: [task],
|
||
|
|
assign_task=assign_task,
|
||
|
|
get_task=lambda _conn, _task_id: task,
|
||
|
|
claim_task=lambda _conn, _task_id, **_kwargs: task,
|
||
|
|
)
|
||
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
||
|
|
|
||
|
|
assert lanes.claim_ready(set(), 1) == [("cassandra", "t_auto")]
|
||
|
|
assert assigned == [("t_auto", "cli-auto")]
|
||
|
|
|
||
|
|
|
||
|
|
def test_corrupt_board_is_quarantined_without_stopping_healthy_lanes(monkeypatch, capsys):
|
||
|
|
class CorruptBoardError(Exception):
|
||
|
|
pass
|
||
|
|
|
||
|
|
task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready")
|
||
|
|
|
||
|
|
class Connection:
|
||
|
|
def close(self):
|
||
|
|
return None
|
||
|
|
|
||
|
|
def connect(*, board):
|
||
|
|
if board == "cassandra":
|
||
|
|
raise CorruptBoardError("integrity_check failed")
|
||
|
|
return Connection()
|
||
|
|
|
||
|
|
fake_db = SimpleNamespace(
|
||
|
|
KanbanDbCorruptError=CorruptBoardError,
|
||
|
|
list_boards=lambda include_archived=False: [
|
||
|
|
{"slug": "cassandra"},
|
||
|
|
{"slug": "healthy"},
|
||
|
|
],
|
||
|
|
scoped_current_board=lambda _board: nullcontext(),
|
||
|
|
connect=connect,
|
||
|
|
recompute_ready=lambda _conn: None,
|
||
|
|
list_tasks=lambda _conn: [task],
|
||
|
|
claim_task=lambda _conn, _task_id, **_kwargs: task,
|
||
|
|
)
|
||
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
||
|
|
lanes.BOARD_CORRUPTION_ERRORS.clear()
|
||
|
|
|
||
|
|
assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")]
|
||
|
|
assert "temporarily skipping Kanban board 'cassandra'" in capsys.readouterr().err
|
||
|
|
|
||
|
|
|
||
|
|
def test_transient_board_scan_failure_does_not_stop_healthy_lanes(monkeypatch, capsys):
|
||
|
|
task = SimpleNamespace(id="t_healthy", assignee="cli-auto", status="ready")
|
||
|
|
|
||
|
|
class Connection:
|
||
|
|
def __init__(self, board):
|
||
|
|
self.board = board
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
return None
|
||
|
|
|
||
|
|
def recompute_ready(connection):
|
||
|
|
if connection.board == "cassandra":
|
||
|
|
raise lanes.sqlite3.OperationalError("disk I/O error")
|
||
|
|
|
||
|
|
fake_db = SimpleNamespace(
|
||
|
|
list_boards=lambda include_archived=False: [
|
||
|
|
{"slug": "cassandra"},
|
||
|
|
{"slug": "healthy"},
|
||
|
|
],
|
||
|
|
scoped_current_board=lambda _board: nullcontext(),
|
||
|
|
connect=lambda board: Connection(board),
|
||
|
|
recompute_ready=recompute_ready,
|
||
|
|
list_tasks=lambda _conn: [task],
|
||
|
|
claim_task=lambda _conn, _task_id, **_kwargs: task,
|
||
|
|
)
|
||
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
||
|
|
lanes.BOARD_CORRUPTION_ERRORS.clear()
|
||
|
|
|
||
|
|
assert lanes.claim_ready(set(), 1) == [("healthy", "t_healthy")]
|
||
|
|
error = capsys.readouterr().err
|
||
|
|
assert "temporarily skipping Kanban board 'cassandra'" in error
|
||
|
|
assert "storage OperationalError: disk I/O error" in error
|
||
|
|
|
||
|
|
|
||
|
|
def test_board_call_retries_storage_faults_on_fresh_connections():
|
||
|
|
connections = []
|
||
|
|
|
||
|
|
class Connection:
|
||
|
|
def __init__(self):
|
||
|
|
self.closed = False
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
self.closed = True
|
||
|
|
|
||
|
|
def connect(*, board):
|
||
|
|
assert board == "cassandra"
|
||
|
|
connection = Connection()
|
||
|
|
connections.append(connection)
|
||
|
|
return connection
|
||
|
|
|
||
|
|
attempts = []
|
||
|
|
|
||
|
|
def operation(_connection):
|
||
|
|
attempts.append(1)
|
||
|
|
if len(attempts) < 3:
|
||
|
|
raise lanes.sqlite3.OperationalError("disk I/O error")
|
||
|
|
return "healthy"
|
||
|
|
|
||
|
|
fake_db = SimpleNamespace(
|
||
|
|
scoped_current_board=lambda _board: nullcontext(),
|
||
|
|
connect=connect,
|
||
|
|
)
|
||
|
|
lanes.BOARD_CORRUPTION_ERRORS.clear()
|
||
|
|
|
||
|
|
assert lanes._board_call(fake_db, "cassandra", operation) == "healthy"
|
||
|
|
assert len(connections) == 3
|
||
|
|
assert all(connection.closed for connection in connections)
|
||
|
|
|
||
|
|
|
||
|
|
def test_accepted_result_survives_failed_finalization_and_replays_exact_run(
|
||
|
|
tmp_path: Path,
|
||
|
|
monkeypatch,
|
||
|
|
):
|
||
|
|
state_root = tmp_path / "cli-lanes"
|
||
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
|
||
|
|
task = SimpleNamespace(
|
||
|
|
id="t_terminal",
|
||
|
|
status="running",
|
||
|
|
result=None,
|
||
|
|
current_run_id=41,
|
||
|
|
assignee="cli-auto",
|
||
|
|
max_runtime_seconds=60,
|
||
|
|
)
|
||
|
|
completions = []
|
||
|
|
comments = []
|
||
|
|
permit_completion = {"value": False}
|
||
|
|
|
||
|
|
class Connection:
|
||
|
|
def close(self):
|
||
|
|
return None
|
||
|
|
|
||
|
|
def complete_task(_conn, _task_id, **kwargs):
|
||
|
|
completions.append(kwargs)
|
||
|
|
if not permit_completion["value"]:
|
||
|
|
return False
|
||
|
|
task.status = "done"
|
||
|
|
task.result = kwargs["result"]
|
||
|
|
task.completed_run_id = kwargs["expected_run_id"]
|
||
|
|
task.current_run_id = None
|
||
|
|
return True
|
||
|
|
|
||
|
|
fake_db = SimpleNamespace(
|
||
|
|
scoped_current_board=lambda _board: nullcontext(),
|
||
|
|
connect=lambda board: Connection(),
|
||
|
|
get_task=lambda _conn, _task_id: task,
|
||
|
|
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
||
|
|
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_terminal"),
|
||
|
|
set_branch_name=lambda *_args: None,
|
||
|
|
set_workspace_path=lambda *_args: None,
|
||
|
|
build_worker_context=lambda *_args: "Finish and verify the objective.",
|
||
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
||
|
|
add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
|
||
|
|
complete_task=complete_task,
|
||
|
|
block_task=lambda *_args, **_kwargs: pytest.fail(
|
||
|
|
"an accepted journaled result must not be converted into a block"
|
||
|
|
),
|
||
|
|
)
|
||
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
||
|
|
monkeypatch.setattr(
|
||
|
|
lanes,
|
||
|
|
"select_route",
|
||
|
|
lambda *_args, **_kwargs: lanes.Route(
|
||
|
|
"codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
|
||
|
|
),
|
||
|
|
)
|
||
|
|
structured = {
|
||
|
|
"status": "completed",
|
||
|
|
"summary": "Verified exact terminal result.",
|
||
|
|
"changed_files": [],
|
||
|
|
"tests_run": ["pytest: passed"],
|
||
|
|
"artifacts": [],
|
||
|
|
"findings": [],
|
||
|
|
"blockers": [],
|
||
|
|
}
|
||
|
|
monkeypatch.setattr(
|
||
|
|
lanes,
|
||
|
|
"run_provider",
|
||
|
|
lambda *_args, **_kwargs: lanes.ProcessResult(0, "", structured, False),
|
||
|
|
)
|
||
|
|
|
||
|
|
lanes.execute_claim("cassandra", "t_terminal")
|
||
|
|
|
||
|
|
candidates = list((state_root / "cassandra").glob("*.candidate-*.json"))
|
||
|
|
terminals = list((state_root / "cassandra").glob("*.terminal.pending.json"))
|
||
|
|
assert len(candidates) == 1
|
||
|
|
assert json.loads(candidates[0].read_text())["structured"] == structured
|
||
|
|
assert len(terminals) == 1
|
||
|
|
assert json.loads(terminals[0].read_text())["kanban_state"] == "pending"
|
||
|
|
assert task.status == "running"
|
||
|
|
assert any("durably journaled" in body for body in comments)
|
||
|
|
|
||
|
|
permit_completion["value"] = True
|
||
|
|
assert lanes.recover_pending_finalizations() == 1
|
||
|
|
assert task.status == "done"
|
||
|
|
assert len(completions) == 2
|
||
|
|
assert not terminals[0].exists()
|
||
|
|
committed = list(
|
||
|
|
(state_root / "cassandra").glob("*.terminal.committed.json")
|
||
|
|
)
|
||
|
|
assert len(committed) == 1
|
||
|
|
terminal = json.loads(committed[0].read_text())
|
||
|
|
assert terminal["kanban_state"] == "committed"
|
||
|
|
assert completions[-1]["result"] == terminal["result"]
|
||
|
|
|
||
|
|
|
||
|
|
def test_restart_does_not_reclaim_an_exact_run_awaiting_finalization(
|
||
|
|
tmp_path: Path,
|
||
|
|
monkeypatch,
|
||
|
|
):
|
||
|
|
state_root = tmp_path / "cli-lanes"
|
||
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
|
||
|
|
state_file = lanes.state_path("cassandra", "t_terminal")
|
||
|
|
terminal_path, _record = lanes._write_terminal_record(
|
||
|
|
state_file,
|
||
|
|
board="cassandra",
|
||
|
|
task_id="t_terminal",
|
||
|
|
run_id=8,
|
||
|
|
structured={
|
||
|
|
"status": "completed",
|
||
|
|
"summary": "done",
|
||
|
|
"changed_files": [],
|
||
|
|
"tests_run": [],
|
||
|
|
"artifacts": [],
|
||
|
|
"findings": [],
|
||
|
|
"blockers": [],
|
||
|
|
},
|
||
|
|
summary="done",
|
||
|
|
metadata={},
|
||
|
|
)
|
||
|
|
assert terminal_path.exists()
|
||
|
|
task = SimpleNamespace(
|
||
|
|
id="t_terminal",
|
||
|
|
status="running",
|
||
|
|
assignee="cli-auto",
|
||
|
|
current_run_id=8,
|
||
|
|
)
|
||
|
|
reclaimed = []
|
||
|
|
|
||
|
|
class Connection:
|
||
|
|
def close(self):
|
||
|
|
return None
|
||
|
|
|
||
|
|
fake_db = SimpleNamespace(
|
||
|
|
list_boards=lambda include_archived=False: [{"slug": "cassandra"}],
|
||
|
|
scoped_current_board=lambda _board: nullcontext(),
|
||
|
|
connect=lambda board: Connection(),
|
||
|
|
list_tasks=lambda _conn: [task],
|
||
|
|
reclaim_task=lambda *_args, **_kwargs: reclaimed.append(True),
|
||
|
|
)
|
||
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
||
|
|
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
||
|
|
|
||
|
|
lanes.recover_orphans()
|
||
|
|
|
||
|
|
assert reclaimed == []
|
||
|
|
|
||
|
|
|
||
|
|
def test_orphan_recovery_passes_the_scanned_run_as_atomic_reclaim_guard(
|
||
|
|
tmp_path: Path,
|
||
|
|
monkeypatch,
|
||
|
|
):
|
||
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
||
|
|
tasks = [
|
||
|
|
SimpleNamespace(
|
||
|
|
id="t_guarded",
|
||
|
|
status="running",
|
||
|
|
assignee="cli-auto",
|
||
|
|
current_run_id=91,
|
||
|
|
),
|
||
|
|
SimpleNamespace(
|
||
|
|
id="t_no_run",
|
||
|
|
status="running",
|
||
|
|
assignee="cli-auto",
|
||
|
|
current_run_id=None,
|
||
|
|
),
|
||
|
|
]
|
||
|
|
reclaimed = []
|
||
|
|
|
||
|
|
class Connection:
|
||
|
|
def close(self):
|
||
|
|
return None
|
||
|
|
|
||
|
|
fake_db = SimpleNamespace(
|
||
|
|
list_boards=lambda include_archived=False: [{"slug": "cassandra"}],
|
||
|
|
scoped_current_board=lambda _board: nullcontext(),
|
||
|
|
connect=lambda board: Connection(),
|
||
|
|
list_tasks=lambda _conn: tasks,
|
||
|
|
reclaim_task=lambda _conn, task_id, **kwargs: (
|
||
|
|
reclaimed.append((task_id, kwargs)) or True
|
||
|
|
),
|
||
|
|
)
|
||
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
||
|
|
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
||
|
|
|
||
|
|
lanes.recover_orphans()
|
||
|
|
|
||
|
|
assert reclaimed == [
|
||
|
|
(
|
||
|
|
"t_guarded",
|
||
|
|
{
|
||
|
|
"reason": "direct CLI lane restarted; provider session will resume",
|
||
|
|
"expected_run_id": 91,
|
||
|
|
},
|
||
|
|
)
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
def test_prepared_evidence_without_pending_still_pins_the_exact_run(
|
||
|
|
tmp_path: Path,
|
||
|
|
monkeypatch,
|
||
|
|
):
|
||
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
||
|
|
pending, record = lanes._write_terminal_record(
|
||
|
|
lanes.state_path("cassandra", "t_prepared_pin"),
|
||
|
|
board="cassandra",
|
||
|
|
task_id="t_prepared_pin",
|
||
|
|
run_id=14,
|
||
|
|
structured=_completed_result("accepted and prepared"),
|
||
|
|
summary="accepted and prepared",
|
||
|
|
metadata={},
|
||
|
|
)
|
||
|
|
identity = lanes._terminal_identity(pending)
|
||
|
|
assert identity is not None
|
||
|
|
lanes._persist_prepared_evidence(identity, record)
|
||
|
|
pending.unlink()
|
||
|
|
|
||
|
|
assert lanes._has_pending_finalization("cassandra", "t_prepared_pin", 14) is True
|
||
|
|
|
||
|
|
|
||
|
|
def test_terminal_replay_never_crosses_into_a_replacement_run(
|
||
|
|
tmp_path: Path,
|
||
|
|
monkeypatch,
|
||
|
|
):
|
||
|
|
state_root = tmp_path / "cli-lanes"
|
||
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
|
||
|
|
state_file = lanes.state_path("cassandra", "t_terminal")
|
||
|
|
terminal_path, _record = lanes._write_terminal_record(
|
||
|
|
state_file,
|
||
|
|
board="cassandra",
|
||
|
|
task_id="t_terminal",
|
||
|
|
run_id=8,
|
||
|
|
structured={
|
||
|
|
"status": "completed",
|
||
|
|
"summary": "old run",
|
||
|
|
"changed_files": [],
|
||
|
|
"tests_run": [],
|
||
|
|
"artifacts": [],
|
||
|
|
"findings": [],
|
||
|
|
"blockers": [],
|
||
|
|
},
|
||
|
|
summary="old run",
|
||
|
|
metadata={},
|
||
|
|
)
|
||
|
|
replacement = SimpleNamespace(
|
||
|
|
id="t_terminal",
|
||
|
|
status="running",
|
||
|
|
result=None,
|
||
|
|
current_run_id=9,
|
||
|
|
)
|
||
|
|
completions = []
|
||
|
|
|
||
|
|
class Connection:
|
||
|
|
def close(self):
|
||
|
|
return None
|
||
|
|
|
||
|
|
fake_db = SimpleNamespace(
|
||
|
|
scoped_current_board=lambda _board: nullcontext(),
|
||
|
|
connect=lambda board: Connection(),
|
||
|
|
get_task=lambda _conn, _task_id: replacement,
|
||
|
|
complete_task=lambda *_args, **_kwargs: completions.append(True),
|
||
|
|
)
|
||
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
||
|
|
|
||
|
|
assert lanes.recover_pending_finalizations() == 0
|
||
|
|
assert completions == []
|
||
|
|
assert not terminal_path.exists()
|
||
|
|
conflicts = list((state_root / "cassandra").glob("*.terminal.conflict-*.json"))
|
||
|
|
assert len(conflicts) == 1
|
||
|
|
conflict = json.loads(conflicts[0].read_text(encoding="utf-8"))
|
||
|
|
assert conflict["result"] == _record["result"]
|
||
|
|
assert conflict["kanban_state"] == "conflict"
|