7a55b259bf
Three fenced worker Pods claim Hermes Kanban runs through a coordinator that owns every state transition, with per-ordinal HMAC authority, a mediated broker-only SCM path, and durable per-ordinal workspaces. Content is the reviewed head of PR #18 (689bcb6e) with PR 16's and PR 19's contributions removed: they were merged in only to validate co-existence and are not prerequisites, so this branch no longer carries them as ancestors. Only PR 14 and PR 15 remain, because the broker boundary and the cli_lane_* decomposition are load-bearing for two of the fixed P0 boundaries. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
333 lines
12 KiB
Python
333 lines
12 KiB
Python
"""Recovery convergence and hidden-authority edge coverage."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from testing.tests.test_hermes_cli_support import (
|
|
_pending_terminal_record,
|
|
lanes,
|
|
)
|
|
|
|
|
|
class _Snapshot:
|
|
def __init__(self, document=None):
|
|
self.document = document
|
|
self.closed = 0
|
|
|
|
def close(self):
|
|
self.closed += 1
|
|
|
|
|
|
def test_terminal_absence_distinguishes_present_missing_and_inaccessible(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
path = tmp_path / "journal"
|
|
path.write_text("data", encoding="utf-8")
|
|
assert lanes._terminal_entry_absent(path) is False
|
|
path.unlink()
|
|
assert lanes._terminal_entry_absent(path) is True
|
|
monkeypatch.setattr(
|
|
lanes.os,
|
|
"open",
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(PermissionError("denied")),
|
|
)
|
|
assert lanes._terminal_entry_absent(path) is False
|
|
|
|
|
|
def _staged_path(root: Path, identity, document: dict) -> tuple[Path, Path]:
|
|
if identity.state in {"pending", "committed"}:
|
|
canonical = lanes._terminal_path(
|
|
lanes.state_path(identity.board, identity.task_id),
|
|
identity.run_id,
|
|
identity.state,
|
|
)
|
|
else:
|
|
canonical = lanes._terminal_evidence_path(identity, identity.state, document)
|
|
digest = hashlib.sha256(canonical.name.encode("utf-8")).hexdigest()[:16]
|
|
return root / identity.board / f".retire.{digest}.{'a' * 16}.0", canonical
|
|
|
|
|
|
@pytest.mark.parametrize("state", ["prepared", "conflict"])
|
|
def test_staged_authority_accepts_valid_evidence_states(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
state: str,
|
|
):
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
|
identity = lanes.TerminalIdentity("board", "task", 5, state)
|
|
document = _pending_terminal_record("board", "task", 5, "accepted")
|
|
document["kanban_state"] = state
|
|
staged, canonical = _staged_path(lanes.STATE_ROOT, identity, document)
|
|
staged.parent.mkdir(parents=True)
|
|
authority = lanes._staged_terminal_authority(staged, _Snapshot(document))
|
|
assert authority == (identity, canonical)
|
|
|
|
|
|
def test_staged_authority_rejects_outside_nested_and_unknown_state(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
|
document = _pending_terminal_record("board", "task", 5, "accepted")
|
|
outside = tmp_path / ".retire.x"
|
|
assert lanes._staged_terminal_authority(outside, _Snapshot(document)) is None
|
|
nested = lanes.STATE_ROOT / "board" / "nested" / ".retire.name"
|
|
assert lanes._staged_terminal_authority(nested, _Snapshot(document)) is None
|
|
board = lanes.STATE_ROOT / "board"
|
|
board.mkdir(parents=True)
|
|
staged = board / f".retire.{'a' * 16}.{'b' * 16}.0"
|
|
document["kanban_state"] = "unknown"
|
|
assert lanes._staged_terminal_authority(staged, _Snapshot(document)) is None
|
|
|
|
valid = _pending_terminal_record("board", "task", 5, "accepted")
|
|
assert lanes._staged_terminal_authority(staged, _Snapshot(valid)) is None
|
|
|
|
|
|
def test_restore_staged_authority_handles_every_first_writer_state(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
document = _pending_terminal_record("board", "task", 6, "accepted")
|
|
canonical = tmp_path / "canonical"
|
|
pending = lanes.TerminalIdentity("board", "task", 6, "pending")
|
|
calls = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_persist_prepared_evidence",
|
|
lambda identity, value: calls.append(("prepared", identity, value)),
|
|
)
|
|
lanes._restore_staged_terminal_authority(pending, canonical, document)
|
|
assert calls[0][0] == "prepared"
|
|
|
|
monkeypatch.setattr(lanes, "_write_json_noreplace", lambda *_args: True)
|
|
committed = lanes.TerminalIdentity("board", "task", 6, "committed")
|
|
lanes._restore_staged_terminal_authority(committed, canonical, document)
|
|
|
|
monkeypatch.setattr(lanes, "_write_json_noreplace", lambda *_args: False)
|
|
monkeypatch.setattr(lanes, "_load_small_json", lambda _path: {})
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_persist_conflict_evidence",
|
|
lambda *args: calls.append(("conflict", *args)),
|
|
)
|
|
lanes._restore_staged_terminal_authority(committed, canonical, document)
|
|
assert calls[-1][0] == "conflict"
|
|
|
|
prepared = lanes.TerminalIdentity("board", "task", 6, "prepared")
|
|
monkeypatch.setattr(lanes, "_terminal_evidence_valid", lambda *_args: False)
|
|
with pytest.raises(OSError, match="conflicting result"):
|
|
lanes._restore_staged_terminal_authority(prepared, canonical, document)
|
|
|
|
|
|
def test_restore_staged_authority_accepts_identical_existing_evidence(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
document = _pending_terminal_record("board", "task", 7, "accepted")
|
|
document["kanban_state"] = "prepared"
|
|
identity = lanes.TerminalIdentity("board", "task", 7, "prepared")
|
|
monkeypatch.setattr(lanes, "_write_json_noreplace", lambda *_args: False)
|
|
monkeypatch.setattr(lanes, "_load_small_json", lambda _path: document)
|
|
monkeypatch.setattr(lanes, "_terminal_evidence_valid", lambda *_args: True)
|
|
lanes._restore_staged_terminal_authority(identity, tmp_path / "existing", document)
|
|
|
|
|
|
def test_staging_recovery_bounds_missing_snapshots_and_restore_errors(
|
|
monkeypatch,
|
|
):
|
|
path = Path("/tmp/staged")
|
|
monkeypatch.setattr(lanes, "_retirement_staging_paths", lambda: [path])
|
|
monkeypatch.setattr(lanes, "_open_terminal_recovery_snapshot", lambda _path: None)
|
|
assert lanes._recover_retirement_staging() == 0
|
|
|
|
identity = lanes.TerminalIdentity("board", "task", 8, "pending")
|
|
snapshot = _Snapshot(_pending_terminal_record("board", "task", 8, "accepted"))
|
|
monkeypatch.setattr(lanes, "_open_terminal_recovery_snapshot", lambda _path: snapshot)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_staged_terminal_authority",
|
|
lambda *_args: (identity, Path("/tmp/canonical")),
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_restore_staged_terminal_authority",
|
|
lambda *_args: (_ for _ in ()).throw(OSError("restore")),
|
|
)
|
|
errors = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_record_board_access_error",
|
|
lambda board, error: errors.append((board, str(error))),
|
|
)
|
|
assert lanes._recover_retirement_staging() == 0
|
|
assert errors == [("board", "restore")]
|
|
assert snapshot.closed == 1
|
|
|
|
|
|
def test_drain_staging_ignores_disappearing_entries(monkeypatch):
|
|
class Vanished:
|
|
def stat(self, **_kwargs):
|
|
raise FileNotFoundError
|
|
|
|
monkeypatch.setattr(lanes, "_retirement_staging_paths", lambda: [Vanished()])
|
|
assert lanes._drain_retirement_staging() == 0
|
|
|
|
|
|
def _prepared_path(tmp_path: Path, monkeypatch) -> tuple[Path, dict]:
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
|
pending = _pending_terminal_record("board", "task", 9, "accepted")
|
|
identity = lanes.TerminalIdentity("board", "task", 9, "pending")
|
|
path = lanes._persist_prepared_evidence(identity, pending)
|
|
return path, pending
|
|
|
|
|
|
def test_prepared_recovery_defers_without_discarding_evidence(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
path, _document = _prepared_path(tmp_path, monkeypatch)
|
|
monkeypatch.setattr(lanes, "_finalize_document_db", lambda *_args: "deferred")
|
|
assert lanes._recover_prepared_finalizations(object()) == 0
|
|
assert path.exists()
|
|
|
|
|
|
def test_prepared_recovery_bounds_db_and_publication_exceptions(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
path, _document = _prepared_path(tmp_path, monkeypatch)
|
|
errors = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_record_board_access_error",
|
|
lambda board, error: errors.append((board, str(error))),
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_finalize_document_db",
|
|
lambda *_args: (_ for _ in ()).throw(OSError("database")),
|
|
)
|
|
assert lanes._recover_prepared_finalizations(object()) == 0
|
|
assert errors[-1] == ("board", "database")
|
|
assert path.exists()
|
|
|
|
monkeypatch.setattr(lanes, "_finalize_document_db", lambda *_args: "committed")
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_promote_prepared_evidence",
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("publish")),
|
|
)
|
|
assert lanes._recover_prepared_finalizations(object()) == 0
|
|
assert errors[-1] == ("board", "publish")
|
|
|
|
|
|
def test_pending_recovery_bounds_finalizer_errors_and_replacement_churn(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
capsys,
|
|
):
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
|
path = lanes._terminal_path(lanes.state_path("board", "task"), 10)
|
|
lanes.atomic_json(path, _pending_terminal_record("board", "task", 10, "accepted"))
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=object()))
|
|
errors = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_record_board_access_error",
|
|
lambda board, error: errors.append((board, str(error))),
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_finalize_terminal_record",
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("finalize")),
|
|
)
|
|
assert lanes.recover_pending_finalizations() == 0
|
|
assert errors[-1] == ("board", "finalize")
|
|
|
|
def invalid(*_args, snapshot=None, **_kwargs):
|
|
snapshot.close()
|
|
return "invalid"
|
|
|
|
monkeypatch.setattr(lanes, "_finalize_terminal_record", invalid)
|
|
assert lanes.recover_pending_finalizations() == 0
|
|
assert "replacement churn deferred" in capsys.readouterr().err
|
|
|
|
|
|
def test_pending_recovery_classifies_foreign_identity(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
|
path = lanes._terminal_path(lanes.state_path("board", "task"), 11)
|
|
lanes.atomic_json(path, _pending_terminal_record("other", "task", 11, "accepted"))
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=object()))
|
|
reasons = []
|
|
|
|
def quarantine(_path, _identity, reason, *, snapshot):
|
|
reasons.append(reason)
|
|
os.unlink(_path)
|
|
|
|
monkeypatch.setattr(lanes, "_quarantine_terminal", quarantine)
|
|
monkeypatch.setattr(lanes, "_recover_exact_run", lambda *_args: False)
|
|
assert lanes.recover_pending_finalizations() == 0
|
|
assert reasons == ["foreign-identity"]
|
|
|
|
|
|
def test_pending_guard_rejects_invalid_run_and_inaccessible_or_bad_artifacts(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
|
assert lanes._has_pending_finalization("board", "task", True) is False
|
|
path = lanes._terminal_path(lanes.state_path("board", "task"), 12)
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text("bad", encoding="utf-8")
|
|
real_stat = Path.stat
|
|
|
|
def denied(candidate, *args, **kwargs):
|
|
if candidate == path:
|
|
raise PermissionError("denied")
|
|
return real_stat(candidate, *args, **kwargs)
|
|
|
|
monkeypatch.setattr(Path, "stat", denied)
|
|
assert lanes._has_pending_finalization("board", "task", 12) is False
|
|
|
|
|
|
def test_pending_guard_scans_invalid_prepared_and_staged_candidates(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
|
base = lanes.state_path("board", "task")
|
|
pending = lanes._terminal_path(base, 13)
|
|
pending.parent.mkdir(parents=True)
|
|
pending.write_text("{}", encoding="utf-8")
|
|
(pending.parent / "task.run-13.terminal.prepared-bad.json").write_text(
|
|
"{}", encoding="utf-8"
|
|
)
|
|
(pending.parent / f"task.run-13.terminal.prepared-{'a' * 32}.json").write_text(
|
|
"{}", encoding="utf-8"
|
|
)
|
|
staged = [
|
|
pending.parent / f".retire.{'b' * 16}.{'c' * 16}.0",
|
|
pending.parent / f".retire.{'d' * 16}.{'e' * 16}.0",
|
|
]
|
|
for path in staged:
|
|
path.write_text("{}", encoding="utf-8")
|
|
snapshots = iter((None, _Snapshot({})))
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_open_terminal_recovery_snapshot",
|
|
lambda _path: next(snapshots),
|
|
)
|
|
monkeypatch.setattr(lanes, "_staged_terminal_authority", lambda *_args: None)
|
|
assert lanes._has_pending_finalization("board", "task", 13) is False
|