461 lines
16 KiB
Python
461 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Bounded replay of pending, prepared, and staged terminal journals."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from cli_lane_board import _record_board_access_error
|
|
from cli_lane_config import (
|
|
STATE_ROOT,
|
|
TerminalIdentity,
|
|
TerminalRecoverySnapshot,
|
|
canonical_run_id,
|
|
)
|
|
from cli_lane_evidence import (
|
|
_persist_conflict_evidence,
|
|
_persist_prepared_evidence,
|
|
_same_terminal_document,
|
|
_terminal_evidence_path,
|
|
_terminal_evidence_valid,
|
|
_write_json_noreplace,
|
|
)
|
|
from cli_lane_files import (
|
|
_LEGACY_RETIRE_NAME,
|
|
_RETIRE_NAME,
|
|
_SAFE_BOARD,
|
|
_terminal_evidence_identity,
|
|
_terminal_identity,
|
|
_terminal_path,
|
|
state_path,
|
|
)
|
|
from cli_lane_finalization import (
|
|
_finalize_document_db,
|
|
_finalize_terminal_record,
|
|
_promote_prepared_evidence,
|
|
_recover_exact_run,
|
|
_resolve_pending_after_winner,
|
|
_retire_snapshot,
|
|
_retire_snapshot_after_db,
|
|
)
|
|
from cli_lane_quarantine import _quarantine_terminal
|
|
from cli_lane_records import (
|
|
_load_small_json,
|
|
_load_terminal_json,
|
|
_open_terminal_recovery_snapshot,
|
|
_terminal_record_valid,
|
|
)
|
|
|
|
|
|
def _terminal_entry_absent(path: Path) -> bool:
|
|
"""Confirm absence through a nofollow directory descriptor."""
|
|
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
|
|
directory_flags |= getattr(os, "O_NOFOLLOW", 0)
|
|
directory = None
|
|
try:
|
|
directory = os.open(path.parent, directory_flags)
|
|
os.stat(path.name, dir_fd=directory, follow_symlinks=False)
|
|
return False
|
|
except FileNotFoundError:
|
|
return True
|
|
except OSError:
|
|
return False
|
|
finally:
|
|
if directory is not None:
|
|
os.close(directory)
|
|
|
|
def _staged_terminal_authority(
|
|
path: Path,
|
|
snapshot: TerminalRecoverySnapshot,
|
|
) -> tuple[TerminalIdentity, Path] | None:
|
|
"""Bind a staged inode to the canonical pending name it was moved from."""
|
|
try:
|
|
relative = path.relative_to(STATE_ROOT)
|
|
except ValueError:
|
|
return None
|
|
if len(relative.parts) == 2:
|
|
board, filename = relative.parts
|
|
elif len(relative.parts) == 3 and relative.parts[1] == "quarantine":
|
|
board, _quarantine, filename = relative.parts
|
|
else:
|
|
return None
|
|
match = _RETIRE_NAME.fullmatch(filename)
|
|
legacy = _LEGACY_RETIRE_NAME.fullmatch(filename)
|
|
document = snapshot.document
|
|
if (
|
|
(match is None and legacy is None)
|
|
or document is None
|
|
or not _SAFE_BOARD.fullmatch(board)
|
|
or board in {".", ".."}
|
|
or document.get("board") != board
|
|
or not isinstance(document.get("task_id"), str)
|
|
or type(document.get("expected_run_id")) is not int
|
|
or canonical_run_id(document.get("expected_run_id")) is None
|
|
):
|
|
return None
|
|
state = document.get("kanban_state")
|
|
if state not in {"pending", "prepared", "committed", "conflict"}:
|
|
return None
|
|
identity = TerminalIdentity(
|
|
board, document["task_id"], document["expected_run_id"], state
|
|
)
|
|
if state in {"pending", "committed"}:
|
|
canonical = _terminal_path(
|
|
state_path(identity.board, identity.task_id),
|
|
identity.run_id,
|
|
state,
|
|
)
|
|
valid = _terminal_record_valid(document, identity)
|
|
else:
|
|
canonical = _terminal_evidence_path(identity, state, document)
|
|
valid = _terminal_evidence_valid(document, identity, state)
|
|
board_dir = STATE_ROOT / board
|
|
if (
|
|
canonical.parent != board_dir
|
|
or path.parent not in {board_dir, board_dir / "quarantine"}
|
|
or (
|
|
match is not None
|
|
and hashlib.sha256(canonical.name.encode("utf-8")).hexdigest()[:16]
|
|
!= match.group("path_digest")
|
|
)
|
|
or not valid
|
|
):
|
|
return None
|
|
return identity, canonical
|
|
|
|
def _retirement_staging_paths() -> list[Path]:
|
|
"""List every directory where identity-safe retirement can stage an inode."""
|
|
paths = list(STATE_ROOT.glob("*/.retire.*"))
|
|
paths.extend(STATE_ROOT.glob("*/quarantine/.retire.*"))
|
|
return sorted(set(paths))
|
|
|
|
def _restore_staged_terminal_authority(
|
|
identity: TerminalIdentity,
|
|
canonical: Path,
|
|
document: dict[str, Any],
|
|
) -> None:
|
|
"""Restore one hidden accepted artifact without executing it prematurely."""
|
|
if identity.state == "pending":
|
|
_persist_prepared_evidence(identity, document)
|
|
return
|
|
if _write_json_noreplace(canonical, document):
|
|
return
|
|
existing = _load_small_json(canonical)
|
|
if identity.state in {"prepared", "conflict"}:
|
|
valid = _terminal_evidence_valid(existing, identity, identity.state)
|
|
else:
|
|
valid = _terminal_record_valid(existing, identity)
|
|
if valid and _same_terminal_document(existing, document):
|
|
return
|
|
if identity.state == "committed":
|
|
_persist_conflict_evidence(
|
|
identity,
|
|
document,
|
|
"staged committed evidence conflicts with the canonical first writer",
|
|
)
|
|
return
|
|
raise OSError("staged terminal evidence path contains a conflicting result")
|
|
|
|
def _recover_retirement_staging() -> int:
|
|
"""Promote valid hidden journals to durable prepared evidence."""
|
|
recovered = 0
|
|
for path in _retirement_staging_paths():
|
|
snapshot = _open_terminal_recovery_snapshot(path)
|
|
if snapshot is None:
|
|
continue
|
|
authority = _staged_terminal_authority(path, snapshot)
|
|
if authority is None:
|
|
try:
|
|
_quarantine_terminal(
|
|
path,
|
|
None,
|
|
"untrusted-retirement-staging",
|
|
snapshot=snapshot,
|
|
)
|
|
finally:
|
|
snapshot.close()
|
|
continue
|
|
identity, canonical = authority
|
|
try:
|
|
_restore_staged_terminal_authority(
|
|
identity,
|
|
canonical,
|
|
snapshot.document or {},
|
|
)
|
|
retirement = _retire_snapshot(
|
|
path,
|
|
snapshot,
|
|
authority_name=canonical.name,
|
|
)
|
|
except Exception as error:
|
|
_record_board_access_error(identity.board, error)
|
|
continue
|
|
finally:
|
|
snapshot.close()
|
|
if retirement in {"retired", "missing"}:
|
|
recovered += 1
|
|
return recovered
|
|
|
|
def _drain_retirement_staging() -> int:
|
|
"""Converge bounded replacement churn without spinning on storage failure."""
|
|
recovered = 0
|
|
previous: tuple[tuple[str, int, int, int, int], ...] | None = None
|
|
for _attempt in range(16):
|
|
surface = []
|
|
for path in _retirement_staging_paths():
|
|
try:
|
|
observed = path.stat(follow_symlinks=False)
|
|
except OSError:
|
|
continue
|
|
surface.append(
|
|
(
|
|
str(path),
|
|
observed.st_dev,
|
|
observed.st_ino,
|
|
observed.st_size,
|
|
observed.st_mtime_ns,
|
|
)
|
|
)
|
|
signature = tuple(surface)
|
|
if not signature or signature == previous:
|
|
break
|
|
previous = signature
|
|
recovered += _recover_retirement_staging()
|
|
return recovered
|
|
|
|
def _recover_prepared_finalizations(kanban_db: Any) -> int:
|
|
"""Recover an accepted result staged before an interrupted DB boundary."""
|
|
recovered = 0
|
|
recovered_identities: set[tuple[str, str, int]] = set()
|
|
prepared_paths = list(STATE_ROOT.glob("*/*.terminal.prepared-*.json"))
|
|
|
|
def durable_order(path: Path) -> tuple[int, str]:
|
|
try:
|
|
return path.stat(follow_symlinks=False).st_mtime_ns, str(path)
|
|
except OSError:
|
|
return 2**63 - 1, str(path)
|
|
|
|
# When no DB winner exists yet, the first durably prepared result owns the
|
|
# run. The DB transaction remains the final arbiter across runner processes.
|
|
for path in sorted(prepared_paths, key=durable_order):
|
|
for _replacement_attempt in range(16):
|
|
identity = _terminal_evidence_identity(path)
|
|
snapshot = _open_terminal_recovery_snapshot(path)
|
|
if snapshot is None:
|
|
break
|
|
document = snapshot.document
|
|
if identity is None or identity.state != "prepared" or document is None:
|
|
try:
|
|
_quarantine_terminal(
|
|
path,
|
|
identity,
|
|
snapshot.invalid_reason or "malformed-prepared-evidence",
|
|
snapshot=snapshot,
|
|
)
|
|
finally:
|
|
snapshot.close()
|
|
continue
|
|
expected = _terminal_evidence_path(identity, "prepared", document)
|
|
if (
|
|
expected.name != path.name
|
|
or not _terminal_evidence_valid(document, identity, "prepared")
|
|
):
|
|
try:
|
|
_quarantine_terminal(
|
|
path,
|
|
identity,
|
|
"foreign-prepared-evidence",
|
|
snapshot=snapshot,
|
|
)
|
|
finally:
|
|
snapshot.close()
|
|
continue
|
|
try:
|
|
outcome = _finalize_document_db(kanban_db, identity, document)
|
|
except Exception as error:
|
|
_record_board_access_error(identity.board, error)
|
|
snapshot.close()
|
|
break
|
|
try:
|
|
if outcome == "committed":
|
|
_promote_prepared_evidence(
|
|
identity,
|
|
document,
|
|
path,
|
|
discard_prepared=False,
|
|
)
|
|
_retire_snapshot_after_db(path, snapshot)
|
|
pending = _terminal_path(
|
|
state_path(identity.board, identity.task_id),
|
|
identity.run_id,
|
|
"pending",
|
|
)
|
|
pending_identity = TerminalIdentity(
|
|
identity.board,
|
|
identity.task_id,
|
|
identity.run_id,
|
|
"pending",
|
|
)
|
|
_resolve_pending_after_winner(
|
|
pending,
|
|
pending_identity,
|
|
document,
|
|
)
|
|
recovered_identity = (
|
|
identity.board,
|
|
identity.task_id,
|
|
identity.run_id,
|
|
)
|
|
if recovered_identity not in recovered_identities:
|
|
recovered_identities.add(recovered_identity)
|
|
recovered += 1
|
|
elif outcome == "stale":
|
|
_persist_conflict_evidence(
|
|
identity,
|
|
document,
|
|
"prepared result no longer matches the authoritative task run",
|
|
)
|
|
_retire_snapshot_after_db(path, snapshot)
|
|
else:
|
|
snapshot.close()
|
|
break
|
|
except Exception as error:
|
|
_record_board_access_error(identity.board, error)
|
|
snapshot.close()
|
|
break
|
|
snapshot.close()
|
|
return recovered
|
|
|
|
def recover_pending_finalizations() -> int:
|
|
"""Replay accepted exact-run results before scheduling more provider work."""
|
|
from hermes_cli import kanban_db
|
|
|
|
_drain_retirement_staging()
|
|
recovered = _recover_prepared_finalizations(kanban_db)
|
|
for path in sorted(STATE_ROOT.glob("*/*.terminal.pending.json")):
|
|
identity = _terminal_identity(path)
|
|
last_invalid_reason = "malformed-name" if identity is None else None
|
|
retired_invalid = False
|
|
for _replacement_attempt in range(16):
|
|
snapshot = _open_terminal_recovery_snapshot(path)
|
|
if snapshot is None:
|
|
if (
|
|
identity is not None
|
|
and retired_invalid
|
|
and _terminal_entry_absent(path)
|
|
):
|
|
_recover_exact_run(
|
|
kanban_db,
|
|
identity,
|
|
last_invalid_reason or "malformed-payload",
|
|
)
|
|
break
|
|
record = snapshot.document or {}
|
|
reason = None
|
|
if identity is None:
|
|
reason = "malformed-name"
|
|
elif not _terminal_record_valid(record):
|
|
reason = snapshot.invalid_reason or "malformed-payload"
|
|
elif not _terminal_record_valid(record, identity):
|
|
reason = "foreign-identity"
|
|
if reason is not None:
|
|
try:
|
|
_quarantine_terminal(
|
|
path,
|
|
identity,
|
|
reason,
|
|
snapshot=snapshot,
|
|
)
|
|
finally:
|
|
snapshot.close()
|
|
retired_invalid = True
|
|
last_invalid_reason = reason
|
|
continue
|
|
assert identity is not None
|
|
try:
|
|
outcome = _finalize_terminal_record(
|
|
kanban_db,
|
|
path,
|
|
record,
|
|
snapshot=snapshot,
|
|
)
|
|
except Exception as error:
|
|
_record_board_access_error(identity.board, error)
|
|
break
|
|
if outcome == "committed":
|
|
recovered += 1
|
|
break
|
|
if outcome in {"invalid", "foreign"}:
|
|
# The pathname changed after the recovery snapshot closed.
|
|
# Reclassify and pin the new inode instead of quarantining it
|
|
# using authority derived from the older entry.
|
|
last_invalid_reason = outcome
|
|
continue
|
|
break
|
|
else:
|
|
print(
|
|
f"terminal journal replacement churn deferred for {path.name}",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
# A quarantine or identity-safe retirement performed above can itself
|
|
# encounter replacement churn. Promote every resulting hidden generation
|
|
# back to first-class prepared/conflict/committed evidence before this
|
|
# recovery pass returns; DB replay, if still needed, occurs on the next
|
|
# bounded dispatcher pass.
|
|
_drain_retirement_staging()
|
|
return recovered
|
|
|
|
def _has_pending_finalization(board: str, task_id: str, run_id: Any) -> bool:
|
|
"""Keep an exact run claimed while its accepted result awaits replay."""
|
|
if type(run_id) is not int or canonical_run_id(run_id) is None:
|
|
return False
|
|
path = _terminal_path(state_path(board, task_id), run_id, "pending")
|
|
try:
|
|
path.stat()
|
|
except FileNotFoundError:
|
|
pass
|
|
except OSError:
|
|
return False
|
|
else:
|
|
identity = _terminal_identity(path)
|
|
if identity is not None:
|
|
record = _load_terminal_json(path, identity)
|
|
if _terminal_record_valid(record, identity):
|
|
return True
|
|
base = state_path(board, task_id)
|
|
for prepared in base.parent.glob(
|
|
f"{base.stem}.run-{run_id}.terminal.prepared-*.json"
|
|
):
|
|
identity = _terminal_evidence_identity(prepared)
|
|
if identity is None or identity.state != "prepared":
|
|
continue
|
|
record = _load_small_json(prepared)
|
|
if _terminal_evidence_valid(record, identity, "prepared"):
|
|
return True
|
|
staged_paths = list(base.parent.glob(".retire.*"))
|
|
quarantine = base.parent / "quarantine"
|
|
if quarantine.is_dir() and not quarantine.is_symlink():
|
|
staged_paths.extend(quarantine.glob(".retire.*"))
|
|
for staged in staged_paths:
|
|
snapshot = _open_terminal_recovery_snapshot(staged)
|
|
if snapshot is None:
|
|
continue
|
|
try:
|
|
authority = _staged_terminal_authority(staged, snapshot)
|
|
if (
|
|
authority is not None
|
|
and authority[0].board == board
|
|
and authority[0].task_id == task_id
|
|
and authority[0].run_id == run_id
|
|
and authority[0].state in {"pending", "prepared", "committed"}
|
|
):
|
|
return True
|
|
finally:
|
|
snapshot.close()
|
|
return False
|