#!/usr/bin/env python3 """Bounded replay of pending, prepared, and staged terminal journals.""" from __future__ import annotations import hashlib import os import sys from pathlib import Path from typing import Any from cli_lane_board import _record_board_access_error from cli_lane_config import ( STATE_ROOT, TerminalIdentity, TerminalRecoverySnapshot, canonical_run_id, ) from cli_lane_evidence import ( _persist_conflict_evidence, _persist_prepared_evidence, _same_terminal_document, _terminal_evidence_path, _terminal_evidence_valid, _write_json_noreplace, ) from cli_lane_files import ( _LEGACY_RETIRE_NAME, _RETIRE_NAME, _SAFE_BOARD, _terminal_evidence_identity, _terminal_identity, _terminal_path, state_path, ) from cli_lane_finalization import ( _finalize_document_db, _finalize_terminal_record, _promote_prepared_evidence, _recover_exact_run, _resolve_pending_after_winner, _retire_snapshot, _retire_snapshot_after_db, ) from cli_lane_quarantine import _quarantine_terminal from cli_lane_records import ( _load_small_json, _load_terminal_json, _open_terminal_recovery_snapshot, _terminal_record_valid, ) def _terminal_entry_absent(path: Path) -> bool: """Confirm absence through a nofollow directory descriptor.""" directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) directory_flags |= getattr(os, "O_NOFOLLOW", 0) directory = None try: directory = os.open(path.parent, directory_flags) os.stat(path.name, dir_fd=directory, follow_symlinks=False) return False except FileNotFoundError: return True except OSError: return False finally: if directory is not None: os.close(directory) def _staged_terminal_authority( path: Path, snapshot: TerminalRecoverySnapshot, ) -> tuple[TerminalIdentity, Path] | None: """Bind a staged inode to the canonical pending name it was moved from.""" try: relative = path.relative_to(STATE_ROOT) except ValueError: return None if len(relative.parts) == 2: board, filename = relative.parts elif len(relative.parts) == 3 and relative.parts[1] == "quarantine": board, _quarantine, filename = relative.parts else: return None match = _RETIRE_NAME.fullmatch(filename) legacy = _LEGACY_RETIRE_NAME.fullmatch(filename) document = snapshot.document if ( (match is None and legacy is None) or document is None or not _SAFE_BOARD.fullmatch(board) or board in {".", ".."} or document.get("board") != board or not isinstance(document.get("task_id"), str) or type(document.get("expected_run_id")) is not int or canonical_run_id(document.get("expected_run_id")) is None ): return None state = document.get("kanban_state") if state not in {"pending", "prepared", "committed", "conflict"}: return None identity = TerminalIdentity( board, document["task_id"], document["expected_run_id"], state ) if state in {"pending", "committed"}: canonical = _terminal_path( state_path(identity.board, identity.task_id), identity.run_id, state, ) valid = _terminal_record_valid(document, identity) else: canonical = _terminal_evidence_path(identity, state, document) valid = _terminal_evidence_valid(document, identity, state) board_dir = STATE_ROOT / board if ( canonical.parent != board_dir or path.parent not in {board_dir, board_dir / "quarantine"} or ( match is not None and hashlib.sha256(canonical.name.encode("utf-8")).hexdigest()[:16] != match.group("path_digest") ) or not valid ): return None return identity, canonical def _retirement_staging_paths() -> list[Path]: """List every directory where identity-safe retirement can stage an inode.""" paths = list(STATE_ROOT.glob("*/.retire.*")) paths.extend(STATE_ROOT.glob("*/quarantine/.retire.*")) return sorted(set(paths)) def _restore_staged_terminal_authority( identity: TerminalIdentity, canonical: Path, document: dict[str, Any], ) -> None: """Restore one hidden accepted artifact without executing it prematurely.""" if identity.state == "pending": _persist_prepared_evidence(identity, document) return if _write_json_noreplace(canonical, document): return existing = _load_small_json(canonical) if identity.state in {"prepared", "conflict"}: valid = _terminal_evidence_valid(existing, identity, identity.state) else: valid = _terminal_record_valid(existing, identity) if valid and _same_terminal_document(existing, document): return if identity.state == "committed": _persist_conflict_evidence( identity, document, "staged committed evidence conflicts with the canonical first writer", ) return raise OSError("staged terminal evidence path contains a conflicting result") def _recover_retirement_staging() -> int: """Promote valid hidden journals to durable prepared evidence.""" recovered = 0 for path in _retirement_staging_paths(): snapshot = _open_terminal_recovery_snapshot(path) if snapshot is None: continue authority = _staged_terminal_authority(path, snapshot) if authority is None: try: _quarantine_terminal( path, None, "untrusted-retirement-staging", snapshot=snapshot, ) finally: snapshot.close() continue identity, canonical = authority try: _restore_staged_terminal_authority( identity, canonical, snapshot.document or {}, ) retirement = _retire_snapshot( path, snapshot, authority_name=canonical.name, ) except Exception as error: _record_board_access_error(identity.board, error) continue finally: snapshot.close() if retirement in {"retired", "missing"}: recovered += 1 return recovered def _drain_retirement_staging() -> int: """Converge bounded replacement churn without spinning on storage failure.""" recovered = 0 previous: tuple[tuple[str, int, int, int, int], ...] | None = None for _attempt in range(16): surface = [] for path in _retirement_staging_paths(): try: observed = path.stat(follow_symlinks=False) except OSError: continue surface.append( ( str(path), observed.st_dev, observed.st_ino, observed.st_size, observed.st_mtime_ns, ) ) signature = tuple(surface) if not signature or signature == previous: break previous = signature recovered += _recover_retirement_staging() return recovered def _recover_prepared_finalizations(kanban_db: Any) -> int: """Recover an accepted result staged before an interrupted DB boundary.""" recovered = 0 recovered_identities: set[tuple[str, str, int]] = set() prepared_paths = list(STATE_ROOT.glob("*/*.terminal.prepared-*.json")) def durable_order(path: Path) -> tuple[int, str]: try: return path.stat(follow_symlinks=False).st_mtime_ns, str(path) except OSError: return 2**63 - 1, str(path) # When no DB winner exists yet, the first durably prepared result owns the # run. The DB transaction remains the final arbiter across runner processes. for path in sorted(prepared_paths, key=durable_order): for _replacement_attempt in range(16): identity = _terminal_evidence_identity(path) snapshot = _open_terminal_recovery_snapshot(path) if snapshot is None: break document = snapshot.document if identity is None or identity.state != "prepared" or document is None: try: _quarantine_terminal( path, identity, snapshot.invalid_reason or "malformed-prepared-evidence", snapshot=snapshot, ) finally: snapshot.close() continue expected = _terminal_evidence_path(identity, "prepared", document) if ( expected.name != path.name or not _terminal_evidence_valid(document, identity, "prepared") ): try: _quarantine_terminal( path, identity, "foreign-prepared-evidence", snapshot=snapshot, ) finally: snapshot.close() continue try: outcome = _finalize_document_db(kanban_db, identity, document) except Exception as error: _record_board_access_error(identity.board, error) snapshot.close() break try: if outcome == "committed": _promote_prepared_evidence( identity, document, path, discard_prepared=False, ) _retire_snapshot_after_db(path, snapshot) pending = _terminal_path( state_path(identity.board, identity.task_id), identity.run_id, "pending", ) pending_identity = TerminalIdentity( identity.board, identity.task_id, identity.run_id, "pending", ) _resolve_pending_after_winner( pending, pending_identity, document, ) recovered_identity = ( identity.board, identity.task_id, identity.run_id, ) if recovered_identity not in recovered_identities: recovered_identities.add(recovered_identity) recovered += 1 elif outcome == "stale": _persist_conflict_evidence( identity, document, "prepared result no longer matches the authoritative task run", ) _retire_snapshot_after_db(path, snapshot) else: snapshot.close() break except Exception as error: _record_board_access_error(identity.board, error) snapshot.close() break snapshot.close() return recovered def recover_pending_finalizations() -> int: """Replay accepted exact-run results before scheduling more provider work.""" from hermes_cli import kanban_db _drain_retirement_staging() recovered = _recover_prepared_finalizations(kanban_db) for path in sorted(STATE_ROOT.glob("*/*.terminal.pending.json")): identity = _terminal_identity(path) last_invalid_reason = "malformed-name" if identity is None else None retired_invalid = False for _replacement_attempt in range(16): snapshot = _open_terminal_recovery_snapshot(path) if snapshot is None: if ( identity is not None and retired_invalid and _terminal_entry_absent(path) ): _recover_exact_run( kanban_db, identity, last_invalid_reason or "malformed-payload", ) break record = snapshot.document or {} reason = None if identity is None: reason = "malformed-name" elif not _terminal_record_valid(record): reason = snapshot.invalid_reason or "malformed-payload" elif not _terminal_record_valid(record, identity): reason = "foreign-identity" if reason is not None: try: _quarantine_terminal( path, identity, reason, snapshot=snapshot, ) finally: snapshot.close() retired_invalid = True last_invalid_reason = reason continue assert identity is not None try: outcome = _finalize_terminal_record( kanban_db, path, record, snapshot=snapshot, ) except Exception as error: _record_board_access_error(identity.board, error) break if outcome == "committed": recovered += 1 break if outcome in {"invalid", "foreign"}: # The pathname changed after the recovery snapshot closed. # Reclassify and pin the new inode instead of quarantining it # using authority derived from the older entry. last_invalid_reason = outcome continue break else: print( f"terminal journal replacement churn deferred for {path.name}", file=sys.stderr, flush=True, ) # A quarantine or identity-safe retirement performed above can itself # encounter replacement churn. Promote every resulting hidden generation # back to first-class prepared/conflict/committed evidence before this # recovery pass returns; DB replay, if still needed, occurs on the next # bounded dispatcher pass. _drain_retirement_staging() return recovered def _has_pending_finalization(board: str, task_id: str, run_id: Any) -> bool: """Keep an exact run claimed while its accepted result awaits replay.""" if type(run_id) is not int or canonical_run_id(run_id) is None: return False path = _terminal_path(state_path(board, task_id), run_id, "pending") try: path.stat() except FileNotFoundError: pass except OSError: return False else: identity = _terminal_identity(path) if identity is not None: record = _load_terminal_json(path, identity) if _terminal_record_valid(record, identity): return True base = state_path(board, task_id) for prepared in base.parent.glob( f"{base.stem}.run-{run_id}.terminal.prepared-*.json" ): identity = _terminal_evidence_identity(prepared) if identity is None or identity.state != "prepared": continue record = _load_small_json(prepared) if _terminal_evidence_valid(record, identity, "prepared"): return True staged_paths = list(base.parent.glob(".retire.*")) quarantine = base.parent / "quarantine" if quarantine.is_dir() and not quarantine.is_symlink(): staged_paths.extend(quarantine.glob(".retire.*")) for staged in staged_paths: snapshot = _open_terminal_recovery_snapshot(staged) if snapshot is None: continue try: authority = _staged_terminal_authority(staged, snapshot) if ( authority is not None and authority[0].board == board and authority[0].task_id == task_id and authority[0].run_id == run_id and authority[0].state in {"pending", "prepared", "committed"} ): return True finally: snapshot.close() return False