#!/usr/bin/env python3 """Recover the known Cassandra Kanban shared-page corruption at pod startup.""" from __future__ import annotations import argparse import hashlib import json import os import re import shutil import sqlite3 import tempfile from contextlib import closing from datetime import datetime, timezone from itertools import islice from pathlib import Path from typing import Iterable from kanban_status_recovery import ( RecoveryRefused, known_task_status_alias_corruption, reconciled_task_status_rows, ) DEFAULT_DATABASE = Path("/opt/data/kanban/boards/cassandra/kanban.db") EXPECTED_TABLES = { "kanban_notify_subs", "task_attachments", "task_comments", "task_events", "task_links", "task_runs", "tasks", } COPY_ORDER = ( "tasks", "task_links", "task_comments", "task_events", "task_runs", "task_attachments", "kanban_notify_subs", ) COMMENTS_INDEX = "idx_comments_task" REBUILDABLE_INDEXES = {"idx_events_run", "idx_events_task"} def _quote(identifier: str) -> str: """Quote one SQLite identifier from the inspected local schema.""" return '"' + identifier.replace('"', '""') + '"' def _connect_read_only(database: Path) -> sqlite3.Connection: """Open a database without permitting the audit phase to mutate it.""" return sqlite3.connect(f"file:{database}?mode=ro", uri=True) def integrity_errors(connection: sqlite3.Connection) -> list[str]: """Return integrity failures, or an empty list for a healthy database.""" rows = [str(row[0]) for row in connection.execute("PRAGMA integrity_check")] return [] if rows == ["ok"] else rows def _schema_tables(connection: sqlite3.Connection) -> set[str]: """Return application tables while excluding SQLite's internal tables.""" return { str(row[0]) for row in connection.execute( "SELECT name FROM sqlite_master " "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" ) } def _known_comments_alias_corruption( connection: sqlite3.Connection, errors: Iterable[str] ) -> bool: """Recognize only the observed task-comments page alias failure.""" row = connection.execute( "SELECT rootpage FROM sqlite_master " "WHERE type = 'table' AND name = 'task_comments'" ).fetchone() if row is None: return False root_page = int(row[0]) allowed = ( re.compile(rf"Tree {root_page} page \d+ cell \d+: 2nd reference to page \d+"), re.compile(rf"Tree {root_page} page \d+ cell \d+: Rowid \d+ out of order"), re.compile(r"(?:NUMERIC|NULL) value in task_comments\.author"), re.compile(rf"row \d+ missing from index {COMMENTS_INDEX}"), ) lines = [ line for error in errors for line in str(error).splitlines() if line and line != "*** in database main ***" ] if not lines or any( not any(pattern.fullmatch(line) for pattern in allowed) for line in lines ): return False text = "\n".join(lines) return all( marker in text for marker in ( "2nd reference to page", "value in task_comments.author", f"missing from index {COMMENTS_INDEX}", ) ) def _known_index_only_corruption(errors: Iterable[str]) -> bool: """Recognize stale entries confined to the reviewed event indexes.""" lines = [ line for error in errors for line in str(error).splitlines() if line and line != "*** in database main ***" ] if not lines: return False pattern = re.compile(r"wrong # of entries in index ([A-Za-z0-9_]+)") indexes: set[str] = set() for line in lines: match = pattern.fullmatch(line) if match is None: return False indexes.add(match.group(1)) return bool(indexes) and indexes <= REBUILDABLE_INDEXES def _schema_entries( connection: sqlite3.Connection, ) -> list[tuple[str, str, str, str]]: """Read the explicit schema objects needed to reconstruct the board.""" return [ (str(kind), str(name), str(table), str(statement)) for kind, name, table, statement in connection.execute( "SELECT type, name, tbl_name, sql FROM sqlite_master " "WHERE sql IS NOT NULL AND type IN ('table', 'index', 'trigger', 'view') " "ORDER BY CASE type WHEN 'table' THEN 0 WHEN 'index' THEN 1 " "WHEN 'trigger' THEN 2 ELSE 3 END, name" ) if name != "sqlite_sequence" ] def _table_columns(connection: sqlite3.Connection, table: str) -> list[str]: """Return columns in their persisted insertion order.""" return [ str(row[1]) for row in connection.execute(f"PRAGMA table_info({_quote(table)})") ] def _copy_table( source: sqlite3.Connection, destination: sqlite3.Connection, table: str, reconciled_rows: dict[str, list[tuple[object, ...]]] | None = None, ) -> int: """Copy one table, using the intact comments index as the trusted row set.""" columns = _table_columns(source, table) if not columns: raise RecoveryRefused(f"table {table} has no inspectable columns") projection = ", ".join(_quote(column) for column in columns) placeholders = ", ".join("?" for _ in columns) insert = f"INSERT INTO {_quote(table)} ({projection}) VALUES ({placeholders})" if reconciled_rows and table in reconciled_rows: rows = iter(reconciled_rows[table]) elif table == "task_comments": # The intact index is the authority for which comments existed, but a # covering lookup follows the damaged table page and can abort. Walk # the table once and retain only rowids present in that intact index. trusted_rowids = { int(row[0]) for row in source.execute( f"SELECT rowid FROM {_quote(table)} " f"INDEXED BY {_quote(COMMENTS_INDEX)}" ) } cursor = source.execute( f"SELECT rowid, {projection} FROM {_quote(table)} NOT INDEXED " "ORDER BY rowid" ) rows = (row[1:] for row in cursor if int(row[0]) in trusted_rowids) else: cursor = source.execute( f"SELECT {projection} FROM {_quote(table)} NOT INDEXED ORDER BY rowid" ) rows = iter(cursor) copied = 0 while batch := list(islice(rows, 500)): destination.executemany(insert, batch) copied += len(batch) if table == "task_comments" and copied != len(trusted_rowids): raise RecoveryRefused("trusted task-comments rows could not all be recovered") return copied def _populate_replacement( source: sqlite3.Connection, replacement: Path, reconciled_rows: dict[str, list[tuple[object, ...]]] | None = None, ) -> dict[str, int]: """Build and verify a clean database using only reviewed source rows.""" entries = _schema_entries(source) tables = [entry for entry in entries if entry[0] == "table"] later = [entry for entry in entries if entry[0] != "table"] counts: dict[str, int] = {} with closing(sqlite3.connect(replacement)) as destination: destination.execute("PRAGMA foreign_keys = OFF") destination.execute("PRAGMA journal_mode = DELETE") destination.execute("BEGIN IMMEDIATE") for _, _, _, statement in tables: destination.execute(statement) for table in COPY_ORDER: counts[table] = _copy_table( source, destination, table, reconciled_rows ) if source.execute( "SELECT 1 FROM sqlite_master WHERE name = 'sqlite_sequence'" ).fetchone(): destination.execute("DELETE FROM sqlite_sequence") destination.executemany( "INSERT INTO sqlite_sequence(name, seq) VALUES (?, ?)", source.execute("SELECT name, seq FROM sqlite_sequence").fetchall(), ) for _, _, _, statement in later: destination.execute(statement) user_version = int(source.execute("PRAGMA user_version").fetchone()[0]) destination.execute(f"PRAGMA user_version = {user_version}") destination.commit() failures = integrity_errors(destination) foreign_key_failures = destination.execute( "PRAGMA foreign_key_check" ).fetchall() if failures or foreign_key_failures: raise RecoveryRefused( "replacement verification failed: " f"integrity={failures!r} foreign_keys={foreign_key_failures!r}" ) return counts def _backup_path(database: Path) -> Path: """Return a unique evidence filename containing the damaged file hash.""" digest = hashlib.sha256(database.read_bytes()).hexdigest()[:16] timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") return database.with_name( f"{database.name}.corrupt.recovery-{timestamp}-{digest}.bak" ) def _durable_copy(source: Path, destination: Path) -> None: """Copy a recovery artifact and flush it before replacing live data.""" with source.open("rb") as reader, destination.open("xb") as writer: shutil.copyfileobj(reader, writer) writer.flush() os.fsync(writer.fileno()) shutil.copystat(source, destination) def recover_database(database: Path, errors: list[str]) -> dict[str, object]: """Atomically replace the known corrupt board while retaining all evidence.""" database = database.resolve() with closing(_connect_read_only(database)) as source: if _schema_tables(source) != EXPECTED_TABLES: raise RecoveryRefused("Kanban schema differs from the reviewed table set") index = source.execute( "SELECT tbl_name FROM sqlite_master WHERE type = 'index' AND name = ?", (COMMENTS_INDEX,), ).fetchone() if index != ("task_comments",): raise RecoveryRefused("trusted task-comments index is unavailable") reconciled_rows = None known_corruption = ( _known_comments_alias_corruption(source, errors) or _known_index_only_corruption(errors) ) if known_task_status_alias_corruption(source, errors): reconciled_rows = reconciled_task_status_rows(source) known_corruption = True if not known_corruption: raise RecoveryRefused( "integrity failure does not match the reviewed corruption" ) descriptor, temporary_name = tempfile.mkstemp( prefix=f".{database.name}.recovery-", dir=database.parent ) os.close(descriptor) replacement = Path(temporary_name) replacement.unlink() try: counts = _populate_replacement( source, replacement, reconciled_rows ) except Exception: replacement.unlink(missing_ok=True) raise backup = _backup_path(database) _durable_copy(database, backup) for suffix in ("-wal", "-shm", "-journal"): journal = database.with_name(database.name + suffix) if journal.exists(): os.replace(journal, backup.with_name(backup.name + suffix)) os.chmod(replacement, database.stat().st_mode & 0o777) os.replace(replacement, database) directory_fd = os.open(database.parent, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) return { "state": "recovered", "database": str(database), "backup": str(backup), "rows": counts, } def repair_if_needed(database: Path = DEFAULT_DATABASE) -> dict[str, object]: """Check one board and recover only its known, evidence-backed failure.""" if not database.is_file(): return {"state": "absent", "database": str(database)} with closing(_connect_read_only(database)) as connection: errors = integrity_errors(connection) if not errors: return {"state": "healthy", "database": str(database)} return recover_database(database, errors) def main() -> int: """Run the startup repair and emit a non-secret machine-readable result.""" parser = argparse.ArgumentParser() parser.add_argument("--database", type=Path, default=DEFAULT_DATABASE) args = parser.parse_args() try: result = repair_if_needed(args.database) except (OSError, sqlite3.DatabaseError, RecoveryRefused) as error: print(f"Kanban recovery refused: {error}", flush=True) return 1 print(json.dumps(result, sort_keys=True), flush=True) return 0 if __name__ == "__main__": raise SystemExit(main())