353 lines
12 KiB
Python
353 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Recover the known Cassandra Kanban shared-page corruption at pod startup."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sqlite3
|
|
import tempfile
|
|
from contextlib import closing
|
|
from datetime import datetime, timezone
|
|
from itertools import islice
|
|
from pathlib import Path
|
|
from typing import Iterable
|
|
|
|
from kanban_status_recovery import (
|
|
RecoveryRefused,
|
|
known_task_status_alias_corruption,
|
|
reconciled_task_status_rows,
|
|
)
|
|
|
|
|
|
DEFAULT_DATABASE = Path("/opt/data/kanban/boards/cassandra/kanban.db")
|
|
EXPECTED_TABLES = {
|
|
"kanban_notify_subs",
|
|
"task_attachments",
|
|
"task_comments",
|
|
"task_events",
|
|
"task_links",
|
|
"task_runs",
|
|
"tasks",
|
|
}
|
|
COPY_ORDER = (
|
|
"tasks",
|
|
"task_links",
|
|
"task_comments",
|
|
"task_events",
|
|
"task_runs",
|
|
"task_attachments",
|
|
"kanban_notify_subs",
|
|
)
|
|
COMMENTS_INDEX = "idx_comments_task"
|
|
REBUILDABLE_INDEXES = {"idx_events_run", "idx_events_task"}
|
|
|
|
|
|
def _quote(identifier: str) -> str:
|
|
"""Quote one SQLite identifier from the inspected local schema."""
|
|
return '"' + identifier.replace('"', '""') + '"'
|
|
|
|
|
|
def _connect_read_only(database: Path) -> sqlite3.Connection:
|
|
"""Open a database without permitting the audit phase to mutate it."""
|
|
return sqlite3.connect(f"file:{database}?mode=ro", uri=True)
|
|
|
|
|
|
def integrity_errors(connection: sqlite3.Connection) -> list[str]:
|
|
"""Return integrity failures, or an empty list for a healthy database."""
|
|
rows = [str(row[0]) for row in connection.execute("PRAGMA integrity_check")]
|
|
return [] if rows == ["ok"] else rows
|
|
|
|
|
|
def _schema_tables(connection: sqlite3.Connection) -> set[str]:
|
|
"""Return application tables while excluding SQLite's internal tables."""
|
|
return {
|
|
str(row[0])
|
|
for row in connection.execute(
|
|
"SELECT name FROM sqlite_master "
|
|
"WHERE type = 'table' AND name NOT LIKE 'sqlite_%'"
|
|
)
|
|
}
|
|
|
|
|
|
def _known_comments_alias_corruption(
|
|
connection: sqlite3.Connection, errors: Iterable[str]
|
|
) -> bool:
|
|
"""Recognize only the observed task-comments page alias failure."""
|
|
row = connection.execute(
|
|
"SELECT rootpage FROM sqlite_master "
|
|
"WHERE type = 'table' AND name = 'task_comments'"
|
|
).fetchone()
|
|
if row is None:
|
|
return False
|
|
root_page = int(row[0])
|
|
allowed = (
|
|
re.compile(rf"Tree {root_page} page \d+ cell \d+: 2nd reference to page \d+"),
|
|
re.compile(rf"Tree {root_page} page \d+ cell \d+: Rowid \d+ out of order"),
|
|
re.compile(r"(?:NUMERIC|NULL) value in task_comments\.author"),
|
|
re.compile(rf"row \d+ missing from index {COMMENTS_INDEX}"),
|
|
)
|
|
lines = [
|
|
line
|
|
for error in errors
|
|
for line in str(error).splitlines()
|
|
if line and line != "*** in database main ***"
|
|
]
|
|
if not lines or any(
|
|
not any(pattern.fullmatch(line) for pattern in allowed) for line in lines
|
|
):
|
|
return False
|
|
text = "\n".join(lines)
|
|
return all(
|
|
marker in text
|
|
for marker in (
|
|
"2nd reference to page",
|
|
"value in task_comments.author",
|
|
f"missing from index {COMMENTS_INDEX}",
|
|
)
|
|
)
|
|
|
|
|
|
def _known_index_only_corruption(errors: Iterable[str]) -> bool:
|
|
"""Recognize stale entries confined to the reviewed event indexes."""
|
|
lines = [
|
|
line
|
|
for error in errors
|
|
for line in str(error).splitlines()
|
|
if line and line != "*** in database main ***"
|
|
]
|
|
if not lines:
|
|
return False
|
|
pattern = re.compile(r"wrong # of entries in index ([A-Za-z0-9_]+)")
|
|
indexes: set[str] = set()
|
|
for line in lines:
|
|
match = pattern.fullmatch(line)
|
|
if match is None:
|
|
return False
|
|
indexes.add(match.group(1))
|
|
return bool(indexes) and indexes <= REBUILDABLE_INDEXES
|
|
|
|
|
|
def _schema_entries(
|
|
connection: sqlite3.Connection,
|
|
) -> list[tuple[str, str, str, str]]:
|
|
"""Read the explicit schema objects needed to reconstruct the board."""
|
|
return [
|
|
(str(kind), str(name), str(table), str(statement))
|
|
for kind, name, table, statement in connection.execute(
|
|
"SELECT type, name, tbl_name, sql FROM sqlite_master "
|
|
"WHERE sql IS NOT NULL AND type IN ('table', 'index', 'trigger', 'view') "
|
|
"ORDER BY CASE type WHEN 'table' THEN 0 WHEN 'index' THEN 1 "
|
|
"WHEN 'trigger' THEN 2 ELSE 3 END, name"
|
|
)
|
|
if name != "sqlite_sequence"
|
|
]
|
|
|
|
|
|
def _table_columns(connection: sqlite3.Connection, table: str) -> list[str]:
|
|
"""Return columns in their persisted insertion order."""
|
|
return [
|
|
str(row[1]) for row in connection.execute(f"PRAGMA table_info({_quote(table)})")
|
|
]
|
|
|
|
|
|
|
|
def _copy_table(
|
|
source: sqlite3.Connection,
|
|
destination: sqlite3.Connection,
|
|
table: str,
|
|
reconciled_rows: dict[str, list[tuple[object, ...]]] | None = None,
|
|
) -> int:
|
|
"""Copy one table, using the intact comments index as the trusted row set."""
|
|
columns = _table_columns(source, table)
|
|
if not columns:
|
|
raise RecoveryRefused(f"table {table} has no inspectable columns")
|
|
projection = ", ".join(_quote(column) for column in columns)
|
|
placeholders = ", ".join("?" for _ in columns)
|
|
insert = f"INSERT INTO {_quote(table)} ({projection}) VALUES ({placeholders})"
|
|
if reconciled_rows and table in reconciled_rows:
|
|
rows = iter(reconciled_rows[table])
|
|
elif table == "task_comments":
|
|
# The intact index is the authority for which comments existed, but a
|
|
# covering lookup follows the damaged table page and can abort. Walk
|
|
# the table once and retain only rowids present in that intact index.
|
|
trusted_rowids = {
|
|
int(row[0])
|
|
for row in source.execute(
|
|
f"SELECT rowid FROM {_quote(table)} "
|
|
f"INDEXED BY {_quote(COMMENTS_INDEX)}"
|
|
)
|
|
}
|
|
cursor = source.execute(
|
|
f"SELECT rowid, {projection} FROM {_quote(table)} NOT INDEXED "
|
|
"ORDER BY rowid"
|
|
)
|
|
rows = (row[1:] for row in cursor if int(row[0]) in trusted_rowids)
|
|
else:
|
|
cursor = source.execute(
|
|
f"SELECT {projection} FROM {_quote(table)} NOT INDEXED ORDER BY rowid"
|
|
)
|
|
rows = iter(cursor)
|
|
copied = 0
|
|
while batch := list(islice(rows, 500)):
|
|
destination.executemany(insert, batch)
|
|
copied += len(batch)
|
|
if table == "task_comments" and copied != len(trusted_rowids):
|
|
raise RecoveryRefused("trusted task-comments rows could not all be recovered")
|
|
return copied
|
|
|
|
|
|
def _populate_replacement(
|
|
source: sqlite3.Connection,
|
|
replacement: Path,
|
|
reconciled_rows: dict[str, list[tuple[object, ...]]] | None = None,
|
|
) -> dict[str, int]:
|
|
"""Build and verify a clean database using only reviewed source rows."""
|
|
entries = _schema_entries(source)
|
|
tables = [entry for entry in entries if entry[0] == "table"]
|
|
later = [entry for entry in entries if entry[0] != "table"]
|
|
counts: dict[str, int] = {}
|
|
with closing(sqlite3.connect(replacement)) as destination:
|
|
destination.execute("PRAGMA foreign_keys = OFF")
|
|
destination.execute("PRAGMA journal_mode = DELETE")
|
|
destination.execute("BEGIN IMMEDIATE")
|
|
for _, _, _, statement in tables:
|
|
destination.execute(statement)
|
|
for table in COPY_ORDER:
|
|
counts[table] = _copy_table(
|
|
source, destination, table, reconciled_rows
|
|
)
|
|
if source.execute(
|
|
"SELECT 1 FROM sqlite_master WHERE name = 'sqlite_sequence'"
|
|
).fetchone():
|
|
destination.execute("DELETE FROM sqlite_sequence")
|
|
destination.executemany(
|
|
"INSERT INTO sqlite_sequence(name, seq) VALUES (?, ?)",
|
|
source.execute("SELECT name, seq FROM sqlite_sequence").fetchall(),
|
|
)
|
|
for _, _, _, statement in later:
|
|
destination.execute(statement)
|
|
user_version = int(source.execute("PRAGMA user_version").fetchone()[0])
|
|
destination.execute(f"PRAGMA user_version = {user_version}")
|
|
destination.commit()
|
|
failures = integrity_errors(destination)
|
|
foreign_key_failures = destination.execute(
|
|
"PRAGMA foreign_key_check"
|
|
).fetchall()
|
|
if failures or foreign_key_failures:
|
|
raise RecoveryRefused(
|
|
"replacement verification failed: "
|
|
f"integrity={failures!r} foreign_keys={foreign_key_failures!r}"
|
|
)
|
|
return counts
|
|
|
|
|
|
def _backup_path(database: Path) -> Path:
|
|
"""Return a unique evidence filename containing the damaged file hash."""
|
|
digest = hashlib.sha256(database.read_bytes()).hexdigest()[:16]
|
|
timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
return database.with_name(
|
|
f"{database.name}.corrupt.recovery-{timestamp}-{digest}.bak"
|
|
)
|
|
|
|
|
|
def _durable_copy(source: Path, destination: Path) -> None:
|
|
"""Copy a recovery artifact and flush it before replacing live data."""
|
|
with source.open("rb") as reader, destination.open("xb") as writer:
|
|
shutil.copyfileobj(reader, writer)
|
|
writer.flush()
|
|
os.fsync(writer.fileno())
|
|
shutil.copystat(source, destination)
|
|
|
|
|
|
def recover_database(database: Path, errors: list[str]) -> dict[str, object]:
|
|
"""Atomically replace the known corrupt board while retaining all evidence."""
|
|
database = database.resolve()
|
|
with closing(_connect_read_only(database)) as source:
|
|
if _schema_tables(source) != EXPECTED_TABLES:
|
|
raise RecoveryRefused("Kanban schema differs from the reviewed table set")
|
|
index = source.execute(
|
|
"SELECT tbl_name FROM sqlite_master WHERE type = 'index' AND name = ?",
|
|
(COMMENTS_INDEX,),
|
|
).fetchone()
|
|
if index != ("task_comments",):
|
|
raise RecoveryRefused("trusted task-comments index is unavailable")
|
|
reconciled_rows = None
|
|
known_corruption = (
|
|
_known_comments_alias_corruption(source, errors)
|
|
or _known_index_only_corruption(errors)
|
|
)
|
|
if known_task_status_alias_corruption(source, errors):
|
|
reconciled_rows = reconciled_task_status_rows(source)
|
|
known_corruption = True
|
|
if not known_corruption:
|
|
raise RecoveryRefused(
|
|
"integrity failure does not match the reviewed corruption"
|
|
)
|
|
descriptor, temporary_name = tempfile.mkstemp(
|
|
prefix=f".{database.name}.recovery-", dir=database.parent
|
|
)
|
|
os.close(descriptor)
|
|
replacement = Path(temporary_name)
|
|
replacement.unlink()
|
|
try:
|
|
counts = _populate_replacement(
|
|
source, replacement, reconciled_rows
|
|
)
|
|
except Exception:
|
|
replacement.unlink(missing_ok=True)
|
|
raise
|
|
|
|
backup = _backup_path(database)
|
|
_durable_copy(database, backup)
|
|
for suffix in ("-wal", "-shm", "-journal"):
|
|
journal = database.with_name(database.name + suffix)
|
|
if journal.exists():
|
|
os.replace(journal, backup.with_name(backup.name + suffix))
|
|
os.chmod(replacement, database.stat().st_mode & 0o777)
|
|
os.replace(replacement, database)
|
|
directory_fd = os.open(database.parent, os.O_RDONLY)
|
|
try:
|
|
os.fsync(directory_fd)
|
|
finally:
|
|
os.close(directory_fd)
|
|
return {
|
|
"state": "recovered",
|
|
"database": str(database),
|
|
"backup": str(backup),
|
|
"rows": counts,
|
|
}
|
|
|
|
|
|
def repair_if_needed(database: Path = DEFAULT_DATABASE) -> dict[str, object]:
|
|
"""Check one board and recover only its known, evidence-backed failure."""
|
|
if not database.is_file():
|
|
return {"state": "absent", "database": str(database)}
|
|
with closing(_connect_read_only(database)) as connection:
|
|
errors = integrity_errors(connection)
|
|
if not errors:
|
|
return {"state": "healthy", "database": str(database)}
|
|
return recover_database(database, errors)
|
|
|
|
|
|
def main() -> int:
|
|
"""Run the startup repair and emit a non-secret machine-readable result."""
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--database", type=Path, default=DEFAULT_DATABASE)
|
|
args = parser.parse_args()
|
|
try:
|
|
result = repair_if_needed(args.database)
|
|
except (OSError, sqlite3.DatabaseError, RecoveryRefused) as error:
|
|
print(f"Kanban recovery refused: {error}", flush=True)
|
|
return 1
|
|
print(json.dumps(result, sort_keys=True), flush=True)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|