287 lines
11 KiB
Python
287 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Evidence-gated reconstruction for stale Kanban task and run status pages."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
from typing import Iterable
|
|
|
|
|
|
TASKS_PK_INDEX = "sqlite_autoindex_tasks_1"
|
|
TASK_STATUS_INDEXES = ("idx_tasks_status", "idx_tasks_assignee_status")
|
|
RUN_STATUS_INDEX = "idx_runs_status"
|
|
|
|
|
|
class RecoveryRefused(RuntimeError):
|
|
"""The database failure is not the reviewed, loss-bounded corruption."""
|
|
|
|
|
|
def _quote(identifier: str) -> str:
|
|
"""Quote one SQLite identifier from the inspected local schema."""
|
|
return '"' + identifier.replace('"', '""') + '"'
|
|
|
|
|
|
def _table_columns(connection: sqlite3.Connection, table: str) -> list[str]:
|
|
"""Return columns in their persisted insertion order."""
|
|
return [
|
|
str(row[1]) for row in connection.execute(f"PRAGMA table_info({_quote(table)})")
|
|
]
|
|
|
|
|
|
def known_task_status_alias_corruption(
|
|
connection: sqlite3.Connection, errors: Iterable[str]
|
|
) -> bool:
|
|
"""Recognize the reviewed tasks-page alias with stale status rows."""
|
|
row = connection.execute(
|
|
"SELECT rootpage FROM sqlite_master "
|
|
"WHERE type = 'table' AND name = 'tasks'"
|
|
).fetchone()
|
|
if row is None:
|
|
return False
|
|
root_page = int(row[0])
|
|
task_indexes = {
|
|
str(index[0])
|
|
for index in connection.execute(
|
|
"SELECT name FROM sqlite_master "
|
|
"WHERE type = 'index' AND tbl_name = 'tasks'"
|
|
)
|
|
}
|
|
allowed_indexes = task_indexes | {RUN_STATUS_INDEX}
|
|
tree_alias = re.compile(
|
|
rf"Tree {root_page} page \d+ cell \d+: 2nd reference to page \d+"
|
|
)
|
|
rowid_order = re.compile(
|
|
rf"Tree {root_page} page \d+ cell \d+: Rowid \d+ out of order"
|
|
)
|
|
wrong_entries = re.compile(r"wrong # of entries in index ([A-Za-z0-9_]+)")
|
|
missing_entry = re.compile(r"row \d+ missing from index ([A-Za-z0-9_]+)")
|
|
lines = [
|
|
line
|
|
for error in errors
|
|
for line in str(error).splitlines()
|
|
if line and line != "*** in database main ***"
|
|
]
|
|
if not lines:
|
|
return False
|
|
saw_alias = False
|
|
saw_order = False
|
|
saw_task_index = False
|
|
saw_task_status = False
|
|
saw_run_status = False
|
|
for line in lines:
|
|
if tree_alias.fullmatch(line):
|
|
saw_alias = True
|
|
continue
|
|
if rowid_order.fullmatch(line):
|
|
saw_order = True
|
|
continue
|
|
match = wrong_entries.fullmatch(line) or missing_entry.fullmatch(line)
|
|
if match is None or match.group(1) not in allowed_indexes:
|
|
return False
|
|
index = match.group(1)
|
|
saw_task_index = saw_task_index or index in task_indexes
|
|
saw_task_status = saw_task_status or index in TASK_STATUS_INDEXES
|
|
saw_run_status = saw_run_status or index == RUN_STATUS_INDEX
|
|
return all(
|
|
(saw_alias, saw_order, saw_task_index, saw_task_status, saw_run_status)
|
|
)
|
|
|
|
|
|
def _indexed_values(
|
|
connection: sqlite3.Connection,
|
|
table: str,
|
|
index: str,
|
|
columns: tuple[str, ...],
|
|
) -> dict[int, tuple[object, ...]]:
|
|
"""Read one value per rowid through a named, independently checked index."""
|
|
projection = ", ".join(_quote(column) for column in columns)
|
|
values: dict[int, tuple[object, ...]] = {}
|
|
for row in connection.execute(
|
|
f"SELECT rowid, {projection} FROM {_quote(table)} "
|
|
f"INDEXED BY {_quote(index)} ORDER BY rowid"
|
|
):
|
|
rowid = int(row[0])
|
|
if rowid in values:
|
|
raise RecoveryRefused(f"duplicate rowid in trusted index {index}")
|
|
values[rowid] = tuple(row[1:])
|
|
return values
|
|
|
|
|
|
def _completed_event(
|
|
connection: sqlite3.Connection, task_id: str, run_id: int
|
|
) -> tuple[int, str]:
|
|
"""Return the matching completion timestamp and retained summary evidence."""
|
|
row = connection.execute(
|
|
"SELECT created_at, payload FROM task_events "
|
|
"WHERE task_id = ? AND run_id = ? AND kind = 'completed' "
|
|
"ORDER BY id DESC LIMIT 1",
|
|
(task_id, run_id),
|
|
).fetchone()
|
|
if row is None:
|
|
raise RecoveryRefused(
|
|
f"status disagreement for {task_id} lacks a completed event"
|
|
)
|
|
try:
|
|
payload = json.loads(str(row[1]))
|
|
except (TypeError, ValueError) as error:
|
|
raise RecoveryRefused(
|
|
f"completed event for {task_id} has invalid payload"
|
|
) from error
|
|
summary = payload.get("summary")
|
|
if not isinstance(summary, str) or not summary.strip():
|
|
raise RecoveryRefused(
|
|
f"completed event for {task_id} lacks retained summary evidence"
|
|
)
|
|
return int(row[0]), summary
|
|
|
|
|
|
def _task_rows(
|
|
connection: sqlite3.Connection, task_columns: list[str]
|
|
) -> dict[int, list[object]]:
|
|
"""Read the unique task row set through its primary-key index."""
|
|
projection = ", ".join(_quote(column) for column in task_columns)
|
|
rows: dict[int, list[object]] = {}
|
|
task_ids: set[str] = set()
|
|
for row in connection.execute(
|
|
f"SELECT rowid, {projection} FROM tasks "
|
|
f"INDEXED BY {_quote(TASKS_PK_INDEX)} ORDER BY id"
|
|
):
|
|
rowid = int(row[0])
|
|
values = list(row[1:])
|
|
task_id = str(values[task_columns.index("id")])
|
|
if rowid in rows or task_id in task_ids:
|
|
raise RecoveryRefused("trusted tasks primary index is not unique")
|
|
rows[rowid] = values
|
|
task_ids.add(task_id)
|
|
return rows
|
|
|
|
|
|
def _reconcile_tasks(
|
|
connection: sqlite3.Connection,
|
|
task_columns: list[str],
|
|
) -> tuple[dict[int, list[object]], dict[int, tuple[int, str]]]:
|
|
"""Reconcile stale task values when two status indexes and an event agree."""
|
|
task_rows = _task_rows(connection, task_columns)
|
|
status_maps = [
|
|
_indexed_values(connection, "tasks", index, ("status",))
|
|
for index in TASK_STATUS_INDEXES
|
|
]
|
|
if status_maps[0] != status_maps[1]:
|
|
raise RecoveryRefused("independent task status indexes disagree")
|
|
if set(task_rows) != set(status_maps[0]):
|
|
raise RecoveryRefused("task primary and status indexes disagree on row set")
|
|
status_column = task_columns.index("status")
|
|
task_id_column = task_columns.index("id")
|
|
run_column = task_columns.index("current_run_id")
|
|
completed_evidence: dict[int, tuple[int, str]] = {}
|
|
for rowid, values in task_rows.items():
|
|
indexed_status = str(status_maps[0][rowid][0])
|
|
if values[status_column] == indexed_status:
|
|
continue
|
|
if indexed_status != "done" or values[run_column] is None:
|
|
raise RecoveryRefused("task status disagreement is not a completed run")
|
|
task_id = str(values[task_id_column])
|
|
run_id = int(values[run_column])
|
|
completed_at, summary = _completed_event(connection, task_id, run_id)
|
|
completed_evidence[run_id] = (completed_at, summary)
|
|
replacements: dict[str, object] = {
|
|
"status": "done",
|
|
"result": summary,
|
|
"completed_at": completed_at,
|
|
"claim_lock": None,
|
|
"claim_expires": None,
|
|
"worker_pid": None,
|
|
"current_run_id": None,
|
|
"block_kind": None,
|
|
"block_recurrences": 0,
|
|
}
|
|
for column, replacement in replacements.items():
|
|
values[task_columns.index(column)] = replacement
|
|
if not completed_evidence:
|
|
raise RecoveryRefused("no evidence-backed task status disagreement was found")
|
|
return task_rows, completed_evidence
|
|
|
|
|
|
def _reconcile_runs(
|
|
connection: sqlite3.Connection,
|
|
run_columns: list[str],
|
|
completed_evidence: dict[int, tuple[int, str]],
|
|
) -> dict[int, list[object]]:
|
|
"""Reconcile stale run values against its index and completed event."""
|
|
projection = ", ".join(_quote(column) for column in run_columns)
|
|
run_rows = {
|
|
int(row[0]): list(row[1:])
|
|
for row in connection.execute(
|
|
f"SELECT rowid, {projection} FROM task_runs NOT INDEXED ORDER BY rowid"
|
|
)
|
|
}
|
|
run_statuses = _indexed_values(
|
|
connection, "task_runs", RUN_STATUS_INDEX, ("status",)
|
|
)
|
|
if set(run_rows) != set(run_statuses):
|
|
raise RecoveryRefused("run table and status index disagree on row set")
|
|
status_column = run_columns.index("status")
|
|
run_id_column = run_columns.index("id")
|
|
task_column = run_columns.index("task_id")
|
|
reconciled_runs: set[int] = set()
|
|
for rowid, values in run_rows.items():
|
|
indexed_status = str(run_statuses[rowid][0])
|
|
if values[status_column] == indexed_status:
|
|
continue
|
|
run_id = int(values[run_id_column])
|
|
if run_id != rowid or indexed_status != "done":
|
|
raise RecoveryRefused("run status disagreement is not a completed run")
|
|
evidence = completed_evidence.get(run_id)
|
|
if evidence is None:
|
|
raise RecoveryRefused("run status disagreement lacks matching task evidence")
|
|
completed_at, summary = evidence
|
|
task_id = str(values[task_column])
|
|
if _completed_event(connection, task_id, run_id) != evidence:
|
|
raise RecoveryRefused("task and run completion evidence disagree")
|
|
replacements = {
|
|
"status": "done",
|
|
"outcome": "completed",
|
|
"ended_at": completed_at,
|
|
"claim_lock": None,
|
|
"claim_expires": None,
|
|
"worker_pid": None,
|
|
"summary": summary,
|
|
"metadata": None,
|
|
"error": None,
|
|
}
|
|
for column, replacement in replacements.items():
|
|
values[run_columns.index(column)] = replacement
|
|
reconciled_runs.add(run_id)
|
|
if reconciled_runs != set(completed_evidence):
|
|
raise RecoveryRefused("completed task and run disagreements do not match")
|
|
return run_rows
|
|
|
|
|
|
def reconciled_task_status_rows(
|
|
connection: sqlite3.Connection,
|
|
) -> dict[str, list[tuple[object, ...]]]:
|
|
"""Reconstruct reviewed stale task/run pages from independent evidence."""
|
|
task_columns = _table_columns(connection, "tasks")
|
|
run_columns = _table_columns(connection, "task_runs")
|
|
required_task_columns = {
|
|
"id", "status", "result", "completed_at", "claim_lock",
|
|
"claim_expires", "worker_pid", "current_run_id", "block_kind",
|
|
"block_recurrences",
|
|
}
|
|
required_run_columns = {
|
|
"id", "task_id", "status", "outcome", "ended_at", "claim_lock",
|
|
"claim_expires", "worker_pid", "summary", "metadata", "error",
|
|
}
|
|
if not required_task_columns <= set(task_columns):
|
|
raise RecoveryRefused("tasks schema lacks status-recovery columns")
|
|
if not required_run_columns <= set(run_columns):
|
|
raise RecoveryRefused("task_runs schema lacks status-recovery columns")
|
|
task_rows, completed_evidence = _reconcile_tasks(connection, task_columns)
|
|
run_rows = _reconcile_runs(connection, run_columns, completed_evidence)
|
|
return {
|
|
"tasks": [tuple(task_rows[rowid]) for rowid in sorted(task_rows)],
|
|
"task_runs": [tuple(run_rows[rowid]) for rowid in sorted(run_rows)],
|
|
}
|