501 lines
26 KiB
Python
501 lines
26 KiB
Python
#!/usr/bin/env python3
|
|
"""Board-local integrity records for Hermes PR continuation cards."""
|
|
from __future__ import annotations
|
|
import hashlib
|
|
import fcntl
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import tempfile
|
|
import time
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from typing import Any, Iterator
|
|
from supervisor_lineage import Lineage
|
|
from publication_retry import PublicationRetryError, validate as validate_retry_receipt
|
|
KANBAN_ROOT = Path(
|
|
os.environ.get("HERMES_KANBAN_HOME", os.environ.get("HERMES_HOME", "/opt/data"))
|
|
) / "kanban/boards"
|
|
STATE_FILE = "supervisor-state.db"
|
|
class SupervisorStateError(ValueError):
|
|
"""A present integrity record cannot be decoded or read safely."""
|
|
def _path(board: str, path: Path | None) -> Path:
|
|
"""Keep integrity rows board-local without sharing Kanban's SQLite file."""
|
|
if path is not None:
|
|
return path
|
|
if not board or "/" in board or "\\" in board:
|
|
raise ValueError("board name is invalid")
|
|
return KANBAN_ROOT / board / STATE_FILE
|
|
def _schema(connection: sqlite3.Connection) -> None:
|
|
"""Install the small coordinator schema in its isolated state database."""
|
|
connection.executescript(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS supervisor_roots (
|
|
board TEXT NOT NULL, root_task_id TEXT NOT NULL, project TEXT NOT NULL,
|
|
branch TEXT NOT NULL, pull_request TEXT NOT NULL, base_branch TEXT NOT NULL,
|
|
live_pr_head TEXT NOT NULL, ready_for_human_merge INTEGER NOT NULL DEFAULT 0,
|
|
ready_commit TEXT NOT NULL DEFAULT '', PRIMARY KEY (board, root_task_id)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS supervisor_children (
|
|
board TEXT NOT NULL, child_task_id TEXT NOT NULL, root_task_id TEXT NOT NULL,
|
|
parent_task_id TEXT NOT NULL, kind TEXT NOT NULL, head_commit TEXT NOT NULL,
|
|
objective_digest TEXT NOT NULL, cycle INTEGER NOT NULL DEFAULT 1,
|
|
PRIMARY KEY (board, child_task_id),
|
|
UNIQUE (board, root_task_id, head_commit, objective_digest)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS publication_retries (
|
|
board TEXT NOT NULL, child_task_id TEXT NOT NULL,
|
|
source_run_id TEXT NOT NULL, source_ordinal INTEGER NOT NULL,
|
|
receipt_json TEXT NOT NULL, issued_run_id TEXT NOT NULL DEFAULT '',
|
|
resolved_run_id TEXT NOT NULL DEFAULT '',
|
|
reissue_count INTEGER NOT NULL DEFAULT 0, retry_after INTEGER NOT NULL DEFAULT 0,
|
|
last_reissued_run_id TEXT NOT NULL DEFAULT '',
|
|
PRIMARY KEY (board, child_task_id)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS publication_retry_provenance (
|
|
board TEXT NOT NULL, child_task_id TEXT NOT NULL, raw_result_sha256 TEXT NOT NULL,
|
|
reconstruction TEXT NOT NULL, PRIMARY KEY (board, child_task_id)
|
|
);
|
|
"""
|
|
)
|
|
def _legacy_rows(board: str, database: Path) -> tuple[list[tuple[Any, ...]], list[tuple[Any, ...]]]:
|
|
"""Read a complete, healthy legacy state table set or refuse migration."""
|
|
if not database.exists():
|
|
return [], []
|
|
try:
|
|
source = sqlite3.connect(f"file:{database}?mode=ro", uri=True)
|
|
try:
|
|
checked = [str(row[0]) for row in source.execute("PRAGMA integrity_check")]
|
|
if checked != ["ok"]:
|
|
raise SupervisorStateError("legacy supervisor state is unreadable")
|
|
names = {str(row[0]) for row in source.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table'"
|
|
)}
|
|
required = {"supervisor_roots", "supervisor_children"}
|
|
if not names & required:
|
|
return [], []
|
|
if not required <= names:
|
|
raise SupervisorStateError("legacy supervisor state is malformed")
|
|
root_columns = {str(row[1]) for row in source.execute("PRAGMA table_info(supervisor_roots)")}
|
|
child_columns = {str(row[1]) for row in source.execute("PRAGMA table_info(supervisor_children)")}
|
|
root_base = {"board", "root_task_id", "project", "branch", "pull_request", "base_branch", "live_pr_head"}
|
|
child_base = {"board", "child_task_id", "root_task_id", "parent_task_id", "kind", "head_commit", "objective_digest"}
|
|
if not root_base <= root_columns or not child_base <= child_columns:
|
|
raise SupervisorStateError("legacy supervisor state schema is malformed")
|
|
ready = "ready_for_human_merge" if "ready_for_human_merge" in root_columns else "0"
|
|
commit = "ready_commit" if "ready_commit" in root_columns else "''"
|
|
cycle = "cycle" if "cycle" in child_columns else "1"
|
|
roots = source.execute(
|
|
"SELECT board,root_task_id,project,branch,pull_request,base_branch,live_pr_head,"
|
|
f"{ready},{commit} FROM supervisor_roots"
|
|
).fetchall()
|
|
children = source.execute(
|
|
"SELECT board,child_task_id,root_task_id,parent_task_id,kind,head_commit,objective_digest,"
|
|
f"{cycle} FROM supervisor_children"
|
|
).fetchall()
|
|
finally:
|
|
source.close()
|
|
except sqlite3.Error as error:
|
|
raise SupervisorStateError("legacy supervisor state is unreadable") from error
|
|
root_ids = {str(row[1]) for row in roots}
|
|
if (any(not row or row[0] != board for row in [*roots, *children])
|
|
or any(str(row[2]) not in root_ids for row in children)):
|
|
raise SupervisorStateError("legacy supervisor state is malformed")
|
|
return roots, children
|
|
def _bootstrap_sidecar(board: str, state: Path) -> None:
|
|
"""Atomically import valid legacy rows once, never creating an empty fallback."""
|
|
lock = state.with_name(f".{STATE_FILE}.lock")
|
|
state.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
with lock.open("a+") as handle:
|
|
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
|
if state.exists():
|
|
return
|
|
roots, children = _legacy_rows(board, state.with_name("kanban.db"))
|
|
descriptor, temporary = tempfile.mkstemp(prefix=f".{STATE_FILE}.", dir=state.parent)
|
|
os.close(descriptor)
|
|
replacement = Path(temporary)
|
|
try:
|
|
connection = sqlite3.connect(replacement)
|
|
try:
|
|
_schema(connection)
|
|
if roots:
|
|
connection.executemany("INSERT INTO supervisor_roots VALUES(?,?,?,?,?,?,?,?,?)", roots)
|
|
connection.executemany("INSERT INTO supervisor_children VALUES(?,?,?,?,?,?,?,?)", children)
|
|
connection.commit()
|
|
finally:
|
|
connection.close()
|
|
descriptor = os.open(replacement, os.O_RDONLY)
|
|
try:
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
os.chmod(replacement, 0o600)
|
|
os.replace(replacement, state)
|
|
directory = os.open(state.parent, os.O_RDONLY)
|
|
try:
|
|
os.fsync(directory)
|
|
finally:
|
|
os.close(directory)
|
|
except Exception:
|
|
replacement.unlink(missing_ok=True)
|
|
raise
|
|
@contextmanager
|
|
def _connect(board: str, path: Path | None = None) -> Iterator[sqlite3.Connection]:
|
|
"""Open one sidecar transaction and always close its SQLite handle."""
|
|
default_path = path is None
|
|
path = _path(board, path)
|
|
if default_path and not path.exists():
|
|
_bootstrap_sidecar(board, path)
|
|
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
connection = sqlite3.connect(path, timeout=5)
|
|
try:
|
|
connection.execute("PRAGMA busy_timeout=5000")
|
|
connection.execute("PRAGMA foreign_keys=ON")
|
|
_schema(connection)
|
|
columns = {row[1] for row in connection.execute("PRAGMA table_info(supervisor_roots)")}
|
|
if "ready_for_human_merge" not in columns:
|
|
connection.execute(
|
|
"ALTER TABLE supervisor_roots ADD COLUMN ready_for_human_merge INTEGER NOT NULL DEFAULT 0"
|
|
)
|
|
if "ready_commit" not in columns:
|
|
connection.execute("ALTER TABLE supervisor_roots ADD COLUMN ready_commit TEXT NOT NULL DEFAULT ''")
|
|
child_columns = {row[1] for row in connection.execute("PRAGMA table_info(supervisor_children)")}
|
|
if "cycle" not in child_columns:
|
|
connection.execute("ALTER TABLE supervisor_children ADD COLUMN cycle INTEGER NOT NULL DEFAULT 1")
|
|
retry_columns = {row[1] for row in connection.execute("PRAGMA table_info(publication_retries)")}
|
|
if "resolved_run_id" not in retry_columns:
|
|
connection.execute("ALTER TABLE publication_retries ADD COLUMN resolved_run_id TEXT NOT NULL DEFAULT ''")
|
|
if "reissue_count" not in retry_columns:
|
|
connection.execute("ALTER TABLE publication_retries ADD COLUMN reissue_count INTEGER NOT NULL DEFAULT 0")
|
|
if "retry_after" not in retry_columns:
|
|
connection.execute("ALTER TABLE publication_retries ADD COLUMN retry_after INTEGER NOT NULL DEFAULT 0")
|
|
if "last_reissued_run_id" not in retry_columns:
|
|
connection.execute("ALTER TABLE publication_retries ADD COLUMN last_reissued_run_id TEXT NOT NULL DEFAULT ''")
|
|
if path.name == STATE_FILE:
|
|
os.chmod(path, 0o600)
|
|
yield connection
|
|
connection.commit()
|
|
except Exception:
|
|
connection.rollback()
|
|
raise
|
|
finally:
|
|
connection.close()
|
|
def _lineage(row: tuple[Any, ...] | None) -> Lineage | None:
|
|
if row is None:
|
|
return None
|
|
try:
|
|
values = tuple(str(value) for value in row[:5])
|
|
except (TypeError, ValueError):
|
|
return None
|
|
if not all(values):
|
|
return None
|
|
return Lineage(*values)
|
|
def get_root(board: str, root_task_id: str, *, path: Path | None = None) -> Lineage | None:
|
|
"""Return one coordinator-issued root lineage, never task-body claims."""
|
|
try:
|
|
with _connect(board, path) as connection:
|
|
row = connection.execute(
|
|
"SELECT root_task_id, branch, pull_request, project, base_branch "
|
|
"FROM supervisor_roots WHERE board=? AND root_task_id=?", (board, root_task_id)
|
|
).fetchone()
|
|
except (OSError, sqlite3.Error) as error:
|
|
raise SupervisorStateError("supervisor root state is unreadable") from error
|
|
if row is None:
|
|
return None
|
|
lineage = _lineage(row)
|
|
if lineage is None:
|
|
raise SupervisorStateError("supervisor root state is malformed")
|
|
return lineage
|
|
def get_live_head(board: str, root_task_id: str, *, path: Path | None = None) -> str:
|
|
"""Return the broker-confirmed head associated with a trusted root."""
|
|
try:
|
|
with _connect(board, path) as connection:
|
|
row = connection.execute(
|
|
"SELECT live_pr_head FROM supervisor_roots WHERE board=? AND root_task_id=?",
|
|
(board, root_task_id),
|
|
).fetchone()
|
|
except (OSError, sqlite3.Error) as error:
|
|
raise SupervisorStateError("supervisor root state is unreadable") from error
|
|
if row is None:
|
|
return ""
|
|
if not isinstance(row[0], str) or not row[0]:
|
|
raise SupervisorStateError("supervisor root state is malformed")
|
|
return row[0]
|
|
def get_child(board: str, child_task_id: str, *, path: Path | None = None) -> dict[str, Any] | None:
|
|
"""Return the only durable authority record for a continuation child."""
|
|
try:
|
|
with _connect(board, path) as connection:
|
|
row = connection.execute(
|
|
"SELECT root_task_id, parent_task_id, kind, head_commit, objective_digest, cycle "
|
|
"FROM supervisor_children WHERE board=? AND child_task_id=?", (board, child_task_id)
|
|
).fetchone()
|
|
except (OSError, sqlite3.Error) as error:
|
|
raise SupervisorStateError("supervisor child state is unreadable") from error
|
|
if row is None:
|
|
return None
|
|
root, parent, kind, head, digest, cycle = (str(value) for value in row)
|
|
lineage = get_root(board, root, path=path)
|
|
if lineage is None or kind not in {"repair", "review"} or not all((parent, head, digest)):
|
|
raise SupervisorStateError("supervisor child state is malformed")
|
|
try:
|
|
parsed_cycle = int(cycle)
|
|
except (TypeError, ValueError) as error:
|
|
raise SupervisorStateError("supervisor child state is malformed") from error
|
|
if parsed_cycle < 1:
|
|
raise SupervisorStateError("supervisor child state is malformed")
|
|
return {"lineage": lineage, "root_task_id": root, "parent_task_id": parent,
|
|
"kind": kind, "head_commit": head, "objective_digest": digest,
|
|
"cycle": parsed_cycle}
|
|
def record_submission(
|
|
board: str, task_id: str, lineage: Lineage, live_pr_head: str, *, path: Path | None = None
|
|
) -> None:
|
|
"""Record a coordinator-verified submission against its immutable root.
|
|
``task_id`` identifies the completing root or child. The coordinator has
|
|
already authenticated that task's signed assignment before calling this
|
|
function; the record deliberately remains keyed by ``root_task_id``.
|
|
"""
|
|
if not board or not task_id or not live_pr_head:
|
|
raise ValueError("submission does not name a task and live head")
|
|
with _connect(board, path) as connection:
|
|
existing = connection.execute(
|
|
"SELECT project,branch,pull_request,base_branch,live_pr_head FROM supervisor_roots "
|
|
"WHERE board=? AND root_task_id=?", (board, lineage.root_task_id)
|
|
).fetchone()
|
|
identity = (lineage.project, lineage.branch, lineage.pull_request, lineage.base_branch)
|
|
if existing is None:
|
|
connection.execute(
|
|
"INSERT INTO supervisor_roots(board,root_task_id,project,branch,pull_request,base_branch,live_pr_head) "
|
|
"VALUES(?,?,?,?,?,?,?)",
|
|
(board, lineage.root_task_id, *identity, live_pr_head),
|
|
)
|
|
elif tuple(existing[:4]) != identity:
|
|
raise ValueError("submission attempts to rewrite immutable root lineage")
|
|
elif existing[4] != live_pr_head:
|
|
connection.execute(
|
|
"UPDATE supervisor_roots SET live_pr_head=?, ready_for_human_merge=0, ready_commit='' "
|
|
"WHERE board=? AND root_task_id=?",
|
|
(live_pr_head, board, lineage.root_task_id),
|
|
)
|
|
def record_live_head(
|
|
board: str, root_task_id: str, live_pr_head: str, *, path: Path | None = None
|
|
) -> None:
|
|
"""Refresh a root's head only after the continuation CLI verified its PR."""
|
|
if not board or not root_task_id or not live_pr_head:
|
|
raise ValueError("root and live head are required")
|
|
with _connect(board, path) as connection:
|
|
changed = connection.execute(
|
|
"UPDATE supervisor_roots SET live_pr_head=?, ready_for_human_merge=0, ready_commit='' "
|
|
"WHERE board=? AND root_task_id=?", (live_pr_head, board, root_task_id)
|
|
).rowcount
|
|
if changed != 1:
|
|
raise ValueError("unknown continuation root")
|
|
def set_ready(
|
|
board: str, root_task_id: str, head_commit: str, *, path: Path | None = None
|
|
) -> None:
|
|
"""Publish human-ready state only for the verified current root head."""
|
|
with _connect(board, path) as connection:
|
|
changed = connection.execute(
|
|
"UPDATE supervisor_roots SET ready_for_human_merge=1, ready_commit=? "
|
|
"WHERE board=? AND root_task_id=? AND live_pr_head=?",
|
|
(head_commit, board, root_task_id, head_commit),
|
|
).rowcount
|
|
if changed != 1:
|
|
raise ValueError("ready head is not the trusted live root head")
|
|
def clear_ready(board: str, root_task_id: str, *, path: Path | None = None) -> None:
|
|
"""Invalidate approval whenever a continuation is queued or head changes."""
|
|
with _connect(board, path) as connection:
|
|
connection.execute(
|
|
"UPDATE supervisor_roots SET ready_for_human_merge=0, ready_commit='' "
|
|
"WHERE board=? AND root_task_id=?", (board, root_task_id)
|
|
)
|
|
def record_child(
|
|
board: str, child_task_id: str, root_task_id: str, parent_task_id: str, kind: str,
|
|
head_commit: str, objective: str, cycle: int = 1, *, path: Path | None = None
|
|
) -> None:
|
|
"""Bind a created child to an existing trusted root and exact continuation."""
|
|
if not get_root(board, root_task_id, path=path) or kind not in {"repair", "review"}:
|
|
raise ValueError("continuation root or kind is invalid")
|
|
digest = objective_digest(objective)
|
|
if cycle < 1:
|
|
raise ValueError("continuation cycle is invalid")
|
|
values = (root_task_id, parent_task_id, kind, head_commit, digest, cycle)
|
|
with _connect(board, path) as connection:
|
|
existing = connection.execute(
|
|
"SELECT root_task_id,parent_task_id,kind,head_commit,objective_digest,cycle "
|
|
"FROM supervisor_children WHERE board=? AND child_task_id=?", (board, child_task_id)
|
|
).fetchone()
|
|
if existing is None:
|
|
collision = connection.execute(
|
|
"SELECT child_task_id FROM supervisor_children WHERE board=? AND root_task_id=? "
|
|
"AND head_commit=? AND objective_digest=?",
|
|
(board, root_task_id, head_commit, digest),
|
|
).fetchone()
|
|
if collision is not None:
|
|
raise ValueError("continuation already maps this root/head/objective to another task")
|
|
connection.execute(
|
|
"INSERT INTO supervisor_children VALUES(?,?,?,?,?,?,?,?)", (board, child_task_id, *values)
|
|
)
|
|
elif tuple(existing) != values:
|
|
raise ValueError("continuation child lineage conflicts with its existing record")
|
|
def objective_digest(objective: str) -> str:
|
|
"""Make duplicate user follow-ups idempotent without retaining extra prose."""
|
|
return hashlib.sha256(objective.strip().encode()).hexdigest()
|
|
def existing_child(
|
|
board: str, root_task_id: str, head_commit: str, objective: str, *, path: Path | None = None
|
|
) -> str:
|
|
"""Return an existing same-objective continuation, if one was created."""
|
|
with _connect(board, path) as connection:
|
|
row = connection.execute(
|
|
"SELECT child_task_id FROM supervisor_children WHERE board=? AND root_task_id=? "
|
|
"AND head_commit=? AND objective_digest=?",
|
|
(board, root_task_id, head_commit, objective_digest(objective)),
|
|
).fetchone()
|
|
return str(row[0]) if row else ""
|
|
def _retry_receipt(
|
|
board: str, child_task_id: str, binding: dict[str, Any], receipt: Any, *, path: Path | None
|
|
) -> tuple[dict[str, Any], int, str]:
|
|
"""Validate the mediator-signed retry receipt against private continuation state."""
|
|
child = get_child(board, child_task_id, path=path)
|
|
if child is None or child["kind"] != "repair" or not isinstance(binding, dict):
|
|
raise ValueError("publication retry lineage is invalid")
|
|
lineage = child["lineage"]
|
|
return validate_retry_receipt(
|
|
receipt, binding, board=board, child_task_id=child_task_id,
|
|
root_task_id=lineage.root_task_id, project=lineage.project, branch=lineage.branch,
|
|
base_branch=lineage.base_branch, live_head=get_live_head(board, lineage.root_task_id, path=path),
|
|
)
|
|
def record_publication_retry(
|
|
board: str, child_task_id: str, binding: dict[str, Any], receipt: Any, *, path: Path | None = None
|
|
) -> None:
|
|
"""Persist one mediator-signed failed-publication receipt for a fresh run."""
|
|
verified, ordinal, source_run_id = _retry_receipt(board, child_task_id, binding, receipt, path=path)
|
|
encoded = json.dumps(verified, separators=(",", ":"), sort_keys=True)
|
|
if len(encoded.encode()) > 48 * 1024:
|
|
raise ValueError("publication retry receipt exceeds its assignment limit")
|
|
with _connect(board, path) as connection:
|
|
existing = connection.execute(
|
|
"SELECT source_run_id,source_ordinal,receipt_json FROM publication_retries "
|
|
"WHERE board=? AND child_task_id=?", (board, child_task_id)
|
|
).fetchone()
|
|
values = (source_run_id, ordinal, encoded)
|
|
if existing is None:
|
|
connection.execute(
|
|
"INSERT INTO publication_retries(board,child_task_id,source_run_id,source_ordinal,receipt_json) "
|
|
"VALUES(?,?,?,?,?)", (board, child_task_id, *values)
|
|
)
|
|
elif tuple(existing) != values:
|
|
raise ValueError("publication retry receipt conflicts with existing evidence")
|
|
def publication_retry(
|
|
board: str, child_task_id: str, run_id: str, *, path: Path | None = None
|
|
) -> dict[str, Any] | None:
|
|
"""Return an unconsumed receipt, or its one exact fresh run after a crash."""
|
|
if not isinstance(run_id, str) or (run_id and (not run_id.isdecimal() or int(run_id) < 1)):
|
|
raise ValueError("publication retry run is invalid")
|
|
if path is None and not _path(board, None).exists():
|
|
return None
|
|
with _connect(board, path) as connection:
|
|
row = connection.execute(
|
|
"SELECT source_run_id,source_ordinal,receipt_json,issued_run_id,resolved_run_id,reissue_count,retry_after FROM publication_retries "
|
|
"WHERE board=? AND child_task_id=?", (board, child_task_id)
|
|
).fetchone()
|
|
if row is None or row[4]:
|
|
return None
|
|
if int(row[6]) > int(time.time()):
|
|
raise PublicationRetryError("publication retry backoff is active")
|
|
if row[3] and row[3] != run_id:
|
|
raise PublicationRetryError("publication retry was already issued")
|
|
try:
|
|
receipt = json.loads(row[2])
|
|
except (TypeError, json.JSONDecodeError) as error:
|
|
raise SupervisorStateError("publication retry receipt is malformed") from error
|
|
verified, ordinal, source_run_id = _retry_receipt(
|
|
board, child_task_id,
|
|
{"board": board, "task_id": child_task_id, "run_id": row[0], "worker_ordinal": row[1],
|
|
"attempt": receipt.get("source", {}).get("attempt") if isinstance(receipt, dict) else None},
|
|
receipt, path=path,
|
|
)
|
|
if source_run_id != row[0] or ordinal != row[1] or (run_id and run_id == source_run_id):
|
|
raise SupervisorStateError("publication retry receipt is inconsistent")
|
|
return verified
|
|
def issue_publication_retry(board: str, child_task_id: str, run_id: str, *, path: Path | None = None) -> None:
|
|
"""Fence a receipt to one fresh assignment after its durable pool row exists."""
|
|
with _connect(board, path) as connection:
|
|
row = connection.execute(
|
|
"SELECT issued_run_id,resolved_run_id,last_reissued_run_id FROM publication_retries "
|
|
"WHERE board=? AND child_task_id=?", (board, child_task_id)
|
|
).fetchone()
|
|
if row is not None and (
|
|
(row[0] == run_id and row[1] in ("", run_id))
|
|
or (not row[0] and not row[1] and row[2] == run_id)
|
|
):
|
|
return
|
|
if publication_retry(board, child_task_id, run_id, path=path) is None:
|
|
raise ValueError("publication retry is unavailable")
|
|
with _connect(board, path) as connection:
|
|
changed = connection.execute(
|
|
"UPDATE publication_retries SET issued_run_id=? WHERE board=? AND child_task_id=? "
|
|
"AND issued_run_id=''", (run_id, board, child_task_id)
|
|
).rowcount
|
|
if changed != 1:
|
|
with _connect(board, path) as connection:
|
|
row = connection.execute(
|
|
"SELECT issued_run_id FROM publication_retries WHERE board=? AND child_task_id=?",
|
|
(board, child_task_id),
|
|
).fetchone()
|
|
if row is None or row[0] != run_id:
|
|
raise SupervisorStateError("publication retry was already issued")
|
|
def resolve_publication_retry(board: str, child_task_id: str, run_id: str, *, path: Path | None = None) -> None:
|
|
"""Mark the one fresh broker-confirmed run as the receipt's trusted resolution."""
|
|
with _connect(board, path) as connection:
|
|
changed = connection.execute(
|
|
"UPDATE publication_retries SET resolved_run_id=? WHERE board=? AND child_task_id=? "
|
|
"AND issued_run_id=? AND resolved_run_id=''", (run_id, board, child_task_id, run_id)
|
|
).rowcount
|
|
if changed != 1:
|
|
with _connect(board, path) as connection:
|
|
row = connection.execute(
|
|
"SELECT issued_run_id,resolved_run_id FROM publication_retries WHERE board=? AND child_task_id=?",
|
|
(board, child_task_id),
|
|
).fetchone()
|
|
if row is None or tuple(row) != (run_id, run_id):
|
|
raise SupervisorStateError("publication retry cannot be resolved")
|
|
def reissue_publication_retry(board: str, child_task_id: str, run_id: str, *, path: Path | None = None) -> bool:
|
|
"""Release one interrupted transient retry once, with a five-minute backoff."""
|
|
with _connect(board, path) as connection:
|
|
changed = connection.execute(
|
|
"UPDATE publication_retries SET issued_run_id='',reissue_count=reissue_count+1,retry_after=?,last_reissued_run_id=? "
|
|
"WHERE board=? AND child_task_id=? AND issued_run_id=? AND resolved_run_id='' AND reissue_count<1",
|
|
(int(time.time()) + 300, run_id, board, child_task_id, run_id),
|
|
).rowcount
|
|
if changed:
|
|
return True
|
|
row = connection.execute(
|
|
"SELECT issued_run_id,resolved_run_id,reissue_count,last_reissued_run_id FROM publication_retries WHERE board=? AND child_task_id=?",
|
|
(board, child_task_id),
|
|
).fetchone()
|
|
if row is not None and not row[0] and not row[1] and row[3] == run_id and int(row[2]) == 1:
|
|
return True
|
|
if row is not None and row[0] == run_id and not row[1] and int(row[2]) >= 1:
|
|
return False
|
|
raise SupervisorStateError("publication retry cannot be reissued")
|
|
def record_publication_retry_provenance(board: str, child_task_id: str, raw_result_sha256: str, *, path: Path | None = None) -> None:
|
|
"""Keep a bootstrap's immutable blocked result hash beside its retry receipt."""
|
|
if not isinstance(raw_result_sha256, str) or len(raw_result_sha256) != 64:
|
|
raise ValueError("publication retry provenance is invalid")
|
|
with _connect(board, path) as connection:
|
|
if connection.execute(
|
|
"SELECT 1 FROM publication_retries WHERE board=? AND child_task_id=?", (board, child_task_id)
|
|
).fetchone() is None:
|
|
raise ValueError("publication retry receipt is unavailable")
|
|
existing = connection.execute(
|
|
"SELECT raw_result_sha256 FROM publication_retry_provenance WHERE board=? AND child_task_id=?",
|
|
(board, child_task_id),
|
|
).fetchone()
|
|
if existing is None:
|
|
connection.execute(
|
|
"INSERT INTO publication_retry_provenance VALUES(?,?,?,?)",
|
|
(board, child_task_id, raw_result_sha256, "operator-reconstructed-from-private-result-log"),
|
|
)
|
|
elif existing[0] != raw_result_sha256:
|
|
raise ValueError("publication retry provenance conflicts with existing evidence")
|