175 lines
8.3 KiB
Python
175 lines
8.3 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Bind coordinator publication blocks to one immutable native event."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import json
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
import supervisor_state
|
||
|
|
|
||
|
|
|
||
|
|
PREFIX = "[hermes-publication-retry-fence:"
|
||
|
|
|
||
|
|
|
||
|
|
def marker(receipt: Any, run_id: str) -> str:
|
||
|
|
"""Return a bounded marker derived only from a validated retry receipt."""
|
||
|
|
digest = receipt.get("result_digest") if isinstance(receipt, dict) else None
|
||
|
|
if not isinstance(digest, str) or len(digest) != 64 or not run_id.isdecimal():
|
||
|
|
raise ValueError("publication retry fence is invalid")
|
||
|
|
return f"{PREFIX}{run_id}:{digest}]"
|
||
|
|
|
||
|
|
|
||
|
|
def _event(connection: Any, task_id: str, run_id: str, value: str) -> int:
|
||
|
|
"""Read the just-written native block event before recording its fence."""
|
||
|
|
row = connection.execute(
|
||
|
|
"SELECT id,run_id,kind,payload FROM task_events WHERE task_id=? ORDER BY id DESC LIMIT 1", (task_id,)
|
||
|
|
).fetchone()
|
||
|
|
try:
|
||
|
|
payload = json.loads(row[3] or "{}")
|
||
|
|
except (TypeError, ValueError, json.JSONDecodeError) as error:
|
||
|
|
raise ValueError("publication retry block event is malformed") from error
|
||
|
|
if (row is None or int(row[0]) < 1 or int(row[1] or 0) != int(run_id)
|
||
|
|
or row[2] != "blocked" or not isinstance(payload, dict)):
|
||
|
|
raise ValueError("publication retry block event is unavailable")
|
||
|
|
if value not in str(payload.get("reason") or ""):
|
||
|
|
raise ValueError("publication retry block event lacks its fence")
|
||
|
|
return int(row[0])
|
||
|
|
|
||
|
|
|
||
|
|
def record(board: str, task_id: str, run_id: str, connection: Any, value: str) -> None:
|
||
|
|
"""Persist a coordinator-owned event identity only after native blocking."""
|
||
|
|
event_id = _event(connection, task_id, run_id, value)
|
||
|
|
with supervisor_state._connect(board) as state:
|
||
|
|
state.execute(
|
||
|
|
"CREATE TABLE IF NOT EXISTS publication_retry_fences("
|
||
|
|
"board TEXT NOT NULL,child_task_id TEXT NOT NULL,run_id TEXT NOT NULL,"
|
||
|
|
"event_id INTEGER NOT NULL,marker TEXT NOT NULL,PRIMARY KEY(board,child_task_id,run_id))"
|
||
|
|
)
|
||
|
|
existing = state.execute(
|
||
|
|
"SELECT event_id,marker FROM publication_retry_fences WHERE board=? AND child_task_id=? AND run_id=?",
|
||
|
|
(board, task_id, run_id),
|
||
|
|
).fetchone()
|
||
|
|
if existing is None:
|
||
|
|
state.execute("INSERT INTO publication_retry_fences VALUES(?,?,?,?,?)", (board, task_id, run_id, event_id, value))
|
||
|
|
elif tuple(existing) != (event_id, value):
|
||
|
|
raise ValueError("publication retry fence conflicts with native history")
|
||
|
|
|
||
|
|
|
||
|
|
def read(board: str, task_id: str, run_id: str) -> tuple[int, str] | None:
|
||
|
|
"""Return one exact coordinator block fence, never a task-body claim."""
|
||
|
|
try:
|
||
|
|
with supervisor_state._connect(board) as state:
|
||
|
|
row = state.execute(
|
||
|
|
"SELECT event_id,marker FROM publication_retry_fences WHERE board=? AND child_task_id=? AND run_id=?",
|
||
|
|
(board, task_id, run_id),
|
||
|
|
).fetchone()
|
||
|
|
except Exception:
|
||
|
|
return None
|
||
|
|
return (int(row[0]), str(row[1])) if row and int(row[0]) > 0 and str(row[1]).startswith(PREFIX) else None
|
||
|
|
|
||
|
|
|
||
|
|
def terminal_receipt(record: Any) -> dict[str, Any] | None:
|
||
|
|
"""Return only a signed-pool publication failure that may reconstruct a fence."""
|
||
|
|
if not isinstance(record, dict):
|
||
|
|
return None
|
||
|
|
result, assignment = record.get("result"), record.get("payload")
|
||
|
|
if not isinstance(result, dict) or not isinstance(assignment, dict):
|
||
|
|
return None
|
||
|
|
structured = result.get("structured")
|
||
|
|
if (not isinstance(structured, dict) or structured.get("status") != "blocked"
|
||
|
|
or result.get("capacity_failure") is not True or result.get("scm_submission") is not None):
|
||
|
|
return None
|
||
|
|
receipt = result.get("scm_resume")
|
||
|
|
if receipt is None and result.get("publication_retry_transient") is True:
|
||
|
|
receipt = assignment.get("scm_resume")
|
||
|
|
source = receipt.get("source") if isinstance(receipt, dict) else None
|
||
|
|
if (not isinstance(source, dict) or source.get("board") != record.get("board")
|
||
|
|
or source.get("task_id") != record.get("task_id")
|
||
|
|
or source.get("worker_ordinal") != record.get("worker_ordinal")):
|
||
|
|
return None
|
||
|
|
return receipt
|
||
|
|
|
||
|
|
|
||
|
|
def recover(board: str, task_id: str, run_id: str, connection: Any, receipt: Any) -> tuple[int, str] | None:
|
||
|
|
"""Reconstruct a missing sidecar fence only from the current native event."""
|
||
|
|
try:
|
||
|
|
with supervisor_state._connect(board) as state:
|
||
|
|
sealed = state.execute(
|
||
|
|
"SELECT receipt_json FROM publication_retries WHERE board=? AND child_task_id=?",
|
||
|
|
(board, task_id),
|
||
|
|
).fetchone()
|
||
|
|
if sealed is None or json.loads(sealed[0]) != receipt:
|
||
|
|
return None
|
||
|
|
value = marker(receipt, run_id)
|
||
|
|
record(board, task_id, run_id, connection, value)
|
||
|
|
except (OSError, TypeError, ValueError, json.JSONDecodeError):
|
||
|
|
return None
|
||
|
|
return read(board, task_id, run_id)
|
||
|
|
|
||
|
|
|
||
|
|
def guarded_unblock(kanban_db: Any, connection: Any, task_id: str, root_task_id: str, run_id: str, fence: tuple[int, str]) -> bool:
|
||
|
|
"""Atomically reopen only the current coordinator-owned native block event."""
|
||
|
|
if (not isinstance(fence, tuple) or len(fence) != 2 or not isinstance(fence[0], int)
|
||
|
|
or fence[0] < 1 or not isinstance(fence[1], str) or not fence[1].startswith(PREFIX)
|
||
|
|
or not str(run_id).isdecimal() or not root_task_id):
|
||
|
|
return False
|
||
|
|
transaction = getattr(kanban_db, "write_txn", None)
|
||
|
|
append_event = getattr(kanban_db, "_append_event", None)
|
||
|
|
if not callable(transaction) or not callable(append_event):
|
||
|
|
return False
|
||
|
|
try:
|
||
|
|
with transaction(connection):
|
||
|
|
event = connection.execute(
|
||
|
|
"SELECT id,run_id,kind,payload FROM task_events WHERE task_id=? ORDER BY id DESC LIMIT 1",
|
||
|
|
(task_id,),
|
||
|
|
).fetchone()
|
||
|
|
try:
|
||
|
|
payload = json.loads(event[3] or "{}") if event is not None else {}
|
||
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
||
|
|
return False
|
||
|
|
if (event is None or int(event[0]) != fence[0] or int(event[1] or 0) != int(run_id)
|
||
|
|
or event[2] != "blocked" or not isinstance(payload, dict)
|
||
|
|
or fence[1] not in str(payload.get("reason") or "")):
|
||
|
|
return False
|
||
|
|
task = connection.execute(
|
||
|
|
"SELECT status,current_run_id FROM tasks WHERE id=?", (task_id,)
|
||
|
|
).fetchone()
|
||
|
|
parent = connection.execute(
|
||
|
|
"SELECT 1 FROM task_links WHERE child_id=? AND parent_id=?", (task_id, root_task_id)
|
||
|
|
).fetchone()
|
||
|
|
if task is None or task[0] != "blocked" or task[1] is not None or parent is None:
|
||
|
|
return False
|
||
|
|
unfinished = connection.execute(
|
||
|
|
"SELECT 1 FROM task_links l JOIN tasks p ON p.id=l.parent_id "
|
||
|
|
"WHERE l.child_id=? AND p.status != 'done' LIMIT 1", (task_id,)
|
||
|
|
).fetchone()
|
||
|
|
status = "todo" if unfinished is not None else "ready"
|
||
|
|
changed = connection.execute(
|
||
|
|
"UPDATE tasks SET status=?,current_run_id=NULL,consecutive_failures=0,last_failure_error=NULL "
|
||
|
|
"WHERE id=? AND status='blocked' AND current_run_id IS NULL", (status, task_id),
|
||
|
|
).rowcount
|
||
|
|
if changed != 1:
|
||
|
|
return False
|
||
|
|
append_event(connection, task_id, "unblocked", {"status": status, "publication_retry_fence": fence[1]})
|
||
|
|
return True
|
||
|
|
except (AttributeError, TypeError, ValueError):
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
def block(kanban_db: Any, connection: Any, board: str, task_id: str, run_id: str, reason: str, kind: str, receipt: Any) -> bool:
|
||
|
|
"""Block normally, then retain a native event fence for an automatic retry."""
|
||
|
|
try:
|
||
|
|
value = marker(receipt, run_id) if receipt is not None else ""
|
||
|
|
except ValueError:
|
||
|
|
# A malformed retained receipt must still park its exact live run, but
|
||
|
|
# can never acquire automatic-retry ownership.
|
||
|
|
value = ""
|
||
|
|
changed = kanban_db.block_task(
|
||
|
|
connection, task_id, reason=f"{reason}\n{value}" if value else reason, kind=kind,
|
||
|
|
expected_run_id=int(run_id),
|
||
|
|
)
|
||
|
|
if changed and value:
|
||
|
|
record(board, task_id, run_id, connection, value)
|
||
|
|
return bool(changed)
|