462 lines
17 KiB
Python
462 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Recoverable maintenance passes for the Hermes execution-pool coordinator.
|
|
|
|
Every pass here runs against a shared Kanban SQLite database that other writers
|
|
(the agent UI, the local CLI lane, crons) hold concurrently, so *any* board call
|
|
can fail transiently. The rules this module enforces:
|
|
|
|
* No pass ever leaves a claimed run without a durable row that some later pass
|
|
will revisit. A Kanban write that fails leaves the row in its retryable state.
|
|
* No single poisoned row, task, or board aborts the work queued behind it.
|
|
* A durable row only reaches a terminal state on authoritative evidence about
|
|
its Kanban run, because terminal rows are the only ones ever collected.
|
|
* A coordinator-side fault is never converted into a Kanban mutation. Only a
|
|
genuine capability failure of the run itself parks that exact run.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
import sys
|
|
from typing import Any
|
|
|
|
import cli_lane_dispatch
|
|
from cli_lane_config import canonical_run_id
|
|
from execution_pool_project import ProjectPolicyError, distributed_workspace_eligible
|
|
from execution_pool_protocol import ProtocolError
|
|
import supervisor_state
|
|
|
|
|
|
RECOVERABLE_BOARD_ERRORS = (OSError, sqlite3.Error)
|
|
DEFERRALS: dict[str, str] = {}
|
|
MAX_DEFERRALS = 64
|
|
|
|
|
|
def _task_value(task: Any, name: str, default: Any = None) -> Any:
|
|
return getattr(task, name, default)
|
|
|
|
|
|
def _defer(context: str, error: BaseException) -> None:
|
|
"""Report one recoverable pool-side failure once per distinct detail.
|
|
|
|
The maintenance loop re-runs every few seconds, so an unchanged failure is
|
|
reported once rather than on every pass. The table is bounded because its
|
|
keys include run identities.
|
|
"""
|
|
detail = f"{type(error).__name__}: {error}"
|
|
if DEFERRALS.get(context) == detail:
|
|
return
|
|
if len(DEFERRALS) >= MAX_DEFERRALS:
|
|
DEFERRALS.clear()
|
|
DEFERRALS[context] = detail
|
|
print(f"pool work deferred for {context}: {detail}", file=sys.stderr, flush=True)
|
|
|
|
|
|
def _settled(context: str) -> None:
|
|
DEFERRALS.pop(context, None)
|
|
|
|
|
|
def _context(record: dict[str, Any]) -> str:
|
|
return f"{record['board']}/{record['task_id']}#{record['run_id']}"
|
|
|
|
|
|
def _resume_ordinal(payload: Any) -> int | None:
|
|
"""Read the coordinator-issued source ordinal without accepting task text."""
|
|
receipt = payload.get("scm_resume") if isinstance(payload, dict) else None
|
|
if receipt is None:
|
|
return None
|
|
source = receipt.get("source") if isinstance(receipt, dict) else None
|
|
ordinal = source.get("worker_ordinal") if isinstance(source, dict) else None
|
|
if isinstance(ordinal, bool) or not isinstance(ordinal, int) or ordinal not in range(3):
|
|
raise ProtocolError("publication retry ordinal is invalid")
|
|
return ordinal
|
|
|
|
|
|
def _issue_retry(record: dict[str, Any]) -> None:
|
|
"""Fence a persisted receipt to its one fresh durable assignment."""
|
|
if _resume_ordinal(record.get("payload")) is not None:
|
|
supervisor_state.issue_publication_retry(
|
|
record["board"], record["task_id"], record["run_id"]
|
|
)
|
|
|
|
|
|
def _pending_retry_ordinal(board: str, task: Any) -> int | None:
|
|
"""Look up an unissued receipt before the ready card receives its fresh run."""
|
|
task_id = str(_task_value(task, "id", "") or "")
|
|
if not task_id:
|
|
return None
|
|
receipt = supervisor_state.publication_retry(board, task_id, "")
|
|
return _resume_ordinal({"scm_resume": receipt}) if receipt is not None else None
|
|
|
|
|
|
def _block_exact(
|
|
kanban_db: Any,
|
|
board: str,
|
|
task_id: str,
|
|
run_id: int | None,
|
|
reason: str,
|
|
*,
|
|
kind: str = "capability",
|
|
) -> None:
|
|
"""Park exactly one run, never letting the write itself escape."""
|
|
try:
|
|
with kanban_db.scoped_current_board(board):
|
|
connection = kanban_db.connect(board=board)
|
|
try:
|
|
kanban_db.block_task(
|
|
connection, task_id, reason=reason, kind=kind,
|
|
expected_run_id=run_id,
|
|
)
|
|
finally:
|
|
connection.close()
|
|
except Exception as error: # noqa: BLE001 - a failed park must not wedge the pass
|
|
_defer(f"{board}/{task_id} block", error)
|
|
|
|
|
|
def recover_results(pool: Any) -> None:
|
|
"""Re-apply each durable result in isolation from its neighbours."""
|
|
for record in pool.store.pending_results():
|
|
try:
|
|
pool.finalize(record)
|
|
_settled(_context(record))
|
|
except Exception as error: # noqa: BLE001 - one bad result must not block the rest
|
|
print(
|
|
f"result recovery deferred: {type(error).__name__}: {error}",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
|
|
|
|
def _lease_failure_state(
|
|
kanban_db: Any, connection: Any, binding: dict[str, Any], run_id: int, retry_exhausted: bool = False
|
|
) -> str:
|
|
"""Classify one exhausted lease from authoritative Kanban evidence only."""
|
|
task = kanban_db.get_task(connection, binding["task_id"])
|
|
current = (
|
|
canonical_run_id(_task_value(task, "current_run_id", None))
|
|
if task is not None
|
|
else None
|
|
)
|
|
if current != run_id:
|
|
return "stale"
|
|
if str(_task_value(task, "status", "") or "") != "running":
|
|
return "finalized"
|
|
if kanban_db.block_task(
|
|
connection,
|
|
binding["task_id"],
|
|
reason=("Publication retry budget exhausted; inspect the retained SCM evidence." if retry_exhausted else
|
|
(
|
|
"Distributed worker lease expired after "
|
|
f"{binding['attempt']} fenced attempts"
|
|
)),
|
|
kind="capability" if retry_exhausted else "transient",
|
|
expected_run_id=run_id,
|
|
):
|
|
return "finalized"
|
|
# Kanban refused the exact-run park yet still reports the run as running, so
|
|
# there is no terminal evidence. Keep the row for the next pass to retry.
|
|
raise ProtocolError("Kanban refused the exact-run lease park")
|
|
|
|
|
|
def record_lease_failure(pool: Any, record: dict[str, Any]) -> None:
|
|
"""Record one exhausted lease against its exact Kanban run, idempotently.
|
|
|
|
The row stays ``lease_failed`` unless this pass obtains authoritative
|
|
evidence: the run moved on or vanished (``stale``), Kanban already parked it
|
|
(``finalized``), or this call parked it (``finalized``). Anything else --
|
|
including a refused write or an unreadable board -- leaves the row for the
|
|
next pass, so no run is ever silently dropped.
|
|
"""
|
|
from hermes_cli import kanban_db
|
|
|
|
binding = pool._binding(record)
|
|
run_id = canonical_run_id(binding["run_id"])
|
|
if run_id is None:
|
|
pool.store.finalize(binding, "stale")
|
|
return
|
|
retry_exhausted = False
|
|
if _resume_ordinal(record.get("payload")) is not None:
|
|
retry_exhausted = not supervisor_state.reissue_publication_retry(
|
|
binding["board"], binding["task_id"], binding["run_id"]
|
|
)
|
|
with pool._kanban_lock, kanban_db.scoped_current_board(binding["board"]):
|
|
connection = kanban_db.connect(board=binding["board"])
|
|
try:
|
|
state = _lease_failure_state(kanban_db, connection, binding, run_id, retry_exhausted)
|
|
finally:
|
|
connection.close()
|
|
pool.store.finalize(binding, state)
|
|
_settled(_context(record))
|
|
|
|
|
|
def recover_lease_failures(pool: Any) -> None:
|
|
"""Retry every released-but-unrecorded lease failure, each in isolation."""
|
|
for record in pool.store.failed_leases():
|
|
try:
|
|
record_lease_failure(pool, record)
|
|
except Exception as error: # noqa: BLE001 - retried on the next pass
|
|
_defer(f"{_context(record)} lease park", error)
|
|
|
|
|
|
def expire_leases(pool: Any) -> None:
|
|
"""Fence dead attempts, then drain every unconfirmed terminal record."""
|
|
pool.store.expire_leases()
|
|
recover_lease_failures(pool)
|
|
|
|
|
|
def _release_moved_run(pool: Any, kanban_db: Any, record: dict[str, Any]) -> None:
|
|
"""Release one durable assignment whose Kanban run is no longer ours."""
|
|
with kanban_db.scoped_current_board(record["board"]):
|
|
connection = kanban_db.connect(board=record["board"])
|
|
try:
|
|
task = kanban_db.get_task(connection, record["task_id"])
|
|
current_run_id = (
|
|
canonical_run_id(_task_value(task, "current_run_id", None))
|
|
if task
|
|
else None
|
|
)
|
|
current = str(current_run_id or "")
|
|
status = str(_task_value(task, "status", "") or "") if task else ""
|
|
finally:
|
|
connection.close()
|
|
if current != record["run_id"] or status != "running":
|
|
pool.store.finalize(record, "stale")
|
|
|
|
|
|
def _adopt_task(
|
|
pool: Any,
|
|
kanban_db: Any,
|
|
connection: Any,
|
|
board: str,
|
|
task: Any,
|
|
ordinals: list[int],
|
|
known: set[tuple[str, str, str]],
|
|
) -> None:
|
|
"""Adopt one orphaned pool-eligible run onto a free ordinal."""
|
|
task_id = str(_task_value(task, "id", "") or "")
|
|
raw_run_id = canonical_run_id(_task_value(task, "current_run_id", None))
|
|
run_id = str(raw_run_id or "")
|
|
assignee = str(_task_value(task, "assignee", "") or "")
|
|
if (
|
|
not task_id
|
|
or not run_id
|
|
or str(_task_value(task, "status", "")) != "running"
|
|
or not assignee.startswith("cli-")
|
|
or not distributed_workspace_eligible(task)
|
|
or (board, task_id, run_id) in known
|
|
):
|
|
return
|
|
try:
|
|
payload = pool.assignment_payload(kanban_db, connection, task, board)
|
|
except RECOVERABLE_BOARD_ERRORS as error:
|
|
_defer(f"{board}/{task_id} recovery payload", error)
|
|
return
|
|
except Exception as error: # noqa: BLE001 - a real capability failure of this run
|
|
_block_exact(
|
|
kanban_db, board, task_id, raw_run_id,
|
|
"Distributed assignment recovery failed: "
|
|
f"{type(error).__name__}: {error}",
|
|
)
|
|
return
|
|
ordinal = _resume_ordinal(payload)
|
|
if ordinal is not None:
|
|
if ordinal not in ordinals:
|
|
_defer(f"{board}/{task_id} recovery retry", ProtocolError("source ordinal is unavailable"))
|
|
return
|
|
else:
|
|
ordinal = ordinals[0]
|
|
binding = {
|
|
"board": board, "task_id": task_id, "run_id": run_id,
|
|
"worker_ordinal": ordinal, "attempt": 1,
|
|
}
|
|
try:
|
|
pool.store.add(binding, payload)
|
|
except ProtocolError as error:
|
|
# A durable row already owns this identity or ordinal. Leave the ordinal
|
|
# free and leave Kanban untouched; a later pass revisits it.
|
|
_defer(f"{board}/{task_id} adoption", error)
|
|
return
|
|
known.add((board, task_id, run_id))
|
|
_issue_retry({**binding, "payload": payload})
|
|
ordinals.remove(ordinal)
|
|
|
|
|
|
def _adopt_board(
|
|
pool: Any,
|
|
kanban_db: Any,
|
|
board: str,
|
|
ordinals: list[int],
|
|
known: set[tuple[str, str, str]],
|
|
) -> None:
|
|
"""Adopt this board's orphaned runs without touching any other board."""
|
|
with kanban_db.scoped_current_board(board):
|
|
connection = kanban_db.connect(board=board)
|
|
try:
|
|
for task in kanban_db.list_tasks(connection):
|
|
if not ordinals:
|
|
return
|
|
_adopt_task(pool, kanban_db, connection, board, task, ordinals, known)
|
|
finally:
|
|
connection.close()
|
|
|
|
|
|
def reconcile(pool: Any) -> None:
|
|
"""Recover the narrow claim/assignment crash gaps without double execution."""
|
|
from hermes_cli import kanban_db
|
|
|
|
for record in pool.store.active_assignments():
|
|
try:
|
|
_issue_retry(record)
|
|
_release_moved_run(pool, kanban_db, record)
|
|
except Exception as error: # noqa: BLE001 - other assignments still reconcile
|
|
_defer(f"{_context(record)} release", error)
|
|
ordinals = pool.store.available_ordinals()
|
|
if not ordinals:
|
|
return
|
|
known = pool.store.known_runs()
|
|
try:
|
|
boards = kanban_db.list_boards(include_archived=False)
|
|
except RECOVERABLE_BOARD_ERRORS as error:
|
|
_defer("board-registry", error)
|
|
return
|
|
_settled("board-registry")
|
|
for raw_board in boards:
|
|
board = cli_lane_dispatch._board_slug(raw_board)
|
|
if not board or not ordinals:
|
|
continue
|
|
try:
|
|
_adopt_board(pool, kanban_db, board, ordinals, known)
|
|
_settled(f"{board} adoption")
|
|
except Exception as error: # noqa: BLE001 - later boards must still run
|
|
_defer(f"{board} adoption", error)
|
|
|
|
|
|
def _materialize(
|
|
pool: Any,
|
|
kanban_db: Any,
|
|
board: str,
|
|
task_id: str,
|
|
ordinal: int,
|
|
known: set[tuple[str, str, str]],
|
|
) -> None:
|
|
"""Turn one freshly claimed run into exactly one durable assignment."""
|
|
database_run_id: int | None = None
|
|
try:
|
|
with kanban_db.scoped_current_board(board):
|
|
connection = kanban_db.connect(board=board)
|
|
try:
|
|
task = kanban_db.get_task(connection, task_id)
|
|
if task is None:
|
|
return
|
|
raw_run = canonical_run_id(
|
|
_task_value(task, "current_run_id", None)
|
|
)
|
|
if not distributed_workspace_eligible(task):
|
|
if raw_run is not None:
|
|
kanban_db.block_task(
|
|
connection, task_id,
|
|
reason=(
|
|
"Distributed claim fenced because an existing "
|
|
"workspace is owned by the local lane"
|
|
),
|
|
kind="capability", expected_run_id=raw_run,
|
|
)
|
|
return
|
|
if raw_run is None:
|
|
raise ProjectPolicyError("claimed task has no canonical run ID")
|
|
database_run_id = raw_run
|
|
payload = pool.assignment_payload(kanban_db, connection, task, board)
|
|
finally:
|
|
connection.close()
|
|
run_id = str(database_run_id)
|
|
expected_ordinal = _resume_ordinal(payload)
|
|
if expected_ordinal is not None and expected_ordinal != ordinal:
|
|
raise ProtocolError("publication retry was not assigned to its source ordinal")
|
|
if (board, task_id, run_id) in known:
|
|
return
|
|
added = pool.store.add(
|
|
{
|
|
"board": board, "task_id": task_id, "run_id": run_id,
|
|
"worker_ordinal": ordinal, "attempt": 1,
|
|
},
|
|
payload,
|
|
)
|
|
if added:
|
|
_issue_retry({
|
|
"board": board, "task_id": task_id, "run_id": run_id, "payload": payload,
|
|
})
|
|
known.add((board, task_id, run_id))
|
|
_settled(f"{board}/{task_id} dispatch")
|
|
except (ProtocolError, *RECOVERABLE_BOARD_ERRORS) as error:
|
|
# Coordinator-side storage/identity conditions. Never convert these into
|
|
# a Kanban capability block; reconcile() adopts the claim on a later pass.
|
|
_defer(f"{board}/{task_id} dispatch", error)
|
|
except Exception as error: # noqa: BLE001 - a real capability failure of this run
|
|
_block_exact(
|
|
kanban_db, board, task_id, database_run_id,
|
|
"Distributed assignment preparation failed: "
|
|
f"{type(error).__name__}: {error}",
|
|
)
|
|
|
|
|
|
def dispatch(pool: Any) -> None:
|
|
"""Claim up to the free ordinal count and materialize assignments."""
|
|
from hermes_cli import kanban_db
|
|
|
|
ordinals = pool.store.available_ordinals()
|
|
if not ordinals:
|
|
return
|
|
retry_ordinals: dict[tuple[str, str], int | None] = {}
|
|
|
|
def retry_ordinal(board: str, task: Any) -> int | None:
|
|
key = (board, str(_task_value(task, "id", "") or ""))
|
|
if key not in retry_ordinals:
|
|
retry_ordinals[key] = _pending_retry_ordinal(board, task)
|
|
return retry_ordinals[key]
|
|
|
|
def eligible(board: str, task: Any) -> bool:
|
|
if not distributed_workspace_eligible(task):
|
|
return False
|
|
try:
|
|
required = retry_ordinal(board, task)
|
|
except (OSError, ValueError) as error:
|
|
_defer(f"{board}/{_task_value(task, 'id', '')} retry lookup", error)
|
|
return False
|
|
return required is None or required in ordinals
|
|
|
|
def priority(board: str, task: Any) -> int:
|
|
try:
|
|
return 0 if retry_ordinal(board, task) is not None else 1
|
|
except (OSError, ValueError) as error:
|
|
_defer(f"{board}/{_task_value(task, 'id', '')} retry lookup", error)
|
|
return 2
|
|
try:
|
|
# Only storage faults are absorbed here. An incompatible claim API must
|
|
# still surface, because silently skipping the eligibility predicate is
|
|
# how the pool would start claiming the local lane's owned workspaces.
|
|
claimed = cli_lane_dispatch.claim_ready(
|
|
set(), len(ordinals),
|
|
eligible,
|
|
claimer="execution-pool",
|
|
priority=priority,
|
|
)
|
|
except RECOVERABLE_BOARD_ERRORS as error:
|
|
_defer("claim-ready", error)
|
|
return
|
|
_settled("claim-ready")
|
|
known = pool.store.known_runs()
|
|
assigned: set[int] = set()
|
|
ordered = sorted(
|
|
claimed,
|
|
key=lambda item: 0 if retry_ordinals.get(item) is not None else 1,
|
|
)
|
|
for board, task_id in ordered:
|
|
ordinal = retry_ordinals.get((board, task_id))
|
|
if ordinal is None:
|
|
ordinal = next((value for value in ordinals if value not in assigned), None)
|
|
if ordinal is None or ordinal in assigned:
|
|
_defer(f"{board}/{task_id} dispatch", ProtocolError("no safe worker ordinal is available"))
|
|
continue
|
|
_materialize(pool, kanban_db, board, task_id, ordinal, known)
|
|
assigned.add(ordinal)
|