496 lines
18 KiB
Python
496 lines
18 KiB
Python
"""Patch exact-run completion, provenance, and reclaim transactions."""
|
|
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from hermes_execution_patch_support import replace_once
|
|
|
|
|
|
|
|
source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes"))
|
|
db_path = source_root / "hermes_cli/kanban_db.py"
|
|
db = db_path.read_text(encoding="utf-8")
|
|
|
|
complete_signature_before = '''def complete_task(
|
|
conn: sqlite3.Connection,
|
|
task_id: str,
|
|
*,
|
|
result: Optional[str] = None,
|
|
summary: Optional[str] = None,
|
|
metadata: Optional[dict] = None,
|
|
created_cards: Optional[Iterable[str]] = None,
|
|
expected_run_id: Optional[int] = None,
|
|
) -> bool:
|
|
'''
|
|
complete_signature_after = '''def complete_task(
|
|
conn: sqlite3.Connection,
|
|
task_id: str,
|
|
*,
|
|
result: Optional[str] = None,
|
|
summary: Optional[str] = None,
|
|
metadata: Optional[dict] = None,
|
|
created_cards: Optional[Iterable[str]] = None,
|
|
expected_run_id: Optional[int] = None,
|
|
replay_ended_run_id: Optional[int] = None,
|
|
) -> bool:
|
|
'''
|
|
db = replace_once(
|
|
db,
|
|
complete_signature_before,
|
|
complete_signature_after,
|
|
"exact ended-run completion signature",
|
|
)
|
|
|
|
complete_branch_before = ''' with write_txn(conn):
|
|
if expected_run_id is None:
|
|
cur = conn.execute(
|
|
"""
|
|
UPDATE tasks
|
|
SET status = 'done',
|
|
result = ?,
|
|
completed_at = ?,
|
|
claim_lock = NULL,
|
|
claim_expires= NULL,
|
|
worker_pid = NULL,
|
|
block_kind = NULL,
|
|
block_recurrences = 0
|
|
WHERE id = ?
|
|
AND status IN ('running', 'ready', 'blocked', 'scheduled')
|
|
""",
|
|
(result, now, task_id),
|
|
)
|
|
else:
|
|
'''
|
|
complete_branch_after = ''' # Journal recovery may arrive after a legacy error path ended the
|
|
# accepted run. Verify that it is still the latest run in this same
|
|
# transaction; a replacement attempt must make the replay fail closed.
|
|
with write_txn(conn):
|
|
replayed_run_id = None
|
|
if expected_run_id is not None and replay_ended_run_id is not None:
|
|
raise ValueError("expected_run_id and replay_ended_run_id are mutually exclusive")
|
|
if replay_ended_run_id is not None:
|
|
replayed_run_id = int(replay_ended_run_id)
|
|
# Replay is permitted only while the exact run-ending transition
|
|
# remains the latest status mutation. Comments and diagnostics do
|
|
# not invalidate durable accepted work, but an unblock, explicit
|
|
# triage specification, decomposition, reclaim, or newer attempt
|
|
# does. BEGIN IMMEDIATE keeps this lineage check and the guarded
|
|
# task/run updates in one transaction.
|
|
last_transition = conn.execute(
|
|
"""
|
|
SELECT kind, run_id
|
|
FROM task_events
|
|
WHERE task_id = ?
|
|
AND kind IN (
|
|
'archived', 'block_loop_detected', 'blocked', 'claimed',
|
|
'completed', 'decomposed', 'dependency_wait', 'gave_up',
|
|
'promoted', 'promoted_manual', 'reclaimed', 'scheduled',
|
|
'specified', 'stale', 'timed_out', 'unblocked'
|
|
)
|
|
ORDER BY id DESC
|
|
LIMIT 1
|
|
""",
|
|
(task_id,),
|
|
).fetchone()
|
|
if (
|
|
last_transition is None
|
|
or last_transition["run_id"] != replayed_run_id
|
|
or last_transition["kind"] not in {
|
|
"block_loop_detected",
|
|
"blocked",
|
|
"reclaimed",
|
|
"stale",
|
|
"timed_out",
|
|
"gave_up",
|
|
}
|
|
):
|
|
return False
|
|
cur = conn.execute(
|
|
"""
|
|
UPDATE tasks
|
|
SET status = 'done',
|
|
result = ?,
|
|
completed_at = ?,
|
|
claim_lock = NULL,
|
|
claim_expires= NULL,
|
|
worker_pid = NULL,
|
|
block_kind = NULL,
|
|
block_recurrences = 0
|
|
WHERE id = ?
|
|
AND status IN ('ready', 'blocked', 'triage')
|
|
AND current_run_id IS NULL
|
|
AND ? = (
|
|
SELECT id FROM task_runs
|
|
WHERE task_id = ?
|
|
ORDER BY id DESC
|
|
LIMIT 1
|
|
)
|
|
AND EXISTS (
|
|
SELECT 1 FROM task_runs
|
|
WHERE id = ?
|
|
AND task_id = ?
|
|
AND ended_at IS NOT NULL
|
|
)
|
|
""",
|
|
(
|
|
result,
|
|
now,
|
|
task_id,
|
|
replayed_run_id,
|
|
task_id,
|
|
replayed_run_id,
|
|
task_id,
|
|
),
|
|
)
|
|
elif expected_run_id is None:
|
|
cur = conn.execute(
|
|
"""
|
|
UPDATE tasks
|
|
SET status = 'done',
|
|
result = ?,
|
|
completed_at = ?,
|
|
claim_lock = NULL,
|
|
claim_expires= NULL,
|
|
worker_pid = NULL,
|
|
block_kind = NULL,
|
|
block_recurrences = 0
|
|
WHERE id = ?
|
|
AND status IN ('running', 'ready', 'blocked', 'scheduled')
|
|
""",
|
|
(result, now, task_id),
|
|
)
|
|
else:
|
|
'''
|
|
db = replace_once(
|
|
db,
|
|
complete_branch_before,
|
|
complete_branch_after,
|
|
"transactional exact ended-run completion",
|
|
)
|
|
|
|
run_completion_before = ''' run_id = _end_run(
|
|
conn, task_id,
|
|
outcome="completed", status="done",
|
|
summary=summary if summary is not None else result,
|
|
metadata=metadata,
|
|
)
|
|
# If complete_task was called on a never-claimed task (ready or
|
|
# blocked → done with no run in flight), synthesize a
|
|
# zero-duration run so the handoff fields are persisted in
|
|
# attempt history instead of silently lost.
|
|
if run_id is None and (summary or metadata or result):
|
|
run_id = _synthesize_ended_run(
|
|
conn, task_id,
|
|
outcome="completed",
|
|
summary=summary if summary is not None else result,
|
|
metadata=metadata,
|
|
)
|
|
'''
|
|
run_completion_after = ''' if replayed_run_id is None:
|
|
run_id = _end_run(
|
|
conn, task_id,
|
|
outcome="completed", status="done",
|
|
summary=summary if summary is not None else result,
|
|
metadata=metadata,
|
|
)
|
|
# If complete_task was called on a never-claimed task (ready or
|
|
# blocked → done with no run in flight), synthesize a
|
|
# zero-duration run so the handoff fields are persisted in
|
|
# attempt history instead of silently lost.
|
|
if run_id is None and (summary or metadata or result):
|
|
run_id = _synthesize_ended_run(
|
|
conn, task_id,
|
|
outcome="completed",
|
|
summary=summary if summary is not None else result,
|
|
metadata=metadata,
|
|
)
|
|
else:
|
|
run_id = replayed_run_id
|
|
replayed = conn.execute(
|
|
"""
|
|
UPDATE task_runs
|
|
SET status = 'done',
|
|
outcome = 'completed',
|
|
summary = ?,
|
|
error = NULL,
|
|
metadata = ?,
|
|
claim_lock = NULL,
|
|
claim_expires = NULL,
|
|
worker_pid = NULL
|
|
WHERE id = ?
|
|
AND task_id = ?
|
|
AND ended_at IS NOT NULL
|
|
""",
|
|
(
|
|
summary if summary is not None else result,
|
|
json.dumps(metadata, ensure_ascii=False) if metadata else None,
|
|
run_id,
|
|
task_id,
|
|
),
|
|
)
|
|
if replayed.rowcount != 1:
|
|
raise RuntimeError("exact ended-run completion lost its run identity")
|
|
conn.execute(
|
|
"UPDATE tasks SET completed_run_id = ? WHERE id = ?",
|
|
(run_id, task_id),
|
|
)
|
|
'''
|
|
db = replace_once(
|
|
db,
|
|
run_completion_before,
|
|
run_completion_after,
|
|
"exact completion run provenance",
|
|
)
|
|
|
|
reclaim_before = '''def reclaim_task(
|
|
conn: sqlite3.Connection,
|
|
task_id: str,
|
|
*,
|
|
reason: Optional[str] = None,
|
|
signal_fn=None,
|
|
) -> bool:
|
|
"""Operator-driven reclaim: release the claim and reset to ``ready``.
|
|
|
|
Unlike :func:`release_stale_claims` which only acts on tasks whose
|
|
``claim_expires`` has passed, this function reclaims immediately
|
|
regardless of TTL. Intended for the dashboard/CLI recovery flow
|
|
when an operator wants to abort a running worker without waiting
|
|
for the TTL to expire (e.g. after seeing a hallucination warning).
|
|
|
|
Returns True if a reclaim happened, False if the task isn't in a
|
|
reclaimable state (not running, or doesn't exist).
|
|
"""
|
|
row = conn.execute(
|
|
"SELECT status, claim_lock, worker_pid FROM tasks WHERE id = ?",
|
|
(task_id,),
|
|
).fetchone()
|
|
if not row:
|
|
return False
|
|
if row["status"] != "running" and row["claim_lock"] is None:
|
|
# Nothing to reclaim — already ready / blocked / done.
|
|
return False
|
|
prev_lock = row["claim_lock"]
|
|
termination = _terminate_reclaimed_worker(
|
|
row["worker_pid"], prev_lock, signal_fn=signal_fn,
|
|
)
|
|
with write_txn(conn):
|
|
cur = conn.execute(
|
|
"UPDATE tasks SET status = 'ready', claim_lock = NULL, "
|
|
"claim_expires = NULL, worker_pid = NULL "
|
|
"WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
|
|
"AND claim_lock IS ?",
|
|
(task_id, prev_lock),
|
|
)
|
|
if cur.rowcount != 1:
|
|
return False
|
|
run_id = _end_run(
|
|
conn, task_id,
|
|
outcome="reclaimed", status="reclaimed",
|
|
error=(
|
|
f"manual_reclaim: {reason}" if reason
|
|
else f"manual_reclaim lock={prev_lock}"
|
|
),
|
|
metadata=termination,
|
|
)
|
|
payload = {
|
|
"manual": True,
|
|
"reason": reason,
|
|
"prev_lock": prev_lock,
|
|
}
|
|
payload.update(termination)
|
|
_append_event(
|
|
conn, task_id, "reclaimed",
|
|
payload,
|
|
run_id=run_id,
|
|
)
|
|
# Operator intervention — they've looked at the task, so the
|
|
# consecutive-failures counter is now stale. Give the next retry
|
|
# a fresh budget. (_clear_failure_counter opens its own write_txn,
|
|
# so it runs after the enclosing one commits.)
|
|
_clear_failure_counter(conn, task_id)
|
|
return True
|
|
'''
|
|
reclaim_after = '''def reclaim_task(
|
|
conn: sqlite3.Connection,
|
|
task_id: str,
|
|
*,
|
|
reason: Optional[str] = None,
|
|
signal_fn=None,
|
|
expected_run_id: Optional[int] = None,
|
|
expected_run_liveness_before: Optional[int] = None,
|
|
) -> bool:
|
|
"""Operator-driven reclaim: release the claim and reset to ``ready``.
|
|
|
|
Unlike :func:`release_stale_claims` which only acts on tasks whose
|
|
``claim_expires`` has passed, this function reclaims immediately
|
|
regardless of TTL. Intended for the dashboard/CLI recovery flow
|
|
when an operator wants to abort a running worker without waiting
|
|
for the TTL to expire (e.g. after seeing a hallucination warning).
|
|
|
|
When ``expected_run_id`` is supplied, the run identity is checked after
|
|
``BEGIN IMMEDIATE`` and included in the guarded update. A replacement run
|
|
is therefore neither signalled nor reclaimed by stale recovery evidence.
|
|
|
|
``expected_run_liveness_before`` adds an atomic heartbeat-age guard for
|
|
watchdog callers. The current run's own heartbeat (or its start time before
|
|
the first heartbeat) must still predate that cutoff inside the same write
|
|
transaction. A heartbeat racing the watchdog therefore wins safely.
|
|
|
|
Returns True if a reclaim happened, False if the task isn't in a
|
|
reclaimable state (not running, or doesn't exist), or its run changed.
|
|
"""
|
|
with write_txn(conn):
|
|
row = conn.execute(
|
|
"SELECT t.status, t.claim_lock, t.worker_pid, t.current_run_id, "
|
|
"COALESCE(r.last_heartbeat_at, r.started_at) AS run_liveness_at "
|
|
"FROM tasks AS t LEFT JOIN task_runs AS r "
|
|
"ON r.id = t.current_run_id WHERE t.id = ?",
|
|
(task_id,),
|
|
).fetchone()
|
|
if not row:
|
|
return False
|
|
if expected_run_id is not None and row["current_run_id"] != expected_run_id:
|
|
return False
|
|
if expected_run_liveness_before is not None:
|
|
liveness_at = row["run_liveness_at"]
|
|
if (
|
|
liveness_at is None
|
|
or int(liveness_at) >= int(expected_run_liveness_before)
|
|
):
|
|
return False
|
|
if row["status"] != "running" and row["claim_lock"] is None:
|
|
# Nothing to reclaim — already ready / blocked / done.
|
|
return False
|
|
prev_lock = row["claim_lock"]
|
|
termination = _terminate_reclaimed_worker(
|
|
row["worker_pid"], prev_lock, signal_fn=signal_fn,
|
|
)
|
|
if expected_run_id is None:
|
|
cur = conn.execute(
|
|
"UPDATE tasks SET status = 'ready', claim_lock = NULL, "
|
|
"claim_expires = NULL, worker_pid = NULL "
|
|
"WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
|
|
"AND claim_lock IS ?",
|
|
(task_id, prev_lock),
|
|
)
|
|
else:
|
|
cur = conn.execute(
|
|
"UPDATE tasks SET status = 'ready', claim_lock = NULL, "
|
|
"claim_expires = NULL, worker_pid = NULL "
|
|
"WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
|
|
"AND claim_lock IS ? AND current_run_id = ?",
|
|
(task_id, prev_lock, int(expected_run_id)),
|
|
)
|
|
if cur.rowcount != 1:
|
|
return False
|
|
run_id = _end_run(
|
|
conn, task_id,
|
|
outcome="reclaimed", status="reclaimed",
|
|
error=(
|
|
f"manual_reclaim: {reason}" if reason
|
|
else f"manual_reclaim lock={prev_lock}"
|
|
),
|
|
metadata=termination,
|
|
)
|
|
payload = {
|
|
"manual": True,
|
|
"reason": reason,
|
|
"prev_lock": prev_lock,
|
|
}
|
|
payload.update(termination)
|
|
_append_event(
|
|
conn, task_id, "reclaimed",
|
|
payload,
|
|
run_id=run_id,
|
|
)
|
|
# Operator intervention — they've looked at the task, so the
|
|
# consecutive-failures counter is now stale. Give the next retry
|
|
# a fresh budget. (_clear_failure_counter opens its own write_txn,
|
|
# so it runs after the enclosing one commits.)
|
|
_clear_failure_counter(conn, task_id)
|
|
return True
|
|
'''
|
|
db = replace_once(
|
|
db,
|
|
reclaim_before,
|
|
reclaim_after,
|
|
"transactional exact-run reclaim",
|
|
)
|
|
|
|
task_field_before = ''' current_run_id: Optional[int] = None
|
|
workflow_template_id: Optional[str] = None
|
|
'''
|
|
task_field_after = ''' current_run_id: Optional[int] = None
|
|
# Durable provenance for the run that authoritatively completed this task.
|
|
completed_run_id: Optional[int] = None
|
|
workflow_template_id: Optional[str] = None
|
|
'''
|
|
db = replace_once(
|
|
db,
|
|
task_field_before,
|
|
task_field_after,
|
|
"completed-run task field",
|
|
)
|
|
|
|
task_row_before = ''' current_run_id=(
|
|
row["current_run_id"] if "current_run_id" in keys else None
|
|
),
|
|
workflow_template_id=(
|
|
'''
|
|
task_row_after = ''' current_run_id=(
|
|
row["current_run_id"] if "current_run_id" in keys else None
|
|
),
|
|
completed_run_id=(
|
|
row["completed_run_id"] if "completed_run_id" in keys else None
|
|
),
|
|
workflow_template_id=(
|
|
'''
|
|
db = replace_once(
|
|
db,
|
|
task_row_before,
|
|
task_row_after,
|
|
"completed-run row mapping",
|
|
)
|
|
|
|
task_schema_before = ''' -- run is in-flight). Denormalised for cheap reads.
|
|
current_run_id INTEGER,
|
|
-- Forward-compat for v2 workflow routing. In v1 the kernel writes
|
|
'''
|
|
task_schema_after = ''' -- run is in-flight). Denormalised for cheap reads.
|
|
current_run_id INTEGER,
|
|
-- Immutable provenance for the task result's authoritative run.
|
|
completed_run_id INTEGER,
|
|
-- Forward-compat for v2 workflow routing. In v1 the kernel writes
|
|
'''
|
|
db = replace_once(
|
|
db,
|
|
task_schema_before,
|
|
task_schema_after,
|
|
"completed-run schema",
|
|
)
|
|
|
|
task_migration_before = ''' if "current_run_id" not in cols:
|
|
_add_column_if_missing(
|
|
conn, "tasks", "current_run_id", "current_run_id INTEGER"
|
|
)
|
|
if "workflow_template_id" not in cols:
|
|
'''
|
|
task_migration_after = ''' if "current_run_id" not in cols:
|
|
_add_column_if_missing(
|
|
conn, "tasks", "current_run_id", "current_run_id INTEGER"
|
|
)
|
|
if "completed_run_id" not in cols:
|
|
_add_column_if_missing(
|
|
conn, "tasks", "completed_run_id", "completed_run_id INTEGER"
|
|
)
|
|
if "workflow_template_id" not in cols:
|
|
'''
|
|
db = replace_once(
|
|
db,
|
|
task_migration_before,
|
|
task_migration_after,
|
|
"completed-run migration",
|
|
)
|
|
|
|
db_path.write_text(db, encoding="utf-8")
|