"""Patch exact-run completion, provenance, and reclaim transactions.""" import os from pathlib import Path from hermes_execution_patch_support import replace_once source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes")) db_path = source_root / "hermes_cli/kanban_db.py" db = db_path.read_text(encoding="utf-8") complete_signature_before = '''def complete_task( conn: sqlite3.Connection, task_id: str, *, result: Optional[str] = None, summary: Optional[str] = None, metadata: Optional[dict] = None, created_cards: Optional[Iterable[str]] = None, expected_run_id: Optional[int] = None, ) -> bool: ''' complete_signature_after = '''def complete_task( conn: sqlite3.Connection, task_id: str, *, result: Optional[str] = None, summary: Optional[str] = None, metadata: Optional[dict] = None, created_cards: Optional[Iterable[str]] = None, expected_run_id: Optional[int] = None, replay_ended_run_id: Optional[int] = None, ) -> bool: ''' db = replace_once( db, complete_signature_before, complete_signature_after, "exact ended-run completion signature", ) complete_branch_before = ''' with write_txn(conn): if expected_run_id is None: cur = conn.execute( """ UPDATE tasks SET status = 'done', result = ?, completed_at = ?, claim_lock = NULL, claim_expires= NULL, worker_pid = NULL, block_kind = NULL, block_recurrences = 0 WHERE id = ? AND status IN ('running', 'ready', 'blocked', 'scheduled') """, (result, now, task_id), ) else: ''' complete_branch_after = ''' # Journal recovery may arrive after a legacy error path ended the # accepted run. Verify that it is still the latest run in this same # transaction; a replacement attempt must make the replay fail closed. with write_txn(conn): replayed_run_id = None if expected_run_id is not None and replay_ended_run_id is not None: raise ValueError("expected_run_id and replay_ended_run_id are mutually exclusive") if replay_ended_run_id is not None: replayed_run_id = int(replay_ended_run_id) # Replay is permitted only while the exact run-ending transition # remains the latest status mutation. Comments and diagnostics do # not invalidate durable accepted work, but an unblock, explicit # triage specification, decomposition, reclaim, or newer attempt # does. BEGIN IMMEDIATE keeps this lineage check and the guarded # task/run updates in one transaction. last_transition = conn.execute( """ SELECT kind, run_id FROM task_events WHERE task_id = ? AND kind IN ( 'archived', 'block_loop_detected', 'blocked', 'claimed', 'completed', 'decomposed', 'dependency_wait', 'gave_up', 'promoted', 'promoted_manual', 'reclaimed', 'scheduled', 'specified', 'stale', 'timed_out', 'unblocked' ) ORDER BY id DESC LIMIT 1 """, (task_id,), ).fetchone() if ( last_transition is None or last_transition["run_id"] != replayed_run_id or last_transition["kind"] not in { "block_loop_detected", "blocked", "reclaimed", "stale", "timed_out", "gave_up", } ): return False cur = conn.execute( """ UPDATE tasks SET status = 'done', result = ?, completed_at = ?, claim_lock = NULL, claim_expires= NULL, worker_pid = NULL, block_kind = NULL, block_recurrences = 0 WHERE id = ? AND status IN ('ready', 'blocked', 'triage') AND current_run_id IS NULL AND ? = ( SELECT id FROM task_runs WHERE task_id = ? ORDER BY id DESC LIMIT 1 ) AND EXISTS ( SELECT 1 FROM task_runs WHERE id = ? AND task_id = ? AND ended_at IS NOT NULL ) """, ( result, now, task_id, replayed_run_id, task_id, replayed_run_id, task_id, ), ) elif expected_run_id is None: cur = conn.execute( """ UPDATE tasks SET status = 'done', result = ?, completed_at = ?, claim_lock = NULL, claim_expires= NULL, worker_pid = NULL, block_kind = NULL, block_recurrences = 0 WHERE id = ? AND status IN ('running', 'ready', 'blocked', 'scheduled') """, (result, now, task_id), ) else: ''' db = replace_once( db, complete_branch_before, complete_branch_after, "transactional exact ended-run completion", ) run_completion_before = ''' run_id = _end_run( conn, task_id, outcome="completed", status="done", summary=summary if summary is not None else result, metadata=metadata, ) # If complete_task was called on a never-claimed task (ready or # blocked → done with no run in flight), synthesize a # zero-duration run so the handoff fields are persisted in # attempt history instead of silently lost. if run_id is None and (summary or metadata or result): run_id = _synthesize_ended_run( conn, task_id, outcome="completed", summary=summary if summary is not None else result, metadata=metadata, ) ''' run_completion_after = ''' if replayed_run_id is None: run_id = _end_run( conn, task_id, outcome="completed", status="done", summary=summary if summary is not None else result, metadata=metadata, ) # If complete_task was called on a never-claimed task (ready or # blocked → done with no run in flight), synthesize a # zero-duration run so the handoff fields are persisted in # attempt history instead of silently lost. if run_id is None and (summary or metadata or result): run_id = _synthesize_ended_run( conn, task_id, outcome="completed", summary=summary if summary is not None else result, metadata=metadata, ) else: run_id = replayed_run_id replayed = conn.execute( """ UPDATE task_runs SET status = 'done', outcome = 'completed', summary = ?, error = NULL, metadata = ?, claim_lock = NULL, claim_expires = NULL, worker_pid = NULL WHERE id = ? AND task_id = ? AND ended_at IS NOT NULL """, ( summary if summary is not None else result, json.dumps(metadata, ensure_ascii=False) if metadata else None, run_id, task_id, ), ) if replayed.rowcount != 1: raise RuntimeError("exact ended-run completion lost its run identity") conn.execute( "UPDATE tasks SET completed_run_id = ? WHERE id = ?", (run_id, task_id), ) ''' db = replace_once( db, run_completion_before, run_completion_after, "exact completion run provenance", ) reclaim_before = '''def reclaim_task( conn: sqlite3.Connection, task_id: str, *, reason: Optional[str] = None, signal_fn=None, ) -> bool: """Operator-driven reclaim: release the claim and reset to ``ready``. Unlike :func:`release_stale_claims` which only acts on tasks whose ``claim_expires`` has passed, this function reclaims immediately regardless of TTL. Intended for the dashboard/CLI recovery flow when an operator wants to abort a running worker without waiting for the TTL to expire (e.g. after seeing a hallucination warning). Returns True if a reclaim happened, False if the task isn't in a reclaimable state (not running, or doesn't exist). """ row = conn.execute( "SELECT status, claim_lock, worker_pid FROM tasks WHERE id = ?", (task_id,), ).fetchone() if not row: return False if row["status"] != "running" and row["claim_lock"] is None: # Nothing to reclaim — already ready / blocked / done. return False prev_lock = row["claim_lock"] termination = _terminate_reclaimed_worker( row["worker_pid"], prev_lock, signal_fn=signal_fn, ) with write_txn(conn): cur = conn.execute( "UPDATE tasks SET status = 'ready', claim_lock = NULL, " "claim_expires = NULL, worker_pid = NULL " "WHERE id = ? AND status IN ('running', 'ready', 'blocked') " "AND claim_lock IS ?", (task_id, prev_lock), ) if cur.rowcount != 1: return False run_id = _end_run( conn, task_id, outcome="reclaimed", status="reclaimed", error=( f"manual_reclaim: {reason}" if reason else f"manual_reclaim lock={prev_lock}" ), metadata=termination, ) payload = { "manual": True, "reason": reason, "prev_lock": prev_lock, } payload.update(termination) _append_event( conn, task_id, "reclaimed", payload, run_id=run_id, ) # Operator intervention — they've looked at the task, so the # consecutive-failures counter is now stale. Give the next retry # a fresh budget. (_clear_failure_counter opens its own write_txn, # so it runs after the enclosing one commits.) _clear_failure_counter(conn, task_id) return True ''' reclaim_after = '''def reclaim_task( conn: sqlite3.Connection, task_id: str, *, reason: Optional[str] = None, signal_fn=None, expected_run_id: Optional[int] = None, ) -> bool: """Operator-driven reclaim: release the claim and reset to ``ready``. Unlike :func:`release_stale_claims` which only acts on tasks whose ``claim_expires`` has passed, this function reclaims immediately regardless of TTL. Intended for the dashboard/CLI recovery flow when an operator wants to abort a running worker without waiting for the TTL to expire (e.g. after seeing a hallucination warning). When ``expected_run_id`` is supplied, the run identity is checked after ``BEGIN IMMEDIATE`` and included in the guarded update. A replacement run is therefore neither signalled nor reclaimed by stale recovery evidence. Returns True if a reclaim happened, False if the task isn't in a reclaimable state (not running, or doesn't exist), or its run changed. """ with write_txn(conn): row = conn.execute( "SELECT status, claim_lock, worker_pid, current_run_id " "FROM tasks WHERE id = ?", (task_id,), ).fetchone() if not row: return False if expected_run_id is not None and row["current_run_id"] != expected_run_id: return False if row["status"] != "running" and row["claim_lock"] is None: # Nothing to reclaim — already ready / blocked / done. return False prev_lock = row["claim_lock"] termination = _terminate_reclaimed_worker( row["worker_pid"], prev_lock, signal_fn=signal_fn, ) if expected_run_id is None: cur = conn.execute( "UPDATE tasks SET status = 'ready', claim_lock = NULL, " "claim_expires = NULL, worker_pid = NULL " "WHERE id = ? AND status IN ('running', 'ready', 'blocked') " "AND claim_lock IS ?", (task_id, prev_lock), ) else: cur = conn.execute( "UPDATE tasks SET status = 'ready', claim_lock = NULL, " "claim_expires = NULL, worker_pid = NULL " "WHERE id = ? AND status IN ('running', 'ready', 'blocked') " "AND claim_lock IS ? AND current_run_id = ?", (task_id, prev_lock, int(expected_run_id)), ) if cur.rowcount != 1: return False run_id = _end_run( conn, task_id, outcome="reclaimed", status="reclaimed", error=( f"manual_reclaim: {reason}" if reason else f"manual_reclaim lock={prev_lock}" ), metadata=termination, ) payload = { "manual": True, "reason": reason, "prev_lock": prev_lock, } payload.update(termination) _append_event( conn, task_id, "reclaimed", payload, run_id=run_id, ) # Operator intervention — they've looked at the task, so the # consecutive-failures counter is now stale. Give the next retry # a fresh budget. (_clear_failure_counter opens its own write_txn, # so it runs after the enclosing one commits.) _clear_failure_counter(conn, task_id) return True ''' db = replace_once( db, reclaim_before, reclaim_after, "transactional exact-run reclaim", ) task_field_before = ''' current_run_id: Optional[int] = None workflow_template_id: Optional[str] = None ''' task_field_after = ''' current_run_id: Optional[int] = None # Durable provenance for the run that authoritatively completed this task. completed_run_id: Optional[int] = None workflow_template_id: Optional[str] = None ''' db = replace_once( db, task_field_before, task_field_after, "completed-run task field", ) task_row_before = ''' current_run_id=( row["current_run_id"] if "current_run_id" in keys else None ), workflow_template_id=( ''' task_row_after = ''' current_run_id=( row["current_run_id"] if "current_run_id" in keys else None ), completed_run_id=( row["completed_run_id"] if "completed_run_id" in keys else None ), workflow_template_id=( ''' db = replace_once( db, task_row_before, task_row_after, "completed-run row mapping", ) task_schema_before = ''' -- run is in-flight). Denormalised for cheap reads. current_run_id INTEGER, -- Forward-compat for v2 workflow routing. In v1 the kernel writes ''' task_schema_after = ''' -- run is in-flight). Denormalised for cheap reads. current_run_id INTEGER, -- Immutable provenance for the task result's authoritative run. completed_run_id INTEGER, -- Forward-compat for v2 workflow routing. In v1 the kernel writes ''' db = replace_once( db, task_schema_before, task_schema_after, "completed-run schema", ) task_migration_before = ''' if "current_run_id" not in cols: _add_column_if_missing( conn, "tasks", "current_run_id", "current_run_id INTEGER" ) if "workflow_template_id" not in cols: ''' task_migration_after = ''' if "current_run_id" not in cols: _add_column_if_missing( conn, "tasks", "current_run_id", "current_run_id INTEGER" ) if "completed_run_id" not in cols: _add_column_if_missing( conn, "tasks", "completed_run_id", "completed_run_id INTEGER" ) if "workflow_template_id" not in cols: ''' db = replace_once( db, task_migration_before, task_migration_after, "completed-run migration", ) db_path.write_text(db, encoding="utf-8")