430 lines
18 KiB
Python
430 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Single-claim provider execution and durable Kanban result handoff."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import json
|
|
import sqlite3
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import cli_lane_goal
|
|
from cli_lane_board import _board_call, _resolve_workspace, _task_context, _task_value
|
|
from cli_lane_config import (
|
|
DEFAULT_MAX_RUNTIME,
|
|
TerminalFinalizationPending,
|
|
)
|
|
from cli_lane_files import (
|
|
_terminal_identity,
|
|
_terminal_path,
|
|
atomic_json,
|
|
load_json,
|
|
state_path,
|
|
)
|
|
from cli_lane_finalization import _finalize_terminal_record, _recover_exact_run
|
|
from cli_lane_prompt import build_prompt, git_handoff, workspace_artifacts
|
|
from cli_lane_provider import run_provider
|
|
from cli_lane_records import _persist_candidate, _write_terminal_record
|
|
from cli_lane_recovery import _has_pending_finalization
|
|
from cli_lane_routing import fresh_unavailable_provider, select_route
|
|
|
|
|
|
def execute_claim(board: str, task_id: str) -> None:
|
|
"""Execute one already-claimed task and commit its outcome to Kanban."""
|
|
from hermes_cli import kanban_db
|
|
|
|
run_id = None
|
|
with kanban_db.scoped_current_board(board):
|
|
conn = kanban_db.connect(board=board)
|
|
try:
|
|
task = kanban_db.get_task(conn, task_id)
|
|
if task is None:
|
|
return
|
|
run_id = _task_value(task, "current_run_id", None)
|
|
assignee = str(_task_value(task, "assignee", "cli-auto") or "cli-auto")
|
|
state_file = state_path(board, task_id)
|
|
state = load_json(state_file)
|
|
state.update(
|
|
{
|
|
"board": board,
|
|
"task_id": task_id,
|
|
"run_id": run_id,
|
|
"assignee": assignee,
|
|
}
|
|
)
|
|
atomic_json(state_file, state)
|
|
log_path = Path(kanban_db.worker_log_path(task_id, board=board))
|
|
workspace = _resolve_workspace(kanban_db, conn, task, board)
|
|
kanban_db.set_workspace_path(conn, task_id, str(workspace))
|
|
context = _task_context(kanban_db, conn, task_id)
|
|
except Exception as error:
|
|
failure_reason = (
|
|
f"Direct CLI lane preparation failed: {type(error).__name__}: {error}"
|
|
)
|
|
conn.close()
|
|
_board_call(
|
|
kanban_db,
|
|
board,
|
|
lambda fresh: kanban_db.block_task(
|
|
fresh,
|
|
task_id,
|
|
reason=failure_reason,
|
|
kind="capability",
|
|
expected_run_id=run_id,
|
|
),
|
|
)
|
|
return
|
|
finally:
|
|
conn.close()
|
|
|
|
def heartbeat(note: str) -> bool:
|
|
try:
|
|
return bool(
|
|
_board_call(
|
|
kanban_db,
|
|
board,
|
|
lambda fresh: kanban_db.heartbeat_worker(
|
|
fresh,
|
|
task_id,
|
|
note=note,
|
|
expected_run_id=run_id,
|
|
),
|
|
)
|
|
)
|
|
except (OSError, sqlite3.Error):
|
|
# The claim TTL is deliberately long. Keep the expensive
|
|
# provider process alive during a transient volume stall and
|
|
# retry on the next heartbeat instead of losing its work.
|
|
return True
|
|
|
|
def comment(body: str) -> None:
|
|
with contextlib.suppress(OSError, sqlite3.Error):
|
|
_board_call(
|
|
kanban_db,
|
|
board,
|
|
lambda fresh: kanban_db.add_comment(
|
|
fresh,
|
|
task_id,
|
|
"cli-lane-runner",
|
|
body,
|
|
),
|
|
)
|
|
# Route state is also written to the durable lane-state file;
|
|
# a later heartbeat or terminal result remains authoritative.
|
|
|
|
try:
|
|
previous_route = state.get("current_route")
|
|
excluded_provider = (
|
|
fresh_unavailable_provider() if assignee == "cli-auto" else None
|
|
)
|
|
unavailable_provider = excluded_provider
|
|
route = select_route(
|
|
context,
|
|
assignee,
|
|
exclude_provider=excluded_provider,
|
|
exclude_reason="is unavailable according to fresh native health"
|
|
if excluded_provider
|
|
else None,
|
|
)
|
|
if excluded_provider:
|
|
comment(
|
|
f"Provider health guard excluded {excluded_provider} before automatic routing.",
|
|
)
|
|
comment(
|
|
f"CLI route: {route.provider}/{route.model} at {route.effort}; classifier={route.classifier}; {route.reason}",
|
|
)
|
|
resume_handoff = ""
|
|
if (
|
|
isinstance(previous_route, dict)
|
|
and previous_route.get("provider")
|
|
and previous_route.get("provider") != route.provider
|
|
):
|
|
try:
|
|
previous_output = log_path.read_text(encoding="utf-8")[-10000:]
|
|
except OSError:
|
|
previous_output = "Previous provider log was unavailable after restart."
|
|
resume_handoff = git_handoff(workspace, previous_output)
|
|
comment(
|
|
f"Restart-time provider change: {previous_route.get('provider')} -> {route.provider}; explicit workspace handoff attached.",
|
|
)
|
|
prompt = build_prompt(context, workspace, resume_handoff)
|
|
max_runtime = int(
|
|
_task_value(task, "max_runtime_seconds", 0) or DEFAULT_MAX_RUNTIME
|
|
)
|
|
goal_mode = bool(_task_value(task, "goal_mode", False))
|
|
goal_max_turns = max(1, int(_task_value(task, "goal_max_turns", 1) or 1))
|
|
goal_turn = max(1, int(state.get("goal_turn", 0) or 0) + 1)
|
|
deadline = time.monotonic() + max_runtime
|
|
while True:
|
|
state["goal_turn"] = goal_turn
|
|
atomic_json(state_file, state)
|
|
remaining = max(1, int(deadline - time.monotonic()))
|
|
result = run_provider(
|
|
route,
|
|
prompt,
|
|
workspace,
|
|
state,
|
|
state_file,
|
|
log_path,
|
|
heartbeat,
|
|
remaining,
|
|
)
|
|
candidate_file = None
|
|
if result.structured:
|
|
candidate_file = _persist_candidate(
|
|
state,
|
|
state_file,
|
|
dict(result.structured),
|
|
route=route,
|
|
returncode=result.returncode,
|
|
goal_turn=goal_turn,
|
|
)
|
|
if result.capacity_failure:
|
|
unavailable_provider = route.provider
|
|
retry_context = (
|
|
context
|
|
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. "
|
|
+ "Select the alternate hosted provider at an appropriate effort."
|
|
)
|
|
alternate = "claude" if route.provider == "codex" else "codex"
|
|
fallback = select_route(
|
|
retry_context,
|
|
f"cli-{alternate}-{route.effort}",
|
|
)
|
|
comment(
|
|
f"Provider fallback: {route.provider} -> {fallback.provider}; Jetson reclassified the retry boundary.",
|
|
)
|
|
result = run_provider(
|
|
fallback,
|
|
build_prompt(
|
|
context,
|
|
workspace,
|
|
git_handoff(workspace, result.output),
|
|
),
|
|
workspace,
|
|
state,
|
|
state_file,
|
|
log_path,
|
|
heartbeat,
|
|
max(1, int(deadline - time.monotonic())),
|
|
)
|
|
route = fallback
|
|
if result.structured:
|
|
candidate_file = _persist_candidate(
|
|
state,
|
|
state_file,
|
|
dict(result.structured),
|
|
route=route,
|
|
returncode=result.returncode,
|
|
goal_turn=goal_turn,
|
|
)
|
|
structured = result.structured
|
|
if structured:
|
|
structured = dict(structured)
|
|
structured["artifacts"] = workspace_artifacts(
|
|
workspace,
|
|
structured.get("artifacts"),
|
|
)
|
|
metadata = {
|
|
"executor": "direct-cli-lane",
|
|
"provider": route.provider,
|
|
"model": route.model,
|
|
"effort": route.effort,
|
|
"classifier": route.classifier,
|
|
"state_file": str(state_file),
|
|
"codex_thread_id": state.get("codex_thread_id"),
|
|
"claude_session_id": state.get("claude_session_id"),
|
|
"goal_mode": goal_mode,
|
|
"goal_turn": goal_turn,
|
|
}
|
|
if candidate_file is not None:
|
|
metadata["candidate_file"] = str(candidate_file)
|
|
if structured:
|
|
for key in (
|
|
"changed_files",
|
|
"tests_run",
|
|
"artifacts",
|
|
"findings",
|
|
"blockers",
|
|
):
|
|
value = structured.get(key)
|
|
metadata[key] = value if isinstance(value, list) else []
|
|
|
|
completion_problem = None
|
|
if structured and result.returncode == 0:
|
|
completion_problem = cli_lane_goal.unfinished_result_reason(structured)
|
|
if (
|
|
structured.get("status") == "completed"
|
|
and completion_problem is None
|
|
and goal_mode
|
|
):
|
|
heartbeat("local goal-completion judge active")
|
|
rejection_history = state.get("goal_rejections", [])
|
|
if not isinstance(rejection_history, list):
|
|
rejection_history = []
|
|
judge_context = context
|
|
if goal_turn > 1 or rejection_history:
|
|
judge_context += (
|
|
"\n\nAuthoritative Hermes goal-controller evidence: "
|
|
f"current turn {goal_turn}/{goal_max_turns}; prior rejected "
|
|
f"reports: {json.dumps(rejection_history[-5:])}."
|
|
)
|
|
accepted, judge_reason = cli_lane_goal.judge_goal_completion(
|
|
judge_context,
|
|
structured,
|
|
)
|
|
metadata["goal_judge_reason"] = judge_reason
|
|
if not accepted:
|
|
completion_problem = f"local goal judge requested continuation: {judge_reason}"
|
|
|
|
if (
|
|
structured
|
|
and structured.get("status") == "completed"
|
|
and result.returncode == 0
|
|
and completion_problem is None
|
|
):
|
|
terminal_file = _terminal_path(state_file, run_id, "pending")
|
|
metadata["terminal_record"] = str(
|
|
_terminal_path(state_file, run_id, "committed")
|
|
)
|
|
try:
|
|
terminal_file, terminal_record = _write_terminal_record(
|
|
state_file,
|
|
board=board,
|
|
task_id=task_id,
|
|
run_id=run_id,
|
|
structured=structured,
|
|
summary=str(structured.get("summary") or "Completed"),
|
|
metadata=metadata,
|
|
)
|
|
outcome = _finalize_terminal_record(
|
|
kanban_db,
|
|
terminal_file,
|
|
terminal_record,
|
|
)
|
|
except Exception as error:
|
|
identity = _terminal_identity(terminal_file)
|
|
replayable = bool(
|
|
identity is not None
|
|
and _has_pending_finalization(
|
|
identity.board,
|
|
identity.task_id,
|
|
identity.run_id,
|
|
)
|
|
)
|
|
if not replayable:
|
|
_recover_exact_run(
|
|
kanban_db,
|
|
identity,
|
|
f"persistence raised {type(error).__name__}",
|
|
)
|
|
raise TerminalFinalizationPending(
|
|
"accepted worker result remains outside the generic block path; "
|
|
f"terminal replayable={replayable}; persistence/finalization "
|
|
f"raised {type(error).__name__}"
|
|
) from error
|
|
if outcome != "committed":
|
|
raise TerminalFinalizationPending(
|
|
"accepted worker result is durably journaled but "
|
|
f"Kanban finalization is {outcome}"
|
|
)
|
|
break
|
|
|
|
can_continue = (
|
|
goal_mode
|
|
and completion_problem is not None
|
|
and goal_turn < goal_max_turns
|
|
and deadline - time.monotonic() > 30
|
|
)
|
|
if can_continue:
|
|
rejection_history = state.get("goal_rejections", [])
|
|
if not isinstance(rejection_history, list):
|
|
rejection_history = []
|
|
state["goal_rejections"] = [
|
|
*rejection_history[-4:],
|
|
completion_problem,
|
|
]
|
|
goal_turn += 1
|
|
comment(
|
|
f"Goal completion rejected; continuing turn {goal_turn}/{goal_max_turns}: {completion_problem}",
|
|
)
|
|
escalation_context = (
|
|
context
|
|
+ "\n\nThe previous worker turn missed its completion quality mark: "
|
|
+ completion_problem
|
|
+ "\nSelect a route that can finish and verify the remaining work."
|
|
)
|
|
excluded = unavailable_provider
|
|
if excluded is None and assignee == "cli-auto":
|
|
excluded = fresh_unavailable_provider()
|
|
next_route = select_route(
|
|
escalation_context,
|
|
assignee,
|
|
exclude_provider=excluded,
|
|
exclude_reason="is unavailable according to fresh native health"
|
|
if excluded
|
|
else None,
|
|
)
|
|
comment(
|
|
f"Goal route {goal_turn}/{goal_max_turns}: "
|
|
f"{next_route.provider}/{next_route.model} at {next_route.effort}; "
|
|
f"classifier={next_route.classifier}; {next_route.reason}",
|
|
)
|
|
handoff = (
|
|
git_handoff(workspace, result.output)
|
|
if next_route.provider != route.provider
|
|
else ""
|
|
)
|
|
route = next_route
|
|
prompt = build_prompt(
|
|
context,
|
|
workspace,
|
|
handoff
|
|
+ "\nGoal-loop continuation: the previous final report was rejected because "
|
|
+ completion_problem
|
|
+ ". Reinspect live state, finish the outstanding work, and return new final evidence.",
|
|
)
|
|
continue
|
|
|
|
reason = completion_problem
|
|
if reason is None and structured:
|
|
blockers = structured.get("blockers", [])
|
|
reason = "; ".join(str(item) for item in blockers)
|
|
reason = reason or str(structured.get("summary") or "")
|
|
if reason is None:
|
|
reason = result.output[-4000:]
|
|
failure_reason = reason or (
|
|
f"{route.provider} worker failed with exit {result.returncode}"
|
|
)
|
|
failure_kind = (
|
|
"transient" if result.capacity_failure else "capability"
|
|
)
|
|
_board_call(
|
|
kanban_db,
|
|
board,
|
|
lambda fresh, failure_reason=failure_reason, failure_kind=failure_kind: kanban_db.block_task(
|
|
fresh,
|
|
task_id,
|
|
reason=failure_reason,
|
|
kind=failure_kind,
|
|
expected_run_id=run_id,
|
|
),
|
|
)
|
|
break
|
|
except TerminalFinalizationPending as error:
|
|
comment(str(error))
|
|
except Exception as error:
|
|
failure_reason = f"Direct CLI lane failed: {type(error).__name__}: {error}"
|
|
_board_call(
|
|
kanban_db,
|
|
board,
|
|
lambda fresh: kanban_db.block_task(
|
|
fresh,
|
|
task_id,
|
|
reason=failure_reason,
|
|
kind="capability",
|
|
expected_run_id=run_id,
|
|
),
|
|
)
|