7a55b259bf
Three fenced worker Pods claim Hermes Kanban runs through a coordinator that owns every state transition, with per-ordinal HMAC authority, a mediated broker-only SCM path, and durable per-ordinal workspaces. Content is the reviewed head of PR #18 (689bcb6e) with PR 16's and PR 19's contributions removed: they were merged in only to validate co-existence and are not prerequisites, so this branch no longer carries them as ancestors. Only PR 14 and PR 15 remain, because the broker boundary and the cli_lane_* decomposition are load-bearing for two of the fixed P0 boundaries. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
232 lines
8.2 KiB
Python
232 lines
8.2 KiB
Python
"""Execution-loop edge coverage for preparation, callbacks, and recovery."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
import sys
|
|
from contextlib import nullcontext
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
from testing.tests.test_hermes_cli_support import _completed_result, lanes
|
|
|
|
|
|
class _Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
|
|
def test_goal_rejection_history_is_list_bounded():
|
|
execution = __import__("cli_lane_execution")
|
|
history = ["first"]
|
|
assert execution._goal_rejections({"goal_rejections": history}) is history
|
|
assert execution._goal_rejections({"goal_rejections": "invalid"}) == []
|
|
|
|
|
|
def test_missing_claim_is_a_noop(tmp_path: Path, monkeypatch):
|
|
db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: _Connection(),
|
|
get_task=lambda *_args: None,
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
|
lanes.execute_claim("cassandra", "missing")
|
|
|
|
|
|
def test_transient_callback_storage_errors_do_not_kill_provider(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
task = SimpleNamespace(
|
|
id="t_callbacks",
|
|
status="running",
|
|
current_run_id=20,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
blocks = []
|
|
db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: _Connection(),
|
|
get_task=lambda *_args: task,
|
|
worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "exercise callbacks",
|
|
heartbeat_worker=lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
|
sqlite3.OperationalError("heartbeat volume")
|
|
),
|
|
add_comment=lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
|
sqlite3.OperationalError("comment volume")
|
|
),
|
|
block_task=lambda *_args, **kwargs: blocks.append(kwargs),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
|
monkeypatch.setattr(lanes, "KANBAN_STORAGE_ATTEMPTS", 1)
|
|
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: "claude")
|
|
route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ())
|
|
monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route)
|
|
|
|
def provider(*args, **_kwargs):
|
|
assert args[6]("still working") is True
|
|
return lanes.ProcessResult(
|
|
1,
|
|
"failed",
|
|
{"status": "incomplete", "blockers": ["synthetic provider failure"]},
|
|
False,
|
|
)
|
|
|
|
monkeypatch.setattr(lanes, "run_provider", provider)
|
|
lanes.execute_claim("cassandra", "t_callbacks")
|
|
|
|
assert blocks and blocks[0]["kind"] == "capability"
|
|
assert blocks[0]["reason"] == "synthetic provider failure"
|
|
|
|
|
|
def test_restart_handoff_survives_missing_prior_log(tmp_path: Path, monkeypatch):
|
|
task = SimpleNamespace(
|
|
id="t_restart",
|
|
status="running",
|
|
current_run_id=21,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
state_root = tmp_path / "lanes"
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
|
|
state_file = lanes.state_path("cassandra", "t_restart")
|
|
lanes.atomic_json(state_file, {"current_route": {"provider": "claude"}})
|
|
blocks = []
|
|
db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: _Connection(),
|
|
get_task=lambda *_args: task,
|
|
worker_log_path=lambda *_args, **_kwargs: tmp_path / "missing.log",
|
|
_resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "resume",
|
|
add_comment=lambda *_args, **_kwargs: None,
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
|
block_task=lambda *_args, **kwargs: blocks.append(kwargs),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
|
route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ())
|
|
monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route)
|
|
handoffs = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"git_handoff",
|
|
lambda _workspace, output: handoffs.append(output) or "handoff",
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"run_provider",
|
|
lambda *_args, **_kwargs: lanes.ProcessResult(1, "failed", None, False),
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_restart")
|
|
|
|
assert handoffs == ["Previous provider log was unavailable after restart."]
|
|
assert blocks
|
|
|
|
|
|
def test_unreplayable_persistence_error_recovers_exact_run(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
task = SimpleNamespace(
|
|
id="t_persist",
|
|
status="running",
|
|
current_run_id=22,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
comments = []
|
|
db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: _Connection(),
|
|
get_task=lambda *_args: task,
|
|
worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "finish",
|
|
add_comment=lambda _conn, _task, _author, body: comments.append(body),
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
|
block_task=lambda *_args, **_kwargs: None,
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"select_route",
|
|
lambda *_args, **_kwargs: lanes.Route(
|
|
"codex", "gpt", "high", "p", "c", "r", 1, ()
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"run_provider",
|
|
lambda *_args, **_kwargs: lanes.ProcessResult(
|
|
0, "", _completed_result("accepted"), False
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_write_terminal_record",
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("disk")),
|
|
)
|
|
monkeypatch.setattr(lanes, "_has_pending_finalization", lambda *_args: False)
|
|
recovered = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_recover_exact_run",
|
|
lambda *args: recovered.append(args) or True,
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_persist")
|
|
|
|
assert recovered and recovered[0][1].run_id == 22
|
|
assert any("terminal replayable=False" in body for body in comments)
|
|
|
|
|
|
def test_unexpected_route_exception_blocks_the_exact_run(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
task = SimpleNamespace(
|
|
id="t_route",
|
|
status="running",
|
|
current_run_id=23,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
blocks = []
|
|
db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: _Connection(),
|
|
get_task=lambda *_args: task,
|
|
worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "route",
|
|
add_comment=lambda *_args, **_kwargs: None,
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
|
block_task=lambda *_args, **kwargs: blocks.append(kwargs),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"select_route",
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("router down")),
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_route")
|
|
|
|
assert blocks[0]["expected_run_id"] == 23
|
|
assert "router down" in blocks[0]["reason"]
|