5f27e50c74
Post-merge fixes after rebasing the distributed worker pool onto the review-goal-semantics train tip: - Pool SCM submission tests target the train's relocated receive-pack scanner: FEATURE_REF_RE now lives in receive_pack_scan, bodies are built via the shared _receive_command helper (valid pack), and the update-rejection assertion matches the train's message. - Take the train's canonical test_hermes_scm_broker, test_hermes_cli_dispatch_runtime and test_hermes_cli_execution_edges, which exercise the train's broker/dispatch/execution behavior. - Runtime staging tests patch os.fchown alongside os.chown so the UID-10000 _write_secret path passes under a non-root gate runner (production ownership behavior unchanged). - Split the jenkins build-evidence contracts out of test_hermes_runtime_access into test_hermes_runtime_evidence to keep both files under the 500-LOC hygiene ceiling after the merge. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
320 lines
12 KiB
Python
320 lines
12 KiB
Python
"""Execution-loop edge coverage for preparation, callbacks, and recovery."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
import sys
|
|
from contextlib import nullcontext
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
from testing.tests.test_hermes_cli_support import _completed_result, lanes
|
|
|
|
|
|
class _Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
|
|
def test_missing_claim_is_a_noop(tmp_path: Path, monkeypatch):
|
|
db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: _Connection(),
|
|
get_task=lambda *_args: None,
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
|
lanes.execute_claim("cassandra", "missing")
|
|
|
|
|
|
def test_transient_callback_storage_errors_do_not_kill_provider(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
task = SimpleNamespace(
|
|
id="t_callbacks",
|
|
status="running",
|
|
current_run_id=20,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
blocks = []
|
|
db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: _Connection(),
|
|
get_task=lambda *_args: task,
|
|
worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "exercise callbacks",
|
|
heartbeat_worker=lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
|
sqlite3.OperationalError("heartbeat volume")
|
|
),
|
|
add_comment=lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
|
sqlite3.OperationalError("comment volume")
|
|
),
|
|
block_task=lambda *_args, **kwargs: blocks.append(kwargs),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
|
monkeypatch.setattr(lanes, "KANBAN_STORAGE_ATTEMPTS", 1)
|
|
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: "claude")
|
|
route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ())
|
|
monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route)
|
|
|
|
def provider(*args, **_kwargs):
|
|
assert args[6]("still working") is True
|
|
return lanes.ProcessResult(1, "failed", None, False)
|
|
|
|
monkeypatch.setattr(lanes, "run_provider", provider)
|
|
lanes.execute_claim("cassandra", "t_callbacks")
|
|
|
|
assert blocks and blocks[0]["kind"] == "capability"
|
|
|
|
|
|
def test_restart_handoff_survives_missing_prior_log(tmp_path: Path, monkeypatch):
|
|
task = SimpleNamespace(
|
|
id="t_restart",
|
|
status="running",
|
|
current_run_id=21,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
state_root = tmp_path / "lanes"
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", state_root)
|
|
state_file = lanes.state_path("cassandra", "t_restart")
|
|
lanes.atomic_json(state_file, {"current_route": {"provider": "claude"}})
|
|
blocks = []
|
|
db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: _Connection(),
|
|
get_task=lambda *_args: task,
|
|
worker_log_path=lambda *_args, **_kwargs: tmp_path / "missing.log",
|
|
_resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "resume",
|
|
add_comment=lambda *_args, **_kwargs: None,
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
|
block_task=lambda *_args, **kwargs: blocks.append(kwargs),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
|
route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ())
|
|
monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route)
|
|
handoffs = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"git_handoff",
|
|
lambda _workspace, output: handoffs.append(output) or "handoff",
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"run_provider",
|
|
lambda *_args, **_kwargs: lanes.ProcessResult(1, "failed", None, False),
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_restart")
|
|
|
|
assert handoffs == ["Previous provider log was unavailable after restart."]
|
|
assert blocks
|
|
|
|
|
|
def test_unreplayable_persistence_error_recovers_exact_run(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
task = SimpleNamespace(
|
|
id="t_persist",
|
|
status="running",
|
|
current_run_id=22,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
comments = []
|
|
db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: _Connection(),
|
|
get_task=lambda *_args: task,
|
|
worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "finish",
|
|
add_comment=lambda _conn, _task, _author, body: comments.append(body),
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
|
block_task=lambda *_args, **_kwargs: None,
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"select_route",
|
|
lambda *_args, **_kwargs: lanes.Route(
|
|
"codex", "gpt", "high", "p", "c", "r", 1, ()
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"run_provider",
|
|
lambda *_args, **_kwargs: lanes.ProcessResult(
|
|
0, "", _completed_result("accepted"), False
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_write_terminal_record",
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("disk")),
|
|
)
|
|
monkeypatch.setattr(lanes, "_has_pending_finalization", lambda *_args: False)
|
|
recovered = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"_recover_exact_run",
|
|
lambda *args: recovered.append(args) or True,
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_persist")
|
|
|
|
assert recovered and recovered[0][1].run_id == 22
|
|
assert any("terminal replayable=False" in body for body in comments)
|
|
|
|
|
|
def test_unexpected_route_exception_blocks_the_exact_run(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
task = SimpleNamespace(
|
|
id="t_route",
|
|
status="running",
|
|
current_run_id=23,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
blocks = []
|
|
db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: _Connection(),
|
|
get_task=lambda *_args: task,
|
|
worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "route",
|
|
add_comment=lambda *_args, **_kwargs: None,
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
|
block_task=lambda *_args, **kwargs: blocks.append(kwargs),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"select_route",
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("router down")),
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_route")
|
|
|
|
assert blocks[0]["expected_run_id"] == 23
|
|
assert "router down" in blocks[0]["reason"]
|
|
|
|
|
|
def test_capacity_fallback_without_evidence_blocks_with_provider_output(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
"""A fallback turn with no structured result blocks on raw provider output."""
|
|
task = SimpleNamespace(
|
|
id="t_nofallback",
|
|
status="running",
|
|
current_run_id=22,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
blocks = []
|
|
db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: _Connection(),
|
|
get_task=lambda *_args: task,
|
|
worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "attempt both providers",
|
|
add_comment=lambda *_args, **_kwargs: None,
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
|
block_task=lambda *_args, **kwargs: blocks.append(kwargs),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
|
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: None)
|
|
codex = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ())
|
|
claude = lanes.Route("claude", "fable", "high", "p", "c", "r", 1, ())
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"select_route",
|
|
lambda _prompt, assignee, **_kwargs: claude
|
|
if assignee == "cli-claude-high"
|
|
else codex,
|
|
)
|
|
monkeypatch.setattr(lanes, "git_handoff", lambda *_args: "handoff")
|
|
reports = [
|
|
lanes.ProcessResult(1, "usage limit reached", None, True),
|
|
lanes.ProcessResult(1, "fallback also failed", None, False),
|
|
]
|
|
monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
|
|
|
|
lanes.execute_claim("cassandra", "t_nofallback")
|
|
|
|
assert reports == []
|
|
assert blocks and "fallback also failed" in blocks[0]["reason"]
|
|
assert blocks[0]["kind"] == "capability"
|
|
|
|
|
|
def test_worker_blockers_become_the_block_reason_without_goal_mode(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
"""A structured non-completed report blocks with its own stated blockers."""
|
|
task = SimpleNamespace(
|
|
id="t_blocked",
|
|
status="running",
|
|
current_run_id=23,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
blocks = []
|
|
db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: _Connection(),
|
|
get_task=lambda *_args: task,
|
|
worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "report the obstacle",
|
|
add_comment=lambda *_args, **_kwargs: None,
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
|
block_task=lambda *_args, **kwargs: blocks.append(kwargs),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
|
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: None)
|
|
route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ())
|
|
monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route)
|
|
structured = {
|
|
**_completed_result("progress only"),
|
|
"status": "blocked",
|
|
"blockers": ["credentials expired", "remote unreachable"],
|
|
}
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"run_provider",
|
|
lambda *_args, **_kwargs: lanes.ProcessResult(0, "out", structured, False),
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_blocked")
|
|
|
|
assert blocks and blocks[0]["reason"] == (
|
|
"credentials expired; remote unreachable"
|
|
)
|