411 lines
14 KiB
Python
411 lines
14 KiB
Python
"""Provider fallback, workspace, and command contracts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from testing.tests.test_hermes_cli_support import (
|
|
Path,
|
|
SimpleNamespace,
|
|
_completed_result,
|
|
json,
|
|
lanes,
|
|
nullcontext,
|
|
pytest,
|
|
sys,
|
|
)
|
|
|
|
|
|
def test_goal_card_continues_after_local_judge_rejects_progress(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
task = SimpleNamespace(
|
|
id="t_goal",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=12,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=300,
|
|
goal_mode=True,
|
|
goal_max_turns=3,
|
|
)
|
|
calls = []
|
|
comments = []
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.",
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
|
add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
|
|
complete_task=lambda *_args, **kwargs: (
|
|
calls.append(("complete", kwargs)) or True
|
|
),
|
|
block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"PROVIDER_HEALTH_PATHS",
|
|
{
|
|
"codex": tmp_path / "provider-health/codex.json",
|
|
"claude": tmp_path / "provider-health/claude.json",
|
|
},
|
|
)
|
|
monkeypatch.setattr(lanes, "fetch_quota_snapshot", lambda *_a, **_k: {})
|
|
lanes.atomic_json(
|
|
lanes.state_path("cassandra", "t_goal"),
|
|
{"goal_rejections": ["prior incomplete report"]},
|
|
)
|
|
claude_low = lanes.Route(
|
|
"claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, ()
|
|
)
|
|
codex_low = lanes.Route(
|
|
"codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, ()
|
|
)
|
|
codex_xhigh = lanes.Route(
|
|
"codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, ()
|
|
)
|
|
route_calls = []
|
|
|
|
def select_route(_prompt, assignee, **kwargs):
|
|
route_calls.append((assignee, kwargs))
|
|
if assignee == "cli-codex-low":
|
|
return codex_low
|
|
if len(route_calls) == 1:
|
|
return claude_low
|
|
return codex_xhigh
|
|
|
|
monkeypatch.setattr(lanes, "select_route", select_route)
|
|
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: None)
|
|
reports = [
|
|
lanes.ProcessResult(1, "authentication expired", None, True),
|
|
lanes.ProcessResult(
|
|
0,
|
|
"first turn",
|
|
{
|
|
"status": "completed",
|
|
"summary": "Focused tests passed.",
|
|
"changed_files": ["src/a.py"],
|
|
"tests_run": ["pytest focused: passed"],
|
|
"artifacts": [],
|
|
"findings": [],
|
|
"blockers": [],
|
|
},
|
|
False,
|
|
),
|
|
lanes.ProcessResult(
|
|
0,
|
|
"second turn",
|
|
{
|
|
"status": "completed",
|
|
"summary": "Full tests passed; commit pushed and remote HEAD verified.",
|
|
"changed_files": ["src/a.py"],
|
|
"tests_run": ["pytest full: passed"],
|
|
"artifacts": [],
|
|
"findings": [],
|
|
"blockers": [],
|
|
},
|
|
False,
|
|
),
|
|
]
|
|
monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
|
|
verdicts = iter(
|
|
[
|
|
(False, "commit, push, and remote verification are missing"),
|
|
(True, "all explicit acceptance criteria have evidence"),
|
|
]
|
|
)
|
|
judge_contexts = []
|
|
|
|
def judge_goal_completion(objective, *_args, **_kwargs):
|
|
judge_contexts.append(objective)
|
|
return next(verdicts)
|
|
|
|
monkeypatch.setattr(
|
|
lanes.cli_lane_goal,
|
|
"judge_goal_completion",
|
|
judge_goal_completion,
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_goal")
|
|
|
|
assert calls[0][0] == "complete"
|
|
assert calls[0][1]["metadata"]["goal_turn"] == 2
|
|
assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments)
|
|
assert any("Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments)
|
|
# The authentication failure keeps Claude excluded at the native-health
|
|
# boundary. Once the retry stays on Codex, quality escalation re-selects
|
|
# the higher lane without inventing another provider outage.
|
|
assert any(
|
|
call[1].get("exclude_provider") == "claude" for call in route_calls
|
|
)
|
|
assert route_calls[-1][0] == "cli-codex-frontier-xhigh"
|
|
assert route_calls[-1][1].get("exclude_provider") is None
|
|
assert "prior rejected reports" in judge_contexts[1]
|
|
assert "commit, push, and remote verification are missing" in judge_contexts[1]
|
|
candidates = sorted(
|
|
(lanes.STATE_ROOT / "cassandra").glob("t_goal.run-12.candidate-*.json")
|
|
)
|
|
assert len(candidates) == 2
|
|
assert json.loads(candidates[0].read_text())["structured"]["summary"] == (
|
|
"Focused tests passed."
|
|
)
|
|
assert json.loads(candidates[1].read_text())["structured"]["summary"].startswith(
|
|
"Full tests passed"
|
|
)
|
|
assert reports == []
|
|
|
|
|
|
def test_capacity_fallback_preserves_first_claude_structured_response(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
task = SimpleNamespace(
|
|
id="t_fallback",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=14,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_fallback"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "Complete with a fallback if needed.",
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
|
add_comment=lambda *_args: None,
|
|
complete_task=lambda _conn, _task_id, **kwargs: (
|
|
setattr(task, "status", "done") or setattr(task, "result", kwargs["result"]) or True
|
|
),
|
|
block_task=lambda *_args, **_kwargs: pytest.fail("fallback should complete"),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"PROVIDER_HEALTH_PATHS",
|
|
{
|
|
"codex": tmp_path / "provider-health/codex.json",
|
|
"claude": tmp_path / "provider-health/claude.json",
|
|
},
|
|
)
|
|
monkeypatch.setattr(lanes, "fetch_quota_snapshot", lambda *_a, **_k: {})
|
|
claude = lanes.Route(
|
|
"claude", "claude-fable-5", "high", "claude-high", "test", "test", 1, ()
|
|
)
|
|
codex = lanes.Route(
|
|
"codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, ()
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"select_route",
|
|
lambda _prompt, assignee, **kwargs: codex
|
|
if kwargs.get("exclude_provider") == "claude"
|
|
else claude,
|
|
)
|
|
reports = [
|
|
lanes.ProcessResult(
|
|
1,
|
|
"subscription capacity exhausted",
|
|
{
|
|
**_completed_result("Claude preserved evidence"),
|
|
"status": "incomplete",
|
|
"blockers": ["subscription capacity exhausted"],
|
|
},
|
|
True,
|
|
),
|
|
lanes.ProcessResult(0, "", _completed_result("Codex completed"), False),
|
|
]
|
|
monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
|
|
|
|
lanes.execute_claim("cassandra", "t_fallback")
|
|
|
|
candidates = sorted(
|
|
(lanes.STATE_ROOT / "cassandra").glob("t_fallback.run-14.candidate-*.json")
|
|
)
|
|
assert len(candidates) == 2
|
|
first, second = [json.loads(path.read_text()) for path in candidates]
|
|
assert (first["provider"], first["structured"]["summary"]) == (
|
|
"claude",
|
|
"Claude preserved evidence",
|
|
)
|
|
assert (second["provider"], second["structured"]["summary"]) == (
|
|
"codex",
|
|
"Codex completed",
|
|
)
|
|
assert task.status == "done"
|
|
|
|
|
|
def test_workspace_preparation_failure_durably_blocks_the_claim(tmp_path: Path, monkeypatch):
|
|
task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto")
|
|
calls = []
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda _task, board: (_ for _ in ()).throw(
|
|
ValueError("not a Git repository")
|
|
),
|
|
block_task=lambda *_args, **kwargs: calls.append(kwargs),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json")
|
|
|
|
lanes.execute_claim("cassandra", "t_bad_worktree")
|
|
|
|
assert calls[0]["kind"] == "capability"
|
|
assert calls[0]["expected_run_id"] == 7
|
|
assert "not a Git repository" in calls[0]["reason"]
|
|
|
|
|
|
def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path):
|
|
workspace = tmp_path / "workspace"
|
|
workspace.mkdir()
|
|
inside = workspace / "report.json"
|
|
outside = tmp_path / "auth.json"
|
|
inside.write_text("{}\n", encoding="utf-8")
|
|
outside.write_text("secret\n", encoding="utf-8")
|
|
|
|
assert lanes.workspace_artifacts(
|
|
workspace,
|
|
["report.json", str(outside), "missing.json"],
|
|
) == [str(inside)]
|
|
|
|
|
|
def test_restart_provider_change_includes_explicit_workspace_handoff(tmp_path: Path, monkeypatch):
|
|
task = SimpleNamespace(
|
|
id="t_resume",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=9,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
|
state_file = lanes.state_path("cassandra", "t_resume")
|
|
state_file.parent.mkdir(parents=True)
|
|
state_file.write_text(
|
|
json.dumps({"current_route": {"provider": "claude"}}),
|
|
encoding="utf-8",
|
|
)
|
|
(tmp_path / "worker.log").write_text("prior provider evidence", encoding="utf-8")
|
|
prompts = []
|
|
|
|
class Connection:
|
|
def close(self):
|
|
return None
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: Connection(),
|
|
get_task=lambda _conn, _task_id: task,
|
|
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_resume"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "resume objective",
|
|
add_comment=lambda *_args: None,
|
|
complete_task=lambda *_args, **_kwargs: True,
|
|
block_task=lambda *_args, **_kwargs: None,
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"select_route",
|
|
lambda *_args, **_kwargs: lanes.Route(
|
|
"codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"git_handoff",
|
|
lambda _workspace, output: f"HANDOFF:{output}",
|
|
)
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"run_provider",
|
|
lambda _route, prompt, *_args, **_kwargs: (
|
|
prompts.append(prompt)
|
|
or lanes.ProcessResult(
|
|
0,
|
|
"",
|
|
{
|
|
"status": "completed",
|
|
"summary": "done",
|
|
"changed_files": [],
|
|
"tests_run": [],
|
|
"artifacts": [],
|
|
"findings": [],
|
|
"blockers": [],
|
|
},
|
|
False,
|
|
)
|
|
),
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_resume")
|
|
|
|
assert "HANDOFF:prior provider evidence" in prompts[0]
|
|
|
|
|
|
def test_provider_commands_are_structured_unattended_and_capped(tmp_path: Path):
|
|
route = lanes.Route("codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, ())
|
|
command = lanes._codex_command(route, "Work.", tmp_path, {}, tmp_path / "result.json")
|
|
assert "--dangerously-bypass-approvals-and-sandbox" in command
|
|
assert "--json" in command
|
|
assert "--output-schema" in command
|
|
assert 'model_reasoning_effort="xhigh"' in command
|
|
|
|
claude_state = {"claude_session_id": "13864642-2985-4f91-bef5-53f145f878e8"}
|
|
claude = lanes._claude_command(
|
|
lanes.Route("claude", "claude-opus-5", "xhigh", "claude-xhigh", "jetson", "vote", 1, ()),
|
|
"Review.",
|
|
claude_state,
|
|
False,
|
|
)
|
|
assert "--dangerously-skip-permissions" in claude
|
|
assert "--output-format" in claude and "stream-json" in claude
|
|
assert "--json-schema" in claude
|
|
assert "--disallowedTools" in claude
|
|
assert "Bash(kubectl apply *)" not in claude
|
|
assert "Bash(flux reconcile *)" not in claude
|
|
assert "max" not in claude
|
|
|
|
|
|
def test_worker_contract_separates_review_findings_from_task_blockers(tmp_path: Path):
|
|
prompt = lanes.build_prompt("Review the change.", tmp_path)
|
|
|
|
assert "put defects and risks in findings" in prompt
|
|
assert "blockers array must be empty whenever status is completed" in prompt
|
|
assert "findings" in lanes.RESULT_SCHEMA["properties"]
|
|
assert set(lanes.RESULT_SCHEMA["required"]) == set(
|
|
lanes.RESULT_SCHEMA["properties"]
|
|
)
|
|
assert "assigned task itself" in lanes.RESULT_SCHEMA["properties"]["blockers"]["description"]
|