248 lines
8.0 KiB
Python
248 lines
8.0 KiB
Python
"""Structured goal execution and local completion judging."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from testing.tests.test_hermes_cli_support import (
|
|
Path,
|
|
SimpleNamespace,
|
|
lanes,
|
|
nullcontext,
|
|
pytest,
|
|
sys,
|
|
)
|
|
|
|
|
|
def test_artifact_gc_is_interval_gated(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
|
monkeypatch.setattr(lanes, "LAST_ARTIFACT_GC", 0.0)
|
|
calls = []
|
|
monkeypatch.setattr(
|
|
lanes, "gc_lane_artifacts", lambda **kwargs: (calls.append(kwargs) or 0)
|
|
)
|
|
|
|
assert lanes.maybe_gc_lane_artifacts(now=1000.0) == 0
|
|
assert lanes.maybe_gc_lane_artifacts(now=1001.0) == 0
|
|
assert len(calls) == 1
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("result", "expected_action"),
|
|
[
|
|
(lanes.ProcessResult(0, "plain text only", None, False), "block"),
|
|
(
|
|
lanes.ProcessResult(
|
|
0,
|
|
"",
|
|
{
|
|
"status": "completed",
|
|
"summary": "done",
|
|
"changed_files": ["src/a.py"],
|
|
"tests_run": ["pytest -q"],
|
|
"artifacts": ["reports/result.json"],
|
|
"findings": [],
|
|
"blockers": [],
|
|
},
|
|
False,
|
|
),
|
|
"complete",
|
|
),
|
|
(
|
|
lanes.ProcessResult(
|
|
0,
|
|
"",
|
|
{
|
|
"status": "completed",
|
|
"summary": "The full test suite is still running.",
|
|
"changed_files": ["src/a.py"],
|
|
"tests_run": ["pytest -q — in progress"],
|
|
"artifacts": [],
|
|
"findings": [],
|
|
"blockers": [],
|
|
},
|
|
False,
|
|
),
|
|
"block",
|
|
),
|
|
],
|
|
)
|
|
def test_claim_requires_structured_evidence_and_surfaces_artifacts(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
result,
|
|
expected_action,
|
|
):
|
|
task = SimpleNamespace(
|
|
id="t_worker",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=4,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=60,
|
|
)
|
|
calls = []
|
|
heartbeats = []
|
|
connections = []
|
|
artifact = tmp_path / "reports/result.json"
|
|
artifact.parent.mkdir()
|
|
artifact.write_text("{}\n", encoding="utf-8")
|
|
|
|
class Connection:
|
|
def __init__(self):
|
|
self.closed = False
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
def connect(*, board):
|
|
assert board == "cassandra"
|
|
connection = Connection()
|
|
connections.append(connection)
|
|
return connection
|
|
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=connect,
|
|
get_task=lambda _conn, _task_id: task,
|
|
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_worker"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "bounded objective",
|
|
heartbeat_worker=lambda _conn, _task_id, *, note, expected_run_id: (
|
|
heartbeats.append((note, expected_run_id)) or True
|
|
),
|
|
add_comment=lambda *_args: None,
|
|
complete_task=lambda *_args, **kwargs: (
|
|
calls.append(("complete", kwargs)) or True
|
|
),
|
|
block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"select_route",
|
|
lambda *_args, **_kwargs: lanes.Route(
|
|
"codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
|
|
),
|
|
)
|
|
def run_provider(*args, **_kwargs):
|
|
assert connections[0].closed
|
|
before_heartbeat = len(connections)
|
|
assert args[6]("working") is True
|
|
assert len(connections) == before_heartbeat + 1
|
|
assert connections[-1].closed
|
|
return result
|
|
|
|
monkeypatch.setattr(lanes, "run_provider", run_provider)
|
|
|
|
lanes.execute_claim("cassandra", "t_worker")
|
|
|
|
assert calls[0][0] == expected_action
|
|
assert heartbeats == [("working", 4)]
|
|
if expected_action == "complete":
|
|
assert calls[0][1]["metadata"]["artifacts"] == [str(artifact)]
|
|
assert calls[0][1]["metadata"]["tests_run"] == ["pytest -q"]
|
|
else:
|
|
assert calls[0][1]["kind"] == "capability"
|
|
assert all(connection.closed for connection in connections)
|
|
|
|
|
|
def test_goal_loop_recovers_from_corrupt_rejection_history(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
"""Corrupt durable rejection history is replaced, not trusted or fatal."""
|
|
task = SimpleNamespace(
|
|
id="t_history",
|
|
status="running",
|
|
result=None,
|
|
current_run_id=6,
|
|
assignee="cli-auto",
|
|
max_runtime_seconds=300,
|
|
goal_mode=True,
|
|
goal_max_turns=2,
|
|
)
|
|
calls = []
|
|
fake_db = SimpleNamespace(
|
|
scoped_current_board=lambda _board: nullcontext(),
|
|
connect=lambda board: SimpleNamespace(close=lambda: None),
|
|
get_task=lambda _conn, _task_id: task,
|
|
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
|
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_history"),
|
|
set_branch_name=lambda *_args: None,
|
|
set_workspace_path=lambda *_args: None,
|
|
build_worker_context=lambda *_args: "achieve the goal with evidence",
|
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
|
add_comment=lambda *_args: None,
|
|
complete_task=lambda *_args, **kwargs: (
|
|
calls.append(("complete", kwargs)) or True
|
|
),
|
|
block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
|
state_file = lanes.state_path("cassandra", "t_history")
|
|
lanes.atomic_json(state_file, {"goal_rejections": "corrupt-history"})
|
|
probes = []
|
|
monkeypatch.setattr(
|
|
lanes,
|
|
"fresh_unavailable_provider",
|
|
lambda: probes.append("health") and None,
|
|
)
|
|
route = lanes.Route(
|
|
"codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
|
|
)
|
|
monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route)
|
|
reports = [
|
|
lanes.ProcessResult(
|
|
0,
|
|
"first turn",
|
|
{
|
|
"status": "completed",
|
|
"summary": "done",
|
|
"changed_files": ["src/a.py"],
|
|
"tests_run": ["pytest -q"],
|
|
"artifacts": [],
|
|
"findings": [],
|
|
"blockers": [],
|
|
},
|
|
False,
|
|
),
|
|
lanes.ProcessResult(
|
|
0,
|
|
"second turn",
|
|
{
|
|
"status": "completed",
|
|
"summary": "done with verification",
|
|
"changed_files": ["src/a.py"],
|
|
"tests_run": ["pytest -q: passed"],
|
|
"artifacts": [],
|
|
"findings": [],
|
|
"blockers": [],
|
|
},
|
|
False,
|
|
),
|
|
]
|
|
monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
|
|
verdicts = iter([(False, "verification evidence missing"), (True, "verified")])
|
|
monkeypatch.setattr(
|
|
lanes.cli_lane_goal,
|
|
"judge_goal_completion",
|
|
lambda *_args, **_kwargs: next(verdicts),
|
|
)
|
|
|
|
lanes.execute_claim("cassandra", "t_history")
|
|
|
|
assert reports == []
|
|
assert calls[0][0] == "complete"
|
|
assert calls[0][1]["metadata"]["goal_turn"] == 2
|
|
assert len(probes) == 2
|
|
state = lanes.load_json(state_file)
|
|
assert state["goal_rejections"] == [
|
|
"local goal judge requested continuation: verification evidence missing"
|
|
]
|