atlas-iac/testing/tests/test_hermes_cli_fallback.py

"""Provider fallback, workspace, and command contracts."""

from __future__ import annotations

from testing.tests.test_hermes_cli_support import (
    Path,
    SimpleNamespace,
    _completed_result,
    json,
    lanes,
    nullcontext,
    pytest,
    sys,
)


def test_goal_card_continues_after_local_judge_rejects_progress(
    tmp_path: Path,
    monkeypatch,
):
    task = SimpleNamespace(
        id="t_goal",
        status="running",
        result=None,
        current_run_id=12,
        assignee="cli-auto",
        max_runtime_seconds=300,
        goal_mode=True,
        goal_max_turns=3,
    )
    calls = []
    comments = []

    class Connection:
        def close(self):
            return None

    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        get_task=lambda _conn, _task_id: task,
        worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
        _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"),
        set_branch_name=lambda *_args: None,
        set_workspace_path=lambda *_args: None,
        build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.",
        heartbeat_worker=lambda *_args, **_kwargs: True,
        add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
        complete_task=lambda *_args, **kwargs: (
            calls.append(("complete", kwargs)) or True
        ),
        block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)),
    )
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
    monkeypatch.setattr(
        lanes,
        "PROVIDER_HEALTH_PATHS",
        {
            "codex": tmp_path / "provider-health/codex.json",
            "claude": tmp_path / "provider-health/claude.json",
        },
    )
    monkeypatch.setattr(lanes, "fetch_quota_snapshot", lambda *_a, **_k: {})
    lanes.atomic_json(
        lanes.state_path("cassandra", "t_goal"),
        {"goal_rejections": ["prior incomplete report"]},
    )
    claude_low = lanes.Route(
        "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, ()
    )
    codex_low = lanes.Route(
        "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, ()
    )
    codex_xhigh = lanes.Route(
        "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, ()
    )
    route_calls = []

    def select_route(_prompt, assignee, **kwargs):
        route_calls.append((assignee, kwargs))
        if assignee == "cli-codex-low":
            return codex_low
        if len(route_calls) == 1:
            return claude_low
        return codex_xhigh

    monkeypatch.setattr(lanes, "select_route", select_route)
    monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: None)
    reports = [
        lanes.ProcessResult(1, "authentication expired", None, True),
        lanes.ProcessResult(
            0,
            "first turn",
            {
                "status": "completed",
                "summary": "Focused tests passed.",
                "changed_files": ["src/a.py"],
                "tests_run": ["pytest focused: passed"],
                "artifacts": [],
                "findings": [],
                "blockers": [],
            },
            False,
        ),
        lanes.ProcessResult(
            0,
            "second turn",
            {
                "status": "completed",
                "summary": "Full tests passed; commit pushed and remote HEAD verified.",
                "changed_files": ["src/a.py"],
                "tests_run": ["pytest full: passed"],
                "artifacts": [],
                "findings": [],
                "blockers": [],
            },
            False,
        ),
    ]
    monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
    verdicts = iter(
        [
            (False, "commit, push, and remote verification are missing"),
            (True, "all explicit acceptance criteria have evidence"),
        ]
    )
    judge_contexts = []

    def judge_goal_completion(objective, *_args, **_kwargs):
        judge_contexts.append(objective)
        return next(verdicts)

    monkeypatch.setattr(
        lanes.cli_lane_goal,
        "judge_goal_completion",
        judge_goal_completion,
    )

    lanes.execute_claim("cassandra", "t_goal")

    assert calls[0][0] == "complete"
    assert calls[0][1]["metadata"]["goal_turn"] == 2
    assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments)
    assert any("Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments)
    assert route_calls[2][1]["exclude_provider"] == "claude"
    assert "prior rejected reports" in judge_contexts[1]
    assert "commit, push, and remote verification are missing" in judge_contexts[1]
    candidates = sorted(
        (lanes.STATE_ROOT / "cassandra").glob("t_goal.run-12.candidate-*.json")
    )
    assert len(candidates) == 2
    assert json.loads(candidates[0].read_text())["structured"]["summary"] == (
        "Focused tests passed."
    )
    assert json.loads(candidates[1].read_text())["structured"]["summary"].startswith(
        "Full tests passed"
    )
    assert reports == []


def test_capacity_fallback_preserves_first_claude_structured_response(
    tmp_path: Path,
    monkeypatch,
):
    task = SimpleNamespace(
        id="t_fallback",
        status="running",
        result=None,
        current_run_id=14,
        assignee="cli-auto",
        max_runtime_seconds=60,
    )

    class Connection:
        def close(self):
            return None

    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        get_task=lambda _conn, _task_id: task,
        worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
        _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_fallback"),
        set_branch_name=lambda *_args: None,
        set_workspace_path=lambda *_args: None,
        build_worker_context=lambda *_args: "Complete with a fallback if needed.",
        heartbeat_worker=lambda *_args, **_kwargs: True,
        add_comment=lambda *_args: None,
        complete_task=lambda _conn, _task_id, **kwargs: (
            setattr(task, "status", "done") or setattr(task, "result", kwargs["result"]) or True
        ),
        block_task=lambda *_args, **_kwargs: pytest.fail("fallback should complete"),
    )
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
    monkeypatch.setattr(
        lanes,
        "PROVIDER_HEALTH_PATHS",
        {
            "codex": tmp_path / "provider-health/codex.json",
            "claude": tmp_path / "provider-health/claude.json",
        },
    )
    monkeypatch.setattr(lanes, "fetch_quota_snapshot", lambda *_a, **_k: {})
    claude = lanes.Route(
        "claude", "claude-fable-5", "high", "claude-high", "test", "test", 1, ()
    )
    codex = lanes.Route(
        "codex", "gpt-5.6-sol", "high", "codex-high", "test", "test", 1, ()
    )
    monkeypatch.setattr(
        lanes,
        "select_route",
        lambda _prompt, assignee, **kwargs: codex
        if kwargs.get("exclude_provider") == "claude"
        else claude,
    )
    reports = [
        lanes.ProcessResult(
            1,
            "subscription capacity exhausted",
            {
                **_completed_result("Claude preserved evidence"),
                "status": "incomplete",
                "blockers": ["subscription capacity exhausted"],
            },
            True,
        ),
        lanes.ProcessResult(0, "", _completed_result("Codex completed"), False),
    ]
    monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))

    lanes.execute_claim("cassandra", "t_fallback")

    candidates = sorted(
        (lanes.STATE_ROOT / "cassandra").glob("t_fallback.run-14.candidate-*.json")
    )
    assert len(candidates) == 2
    first, second = [json.loads(path.read_text()) for path in candidates]
    assert (first["provider"], first["structured"]["summary"]) == (
        "claude",
        "Claude preserved evidence",
    )
    assert (second["provider"], second["structured"]["summary"]) == (
        "codex",
        "Codex completed",
    )
    assert task.status == "done"


def test_workspace_preparation_failure_durably_blocks_the_claim(tmp_path: Path, monkeypatch):
    task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto")
    calls = []

    class Connection:
        def close(self):
            return None

    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        get_task=lambda _conn, _task_id: task,
        worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
        _resolve_worktree_workspace=lambda _task, board: (_ for _ in ()).throw(
            ValueError("not a Git repository")
        ),
        block_task=lambda *_args, **kwargs: calls.append(kwargs),
    )
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
    monkeypatch.setattr(lanes, "state_path", lambda _board, _task_id: tmp_path / "state.json")

    lanes.execute_claim("cassandra", "t_bad_worktree")

    assert calls[0]["kind"] == "capability"
    assert calls[0]["expected_run_id"] == 7
    assert "not a Git repository" in calls[0]["reason"]


def test_artifacts_cannot_escape_the_task_worktree(tmp_path: Path):
    workspace = tmp_path / "workspace"
    workspace.mkdir()
    inside = workspace / "report.json"
    outside = tmp_path / "auth.json"
    inside.write_text("{}\n", encoding="utf-8")
    outside.write_text("secret\n", encoding="utf-8")

    assert lanes.workspace_artifacts(
        workspace,
        ["report.json", str(outside), "missing.json"],
    ) == [str(inside)]


def test_restart_provider_change_includes_explicit_workspace_handoff(tmp_path: Path, monkeypatch):
    task = SimpleNamespace(
        id="t_resume",
        status="running",
        result=None,
        current_run_id=9,
        assignee="cli-auto",
        max_runtime_seconds=60,
    )
    monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
    state_file = lanes.state_path("cassandra", "t_resume")
    state_file.parent.mkdir(parents=True)
    state_file.write_text(
        json.dumps({"current_route": {"provider": "claude"}}),
        encoding="utf-8",
    )
    (tmp_path / "worker.log").write_text("prior provider evidence", encoding="utf-8")
    prompts = []

    class Connection:
        def close(self):
            return None

    fake_db = SimpleNamespace(
        scoped_current_board=lambda _board: nullcontext(),
        connect=lambda board: Connection(),
        get_task=lambda _conn, _task_id: task,
        worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
        _resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_resume"),
        set_branch_name=lambda *_args: None,
        set_workspace_path=lambda *_args: None,
        build_worker_context=lambda *_args: "resume objective",
        add_comment=lambda *_args: None,
        complete_task=lambda *_args, **_kwargs: True,
        block_task=lambda *_args, **_kwargs: None,
    )
    monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
    monkeypatch.setattr(
        lanes,
        "select_route",
        lambda *_args, **_kwargs: lanes.Route(
            "codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
        ),
    )
    monkeypatch.setattr(
        lanes,
        "git_handoff",
        lambda _workspace, output: f"HANDOFF:{output}",
    )
    monkeypatch.setattr(
        lanes,
        "run_provider",
        lambda _route, prompt, *_args, **_kwargs: (
            prompts.append(prompt)
            or lanes.ProcessResult(
                0,
                "",
                {
                    "status": "completed",
                    "summary": "done",
                    "changed_files": [],
                    "tests_run": [],
                    "artifacts": [],
                    "findings": [],
                    "blockers": [],
                },
                False,
            )
        ),
    )

    lanes.execute_claim("cassandra", "t_resume")

    assert "HANDOFF:prior provider evidence" in prompts[0]


def test_provider_commands_are_structured_unattended_and_capped(tmp_path: Path):
    route = lanes.Route("codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, ())
    command = lanes._codex_command(route, "Work.", tmp_path, {}, tmp_path / "result.json")
    assert "--dangerously-bypass-approvals-and-sandbox" in command
    assert "--json" in command
    assert "--output-schema" in command
    assert 'model_reasoning_effort="xhigh"' in command

    claude_state = {"claude_session_id": "13864642-2985-4f91-bef5-53f145f878e8"}
    claude = lanes._claude_command(
        lanes.Route("claude", "claude-opus-5", "xhigh", "claude-xhigh", "jetson", "vote", 1, ()),
        "Review.",
        claude_state,
        False,
    )
    assert "--dangerously-skip-permissions" in claude
    assert "--output-format" in claude and "stream-json" in claude
    assert "--json-schema" in claude
    assert "--disallowedTools" in claude
    assert "Bash(kubectl apply *)" not in claude
    assert "Bash(flux reconcile *)" not in claude
    assert "max" not in claude


def test_worker_contract_separates_review_findings_from_task_blockers(tmp_path: Path):
    prompt = lanes.build_prompt("Review the change.", tmp_path)

    assert "put defects and risks in findings" in prompt
    assert "blockers array must be empty whenever status is completed" in prompt
    assert "findings" in lanes.RESULT_SCHEMA["properties"]
    assert set(lanes.RESULT_SCHEMA["required"]) == set(
        lanes.RESULT_SCHEMA["properties"]
    )
    assert "assigned task itself" in lanes.RESULT_SCHEMA["properties"]["blockers"]["description"]