atlas-iac/testing/tests/test_hermes_cli_review_roles.py

"""Role-aware lane regressions for the goal-completion incident.

The instrumented single-claim harness lives in
``test_hermes_cli_review_lane`` so both lane suites drive the same fake board
and the same ``execute_claim`` entry point.
"""

from __future__ import annotations

from test_hermes_cli_review_lane import (
    BLOCK_REVIEW,
    IMPLEMENTATION_CARD,
    REVIEW_CARD,
    _Lane,
    _no_judge,
    _result,
    _task,
    execution,
    goal,
    lanes,
)


BARE_TOKEN_REVIEW = dict(
    BLOCK_REVIEW,
    summary=(
        "Independent read-only review of PR #18 concludes BLOCK: the coordinator "
        "drops its lease on the first tick and deployment verification is pending "
        "on two of three nodes, so the change is unfit to ship."
    ),
)


def test_a_bare_token_review_with_artifact_prose_finalizes_on_turn_one(
    tmp_path, monkeypatch
):
    """The end-to-end shape that burned every goal turn before the role reached
    ``unfinished_result_reason``: a bare uppercase verdict token whose prose
    describes the *reviewed* deployment as pending."""
    monkeypatch.setattr(goal.urllib.request, "urlopen", _no_judge)
    lane = _Lane(
        tmp_path,
        monkeypatch,
        card=REVIEW_CARD,
        task=_task(goal_max_turns=3),
        reports=[_result(**BARE_TOKEN_REVIEW)],
    )

    execution.execute_claim("titan-iac", "t_card")

    action, kwargs = lane.terminal
    assert action == "complete"
    assert lane.rejections() == []
    assert len(lane.prompts) == 1
    assert kwargs["metadata"]["task_role"] == goal.REVIEW_ROLE
    assert kwargs["metadata"]["task_role_source"] == "inferred"
    assert "BLOCK verdict with 1 finding(s)" in kwargs["metadata"]["goal_judge_reason"]


def test_single_shot_review_completes_instead_of_discarding_its_verdict(
    tmp_path, monkeypatch
):
    monkeypatch.setattr(goal.urllib.request, "urlopen", _no_judge)
    lane = _Lane(
        tmp_path,
        monkeypatch,
        card=REVIEW_CARD,
        task=_task(goal_mode=False, goal_max_turns=1),
        reports=[_result(**BARE_TOKEN_REVIEW)],
    )

    execution.execute_claim("titan-iac", "t_card")

    action, kwargs = lane.terminal
    assert action == "complete"
    assert kwargs["metadata"]["task_role"] == goal.REVIEW_ROLE
    assert "goal_judge_reason" not in kwargs["metadata"]


def test_single_shot_review_without_a_verdict_fails_closed(tmp_path, monkeypatch):
    """The documented single-shot change: a verdictless review is not finished."""
    monkeypatch.setattr(goal.urllib.request, "urlopen", _no_judge)
    lane = _Lane(
        tmp_path,
        monkeypatch,
        card=REVIEW_CARD,
        task=_task(goal_mode=False, goal_max_turns=1),
        reports=[_result(summary="Review complete; residual risks are listed.")],
    )

    execution.execute_claim("titan-iac", "t_card")

    action, kwargs = lane.terminal
    assert action == "block"
    assert kwargs["reason"].startswith(goal.READ_ONLY_GUARD)
    assert "declares no explicit SHIP or BLOCK verdict" in kwargs["reason"]


def test_single_shot_implementation_still_blocks_on_unfinished_evidence(
    tmp_path, monkeypatch
):
    lane = _Lane(
        tmp_path,
        monkeypatch,
        card=IMPLEMENTATION_CARD,
        task=_task(goal_mode=False, goal_max_turns=1),
        reports=[
            lanes.ProcessResult(
                0,
                "turn one",
                {
                    **BLOCK_REVIEW,
                    "summary": "The broad rerun remains active before the push.",
                    "changed_files": ["src/a.py"],
                },
                False,
            )
        ],
    )

    execution.execute_claim("titan-iac", "t_card")

    action, kwargs = lane.terminal
    assert action == "block"
    assert "completion evidence says work is unfinished" in kwargs["reason"]
    assert goal.READ_ONLY_GUARD not in kwargs["reason"]


def test_goal_controller_evidence_stays_outside_the_card(tmp_path, monkeypatch):
    contexts: list[str] = []

    def judge(objective, *_args, **_kwargs):
        contexts.append(objective)
        return (len(contexts) > 1, "ok" if len(contexts) > 1 else "push the branch")

    monkeypatch.setattr(goal, "judge_goal_completion", judge)
    implementation = {
        **BLOCK_REVIEW,
        "summary": "Focused tests passed and the branch was pushed.",
        "changed_files": ["src/a.py"],
        "findings": [],
    }
    lane = _Lane(
        tmp_path,
        monkeypatch,
        card=IMPLEMENTATION_CARD,
        task=_task(goal_max_turns=2),
        reports=[
            lanes.ProcessResult(0, "one", dict(implementation), False),
            lanes.ProcessResult(0, "two", dict(implementation), False),
        ],
    )

    execution.execute_claim("titan-iac", "t_card")

    assert lane.terminal[0] == "complete"
    assert goal.CONTROLLER_EVIDENCE_HEADING in contexts[1]
    assert "push the branch" in contexts[1]
    assert "push the branch" not in goal.card_scope(contexts[1])