atlas-iac/testing/tests/test_hermes_cli_review_goal.py

"""Task-role and verdict-contract tests for the Hermes goal judge."""

from __future__ import annotations

import importlib.util
import json
import sys
from pathlib import Path

import pytest


SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/cli_lane_goal.py"
SPEC = importlib.util.spec_from_file_location("cli_lane_review_goal_test", SCRIPT)
assert SPEC and SPEC.loader
goal = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = goal
SPEC.loader.exec_module(goal)


REVIEW_CARD = """# Kanban task t_review: Independent read-only review of the worker pool

Assignee: cli-claude-xhigh

## Body
Perform a fresh independent read-only release review of the pool implementation.
Do not edit files, comment, commit, push, merge, publish, or deploy.
Return strict SHIP or BLOCK with exact file/line and reproducible inputs.
"""
IMPLEMENTATION_CARD = """# Kanban task t_impl: Repair the pool blockers

Assignee: cli-auto

## Body
Repair every reported blocker, run the focused suites, and open a draft PR.
Return exact pushed SHA and evidence.
"""
POLLUTED_CARD = REVIEW_CARD + """
## Recent work by @cli-claude-xhigh
- t_prior: Implemented and pushed the pool. PR #18 is open at the exact pushed SHA.

## Comment thread
- shepherd: commit and push the repairs before returning.
"""


def _review(**overrides):
    value = {
        "status": "completed",
        "summary": (
            "Independent read-only review of PR #18 at head 2000252. Verdict: BLOCK. "
            "Five reproducible P0 defects fire on the coordinator's first tick and the "
            "review worktree was left pristine."
        ),
        "changed_files": [],
        "tests_run": ["pytest testing/tests/test_pool.py: 82 passed"],
        "artifacts": ["/opt/data/workspace/evidence/t_review/RUN.md"],
        "findings": ["P0-1 BLOCKER - coordinator.py:248 compares int to str run ids."],
        "blockers": [],
    }
    value.update(overrides)
    return value


def _implementation(**overrides):
    value = {
        "status": "completed",
        "summary": "All acceptance criteria passed and the branch was pushed.",
        "changed_files": ["src/example.py"],
        "tests_run": ["pytest -q: 12 passed"],
        "artifacts": [],
        "findings": [],
        "blockers": [],
    }
    value.update(overrides)
    return value


def _no_judge(*_args, **_kwargs):
    raise AssertionError("the model judge must not be consulted for a review card")


def test_completed_block_review_finalizes_without_consulting_the_model():
    accepted, reason = goal.judge_goal_completion(
        REVIEW_CARD,
        _review(),
        open_request=_no_judge,
    )

    assert accepted is True
    assert "BLOCK verdict with 1 finding(s)" in reason
    assert "resolved by inferred" in reason


def test_completed_ship_review_finalizes_on_its_own_evidence():
    accepted, reason = goal.judge_goal_completion(
        REVIEW_CARD,
        _review(
            summary=(
                "Independent read-only review of PR #18 at head 2000252. Verdict: SHIP. "
                "Every prioritized boundary was exercised and no defect survived."
            ),
            findings=[],
        ),
        open_request=_no_judge,
    )

    assert accepted is True
    assert "SHIP verdict with 0 finding(s)" in reason


@pytest.mark.parametrize(
    ("role_line", "expected"),
    [
        ("Hermes-Task-Role: review", goal.REVIEW_ROLE),
        ("Task-Role: audit", goal.REVIEW_ROLE),
        ("- Hermes Task Role = diagnostic", goal.REVIEW_ROLE),
        ("Hermes-Expected-Output: verdict", goal.REVIEW_ROLE),
        ("Hermes-Task-Role: implementation", goal.IMPLEMENTATION_ROLE),
    ],
)
def test_explicit_card_metadata_decides_the_task_role(role_line, expected):
    card = IMPLEMENTATION_CARD.replace("## Body\n", f"## Body\n{role_line}\n")

    assert goal.task_role(card, _review())[0] == expected
    assert goal.task_role(card, _review())[1] == "directive"


def test_explicit_review_metadata_cannot_self_certify_a_changed_file_report():
    card = REVIEW_CARD.replace("## Body\n", "## Body\nHermes-Task-Role: review\n")
    mutated = _review(changed_files=["services/hermes/scripts/cli_lane_goal.py"])

    assert goal.task_role(card, mutated) == (goal.IMPLEMENTATION_ROLE, "conflict")
    accepted, reason = goal.judge_goal_completion(
        card,
        mutated,
        open_request=lambda *_a, **_k: JudgeResponse("continue", "the card forbade edits"),
    )

    assert accepted is False
    assert reason == "the card forbade edits"
    # The declaration still holds when the report really did change nothing.
    assert goal.task_role(card, _review()) == (goal.REVIEW_ROLE, "directive")


def test_appended_run_history_cannot_reassign_the_card_role():
    assert goal.task_role(POLLUTED_CARD, _review()) == (goal.REVIEW_ROLE, "inferred")
    assert "Recent work by" not in goal.card_scope(POLLUTED_CARD)
    assert "Do not edit files" in goal.card_scope(POLLUTED_CARD)


def test_implementation_card_never_infers_a_review_role():
    assert goal.task_role(IMPLEMENTATION_CARD, _review()) == (
        goal.IMPLEMENTATION_ROLE,
        "default",
    )


@pytest.mark.parametrize(
    ("result", "expected"),
    [
        (_review(summary="Verdict: BLOCK."), "carries no reviewable rationale"),
        (_review(findings=[]), "BLOCK verdict reports no findings"),
        (
            _review(summary=_review()["summary"].replace("Verdict: BLOCK.", "")),
            "declares no explicit SHIP or BLOCK verdict",
        ),
        (
            _review(verdict="SHIP"),
            "declares both SHIP and BLOCK verdicts",
        ),
        (
            _review(verdict="maybe"),
            "review verdict field is not SHIP or BLOCK",
        ),
        (
            _review(
                summary=(
                    "The reviewed change is fine overall. SHIP is defensible but BLOCK "
                    "is also arguable given the unresolved capacity question."
                ),
            ),
            "names both SHIP and BLOCK without a verdict line",
        ),
        (
            _review(
                summary=(
                    "Independent read-only review of PR #18. Verdict: SHIP. Every "
                    "prioritized boundary was exercised and nothing survived."
                ),
                findings=[],
                tests_run=[],
                artifacts=[],
            ),
            "no findings, tests, or artifacts as evidence",
        ),
    ],
)
def test_malformed_review_completion_fails_closed(result, expected):
    accepted, reason = goal.judge_goal_completion(
        REVIEW_CARD,
        result,
        open_request=_no_judge,
    )

    assert accepted is False
    assert expected in reason
    assert reason.startswith(goal.READ_ONLY_GUARD)


def test_rejected_review_never_demands_an_implementation_edit():
    accepted, reason = goal.judge_goal_completion(
        REVIEW_CARD,
        _review(findings=[]),
        open_request=_no_judge,
    )

    assert accepted is False
    assert "do not modify the reviewed implementation" in reason


def test_incomplete_review_turn_is_resumed_under_the_read_only_guard():
    accepted, reason = goal.judge_goal_completion(
        REVIEW_CARD,
        _review(status="incomplete", summary="Only two of six boundaries were read."),
        open_request=_no_judge,
    )

    assert accepted is False
    assert reason.startswith(goal.READ_ONLY_GUARD)
    assert "Only two of six boundaries were read." in reason


def test_blocked_review_reports_its_own_obstacle_not_a_verdict():
    result = _review(
        status="blocked",
        summary="The pull request head could not be resolved from any SCM mirror.",
        blockers=["Gitea API is unreachable from the review worktree."],
    )

    assert goal.unfinished_result_reason(result, role=goal.REVIEW_ROLE) is None
    assert goal.unfinished_result_reason(result) is None
    accepted, reason = goal.judge_goal_completion(
        REVIEW_CARD, result, open_request=_no_judge
    )
    assert accepted is False
    assert "rather than a finished task" in reason


def test_conflicting_role_directives_hold_the_card_to_implementation():
    card = REVIEW_CARD.replace(
        "## Body\n",
        "## Body\nHermes-Task-Role: review\nHermes-Task-Role: implementation\n",
    )

    assert goal.task_role(card, _review()) == (goal.IMPLEMENTATION_ROLE, "directive")


def test_incomplete_review_summary_is_bounded_before_it_leaves_the_judge():
    accepted, reason = goal.judge_goal_completion(
        REVIEW_CARD,
        _review(status="incomplete", summary="pending\n" + "y" * 4000),
        open_request=_no_judge,
    )

    assert accepted is False
    assert reason.startswith(goal.READ_ONLY_GUARD)
    assert len(reason) <= goal.JUDGE_REASON_LIMIT


def test_review_claiming_completion_with_blockers_is_rejected():
    accepted, reason = goal.judge_goal_completion(
        REVIEW_CARD,
        _review(blockers=["the diff could not be resolved"]),
        open_request=_no_judge,
    )

    assert accepted is False
    assert "worker reported blockers while claiming completion" in reason


def test_review_prose_about_the_reviewed_artifact_is_not_unfinished_work():
    result = _review(
        summary=(
            "Independent read-only review. Verdict: BLOCK. The reviewed branch's CI "
            "build is still running and its migration remains pending, which is why "
            "the change is unfit to ship."
        ),
        tests_run=["upstream pipeline: build is still running on the reviewed head"],
    )

    assert goal.unfinished_result_reason(result, role=goal.REVIEW_ROLE) is None
    accepted, _reason = goal.judge_goal_completion(
        REVIEW_CARD,
        result,
        open_request=_no_judge,
    )
    assert accepted is True


def test_implementation_prose_about_unfinished_work_is_still_rejected():
    result = _implementation(summary="The broad rerun remains active before the push.")

    assert goal.unfinished_result_reason(result, role=goal.IMPLEMENTATION_ROLE)
    assert goal.judge_goal_completion(
        IMPLEMENTATION_CARD, result, open_request=_no_judge
    ) == (False, "completion evidence says work is unfinished: remains active")


class JudgeResponse:
    def __init__(self, verdict: str, reason: str):
        self.body = json.dumps(
            {"choices": [{"message": {"content": json.dumps(
                {"verdict": verdict, "reason": reason}
            )}}]}
        ).encode()

    def __enter__(self):
        return self

    def __exit__(self, *_args):
        return False

    def read(self):
        return self.body


def test_implementation_card_keeps_the_fail_closed_model_judge():
    observed = {}

    def request(req, timeout):
        observed["payload"] = json.loads(req.data)
        observed["timeout"] = timeout
        return JudgeResponse("continue", "the remote head was never verified")

    accepted, reason = goal.judge_goal_completion(
        IMPLEMENTATION_CARD,
        _implementation(),
        open_request=request,
    )

    assert accepted is False
    assert reason == "the remote head was never verified"
    role = json.loads(observed["payload"]["messages"][1]["content"])["task_role"]
    assert role == goal.IMPLEMENTATION_ROLE
    assert observed["timeout"] == 120


def test_implementation_completion_still_finalizes_on_an_accepting_judge():
    accepted, reason = goal.judge_goal_completion(
        IMPLEMENTATION_CARD,
        _implementation(),
        open_request=lambda *_a, **_k: JudgeResponse("complete", "every criterion has evidence"),
    )

    assert accepted is True
    assert reason == "every criterion has evidence"


def test_ambiguous_card_without_a_verdict_report_stays_on_the_model_judge():
    ambiguous = """# Kanban task t_mixed: Review and repair the pool

## Body
Review the pool, then repair what you find. Do not modify unrelated files.
Return a verdict and open a draft PR with the repairs.
"""

    accepted, _reason = goal.judge_goal_completion(
        ambiguous,
        _review(),
        open_request=lambda *_a, **_k: JudgeResponse("continue", "the repairs are missing"),
    )

    assert goal.task_role(ambiguous, _review())[0] == goal.IMPLEMENTATION_ROLE
    assert accepted is False


def test_judge_reason_is_bounded_sanitized_and_secret_free():
    leaky = "line one\r\n\x07 token ghp_" + "A" * 24 + " " + "x" * 900

    accepted, reason = goal.judge_goal_completion(
        IMPLEMENTATION_CARD,
        _implementation(),
        open_request=lambda *_a, **_k: JudgeResponse("continue", leaky),
    )

    assert accepted is False
    assert len(reason) <= goal.JUDGE_REASON_LIMIT
    assert "ghp_" not in reason
    assert "[redacted]" in reason
    assert "\n" not in reason and "\r" not in reason and "\x07" not in reason


def test_unavailable_judge_still_fails_closed_for_implementation_cards():
    accepted, reason = goal.judge_goal_completion(
        IMPLEMENTATION_CARD,
        _implementation(),
        open_request=lambda *_a, **_k: JudgeResponse("unknown", "bad"),
    )

    assert accepted is False
    assert "local completion judge unavailable" in reason


def test_bare_uppercase_verdict_token_is_accepted_as_a_last_resort():
    result = _review(
        summary=(
            "Independent read-only review of PR #18 at head 2000252 concludes BLOCK "
            "because five reproducible P0 defects fire on the first coordinator tick."
        ),
    )

    assert goal.review_verdict(result) == ("BLOCK", None)
    accepted, _reason = goal.judge_goal_completion(
        REVIEW_CARD, result, open_request=_no_judge
    )
    assert accepted is True


def test_unknown_role_directive_falls_through_to_the_next_contract_surface():
    card = IMPLEMENTATION_CARD.replace(
        "## Body\n",
        "## Body\nHermes-Task-Role: gardener\nHermes-Expected-Output: verdict\n",
    )

    assert goal.task_role(card, _review()) == (goal.REVIEW_ROLE, "directive")


def test_unrecognized_directives_leave_the_role_to_inference():
    card = REVIEW_CARD.replace("## Body\n", "## Body\nHermes-Task-Role: gardener\n")

    assert goal.task_role(card, _review()) == (goal.REVIEW_ROLE, "inferred")


def test_malformed_evidence_fields_are_read_without_raising():
    assert goal._strings("not-a-list") == []
    result = _implementation(tests_run="pytest -q", summary="Everything shipped.")

    assert goal.unfinished_result_reason(result, role=goal.IMPLEMENTATION_ROLE) is None


def test_a_blocked_implementation_report_is_rejected_without_the_read_only_guard():
    accepted, reason = goal.judge_goal_completion(
        IMPLEMENTATION_CARD,
        _implementation(status="blocked", summary="The remote was unreachable."),
        open_request=_no_judge,
    )

    assert accepted is False
    assert reason == "worker reported status 'blocked' rather than a finished task"


def test_implementation_rejection_reason_carries_no_read_only_guard():
    accepted, reason = goal.judge_goal_completion(
        IMPLEMENTATION_CARD,
        _implementation(blockers=["the remote was unreachable"]),
        open_request=_no_judge,
    )

    assert accepted is False
    assert reason == "worker reported blockers while claiming completion"


def test_oversized_objectives_and_empty_reasons_stay_bounded():
    compacted = goal._bounded("a" * 9000, 6000)

    assert len(compacted) == 6000
    assert "[goal evidence compacted]" in compacted
    assert goal.sanitize_reason("   ") == "local judge supplied no reason"


def test_judging_never_mutates_the_reported_result():
    result = _review()
    snapshot = json.dumps(result, sort_keys=True)

    goal.judge_goal_completion(REVIEW_CARD, result, open_request=_no_judge)

    assert json.dumps(result, sort_keys=True) == snapshot