atlas-iac/testing/tests/test_hermes_handoff_evaluators.py

"""Contracts for the text-shaped handoff acceptance evaluators."""

from __future__ import annotations

from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step

evaluators = load_handoff_module("hermes_handoff_evaluators")
model = load_handoff_module("hermes_handoff_model")

REVIEW = model.REVIEW
ATTEMPT = model.ATTEMPT
FORBIDDEN = outcome(stderr="Error from server (Forbidden): pods is forbidden", returncode=1)


def deny_spec() -> object:
    return spec(
        "denied",
        steps=(step("review", kind=REVIEW), step("attempt", kind=ATTEMPT)),
    )


def evaluate(built, outcomes):
    return evaluators.evaluate(built, outcomes)


def test_a_deny_check_passes_only_on_a_review_no_and_a_live_refusal() -> None:
    evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN})

    assert evaluation.status == model.PASS
    assert evaluation.evidence == {"review": "no", "attempt": "refused"}


def test_a_deny_check_fails_when_the_review_says_the_authority_exists() -> None:
    evaluation = evaluate(deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN})

    assert evaluation.status == model.FAIL
    assert "reported yes" in evaluation.reason


def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> None:
    evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()})

    assert evaluation.status == model.FAIL
    assert "must be refused" in evaluation.reason


def test_a_request_that_merely_broke_is_not_evidence_of_a_refusal() -> None:
    broken = outcome(stderr="connection refused", returncode=7)
    evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": broken})

    assert evaluation.status == model.NOT_RUN
    assert "recognisable refusal" in evaluation.reason


def test_a_deny_check_without_a_live_attempt_refuses_to_classify() -> None:
    """An authorization review states policy; only an attempt proves enforcement."""
    review_only = spec("denied", steps=(step("review", kind=REVIEW),))

    evaluation = evaluate(review_only, {"review": outcome(stdout="no")})

    assert evaluation.status == model.NOT_RUN
    assert "require a real attempt" in evaluation.reason


def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> None:
    built = spec("allowed", steps=(step("review", kind=REVIEW), step("read")))

    assert evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status == model.PASS

    denied = evaluate(built, {"review": outcome(stdout="no"), "read": outcome()})
    assert denied.status == model.FAIL and "reported no" in denied.reason

    failed = evaluate(built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)})
    assert failed.status == model.FAIL and "step read failed" in failed.reason


def test_names_absent_flags_exact_names_and_substrings() -> None:
    built = spec(
        "names_absent",
        {"step": "env", "names": ("GITEA_TOKEN",), "contains": ("API_KEY",)},
        steps=(step("env"),),
    )

    clean = evaluate(built, {"env": outcome(stdout="PATH\nHOME\n")})
    assert clean.status == model.PASS and clean.evidence["observed_count"] == 2

    leaked = evaluate(built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")})
    assert leaked.status == model.FAIL
    assert leaked.evidence["offenders"] == ["GITEA_TOKEN", "OPENAI_API_KEY"]

    unreadable = evaluate(built, {"env": outcome(returncode=1)})
    assert unreadable.status == model.NOT_RUN


def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> None:
    built = spec(
        "names_present",
        {"step": "init", "names": ("patch-web-session-activity",), "contains": ("@sha256:",)},
        steps=(step("init"),),
    )

    good = outcome(stdout="patch-web-session-activity\nimage@sha256:abc\n")
    assert evaluate(built, {"init": good}).status == model.PASS

    partial = evaluate(built, {"init": outcome(stdout="patch-web-session-activity\n")})
    assert partial.status == model.FAIL and "@sha256:" in partial.reason

    empty = spec("names_present", {"step": "init"}, steps=(step("init"),))
    assert evaluate(empty, {"init": outcome(stdout="")}).status == model.NOT_RUN


def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None:
    single = spec("stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),))
    assert evaluate(single, {"s": outcome(stdout=" locked \n")}).status == model.PASS

    mismatch = evaluate(single, {"s": outcome(stdout="unlocked")})
    assert mismatch.status == model.FAIL and mismatch.evidence["observed"] == "unlocked"

    several = spec(
        "stdout_matches", {"step": "s", "equals": ["600 root:root", "absent"]}, steps=(step("s"),)
    )
    assert evaluate(several, {"s": outcome(stdout="absent")}).status == model.PASS

    assert evaluate(single, {"s": outcome(returncode=1)}).status == model.NOT_RUN


def test_distinct_count_separates_replica_count_from_actual_spread() -> None:
    built = spec(
        "distinct_count", {"step": "n", "minimum": 3, "total_equals": 3}, steps=(step("n"),)
    )

    spread = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-06\ntitan-07\n")})
    assert spread.status == model.PASS and spread.evidence["distinct"] == 3

    stacked = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-05\ntitan-05\n")})
    assert stacked.status == model.FAIL and "distinct values" in stacked.reason

    extra = spec("distinct_count", {"step": "n", "minimum": 1, "total_equals": 2}, steps=(step("n"),))
    over = evaluate(extra, {"n": outcome(stdout="a\nb\nc\n")})
    assert over.status == model.FAIL and "expected exactly 2" in over.reason

    assert evaluate(built, {"n": outcome(stdout="")}).status == model.NOT_RUN
    assert evaluate(built, {"n": outcome(returncode=1)}).status == model.NOT_RUN


def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None:
    built = spec(
        "vantages_agree",
        {"steps": ("operator", "self")},
        steps=(step("operator", vantage="operator"), step("self")),
    )

    agree = evaluate(
        built,
        {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/b\nns/a\n")},
    )
    assert agree.status == model.PASS

    differ = evaluate(
        built, {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")}
    )
    assert differ.status == model.FAIL and "disagree" in differ.reason

    empty = evaluate(built, {"operator": outcome(stdout=""), "self": outcome(stdout="")})
    assert empty.status == model.NOT_RUN

    broken = evaluate(built, {"operator": outcome(returncode=1), "self": outcome()})
    assert broken.status == model.NOT_RUN


def test_steps_agree_is_the_same_rule_under_a_neutral_name() -> None:
    assert evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"]


def test_lines_match_catches_a_blank_leading_column() -> None:
    built = spec(
        "lines_match",
        {"step": "s", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
        steps=(step("s"),),
    )
    table = "Preview   Src   ID\n-------\nwork on task   cli   2026\n"

    assert evaluate(built, {"s": outcome(stdout=table)}).status == model.PASS

    blank = "Preview   Src   ID\n-------\n          cli   2026\n"
    failed = evaluate(built, {"s": outcome(stdout=blank)})
    assert failed.status == model.FAIL and failed.evidence["offenders"]

    assert evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status == model.NOT_RUN
    assert evaluate(built, {"s": outcome(returncode=1)}).status == model.NOT_RUN


def test_not_armed_is_the_fixed_skip_a_default_run_must_produce() -> None:
    built = spec("not_armed", {"reason": "ephemeral mutation mode is not armed"})

    evaluation = evaluate(built, {})

    assert evaluation.status == model.NOT_RUN
    assert evaluation.evidence == {"armed": False}
    assert evaluation.reason == "ephemeral mutation mode is not armed"
    assert evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed"