atlas-iac/testing/tests/test_hermes_handoff_evaluators.py

"""Contracts for the text-shaped handoff acceptance evaluators."""

from __future__ import annotations

import json

import pytest

from testing.tests.test_hermes_handoff_support import (
    load_handoff_module,
    outcome,
    spec,
    step,
)

evaluators = load_handoff_module("hermes_handoff_evaluators")
model = load_handoff_module("hermes_handoff_model")

REVIEW = model.REVIEW
ATTEMPT = model.ATTEMPT
FORBIDDEN = outcome(
    stderr="Error from server (Forbidden): pods is forbidden", returncode=1
)


def deny_spec() -> object:
    return spec(
        "denied",
        steps=(step("review", kind=REVIEW), step("attempt", kind=ATTEMPT)),
    )


def evaluate(built, outcomes):
    return evaluators.evaluate(built, outcomes)


def test_a_deny_check_passes_only_on_a_review_no_and_a_live_refusal() -> None:
    evaluation = evaluate(
        deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN}
    )

    assert evaluation.status == model.PASS
    assert evaluation.evidence == {"review": "no", "attempt": "refused"}


def test_a_deny_check_fails_when_the_review_says_the_authority_exists() -> None:
    evaluation = evaluate(
        deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN}
    )

    assert evaluation.status == model.FAIL
    assert "reported yes" in evaluation.reason


def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> None:
    evaluation = evaluate(
        deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()}
    )

    assert evaluation.status == model.FAIL
    assert "must be refused" in evaluation.reason


def test_a_request_that_merely_broke_is_not_evidence_of_a_refusal() -> None:
    broken = outcome(stderr="connection refused", returncode=7)
    evaluation = evaluate(
        deny_spec(), {"review": outcome(stdout="no"), "attempt": broken}
    )

    assert evaluation.status == model.NOT_RUN
    assert "recognisable refusal" in evaluation.reason


def test_a_deny_check_without_a_live_attempt_refuses_to_classify() -> None:
    """An authorization review states policy; only an attempt proves enforcement."""
    review_only = spec("denied", steps=(step("review", kind=REVIEW),))

    evaluation = evaluate(review_only, {"review": outcome(stdout="no")})

    assert evaluation.status == model.NOT_RUN
    assert "require a real attempt" in evaluation.reason


def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> (
    None
):
    built = spec("allowed", steps=(step("review", kind=REVIEW), step("read")))

    assert (
        evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status
        == model.PASS
    )

    denied = evaluate(built, {"review": outcome(stdout="no"), "read": outcome()})
    assert denied.status == model.FAIL and "reported no" in denied.reason

    failed = evaluate(
        built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)}
    )
    assert failed.status == model.FAIL and "step read failed" in failed.reason


def test_names_absent_flags_exact_names_and_substrings() -> None:
    built = spec(
        "names_absent",
        {"step": "env", "names": ("GITEA_TOKEN",), "contains": ("API_KEY",)},
        steps=(step("env"),),
    )

    clean = evaluate(built, {"env": outcome(stdout="PATH\nHOME\n")})
    assert clean.status == model.PASS and clean.evidence["observed_count"] == 2

    leaked = evaluate(
        built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")}
    )
    assert leaked.status == model.FAIL
    assert leaked.evidence["offenders"] == ["GITEA_TOKEN", "OPENAI_API_KEY"]

    unreadable = evaluate(built, {"env": outcome(returncode=1)})
    assert unreadable.status == model.NOT_RUN


def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> (
    None
):
    built = spec(
        "names_present",
        {
            "step": "init",
            "names": ("patch-web-session-activity",),
            "contains": ("@sha256:",),
        },
        steps=(step("init"),),
    )

    good = outcome(stdout="patch-web-session-activity\nimage@sha256:abc\n")
    assert evaluate(built, {"init": good}).status == model.PASS

    partial = evaluate(built, {"init": outcome(stdout="patch-web-session-activity\n")})
    assert partial.status == model.FAIL and "@sha256:" in partial.reason

    empty = spec("names_present", {"step": "init"}, steps=(step("init"),))
    assert evaluate(empty, {"init": outcome(stdout="")}).status == model.NOT_RUN


def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None:
    single = spec(
        "stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),)
    )
    assert evaluate(single, {"s": outcome(stdout=" locked \n")}).status == model.PASS

    mismatch = evaluate(single, {"s": outcome(stdout="unlocked")})
    assert mismatch.status == model.FAIL and mismatch.evidence["observed"] == "unlocked"

    several = spec(
        "stdout_matches",
        {"step": "s", "equals": ["600 root:root", "absent"]},
        steps=(step("s"),),
    )
    assert evaluate(several, {"s": outcome(stdout="absent")}).status == model.PASS

    assert evaluate(single, {"s": outcome(returncode=1)}).status == model.NOT_RUN


def test_distinct_count_separates_replica_count_from_actual_spread() -> None:
    built = spec(
        "distinct_count",
        {"step": "n", "minimum": 3, "total_equals": 3},
        steps=(step("n"),),
    )

    spread = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-06\ntitan-07\n")})
    assert spread.status == model.PASS and spread.evidence["distinct"] == 3

    stacked = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-05\ntitan-05\n")})
    assert stacked.status == model.FAIL and "distinct values" in stacked.reason

    extra = spec(
        "distinct_count",
        {"step": "n", "minimum": 1, "total_equals": 2},
        steps=(step("n"),),
    )
    over = evaluate(extra, {"n": outcome(stdout="a\nb\nc\n")})
    assert over.status == model.FAIL and "expected exactly 2" in over.reason

    assert evaluate(built, {"n": outcome(stdout="")}).status == model.NOT_RUN
    assert evaluate(built, {"n": outcome(returncode=1)}).status == model.NOT_RUN


def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None:
    built = spec(
        "vantages_agree",
        {"steps": ("operator", "self")},
        steps=(step("operator", vantage="operator"), step("self")),
    )

    agree = evaluate(
        built,
        {
            "operator": outcome(stdout="ns/a\nns/b\n"),
            "self": outcome(stdout="ns/b\nns/a\n"),
        },
    )
    assert agree.status == model.PASS

    differ = evaluate(
        built,
        {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")},
    )
    assert differ.status == model.FAIL and "disagree" in differ.reason

    empty = evaluate(
        built, {"operator": outcome(stdout=""), "self": outcome(stdout="")}
    )
    assert empty.status == model.NOT_RUN

    broken = evaluate(built, {"operator": outcome(returncode=1), "self": outcome()})
    assert broken.status == model.NOT_RUN


def test_steps_agree_is_the_same_rule_under_a_neutral_name() -> None:
    assert (
        evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"]
    )


def test_lines_match_catches_a_blank_leading_column() -> None:
    built = spec(
        "lines_match",
        {"step": "s", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
        steps=(step("s"),),
    )
    table = "Preview   Src   ID\n-------\nwork on task   cli   2026\n"

    assert evaluate(built, {"s": outcome(stdout=table)}).status == model.PASS

    blank = "Preview   Src   ID\n-------\n          cli   2026\n"
    failed = evaluate(built, {"s": outcome(stdout=blank)})
    assert failed.status == model.FAIL and failed.evidence["offenders"]

    assert (
        evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status
        == model.NOT_RUN
    )
    assert evaluate(built, {"s": outcome(returncode=1)}).status == model.NOT_RUN


def test_not_armed_is_the_fixed_skip_a_default_run_must_produce() -> None:
    built = spec("not_armed", {"reason": "ephemeral mutation mode is not armed"})

    evaluation = evaluate(built, {})

    assert evaluation.status == model.NOT_RUN
    assert evaluation.evidence == {"armed": False}
    assert evaluation.reason == "ephemeral mutation mode is not armed"
    assert (
        evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed"
    )


def test_truncation_fails_closed_before_every_registered_evaluator() -> None:
    """No evaluator may reinterpret a clipped required capture as PASS."""
    clipped = outcome(stdout="apparently good", truncated=True)
    for name in evaluators.EVALUATORS:
        built = spec(name, {"step": "s"}, steps=(step("s"),))
        evaluation = evaluate(built, {"s": clipped})
        assert evaluation.status == model.NOT_RUN, name
        assert "truncated" in evaluation.reason, name


def test_missing_required_steps_fail_closed_inside_every_step_evaluator() -> None:
    for name in evaluators.EVALUATORS:
        if name == "not_armed":
            continue
        built = spec(name, {"step": "s"}, steps=(step("s"),))
        assert evaluate(built, {}).status == model.NOT_RUN, name


def test_pool_topology_requires_three_ordinals_nodes_claims_and_ready_state() -> None:
    built = spec(
        "pool_topology",
        {
            "state_step": "state",
            "worker_step": "workers",
            "replicas": 3,
            "name": "worker",
        },
        steps=(step("state"), step("workers")),
    )
    state = outcome(stdout="3\t3\t3\tfalse")
    workers = outcome(
        stdout=(
            "worker-0\tnode-a\tRunning\tTrue\tclaim-0,\n"
            "worker-1\tnode-b\tRunning\tTrue\tclaim-1,\n"
            "worker-2\tnode-c\tRunning\tTrue\tclaim-2,\n"
        )
    )
    assert evaluate(built, {"state": state, "workers": workers}).status == model.PASS
    stacked = outcome(stdout=workers.stdout.replace("node-b", "node-a"))
    assert evaluate(built, {"state": state, "workers": stacked}).status == model.FAIL
    assert (
        evaluate(built, {"state": outcome(stdout="3\t3"), "workers": workers}).status
        == model.NOT_RUN
    )
    assert (
        evaluate(
            built, {"state": outcome(stdout="x\t3\t3\tfalse"), "workers": workers}
        ).status
        == model.NOT_RUN
    )
    assert (
        evaluate(
            built, {"state": outcome(stdout="3\t2\t3\tfalse"), "workers": workers}
        ).status
        == model.FAIL
    )
    malformed = outcome(stdout="worker-0\tnode-a\n")
    assert (
        evaluate(built, {"state": state, "workers": malformed}).status == model.NOT_RUN
    )
    bad_worker = outcome(stdout=workers.stdout.replace("Running", "Pending", 1))
    assert evaluate(built, {"state": state, "workers": bad_worker}).status == model.FAIL
    duplicate = outcome(stdout=workers.stdout.replace("worker-1", "worker-0"))
    assert evaluate(built, {"state": state, "workers": duplicate}).status == model.FAIL
    wrong_ordinals = outcome(stdout=workers.stdout.replace("worker-2", "worker-3"))
    assert (
        evaluate(built, {"state": state, "workers": wrong_ordinals}).status
        == model.FAIL
    )


MAIN = "1" * 40
HEAD = "2" * 40
BUILD = MAIN
DIGEST = "sha256:" + "4" * 64
IMAGE = f"registry.example/hermes:git-{BUILD}-build-7@{DIGEST}"


def lineage_spec():
    keys = (
        "remote-main",
        "reviewed-pr",
        "build-source",
        "merge-ancestry",
        "deployment",
        "pods",
        "replicasets",
        "flux",
    )
    return spec(
        "release_lineage",
        {
            "main_sha": MAIN,
            "head_sha": HEAD,
            "head_ref": "feature/hermes-full-handoff-acceptance",
            "pr_number": 19,
            "image": IMAGE,
            "build_sha": BUILD,
            "deployment_revision": "7",
        },
        steps=tuple(step(key) for key in keys),
    )


def lineage_outcomes():
    pull = {
        "number": 19,
        "state": "closed",
        "draft": False,
        "merged": True,
        # The recorded base SHA is the pre-merge base, not the current main.
        "base": {"ref": "main", "sha": "3" * 40},
        "head": {"ref": "feature/hermes-full-handoff-acceptance", "sha": HEAD},
    }
    return {
        "remote-main": outcome(stdout=f"{MAIN}\trefs/heads/main\n"),
        "reviewed-pr": outcome(stdout=json.dumps(pull)),
        "build-source": outcome(stdout=BUILD),
        "merge-ancestry": outcome(),
        "deployment": outcome(stdout=f"5\t5\t2\t2\t7\t{IMAGE}"),
        "replicasets": outcome(
            stdout=f"rs-old\t6\t0\t0\t{IMAGE}\told\nrs-live\t7\t2\t2\t{IMAGE}\thash7\n"
        ),
        "pods": outcome(
            stdout=(
                f"pod-a\tRunning\tTrue\t{IMAGE}\tdocker-pullable://hermes@{DIGEST}\thash7\n"
                f"pod-b\tRunning\tTrue\t{IMAGE}\tcontainerd://hermes@{DIGEST}\thash7\n"
            )
        ),
        "flux": outcome(stdout=f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{MAIN}"),
    }


def test_release_lineage_atomically_binds_every_release_identity() -> None:
    assert evaluate(lineage_spec(), lineage_outcomes()).status == model.PASS


@pytest.mark.parametrize(
    "patch",
    [
        {"state": "open", "draft": True, "merged": False},
        {"state": "closed", "merged": False},
        {"head": {"ref": "feature/hermes-full-handoff-acceptance", "sha": "9" * 40}},
    ],
    ids=["still-open-draft", "closed-without-merging", "wrong-head-sha"],
)
def test_release_lineage_accepts_only_the_merged_terminal_pr_state(patch) -> None:
    """After integration, only the merged PR proves the reviewed code runs."""
    outcomes = lineage_outcomes()
    pull = {**json.loads(outcomes["reviewed-pr"].stdout), **patch}
    outcomes["reviewed-pr"] = outcome(stdout=json.dumps(pull))

    assert evaluate(lineage_spec(), outcomes).status == model.FAIL


def test_release_lineage_requires_the_merged_head_to_be_an_ancestor_of_main() -> None:
    not_ancestor = lineage_outcomes()
    not_ancestor["merge-ancestry"] = outcome(returncode=1)
    assert evaluate(lineage_spec(), not_ancestor).status == model.FAIL

    undecidable = lineage_outcomes()
    undecidable["merge-ancestry"] = outcome(returncode=128)
    assert evaluate(lineage_spec(), undecidable).status == model.NOT_RUN


@pytest.mark.parametrize(
    ("key", "bad"),
    [
        ("remote-main", ""),
        ("reviewed-pr", "{}"),
        ("build-source", HEAD),
        ("deployment", "5\t4\t2\t2\t7\tbad"),
        ("replicasets", "malformed"),
        ("pods", "malformed"),
        ("flux", "hermes\t9\t9\tfalse\tUnknown\tmain@sha1:" + MAIN),
    ],
)
def test_release_lineage_fails_closed_on_each_malformed_or_mismatched_source(
    key, bad
) -> None:
    outcomes = lineage_outcomes()
    outcomes[key] = outcome(stdout=bad)
    assert evaluate(lineage_spec(), outcomes).status in {model.FAIL, model.NOT_RUN}


def test_release_lineage_rejects_structural_partial_failure_variants() -> None:
    variants = []
    malformed_expect = lineage_spec()
    malformed_expect = model.CheckSpec(
        **{
            **malformed_expect.__dict__,
            "expect": {**malformed_expect.expect, "main_sha": "bad"},
        }
    )
    variants.append((malformed_expect, lineage_outcomes()))
    malformed_image = lineage_spec()
    malformed_image = model.CheckSpec(
        **{
            **malformed_image.__dict__,
            "expect": {
                **malformed_image.expect,
                "image": f"registry.example/hermes@{DIGEST}",
            },
        }
    )
    variants.append((malformed_image, lineage_outcomes()))
    for key, value in (
        ("reviewed-pr", "[]"),
        ("deployment", f"x\tx\t2\t2\t7\t{IMAGE}"),
        ("replicasets", f"rs\t7\t1\t2\t{IMAGE}\thash7"),
        ("replicasets", f"a\t7\t2\t2\t{IMAGE}\th1\nb\t7\t2\t2\t{IMAGE}\th2"),
        ("replicasets", f"rs\t6\t2\t2\t{IMAGE}\thash7"),
        ("pods", f"pod-a\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7"),
        (
            "pods",
            f"pod-a\tPending\tFalse\t{IMAGE}\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7",
        ),
        (
            "pods",
            f"pod-a\tRunning\tTrue\tbad\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7",
        ),
        ("flux", f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{HEAD}"),
    ):
        outcomes = lineage_outcomes()
        outcomes[key] = outcome(stdout=value)
        variants.append((lineage_spec(), outcomes))
    for built, outcomes in variants:
        assert evaluate(built, outcomes).status in {model.FAIL, model.NOT_RUN}