"""Contracts for the text-shaped handoff acceptance evaluators.""" from __future__ import annotations import json import pytest from testing.tests.test_hermes_handoff_support import ( load_handoff_module, outcome, spec, step, ) evaluators = load_handoff_module("hermes_handoff_evaluators") model = load_handoff_module("hermes_handoff_model") REVIEW = model.REVIEW ATTEMPT = model.ATTEMPT FORBIDDEN = outcome( stderr="Error from server (Forbidden): pods is forbidden", returncode=1 ) def deny_spec() -> object: return spec( "denied", steps=(step("review", kind=REVIEW), step("attempt", kind=ATTEMPT)), ) def evaluate(built, outcomes): return evaluators.evaluate(built, outcomes) def test_a_deny_check_passes_only_on_a_review_no_and_a_live_refusal() -> None: evaluation = evaluate( deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN} ) assert evaluation.status == model.PASS assert evaluation.evidence == {"review": "no", "attempt": "refused"} def test_a_deny_check_fails_when_the_review_says_the_authority_exists() -> None: evaluation = evaluate( deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN} ) assert evaluation.status == model.FAIL assert "reported yes" in evaluation.reason def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> None: evaluation = evaluate( deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()} ) assert evaluation.status == model.FAIL assert "must be refused" in evaluation.reason def test_a_request_that_merely_broke_is_not_evidence_of_a_refusal() -> None: broken = outcome(stderr="connection refused", returncode=7) evaluation = evaluate( deny_spec(), {"review": outcome(stdout="no"), "attempt": broken} ) assert evaluation.status == model.NOT_RUN assert "recognisable refusal" in evaluation.reason def test_a_deny_check_without_a_live_attempt_refuses_to_classify() -> None: """An authorization review states policy; only an attempt proves enforcement.""" review_only = spec("denied", steps=(step("review", kind=REVIEW),)) evaluation = evaluate(review_only, {"review": outcome(stdout="no")}) assert evaluation.status == model.NOT_RUN assert "require a real attempt" in evaluation.reason def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> ( None ): built = spec("allowed", steps=(step("review", kind=REVIEW), step("read"))) assert ( evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status == model.PASS ) denied = evaluate(built, {"review": outcome(stdout="no"), "read": outcome()}) assert denied.status == model.FAIL and "reported no" in denied.reason failed = evaluate( built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)} ) assert failed.status == model.FAIL and "step read failed" in failed.reason def test_names_absent_flags_exact_names_and_substrings() -> None: built = spec( "names_absent", {"step": "env", "names": ("GITEA_TOKEN",), "contains": ("API_KEY",)}, steps=(step("env"),), ) clean = evaluate(built, {"env": outcome(stdout="PATH\nHOME\n")}) assert clean.status == model.PASS and clean.evidence["observed_count"] == 2 leaked = evaluate( built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")} ) assert leaked.status == model.FAIL assert leaked.evidence["offenders"] == ["GITEA_TOKEN", "OPENAI_API_KEY"] unreadable = evaluate(built, {"env": outcome(returncode=1)}) assert unreadable.status == model.NOT_RUN def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> ( None ): built = spec( "names_present", { "step": "init", "names": ("patch-web-session-activity",), "contains": ("@sha256:",), }, steps=(step("init"),), ) good = outcome(stdout="patch-web-session-activity\nimage@sha256:abc\n") assert evaluate(built, {"init": good}).status == model.PASS partial = evaluate(built, {"init": outcome(stdout="patch-web-session-activity\n")}) assert partial.status == model.FAIL and "@sha256:" in partial.reason empty = spec("names_present", {"step": "init"}, steps=(step("init"),)) assert evaluate(empty, {"init": outcome(stdout="")}).status == model.NOT_RUN def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None: single = spec( "stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),) ) assert evaluate(single, {"s": outcome(stdout=" locked \n")}).status == model.PASS mismatch = evaluate(single, {"s": outcome(stdout="unlocked")}) assert mismatch.status == model.FAIL and mismatch.evidence["observed"] == "unlocked" several = spec( "stdout_matches", {"step": "s", "equals": ["600 root:root", "absent"]}, steps=(step("s"),), ) assert evaluate(several, {"s": outcome(stdout="absent")}).status == model.PASS assert evaluate(single, {"s": outcome(returncode=1)}).status == model.NOT_RUN def test_distinct_count_separates_replica_count_from_actual_spread() -> None: built = spec( "distinct_count", {"step": "n", "minimum": 3, "total_equals": 3}, steps=(step("n"),), ) spread = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-06\ntitan-07\n")}) assert spread.status == model.PASS and spread.evidence["distinct"] == 3 stacked = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-05\ntitan-05\n")}) assert stacked.status == model.FAIL and "distinct values" in stacked.reason extra = spec( "distinct_count", {"step": "n", "minimum": 1, "total_equals": 2}, steps=(step("n"),), ) over = evaluate(extra, {"n": outcome(stdout="a\nb\nc\n")}) assert over.status == model.FAIL and "expected exactly 2" in over.reason assert evaluate(built, {"n": outcome(stdout="")}).status == model.NOT_RUN assert evaluate(built, {"n": outcome(returncode=1)}).status == model.NOT_RUN def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None: built = spec( "vantages_agree", {"steps": ("operator", "self")}, steps=(step("operator", vantage="operator"), step("self")), ) agree = evaluate( built, { "operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/b\nns/a\n"), }, ) assert agree.status == model.PASS differ = evaluate( built, {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")}, ) assert differ.status == model.FAIL and "disagree" in differ.reason empty = evaluate( built, {"operator": outcome(stdout=""), "self": outcome(stdout="")} ) assert empty.status == model.NOT_RUN broken = evaluate(built, {"operator": outcome(returncode=1), "self": outcome()}) assert broken.status == model.NOT_RUN def test_steps_agree_is_the_same_rule_under_a_neutral_name() -> None: assert ( evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"] ) def test_lines_match_catches_a_blank_leading_column() -> None: built = spec( "lines_match", {"step": "s", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, steps=(step("s"),), ) table = "Preview Src ID\n-------\nwork on task cli 2026\n" assert evaluate(built, {"s": outcome(stdout=table)}).status == model.PASS blank = "Preview Src ID\n-------\n cli 2026\n" failed = evaluate(built, {"s": outcome(stdout=blank)}) assert failed.status == model.FAIL and failed.evidence["offenders"] assert ( evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status == model.NOT_RUN ) assert evaluate(built, {"s": outcome(returncode=1)}).status == model.NOT_RUN def test_not_armed_is_the_fixed_skip_a_default_run_must_produce() -> None: built = spec("not_armed", {"reason": "ephemeral mutation mode is not armed"}) evaluation = evaluate(built, {}) assert evaluation.status == model.NOT_RUN assert evaluation.evidence == {"armed": False} assert evaluation.reason == "ephemeral mutation mode is not armed" assert ( evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed" ) def test_truncation_fails_closed_before_every_registered_evaluator() -> None: """No evaluator may reinterpret a clipped required capture as PASS.""" clipped = outcome(stdout="apparently good", truncated=True) for name in evaluators.EVALUATORS: built = spec(name, {"step": "s"}, steps=(step("s"),)) evaluation = evaluate(built, {"s": clipped}) assert evaluation.status == model.NOT_RUN, name assert "truncated" in evaluation.reason, name def test_missing_required_steps_fail_closed_inside_every_step_evaluator() -> None: for name in evaluators.EVALUATORS: if name == "not_armed": continue built = spec(name, {"step": "s"}, steps=(step("s"),)) assert evaluate(built, {}).status == model.NOT_RUN, name def test_pool_topology_requires_three_ordinals_nodes_claims_and_ready_state() -> None: built = spec( "pool_topology", { "state_step": "state", "worker_step": "workers", "replicas": 3, "name": "worker", }, steps=(step("state"), step("workers")), ) state = outcome(stdout="3\t3\t3\tfalse") workers = outcome( stdout=( "worker-0\tnode-a\tRunning\tTrue\tclaim-0,\n" "worker-1\tnode-b\tRunning\tTrue\tclaim-1,\n" "worker-2\tnode-c\tRunning\tTrue\tclaim-2,\n" ) ) assert evaluate(built, {"state": state, "workers": workers}).status == model.PASS stacked = outcome(stdout=workers.stdout.replace("node-b", "node-a")) assert evaluate(built, {"state": state, "workers": stacked}).status == model.FAIL assert ( evaluate(built, {"state": outcome(stdout="3\t3"), "workers": workers}).status == model.NOT_RUN ) assert ( evaluate( built, {"state": outcome(stdout="x\t3\t3\tfalse"), "workers": workers} ).status == model.NOT_RUN ) assert ( evaluate( built, {"state": outcome(stdout="3\t2\t3\tfalse"), "workers": workers} ).status == model.FAIL ) malformed = outcome(stdout="worker-0\tnode-a\n") assert ( evaluate(built, {"state": state, "workers": malformed}).status == model.NOT_RUN ) bad_worker = outcome(stdout=workers.stdout.replace("Running", "Pending", 1)) assert evaluate(built, {"state": state, "workers": bad_worker}).status == model.FAIL duplicate = outcome(stdout=workers.stdout.replace("worker-1", "worker-0")) assert evaluate(built, {"state": state, "workers": duplicate}).status == model.FAIL wrong_ordinals = outcome(stdout=workers.stdout.replace("worker-2", "worker-3")) assert ( evaluate(built, {"state": state, "workers": wrong_ordinals}).status == model.FAIL ) MAIN = "1" * 40 HEAD = "2" * 40 BUILD = MAIN DIGEST = "sha256:" + "4" * 64 IMAGE = f"registry.example/hermes:git-{BUILD}-build-7@{DIGEST}" def lineage_spec(): keys = ( "remote-main", "reviewed-pr", "build-source", "merge-ancestry", "deployment", "pods", "replicasets", "flux", ) return spec( "release_lineage", { "main_sha": MAIN, "head_sha": HEAD, "head_ref": "feature/hermes-full-handoff-acceptance", "pr_number": 19, "image": IMAGE, "build_sha": BUILD, "deployment_revision": "7", }, steps=tuple(step(key) for key in keys), ) def lineage_outcomes(): pull = { "number": 19, "state": "closed", "draft": False, "merged": True, # The recorded base SHA is the pre-merge base, not the current main. "base": {"ref": "main", "sha": "3" * 40}, "head": {"ref": "feature/hermes-full-handoff-acceptance", "sha": HEAD}, } return { "remote-main": outcome(stdout=f"{MAIN}\trefs/heads/main\n"), "reviewed-pr": outcome(stdout=json.dumps(pull)), "build-source": outcome(stdout=BUILD), "merge-ancestry": outcome(), "deployment": outcome(stdout=f"5\t5\t2\t2\t7\t{IMAGE}"), "replicasets": outcome( stdout=f"rs-old\t6\t0\t0\t{IMAGE}\told\nrs-live\t7\t2\t2\t{IMAGE}\thash7\n" ), "pods": outcome( stdout=( f"pod-a\tRunning\tTrue\t{IMAGE}\tdocker-pullable://hermes@{DIGEST}\thash7\n" f"pod-b\tRunning\tTrue\t{IMAGE}\tcontainerd://hermes@{DIGEST}\thash7\n" ) ), "flux": outcome(stdout=f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{MAIN}"), } def test_release_lineage_atomically_binds_every_release_identity() -> None: assert evaluate(lineage_spec(), lineage_outcomes()).status == model.PASS @pytest.mark.parametrize( "patch", [ {"state": "open", "draft": True, "merged": False}, {"state": "closed", "merged": False}, {"head": {"ref": "feature/hermes-full-handoff-acceptance", "sha": "9" * 40}}, ], ids=["still-open-draft", "closed-without-merging", "wrong-head-sha"], ) def test_release_lineage_accepts_only_the_merged_terminal_pr_state(patch) -> None: """After integration, only the merged PR proves the reviewed code runs.""" outcomes = lineage_outcomes() pull = {**json.loads(outcomes["reviewed-pr"].stdout), **patch} outcomes["reviewed-pr"] = outcome(stdout=json.dumps(pull)) assert evaluate(lineage_spec(), outcomes).status == model.FAIL def test_release_lineage_requires_the_merged_head_to_be_an_ancestor_of_main() -> None: not_ancestor = lineage_outcomes() not_ancestor["merge-ancestry"] = outcome(returncode=1) assert evaluate(lineage_spec(), not_ancestor).status == model.FAIL undecidable = lineage_outcomes() undecidable["merge-ancestry"] = outcome(returncode=128) assert evaluate(lineage_spec(), undecidable).status == model.NOT_RUN @pytest.mark.parametrize( ("key", "bad"), [ ("remote-main", ""), ("reviewed-pr", "{}"), ("build-source", HEAD), ("deployment", "5\t4\t2\t2\t7\tbad"), ("replicasets", "malformed"), ("pods", "malformed"), ("flux", "hermes\t9\t9\tfalse\tUnknown\tmain@sha1:" + MAIN), ], ) def test_release_lineage_fails_closed_on_each_malformed_or_mismatched_source( key, bad ) -> None: outcomes = lineage_outcomes() outcomes[key] = outcome(stdout=bad) assert evaluate(lineage_spec(), outcomes).status in {model.FAIL, model.NOT_RUN} def test_release_lineage_rejects_structural_partial_failure_variants() -> None: variants = [] malformed_expect = lineage_spec() malformed_expect = model.CheckSpec( **{ **malformed_expect.__dict__, "expect": {**malformed_expect.expect, "main_sha": "bad"}, } ) variants.append((malformed_expect, lineage_outcomes())) malformed_image = lineage_spec() malformed_image = model.CheckSpec( **{ **malformed_image.__dict__, "expect": { **malformed_image.expect, "image": f"registry.example/hermes@{DIGEST}", }, } ) variants.append((malformed_image, lineage_outcomes())) for key, value in ( ("reviewed-pr", "[]"), ("deployment", f"x\tx\t2\t2\t7\t{IMAGE}"), ("replicasets", f"rs\t7\t1\t2\t{IMAGE}\thash7"), ("replicasets", f"a\t7\t2\t2\t{IMAGE}\th1\nb\t7\t2\t2\t{IMAGE}\th2"), ("replicasets", f"rs\t6\t2\t2\t{IMAGE}\thash7"), ("pods", f"pod-a\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7"), ( "pods", f"pod-a\tPending\tFalse\t{IMAGE}\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7", ), ( "pods", f"pod-a\tRunning\tTrue\tbad\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7", ), ("flux", f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{HEAD}"), ): outcomes = lineage_outcomes() outcomes[key] = outcome(stdout=value) variants.append((lineage_spec(), outcomes)) for built, outcomes in variants: assert evaluate(built, outcomes).status in {model.FAIL, model.NOT_RUN}