"""Contracts for the text-shaped handoff acceptance evaluators.""" from __future__ import annotations from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step evaluators = load_handoff_module("hermes_handoff_evaluators") model = load_handoff_module("hermes_handoff_model") REVIEW = model.REVIEW ATTEMPT = model.ATTEMPT FORBIDDEN = outcome(stderr="Error from server (Forbidden): pods is forbidden", returncode=1) def deny_spec() -> object: return spec( "denied", steps=(step("review", kind=REVIEW), step("attempt", kind=ATTEMPT)), ) def evaluate(built, outcomes): return evaluators.evaluate(built, outcomes) def test_a_deny_check_passes_only_on_a_review_no_and_a_live_refusal() -> None: evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN}) assert evaluation.status == model.PASS assert evaluation.evidence == {"review": "no", "attempt": "refused"} def test_a_deny_check_fails_when_the_review_says_the_authority_exists() -> None: evaluation = evaluate(deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN}) assert evaluation.status == model.FAIL assert "reported yes" in evaluation.reason def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> None: evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()}) assert evaluation.status == model.FAIL assert "must be refused" in evaluation.reason def test_a_request_that_merely_broke_is_not_evidence_of_a_refusal() -> None: broken = outcome(stderr="connection refused", returncode=7) evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": broken}) assert evaluation.status == model.NOT_RUN assert "recognisable refusal" in evaluation.reason def test_a_deny_check_without_a_live_attempt_refuses_to_classify() -> None: """An authorization review states policy; only an attempt proves enforcement.""" review_only = spec("denied", steps=(step("review", kind=REVIEW),)) evaluation = evaluate(review_only, {"review": outcome(stdout="no")}) assert evaluation.status == model.NOT_RUN assert "require a real attempt" in evaluation.reason def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> None: built = spec("allowed", steps=(step("review", kind=REVIEW), step("read"))) assert evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status == model.PASS denied = evaluate(built, {"review": outcome(stdout="no"), "read": outcome()}) assert denied.status == model.FAIL and "reported no" in denied.reason failed = evaluate(built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)}) assert failed.status == model.FAIL and "step read failed" in failed.reason def test_names_absent_flags_exact_names_and_substrings() -> None: built = spec( "names_absent", {"step": "env", "names": ("GITEA_TOKEN",), "contains": ("API_KEY",)}, steps=(step("env"),), ) clean = evaluate(built, {"env": outcome(stdout="PATH\nHOME\n")}) assert clean.status == model.PASS and clean.evidence["observed_count"] == 2 leaked = evaluate(built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")}) assert leaked.status == model.FAIL assert leaked.evidence["offenders"] == ["GITEA_TOKEN", "OPENAI_API_KEY"] unreadable = evaluate(built, {"env": outcome(returncode=1)}) assert unreadable.status == model.NOT_RUN def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> None: built = spec( "names_present", {"step": "init", "names": ("patch-web-session-activity",), "contains": ("@sha256:",)}, steps=(step("init"),), ) good = outcome(stdout="patch-web-session-activity\nimage@sha256:abc\n") assert evaluate(built, {"init": good}).status == model.PASS partial = evaluate(built, {"init": outcome(stdout="patch-web-session-activity\n")}) assert partial.status == model.FAIL and "@sha256:" in partial.reason empty = spec("names_present", {"step": "init"}, steps=(step("init"),)) assert evaluate(empty, {"init": outcome(stdout="")}).status == model.NOT_RUN def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None: single = spec("stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),)) assert evaluate(single, {"s": outcome(stdout=" locked \n")}).status == model.PASS mismatch = evaluate(single, {"s": outcome(stdout="unlocked")}) assert mismatch.status == model.FAIL and mismatch.evidence["observed"] == "unlocked" several = spec( "stdout_matches", {"step": "s", "equals": ["600 root:root", "absent"]}, steps=(step("s"),) ) assert evaluate(several, {"s": outcome(stdout="absent")}).status == model.PASS assert evaluate(single, {"s": outcome(returncode=1)}).status == model.NOT_RUN def test_distinct_count_separates_replica_count_from_actual_spread() -> None: built = spec( "distinct_count", {"step": "n", "minimum": 3, "total_equals": 3}, steps=(step("n"),) ) spread = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-06\ntitan-07\n")}) assert spread.status == model.PASS and spread.evidence["distinct"] == 3 stacked = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-05\ntitan-05\n")}) assert stacked.status == model.FAIL and "distinct values" in stacked.reason extra = spec("distinct_count", {"step": "n", "minimum": 1, "total_equals": 2}, steps=(step("n"),)) over = evaluate(extra, {"n": outcome(stdout="a\nb\nc\n")}) assert over.status == model.FAIL and "expected exactly 2" in over.reason assert evaluate(built, {"n": outcome(stdout="")}).status == model.NOT_RUN assert evaluate(built, {"n": outcome(returncode=1)}).status == model.NOT_RUN def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None: built = spec( "vantages_agree", {"steps": ("operator", "self")}, steps=(step("operator", vantage="operator"), step("self")), ) agree = evaluate( built, {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/b\nns/a\n")}, ) assert agree.status == model.PASS differ = evaluate( built, {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")} ) assert differ.status == model.FAIL and "disagree" in differ.reason empty = evaluate(built, {"operator": outcome(stdout=""), "self": outcome(stdout="")}) assert empty.status == model.NOT_RUN broken = evaluate(built, {"operator": outcome(returncode=1), "self": outcome()}) assert broken.status == model.NOT_RUN def test_steps_agree_is_the_same_rule_under_a_neutral_name() -> None: assert evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"] def test_lines_match_catches_a_blank_leading_column() -> None: built = spec( "lines_match", {"step": "s", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, steps=(step("s"),), ) table = "Preview Src ID\n-------\nwork on task cli 2026\n" assert evaluate(built, {"s": outcome(stdout=table)}).status == model.PASS blank = "Preview Src ID\n-------\n cli 2026\n" failed = evaluate(built, {"s": outcome(stdout=blank)}) assert failed.status == model.FAIL and failed.evidence["offenders"] assert evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status == model.NOT_RUN assert evaluate(built, {"s": outcome(returncode=1)}).status == model.NOT_RUN def test_not_armed_is_the_fixed_skip_a_default_run_must_produce() -> None: built = spec("not_armed", {"reason": "ephemeral mutation mode is not armed"}) evaluation = evaluate(built, {}) assert evaluation.status == model.NOT_RUN assert evaluation.evidence == {"armed": False} assert evaluation.reason == "ephemeral mutation mode is not armed" assert evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed"