a5cafddd45
The reviewed PR stack is now merged into main, so an open draft PR #19 is no longer proof that the reviewed code is what runs. The mandatory release.exact-lineage-is-running check now requires the merged terminal state instead: PR #19 closed with merged=true, base main, the existing feature ref, and the exact reviewed head, plus a new merge-ancestry step that proves the reviewed head is an ancestor of the pinned origin/main via git merge-base --is-ancestor. An open PR, a PR closed without merging, a mismatched head, or a head that is not a proven ancestor of main still fails closed; an undecidable ancestry probe is NOT_RUN. The recorded pre-merge base SHA is no longer compared against current main. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
489 lines
16 KiB
Python
489 lines
16 KiB
Python
"""Contracts for the text-shaped handoff acceptance evaluators."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from testing.tests.test_hermes_handoff_support import (
|
|
load_handoff_module,
|
|
outcome,
|
|
spec,
|
|
step,
|
|
)
|
|
|
|
evaluators = load_handoff_module("hermes_handoff_evaluators")
|
|
model = load_handoff_module("hermes_handoff_model")
|
|
|
|
REVIEW = model.REVIEW
|
|
ATTEMPT = model.ATTEMPT
|
|
FORBIDDEN = outcome(
|
|
stderr="Error from server (Forbidden): pods is forbidden", returncode=1
|
|
)
|
|
|
|
|
|
def deny_spec() -> object:
|
|
return spec(
|
|
"denied",
|
|
steps=(step("review", kind=REVIEW), step("attempt", kind=ATTEMPT)),
|
|
)
|
|
|
|
|
|
def evaluate(built, outcomes):
|
|
return evaluators.evaluate(built, outcomes)
|
|
|
|
|
|
def test_a_deny_check_passes_only_on_a_review_no_and_a_live_refusal() -> None:
|
|
evaluation = evaluate(
|
|
deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN}
|
|
)
|
|
|
|
assert evaluation.status == model.PASS
|
|
assert evaluation.evidence == {"review": "no", "attempt": "refused"}
|
|
|
|
|
|
def test_a_deny_check_fails_when_the_review_says_the_authority_exists() -> None:
|
|
evaluation = evaluate(
|
|
deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN}
|
|
)
|
|
|
|
assert evaluation.status == model.FAIL
|
|
assert "reported yes" in evaluation.reason
|
|
|
|
|
|
def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> None:
|
|
evaluation = evaluate(
|
|
deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()}
|
|
)
|
|
|
|
assert evaluation.status == model.FAIL
|
|
assert "must be refused" in evaluation.reason
|
|
|
|
|
|
def test_a_request_that_merely_broke_is_not_evidence_of_a_refusal() -> None:
|
|
broken = outcome(stderr="connection refused", returncode=7)
|
|
evaluation = evaluate(
|
|
deny_spec(), {"review": outcome(stdout="no"), "attempt": broken}
|
|
)
|
|
|
|
assert evaluation.status == model.NOT_RUN
|
|
assert "recognisable refusal" in evaluation.reason
|
|
|
|
|
|
def test_a_deny_check_without_a_live_attempt_refuses_to_classify() -> None:
|
|
"""An authorization review states policy; only an attempt proves enforcement."""
|
|
review_only = spec("denied", steps=(step("review", kind=REVIEW),))
|
|
|
|
evaluation = evaluate(review_only, {"review": outcome(stdout="no")})
|
|
|
|
assert evaluation.status == model.NOT_RUN
|
|
assert "require a real attempt" in evaluation.reason
|
|
|
|
|
|
def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> (
|
|
None
|
|
):
|
|
built = spec("allowed", steps=(step("review", kind=REVIEW), step("read")))
|
|
|
|
assert (
|
|
evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status
|
|
== model.PASS
|
|
)
|
|
|
|
denied = evaluate(built, {"review": outcome(stdout="no"), "read": outcome()})
|
|
assert denied.status == model.FAIL and "reported no" in denied.reason
|
|
|
|
failed = evaluate(
|
|
built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)}
|
|
)
|
|
assert failed.status == model.FAIL and "step read failed" in failed.reason
|
|
|
|
|
|
def test_names_absent_flags_exact_names_and_substrings() -> None:
|
|
built = spec(
|
|
"names_absent",
|
|
{"step": "env", "names": ("GITEA_TOKEN",), "contains": ("API_KEY",)},
|
|
steps=(step("env"),),
|
|
)
|
|
|
|
clean = evaluate(built, {"env": outcome(stdout="PATH\nHOME\n")})
|
|
assert clean.status == model.PASS and clean.evidence["observed_count"] == 2
|
|
|
|
leaked = evaluate(
|
|
built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")}
|
|
)
|
|
assert leaked.status == model.FAIL
|
|
assert leaked.evidence["offenders"] == ["GITEA_TOKEN", "OPENAI_API_KEY"]
|
|
|
|
unreadable = evaluate(built, {"env": outcome(returncode=1)})
|
|
assert unreadable.status == model.NOT_RUN
|
|
|
|
|
|
def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> (
|
|
None
|
|
):
|
|
built = spec(
|
|
"names_present",
|
|
{
|
|
"step": "init",
|
|
"names": ("patch-web-session-activity",),
|
|
"contains": ("@sha256:",),
|
|
},
|
|
steps=(step("init"),),
|
|
)
|
|
|
|
good = outcome(stdout="patch-web-session-activity\nimage@sha256:abc\n")
|
|
assert evaluate(built, {"init": good}).status == model.PASS
|
|
|
|
partial = evaluate(built, {"init": outcome(stdout="patch-web-session-activity\n")})
|
|
assert partial.status == model.FAIL and "@sha256:" in partial.reason
|
|
|
|
empty = spec("names_present", {"step": "init"}, steps=(step("init"),))
|
|
assert evaluate(empty, {"init": outcome(stdout="")}).status == model.NOT_RUN
|
|
|
|
|
|
def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None:
|
|
single = spec(
|
|
"stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),)
|
|
)
|
|
assert evaluate(single, {"s": outcome(stdout=" locked \n")}).status == model.PASS
|
|
|
|
mismatch = evaluate(single, {"s": outcome(stdout="unlocked")})
|
|
assert mismatch.status == model.FAIL and mismatch.evidence["observed"] == "unlocked"
|
|
|
|
several = spec(
|
|
"stdout_matches",
|
|
{"step": "s", "equals": ["600 root:root", "absent"]},
|
|
steps=(step("s"),),
|
|
)
|
|
assert evaluate(several, {"s": outcome(stdout="absent")}).status == model.PASS
|
|
|
|
assert evaluate(single, {"s": outcome(returncode=1)}).status == model.NOT_RUN
|
|
|
|
|
|
def test_distinct_count_separates_replica_count_from_actual_spread() -> None:
|
|
built = spec(
|
|
"distinct_count",
|
|
{"step": "n", "minimum": 3, "total_equals": 3},
|
|
steps=(step("n"),),
|
|
)
|
|
|
|
spread = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-06\ntitan-07\n")})
|
|
assert spread.status == model.PASS and spread.evidence["distinct"] == 3
|
|
|
|
stacked = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-05\ntitan-05\n")})
|
|
assert stacked.status == model.FAIL and "distinct values" in stacked.reason
|
|
|
|
extra = spec(
|
|
"distinct_count",
|
|
{"step": "n", "minimum": 1, "total_equals": 2},
|
|
steps=(step("n"),),
|
|
)
|
|
over = evaluate(extra, {"n": outcome(stdout="a\nb\nc\n")})
|
|
assert over.status == model.FAIL and "expected exactly 2" in over.reason
|
|
|
|
assert evaluate(built, {"n": outcome(stdout="")}).status == model.NOT_RUN
|
|
assert evaluate(built, {"n": outcome(returncode=1)}).status == model.NOT_RUN
|
|
|
|
|
|
def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None:
|
|
built = spec(
|
|
"vantages_agree",
|
|
{"steps": ("operator", "self")},
|
|
steps=(step("operator", vantage="operator"), step("self")),
|
|
)
|
|
|
|
agree = evaluate(
|
|
built,
|
|
{
|
|
"operator": outcome(stdout="ns/a\nns/b\n"),
|
|
"self": outcome(stdout="ns/b\nns/a\n"),
|
|
},
|
|
)
|
|
assert agree.status == model.PASS
|
|
|
|
differ = evaluate(
|
|
built,
|
|
{"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")},
|
|
)
|
|
assert differ.status == model.FAIL and "disagree" in differ.reason
|
|
|
|
empty = evaluate(
|
|
built, {"operator": outcome(stdout=""), "self": outcome(stdout="")}
|
|
)
|
|
assert empty.status == model.NOT_RUN
|
|
|
|
broken = evaluate(built, {"operator": outcome(returncode=1), "self": outcome()})
|
|
assert broken.status == model.NOT_RUN
|
|
|
|
|
|
def test_steps_agree_is_the_same_rule_under_a_neutral_name() -> None:
|
|
assert (
|
|
evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"]
|
|
)
|
|
|
|
|
|
def test_lines_match_catches_a_blank_leading_column() -> None:
|
|
built = spec(
|
|
"lines_match",
|
|
{"step": "s", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
|
|
steps=(step("s"),),
|
|
)
|
|
table = "Preview Src ID\n-------\nwork on task cli 2026\n"
|
|
|
|
assert evaluate(built, {"s": outcome(stdout=table)}).status == model.PASS
|
|
|
|
blank = "Preview Src ID\n-------\n cli 2026\n"
|
|
failed = evaluate(built, {"s": outcome(stdout=blank)})
|
|
assert failed.status == model.FAIL and failed.evidence["offenders"]
|
|
|
|
assert (
|
|
evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status
|
|
== model.NOT_RUN
|
|
)
|
|
assert evaluate(built, {"s": outcome(returncode=1)}).status == model.NOT_RUN
|
|
|
|
|
|
def test_not_armed_is_the_fixed_skip_a_default_run_must_produce() -> None:
|
|
built = spec("not_armed", {"reason": "ephemeral mutation mode is not armed"})
|
|
|
|
evaluation = evaluate(built, {})
|
|
|
|
assert evaluation.status == model.NOT_RUN
|
|
assert evaluation.evidence == {"armed": False}
|
|
assert evaluation.reason == "ephemeral mutation mode is not armed"
|
|
assert (
|
|
evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed"
|
|
)
|
|
|
|
|
|
def test_truncation_fails_closed_before_every_registered_evaluator() -> None:
|
|
"""No evaluator may reinterpret a clipped required capture as PASS."""
|
|
clipped = outcome(stdout="apparently good", truncated=True)
|
|
for name in evaluators.EVALUATORS:
|
|
built = spec(name, {"step": "s"}, steps=(step("s"),))
|
|
evaluation = evaluate(built, {"s": clipped})
|
|
assert evaluation.status == model.NOT_RUN, name
|
|
assert "truncated" in evaluation.reason, name
|
|
|
|
|
|
def test_missing_required_steps_fail_closed_inside_every_step_evaluator() -> None:
|
|
for name in evaluators.EVALUATORS:
|
|
if name == "not_armed":
|
|
continue
|
|
built = spec(name, {"step": "s"}, steps=(step("s"),))
|
|
assert evaluate(built, {}).status == model.NOT_RUN, name
|
|
|
|
|
|
def test_pool_topology_requires_three_ordinals_nodes_claims_and_ready_state() -> None:
|
|
built = spec(
|
|
"pool_topology",
|
|
{
|
|
"state_step": "state",
|
|
"worker_step": "workers",
|
|
"replicas": 3,
|
|
"name": "worker",
|
|
},
|
|
steps=(step("state"), step("workers")),
|
|
)
|
|
state = outcome(stdout="3\t3\t3\tfalse")
|
|
workers = outcome(
|
|
stdout=(
|
|
"worker-0\tnode-a\tRunning\tTrue\tclaim-0,\n"
|
|
"worker-1\tnode-b\tRunning\tTrue\tclaim-1,\n"
|
|
"worker-2\tnode-c\tRunning\tTrue\tclaim-2,\n"
|
|
)
|
|
)
|
|
assert evaluate(built, {"state": state, "workers": workers}).status == model.PASS
|
|
stacked = outcome(stdout=workers.stdout.replace("node-b", "node-a"))
|
|
assert evaluate(built, {"state": state, "workers": stacked}).status == model.FAIL
|
|
assert (
|
|
evaluate(built, {"state": outcome(stdout="3\t3"), "workers": workers}).status
|
|
== model.NOT_RUN
|
|
)
|
|
assert (
|
|
evaluate(
|
|
built, {"state": outcome(stdout="x\t3\t3\tfalse"), "workers": workers}
|
|
).status
|
|
== model.NOT_RUN
|
|
)
|
|
assert (
|
|
evaluate(
|
|
built, {"state": outcome(stdout="3\t2\t3\tfalse"), "workers": workers}
|
|
).status
|
|
== model.FAIL
|
|
)
|
|
malformed = outcome(stdout="worker-0\tnode-a\n")
|
|
assert (
|
|
evaluate(built, {"state": state, "workers": malformed}).status == model.NOT_RUN
|
|
)
|
|
bad_worker = outcome(stdout=workers.stdout.replace("Running", "Pending", 1))
|
|
assert evaluate(built, {"state": state, "workers": bad_worker}).status == model.FAIL
|
|
duplicate = outcome(stdout=workers.stdout.replace("worker-1", "worker-0"))
|
|
assert evaluate(built, {"state": state, "workers": duplicate}).status == model.FAIL
|
|
wrong_ordinals = outcome(stdout=workers.stdout.replace("worker-2", "worker-3"))
|
|
assert (
|
|
evaluate(built, {"state": state, "workers": wrong_ordinals}).status
|
|
== model.FAIL
|
|
)
|
|
|
|
|
|
MAIN = "1" * 40
|
|
HEAD = "2" * 40
|
|
BUILD = MAIN
|
|
DIGEST = "sha256:" + "4" * 64
|
|
IMAGE = f"registry.example/hermes:git-{BUILD}-build-7@{DIGEST}"
|
|
|
|
|
|
def lineage_spec():
|
|
keys = (
|
|
"remote-main",
|
|
"reviewed-pr",
|
|
"build-source",
|
|
"merge-ancestry",
|
|
"deployment",
|
|
"pods",
|
|
"replicasets",
|
|
"flux",
|
|
)
|
|
return spec(
|
|
"release_lineage",
|
|
{
|
|
"main_sha": MAIN,
|
|
"head_sha": HEAD,
|
|
"head_ref": "feature/hermes-full-handoff-acceptance",
|
|
"pr_number": 19,
|
|
"image": IMAGE,
|
|
"build_sha": BUILD,
|
|
"deployment_revision": "7",
|
|
},
|
|
steps=tuple(step(key) for key in keys),
|
|
)
|
|
|
|
|
|
def lineage_outcomes():
|
|
pull = {
|
|
"number": 19,
|
|
"state": "closed",
|
|
"draft": False,
|
|
"merged": True,
|
|
# The recorded base SHA is the pre-merge base, not the current main.
|
|
"base": {"ref": "main", "sha": "3" * 40},
|
|
"head": {"ref": "feature/hermes-full-handoff-acceptance", "sha": HEAD},
|
|
}
|
|
return {
|
|
"remote-main": outcome(stdout=f"{MAIN}\trefs/heads/main\n"),
|
|
"reviewed-pr": outcome(stdout=json.dumps(pull)),
|
|
"build-source": outcome(stdout=BUILD),
|
|
"merge-ancestry": outcome(),
|
|
"deployment": outcome(stdout=f"5\t5\t2\t2\t7\t{IMAGE}"),
|
|
"replicasets": outcome(
|
|
stdout=f"rs-old\t6\t0\t0\t{IMAGE}\told\nrs-live\t7\t2\t2\t{IMAGE}\thash7\n"
|
|
),
|
|
"pods": outcome(
|
|
stdout=(
|
|
f"pod-a\tRunning\tTrue\t{IMAGE}\tdocker-pullable://hermes@{DIGEST}\thash7\n"
|
|
f"pod-b\tRunning\tTrue\t{IMAGE}\tcontainerd://hermes@{DIGEST}\thash7\n"
|
|
)
|
|
),
|
|
"flux": outcome(stdout=f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{MAIN}"),
|
|
}
|
|
|
|
|
|
def test_release_lineage_atomically_binds_every_release_identity() -> None:
|
|
assert evaluate(lineage_spec(), lineage_outcomes()).status == model.PASS
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"patch",
|
|
[
|
|
{"state": "open", "draft": True, "merged": False},
|
|
{"state": "closed", "merged": False},
|
|
{"head": {"ref": "feature/hermes-full-handoff-acceptance", "sha": "9" * 40}},
|
|
],
|
|
ids=["still-open-draft", "closed-without-merging", "wrong-head-sha"],
|
|
)
|
|
def test_release_lineage_accepts_only_the_merged_terminal_pr_state(patch) -> None:
|
|
"""After integration, only the merged PR proves the reviewed code runs."""
|
|
outcomes = lineage_outcomes()
|
|
pull = {**json.loads(outcomes["reviewed-pr"].stdout), **patch}
|
|
outcomes["reviewed-pr"] = outcome(stdout=json.dumps(pull))
|
|
|
|
assert evaluate(lineage_spec(), outcomes).status == model.FAIL
|
|
|
|
|
|
def test_release_lineage_requires_the_merged_head_to_be_an_ancestor_of_main() -> None:
|
|
not_ancestor = lineage_outcomes()
|
|
not_ancestor["merge-ancestry"] = outcome(returncode=1)
|
|
assert evaluate(lineage_spec(), not_ancestor).status == model.FAIL
|
|
|
|
undecidable = lineage_outcomes()
|
|
undecidable["merge-ancestry"] = outcome(returncode=128)
|
|
assert evaluate(lineage_spec(), undecidable).status == model.NOT_RUN
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("key", "bad"),
|
|
[
|
|
("remote-main", ""),
|
|
("reviewed-pr", "{}"),
|
|
("build-source", HEAD),
|
|
("deployment", "5\t4\t2\t2\t7\tbad"),
|
|
("replicasets", "malformed"),
|
|
("pods", "malformed"),
|
|
("flux", "hermes\t9\t9\tfalse\tUnknown\tmain@sha1:" + MAIN),
|
|
],
|
|
)
|
|
def test_release_lineage_fails_closed_on_each_malformed_or_mismatched_source(
|
|
key, bad
|
|
) -> None:
|
|
outcomes = lineage_outcomes()
|
|
outcomes[key] = outcome(stdout=bad)
|
|
assert evaluate(lineage_spec(), outcomes).status in {model.FAIL, model.NOT_RUN}
|
|
|
|
|
|
def test_release_lineage_rejects_structural_partial_failure_variants() -> None:
|
|
variants = []
|
|
malformed_expect = lineage_spec()
|
|
malformed_expect = model.CheckSpec(
|
|
**{
|
|
**malformed_expect.__dict__,
|
|
"expect": {**malformed_expect.expect, "main_sha": "bad"},
|
|
}
|
|
)
|
|
variants.append((malformed_expect, lineage_outcomes()))
|
|
malformed_image = lineage_spec()
|
|
malformed_image = model.CheckSpec(
|
|
**{
|
|
**malformed_image.__dict__,
|
|
"expect": {
|
|
**malformed_image.expect,
|
|
"image": f"registry.example/hermes@{DIGEST}",
|
|
},
|
|
}
|
|
)
|
|
variants.append((malformed_image, lineage_outcomes()))
|
|
for key, value in (
|
|
("reviewed-pr", "[]"),
|
|
("deployment", f"x\tx\t2\t2\t7\t{IMAGE}"),
|
|
("replicasets", f"rs\t7\t1\t2\t{IMAGE}\thash7"),
|
|
("replicasets", f"a\t7\t2\t2\t{IMAGE}\th1\nb\t7\t2\t2\t{IMAGE}\th2"),
|
|
("replicasets", f"rs\t6\t2\t2\t{IMAGE}\thash7"),
|
|
("pods", f"pod-a\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7"),
|
|
(
|
|
"pods",
|
|
f"pod-a\tPending\tFalse\t{IMAGE}\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7",
|
|
),
|
|
(
|
|
"pods",
|
|
f"pod-a\tRunning\tTrue\tbad\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7",
|
|
),
|
|
("flux", f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{HEAD}"),
|
|
):
|
|
outcomes = lineage_outcomes()
|
|
outcomes[key] = outcome(stdout=value)
|
|
variants.append((lineage_spec(), outcomes))
|
|
for built, outcomes in variants:
|
|
assert evaluate(built, outcomes).status in {model.FAIL, model.NOT_RUN}
|