Decides whether the Hermes platform handoff is fit to release, and refuses to round an absence of evidence up to a pass. The harness is read-only by default and classifies 71 checks PASS / FAIL / NOT_RUN / NOT_APPLICABLE. Any mandatory FAIL or NOT_RUN is NO_GO, and so is a harness-level problem: an unreachable vantage, a catalog entry whose evidence no longer exists, an expired deadline, or an evaluator that raised. Evidence comes from two vantages that cannot cover for each other: an external read-only operator kubeconfig, and the Hermes agent probing itself from inside its own pod. Before any check runs, the harness asks each vantage who it is and stops if they are the same principal, because dual-vantage evidence from one identity is a restatement rather than a corroboration. `--as` is rejected for every operator-side command and reachable only as the inner command of a `kubectl exec`, so impersonation can never stand in for a real self-probe. A deny check needs a live refused request, not only an authorization review. Two safety properties are structural rather than conventional, enforced where an argv becomes a subprocess: the default mode mutates nothing (mutating verbs require a server dry run; there is deliberately no live TokenRequest probe, because a successful one would mint a real credential), and no probe can pull a credential value into a report (no vault/sops/curl, secrets readable only with -o name, environment probes list names, shell only through frozen reviewed templates). Captures are bounded before they are screened, and the rendered report is re-screened before it is written. Mutation lives behind a separate arming flag with an exact confirmation phrase, a caller-supplied unique ref, a preflight that refuses a protected push target before any network call, and a cleanup whose verification is itself mandatory. A default run reports those four checks NOT_RUN. The catalog is declarative so a reviewer reads what is asserted rather than how it is plumbed, and so structural properties can be proven over every entry before a run. Catalog drift surfaces as NOT_RUN, which stops the release. docs/hermes_full_handoff_acceptance.md carries the merge order for PRs #14-#18 on top of the merged #13 baseline, the image build and Flux rollout, the rollback point for each step, the go/no-go checklist, and the limits that are asserted rather than exercised. Validation: 295 handoff tests pass with 100% line coverage on all 15 new modules; the full unit suite is 647 passed with two failures that reproduce unchanged on origin/main; Ruff, py_compile, kustomize render, and a diff credential screen are clean; a live read-only run against Atlas returns NO_GO for the pre-merge cluster with no unscreened fields in the report.
272 lines
9.7 KiB
Python
272 lines
9.7 KiB
Python
"""Contracts for the separately armed, self-cleaning mutation mode."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
|
|
import pytest
|
|
|
|
from testing.tests.test_hermes_handoff_support import (
|
|
FakeClock,
|
|
FakeSpawn,
|
|
load_handoff_module,
|
|
)
|
|
|
|
ephemeral = load_handoff_module("hermes_handoff_ephemeral")
|
|
exec_module = load_handoff_module("hermes_handoff_exec")
|
|
policy = load_handoff_module("hermes_handoff_policy")
|
|
model = load_handoff_module("hermes_handoff_model")
|
|
|
|
REPO = "atlas/titan-iac"
|
|
TOKEN = "acceptance-20260817a"
|
|
VANTAGE = exec_module.operator_vantage()
|
|
|
|
|
|
def request(**overrides) -> object:
|
|
fields = {
|
|
"repo": REPO,
|
|
"remote": "origin",
|
|
"token": TOKEN,
|
|
"confirmation": ephemeral.CONFIRMATION,
|
|
"base": "main",
|
|
}
|
|
fields.update(overrides)
|
|
return ephemeral.ArmRequest(**fields)
|
|
|
|
|
|
def completed(stdout: str = "", stderr: str = "", returncode: int = 0):
|
|
return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr)
|
|
|
|
|
|
def armed_runner(results: list) -> tuple:
|
|
spawn = FakeSpawn(results)
|
|
runner = exec_module.Runner(
|
|
mode=policy.ARMED, clock=FakeClock(), spawn=spawn, environ={}, deadline_seconds=3600
|
|
)
|
|
return (runner, spawn)
|
|
|
|
|
|
def by_id(results: list) -> dict:
|
|
return {result.spec.id: result for result in results}
|
|
|
|
|
|
def test_the_ephemeral_ref_is_derived_from_the_caller_supplied_token() -> None:
|
|
assert request().ref == f"{ephemeral.REF_PREFIX}/{TOKEN}"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"ref", ["main", "master", "refs/heads/main", "HEAD", "Production", "feature/x", ""]
|
|
)
|
|
def test_protected_and_malformed_push_targets_are_refused(ref: str) -> None:
|
|
with pytest.raises(ephemeral.ArmingError):
|
|
ephemeral.assert_push_target_allowed(ref)
|
|
|
|
|
|
def test_an_ephemeral_ref_is_the_only_accepted_push_target() -> None:
|
|
ephemeral.assert_push_target_allowed(f"{ephemeral.REF_PREFIX}/{TOKEN}")
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("overrides", "fragment"),
|
|
[
|
|
({"confirmation": "yes please"}, "confirmation phrase"),
|
|
({"confirmation": ""}, "confirmation phrase"),
|
|
({"repo": "atlas/other"}, "is not the expected"),
|
|
({"token": "short"}, "ephemeral token"),
|
|
({"token": "Has-Capitals-Here"}, "ephemeral token"),
|
|
({"token": ""}, "ephemeral token"),
|
|
({"base": ""}, "base branch is required"),
|
|
],
|
|
)
|
|
def test_preflight_refuses_before_any_network_call(overrides: dict, fragment: str) -> None:
|
|
with pytest.raises(ephemeral.ArmingError) as caught:
|
|
ephemeral.preflight(request(**overrides), REPO)
|
|
assert fragment in str(caught.value)
|
|
|
|
|
|
def test_preflight_accepts_a_correctly_armed_request() -> None:
|
|
ephemeral.preflight(request(), REPO)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("raw", "expected"), [("refs/heads/Main", "main"), (" main ", "main"), ("/x/", "x")]
|
|
)
|
|
def test_branch_names_are_normalised_before_comparison(raw: str, expected: str) -> None:
|
|
assert ephemeral.normalise_branch(raw) == expected
|
|
|
|
|
|
def test_the_protected_branch_guard_is_exercised_on_every_armed_run() -> None:
|
|
"""A guard checked once is a guard nobody notices regressing."""
|
|
runner, _ = armed_runner([completed(stdout="", returncode=2)])
|
|
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
|
guard = results["ephemeral.protected-branch-refusal"]
|
|
|
|
assert guard.status == model.PASS
|
|
assert guard.evidence["accepted"] == []
|
|
assert "main" in guard.evidence["refused"]
|
|
|
|
|
|
def test_a_broken_protected_branch_guard_fails_its_own_self_test(monkeypatch) -> None:
|
|
"""If the guard ever stops refusing, the armed run says so instead of pushing."""
|
|
monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None)
|
|
runner, _ = armed_runner([completed(stdout="", returncode=2)])
|
|
|
|
guard = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[
|
|
"ephemeral.protected-branch-refusal"
|
|
]
|
|
|
|
assert guard.status == model.FAIL
|
|
assert "main" in guard.evidence["accepted"]
|
|
assert "the guard accepted" in guard.reason
|
|
|
|
|
|
def test_a_full_armed_run_pushes_opens_a_draft_and_removes_both() -> None:
|
|
runner, spawn = armed_runner(
|
|
[
|
|
completed(stdout=""), # ls-remote: ref unused
|
|
completed(stdout=""), # push
|
|
completed(stdout=json.dumps({"number": 42, "draft": True})), # create draft
|
|
completed(stdout=""), # close
|
|
completed(stdout=""), # delete branch
|
|
completed(stdout=""), # ls-remote: gone
|
|
completed(stdout=json.dumps({"number": 42, "state": "closed"})),
|
|
]
|
|
)
|
|
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
|
|
|
assert results["ephemeral.feature-branch-push"].status == model.PASS
|
|
assert results["ephemeral.draft-pull-request"].status == model.PASS
|
|
assert results["ephemeral.draft-pull-request"].evidence["pull_request"] == 42
|
|
assert results["ephemeral.cleanup-verified"].status == model.PASS
|
|
assert all(result.spec.mandatory for result in results.values())
|
|
assert all(result.spec.scope == model.EPHEMERAL for result in results.values())
|
|
|
|
pushed = [call for call in spawn.calls if call["argv"][:2] == ["git", "push"]]
|
|
assert pushed[0]["argv"][-1] == f"HEAD:refs/heads/{request().ref}"
|
|
assert "--force" not in pushed[0]["argv"]
|
|
|
|
titles = [arg for call in spawn.calls for arg in call["argv"] if arg.startswith("title=")]
|
|
assert titles[0].startswith(f"title={ephemeral.DRAFT_TITLE_PREFIX}")
|
|
|
|
|
|
def test_an_existing_ref_stops_the_run_before_it_writes() -> None:
|
|
runner, spawn = armed_runner([completed(stdout="abc123\trefs/heads/x\n")])
|
|
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
|
|
|
assert results["ephemeral.feature-branch-push"].status == model.FAIL
|
|
assert "already exists" in results["ephemeral.feature-branch-push"].reason
|
|
assert "ephemeral.draft-pull-request" not in results
|
|
assert len(spawn.calls) == 1
|
|
|
|
|
|
def test_an_unverifiable_ref_check_is_not_run_rather_than_assumed_free() -> None:
|
|
runner, _ = armed_runner([completed(stderr="network down", returncode=128)])
|
|
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
|
|
|
assert results["ephemeral.feature-branch-push"].status == model.NOT_RUN
|
|
|
|
|
|
def test_a_failed_push_still_reports_cleanup_and_skips_the_draft() -> None:
|
|
runner, _ = armed_runner(
|
|
[
|
|
completed(stdout=""),
|
|
completed(stderr="denied", returncode=1),
|
|
completed(stdout=""), # delete branch
|
|
completed(stdout=""), # ls-remote: gone
|
|
]
|
|
)
|
|
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
|
|
|
assert results["ephemeral.feature-branch-push"].status == model.FAIL
|
|
assert results["ephemeral.draft-pull-request"].status == model.NOT_RUN
|
|
assert results["ephemeral.cleanup-verified"].status == model.PASS
|
|
|
|
|
|
def test_a_draft_response_without_a_number_is_a_failure() -> None:
|
|
runner, _ = armed_runner(
|
|
[
|
|
completed(stdout=""),
|
|
completed(stdout=""),
|
|
completed(stdout="{}"),
|
|
completed(stdout=""),
|
|
completed(stdout=""),
|
|
]
|
|
)
|
|
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
|
|
|
assert results["ephemeral.draft-pull-request"].status == model.FAIL
|
|
|
|
|
|
def test_a_branch_that_survives_cleanup_fails_the_cleanup_check() -> None:
|
|
runner, _ = armed_runner(
|
|
[
|
|
completed(stdout=""),
|
|
completed(stdout=""),
|
|
completed(stdout=json.dumps({"number": 7})),
|
|
completed(stderr="cannot close", returncode=1),
|
|
completed(stderr="cannot delete", returncode=1),
|
|
completed(stdout="abc123\trefs/heads/x\n"),
|
|
completed(stdout=json.dumps({"number": 7, "state": "open"})),
|
|
]
|
|
)
|
|
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
|
cleanup = results["ephemeral.cleanup-verified"]
|
|
|
|
assert cleanup.status == model.FAIL
|
|
assert "could not be closed" in cleanup.reason
|
|
assert "still exists" in cleanup.reason
|
|
assert "still open" in cleanup.reason
|
|
|
|
|
|
def test_cleanup_that_cannot_be_verified_is_a_failure_not_a_pass() -> None:
|
|
runner, _ = armed_runner(
|
|
[
|
|
completed(stdout=""),
|
|
completed(stdout=""),
|
|
completed(stdout=json.dumps({"number": 7})),
|
|
completed(stdout=""),
|
|
completed(stdout=""),
|
|
subprocess.TimeoutExpired(cmd="git", timeout=1.0),
|
|
completed(stderr="gone", returncode=1),
|
|
]
|
|
)
|
|
|
|
cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))["ephemeral.cleanup-verified"]
|
|
|
|
assert cleanup.status == model.FAIL
|
|
assert "could not verify removal" in cleanup.reason
|
|
assert "could not confirm pull request #7 is closed" in cleanup.reason
|
|
|
|
|
|
def test_the_armed_results_replace_the_default_not_run_placeholders() -> None:
|
|
runner, _ = armed_runner([completed(stdout="abc\trefs/heads/x\n")])
|
|
identifiers = {result.spec.id for result in ephemeral.run_armed(runner, VANTAGE, request())}
|
|
|
|
assert identifiers <= {
|
|
"ephemeral.protected-branch-refusal",
|
|
"ephemeral.feature-branch-push",
|
|
"ephemeral.draft-pull-request",
|
|
"ephemeral.cleanup-verified",
|
|
}
|
|
|
|
|
|
def test_a_read_only_runner_cannot_execute_the_armed_flow() -> None:
|
|
"""The mode, not the caller, is what opens the mutations."""
|
|
spawn = FakeSpawn([completed(stdout=""), completed(stdout=""), completed(stdout="")])
|
|
runner = exec_module.Runner(clock=FakeClock(), spawn=spawn, environ={})
|
|
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
|
push = results["ephemeral.feature-branch-push"]
|
|
|
|
assert push.status == model.FAIL
|
|
assert "policy" in push.reason
|
|
assert not any(call["argv"][:2] == ["git", "push"] for call in spawn.calls)
|