atlas-iac/testing/tests/test_hermes_handoff_ephemeral.py
Hermes Agent 8f00545828 hermes: add a fail-closed full-handoff acceptance harness
Decides whether the Hermes platform handoff is fit to release, and refuses
to round an absence of evidence up to a pass.

The harness is read-only by default and classifies 71 checks PASS / FAIL /
NOT_RUN / NOT_APPLICABLE. Any mandatory FAIL or NOT_RUN is NO_GO, and so is a
harness-level problem: an unreachable vantage, a catalog entry whose evidence
no longer exists, an expired deadline, or an evaluator that raised.

Evidence comes from two vantages that cannot cover for each other: an external
read-only operator kubeconfig, and the Hermes agent probing itself from inside
its own pod. Before any check runs, the harness asks each vantage who it is and
stops if they are the same principal, because dual-vantage evidence from one
identity is a restatement rather than a corroboration. `--as` is rejected for
every operator-side command and reachable only as the inner command of a
`kubectl exec`, so impersonation can never stand in for a real self-probe. A
deny check needs a live refused request, not only an authorization review.

Two safety properties are structural rather than conventional, enforced where
an argv becomes a subprocess: the default mode mutates nothing (mutating verbs
require a server dry run; there is deliberately no live TokenRequest probe,
because a successful one would mint a real credential), and no probe can pull a
credential value into a report (no vault/sops/curl, secrets readable only with
-o name, environment probes list names, shell only through frozen reviewed
templates). Captures are bounded before they are screened, and the rendered
report is re-screened before it is written.

Mutation lives behind a separate arming flag with an exact confirmation phrase,
a caller-supplied unique ref, a preflight that refuses a protected push target
before any network call, and a cleanup whose verification is itself mandatory.
A default run reports those four checks NOT_RUN.

The catalog is declarative so a reviewer reads what is asserted rather than how
it is plumbed, and so structural properties can be proven over every entry
before a run. Catalog drift surfaces as NOT_RUN, which stops the release.

docs/hermes_full_handoff_acceptance.md carries the merge order for PRs #14-#18
on top of the merged #13 baseline, the image build and Flux rollout, the
rollback point for each step, the go/no-go checklist, and the limits that are
asserted rather than exercised.

Validation: 295 handoff tests pass with 100% line coverage on all 15 new
modules; the full unit suite is 647 passed with two failures that reproduce
unchanged on origin/main; Ruff, py_compile, kustomize render, and a diff
credential screen are clean; a live read-only run against Atlas returns NO_GO
for the pre-merge cluster with no unscreened fields in the report.
2026-08-17 10:14:17 +00:00

272 lines
9.7 KiB
Python

"""Contracts for the separately armed, self-cleaning mutation mode."""
from __future__ import annotations
import json
import subprocess
import pytest
from testing.tests.test_hermes_handoff_support import (
FakeClock,
FakeSpawn,
load_handoff_module,
)
ephemeral = load_handoff_module("hermes_handoff_ephemeral")
exec_module = load_handoff_module("hermes_handoff_exec")
policy = load_handoff_module("hermes_handoff_policy")
model = load_handoff_module("hermes_handoff_model")
REPO = "atlas/titan-iac"
TOKEN = "acceptance-20260817a"
VANTAGE = exec_module.operator_vantage()
def request(**overrides) -> object:
fields = {
"repo": REPO,
"remote": "origin",
"token": TOKEN,
"confirmation": ephemeral.CONFIRMATION,
"base": "main",
}
fields.update(overrides)
return ephemeral.ArmRequest(**fields)
def completed(stdout: str = "", stderr: str = "", returncode: int = 0):
return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr)
def armed_runner(results: list) -> tuple:
spawn = FakeSpawn(results)
runner = exec_module.Runner(
mode=policy.ARMED, clock=FakeClock(), spawn=spawn, environ={}, deadline_seconds=3600
)
return (runner, spawn)
def by_id(results: list) -> dict:
return {result.spec.id: result for result in results}
def test_the_ephemeral_ref_is_derived_from_the_caller_supplied_token() -> None:
assert request().ref == f"{ephemeral.REF_PREFIX}/{TOKEN}"
@pytest.mark.parametrize(
"ref", ["main", "master", "refs/heads/main", "HEAD", "Production", "feature/x", ""]
)
def test_protected_and_malformed_push_targets_are_refused(ref: str) -> None:
with pytest.raises(ephemeral.ArmingError):
ephemeral.assert_push_target_allowed(ref)
def test_an_ephemeral_ref_is_the_only_accepted_push_target() -> None:
ephemeral.assert_push_target_allowed(f"{ephemeral.REF_PREFIX}/{TOKEN}")
@pytest.mark.parametrize(
("overrides", "fragment"),
[
({"confirmation": "yes please"}, "confirmation phrase"),
({"confirmation": ""}, "confirmation phrase"),
({"repo": "atlas/other"}, "is not the expected"),
({"token": "short"}, "ephemeral token"),
({"token": "Has-Capitals-Here"}, "ephemeral token"),
({"token": ""}, "ephemeral token"),
({"base": ""}, "base branch is required"),
],
)
def test_preflight_refuses_before_any_network_call(overrides: dict, fragment: str) -> None:
with pytest.raises(ephemeral.ArmingError) as caught:
ephemeral.preflight(request(**overrides), REPO)
assert fragment in str(caught.value)
def test_preflight_accepts_a_correctly_armed_request() -> None:
ephemeral.preflight(request(), REPO)
@pytest.mark.parametrize(
("raw", "expected"), [("refs/heads/Main", "main"), (" main ", "main"), ("/x/", "x")]
)
def test_branch_names_are_normalised_before_comparison(raw: str, expected: str) -> None:
assert ephemeral.normalise_branch(raw) == expected
def test_the_protected_branch_guard_is_exercised_on_every_armed_run() -> None:
"""A guard checked once is a guard nobody notices regressing."""
runner, _ = armed_runner([completed(stdout="", returncode=2)])
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
guard = results["ephemeral.protected-branch-refusal"]
assert guard.status == model.PASS
assert guard.evidence["accepted"] == []
assert "main" in guard.evidence["refused"]
def test_a_broken_protected_branch_guard_fails_its_own_self_test(monkeypatch) -> None:
"""If the guard ever stops refusing, the armed run says so instead of pushing."""
monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None)
runner, _ = armed_runner([completed(stdout="", returncode=2)])
guard = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[
"ephemeral.protected-branch-refusal"
]
assert guard.status == model.FAIL
assert "main" in guard.evidence["accepted"]
assert "the guard accepted" in guard.reason
def test_a_full_armed_run_pushes_opens_a_draft_and_removes_both() -> None:
runner, spawn = armed_runner(
[
completed(stdout=""), # ls-remote: ref unused
completed(stdout=""), # push
completed(stdout=json.dumps({"number": 42, "draft": True})), # create draft
completed(stdout=""), # close
completed(stdout=""), # delete branch
completed(stdout=""), # ls-remote: gone
completed(stdout=json.dumps({"number": 42, "state": "closed"})),
]
)
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
assert results["ephemeral.feature-branch-push"].status == model.PASS
assert results["ephemeral.draft-pull-request"].status == model.PASS
assert results["ephemeral.draft-pull-request"].evidence["pull_request"] == 42
assert results["ephemeral.cleanup-verified"].status == model.PASS
assert all(result.spec.mandatory for result in results.values())
assert all(result.spec.scope == model.EPHEMERAL for result in results.values())
pushed = [call for call in spawn.calls if call["argv"][:2] == ["git", "push"]]
assert pushed[0]["argv"][-1] == f"HEAD:refs/heads/{request().ref}"
assert "--force" not in pushed[0]["argv"]
titles = [arg for call in spawn.calls for arg in call["argv"] if arg.startswith("title=")]
assert titles[0].startswith(f"title={ephemeral.DRAFT_TITLE_PREFIX}")
def test_an_existing_ref_stops_the_run_before_it_writes() -> None:
runner, spawn = armed_runner([completed(stdout="abc123\trefs/heads/x\n")])
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
assert results["ephemeral.feature-branch-push"].status == model.FAIL
assert "already exists" in results["ephemeral.feature-branch-push"].reason
assert "ephemeral.draft-pull-request" not in results
assert len(spawn.calls) == 1
def test_an_unverifiable_ref_check_is_not_run_rather_than_assumed_free() -> None:
runner, _ = armed_runner([completed(stderr="network down", returncode=128)])
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
assert results["ephemeral.feature-branch-push"].status == model.NOT_RUN
def test_a_failed_push_still_reports_cleanup_and_skips_the_draft() -> None:
runner, _ = armed_runner(
[
completed(stdout=""),
completed(stderr="denied", returncode=1),
completed(stdout=""), # delete branch
completed(stdout=""), # ls-remote: gone
]
)
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
assert results["ephemeral.feature-branch-push"].status == model.FAIL
assert results["ephemeral.draft-pull-request"].status == model.NOT_RUN
assert results["ephemeral.cleanup-verified"].status == model.PASS
def test_a_draft_response_without_a_number_is_a_failure() -> None:
runner, _ = armed_runner(
[
completed(stdout=""),
completed(stdout=""),
completed(stdout="{}"),
completed(stdout=""),
completed(stdout=""),
]
)
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
assert results["ephemeral.draft-pull-request"].status == model.FAIL
def test_a_branch_that_survives_cleanup_fails_the_cleanup_check() -> None:
runner, _ = armed_runner(
[
completed(stdout=""),
completed(stdout=""),
completed(stdout=json.dumps({"number": 7})),
completed(stderr="cannot close", returncode=1),
completed(stderr="cannot delete", returncode=1),
completed(stdout="abc123\trefs/heads/x\n"),
completed(stdout=json.dumps({"number": 7, "state": "open"})),
]
)
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
cleanup = results["ephemeral.cleanup-verified"]
assert cleanup.status == model.FAIL
assert "could not be closed" in cleanup.reason
assert "still exists" in cleanup.reason
assert "still open" in cleanup.reason
def test_cleanup_that_cannot_be_verified_is_a_failure_not_a_pass() -> None:
runner, _ = armed_runner(
[
completed(stdout=""),
completed(stdout=""),
completed(stdout=json.dumps({"number": 7})),
completed(stdout=""),
completed(stdout=""),
subprocess.TimeoutExpired(cmd="git", timeout=1.0),
completed(stderr="gone", returncode=1),
]
)
cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))["ephemeral.cleanup-verified"]
assert cleanup.status == model.FAIL
assert "could not verify removal" in cleanup.reason
assert "could not confirm pull request #7 is closed" in cleanup.reason
def test_the_armed_results_replace_the_default_not_run_placeholders() -> None:
runner, _ = armed_runner([completed(stdout="abc\trefs/heads/x\n")])
identifiers = {result.spec.id for result in ephemeral.run_armed(runner, VANTAGE, request())}
assert identifiers <= {
"ephemeral.protected-branch-refusal",
"ephemeral.feature-branch-push",
"ephemeral.draft-pull-request",
"ephemeral.cleanup-verified",
}
def test_a_read_only_runner_cannot_execute_the_armed_flow() -> None:
"""The mode, not the caller, is what opens the mutations."""
spawn = FakeSpawn([completed(stdout=""), completed(stdout=""), completed(stdout="")])
runner = exec_module.Runner(clock=FakeClock(), spawn=spawn, environ={})
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
push = results["ephemeral.feature-branch-push"]
assert push.status == model.FAIL
assert "policy" in push.reason
assert not any(call["argv"][:2] == ["git", "push"] for call in spawn.calls)