272 lines
9.7 KiB
Python
272 lines
9.7 KiB
Python
|
|
"""Contracts for the separately armed, self-cleaning mutation mode."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import json
|
||
|
|
import subprocess
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from testing.tests.test_hermes_handoff_support import (
|
||
|
|
FakeClock,
|
||
|
|
FakeSpawn,
|
||
|
|
load_handoff_module,
|
||
|
|
)
|
||
|
|
|
||
|
|
ephemeral = load_handoff_module("hermes_handoff_ephemeral")
|
||
|
|
exec_module = load_handoff_module("hermes_handoff_exec")
|
||
|
|
policy = load_handoff_module("hermes_handoff_policy")
|
||
|
|
model = load_handoff_module("hermes_handoff_model")
|
||
|
|
|
||
|
|
REPO = "atlas/titan-iac"
|
||
|
|
TOKEN = "acceptance-20260817a"
|
||
|
|
VANTAGE = exec_module.operator_vantage()
|
||
|
|
|
||
|
|
|
||
|
|
def request(**overrides) -> object:
|
||
|
|
fields = {
|
||
|
|
"repo": REPO,
|
||
|
|
"remote": "origin",
|
||
|
|
"token": TOKEN,
|
||
|
|
"confirmation": ephemeral.CONFIRMATION,
|
||
|
|
"base": "main",
|
||
|
|
}
|
||
|
|
fields.update(overrides)
|
||
|
|
return ephemeral.ArmRequest(**fields)
|
||
|
|
|
||
|
|
|
||
|
|
def completed(stdout: str = "", stderr: str = "", returncode: int = 0):
|
||
|
|
return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr)
|
||
|
|
|
||
|
|
|
||
|
|
def armed_runner(results: list) -> tuple:
|
||
|
|
spawn = FakeSpawn(results)
|
||
|
|
runner = exec_module.Runner(
|
||
|
|
mode=policy.ARMED, clock=FakeClock(), spawn=spawn, environ={}, deadline_seconds=3600
|
||
|
|
)
|
||
|
|
return (runner, spawn)
|
||
|
|
|
||
|
|
|
||
|
|
def by_id(results: list) -> dict:
|
||
|
|
return {result.spec.id: result for result in results}
|
||
|
|
|
||
|
|
|
||
|
|
def test_the_ephemeral_ref_is_derived_from_the_caller_supplied_token() -> None:
|
||
|
|
assert request().ref == f"{ephemeral.REF_PREFIX}/{TOKEN}"
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"ref", ["main", "master", "refs/heads/main", "HEAD", "Production", "feature/x", ""]
|
||
|
|
)
|
||
|
|
def test_protected_and_malformed_push_targets_are_refused(ref: str) -> None:
|
||
|
|
with pytest.raises(ephemeral.ArmingError):
|
||
|
|
ephemeral.assert_push_target_allowed(ref)
|
||
|
|
|
||
|
|
|
||
|
|
def test_an_ephemeral_ref_is_the_only_accepted_push_target() -> None:
|
||
|
|
ephemeral.assert_push_target_allowed(f"{ephemeral.REF_PREFIX}/{TOKEN}")
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
("overrides", "fragment"),
|
||
|
|
[
|
||
|
|
({"confirmation": "yes please"}, "confirmation phrase"),
|
||
|
|
({"confirmation": ""}, "confirmation phrase"),
|
||
|
|
({"repo": "atlas/other"}, "is not the expected"),
|
||
|
|
({"token": "short"}, "ephemeral token"),
|
||
|
|
({"token": "Has-Capitals-Here"}, "ephemeral token"),
|
||
|
|
({"token": ""}, "ephemeral token"),
|
||
|
|
({"base": ""}, "base branch is required"),
|
||
|
|
],
|
||
|
|
)
|
||
|
|
def test_preflight_refuses_before_any_network_call(overrides: dict, fragment: str) -> None:
|
||
|
|
with pytest.raises(ephemeral.ArmingError) as caught:
|
||
|
|
ephemeral.preflight(request(**overrides), REPO)
|
||
|
|
assert fragment in str(caught.value)
|
||
|
|
|
||
|
|
|
||
|
|
def test_preflight_accepts_a_correctly_armed_request() -> None:
|
||
|
|
ephemeral.preflight(request(), REPO)
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
("raw", "expected"), [("refs/heads/Main", "main"), (" main ", "main"), ("/x/", "x")]
|
||
|
|
)
|
||
|
|
def test_branch_names_are_normalised_before_comparison(raw: str, expected: str) -> None:
|
||
|
|
assert ephemeral.normalise_branch(raw) == expected
|
||
|
|
|
||
|
|
|
||
|
|
def test_the_protected_branch_guard_is_exercised_on_every_armed_run() -> None:
|
||
|
|
"""A guard checked once is a guard nobody notices regressing."""
|
||
|
|
runner, _ = armed_runner([completed(stdout="", returncode=2)])
|
||
|
|
|
||
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
||
|
|
guard = results["ephemeral.protected-branch-refusal"]
|
||
|
|
|
||
|
|
assert guard.status == model.PASS
|
||
|
|
assert guard.evidence["accepted"] == []
|
||
|
|
assert "main" in guard.evidence["refused"]
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_broken_protected_branch_guard_fails_its_own_self_test(monkeypatch) -> None:
|
||
|
|
"""If the guard ever stops refusing, the armed run says so instead of pushing."""
|
||
|
|
monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None)
|
||
|
|
runner, _ = armed_runner([completed(stdout="", returncode=2)])
|
||
|
|
|
||
|
|
guard = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[
|
||
|
|
"ephemeral.protected-branch-refusal"
|
||
|
|
]
|
||
|
|
|
||
|
|
assert guard.status == model.FAIL
|
||
|
|
assert "main" in guard.evidence["accepted"]
|
||
|
|
assert "the guard accepted" in guard.reason
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_full_armed_run_pushes_opens_a_draft_and_removes_both() -> None:
|
||
|
|
runner, spawn = armed_runner(
|
||
|
|
[
|
||
|
|
completed(stdout=""), # ls-remote: ref unused
|
||
|
|
completed(stdout=""), # push
|
||
|
|
completed(stdout=json.dumps({"number": 42, "draft": True})), # create draft
|
||
|
|
completed(stdout=""), # close
|
||
|
|
completed(stdout=""), # delete branch
|
||
|
|
completed(stdout=""), # ls-remote: gone
|
||
|
|
completed(stdout=json.dumps({"number": 42, "state": "closed"})),
|
||
|
|
]
|
||
|
|
)
|
||
|
|
|
||
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
||
|
|
|
||
|
|
assert results["ephemeral.feature-branch-push"].status == model.PASS
|
||
|
|
assert results["ephemeral.draft-pull-request"].status == model.PASS
|
||
|
|
assert results["ephemeral.draft-pull-request"].evidence["pull_request"] == 42
|
||
|
|
assert results["ephemeral.cleanup-verified"].status == model.PASS
|
||
|
|
assert all(result.spec.mandatory for result in results.values())
|
||
|
|
assert all(result.spec.scope == model.EPHEMERAL for result in results.values())
|
||
|
|
|
||
|
|
pushed = [call for call in spawn.calls if call["argv"][:2] == ["git", "push"]]
|
||
|
|
assert pushed[0]["argv"][-1] == f"HEAD:refs/heads/{request().ref}"
|
||
|
|
assert "--force" not in pushed[0]["argv"]
|
||
|
|
|
||
|
|
titles = [arg for call in spawn.calls for arg in call["argv"] if arg.startswith("title=")]
|
||
|
|
assert titles[0].startswith(f"title={ephemeral.DRAFT_TITLE_PREFIX}")
|
||
|
|
|
||
|
|
|
||
|
|
def test_an_existing_ref_stops_the_run_before_it_writes() -> None:
|
||
|
|
runner, spawn = armed_runner([completed(stdout="abc123\trefs/heads/x\n")])
|
||
|
|
|
||
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
||
|
|
|
||
|
|
assert results["ephemeral.feature-branch-push"].status == model.FAIL
|
||
|
|
assert "already exists" in results["ephemeral.feature-branch-push"].reason
|
||
|
|
assert "ephemeral.draft-pull-request" not in results
|
||
|
|
assert len(spawn.calls) == 1
|
||
|
|
|
||
|
|
|
||
|
|
def test_an_unverifiable_ref_check_is_not_run_rather_than_assumed_free() -> None:
|
||
|
|
runner, _ = armed_runner([completed(stderr="network down", returncode=128)])
|
||
|
|
|
||
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
||
|
|
|
||
|
|
assert results["ephemeral.feature-branch-push"].status == model.NOT_RUN
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_failed_push_still_reports_cleanup_and_skips_the_draft() -> None:
|
||
|
|
runner, _ = armed_runner(
|
||
|
|
[
|
||
|
|
completed(stdout=""),
|
||
|
|
completed(stderr="denied", returncode=1),
|
||
|
|
completed(stdout=""), # delete branch
|
||
|
|
completed(stdout=""), # ls-remote: gone
|
||
|
|
]
|
||
|
|
)
|
||
|
|
|
||
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
||
|
|
|
||
|
|
assert results["ephemeral.feature-branch-push"].status == model.FAIL
|
||
|
|
assert results["ephemeral.draft-pull-request"].status == model.NOT_RUN
|
||
|
|
assert results["ephemeral.cleanup-verified"].status == model.PASS
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_draft_response_without_a_number_is_a_failure() -> None:
|
||
|
|
runner, _ = armed_runner(
|
||
|
|
[
|
||
|
|
completed(stdout=""),
|
||
|
|
completed(stdout=""),
|
||
|
|
completed(stdout="{}"),
|
||
|
|
completed(stdout=""),
|
||
|
|
completed(stdout=""),
|
||
|
|
]
|
||
|
|
)
|
||
|
|
|
||
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
||
|
|
|
||
|
|
assert results["ephemeral.draft-pull-request"].status == model.FAIL
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_branch_that_survives_cleanup_fails_the_cleanup_check() -> None:
|
||
|
|
runner, _ = armed_runner(
|
||
|
|
[
|
||
|
|
completed(stdout=""),
|
||
|
|
completed(stdout=""),
|
||
|
|
completed(stdout=json.dumps({"number": 7})),
|
||
|
|
completed(stderr="cannot close", returncode=1),
|
||
|
|
completed(stderr="cannot delete", returncode=1),
|
||
|
|
completed(stdout="abc123\trefs/heads/x\n"),
|
||
|
|
completed(stdout=json.dumps({"number": 7, "state": "open"})),
|
||
|
|
]
|
||
|
|
)
|
||
|
|
|
||
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
||
|
|
cleanup = results["ephemeral.cleanup-verified"]
|
||
|
|
|
||
|
|
assert cleanup.status == model.FAIL
|
||
|
|
assert "could not be closed" in cleanup.reason
|
||
|
|
assert "still exists" in cleanup.reason
|
||
|
|
assert "still open" in cleanup.reason
|
||
|
|
|
||
|
|
|
||
|
|
def test_cleanup_that_cannot_be_verified_is_a_failure_not_a_pass() -> None:
|
||
|
|
runner, _ = armed_runner(
|
||
|
|
[
|
||
|
|
completed(stdout=""),
|
||
|
|
completed(stdout=""),
|
||
|
|
completed(stdout=json.dumps({"number": 7})),
|
||
|
|
completed(stdout=""),
|
||
|
|
completed(stdout=""),
|
||
|
|
subprocess.TimeoutExpired(cmd="git", timeout=1.0),
|
||
|
|
completed(stderr="gone", returncode=1),
|
||
|
|
]
|
||
|
|
)
|
||
|
|
|
||
|
|
cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))["ephemeral.cleanup-verified"]
|
||
|
|
|
||
|
|
assert cleanup.status == model.FAIL
|
||
|
|
assert "could not verify removal" in cleanup.reason
|
||
|
|
assert "could not confirm pull request #7 is closed" in cleanup.reason
|
||
|
|
|
||
|
|
|
||
|
|
def test_the_armed_results_replace_the_default_not_run_placeholders() -> None:
|
||
|
|
runner, _ = armed_runner([completed(stdout="abc\trefs/heads/x\n")])
|
||
|
|
identifiers = {result.spec.id for result in ephemeral.run_armed(runner, VANTAGE, request())}
|
||
|
|
|
||
|
|
assert identifiers <= {
|
||
|
|
"ephemeral.protected-branch-refusal",
|
||
|
|
"ephemeral.feature-branch-push",
|
||
|
|
"ephemeral.draft-pull-request",
|
||
|
|
"ephemeral.cleanup-verified",
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_read_only_runner_cannot_execute_the_armed_flow() -> None:
|
||
|
|
"""The mode, not the caller, is what opens the mutations."""
|
||
|
|
spawn = FakeSpawn([completed(stdout=""), completed(stdout=""), completed(stdout="")])
|
||
|
|
runner = exec_module.Runner(clock=FakeClock(), spawn=spawn, environ={})
|
||
|
|
|
||
|
|
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
||
|
|
push = results["ephemeral.feature-branch-push"]
|
||
|
|
|
||
|
|
assert push.status == model.FAIL
|
||
|
|
assert "policy" in push.reason
|
||
|
|
assert not any(call["argv"][:2] == ["git", "push"] for call in spawn.calls)
|