Decides whether the Hermes platform handoff is fit to release, and refuses to round an absence of evidence up to a pass. The harness is read-only by default and classifies 71 checks PASS / FAIL / NOT_RUN / NOT_APPLICABLE. Any mandatory FAIL or NOT_RUN is NO_GO, and so is a harness-level problem: an unreachable vantage, a catalog entry whose evidence no longer exists, an expired deadline, or an evaluator that raised. Evidence comes from two vantages that cannot cover for each other: an external read-only operator kubeconfig, and the Hermes agent probing itself from inside its own pod. Before any check runs, the harness asks each vantage who it is and stops if they are the same principal, because dual-vantage evidence from one identity is a restatement rather than a corroboration. `--as` is rejected for every operator-side command and reachable only as the inner command of a `kubectl exec`, so impersonation can never stand in for a real self-probe. A deny check needs a live refused request, not only an authorization review. Two safety properties are structural rather than conventional, enforced where an argv becomes a subprocess: the default mode mutates nothing (mutating verbs require a server dry run; there is deliberately no live TokenRequest probe, because a successful one would mint a real credential), and no probe can pull a credential value into a report (no vault/sops/curl, secrets readable only with -o name, environment probes list names, shell only through frozen reviewed templates). Captures are bounded before they are screened, and the rendered report is re-screened before it is written. Mutation lives behind a separate arming flag with an exact confirmation phrase, a caller-supplied unique ref, a preflight that refuses a protected push target before any network call, and a cleanup whose verification is itself mandatory. A default run reports those four checks NOT_RUN. The catalog is declarative so a reviewer reads what is asserted rather than how it is plumbed, and so structural properties can be proven over every entry before a run. Catalog drift surfaces as NOT_RUN, which stops the release. docs/hermes_full_handoff_acceptance.md carries the merge order for PRs #14-#18 on top of the merged #13 baseline, the image build and Flux rollout, the rollback point for each step, the go/no-go checklist, and the limits that are asserted rather than exercised. Validation: 295 handoff tests pass with 100% line coverage on all 15 new modules; the full unit suite is 647 passed with two failures that reproduce unchanged on origin/main; Ruff, py_compile, kustomize render, and a diff credential screen are clean; a live read-only run against Atlas returns NO_GO for the pre-merge cluster with no unscreened fields in the report.
314 lines
11 KiB
Python
314 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""The separately armed, self-cleaning mutation mode of the acceptance harness.
|
|
|
|
Proving that a worker can still open a draft pull request means actually opening
|
|
one, which is the only part of this harness that writes anything. It is
|
|
therefore a distinct mode with a distinct policy: an explicit confirmation
|
|
phrase, a caller-supplied unique ref, a preflight that refuses a protected
|
|
target before any network call, and a cleanup whose *verification* is itself a
|
|
mandatory check.
|
|
|
|
The push target is guarded, not the pull-request base. Opening a pull request
|
|
against `main` writes nothing to `main`; pushing to it does. The refusal is
|
|
checked by name and by pattern, and the arming self-test exercises it on every
|
|
armed run rather than trusting that it was checked once.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from collections.abc import Sequence
|
|
from dataclasses import dataclass
|
|
|
|
from hermes_handoff_exec import Outcome, Runner, Vantage
|
|
from hermes_handoff_model import EPHEMERAL, FAIL, NOT_RUN, PASS, CheckResult, CheckSpec
|
|
from hermes_handoff_policy import GITEA_CLIENT
|
|
|
|
CONFIRMATION = "ARM EPHEMERAL HERMES HANDOFF PUSH"
|
|
REF_PREFIX = "ephemeral/hermes-handoff-acceptance"
|
|
REF_RE = re.compile(rf"{REF_PREFIX}/[a-z0-9][a-z0-9-]{{7,63}}\Z")
|
|
TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9-]{7,63}\Z")
|
|
DRAFT_TITLE_PREFIX = "WIP: "
|
|
PROTECTED_NAMES = frozenset(
|
|
{
|
|
"default",
|
|
"develop",
|
|
"head",
|
|
"main",
|
|
"master",
|
|
"prod",
|
|
"production",
|
|
"release",
|
|
"stable",
|
|
"trunk",
|
|
}
|
|
)
|
|
|
|
|
|
class ArmingError(ValueError):
|
|
"""Raised when an ephemeral run is not safe to start."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ArmRequest:
|
|
"""Everything the armed mode needs, and nothing it can infer for itself."""
|
|
|
|
repo: str
|
|
remote: str
|
|
token: str
|
|
confirmation: str
|
|
base: str = "main"
|
|
|
|
@property
|
|
def ref(self) -> str:
|
|
"""Return the exact, unique branch this run is allowed to create."""
|
|
return f"{REF_PREFIX}/{self.token}"
|
|
|
|
|
|
def normalise_branch(name: str) -> str:
|
|
"""Return a branch name stripped of ref decoration, for comparison."""
|
|
trimmed = name.strip().removeprefix("refs/heads/").strip("/")
|
|
return trimmed.lower()
|
|
|
|
|
|
def assert_push_target_allowed(ref: str) -> None:
|
|
"""Raise unless ``ref`` is an ephemeral branch this harness may create."""
|
|
if normalise_branch(ref) in PROTECTED_NAMES:
|
|
raise ArmingError(f"refusing to push to protected branch {ref!r}")
|
|
if not REF_RE.fullmatch(ref):
|
|
raise ArmingError(f"push target {ref!r} is not an ephemeral acceptance ref")
|
|
|
|
|
|
def preflight(request: ArmRequest, expected_repo: str) -> None:
|
|
"""Validate an arming request before any network call is made."""
|
|
if request.confirmation != CONFIRMATION:
|
|
raise ArmingError("confirmation phrase does not match")
|
|
if request.repo != expected_repo:
|
|
raise ArmingError(f"repository {request.repo!r} is not the expected {expected_repo!r}")
|
|
if not TOKEN_RE.fullmatch(request.token):
|
|
raise ArmingError("ephemeral token must be 8-64 lowercase alphanumeric or dash characters")
|
|
assert_push_target_allowed(request.ref)
|
|
if not normalise_branch(request.base):
|
|
raise ArmingError("a pull-request base branch is required")
|
|
|
|
|
|
def _spec(identifier: str, title: str, rationale: str = "") -> CheckSpec:
|
|
return CheckSpec(
|
|
id=f"ephemeral.{identifier}",
|
|
title=title,
|
|
group="ephemeral",
|
|
rule="armed",
|
|
steps=(),
|
|
mandatory=True,
|
|
scope=EPHEMERAL,
|
|
rationale=rationale,
|
|
)
|
|
|
|
|
|
def _result(
|
|
identifier: str,
|
|
title: str,
|
|
status: str,
|
|
reason: str,
|
|
outcomes: Sequence[Outcome] = (),
|
|
evidence: dict | None = None,
|
|
rationale: str = "",
|
|
) -> CheckResult:
|
|
return CheckResult(
|
|
spec=_spec(identifier, title, rationale),
|
|
status=status,
|
|
reason=reason,
|
|
evidence=evidence or {},
|
|
outcomes=list(outcomes),
|
|
)
|
|
|
|
|
|
def _protected_refusal_result() -> CheckResult:
|
|
"""Exercise the protected-branch guard on this run, not on a past one."""
|
|
refused: list[str] = []
|
|
accepted: list[str] = []
|
|
for candidate in ("main", "master", "refs/heads/main", "HEAD", f"{REF_PREFIX}/x"):
|
|
try:
|
|
assert_push_target_allowed(candidate)
|
|
except ArmingError:
|
|
refused.append(candidate)
|
|
else:
|
|
accepted.append(candidate)
|
|
evidence = {"refused": refused, "accepted": accepted}
|
|
if accepted:
|
|
return _result(
|
|
"protected-branch-refusal",
|
|
"Arming against a protected branch is refused before any network call",
|
|
FAIL,
|
|
f"the guard accepted {accepted}",
|
|
evidence=evidence,
|
|
)
|
|
return _result(
|
|
"protected-branch-refusal",
|
|
"Arming against a protected branch is refused before any network call",
|
|
PASS,
|
|
"every protected and malformed push target was refused",
|
|
evidence=evidence,
|
|
)
|
|
|
|
|
|
def _detail(outcome: Outcome) -> str:
|
|
"""Return the most informative bounded explanation an outcome carries."""
|
|
return outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}"
|
|
|
|
|
|
def _pull_request_number(outcome: Outcome) -> int | None:
|
|
match = re.search(r'"number"\s*:\s*(\d+)', outcome.stdout)
|
|
return int(match.group(1)) if match else None
|
|
|
|
|
|
def run_armed(runner: Runner, vantage: Vantage, request: ArmRequest) -> list[CheckResult]:
|
|
"""Push a unique ephemeral ref, open a draft pull request, and remove both.
|
|
|
|
Every step is checked and every artifact is removed. Cleanup runs even when
|
|
an earlier step failed, and the cleanup verification is reported as its own
|
|
mandatory check so a half-removed branch cannot pass as success.
|
|
"""
|
|
api = f"/api/v1/repos/{request.repo}"
|
|
results = [_protected_refusal_result()]
|
|
outcomes: list[Outcome] = []
|
|
|
|
existing = runner.run(("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage)
|
|
outcomes.append(existing)
|
|
if not existing.ok:
|
|
results.append(
|
|
_result(
|
|
"feature-branch-push",
|
|
"A unique ephemeral feature branch can be pushed and removed",
|
|
NOT_RUN,
|
|
f"could not confirm the ref is unused: {_detail(existing)}",
|
|
outcomes,
|
|
)
|
|
)
|
|
return results
|
|
if existing.stdout.strip():
|
|
results.append(
|
|
_result(
|
|
"feature-branch-push",
|
|
"A unique ephemeral feature branch can be pushed and removed",
|
|
FAIL,
|
|
f"{request.ref} already exists on {request.remote}; pick a fresh token",
|
|
outcomes,
|
|
)
|
|
)
|
|
return results
|
|
|
|
pushed = runner.run(
|
|
("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"), vantage
|
|
)
|
|
outcomes.append(pushed)
|
|
results.append(
|
|
_result(
|
|
"feature-branch-push",
|
|
"A unique ephemeral feature branch can be pushed and removed",
|
|
PASS if pushed.ok else FAIL,
|
|
f"pushed {request.ref}" if pushed.ok else f"push failed: {_detail(pushed)}",
|
|
[pushed],
|
|
{"ref": request.ref},
|
|
)
|
|
)
|
|
|
|
number: int | None = None
|
|
if pushed.ok:
|
|
created = runner.run(
|
|
(
|
|
GITEA_CLIENT,
|
|
"POST",
|
|
f"{api}/pulls",
|
|
"--field",
|
|
f"title={DRAFT_TITLE_PREFIX}Hermes handoff acceptance ephemeral probe",
|
|
"--field",
|
|
f"head={request.ref}",
|
|
"--field",
|
|
f"base={normalise_branch(request.base)}",
|
|
"--field",
|
|
"body=Ephemeral acceptance probe. Closed and deleted by the harness.",
|
|
),
|
|
vantage,
|
|
)
|
|
outcomes.append(created)
|
|
number = _pull_request_number(created) if created.ok else None
|
|
results.append(
|
|
_result(
|
|
"draft-pull-request",
|
|
"A draft pull request can be opened and closed against the ephemeral branch",
|
|
PASS if created.ok and number else FAIL,
|
|
f"opened draft pull request #{number}"
|
|
if created.ok and number
|
|
else f"draft creation failed: {_detail(created)}",
|
|
[created],
|
|
{"pull_request": number},
|
|
)
|
|
)
|
|
else:
|
|
results.append(
|
|
_result(
|
|
"draft-pull-request",
|
|
"A draft pull request can be opened and closed against the ephemeral branch",
|
|
NOT_RUN,
|
|
"the ephemeral branch was never pushed",
|
|
)
|
|
)
|
|
|
|
results.append(_cleanup(runner, vantage, request, api, number))
|
|
return results
|
|
|
|
|
|
def _cleanup(
|
|
runner: Runner, vantage: Vantage, request: ArmRequest, api: str, number: int | None
|
|
) -> CheckResult:
|
|
"""Close the pull request, delete the ref, and prove both are gone."""
|
|
outcomes: list[Outcome] = []
|
|
problems: list[str] = []
|
|
|
|
if number is not None:
|
|
closed = runner.run(
|
|
(GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"), vantage
|
|
)
|
|
outcomes.append(closed)
|
|
if not closed.ok:
|
|
problems.append(f"pull request #{number} could not be closed")
|
|
|
|
deleted = runner.run((GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage)
|
|
outcomes.append(deleted)
|
|
|
|
remaining = runner.run(
|
|
("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage
|
|
)
|
|
outcomes.append(remaining)
|
|
if not remaining.ran:
|
|
problems.append(f"could not verify removal: {_detail(remaining)}")
|
|
elif remaining.stdout.strip():
|
|
problems.append(f"{request.ref} still exists on {request.remote}")
|
|
|
|
if number is not None:
|
|
state = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage)
|
|
outcomes.append(state)
|
|
if not state.ok:
|
|
problems.append(f"could not confirm pull request #{number} is closed")
|
|
elif '"state":"open"' in state.stdout.replace(" ", ""):
|
|
problems.append(f"pull request #{number} is still open")
|
|
|
|
if problems:
|
|
return _result(
|
|
"cleanup-verified",
|
|
"Every ephemeral ref and pull request is removed and the removal is verified",
|
|
FAIL,
|
|
"; ".join(problems),
|
|
outcomes,
|
|
{"ref": request.ref, "pull_request": number},
|
|
)
|
|
return _result(
|
|
"cleanup-verified",
|
|
"Every ephemeral ref and pull request is removed and the removal is verified",
|
|
PASS,
|
|
"the ephemeral ref and pull request are gone and their absence was re-checked",
|
|
outcomes,
|
|
{"ref": request.ref, "pull_request": number},
|
|
)
|