diff --git a/docs/hermes_full_handoff_acceptance.md b/docs/hermes_full_handoff_acceptance.md new file mode 100644 index 00000000..8b9103c2 --- /dev/null +++ b/docs/hermes_full_handoff_acceptance.md @@ -0,0 +1,323 @@ +# Hermes full-handoff acceptance harness and release runbook + +`scripts/ops/hermes_handoff_acceptance.py` decides whether the Hermes platform +handoff is fit to release. It is read-only by default, it classifies every check +`PASS` / `FAIL` / `NOT_RUN` / `NOT_APPLICABLE`, and **any mandatory `FAIL` or +`NOT_RUN` makes the whole run `NO_GO`**. Absence of evidence is never rounded up +to evidence of absence: a probe that could not run, a catalog entry whose +evidence no longer exists, an expired deadline, or a harness bug all land in +`NOT_RUN` and stop the release. + +``` +scripts/ops/hermes_handoff_acceptance.py \ + --context atlas-operator \ + --node-count 21 \ + --expected-suspension flux-system/bstein-dev-home-migrations \ + --output build/hermes-handoff-acceptance.json +``` + +Exit status is `0` for `GO` and `1` for `NO_GO`. The JSON report goes to +`--output` (or stdout); a short human summary goes to stderr unless +`--json-only` is passed. + +## Why two vantages + +Every authority claim is checked from two places that cannot cover for each +other: + +| Vantage | What it is | What it proves | +| --- | --- | --- | +| `operator` | An external, read-only kubeconfig | Desired state: bindings, manifests, workloads, Flux health, forge state | +| `self` | `kubectl exec` into the running `hermes-agent` container | Enforced state: what the agent's own service-account token can actually do | +| `switchyard`, `node`, `chat` | `kubectl exec` into the Switchyard, node-hardening, and chat pods | Evidence that only exists inside those workloads | + +Two rules keep this honest: + +* **The harness cannot impersonate from outside.** `--as`, `--as-group`, and + `--as-uid` are rejected by policy for any operator-side command. They are + reachable only as the *inner* command of a `kubectl exec`, where the identity + is testing its own refusal to impersonate rather than borrowing authority. + `kubectl auth can-i --as` is therefore not available as a substitute for a + real self-probe. +* **The two vantages must be different principals.** Before any check runs, the + harness asks each vantage `kubectl auth whoami` and compares. If the operator + and the in-pod probe authenticate as the same principal, the dual-vantage + evidence is a restatement rather than a corroboration, and the run is `NO_GO` + with `the operator and in-pod vantages authenticate as the same principal`. + Run the harness from a genuinely external kubeconfig — not from inside the + Hermes owner pod. + +Deny checks additionally require a **real refused request**, not only an +authorization review. `kubectl auth can-i` reports what RBAC says; only an +attempt proves the API server enforces it. Mutating attempts go through +`--dry-run=server`: the API server authorises the request and then discards it, +so a refusal is genuine and nothing is written either way. + +## Safety properties + +These hold structurally, in `hermes_handoff_policy.py`, so a catalog mistake is +a policy error before a run rather than a mutation or a leak during one. + +* **No mutation in the default mode.** Only read subcommands are reachable; + `apply`, `create`, `delete`, `patch`, `replace`, and `scale` require a dry-run + flag. `git push` and every non-`GET` forge method are refused outside the + armed mode. There is deliberately no live TokenRequest probe — a successful + one would mint a real credential. +* **No credential is ever read.** `vault`, `sops`, `age`, `gpg`, `openssl`, + `curl`, and `wget` are not on the binary allowlist. Secrets may only be read + with `-o name`. Environment probes list variable *names* and never values. + Provider health files are read through a frozen template that projects an + allow-listed set of non-secret status fields. The node account's lock state is + reported without reading its password hash; its authorized keys are counted, + never printed. +* **Shell is frozen.** `sh -c` is reachable only through the reviewed, + parameter-validated templates in `SHELL_TEMPLATES`. Parameters may not contain + shell metacharacters. +* **Output is bounded and screened.** Each capture is truncated to a byte budget + *before* it is screened, so a large dump cannot outrun the screen. Bulk + evidence (the routing-log tail, the Kanban listing) is parsed but replaced with + `[output not recorded]` in the artifact. The screen preserves JSON quoting, so + screened structured output is still parsable. +* **The report is re-checked before it is published.** `unscreened_fields` walks + the rendered payload; if anything still looks credential-shaped the report is + replaced with a `NO_GO` stub naming the offending pointers, and the file that + would have leaked is never written. + +## What the catalog asserts + +The catalog is declarative (`hermes_handoff_catalog.py` plus the four +`hermes_handoff_checks_*.py` modules) so a reviewer reads what is asserted, not +how it is plumbed. 71 checks in 14 groups: + +| Group | Assertions | +| --- | --- | +| `baseline` | `origin/main` descends from the merged worker-isolation baseline; every dependency pull request is merged into `main`; the exact open pull-request heads are recorded | +| `identity` | Codex authenticates as `codex-chatgpt-subscription`; Claude as `firstParty` / `claude.ai` / `claude-code-cli-subscription`; both evidence files are fresh; no provider API-key variable exists in the running process **or** in the manifest | +| `routing` | Switchyard has a ready replica; the routing log covers both providers, the `route` and `worker` lanes, medium/high/xhigh effort, and at least one real fallback; the newest record is fresh; both providers record a measured latency | +| `scopes` | Chat, agent, and triage each declare their own `HERMES_AUTO_ROUTER_PROFILE`, on distinct hostnames | +| `access-denied` | No cluster-admin binding for the agent; Secrets, service-account tokens, impersonation, workload mutation, arbitrary pod exec, attach, port-forward, and control-plane writes are all refused | +| `access-allowed` | get/list/watch on core objects, pod logs, and Flux/Helm status still work; the operator and in-pod namespace inventories agree | +| `gitops` | No unexpected Kustomization or HelmRelease suspension, and nothing unhealthy | +| `forge` | No forge credential in the worker environment or home; the forge identity is not an administrator; repository authority is pull/push only; an administrative route is live-refused; clone/fetch still works | +| `nodes` | The dedicated `hermes-agent` account is locked, has the expected uid/gid/home/shell, joins no supplementary group, has no sudoers fragment, holds exactly one authorized key; K3s credentials, K3s state, kubelet state, and the container runtime are root-only; the audit covers every node | +| `build` | The image builder service account mounts no token; the build pod is unprivileged with exactly `CHOWN, FOWNER, DAC_OVERRIDE, SETGID, SETUID`; the Harbor immutable-tag job succeeded; the running image is digest-pinned | +| `reliability` | The finalization/replay/session patches are running in the pod; the regression suites are present on `main` | +| `pool` | Three ready workers, on distinct nodes, with distinct volumes, mounting no API token and none of the coordinator's claims | +| `surfaces` | Kanban activity and durable sessions are visible and render without blank rows; the agent and chat session patches are deployed; Telegram topic state is on durable storage | +| `ephemeral` | Feature-branch push, draft pull request, protected-branch refusal, and verified cleanup — `NOT_RUN` unless armed | + +### Site configuration + +A few checks are emitted only when the operator declares what to expect, +because a `names_present` assertion over an empty expectation would pass on any +workload at all — which reads as coverage without being any: + +| Flag | Enables | +| --- | --- | +| `--pool-worker-env NAME` (repeatable) | `pool.assignment-safety-knobs-are-configured` | +| `--chat-config-revision REV` | `surfaces.chat-runs-the-telegram-topic-revision` | +| `--expect-telegram-sessions` with `--chat-ordinal N` | `surfaces.chat-telegram-sessions-are-continuous` | + +`--node-count` must be the real node count; the default of `1` makes the +fleet-coverage check vacuous. + +### Catalog drift is a NO_GO, not a pass + +If a workload, field, or manifest is renamed, the affected checks report +`NOT_RUN` because the evidence they name is no longer there. That is the +intended behaviour: the release stops until either the platform or the catalog +is corrected. Do not "fix" a `NOT_RUN` by deleting the check. + +## Ephemeral mutation mode + +Proving that a worker can still open a draft pull request means actually opening +one. That is the only part of the harness that writes anything, and it is a +separate mode: + +``` +scripts/ops/hermes_handoff_acceptance.py \ + --context atlas-operator \ + --arm-ephemeral-push \ + --confirm "ARM EPHEMERAL HERMES HANDOFF PUSH" \ + --ephemeral-token acceptance-20260817a \ + --output build/hermes-handoff-acceptance-armed.json +``` + +* The confirmation phrase must match exactly. +* `--ephemeral-token` is 8–64 lowercase alphanumeric/dash characters and must be + unique per run; the ref is always + `ephemeral/hermes-handoff-acceptance/`. +* **Preflight runs before any network call**: wrong phrase, wrong repository, + malformed token, or a protected push target all abort with `arming refused` + and a `NO_GO` report. `main`, `master`, `HEAD`, `develop`, `release`, + `production`, `stable`, `trunk`, `prod`, and `default` are refused by name + (case- and `refs/heads/`-insensitive), and anything that is not an ephemeral + acceptance ref is refused by pattern. +* The guard is **exercised on every armed run** by + `ephemeral.protected-branch-refusal`, rather than trusting that it was checked + once. +* The push target is guarded; the pull-request *base* is not. Opening a draft + pull request against `main` writes nothing to `main`. The draft title always + starts with `WIP: `. +* Cleanup closes the pull request, deletes the ref, and then **re-reads both** to + confirm they are gone. `ephemeral.cleanup-verified` is mandatory: a half-removed + branch fails the run rather than passing it. Cleanup runs even when an earlier + step failed. +* `git push --force`, `--force-with-lease`, `--mirror`, `--all`, and `--tags` + are refused even in armed mode. + +A default run reports all four ephemeral checks `NOT_RUN` and non-mandatory, so +a read-only sweep can still reach `GO`. + +## Release order + +### Baseline, already merged and deployed + +PR #13 (`hermes: harden worker isolation and blocked-task semantics`) merged as +`ab346f55509d584e457fe26cf90be3078f7a375c`. Every check below assumes it is +present; `baseline.origin-main-descends-merged-work` asserts it. Override with +`--baseline-commit` only when a later merge legitimately replaces it. + +### Pending work, observed 2026-08-17T10:07Z + +These heads move. The harness re-fetches them at run time via +`baseline.pending-pull-requests-recorded`; the table is the snapshot this +document was written against, not an input. + +| PR | Branch | Head at observation | State | Delivers | +| --- | --- | --- | --- | --- | +| #14 | `feature/hermes-safe-gitea-pr-client` | `3cd8a17e772978fb554d88dd1c59777786be3991` | draft | Bounded forge authority, observer RBAC (removes `hermes-agent-cluster-admin`), node account hardening | +| #15 | `fix/hermes-result-decomposition-reliability` | `7609cd77e87a64e332b2f0d670c1d7a4f62b3d9e` | draft | Result preservation, duplicate-fanout prevention, execution-safety patch | +| #16 | `feature/hermes-unprivileged-image-builder` | `5ab0eba619779b3dfed2eb09d5c02f0a9c92dd0a` | ready | Daemonless Kaniko build lane, tokenless builder, Harbor immutable-tag rule | +| #17 | `feature/hermes-three-lane-placement` | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | draft | Three concurrent CLI lanes on the larger ARM64 nodes | +| #18 | `feature/hermes-distributed-worker-pool` | `20002527512235b29054f3636b02418168373009` | draft | Three-node fenced execution pool with coordinator-only state ownership | + +Base at observation: `30259b526be5b2e226620aeebd9fb5f57dc88b7d`. + +### Merge order and why + +1. **#14 first.** It is the only change that *reduces* authority: it deletes + `services/hermes/agent-rbac.yaml` (the `cluster-admin` binding) and replaces + it with the namespaced/cluster observer roles, and it hardens the node + account. Merging it last would mean every earlier acceptance run measured a + cluster-admin agent. Every `access-denied` and `nodes` check depends on it. +2. **#16 second.** The image lane must exist before anything that needs a new + agent image is rolled out. It also touches `testing/quality_contract.json`, + so merging it early keeps later rebases small. `build.*` depends on it. +3. **#15 third.** It changes `dockerfiles/Dockerfile.hermes-agent`, which #16 + also touches — take #16's shape and rebase #15 onto it. `reliability.*` + depends on it. +4. **#17 fourth.** Placement only; it edits `services/hermes/agent-deployment.yaml` + and `testing/tests/test_hermes_cli_lanes.py`, both of which #14, #15, and #18 + also touch. Merging it after them avoids resolving the same conflict twice. +5. **#18 last.** The pool depends on the bounded forge client from #14, the + execution-safety patch from #15, and the agent image from #16. `pool.*` + depends on it. + +Overlapping files to expect (rebase, do not merge blind): `services/hermes/agent-deployment.yaml` +(#12, #14, #16, #17, #18), `services/hermes/kustomization.yaml` (#14, #16, #18), +`services/hermes/scripts/stage_runtime_access.py` (#14, #16, #18), +`services/vault/scripts/vault_k8s_auth_configure.sh` (#14, #16, #18), +`dockerfiles/Dockerfile.hermes-agent` (#15, #16), +`testing/tests/test_hermes_cli_lanes.py` (#14, #15, #17, #18), +`testing/tests/test_hermes_runtime_access.py` (#16, #18). + +PR #12 (AI usage exporter) is independent and can merge at any point; it touches +`services/hermes/agent-deployment.yaml`, so land it before #17 if convenient. + +### Image build and Flux rollout + +1. Merge in the order above; observe the default-branch Jenkins build to a + terminal result after each merge + (`/opt/coordinator/jenkins_build_evidence.py titan-iac --branch main --wait`). +2. After #16 is merged, run the `hermes-agent-image` pipeline. It publishes an + immutable `git--build-` tag and emits + `build/hermes-agent-release/hermes-image-update.patch`. +3. Apply that patch to `services/hermes/kustomization.yaml` on a branch, review, + and merge. The digest is what pins the deployment; + `build.deployed-image-is-digest-pinned` asserts it. +4. Let Flux reconcile. Confirm with `flux get kustomizations -A` that `hermes` + and `hermes-chat` are `Ready`, then wait for the rollout to settle. +5. Re-run the harness. Do not run it mid-rollout: `gitops.*` tolerates an + in-flight reconcile (`Reconciling=True` with `Ready=False`) but a half-rolled + deployment will legitimately fail the `identity`, `routing`, and `surfaces` + groups. + +### Rollback points + +| After | Roll back by | Restores | +| --- | --- | --- | +| #14 merge | Revert the merge commit; Flux restores `hermes-agent-cluster-admin` | Cluster-admin agent, previous node account state | +| #16 merge | Revert the merge; the previously pinned digest is unchanged in `kustomization.yaml` | Previous build lane; published immutable tags are not deleted | +| Image digest bump | Revert the `kustomization.yaml` commit to the prior digest | Previous agent image; Harbor immutability means the old tag is still exactly what it was | +| #15 merge | Revert the merge; the init-container patch disappears on the next rollout | Previous finalization/replay behaviour | +| #17 merge | Revert the merge | Previous lane concurrency and node placement | +| #18 merge | Revert the merge; the StatefulSet and its PVCs are removed by Flux pruning | Single-coordinator execution | + +Every rollback is a Git revert reconciled by Flux. Do not roll back by direct +`kubectl` edit; the next reconcile would undo it. + +## Go / no-go checklist + +Run in order. Stop at the first item that does not hold. + +1. `origin/main` is `ab346f55509d584e457fe26cf90be3078f7a375c` or a descendant. +2. PRs #14, #15, #16, #17, #18 are all merged into `main`, in that order, each + with a green default-branch build. +3. The agent image has been built by the #16 lane, is pinned by digest in + `services/hermes/kustomization.yaml`, and Flux has finished reconciling. +4. The harness is run from an **external operator kubeconfig**, not from inside + the Hermes owner pod, with `--node-count` set to the real node count and every + deliberate Flux suspension declared via `--expected-suspension`. +5. The default read-only run reports `GO` — no mandatory `FAIL`, no mandatory + `NOT_RUN`, no `harness_errors`. +6. The armed run reports `PASS` for all four `ephemeral.*` checks, including + `ephemeral.cleanup-verified`, and the ephemeral ref and pull request are + confirmed gone. +7. The JSON report is archived with the exact `origin/main` commit and the + recorded pull-request heads from `baseline.pending-pull-requests-recorded`. + +Only then is the handoff releasable. + +## Known limits + +Stated so nobody mistakes a proxy for the thing itself. + +* **Merge, approve, and close authority is asserted, not exercised.** The + harness never attempts a merge. It proves the forge identity holds + `permissions.admin: false` and is live-refused on an administrative route; + the route-level allowlist is covered by PR #14's own regression suite. +* **Attach and port-forward have no side-effect-free live attempt.** Their + authorization reviews ride alongside a real, refused control-plane write in the + same check, and pod exec — the one subresource with a harmless real attempt — + has its own check. +* **TokenRequest is asserted by review.** A successful live attempt would mint a + real credential, so the check pairs the `serviceaccounts/token` review with a + refused `serviceaccount` create in the same RBAC family. +* **Branch protection is not read.** The harness identity is not a repository + administrator, so `GET /branch_protections` is refused — which is itself + recorded as evidence. Protected-push refusal is guaranteed by the harness's own + preflight, exercised on every armed run, not by reading the forge's config. +* **Stale/duplicate assignment safety is asserted as deployed configuration.** + The behaviour itself is covered by the pool's regression suite; the harness + checks that the workers carry the configuration and that the coordinator keeps + sole ownership of its claims. +* **Telegram session continuity is per tenant** and depends on there having been + Telegram traffic, so it is off by default. The always-on check is that topic + and tenant state live on a persistent claim rather than in the pod. +* **The routing window is a tail**, 600 records by default. A property that did + not occur inside that window is reported as missing rather than absent; widen + the window or wait for traffic rather than removing the assertion. + +## Tests + +``` +python3 -m pytest testing/tests/ -k handoff +python3 -m coverage run --source=scripts/ops --include="*/hermes_handoff_*.py" \ + -m pytest testing/tests/ -k handoff && python3 -m coverage report +``` + +The suite covers parsing, redaction, timeouts and deadlines, partial failure, +zero state, dual-vantage mismatch, mandatory skips, catalog structure, and +ephemeral arming and cleanup — including the cases where cleanup cannot be +verified. diff --git a/scripts/ops/hermes_handoff_acceptance.py b/scripts/ops/hermes_handoff_acceptance.py new file mode 100755 index 00000000..7be6de29 --- /dev/null +++ b/scripts/ops/hermes_handoff_acceptance.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +"""Fail-closed acceptance harness for the Hermes full handoff. + +Read-only by default. The harness collects evidence from two genuinely separate +places — an external operator vantage and the Hermes agent probing itself from +inside its own pod — classifies every check PASS / FAIL / NOT_RUN / +NOT_APPLICABLE, and emits machine-readable JSON alongside a short human summary. + +Any mandatory FAIL or NOT_RUN is a NO_GO, and so is a harness-level problem: an +unreachable vantage, a catalog whose evidence no longer exists, an expired +deadline, or two vantages that turn out to be the same principal. Nothing here +rounds an absence of evidence up to a pass. + + scripts/ops/hermes_handoff_acceptance.py --context atlas-operator + scripts/ops/hermes_handoff_acceptance.py --output build/handoff.json + +Mutation lives behind a separate arming flag, needs an exact confirmation +phrase and a unique ref, refuses protected push targets before any network +call, and verifies its own cleanup. A default run reports those checks NOT_RUN. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import sys +from collections.abc import Sequence +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from hermes_handoff_catalog import Targets # noqa: E402 +from hermes_handoff_ephemeral import ( # noqa: E402 + CONFIRMATION, + ArmingError, + ArmRequest, + preflight, + run_armed, +) +from hermes_handoff_exec import ( # noqa: E402 + DEFAULT_COMMAND_TIMEOUT, + DEFAULT_RUN_DEADLINE, + SELF, + Runner, +) +from hermes_handoff_model import GO, Report, unscreened_fields, utc_now # noqa: E402 +from hermes_handoff_policy import ARMED, READ_ONLY # noqa: E402 +from hermes_handoff_redaction import DEFAULT_MAX_BYTES # noqa: E402 +from hermes_handoff_run import ( # noqa: E402 + build_catalog, + build_report, + resolve_vantages, + unavailable_report, +) + +EXIT_GO = 0 +EXIT_NO_GO = 1 +EXIT_USAGE = 2 + + +def build_parser() -> argparse.ArgumentParser: + """Return the command-line interface for the harness.""" + parser = argparse.ArgumentParser( + prog="hermes_handoff_acceptance", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--namespace", default="hermes", help="Hermes namespace.") + parser.add_argument("--context", help="Kubernetes context for the operator vantage.") + parser.add_argument("--kubeconfig", help="Kubeconfig for the operator vantage.") + parser.add_argument("--output", type=Path, help="Write the JSON report here instead of stdout.") + parser.add_argument("--json-only", action="store_true", help="Suppress the human summary.") + parser.add_argument( + "--command-timeout", type=float, default=DEFAULT_COMMAND_TIMEOUT, help="Per-command seconds." + ) + parser.add_argument( + "--deadline", type=float, default=DEFAULT_RUN_DEADLINE, help="Whole-run seconds." + ) + parser.add_argument( + "--max-output-bytes", type=int, default=DEFAULT_MAX_BYTES, help="Per-capture byte budget." + ) + parser.add_argument( + "--baseline-commit", default=Targets.baseline_commit, help="Commit main must descend from." + ) + parser.add_argument( + "--dependency-pr", + type=int, + action="append", + default=None, + help="Pull request that must be merged before this release is certifiable (repeatable).", + ) + parser.add_argument( + "--expected-suspension", + action="append", + default=None, + metavar="NAMESPACE/NAME", + help="Flux object allowed to be suspended (repeatable).", + ) + parser.add_argument("--node-count", type=int, default=1, help="Nodes the fleet must cover.") + parser.add_argument("--pool-replicas", type=int, default=3, help="Expected pool worker count.") + parser.add_argument( + "--chat-config-revision", default="", help="Expected chat pod-template config revision." + ) + parser.add_argument( + "--chat-ordinal", + type=int, + default=0, + help="Chat tenant ordinal that carries the Telegram identity.", + ) + parser.add_argument( + "--expect-telegram-sessions", + action="store_true", + help="Require the selected chat tenant to hold durable Telegram-sourced sessions.", + ) + parser.add_argument( + "--pool-worker-env", + action="append", + default=None, + help="Environment variable the pool workers must declare (repeatable).", + ) + parser.add_argument( + "--max-evidence-age", + type=int, + default=86_400, + help="Seconds before provider and routing evidence is treated as stale.", + ) + armed = parser.add_argument_group("ephemeral mutation mode") + armed.add_argument( + "--arm-ephemeral-push", + action="store_true", + help="Permit one ephemeral branch push and draft pull request, then remove both.", + ) + armed.add_argument("--confirm", default="", help=f"Exact confirmation phrase: {CONFIRMATION!r}.") + armed.add_argument("--ephemeral-token", default="", help="Unique 8-64 character ref suffix.") + armed.add_argument("--ephemeral-base", default="main", help="Pull-request base branch.") + armed.add_argument("--remote", default="origin", help="Git remote for the ephemeral push.") + armed.add_argument("--repo", default=Targets.repo, help="owner/name of the forge repository.") + return parser + + +def targets_from_args(arguments: argparse.Namespace, now: dt.datetime) -> Targets: + """Return the catalog targets a parsed command line describes.""" + defaults = Targets() + return Targets( + namespace=arguments.namespace, + baseline_commit=arguments.baseline_commit, + dependency_pull_requests=tuple(arguments.dependency_pr or defaults.dependency_pull_requests), + expected_suspensions=tuple(arguments.expected_suspension or ()), + node_count=arguments.node_count, + pool_replicas=arguments.pool_replicas, + chat_config_revision=arguments.chat_config_revision, + chat_ordinal=arguments.chat_ordinal, + expect_telegram_sessions=arguments.expect_telegram_sessions, + pool_worker_env=tuple(arguments.pool_worker_env or ()), + max_evidence_age_seconds=arguments.max_evidence_age, + repo=arguments.repo, + remote=arguments.remote, + now=now, + ) + + +def emit(report: Report, arguments: argparse.Namespace, stream=None) -> list[str]: + """Write the report, refusing to publish one that still looks credential-shaped.""" + payload = report.as_dict() + offenders = unscreened_fields(payload) + if offenders: + payload = { + "harness": payload["harness"], + "decision": "NO_GO", + "harness_errors": [ + "the rendered report failed its own credential screening at " + + ", ".join(offenders[:8]) + ], + } + rendered = json.dumps(payload, indent=2, sort_keys=True, default=str) + if arguments.output: + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(rendered + "\n", encoding="utf-8") + else: + print(rendered, file=stream or sys.stdout) + if not arguments.json_only: + print(report.render_summary(), file=sys.stderr) + return offenders + + +def _armed_results(arguments: argparse.Namespace, runner: Runner, vantages) -> list: + request = ArmRequest( + repo=arguments.repo, + remote=arguments.remote, + token=arguments.ephemeral_token, + confirmation=arguments.confirm, + base=arguments.ephemeral_base, + ) + preflight(request, arguments.repo) + vantage = vantages.get(SELF) or vantages["operator"] + return run_armed(runner, vantage, request) + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the acceptance sweep and return the process exit status.""" + arguments = build_parser().parse_args(argv) + now = dt.datetime.now(dt.timezone.utc) + started_at = utc_now(lambda: now) + targets = targets_from_args(arguments, now) + mode = ARMED if arguments.arm_ephemeral_push else READ_ONLY + + runner = Runner( + mode=mode, + command_timeout=arguments.command_timeout, + deadline_seconds=arguments.deadline, + max_bytes=arguments.max_output_bytes, + ) + vantages, records = resolve_vantages(runner, targets, arguments.kubeconfig, arguments.context) + + armed_results: list = [] + if arguments.arm_ephemeral_push: + try: + armed_results = _armed_results(arguments, runner, vantages) + except ArmingError as exc: + report = unavailable_report(mode, started_at, f"arming refused: {exc}", targets) + emit(report, arguments) + return EXIT_NO_GO + + specs = build_catalog(targets) + report = build_report( + runner, targets, specs, vantages, records, mode, started_at, armed_results + ) + offenders = emit(report, arguments) + if offenders: + return EXIT_NO_GO + return EXIT_GO if report.decision == GO else EXIT_NO_GO + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/scripts/ops/hermes_handoff_catalog.py b/scripts/ops/hermes_handoff_catalog.py new file mode 100644 index 00000000..b59ff2b9 --- /dev/null +++ b/scripts/ops/hermes_handoff_catalog.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Declarative acceptance catalog: baseline, identity, routing, scopes, access. + +The catalog is data on purpose. A reviewer should be able to read what the +release is asserted to satisfy without reading any plumbing, and the harness +should be able to prove structural properties over every entry — both vantages +represented, no impersonation from outside a pod, no mutation outside a server +dry run — before it runs a single command. + +Catalog drift is a NO_GO, not a pass. If a manifest, a field name, or a workload +is renamed, the affected checks report ``NOT_RUN`` because the evidence they +name is not there, and the release stops until either the platform or the +catalog is corrected. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field +from typing import Any + +from hermes_handoff_model import ALWAYS, ATTEMPT, READ, REVIEW, CheckSpec, Step + +OPERATOR = "operator" +SELF = "self" +SWITCHYARD = "switchyard" +NODE = "node" +CHAT = "chat" +VANTAGE_NAMES = (OPERATOR, SELF, SWITCHYARD, NODE, CHAT) + +# `namespace/namesuspendType=Status,...` — one compact row per object. +FLUX_PROJECTION = ( + "jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{'\\t'}" + "{.spec.suspend}{'\\t'}{range .status.conditions[*]}{.type}={.status},{end}{'\\n'}{end}" +).replace("'", '"') + +ENV_NAME_PROJECTION = ( + "jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{'\\n'}{end}{end}" +).replace("'", '"') + +INIT_NAME_PROJECTION = ( + "jsonpath={range .spec.template.spec.initContainers[*]}{.name}{'\\n'}{end}" +).replace("'", '"') + +NODE_NAME_PROJECTION = "jsonpath={range .items[*]}{.spec.nodeName}{'\\n'}{end}".replace("'", '"') + +PROVIDER_API_KEY_NAMES = ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + "AZURE_OPENAI_API_KEY", + "CLAUDE_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", +) +FORGE_CREDENTIAL_NAMES = ("GITEA_PASSWORD", "GITEA_TOKEN", "GITEA_USERNAME", "GIT_PASSWORD") + +CODEX_HEALTH = "/opt/data/provider-health/codex.json" +CLAUDE_HEALTH = "/opt/data/provider-health/claude.json" +CODEX_HEALTH_FIELDS = "authenticated,transport,state,model,effort,latency_ms,checked_at" +CLAUDE_HEALTH_FIELDS = ( + "authenticated,api_provider,auth_method,transport,state,subscription_type," + "model,effort,latency_ms,checked_at" +) +ROUTING_LOG = "/var/lib/switchyard/routing.jsonl" + + +@dataclass(frozen=True) +class Targets: + """Runtime names and expected values the catalog is written against. + + Everything a site can legitimately rename lives here so the assertions stay + readable and a rename is a configuration change rather than a code change. + """ + + namespace: str = "hermes" + agent_deployment: str = "hermes-agent" + agent_container: str = "hermes" + chat_statefulset: str = "hermes-chat-tenant" + chat_container: str = "hermes" + chat_ordinal: int = 0 + chat_router_deployment: str = "hermes-chat-router" + triage_deployment: str = "hermes" + triage_container: str = "hermes" + switchyard_deployment: str = "hermes-switchyard" + switchyard_container: str = "switchyard" + node_daemonset: str = "hermes-node-ssh-access" + node_container: str = "key-reconciler" + pool_statefulset: str = "hermes-execution-worker" + pool_replicas: int = 3 + pool_selector: str = "app=hermes-execution-worker" + coordinator_claims: tuple[str, ...] = ("hermes-agent-home", "hermes-agent-workspace") + node_account: str = "hermes-agent" + node_account_identity: str = "1200:1200:/home/hermes-agent:/bin/bash" + node_count: int = 1 + repo: str = "atlas/titan-iac" + remote: str = "origin" + baseline_commit: str = "ab346f55509d584e457fe26cf90be3078f7a375c" + dependency_pull_requests: tuple[int, ...] = (14, 15, 16, 17, 18) + expected_suspensions: tuple[str, ...] = () + harbor_namespace: str = "harbor" + harbor_immutability_job: str = "hermes-agent-immutability" + builder_namespace: str = "jenkins" + builder_serviceaccount: str = "hermes-image-builder" + builder_pipeline: str = "ci/Jenkinsfile.hermes-agent-image" + builder_capabilities: str = 'add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]' + agent_init_containers: tuple[str, ...] = ("patch-web-session-activity",) + chat_init_containers: tuple[str, ...] = ("patch-api-server-sessions",) + chat_config_revision: str = "" + pool_worker_env: tuple[str, ...] = () + max_evidence_age_seconds: int = 86_400 + routing_tail_lines: int = 600 + routing_tail_bytes: int = 512 * 1024 + listing_bytes: int = 512 * 1024 + git_askpass: str = "/opt/coordinator/gitea_askpass.sh" + expect_telegram_sessions: bool = False + now: Any = None + extra: dict[str, Any] = field(default_factory=dict) + + +def step( + key: str, + vantage: str, + *argv: str, + kind: str = READ, + optional: bool = False, + record: bool = True, + max_bytes: int | None = None, +) -> Step: + """Return a probe step addressed to one vantage.""" + return Step( + key=key, + vantage=vantage, + argv=tuple(argv), + kind=kind, + optional=optional, + record=record, + max_bytes=max_bytes, + ) + + +def check( + identifier: str, + title: str, + group: str, + rule: str, + steps: Sequence[Step], + expect: dict[str, Any] | None = None, + mandatory: bool = True, + scope: str = ALWAYS, + rationale: str = "", +) -> CheckSpec: + """Return one catalog entry.""" + return CheckSpec( + id=identifier, + title=title, + group=group, + rule=rule, + steps=tuple(steps), + expect=expect or {}, + mandatory=mandatory, + scope=scope, + rationale=rationale, + ) + + +def deny( + identifier: str, + title: str, + review: Sequence[str], + attempt: Sequence[str], + rationale: str, + group: str = "access-denied", +) -> CheckSpec: + """Return a deny check that pairs a self review with a real refused request.""" + return check( + identifier, + title, + group, + "denied", + [ + step("review", SELF, *review, kind=REVIEW), + step("attempt", SELF, *attempt, kind=ATTEMPT), + ], + rationale=rationale, + ) diff --git a/scripts/ops/hermes_handoff_checks_access.py b/scripts/ops/hermes_handoff_checks_access.py new file mode 100644 index 00000000..a863e0f3 --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_access.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Acceptance checks for what Hermes may and may not do to the cluster. + +Every deny check pairs an in-pod authorization review with a real request that +must be refused, because a review reports what RBAC says and only an attempt +proves the API server enforces it. Mutating attempts go through a server dry +run: the API server authorises the request and discards it, so the refusal is +genuine and nothing is written either way. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + FLUX_PROJECTION, + OPERATOR, + SELF, + Targets, + check, + deny, + step, +) +from hermes_handoff_model import ATTEMPT, REVIEW, CheckSpec +from hermes_handoff_policy import shell + + +def _access(targets: Targets) -> list[CheckSpec]: + namespace = targets.namespace + return [ + check( + "access.no-cluster-admin-binding-for-agent", + "No ClusterRoleBinding grants the agent service account cluster-admin", + "access-denied", + "names_absent", + [step("bindings", OPERATOR, "kubectl", "get", "clusterrolebindings", "-o", "name")], + {"step": "bindings", "contains": ("hermes-agent-cluster-admin",)}, + rationale="The operator half of the pair; the in-pod attempts below are the enforcement half.", + ), + deny( + "access.secrets-are-denied", + "Hermes cannot read Secrets cluster-wide", + ("kubectl", "auth", "can-i", "get", "secrets", "--all-namespaces"), + ("kubectl", "get", "secrets", "--all-namespaces", "-o", "name"), + "Even a regression that granted this could only return object names: the probe is pinned to -o name.", + ), + check( + "access.service-account-tokens-are-denied", + "Hermes cannot mint a service-account token", + "access-denied", + "denied", + [ + step( + "review-token", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "serviceaccounts/token", + "--namespace", + namespace, + kind=REVIEW, + ), + step( + "review-serviceaccount", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "serviceaccounts", + "--namespace", + namespace, + kind=REVIEW, + ), + step( + "attempt", + SELF, + "kubectl", + "--namespace", + namespace, + "create", + "serviceaccount", + "hermes-acceptance-probe", + "--dry-run=server", + kind=ATTEMPT, + ), + ], + rationale="TokenRequest has no side-effect-free live attempt — a successful one would mint a real credential — so it is asserted by review alongside a refused write in the same RBAC family.", + ), + deny( + "access.impersonation-is-denied", + "Hermes cannot impersonate another identity", + ("kubectl", "auth", "can-i", "impersonate", "users", "--all-namespaces"), + ("kubectl", "get", "namespaces", "--as", "system:admin", "-o", "name"), + "`--as` is reachable only from inside the pod, where the identity is testing its own refusal.", + ), + deny( + "access.workload-mutation-is-denied", + "Hermes cannot mutate its own workload", + ("kubectl", "auth", "can-i", "patch", "deployments", "--namespace", namespace), + ( + "kubectl", + "--namespace", + namespace, + "patch", + f"deploy/{targets.agent_deployment}", + "--type=merge", + "--patch", + '{"metadata":{"annotations":{"acceptance.bstein.dev/probe":"deny"}}}', + "--dry-run=server", + ), + "A server dry run is authorised and then discarded, so the refusal is real and nothing is written.", + ), + deny( + "access.pod-exec-is-denied", + "Hermes cannot exec into an arbitrary pod", + ("kubectl", "auth", "can-i", "create", "pods/exec", "--all-namespaces"), + ( + "kubectl", + "--namespace", + namespace, + "exec", + f"deploy/{targets.switchyard_deployment}", + "--container", + targets.switchyard_container, + "--", + *shell("path_readable", path="/tmp"), + ), + "Exec is the one subresource with a harmless real attempt; attach and port-forward are covered by their own reviews.", + ), + check( + "access.attach-portforward-and-kube-system-writes-are-denied", + "Hermes holds no attach, port-forward, or control-plane write authority", + "access-denied", + "denied", + [ + step("review-attach", SELF, "kubectl", "auth", "can-i", "create", "pods/attach", "--all-namespaces", kind=REVIEW), + step("review-portforward", SELF, "kubectl", "auth", "can-i", "create", "pods/portforward", "--all-namespaces", kind=REVIEW), + step("review-create", SELF, "kubectl", "auth", "can-i", "create", "configmaps", "--namespace", "kube-system", kind=REVIEW), + step( + "attempt", + SELF, + "kubectl", + "--namespace", + "kube-system", + "create", + "configmap", + "hermes-acceptance-probe", + "--dry-run=server", + kind=ATTEMPT, + ), + ], + rationale="Attach and port-forward have no side-effect-free live attempt, so their reviews ride alongside a real, refused control-plane write.", + ), + check( + "access.required-reads-are-allowed", + "Hermes retains the reads its operator role depends on", + "access-allowed", + "allowed", + [ + step("watch-review", SELF, "kubectl", "auth", "can-i", "watch", "pods", "--namespace", namespace, kind=REVIEW), + step("pods", SELF, "kubectl", "--namespace", namespace, "get", "pods", "-o", "name"), + step("namespaces", SELF, "kubectl", "get", "namespaces", "-o", "name"), + step("deployments", SELF, "kubectl", "--namespace", namespace, "get", "deployments", "-o", "name"), + step("nodes", SELF, "kubectl", "get", "nodes", "-o", "name"), + ], + ), + check( + "access.pod-logs-are-allowed", + "Hermes can still read pod logs", + "access-allowed", + "allowed", + [ + step( + "logs", + SELF, + "kubectl", + "--namespace", + targets.namespace, + "logs", + f"deploy/{targets.switchyard_deployment}", + "--container", + targets.switchyard_container, + "--tail", + "1", + record=False, + ) + ], + ), + check( + "access.flux-and-helm-status-are-allowed", + "Hermes can read Flux and Helm reconciliation status", + "access-allowed", + "allowed", + [ + step("kustomizations", SELF, "kubectl", "get", "kustomizations.kustomize.toolkit.fluxcd.io", "--all-namespaces", "-o", "name", record=False), + step("helmreleases", SELF, "kubectl", "get", "helmreleases.helm.toolkit.fluxcd.io", "--all-namespaces", "-o", "name", record=False), + ], + ), + check( + "access.namespace-view-agrees-across-vantages", + "The operator and in-pod vantages see the same namespace inventory", + "access-allowed", + "vantages_agree", + [ + step("operator", OPERATOR, "kubectl", "get", "namespaces", "-o", "name", record=False), + step("self", SELF, "kubectl", "get", "namespaces", "-o", "name", record=False), + ], + {"steps": ("operator", "self")}, + rationale="A disagreement here means one vantage is not seeing the cluster the other is certifying.", + ), + ] + + +def _gitops(targets: Targets) -> list[CheckSpec]: + return [ + check( + f"gitops.{kind.split('.')[0]}-reconcile-cleanly", + f"No unexpected {label} suspension or unhealthy reconciliation", + "gitops", + "flux_health", + [step("objects", OPERATOR, "kubectl", "get", kind, "--all-namespaces", "-o", FLUX_PROJECTION, record=False)], + {"step": "objects", "expected_suspensions": targets.expected_suspensions}, + ) + for kind, label in ( + ("kustomizations.kustomize.toolkit.fluxcd.io", "Kustomization"), + ("helmreleases.helm.toolkit.fluxcd.io", "HelmRelease"), + ) + ] + + +def access_checks(targets: Targets) -> list[CheckSpec]: + """Return the access-denied, access-allowed, and GitOps health checks.""" + return [*_access(targets), *_gitops(targets)] diff --git a/scripts/ops/hermes_handoff_checks_delivery.py b/scripts/ops/hermes_handoff_checks_delivery.py new file mode 100644 index 00000000..a3dbaad7 --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_delivery.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python3 +"""Acceptance checks for forge authority, node accounts, images, and regressions. + +What the platform is allowed to do outside its own namespace: which forge +authority a worker holds, what a node account is and is not, how an image is +built and pinned, and whether the fixes the release depends on are actually +running rather than merely merged. + +Where a property has no side-effect-free live probe — a merge that must never be +attempted, a branch protection the harness identity cannot read — the entry says +so in its rationale and asserts the nearest fact that *is* observable, rather +than quietly asserting nothing. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + FORGE_CREDENTIAL_NAMES, + INIT_NAME_PROJECTION, + NODE, + NODE_NAME_PROJECTION, + OPERATOR, + SELF, + Targets, + check, + step, +) +from hermes_handoff_model import ATTEMPT, CheckSpec +from hermes_handoff_policy import GITEA_CLIENT, shell + +HOST_ETC = "/host-etc" +HOST_HOME = "/host-home" + + +def _forge(targets: Targets) -> list[CheckSpec]: + api = f"/api/v1/repos/{targets.repo}" + return [ + check( + "forge.worker-holds-no-git-credential-variable", + "No forge credential is exported into the worker process", + "forge", + "names_absent", + [step("env", SELF, *shell("env_names"))], + {"step": "env", "names": FORGE_CREDENTIAL_NAMES}, + rationale="Authentication reaches Git through an askpass helper that reads runtime state, never through the environment.", + ), + check( + "forge.worker-holds-no-credential-file", + "The worker home has no stored Git credential file", + "forge", + "stdout_matches", + [step("credentials", SELF, *shell("path_readable", path="/opt/data/.git-credentials"))], + {"step": "credentials", "equals": "unreadable"}, + ), + check( + "forge.identity-is-not-an-administrator", + "The forge identity the worker uses holds no administrative rights", + "forge", + "json_field", + [step("user", SELF, GITEA_CLIENT, "GET", "/api/v1/user")], + {"step": "user", "fields": {"is_admin": False}}, + ), + check( + "forge.repository-authority-is-push-only", + "The worker can push a branch but cannot administer the repository", + "forge", + "json_field", + [step("repo", SELF, GITEA_CLIENT, "GET", api)], + { + "step": "repo", + "fields": { + "permissions.admin": False, + "permissions.push": True, + "permissions.pull": True, + }, + }, + rationale="Merge, approve, close, and protection changes all require authority this identity is proven not to hold.", + ), + check( + "forge.administrative-route-is-refused", + "An administrative forge route is refused for this identity", + "forge", + "denied", + [ + step( + "attempt", + SELF, + GITEA_CLIENT, + "GET", + f"{api}/branch_protections", + kind=ATTEMPT, + ) + ], + rationale="A live refusal, not a configuration reading. The same lack of authority is what blocks a merge route.", + ), + check( + "forge.clone-and-fetch-still-work", + "The worker can still read the repository it is expected to work in", + "forge", + "allowed", + [ + step( + "ls-remote", + SELF, + *shell( + "git_ls_remote", + askpass=targets.git_askpass, + url=f"https://scm.bstein.dev/{targets.repo}.git", + ), + ) + ], + ), + ] + + +def _nodes(targets: Targets) -> list[CheckSpec]: + account = targets.node_account + return [ + check( + "nodes.account-password-is-locked", + "The dedicated node account has no usable password", + "nodes", + "stdout_matches", + [ + step( + "lock", + NODE, + *shell("account_lock_state", account=account, path=f"{HOST_ETC}/shadow"), + ) + ], + {"step": "lock", "equals": "locked"}, + rationale="The probe reports the lock marker only; the password hash is never read.", + ), + check( + "nodes.account-identity-is-dedicated", + "The node account is the dedicated Hermes identity, not a shared one", + "nodes", + "stdout_matches", + [ + step( + "identity", + NODE, + *shell("account_identity", account=account, path=f"{HOST_ETC}/passwd"), + ) + ], + {"step": "identity", "equals": targets.node_account_identity}, + ), + check( + "nodes.account-holds-no-supplementary-groups", + "The node account joins no supplementary or administrative group", + "nodes", + "stdout_matches", + [ + step( + "groups", + NODE, + *shell("account_groups", account=account, path=f"{HOST_ETC}/group"), + ) + ], + {"step": "groups", "equals": ""}, + rationale="sudo, docker, adm, and disk are all group grants; an empty membership list rules out every one of them.", + ), + check( + "nodes.account-has-no-sudo-policy", + "No sudoers fragment names the node account", + "nodes", + "stdout_matches", + [step("sudoers", NODE, *shell("path_mode", path=f"{HOST_ETC}/sudoers.d/{account}"))], + {"step": "sudoers", "equals": "absent"}, + ), + check( + "nodes.account-holds-a-single-authorized-key", + "Exactly one authorized key is installed for the node account", + "nodes", + "stdout_matches", + [ + step( + "keys", + NODE, + *shell("line_count", path=f"{HOST_HOME}/{account}/.ssh/authorized_keys"), + ) + ], + {"step": "keys", "equals": "1"}, + rationale="A count, never the key material. A second key is how a human identity re-enters an account that is meant to be dedicated.", + ), + *[ + check( + f"nodes.{label}-is-root-only", + f"{title} remains unreadable to the node account", + "nodes", + "stdout_matches", + [step("mode", NODE, *shell("path_mode", path=path))], + {"step": "mode", "equals": expected}, + ) + for label, title, path, expected in ( + ( + "k3s-credentials", + "The K3s admin kubeconfig", + f"{HOST_ETC}/rancher/k3s/k3s.yaml", + ["600 root:root", "absent"], + ), + ("k3s-state", "K3s server state", "/host-k3s", ["700 root:root", "absent"]), + ("kubelet-state", "Kubelet state", "/host-kubelet", ["700 root:root", "750 root:root", "absent"]), + ("container-runtime", "The container runtime socket directory", "/host-run-containerd", ["710 root:root", "700 root:root", "absent"]), + ) + ], + check( + "nodes.hardening-covers-every-node", + "The node account audit runs on every node in the cluster", + "nodes", + "distinct_count", + [ + step( + "coverage", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "pods", + "--selector", + f"app={targets.node_daemonset}", + "-o", + NODE_NAME_PROJECTION, + ) + ], + {"step": "coverage", "minimum": targets.node_count}, + rationale="A hardened fleet with one unenrolled node is an unhardened fleet.", + ), + ] + + +def _build(targets: Targets) -> list[CheckSpec]: + pipeline = f"{targets.remote}/main:{targets.builder_pipeline}" + return [ + check( + "build.builder-service-account-is-tokenless", + "The image builder service account mounts no API token", + "build", + "json_field", + [ + step( + "serviceaccount", + OPERATOR, + "kubectl", + "--namespace", + targets.builder_namespace, + "get", + "serviceaccount", + targets.builder_serviceaccount, + "-o", + "json", + ) + ], + {"step": "serviceaccount", "fields": {"automountServiceAccountToken": False}}, + ), + check( + "build.builder-runs-unprivileged-with-exact-capabilities", + "The build pod is unprivileged and adds only the expected capabilities", + "build", + "names_present", + [step("pipeline", OPERATOR, "git", "show", pipeline, record=False)], + { + "step": "pipeline", + "names": ( + targets.builder_capabilities, + "privileged: false", + "allowPrivilegeEscalation: false", + "automountServiceAccountToken: false", + 'drop: ["ALL"]', + ), + }, + rationale="Exact lines, not a substring sweep: a widened capability set has to change one of these lines to take effect.", + ), + check( + "build.harbor-immutability-rule-is-applied", + "The Harbor immutable-tag rule for the agent image has been applied", + "build", + "json_numeric", + [ + step( + "job", + OPERATOR, + "kubectl", + "--namespace", + targets.harbor_namespace, + "get", + "job", + targets.harbor_immutability_job, + "-o", + "json", + ) + ], + {"step": "job", "fields": {"status.succeeded": {"min": 1}}}, + rationale="The job verifies the rule it creates, so a completed run is evidence the rule matched the expected contract.", + ), + check( + "build.deployed-image-is-digest-pinned", + "The running agent image is pinned by digest, not by a mutable tag", + "build", + "names_present", + [ + step( + "image", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.agent_deployment}", + "-o", + 'jsonpath={range .spec.template.spec.containers[*]}{.image}{"\\n"}{end}', + ) + ], + {"step": "image", "contains": ("@sha256:",)}, + ), + ] + + +def _reliability(targets: Targets) -> list[CheckSpec]: + return [ + check( + "reliability.finalization-and-replay-patches-are-deployed", + "The agent workload runs the finalization, replay, and session patches", + "reliability", + "names_present", + [ + step( + "init", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.agent_deployment}", + "-o", + INIT_NAME_PROJECTION, + ) + ], + {"step": "init", "names": targets.agent_init_containers}, + rationale="The regression suites live in the repository; this asserts the fixes they cover are actually running.", + ), + check( + "reliability.regression-suites-are-present-on-main", + "The decomposition and worker-recovery regression suites are on main", + "reliability", + "names_present", + [ + step( + "suites", + OPERATOR, + "git", + "ls-tree", + "--name-only", + f"{targets.remote}/main", + "testing/tests/", + ) + ], + { + "step": "suites", + "names": ( + "testing/tests/test_hermes_cli_lanes.py", + "testing/tests/test_hermes_coordinator.py", + "testing/tests/test_hermes_worker_recovery.py", + ), + }, + ), + ] + + +def delivery_checks(targets: Targets) -> list[CheckSpec]: + """Return the forge, node, build, and reliability checks.""" + return [*_forge(targets), *_nodes(targets), *_build(targets), *_reliability(targets)] diff --git a/scripts/ops/hermes_handoff_checks_platform.py b/scripts/ops/hermes_handoff_checks_platform.py new file mode 100644 index 00000000..d1986976 --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_platform.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""Acceptance checks for the release baseline, provider identity, and routing. + +These are the facts a handoff rests on before any authority question is asked: +that the tree under test is the merged one, that both providers authenticate +through the subscriptions they are supposed to, that Switchyard actually decided +the routes it claims to, and that chat, agent, and triage are still three +distinct scopes rather than one surface wearing three hostnames. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + CLAUDE_HEALTH, + CLAUDE_HEALTH_FIELDS, + CODEX_HEALTH, + CODEX_HEALTH_FIELDS, + ENV_NAME_PROJECTION, + OPERATOR, + PROVIDER_API_KEY_NAMES, + ROUTING_LOG, + SELF, + SWITCHYARD, + Targets, + check, + step, +) +from hermes_handoff_model import CheckSpec +from hermes_handoff_policy import GITEA_CLIENT, shell + + +def _baseline(targets: Targets) -> list[CheckSpec]: + api = f"/api/v1/repos/{targets.repo}" + checks = [ + check( + "baseline.origin-main-descends-merged-work", + "origin/main descends from the merged Hermes isolation baseline", + "baseline", + "allowed", + [ + step( + "ancestor", + OPERATOR, + "git", + "merge-base", + "--is-ancestor", + targets.baseline_commit, + f"{targets.remote}/main", + ) + ], + rationale="Everything downstream assumes the merged worker-isolation baseline is present.", + ), + check( + "baseline.pending-pull-requests-recorded", + "Exact open pull-request heads are recorded, not assumed", + "baseline", + "json_record", + [step("open", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls?state=open&limit=50")], + { + "step": "open", + "record": { + "numbers": "[].number", + "head_refs": "[].head.ref", + "head_shas": "[].head.sha", + "base_shas": "[].base.sha", + "drafts": "[].draft", + "mergeable": "[].mergeable", + }, + }, + rationale="The runbook has to name the heads it was verified against; stale heads are how a release verifies the wrong tree.", + ), + check( + "baseline.operator-checkout-clean", + "The operator checkout has no uncommitted drift", + "baseline", + "stdout_matches", + [step("status", OPERATOR, "git", "status", "--porcelain")], + {"step": "status", "equals": ""}, + mandatory=False, + rationale="Advisory: a dirty checkout does not invalidate cluster evidence, but it does muddy what was compared.", + ), + ] + checks += [ + check( + f"baseline.dependency-pr-{number}-merged", + f"Pull request #{number} is merged into main", + "baseline", + "json_field", + [step("pr", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls/{number}")], + {"step": "pr", "fields": {"merged": True, "base.ref": "main"}}, + rationale="This harness certifies the post-merge platform; an unmerged dependency means it is measuring something else.", + ) + for number in targets.dependency_pull_requests + ] + return checks + + +def _identity(targets: Targets) -> list[CheckSpec]: + codex = shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS) + claude = shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS) + agent = f"deploy/{targets.agent_deployment}" + return [ + check( + "identity.codex-is-chatgpt-subscription", + "Codex authenticates through the ChatGPT subscription, not an API key", + "identity", + "json_field", + [step("health", SELF, *codex)], + { + "step": "health", + "fields": { + "authenticated": True, + "transport": "codex-chatgpt-subscription", + "state": "available", + }, + }, + rationale="Only allow-listed status fields are read; the credential itself is never opened.", + ), + check( + "identity.codex-evidence-is-fresh", + "Codex authentication evidence is current", + "identity", + "json_recent", + [step("health", SELF, *codex)], + { + "step": "health", + "now": targets.now, + "fields": {"checked_at": targets.max_evidence_age_seconds}, + }, + rationale="Stale provider evidence describes a state the release no longer has.", + ), + check( + "identity.claude-is-firstparty-subscription", + "Claude authenticates as a claude.ai first-party subscription", + "identity", + "json_field", + [step("health", SELF, *claude)], + { + "step": "health", + "fields": { + "authenticated": True, + "api_provider": "firstParty", + "auth_method": "claude.ai", + "transport": "claude-code-cli-subscription", + "state": "available", + }, + }, + ), + check( + "identity.claude-evidence-is-fresh", + "Claude authentication evidence is current", + "identity", + "json_recent", + [step("health", SELF, *claude)], + { + "step": "health", + "now": targets.now, + "fields": {"checked_at": targets.max_evidence_age_seconds}, + }, + ), + check( + "identity.no-provider-api-key-in-pod", + "No provider API-key variable is set in the running agent process", + "identity", + "names_absent", + [step("env", SELF, *shell("env_names"))], + { + "step": "env", + "names": PROVIDER_API_KEY_NAMES, + "contains": ("API_KEY",), + }, + rationale="Names only. The probe lists variable names and never their values.", + ), + check( + "identity.no-provider-api-key-in-manifest", + "No provider API-key variable is declared on the agent workload", + "identity", + "names_absent", + [ + step( + "env", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + agent, + "-o", + ENV_NAME_PROJECTION, + ) + ], + {"step": "env", "names": PROVIDER_API_KEY_NAMES, "contains": ("API_KEY",)}, + rationale="The operator counterpart to the in-pod probe: desired state and running state are asserted separately.", + ), + ] + + +def _routing(targets: Targets) -> list[CheckSpec]: + switchyard = f"deploy/{targets.switchyard_deployment}" + tail = shell("tail_lines", path=ROUTING_LOG, limit=str(targets.routing_tail_lines)) + routing_step = step( + "routing", SWITCHYARD, *tail, record=False, max_bytes=targets.routing_tail_bytes + ) + return [ + check( + "routing.switchyard-is-available", + "Switchyard has a ready replica serving every model-call boundary", + "routing", + "json_numeric", + [ + step( + "deployment", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + switchyard, + "-o", + "json", + ) + ], + {"step": "deployment", "fields": {"status.readyReplicas": {"min": 1}}}, + ), + check( + "routing.provider-effort-and-fallback-evidence", + "Routing evidence covers both providers, all graded efforts, and a real fallback", + "routing", + "routing_evidence", + [routing_step], + { + "step": "routing", + "providers": ("codex", "claude"), + "efforts": ("medium", "high", "xhigh"), + "lanes": ("route", "worker"), + "require_fallback_evidence": True, + "max_age_seconds": targets.max_evidence_age_seconds, + "now": targets.now, + }, + rationale="The routing log records the decided route per boundary; the tail is parsed here and never pasted into the report.", + ), + check( + "routing.codex-latency-recorded", + "Codex boundary latency is measured, not assumed", + "routing", + "json_numeric", + [step("health", SELF, *shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS))], + {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, + ), + check( + "routing.claude-latency-recorded", + "Claude boundary latency is measured, not assumed", + "routing", + "json_numeric", + [step("health", SELF, *shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS))], + {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, + ), + ] + + +def _scope_profile( + targets: Targets, name: str, workload: str, container: str, expected: str +) -> CheckSpec: + projection = ( + f'jsonpath={{.spec.template.spec.containers[?(@.name=="{container}")]' + '.env[?(@.name=="HERMES_AUTO_ROUTER_PROFILE")].value}' + ) + return check( + f"scopes.{name}-profile-is-distinct", + f"The {name} surface runs under its own routing profile", + "scopes", + "stdout_matches", + [ + step( + "profile", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + workload, + "-o", + projection, + ) + ], + {"step": "profile", "equals": expected}, + rationale="Chat, agent, and triage must not collapse into one scope; each states its own profile.", + ) + + +def _scopes(targets: Targets) -> list[CheckSpec]: + return [ + _scope_profile( + targets, "agent", f"deploy/{targets.agent_deployment}", targets.agent_container, "agent" + ), + _scope_profile( + targets, + "chat", + f"statefulset/{targets.chat_statefulset}", + targets.chat_container, + "chat", + ), + _scope_profile( + targets, + "triage", + f"deploy/{targets.triage_deployment}", + targets.triage_container, + "triage", + ), + check( + "scopes.surfaces-have-distinct-hosts", + "Chat, agent, and triage are published on distinct hostnames", + "scopes", + "distinct_count", + [ + step( + "hosts", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "ingress", + "-o", + 'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}', + ) + ], + {"step": "hosts", "minimum": 3}, + ), + ] + + +def platform_checks(targets: Targets) -> list[CheckSpec]: + """Return the baseline, identity, routing, and scope checks.""" + return [*_baseline(targets), *_identity(targets), *_routing(targets), *_scopes(targets)] diff --git a/scripts/ops/hermes_handoff_checks_workers.py b/scripts/ops/hermes_handoff_checks_workers.py new file mode 100644 index 00000000..a2e1abea --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_workers.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +"""Acceptance checks for the distributed worker pool and the visible surfaces. + +A pool is only distributed if its workers occupy distinct nodes and distinct +volumes and leave state ownership with the coordinator; a replica count alone +satisfies none of that. A surface is only working if the work it did is visible +in it, which is why the session and Kanban checks look at the same data the UI +renders rather than at whether a pod is Running. + +The ephemeral entries are placeholders in a default run. They report NOT_RUN +until the mutation mode is separately armed, and the armed mode replaces them +with real, mandatory results. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + CHAT, + INIT_NAME_PROJECTION, + NODE_NAME_PROJECTION, + OPERATOR, + SELF, + Targets, + check, +) +from hermes_handoff_catalog import step as step +from hermes_handoff_model import EPHEMERAL, CheckSpec + +KANBAN_STATUSES = [ + "archived", + "blocked", + "done", + "ready", + "review", + "running", + "scheduled", + "todo", + "triage", +] + + +def _pool(targets: Targets) -> list[CheckSpec]: + selector = targets.pool_selector + return [ + check( + "pool.three-workers-are-ready", + "The distributed execution pool has its full complement of ready workers", + "pool", + "json_numeric", + [ + step( + "statefulset", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + "json", + ) + ], + { + "step": "statefulset", + "fields": { + "status.readyReplicas": {"min": targets.pool_replicas}, + "spec.replicas": {"min": targets.pool_replicas}, + }, + }, + ), + check( + "pool.workers-occupy-distinct-nodes", + "Each pool worker runs on its own node", + "pool", + "distinct_count", + [ + step( + "nodes", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "pods", + "--selector", + selector, + "-o", + NODE_NAME_PROJECTION, + ) + ], + { + "step": "nodes", + "minimum": targets.pool_replicas, + "total_equals": targets.pool_replicas, + }, + rationale="Three replicas co-scheduled onto one node satisfy a replica count and defeat the isolation it stands for.", + ), + check( + "pool.workers-own-distinct-volumes", + "Each pool worker has its own durable workspace", + "pool", + "distinct_count", + [ + step( + "claims", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "persistentvolumeclaims", + "--selector", + selector, + "-o", + "name", + ) + ], + {"step": "claims", "minimum": targets.pool_replicas}, + ), + check( + "pool.workers-are-tokenless", + "Pool workers mount no Kubernetes API token", + "pool", + "json_field", + [ + step( + "statefulset", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + "json", + ) + ], + { + "step": "statefulset", + "fields": {"spec.template.spec.automountServiceAccountToken": False}, + }, + rationale="A fenced execution worker needs no cluster authority; holding a token is the failure, not the exception.", + ), + check( + "pool.coordinator-retains-sole-state-ownership", + "No pool worker mounts the coordinator's durable state", + "pool", + "names_absent", + [ + step( + "claims", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}', + ) + ], + {"step": "claims", "names": targets.coordinator_claims}, + rationale="Two writers on one SQLite state file is the duplicate-work and lost-result failure this pool exists to avoid.", + ), + *_pool_env(targets), + ] + + +def _pool_env(targets: Targets) -> list[CheckSpec]: + """Return the assignment-safety check only when a site declares what to expect. + + A ``names_present`` assertion over an empty expectation would pass on any + workload at all, which is worse than not asserting: it reads as coverage. + """ + if not targets.pool_worker_env: + return [] + return [ + check( + "pool.assignment-safety-knobs-are-configured", + "Pool workers carry the lease and de-duplication configuration", + "pool", + "names_present", + [ + step( + "env", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + 'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}', + ) + ], + {"step": "env", "names": targets.pool_worker_env}, + rationale="Deployed configuration only; the stale-claim and duplicate-dispatch behaviour itself is covered by the pool's own regression suite.", + ) + ] + + +def _surfaces(targets: Targets) -> list[CheckSpec]: + checks = [ + check( + "surfaces.kanban-activity-is-visible", + "Durable Kanban activity is visible to the agent surface", + "surfaces", + "all_of_field", + [ + step( + "tasks", + SELF, + "hermes", + "kanban", + "list", + "--json", + record=False, + max_bytes=targets.listing_bytes, + ) + ], + {"step": "tasks", "key_field": "id", "fields": {"status": KANBAN_STATUSES}}, + rationale="Non-empty and well-formed. An empty board is NOT_RUN, because it cannot distinguish a quiet week from a broken read.", + ), + check( + "surfaces.durable-sessions-render-without-blank-rows", + "Durable sessions are listed with a preview, a source, and an id", + "surfaces", + "lines_match", + [step("sessions", SELF, "hermes", "sessions", "list", "--limit", "20", record=False)], + {"step": "sessions", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + rationale="A row whose leading column is blank is the session-render regression seen from the data side.", + ), + check( + "surfaces.agent-ui-session-patches-are-deployed", + "The agent surface runs the session-activity rendering patches", + "surfaces", + "names_present", + [ + step( + "init", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.agent_deployment}", + "-o", + INIT_NAME_PROJECTION, + ) + ], + {"step": "init", "names": ("patch-web-session-activity", "patch-api-server-sessions")}, + rationale="Blank session rows were a rendering regression; this asserts the patches that fixed it are in the running pod.", + ), + check( + "surfaces.chat-session-patches-are-deployed", + "The chat surface runs the API-session continuity patches", + "surfaces", + "names_present", + [ + step( + "init", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.chat_statefulset}", + "-o", + INIT_NAME_PROJECTION, + ) + ], + {"step": "init", "names": targets.chat_init_containers}, + ), + check( + "surfaces.chat-telegram-topic-state-is-durable", + "Telegram topic and tenant state is held on durable storage", + "surfaces", + "names_present", + [ + step( + "claims", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.chat_router_deployment}", + "-o", + 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}', + ) + ], + {"step": "claims", "contains": ("state",)}, + rationale="Topic selection survives a router restart only if the tenant state file is on a claim rather than in the pod.", + ), + ] + if targets.expect_telegram_sessions: + checks.append( + check( + "surfaces.chat-telegram-sessions-are-continuous", + "The selected chat tenant still holds durable Telegram-sourced sessions", + "surfaces", + "lines_match", + [ + step( + "telegram", + CHAT, + "hermes", + "sessions", + "list", + "--source", + "telegram", + "--limit", + "20", + record=False, + ) + ], + {"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.", + ) + ) + if targets.chat_config_revision: + checks.append( + check( + "surfaces.chat-runs-the-telegram-topic-revision", + "The chat surface runs the durable Telegram topic configuration", + "surfaces", + "stdout_matches", + [ + step( + "revision", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.chat_statefulset}", + "-o", + "jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}", + ) + ], + {"step": "revision", "equals": targets.chat_config_revision}, + rationale="Telegram topic continuity is carried by the config revision the pod template pins.", + ) + ) + return checks + + +def _ephemeral() -> list[CheckSpec]: + return [ + check( + f"ephemeral.{name}", + title, + "ephemeral", + "not_armed", + [], + {"reason": reason}, + mandatory=False, + scope=EPHEMERAL, + rationale="Mutating checks are separately armed. A default run reports them NOT_RUN and never touches the forge.", + ) + for name, title, reason in ( + ( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + "ephemeral mutation mode is not armed", + ), + ( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + "ephemeral mutation mode is not armed", + ), + ( + "protected-branch-refusal", + "Arming against a protected branch is refused before any network call", + "ephemeral mutation mode is not armed", + ), + ( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + "ephemeral mutation mode is not armed", + ), + ) + ] + + +def worker_checks(targets: Targets) -> list[CheckSpec]: + """Return the pool, surface, and ephemeral-placeholder checks.""" + return [*_pool(targets), *_surfaces(targets), *_ephemeral()] diff --git a/scripts/ops/hermes_handoff_ephemeral.py b/scripts/ops/hermes_handoff_ephemeral.py new file mode 100644 index 00000000..5456482b --- /dev/null +++ b/scripts/ops/hermes_handoff_ephemeral.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""The separately armed, self-cleaning mutation mode of the acceptance harness. + +Proving that a worker can still open a draft pull request means actually opening +one, which is the only part of this harness that writes anything. It is +therefore a distinct mode with a distinct policy: an explicit confirmation +phrase, a caller-supplied unique ref, a preflight that refuses a protected +target before any network call, and a cleanup whose *verification* is itself a +mandatory check. + +The push target is guarded, not the pull-request base. Opening a pull request +against `main` writes nothing to `main`; pushing to it does. The refusal is +checked by name and by pattern, and the arming self-test exercises it on every +armed run rather than trusting that it was checked once. +""" + +from __future__ import annotations + +import re +from collections.abc import Sequence +from dataclasses import dataclass + +from hermes_handoff_exec import Outcome, Runner, Vantage +from hermes_handoff_model import EPHEMERAL, FAIL, NOT_RUN, PASS, CheckResult, CheckSpec +from hermes_handoff_policy import GITEA_CLIENT + +CONFIRMATION = "ARM EPHEMERAL HERMES HANDOFF PUSH" +REF_PREFIX = "ephemeral/hermes-handoff-acceptance" +REF_RE = re.compile(rf"{REF_PREFIX}/[a-z0-9][a-z0-9-]{{7,63}}\Z") +TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9-]{7,63}\Z") +DRAFT_TITLE_PREFIX = "WIP: " +PROTECTED_NAMES = frozenset( + { + "default", + "develop", + "head", + "main", + "master", + "prod", + "production", + "release", + "stable", + "trunk", + } +) + + +class ArmingError(ValueError): + """Raised when an ephemeral run is not safe to start.""" + + +@dataclass(frozen=True) +class ArmRequest: + """Everything the armed mode needs, and nothing it can infer for itself.""" + + repo: str + remote: str + token: str + confirmation: str + base: str = "main" + + @property + def ref(self) -> str: + """Return the exact, unique branch this run is allowed to create.""" + return f"{REF_PREFIX}/{self.token}" + + +def normalise_branch(name: str) -> str: + """Return a branch name stripped of ref decoration, for comparison.""" + trimmed = name.strip().removeprefix("refs/heads/").strip("/") + return trimmed.lower() + + +def assert_push_target_allowed(ref: str) -> None: + """Raise unless ``ref`` is an ephemeral branch this harness may create.""" + if normalise_branch(ref) in PROTECTED_NAMES: + raise ArmingError(f"refusing to push to protected branch {ref!r}") + if not REF_RE.fullmatch(ref): + raise ArmingError(f"push target {ref!r} is not an ephemeral acceptance ref") + + +def preflight(request: ArmRequest, expected_repo: str) -> None: + """Validate an arming request before any network call is made.""" + if request.confirmation != CONFIRMATION: + raise ArmingError("confirmation phrase does not match") + if request.repo != expected_repo: + raise ArmingError(f"repository {request.repo!r} is not the expected {expected_repo!r}") + if not TOKEN_RE.fullmatch(request.token): + raise ArmingError("ephemeral token must be 8-64 lowercase alphanumeric or dash characters") + assert_push_target_allowed(request.ref) + if not normalise_branch(request.base): + raise ArmingError("a pull-request base branch is required") + + +def _spec(identifier: str, title: str, rationale: str = "") -> CheckSpec: + return CheckSpec( + id=f"ephemeral.{identifier}", + title=title, + group="ephemeral", + rule="armed", + steps=(), + mandatory=True, + scope=EPHEMERAL, + rationale=rationale, + ) + + +def _result( + identifier: str, + title: str, + status: str, + reason: str, + outcomes: Sequence[Outcome] = (), + evidence: dict | None = None, + rationale: str = "", +) -> CheckResult: + return CheckResult( + spec=_spec(identifier, title, rationale), + status=status, + reason=reason, + evidence=evidence or {}, + outcomes=list(outcomes), + ) + + +def _protected_refusal_result() -> CheckResult: + """Exercise the protected-branch guard on this run, not on a past one.""" + refused: list[str] = [] + accepted: list[str] = [] + for candidate in ("main", "master", "refs/heads/main", "HEAD", f"{REF_PREFIX}/x"): + try: + assert_push_target_allowed(candidate) + except ArmingError: + refused.append(candidate) + else: + accepted.append(candidate) + evidence = {"refused": refused, "accepted": accepted} + if accepted: + return _result( + "protected-branch-refusal", + "Arming against a protected branch is refused before any network call", + FAIL, + f"the guard accepted {accepted}", + evidence=evidence, + ) + return _result( + "protected-branch-refusal", + "Arming against a protected branch is refused before any network call", + PASS, + "every protected and malformed push target was refused", + evidence=evidence, + ) + + +def _detail(outcome: Outcome) -> str: + """Return the most informative bounded explanation an outcome carries.""" + return outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}" + + +def _pull_request_number(outcome: Outcome) -> int | None: + match = re.search(r'"number"\s*:\s*(\d+)', outcome.stdout) + return int(match.group(1)) if match else None + + +def run_armed(runner: Runner, vantage: Vantage, request: ArmRequest) -> list[CheckResult]: + """Push a unique ephemeral ref, open a draft pull request, and remove both. + + Every step is checked and every artifact is removed. Cleanup runs even when + an earlier step failed, and the cleanup verification is reported as its own + mandatory check so a half-removed branch cannot pass as success. + """ + api = f"/api/v1/repos/{request.repo}" + results = [_protected_refusal_result()] + outcomes: list[Outcome] = [] + + existing = runner.run(("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage) + outcomes.append(existing) + if not existing.ok: + results.append( + _result( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + NOT_RUN, + f"could not confirm the ref is unused: {_detail(existing)}", + outcomes, + ) + ) + return results + if existing.stdout.strip(): + results.append( + _result( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + FAIL, + f"{request.ref} already exists on {request.remote}; pick a fresh token", + outcomes, + ) + ) + return results + + pushed = runner.run( + ("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"), vantage + ) + outcomes.append(pushed) + results.append( + _result( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + PASS if pushed.ok else FAIL, + f"pushed {request.ref}" if pushed.ok else f"push failed: {_detail(pushed)}", + [pushed], + {"ref": request.ref}, + ) + ) + + number: int | None = None + if pushed.ok: + created = runner.run( + ( + GITEA_CLIENT, + "POST", + f"{api}/pulls", + "--field", + f"title={DRAFT_TITLE_PREFIX}Hermes handoff acceptance ephemeral probe", + "--field", + f"head={request.ref}", + "--field", + f"base={normalise_branch(request.base)}", + "--field", + "body=Ephemeral acceptance probe. Closed and deleted by the harness.", + ), + vantage, + ) + outcomes.append(created) + number = _pull_request_number(created) if created.ok else None + results.append( + _result( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + PASS if created.ok and number else FAIL, + f"opened draft pull request #{number}" + if created.ok and number + else f"draft creation failed: {_detail(created)}", + [created], + {"pull_request": number}, + ) + ) + else: + results.append( + _result( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + NOT_RUN, + "the ephemeral branch was never pushed", + ) + ) + + results.append(_cleanup(runner, vantage, request, api, number)) + return results + + +def _cleanup( + runner: Runner, vantage: Vantage, request: ArmRequest, api: str, number: int | None +) -> CheckResult: + """Close the pull request, delete the ref, and prove both are gone.""" + outcomes: list[Outcome] = [] + problems: list[str] = [] + + if number is not None: + closed = runner.run( + (GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"), vantage + ) + outcomes.append(closed) + if not closed.ok: + problems.append(f"pull request #{number} could not be closed") + + deleted = runner.run((GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage) + outcomes.append(deleted) + + remaining = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage + ) + outcomes.append(remaining) + if not remaining.ran: + problems.append(f"could not verify removal: {_detail(remaining)}") + elif remaining.stdout.strip(): + problems.append(f"{request.ref} still exists on {request.remote}") + + if number is not None: + state = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage) + outcomes.append(state) + if not state.ok: + problems.append(f"could not confirm pull request #{number} is closed") + elif '"state":"open"' in state.stdout.replace(" ", ""): + problems.append(f"pull request #{number} is still open") + + if problems: + return _result( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + FAIL, + "; ".join(problems), + outcomes, + {"ref": request.ref, "pull_request": number}, + ) + return _result( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + PASS, + "the ephemeral ref and pull request are gone and their absence was re-checked", + outcomes, + {"ref": request.ref, "pull_request": number}, + ) diff --git a/scripts/ops/hermes_handoff_evaluators.py b/scripts/ops/hermes_handoff_evaluators.py new file mode 100644 index 00000000..72e3558c --- /dev/null +++ b/scripts/ops/hermes_handoff_evaluators.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +"""Text-shaped evaluators for the handoff acceptance harness. + +These rules cover evidence that arrives as a refusal, an exit status, or a list +of names. The ``denied`` rule is the load-bearing one: it refuses to classify on +an authorization review alone — ``kubectl auth can-i`` reports what RBAC says, +not what the API server does — so a real request must have been made and +refused before it will pass. + +Importing this module also registers the JSON-shaped rules, so a caller only has +to import one place to have the full catalog vocabulary available. +""" + +from __future__ import annotations + +import re +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_json_rules import evaluate_flux_health # noqa: F401 (registers rules) +from hermes_handoff_model import ATTEMPT, FAIL, NOT_RUN, PASS, REVIEW, CheckSpec +from hermes_handoff_rules import ( + EVALUATORS, + Evaluation, + evaluate, + evaluator, + lines, + looks_denied, + missing_steps, + step_keys, +) + +__all__ = ["EVALUATORS", "Evaluation", "evaluate", "evaluator"] + + +@evaluator("denied") +def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass only when the review says no and a real request was refused.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + attempts = step_keys(spec, ATTEMPT) + if not attempts: + return Evaluation(NOT_RUN, "deny checks require a real attempt, not only a review") + evidence: dict[str, Any] = {} + for key in step_keys(spec, REVIEW): + verdict = outcomes[key].stdout.strip().lower() + evidence[key] = verdict + if verdict != "no": + return Evaluation( + FAIL, f"authorization review {key} reported {verdict or 'nothing'}", evidence + ) + for key in attempts: + outcome = outcomes[key] + if outcome.ok: + return Evaluation(FAIL, f"attempt {key} succeeded but must be refused", evidence) + if not looks_denied(outcome): + return Evaluation( + NOT_RUN, + f"attempt {key} failed without a recognisable refusal (rc={outcome.returncode})", + evidence, + ) + evidence[key] = "refused" + return Evaluation(PASS, "review and live attempt were both refused", evidence) + + +@evaluator("allowed") +def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every required step succeeded and every review said yes.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + evidence: dict[str, Any] = {} + for step in spec.steps: + outcome = outcomes[step.key] + if step.kind == REVIEW: + verdict = outcome.stdout.strip().lower() + evidence[step.key] = verdict + if verdict != "yes": + return Evaluation( + FAIL, + f"authorization review {step.key} reported {verdict or 'nothing'}", + evidence, + ) + continue + if not outcome.ok: + return Evaluation(FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence) + evidence[step.key] = "succeeded" + return Evaluation(PASS, "required access is present", evidence) + + +def _matched(observed: set[str], patterns: tuple[str, ...]) -> set[str]: + return {name for pattern in patterns for name in observed if pattern in name} + + +def _line_step(spec: CheckSpec, outcomes: dict[str, Outcome]) -> tuple[set[str], str] | Evaluation: + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + return (set(lines(outcome)), key) + + +@evaluator("names_absent") +def evaluate_names_absent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when no forbidden name or substring appears in a step's lines.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + resolved = _line_step(spec, outcomes) + if isinstance(resolved, Evaluation): + return resolved + observed, _ = resolved + offenders = sorted( + (observed & set(spec.expect.get("names", ()))) + | _matched(observed, tuple(spec.expect.get("contains", ()))) + ) + evidence = {"observed_count": len(observed), "offenders": offenders} + if offenders: + return Evaluation(FAIL, f"forbidden names present: {', '.join(offenders)}", evidence) + return Evaluation(PASS, "no forbidden names are present", evidence) + + +@evaluator("names_present") +def evaluate_names_present(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every required name, and every required substring, appears.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + resolved = _line_step(spec, outcomes) + if isinstance(resolved, Evaluation): + return resolved + observed, _ = resolved + missing = sorted(set(spec.expect.get("names", ())) - observed) + missing += sorted( + pattern + for pattern in spec.expect.get("contains", ()) + if not any(pattern in name for name in observed) + ) + evidence = {"observed_count": len(observed), "missing": missing} + if missing: + return Evaluation(FAIL, f"required names absent: {', '.join(missing)}", evidence) + if not observed: + return Evaluation(NOT_RUN, "step produced no lines to examine", evidence) + return Evaluation(PASS, "every required name is present", evidence) + + +@evaluator("stdout_matches") +def evaluate_stdout_matches(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Compare a step's trimmed stdout against an expected value or set.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + actual = outcome.stdout.strip() + expected = spec.expect["equals"] + allowed = expected if isinstance(expected, list) else [expected] + evidence = {"observed": actual} + if actual not in allowed: + return Evaluation(FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence) + return Evaluation(PASS, "output matches the expected value", evidence) + + +@evaluator("distinct_count") +def evaluate_distinct_count(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when a step yields enough distinct rows — and, optionally, exactly N. + + Distinctness is the point for placement checks: three worker pods pinned to + one node satisfy a replica count and defeat the isolation the replica count + was standing in for. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + values = lines(outcome) + distinct = sorted(set(values)) + minimum = spec.expect["minimum"] + evidence = {"total": len(values), "distinct": len(distinct), "values": distinct[:16]} + if not values: + return Evaluation(NOT_RUN, f"step {key} produced no rows to count", evidence) + if len(distinct) < minimum: + return Evaluation( + FAIL, f"{len(distinct)} distinct values, expected at least {minimum}", evidence + ) + total_equals = spec.expect.get("total_equals") + if total_equals is not None and len(values) != total_equals: + return Evaluation(FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence) + return Evaluation(PASS, f"{len(distinct)} distinct values observed", evidence) + + +@evaluator("lines_match") +def evaluate_lines_match(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every data row of a tabular listing matches a shape. + + Some surfaces only speak tables. A blank leading column is exactly what the + session-render regression looked like, so asserting the shape of each row is + a real check rather than a proxy for one. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + pattern = re.compile(spec.expect["pattern"]) + rows = outcome.stdout.splitlines()[spec.expect.get("skip", 0) :] + rows = [row for row in rows if row.strip()] + offenders = [row[:60] for row in rows if not pattern.search(row)] + evidence = {"rows": len(rows), "offenders": offenders[:8]} + if len(rows) < spec.expect.get("minimum", 1): + return Evaluation( + NOT_RUN, f"step {key} produced {len(rows)} data rows, expected at least " + f"{spec.expect.get('minimum', 1)}", evidence + ) + if offenders: + return Evaluation(FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence) + return Evaluation(PASS, f"all {len(rows)} rows match the expected shape", evidence) + + +@evaluator("vantages_agree") +def evaluate_vantages_agree(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass only when two independent observations report the same fact. + + A disagreement is a finding in its own right: it usually means the manifest + on record and the workload actually running have drifted apart, and either + one alone would have looked clean. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + left_key, right_key = spec.expect["steps"] + observations: dict[str, list[str]] = {} + for key in (left_key, right_key): + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + observations[key] = sorted(lines(outcome)) + evidence = {key: value[:16] for key, value in observations.items()} + if observations[left_key] != observations[right_key]: + return Evaluation(FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence) + if not observations[left_key]: + return Evaluation(NOT_RUN, "both vantages returned nothing to compare", evidence) + return Evaluation(PASS, "both vantages report the same state", evidence) + + +@evaluator("not_armed") +def evaluate_not_armed(spec: CheckSpec, _outcomes: dict[str, Outcome]) -> Evaluation: + """Report the fixed NOT_RUN a default, read-only run must produce.""" + return Evaluation( + NOT_RUN, + spec.expect.get("reason", "ephemeral mutation mode is not armed"), + {"armed": False}, + ) + + +# Two observations of the same fact agree whether or not they came from +# different vantages; the dual-vantage case is just the important one. +EVALUATORS["steps_agree"] = evaluate_vantages_agree diff --git a/scripts/ops/hermes_handoff_exec.py b/scripts/ops/hermes_handoff_exec.py new file mode 100644 index 00000000..200d82e0 --- /dev/null +++ b/scripts/ops/hermes_handoff_exec.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +"""Bounded, screened command execution for the handoff acceptance harness. + +Every probe runs through :class:`Runner`, which owns the three budgets a +long-running acceptance sweep needs: a per-command timeout, a whole-run +deadline, and a per-capture byte budget. Exhausting the deadline does not +silently shorten the sweep — the remaining probes report ``NOT_RUN``, which is a +NO_GO, so a slow cluster can never be mistaken for a clean one. + +Subprocesses never inherit the caller's environment. A probe that echoed +`$ANTHROPIC_API_KEY` would otherwise be one typo away from a leak, so the child +environment is rebuilt from an allowlist of names whose values the harness +already needs to reach the cluster and the forge. +""" + +from __future__ import annotations + +import os +import shlex +import subprocess +import time +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field + +from hermes_handoff_policy import READ_ONLY, PolicyError, check_argv +from hermes_handoff_redaction import DEFAULT_MAX_BYTES, redact, safe_text + +DEFAULT_COMMAND_TIMEOUT = 25.0 +DEFAULT_RUN_DEADLINE = 900.0 + +# Names whose values the harness needs to reach the cluster and the forge. The +# values are handed to the child process and are never recorded. +ENV_PASSTHROUGH = ( + "GIT_ASKPASS", + "GITEA_BASE_URL", + "HERMES_HOME", + "HOME", + "JENKINS_BASE_URL", + "KUBECONFIG", + "LANG", + "PATH", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_FILE", + "VAULT_ADDR", +) + +DEADLINE_ERROR = "deadline-exceeded" +TIMEOUT_ERROR = "timeout" + +OPERATOR = "operator" +SELF = "self" + +# Tools that accept a `--context` flag directly after the binary name. +CONTEXT_AWARE_BINARIES = ("kubectl", "flux", "helm") + + +@dataclass(frozen=True) +class Outcome: + """The screened, bounded result of one probe command.""" + + argv: tuple[str, ...] + vantage: str + returncode: int | None = None + stdout: str = "" + stderr: str = "" + truncated: bool = False + duration_ms: int = 0 + error: str | None = None + + @property + def ok(self) -> bool: + """Report whether the command ran to completion and succeeded.""" + return self.error is None and self.returncode == 0 + + @property + def ran(self) -> bool: + """Report whether the command reached the target at all.""" + return self.error is None + + @property + def combined(self) -> str: + """Return stdout and stderr joined, for message matching.""" + return "\n".join(part for part in (self.stdout, self.stderr) if part) + + def as_dict(self) -> dict[str, object]: + """Return a JSON-serialisable view of the outcome.""" + return { + "command": redact(shlex.join(self.argv)), + "vantage": self.vantage, + "returncode": self.returncode, + "stdout": self.stdout, + "stderr": self.stderr, + "truncated": self.truncated, + "duration_ms": self.duration_ms, + "error": self.error, + } + + +@dataclass(frozen=True) +class Vantage: + """A named identity and transport a probe can be observed from. + + ``prefix`` is how the in-pod vantage is reached: a ``kubectl exec`` into the + target container, so the inner command authenticates as the pod's own + service account instead of as the operator. ``inserts`` carries per-binary + flags — the operator's `--context`, which must not follow the command into + the pod. + """ + + name: str + prefix: tuple[str, ...] = () + inserts: Mapping[str, tuple[str, ...]] = field(default_factory=dict) + env: Mapping[str, str] = field(default_factory=dict) + description: str = "" + + def wrap(self, argv: Sequence[str]) -> tuple[str, ...]: + """Return ``argv`` addressed to this vantage.""" + command = tuple(argv) + if command: + extra = self.inserts.get(command[0].rsplit("/", 1)[-1]) + if extra: + command = (command[0], *extra, *command[1:]) + return (*self.prefix, *command) + + +def build_environment(vantage: Vantage, source: Mapping[str, str] | None = None) -> dict[str, str]: + """Return the child environment: allow-listed names plus vantage overrides.""" + origin = os.environ if source is None else source + environment = {name: origin[name] for name in ENV_PASSTHROUGH if name in origin} + environment.setdefault("LC_ALL", "C") + environment.update(vantage.env) + return environment + + +def operator_vantage(kubeconfig: str | None = None, context: str | None = None) -> Vantage: + """Return the external read-only operator vantage.""" + inserts = ( + {binary: ("--context", context) for binary in CONTEXT_AWARE_BINARIES} if context else {} + ) + detail = " ".join( + part for part in (f"context={context}" if context else "", "kubeconfig-pinned" if kubeconfig else "") if part + ) + return Vantage( + name=OPERATOR, + inserts=inserts, + env={"KUBECONFIG": kubeconfig} if kubeconfig else {}, + description=f"operator {detail}".strip(), + ) + + +def pod_vantage( + namespace: str, + pod: str, + container: str, + operator: Vantage | None = None, +) -> Vantage: + """Return the in-pod vantage: the inner command authenticates as the pod.""" + reach = operator.inserts.get("kubectl", ()) if operator else () + return Vantage( + name=SELF, + prefix=( + "kubectl", + *reach, + "exec", + "--namespace", + namespace, + pod, + "--container", + container, + "--", + ), + env=dict(operator.env) if operator else {}, + description=f"in-pod {namespace}/{pod}[{container}]", + ) + + +class Runner: + """Run probe commands under policy, timeout, deadline, and byte budgets.""" + + def __init__( + self, + mode: str = READ_ONLY, + command_timeout: float = DEFAULT_COMMAND_TIMEOUT, + deadline_seconds: float = DEFAULT_RUN_DEADLINE, + max_bytes: int = DEFAULT_MAX_BYTES, + clock: Callable[[], float] = time.monotonic, + spawn: Callable[..., subprocess.CompletedProcess] = subprocess.run, + environ: Mapping[str, str] | None = None, + ) -> None: + self.mode = mode + self.command_timeout = command_timeout + self.max_bytes = max_bytes + self._clock = clock + self._spawn = spawn + self._environ = environ + self._deadline_at = clock() + deadline_seconds + self.commands_run = 0 + + @property + def remaining_seconds(self) -> float: + """Return the seconds left before the whole-run deadline.""" + return self._deadline_at - self._clock() + + def run( + self, argv: Sequence[str], vantage: Vantage, max_bytes: int | None = None + ) -> Outcome: + """Run one command, returning a screened outcome and never raising.""" + budget_bytes = self.max_bytes if max_bytes is None else max_bytes + addressed = vantage.wrap(argv) + try: + check_argv(addressed, self.mode) + except PolicyError as exc: + return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}") + + budget = min(self.command_timeout, self.remaining_seconds) + if budget <= 0: + return Outcome(argv=addressed, vantage=vantage.name, error=DEADLINE_ERROR) + + started = self._clock() + self.commands_run += 1 + try: + completed = self._spawn( + list(addressed), + capture_output=True, + text=True, + timeout=budget, + check=False, + env=build_environment(vantage, self._environ), + ) + except subprocess.TimeoutExpired: + return Outcome( + argv=addressed, + vantage=vantage.name, + error=TIMEOUT_ERROR, + duration_ms=int((self._clock() - started) * 1000), + ) + except OSError as exc: + return Outcome( + argv=addressed, vantage=vantage.name, error=f"spawn: {exc.strerror or exc}" + ) + + stdout, cut_out = safe_text(completed.stdout or "", budget_bytes) + stderr, cut_err = safe_text(completed.stderr or "", budget_bytes) + return Outcome( + argv=addressed, + vantage=vantage.name, + returncode=completed.returncode, + stdout=stdout, + stderr=stderr, + truncated=cut_out or cut_err, + duration_ms=int((self._clock() - started) * 1000), + ) diff --git a/scripts/ops/hermes_handoff_json_rules.py b/scripts/ops/hermes_handoff_json_rules.py new file mode 100644 index 00000000..6c10e52d --- /dev/null +++ b/scripts/ops/hermes_handoff_json_rules.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""JSON-shaped evaluators for the handoff acceptance harness. + +These rules cover the evidence that arrives as structured output — Kubernetes +objects, forge pull requests, provider health files, and the two append-only +evidence logs the platform keeps. They share the fail-closed contract of every +other rule: an absent path, an unparsable payload, or an empty result set is +``NOT_RUN``, never a quiet pass. +""" + +from __future__ import annotations + +import datetime as dt +import json +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_model import FAIL, NOT_RUN, PASS, CheckSpec +from hermes_handoff_rules import ( + Evaluation, + dotted, + evaluator, + is_list, + missing_steps, + parsed_step, +) + + +@evaluator("json_field") +def evaluate_json_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Compare dotted JSON paths of one step's output against expected values.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + evidence: dict[str, Any] = {} + mismatches: list[str] = [] + for path, expected in spec.expect["fields"].items(): + try: + actual = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + evidence[path] = actual + allowed = expected if isinstance(expected, list) else [expected] + if actual not in allowed: + mismatches.append(f"{path}={actual!r} expected one of {allowed!r}") + if mismatches: + return Evaluation(FAIL, "; ".join(mismatches), evidence) + return Evaluation(PASS, "observed fields match the release contract", evidence) + + +@evaluator("json_numeric") +def evaluate_json_numeric(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Compare numeric JSON paths against inclusive minimum/maximum bounds.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + evidence: dict[str, Any] = {} + problems: list[str] = [] + for path, bounds in spec.expect["fields"].items(): + try: + actual = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + if not isinstance(actual, (int, float)) or isinstance(actual, bool): + return Evaluation(NOT_RUN, f"path {path} is not numeric in step {key}", evidence) + evidence[path] = actual + if "min" in bounds and actual < bounds["min"]: + problems.append(f"{path}={actual} below minimum {bounds['min']}") + if "max" in bounds and actual > bounds["max"]: + problems.append(f"{path}={actual} above maximum {bounds['max']}") + if problems: + return Evaluation(FAIL, "; ".join(problems), evidence) + return Evaluation(PASS, "numeric fields are within the expected bounds", evidence) + + +@evaluator("json_recent") +def evaluate_json_recent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when a timestamp field is fresh enough to be evidence about now. + + Stale provider or routing evidence is worse than none: it describes a state + the release no longer has. A timestamp older than the allowed age is a FAIL, + and one the harness cannot parse is NOT_RUN. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + now = spec.expect["now"] + evidence: dict[str, Any] = {} + stale: list[str] = [] + for path, max_age in spec.expect["fields"].items(): + try: + raw = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + try: + observed = dt.datetime.fromisoformat(str(raw).replace("Z", "+00:00")) + except ValueError: + return Evaluation(NOT_RUN, f"path {path} is not an ISO timestamp", evidence) + if observed.tzinfo is None: + observed = observed.replace(tzinfo=dt.timezone.utc) + age = (now - observed).total_seconds() + evidence[path] = {"age_seconds": int(age)} + if age > max_age: + stale.append(f"{path} is {int(age)}s old, limit {max_age}s") + if stale: + return Evaluation(FAIL, "; ".join(stale), evidence) + return Evaluation(PASS, "observed timestamps are fresh", evidence) + + +@evaluator("all_of_field") +def evaluate_all_of_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every selected element of a JSON array matches expected fields.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + try: + elements = dotted(payload, spec.expect.get("array", "")) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"array path is absent from step {key}") + if not is_list(elements): + return Evaluation(NOT_RUN, f"step {key} did not return an array") + key_field = spec.expect.get("key_field", "") + wanted = spec.expect.get("keys") + exempt = set(spec.expect.get("exempt_keys", ())) + seen: list[Any] = [] + problems: list[str] = [] + evidence: dict[str, Any] = {} + for element in elements: + identity = dotted(element, key_field) if key_field else None + if wanted is not None and identity not in wanted: + continue + if identity in exempt: + continue + seen.append(identity) + for path in spec.expect.get("non_empty", ()): + try: + value = dotted(element, path) + except (KeyError, ValueError, IndexError): + value = None + if value in (None, "", [], {}): + problems.append(f"{identity}: {path} is empty") + for path, expected in spec.expect.get("fields", {}).items(): + try: + actual = dotted(element, path) + except (KeyError, ValueError, IndexError): + actual = None + allowed = expected if isinstance(expected, list) else [expected] + if actual not in allowed: + problems.append(f"{identity}: {path}={actual!r} expected one of {allowed!r}") + evidence["examined"] = seen[:32] + if wanted is not None: + absent = [item for item in wanted if item not in seen] + if absent: + return Evaluation(NOT_RUN, f"expected keys absent from step {key}: {absent}", evidence) + elif not seen: + return Evaluation(NOT_RUN, f"step {key} returned nothing to examine", evidence) + if problems: + return Evaluation(FAIL, "; ".join(problems[:8]), evidence) + return Evaluation(PASS, f"{len(seen)} element(s) match the release contract", evidence) + + +@evaluator("json_record") +def evaluate_json_record(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Capture named JSON paths as evidence; pass when all of them resolve. + + Recording is a release requirement in its own right — the runbook has to + name the exact heads it was verified against — so an unreachable source is a + NOT_RUN rather than an empty section. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + evidence: dict[str, Any] = {} + for label, path in spec.expect["record"].items(): + try: + evidence[label] = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + return Evaluation(PASS, "state recorded", evidence) + + + +@evaluator("routing_evidence") +def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Summarise a Switchyard routing log tail into provider/effort coverage. + + Route identifiers are ``///``, so provider, + model family, effort tier, and lane all come from the same field. Fallback + and failure evidence is a separate ``fallback_reason`` on the record that + was retried. The parser tolerates a clipped first line: a tail of a growing + file starts mid-record, and discarding one row is better than a NOT_RUN that + reads as an outage. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + records = [] + for line in outcome.stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + records.append(json.loads(line)) + except ValueError: + continue + routes = [ + str(record.get("model") or "").split("/") + for record in records + if str(record.get("model") or "").count("/") == 3 + ] + fallbacks = [record for record in records if record.get("fallback_reason")] + stamps = sorted(str(record.get("ts")) for record in records if record.get("ts")) + evidence: dict[str, Any] = { + "records": len(records), + "routed": len(routes), + "lanes": sorted({route[0] for route in routes}), + "providers": sorted({route[1] for route in routes}), + "families": sorted({route[2] for route in routes}), + "efforts": sorted({route[3] for route in routes}), + "fallback_events": len(fallbacks), + "fallback_reasons": sorted({str(record["fallback_reason"]) for record in fallbacks}), + "latest_ts": stamps[-1] if stamps else "", + } + if not records: + return Evaluation(NOT_RUN, "routing log tail contained no parsable records", evidence) + missing = [ + f"{label}:{item}" + for label, wanted in ( + ("provider", spec.expect.get("providers", ())), + ("effort", spec.expect.get("efforts", ())), + ("lane", spec.expect.get("lanes", ())), + ) + for item in wanted + if item not in evidence[f"{label}s"] + ] + if spec.expect.get("require_fallback_evidence") and not fallbacks: + missing.append("fallback:none-observed") + max_age = spec.expect.get("max_age_seconds") + now = spec.expect.get("now") + if max_age and now: + try: + newest = dt.datetime.fromisoformat(evidence["latest_ts"].replace("Z", "+00:00")) + except ValueError: + return Evaluation(NOT_RUN, "routing log has no parsable newest timestamp", evidence) + age = (now - newest).total_seconds() + evidence["age_seconds"] = int(age) + if age > max_age: + missing.append(f"freshness:{int(age)}s-exceeds-{max_age}s") + if missing: + return Evaluation(FAIL, f"routing evidence missing {', '.join(missing)}", evidence) + return Evaluation(PASS, "routing evidence covers the required lanes", evidence) + + +@evaluator("flux_health") +def evaluate_flux_health(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Classify a compact Flux projection into suspensions and unhealthy objects. + + Each row is ``namespace/namesuspendType=Status,...``. An object is + unhealthy when it reports ``Healthy=False``, or ``Ready=False`` with no + Healthy condition and no in-flight ``Reconciling=True``. That last clause + matters: Flux flips Ready to False for the duration of a reconcile, so + without it the verdict depends on when the snapshot was taken. Suspended + objects are excluded from the health tally and judged against the + expected-suspension list instead, because a deliberate park is a different + fact from a broken reconcile. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + expected = set(spec.expect.get("expected_suspensions", ())) + suspended: list[str] = [] + unhealthy: list[str] = [] + total = 0 + for line in outcome.stdout.splitlines(): + if not line.strip(): + continue + total += 1 + name, _, rest = line.partition("\t") + suspend, _, conditions = rest.partition("\t") + if suspend.strip().lower() == "true": + suspended.append(name.strip()) + continue + states = dict( + pair.split("=", 1) for pair in conditions.strip().strip(",").split(",") if "=" in pair + ) + healthy = states.get("Healthy") + reconciling = states.get("Reconciling") == "True" + ready_failed = states.get("Ready") == "False" + if healthy == "False" or (healthy is None and ready_failed and not reconciling): + unhealthy.append(name.strip()) + unexpected = sorted(set(suspended) - expected) + evidence = { + "total": total, + "suspended": sorted(suspended), + "unexpected_suspensions": unexpected, + "unhealthy": sorted(unhealthy), + } + if not total: + return Evaluation(NOT_RUN, f"step {key} returned no Flux objects", evidence) + problems = [] + if unexpected: + problems.append(f"unexpected suspensions: {', '.join(unexpected)}") + if unhealthy: + problems.append(f"unhealthy: {', '.join(sorted(unhealthy)[:8])}") + if problems: + return Evaluation(FAIL, "; ".join(problems), evidence) + return Evaluation(PASS, f"{total} Flux objects reconciled as expected", evidence) diff --git a/scripts/ops/hermes_handoff_model.py b/scripts/ops/hermes_handoff_model.py new file mode 100644 index 00000000..d34b9ebc --- /dev/null +++ b/scripts/ops/hermes_handoff_model.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""Result model, aggregation, and rendering for the handoff acceptance harness. + +The classification is deliberately four-valued. ``NOT_APPLICABLE`` is the only +way a check can be excused, and a check can only reach it by matching a scope +rule that a reviewer wrote down; anything the harness merely failed to observe +is ``NOT_RUN``. Because a mandatory ``NOT_RUN`` is a NO_GO exactly like a +``FAIL``, an unreachable cluster, an exhausted deadline, or a probe nobody +maintained can never round up to a release. +""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_redaction import contains_credential_shape, scrub + +PASS = "PASS" +FAIL = "FAIL" +NOT_RUN = "NOT_RUN" +NOT_APPLICABLE = "NOT_APPLICABLE" +STATUSES = (PASS, FAIL, NOT_RUN, NOT_APPLICABLE) + +GO = "GO" +NO_GO = "NO_GO" + +ALWAYS = "always" +EPHEMERAL = "ephemeral" + +HARNESS = "hermes-handoff-acceptance" +SCHEMA_VERSION = 1 + + +REVIEW = "review" +ATTEMPT = "attempt" +READ = "read" +STEP_KINDS = (REVIEW, ATTEMPT, READ) + + +@dataclass(frozen=True) +class Step: + """One command a check needs, and the vantage it must be observed from. + + ``kind`` separates a self access review (``review``) from a real request the + identity is expected to be refused (``attempt``). A deny check needs both: + an authorization review states policy, only an attempt proves the API server + enforces it. + + ``record`` is cleared for bulk evidence — a routing log tail, a large object + listing — which the harness parses but must not paste into the artifact. + ``max_bytes`` raises the capture budget for that same bulk evidence, where a + default-sized truncation would silently narrow the window being examined. + """ + + key: str + vantage: str + argv: tuple[str, ...] + kind: str = READ + optional: bool = False + record: bool = True + max_bytes: int | None = None + + +@dataclass(frozen=True) +class CheckSpec: + """A declarative acceptance check. + + ``rule`` names an evaluator; ``expect`` carries its parameters. Keeping the + catalog declarative means a reviewer reads what is asserted, not how it is + plumbed, and the harness can prove structural properties over every check + (no impersonation, no mutation, both vantages present) before running any. + """ + + id: str + title: str + group: str + rule: str + steps: tuple[Step, ...] = () + expect: Mapping[str, Any] = field(default_factory=dict) + mandatory: bool = True + scope: str = ALWAYS + rationale: str = "" + + @property + def vantages(self) -> tuple[str, ...]: + """Return the distinct vantages this check draws evidence from.""" + seen: list[str] = [] + for step in self.steps: + if step.vantage not in seen: + seen.append(step.vantage) + return tuple(seen) + + +@dataclass +class CheckResult: + """The classified outcome of one acceptance check.""" + + spec: CheckSpec + status: str + reason: str = "" + evidence: dict[str, Any] = field(default_factory=dict) + outcomes: list[Outcome] = field(default_factory=list) + + @property + def blocking(self) -> bool: + """Report whether this result alone forces NO_GO.""" + return self.spec.mandatory and self.status in {FAIL, NOT_RUN} + + def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: + """Return a JSON-serialisable, screened view of the result.""" + payload: dict[str, Any] = { + "id": self.spec.id, + "title": self.spec.title, + "group": self.spec.group, + "rule": self.spec.rule, + "scope": self.spec.scope, + "mandatory": self.spec.mandatory, + "vantages": list(self.spec.vantages), + "status": self.status, + "reason": self.reason, + "blocking": self.blocking, + "evidence": self.evidence, + } + if self.spec.rationale: + payload["rationale"] = self.spec.rationale + if include_outcomes: + payload["commands"] = [outcome.as_dict() for outcome in self.outcomes] + return scrub(payload) + + +@dataclass +class VantageRecord: + """What the harness proved about one vantage before trusting its evidence.""" + + name: str + description: str + identity: str = "" + available: bool = False + detail: str = "" + + def as_dict(self) -> dict[str, Any]: + """Return a JSON-serialisable view of the vantage.""" + return scrub( + { + "name": self.name, + "description": self.description, + "identity": self.identity, + "available": self.available, + "detail": self.detail, + } + ) + + +@dataclass +class Report: + """The whole acceptance run: metadata, per-check results, and a verdict.""" + + mode: str + started_at: str + baseline: dict[str, Any] = field(default_factory=dict) + vantages: list[VantageRecord] = field(default_factory=list) + results: list[CheckResult] = field(default_factory=list) + harness_errors: list[str] = field(default_factory=list) + finished_at: str = "" + + @property + def counts(self) -> dict[str, int]: + """Return the number of results in each status.""" + tally = dict.fromkeys(STATUSES, 0) + for result in self.results: + tally[result.status] = tally.get(result.status, 0) + 1 + return tally + + @property + def blocking(self) -> list[CheckResult]: + """Return every result that forces NO_GO, most severe first.""" + order = {FAIL: 0, NOT_RUN: 1} + return sorted( + (result for result in self.results if result.blocking), + key=lambda result: (order.get(result.status, 2), result.spec.id), + ) + + @property + def decision(self) -> str: + """Return GO only when nothing mandatory failed, skipped, or errored.""" + if self.harness_errors or self.blocking: + return NO_GO + return GO + + def group_summary(self) -> dict[str, dict[str, int]]: + """Return per-group status counts, for the human summary.""" + groups: dict[str, dict[str, int]] = {} + for result in self.results: + bucket = groups.setdefault(result.spec.group, dict.fromkeys(STATUSES, 0)) + bucket[result.status] = bucket.get(result.status, 0) + 1 + return groups + + def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: + """Return the machine-readable report.""" + return { + "harness": HARNESS, + "schema_version": SCHEMA_VERSION, + "mode": self.mode, + "started_at": self.started_at, + "finished_at": self.finished_at, + "decision": self.decision, + "counts": self.counts, + "baseline": scrub(self.baseline), + "vantages": [vantage.as_dict() for vantage in self.vantages], + "harness_errors": [scrub(error) for error in self.harness_errors], + "blocking": [result.spec.id for result in self.blocking], + "checks": [result.as_dict(include_outcomes) for result in self.results], + } + + def render_summary(self, width: int = 88) -> str: + """Return the concise human summary printed alongside the JSON.""" + counts = self.counts + lines = [ + f"{HARNESS}: {self.decision}", + f" mode={self.mode} started={self.started_at} checks={len(self.results)}", + " " + " ".join(f"{status}={counts[status]}" for status in STATUSES), + ] + for vantage in self.vantages: + state = "available" if vantage.available else "unavailable" + identity = vantage.identity or "unknown" + lines.append(f" vantage {vantage.name}: {state} identity={identity}") + for error in self.harness_errors: + lines.append(f" harness error: {error}") + if self.blocking: + lines.append(" blocking:") + for result in self.blocking: + lines.append(f" {result.status:<15}{result.spec.id}") + lines.append(f" {'':<15}{result.reason[:width]}") + groups = self.group_summary() + lines.append(" groups:") + for name in sorted(groups): + bucket = groups[name] + detail = " ".join(f"{status}={bucket[status]}" for status in STATUSES if bucket[status]) + lines.append(f" {name:<18}{detail}") + return "\n".join(lines) + + +def utc_now(clock: Any = None) -> str: + """Return an ISO-8601 UTC timestamp, injectable for deterministic tests.""" + now = clock() if clock else dt.datetime.now(dt.timezone.utc) + return now.replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def unscreened_fields(payload: Any, path: str = "$") -> list[str]: + """Return JSON pointers whose text still looks credential-shaped. + + The report is already screened field by field; this is the invariant check + that runs before anything is written, so a screening gap fails the run + instead of publishing an artifact nobody re-read. + """ + offenders: list[str] = [] + if isinstance(payload, str): + if contains_credential_shape(payload): + offenders.append(path) + elif isinstance(payload, Mapping): + for key, value in payload.items(): + offenders.extend(unscreened_fields(value, f"{path}.{key}")) + elif isinstance(payload, Sequence) and not isinstance(payload, (str, bytes)): + for index, value in enumerate(payload): + offenders.extend(unscreened_fields(value, f"{path}[{index}]")) + return offenders diff --git a/scripts/ops/hermes_handoff_policy.py b/scripts/ops/hermes_handoff_policy.py new file mode 100644 index 00000000..473dd782 --- /dev/null +++ b/scripts/ops/hermes_handoff_policy.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +"""Fail-closed command policy for the Hermes handoff acceptance harness. + +Two properties have to hold structurally, not by convention: + +* the default run mutates nothing, and +* no probe can pull a credential value into the report. + +Both are enforced here, where an argv becomes a subprocess, so a mistake in the +check catalog surfaces as a policy error when the catalog is built rather than +as a mutation or a leak when it runs. `--as` is rejected on the operator side: +impersonation is not evidence about what Hermes can do, and a harness able to +impersonate would be tempted to substitute it for a real self-probe. It stays +available to the inner command of a `kubectl exec`, where an identity is +probing its own refusal to impersonate rather than borrowing authority. + +Shell is reachable only through frozen templates. Free-form `sh -c` cannot be +screened reliably, but a few probes genuinely need field extraction (report that +a password field is locked without ever reading its hash) or an explicit helper +path, so those scripts are named, reviewed here, and parameter-validated instead +of assembled by callers. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator + +READ_ONLY = "read-only" +ARMED = "ephemeral-armed" +MODES = (READ_ONLY, ARMED) + +# Rejected everywhere. Impersonation and inline credentials have no honest use +# in an acceptance probe. +FORBIDDEN_ARGS = ( + "--as", + "--as-group", + "--as-uid", + "--token", + "--password", + "--username", + "--client-key", + "--client-certificate", +) +# Reachable only from inside a pod, where an identity is probing its own +# refusal to impersonate rather than borrowing someone else's authority. +IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid") +FORBIDDEN_BINARIES = frozenset({"vault", "sops", "age", "gpg", "openssl", "curl", "wget"}) +GITEA_CLIENT = "/opt/coordinator/gitea_api.py" +ALLOWED_BINARIES = frozenset({"kubectl", "flux", "helm", "git", "hermes", "sh", GITEA_CLIENT}) + +# Flags that consume the following token, so it is never a subcommand. +VALUE_FLAGS = frozenset( + { + "-c", + "-l", + "-n", + "-o", + "--cluster", + "--container", + "--context", + "--field-selector", + "--kubeconfig", + "--namespace", + "--output", + "--request-timeout", + "--selector", + "--server", + "--since", + "--tail", + "--user", + } +) + +KUBECTL_READ_SUBCOMMANDS = frozenset( + { + "api-resources", + "api-versions", + "auth", + "cluster-info", + "config", + "describe", + "events", + "explain", + "get", + "logs", + "top", + "version", + } +) +# Reachable only through a dry run: the API server authorises the request and +# discards it. A rejection is the evidence, and nothing is persisted either way. +KUBECTL_DRY_RUN_SUBCOMMANDS = frozenset( + {"annotate", "apply", "create", "delete", "label", "patch", "replace", "scale", "set"} +) +KUBECTL_EXEC_SUBCOMMANDS = frozenset({"exec"}) +KUBECTL_SUBCOMMANDS = ( + KUBECTL_READ_SUBCOMMANDS | KUBECTL_DRY_RUN_SUBCOMMANDS | KUBECTL_EXEC_SUBCOMMANDS +) +DRY_RUN_FLAGS = ("--dry-run=server", "--dry-run=client") + +# Resources whose objects carry credential material. Reading them is permitted +# only in name-only output form, so a regression that grants access still cannot +# deliver a secret value into a report. +NAME_ONLY_RESOURCES = frozenset({"secret", "secrets"}) + +FLUX_READ_SUBCOMMANDS = frozenset( + {"check", "diff", "events", "export", "get", "stats", "trace", "tree", "version"} +) +HELM_READ_SUBCOMMANDS = frozenset({"get", "history", "list", "status", "version"}) +GIT_READ_SUBCOMMANDS = frozenset( + { + "cat-file", + "config", + "diff", + "fetch", + "for-each-ref", + "log", + "ls-remote", + "ls-tree", + "merge-base", + "remote", + "rev-list", + "rev-parse", + "show", + "status", + } +) +GIT_ARMED_SUBCOMMANDS = frozenset({"push"}) +GIT_FORCE_FLAGS = frozenset({"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"}) +GITEA_READ_METHODS = frozenset({"GET"}) +GITEA_ARMED_METHODS = frozenset({"POST", "PATCH", "DELETE"}) + +# The Hermes CLI is the only honest source for durable session and Kanban +# activity, so a narrow read-only slice of it is on the allowlist. +HERMES_READ_COMMANDS = frozenset({("kanban", "list"), ("kanban", "show"), ("sessions", "list"), ("status",)}) + +# Paths that hold credential material. A probe may prove one exists or is +# unreadable; it may never read it outside a frozen template. +SECRET_PATH_RE = re.compile( + r"(?:^|/)(?:vault/secrets|var/run/secrets|runtime-access|\.credentials\.json" + r"|auth\.json|authorized_keys|id_[a-z0-9]+|\.netrc|shadow|gshadow)(?:$|/)" +) + +_PARAMETER_RE = re.compile(r"[A-Za-z0-9/][A-Za-z0-9._/@:,-]{0,255}\Z") +_PLACEHOLDER_RE = re.compile(r"\{([a-z_]+)\}") + +# Frozen shell templates. Each exists because a plain argv cannot express the +# safety property its probe needs. +SHELL_TEMPLATES: dict[str, str] = { + # Report an account's lock state without reading its password hash. + "account_lock_state": ( + 'awk -F: \'$1=="{account}"{{print ($2=="!"||$2=="*"||$2=="!!")?"locked":"unlocked"}}\' {path}' + ), + # Report the account record's non-secret identity fields. + "account_identity": 'awk -F: \'$1=="{account}"{{print $3":"$4":"$6":"$7}}\' {path}', + # Report supplementary group names the account joins, never their members. + "account_groups": "awk -F: '$4~/(^|,){account}(,|$)/{{print $1}}' {path}", + # Prove a path's presence and mode without reading its bytes. + "path_mode": 'if [ -e {path} ]; then stat -c "%a %U:%G" {path}; else echo absent; fi', + # Prove whether a path is readable by this identity, without printing it. + "path_readable": 'if [ -r {path} ]; then echo readable; else echo unreadable; fi', + # List environment variable names only. Values never leave the container. + "env_names": "env | sed -n 's/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/\\1/p' | sort", + # Report a JSON document's top-level key names without any of its values. + "json_keys": ( + 'python3 -c "import json,sys;print(chr(10).join(sorted(json.load(open(sys.argv[1])))))" {path}' + ), + # Report allow-listed non-secret status fields from a provider health file. + "json_fields": ( + 'python3 -c "import json,sys;d=json.load(open(sys.argv[1]));' + 'print(json.dumps({{k:d.get(k) for k in sys.argv[2].split(chr(44))}}))" {path} {fields}' + ), + # Count non-empty lines, for authorized-key inventories. + "line_count": "awk 'NF{{n++}} END{{print n+0}}' {path}", + # Tail an append-only evidence log. The harness parses it; the container is + # not assumed to carry an interpreter. + "tail_lines": "tail -n {limit} {path}", + # Prove the worker can still read the repository. `kubectl exec` does not + # inherit the coordinator process environment, so the askpass helper is + # named explicitly rather than assumed; the helper reads runtime state and + # the credential never reaches this command line. + "git_ls_remote": "GIT_ASKPASS={askpass} git ls-remote {url} refs/heads/main", +} + + +class PolicyError(ValueError): + """Raised when a requested command falls outside the safe boundary.""" + + +def template_parameters(template_name: str) -> set[str]: + """Return the parameter names a frozen template requires.""" + template = SHELL_TEMPLATES.get(template_name) + if template is None: + raise PolicyError(f"unknown shell template: {template_name}") + return set(_PLACEHOLDER_RE.findall(template)) + + +def render_shell(template_name: str, **parameters: str) -> str: + """Return a frozen shell template with validated parameters substituted.""" + required = template_parameters(template_name) + template = SHELL_TEMPLATES[template_name] + if set(parameters) != required: + raise PolicyError(f"shell template {template_name} expects parameters {sorted(required)}") + for key, value in parameters.items(): + if not _PARAMETER_RE.fullmatch(value): + raise PolicyError(f"unsafe shell parameter for {template_name}.{key}") + return template.format(**parameters) + + +_RENDERED: set[str] = set() + + +def shell(template_name: str, **parameters: str) -> tuple[str, ...]: + """Return a validated ``sh -c`` argv built from a frozen template.""" + script = render_shell(template_name, **parameters) + _RENDERED.add(script) + return ("sh", "-c", script) + + +def positionals(argv: tuple[str, ...], start: int = 1) -> Iterator[str]: + """Yield positional arguments, skipping flags and the values they consume.""" + index = start + while index < len(argv): + item = argv[index] + if item == "--": + return + if item.startswith("-"): + if item in VALUE_FLAGS: + index += 2 + continue + index += 1 + continue + yield item + index += 1 + + +def _reject_forbidden_arguments(argv: tuple[str, ...], allow_impersonation: bool = False) -> None: + permitted = IMPERSONATION_ARGS if allow_impersonation else () + boundary = argv.index("--") if "--" in argv else len(argv) + for argument in argv[:boundary]: + for forbidden in FORBIDDEN_ARGS: + if forbidden in permitted: + continue + if argument == forbidden or argument.startswith(f"{forbidden}="): + raise PolicyError(f"forbidden argument: {forbidden}") + + +def _reject_secret_paths(argv: tuple[str, ...]) -> None: + boundary = argv.index("--") if "--" in argv else len(argv) + for argument in argv[:boundary]: + if SECRET_PATH_RE.search(argument): + raise PolicyError("probes may not name a credential path outside a frozen template") + + +def _assert_name_only_output(argv: tuple[str, ...]) -> None: + if "--output=name" in argv or "-oname" in argv: + return + for index, item in enumerate(argv[:-1]): + if item in {"-o", "--output"} and argv[index + 1] == "name": + return + raise PolicyError("credential-bearing resources may only be read with -o name") + + +def _check_kubectl(argv: tuple[str, ...], mode: str, allow_impersonation: bool) -> None: + words = list(positionals(argv)) + subcommand = next((word for word in words if word in KUBECTL_SUBCOMMANDS), None) + if subcommand is None: + raise PolicyError("kubectl subcommand is outside the read-only boundary") + if subcommand != words[0]: + raise PolicyError("kubectl subcommand must precede its resource arguments") + resource = words[1].split(".")[0] if len(words) > 1 else "" + if subcommand in {"get", "describe"} and resource in NAME_ONLY_RESOURCES: + _assert_name_only_output(argv) + if subcommand in KUBECTL_READ_SUBCOMMANDS: + return + if subcommand in KUBECTL_EXEC_SUBCOMMANDS: + if "--" not in argv: + raise PolicyError("kubectl exec requires an explicit -- command boundary") + inner = argv[argv.index("--") + 1 :] + if not inner: + raise PolicyError("kubectl exec requires an inner command") + check_argv(inner, mode, allow_impersonation=True) + return + if any(flag in argv for flag in DRY_RUN_FLAGS): + return + raise PolicyError(f"kubectl {subcommand} requires a dry run") + + +def _first_positional(binary: str, argv: tuple[str, ...]) -> str: + word = next(positionals(argv), None) + if word is None: + raise PolicyError(f"{binary} requires a subcommand") + return word + + +def _check_read_only_tool(binary: str, argv: tuple[str, ...], allowed: frozenset[str]) -> None: + subcommand = _first_positional(binary, argv) + if subcommand not in allowed: + raise PolicyError(f"{binary} {subcommand} is outside the read-only boundary") + + +def _check_git(argv: tuple[str, ...], mode: str) -> None: + subcommand = _first_positional("git", argv) + if subcommand == "config" and not {"--get", "--get-all", "--list"} & set(argv): + raise PolicyError("git config may only read") + if subcommand in GIT_READ_SUBCOMMANDS: + return + if subcommand in GIT_ARMED_SUBCOMMANDS and mode == ARMED: + if GIT_FORCE_FLAGS & set(argv): + raise PolicyError("git push may not force or fan out across refs") + return + raise PolicyError(f"git {subcommand} is not permitted in {mode} mode") + + +def _check_hermes(argv: tuple[str, ...]) -> None: + words = tuple(positionals(argv)) + if not words: + raise PolicyError("hermes requires a subcommand") + if words[:2] not in HERMES_READ_COMMANDS and words[:1] not in HERMES_READ_COMMANDS: + raise PolicyError(f"hermes {' '.join(words[:2])} is outside the read-only boundary") + + +def _check_gitea(argv: tuple[str, ...], mode: str) -> None: + if len(argv) < 3: + raise PolicyError("gitea_api.py requires a method and a path") + method = argv[1].upper() + if method in GITEA_READ_METHODS: + return + if method in GITEA_ARMED_METHODS and mode == ARMED: + return + raise PolicyError(f"gitea_api.py {method} is not permitted in {mode} mode") + + +def _check_shell(argv: tuple[str, ...]) -> None: + if len(argv) != 3 or argv[1] != "-c": + raise PolicyError("sh is only reachable as sh -c ") + if argv[2] not in _RENDERED: + raise PolicyError("sh -c script is not a frozen template rendering") + + +def check_argv( + argv: tuple[str, ...], mode: str = READ_ONLY, allow_impersonation: bool = False +) -> None: + """Raise :class:`PolicyError` unless ``argv`` is safe to run in ``mode``. + + ``allow_impersonation`` is set only for the inner command of a + ``kubectl exec``. Hermes probing its own refusal to impersonate is evidence; + the operator impersonating Hermes is not, so `--as` stays unreachable from + the outer, operator-side argv. + """ + if mode not in MODES: + raise PolicyError(f"unknown mode: {mode}") + if not argv: + raise PolicyError("empty command") + binary = argv[0] if argv[0] == GITEA_CLIENT else argv[0].rsplit("/", 1)[-1] + if binary in FORBIDDEN_BINARIES: + raise PolicyError(f"forbidden binary: {binary}") + if binary not in ALLOWED_BINARIES: + raise PolicyError(f"binary is not on the harness allowlist: {binary}") + _reject_forbidden_arguments(argv, allow_impersonation) + if binary == "sh": + _check_shell(argv) + return + _reject_secret_paths(argv) + if binary == "kubectl": + _check_kubectl(argv, mode, allow_impersonation) + elif binary == "flux": + _check_read_only_tool("flux", argv, FLUX_READ_SUBCOMMANDS) + elif binary == "helm": + _check_read_only_tool("helm", argv, HELM_READ_SUBCOMMANDS) + elif binary == "git": + _check_git(argv, mode) + elif binary == "hermes": + _check_hermes(argv) + else: + _check_gitea(argv, mode) diff --git a/scripts/ops/hermes_handoff_redaction.py b/scripts/ops/hermes_handoff_redaction.py new file mode 100644 index 00000000..f48344a5 --- /dev/null +++ b/scripts/ops/hermes_handoff_redaction.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Credential redaction and output bounding for the handoff acceptance harness. + +Every string the harness records — command output, failure reasons, evidence +values — passes through :func:`redact` before it reaches a report. Screening is +a fail-closed accident barrier, not proof that text is secret-free: a short +secret under an innocuous key is not reliably distinguishable from prose. The +harness therefore also never asks for a secret in the first place. Probes read +names, shapes, and status fields; they never read environment values, token +files, private keys, cookies, or Vault responses. + +Bounding is part of the same barrier. An unbounded capture is how a debug dump +or a stack trace carrying an authorization header ends up in an artifact, so +captures are truncated at a byte budget before they are screened. +""" + +from __future__ import annotations + +import re +from typing import Any + +REDACTED = "[redacted]" +DEFAULT_MAX_BYTES = 64 * 1024 +TRUNCATION_NOTE = "[truncated]" + +# Keys whose value is never safe to record, matched against a normalised +# (lowercased, separator-stripped) form of the key. +SENSITIVE_KEY_SUFFIXES = ( + "accesskey", + "accesskeyid", + "apikey", + "authtoken", + "clientsecret", + "connectionstring", + "cookie", + "credential", + "credentials", + "passphrase", + "passwd", + "password", + "privatekey", + "secret", + "secretkey", + "sessionkey", + "signature", + "signingkey", + "token", + "webhook", +) +SENSITIVE_KEY_EXACT = frozenset( + { + "auth", + "authorization", + "bearer", + "pat", + "pw", + "sas", + "setcookie", + } +) +# A bare `key:` is a structural field name across Kubernetes and TOML far more +# often than it is a credential, so it is screened by value shape and by the +# token patterns rather than by name. The compound forms above still match. + +# Values that cannot hold a credential: booleans, numbers, and anything too +# short to be one. Screening them corrupts structured evidence — a redacted +# `automountServiceAccountToken` is exactly the fact a reader needs — without +# protecting anything. +MIN_SECRET_LENGTH = 8 +NON_SECRET_LITERALS = frozenset({"true", "false", "null", "none", "yes", "no", "0", "1", ""}) +_KEY_NOISE_RE = re.compile(r"[^a-z0-9]+") + +# Ordered: structural matches first so a nested token is not partially rewritten +# by a looser rule. +_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile( + r"-----BEGIN[ A-Z0-9-]{0,64}-----.*?-----END[ A-Z0-9-]{0,64}-----", + re.DOTALL, + ), + re.compile(r"(?i)\b(?:authorization|proxy-authorization)\s*:\s*\S.*"), + re.compile(r"(?i)\b(?:set-)?cookie\s*:\s*\S.*"), + re.compile(r"(?i)\b(?:bearer|basic|token)\s+[A-Za-z0-9+/_=.:~-]{8,}"), + re.compile(r"(?i)\b[a-z][a-z0-9+.-]{1,20}://[^\s/@:]{1,128}:[^\s/@]{1,256}@"), + re.compile(r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}"), + re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}"), + re.compile(r"(?["']?) + (?P[A-Za-z][A-Za-z0-9_.\ -]{0,63}) + (?P=quote) + (?P\s{0,8}[:=]\s{0,8}) + (?P"[^"\n]{0,4096}"|'[^'\n]{0,4096}'|[^\s,;}\n][^,;}\n]{0,4096}) + """ +) + +# Long, mixed-class runs. Deliberately conservative: prose and hex digests of +# fewer than 44 characters are left alone so commit SHAs stay readable. +_ENTROPY_RE = re.compile(r"(? str: + """Return a key lowercased with separators removed, for suffix matching.""" + return _KEY_NOISE_RE.sub("", key.lower()) + + +def is_sensitive_key(key: str) -> bool: + """Report whether a key name means its value must never be recorded.""" + compact = normalise_key(key) + if not compact: + return False + if compact in SENSITIVE_KEY_EXACT: + return True + return compact.endswith(SENSITIVE_KEY_SUFFIXES) + + +def _is_high_entropy(candidate: str) -> bool: + """Report whether a long run mixes enough character classes to be a token.""" + classes = sum( + ( + any(character.islower() for character in candidate), + any(character.isupper() for character in candidate), + any(character.isdigit() for character in candidate), + any(character in "+/_=-" for character in candidate), + ) + ) + return classes >= 3 + + +def could_hold_secret(value: str) -> bool: + """Report whether a raw value is long and opaque enough to be a credential.""" + stripped = value.strip().strip("\"'") + if stripped.lower() in NON_SECRET_LITERALS or len(stripped) < MIN_SECRET_LENGTH: + return False + try: + float(stripped) + except ValueError: + return True + return False + + +def _redact_assignment(match: re.Match[str]) -> str: + value = match.group("value") + if not is_sensitive_key(match.group("key")) or not could_hold_secret(value): + return match.group(0) + # Preserve the original quoting so screened JSON stays parsable JSON. + quoted = value.strip()[:1] in {'"', "'"} + replacement = f'"{REDACTED}"' if quoted else REDACTED + key_quote = match.group("quote") + return f"{key_quote}{match.group('key')}{key_quote}{match.group('gap')}{replacement}" + + +def _redact_entropy(match: re.Match[str]) -> str: + return REDACTED if _is_high_entropy(match.group(0)) else match.group(0) + + +def redact(text: str) -> str: + """Return ``text`` with credential-shaped substrings replaced.""" + if not text: + return text + for pattern in _PATTERNS: + text = pattern.sub(REDACTED, text) + text = _ASSIGNMENT_RE.sub(_redact_assignment, text) + return _ENTROPY_RE.sub(_redact_entropy, text) + + +def bound(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: + """Truncate ``text`` to a UTF-8 byte budget, reporting whether it was cut.""" + if max_bytes <= 0: + return ("", bool(text)) + encoded = text.encode("utf-8", "replace") + if len(encoded) <= max_bytes: + return (text, False) + kept = encoded[:max_bytes].decode("utf-8", "ignore") + return (f"{kept}\n{TRUNCATION_NOTE}", True) + + +def safe_text(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: + """Bound then redact, in that order, so a huge dump cannot outrun screening.""" + bounded, truncated = bound(text, max_bytes) + return (redact(bounded), truncated) + + +def scrub(value: Any, max_bytes: int = DEFAULT_MAX_BYTES) -> Any: + """Recursively bound and redact a JSON-shaped value. + + Mapping keys are preserved (names are evidence); values under a sensitive + key are dropped outright rather than pattern-screened, because a short + secret would survive screening. + """ + if isinstance(value, str): + return safe_text(value, max_bytes)[0] + if isinstance(value, dict): + return { + str(key): REDACTED + if is_sensitive_key(str(key)) and could_hold_secret(str(item)) + else scrub(item, max_bytes) + for key, item in value.items() + } + if isinstance(value, (list, tuple)): + return [scrub(item, max_bytes) for item in value] + return value + + +def contains_credential_shape(text: str) -> bool: + """Report whether screening would rewrite ``text``. + + Used by the harness self-test to prove a rendered report is clean before it + is written, and by the unit suite as an assertion helper. + """ + return redact(text) != text diff --git a/scripts/ops/hermes_handoff_rules.py b/scripts/ops/hermes_handoff_rules.py new file mode 100644 index 00000000..cf9b5b97 --- /dev/null +++ b/scripts/ops/hermes_handoff_rules.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Shared primitives and dispatch for handoff acceptance evaluators. + +Evaluators are registered by name so the check catalog stays declarative: an +entry says *what* it asserts, not how to plumb it. The dispatch here is the one +place that turns an unexpected exception into ``NOT_RUN`` — a harness bug must +never be indistinguishable from a passing release. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_model import NOT_RUN, CheckSpec + +DENIAL_MARKERS = ( + "forbidden", + "is not allowed", + "unauthorized", + "permission denied", + "cannot get", + "cannot list", + "cannot create", + "cannot delete", + "cannot patch", + "not permitted", + "no such route", + "404 not found", + "403", +) + + +@dataclass +class Evaluation: + """A classification plus the small facts that justify it.""" + + status: str + reason: str + evidence: dict[str, Any] = field(default_factory=dict) + + +Evaluator = Callable[[CheckSpec, dict[str, Outcome]], Evaluation] +EVALUATORS: dict[str, Evaluator] = {} + + +def evaluator(name: str) -> Callable[[Evaluator], Evaluator]: + """Register an evaluator under the rule name catalog entries use.""" + + def register(function: Evaluator) -> Evaluator: + EVALUATORS[name] = function + return function + + return register + + +def missing_steps(spec: CheckSpec, outcomes: Mapping[str, Outcome]) -> Evaluation | None: + """Return NOT_RUN when a required step never produced usable evidence.""" + for step in spec.steps: + if step.optional: + continue + outcome = outcomes.get(step.key) + if outcome is None: + return Evaluation(NOT_RUN, f"step {step.key} was never executed") + if not outcome.ran: + return Evaluation(NOT_RUN, f"step {step.key} did not run: {outcome.error}") + return None + + +def looks_denied(outcome: Outcome) -> bool: + """Report whether a real request was refused by policy rather than failing.""" + if outcome.ok: + return False + haystack = outcome.combined.lower() + return any(marker in haystack for marker in DENIAL_MARKERS) + + +def step_keys(spec: CheckSpec, kind: str) -> tuple[str, ...]: + """Return the keys of every step of one kind.""" + return tuple(step.key for step in spec.steps if step.kind == kind) + + +def is_list(value: Any) -> bool: + """Report whether a value is a JSON array rather than a string.""" + return isinstance(value, Sequence) and not isinstance(value, (str, bytes)) + + +def dotted(payload: Any, path: str) -> Any: + """Return a value addressed by a dotted path. + + ``a.b[0].c`` indexes; ``a[].b`` maps the remainder of the path over a list, + which is how the catalog records one field from every pull request or every + Flux object without a bespoke parser per check. + """ + current = payload + segments = [segment for segment in path.split(".") if segment] + for position, raw in enumerate(segments): + name, bracket, indexes = raw.partition("[") + if name: + if not isinstance(current, Mapping) or name not in current: + raise KeyError(path) + current = current[name] + if not bracket: + continue + for chunk in indexes.rstrip("]").split("]["): + if not is_list(current): + raise KeyError(path) + if chunk == "": + remainder = ".".join(segments[position + 1 :]) + return [dotted(item, remainder) if remainder else item for item in current] + offset = int(chunk) + if offset >= len(current): + raise KeyError(path) + current = current[offset] + return current + + +def parse_json(outcome: Outcome) -> Any: + """Parse a step's stdout as JSON, raising ``ValueError`` when it is not. + + Control characters are tolerated. Task bodies and session titles routinely + carry raw newlines and tabs, and refusing to read a report because a human + pasted a tab into a Kanban card would be a false NO_GO. + """ + text = outcome.stdout.strip() + if not text: + raise ValueError("empty output") + return json.loads(text, strict=False) + + +def lines(outcome: Outcome) -> list[str]: + """Return non-empty, stripped stdout lines.""" + return [line.strip() for line in outcome.stdout.splitlines() if line.strip()] + + +def parsed_step(spec: CheckSpec, outcomes: dict[str, Outcome]) -> tuple[Any, str] | Evaluation: + """Return the parsed payload of ``expect['step']`` or the NOT_RUN to report.""" + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + try: + return (parse_json(outcome), key) + except ValueError as exc: + return Evaluation(NOT_RUN, f"step {key} did not return JSON: {exc}") + + +def evaluate(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Dispatch to the named evaluator, converting any surprise into NOT_RUN.""" + rule = EVALUATORS.get(spec.rule) + if rule is None: + return Evaluation(NOT_RUN, f"unknown evaluator: {spec.rule}") + try: + return rule(spec, outcomes) + except Exception as exc: # noqa: BLE001 - a harness bug must never read as PASS + return Evaluation(NOT_RUN, f"evaluator {spec.rule} raised {type(exc).__name__}: {exc}") diff --git a/scripts/ops/hermes_handoff_run.py b/scripts/ops/hermes_handoff_run.py new file mode 100644 index 00000000..cdfaf3c2 --- /dev/null +++ b/scripts/ops/hermes_handoff_run.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 +"""Orchestration for the Hermes full-handoff acceptance harness. + +Three things happen before any check runs, and each of them can stop the run on +its own: + +* the catalog is validated structurally — every rule exists, every deny check + carries a real attempt, no operator-side step can impersonate; +* each vantage is resolved and asked who it is; and +* the operator and in-pod identities are compared. If they are the same + principal there is only one vantage, the dual-vantage evidence would be a + restatement rather than a corroboration, and the run is a NO_GO. + +Everything after that is uniform: run a check's steps against its vantage, hand +the outcomes to the named evaluator, and record the classification. A vantage +that could not be resolved yields ``NOT_RUN`` for every check that needs it, +which is exactly the fail-closed answer. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping, Sequence + +from hermes_handoff_catalog import ( + CHAT, + NODE, + OPERATOR, + SELF, + SWITCHYARD, + VANTAGE_NAMES, + Targets, +) +from hermes_handoff_checks_access import access_checks +from hermes_handoff_checks_delivery import delivery_checks +from hermes_handoff_checks_platform import platform_checks +from hermes_handoff_checks_workers import worker_checks +from hermes_handoff_evaluators import EVALUATORS, evaluate +from hermes_handoff_exec import Outcome, Runner, Vantage, operator_vantage, pod_vantage +from hermes_handoff_model import ( + ATTEMPT, + EPHEMERAL, + NOT_RUN, + STEP_KINDS, + CheckResult, + CheckSpec, + Report, + VantageRecord, + utc_now, +) +from hermes_handoff_policy import IMPERSONATION_ARGS + +UNRECORDED = "[output not recorded]" + + +def build_catalog(targets: Targets) -> list[CheckSpec]: + """Return the whole acceptance catalog for a set of targets.""" + return [ + *platform_checks(targets), + *access_checks(targets), + *delivery_checks(targets), + *worker_checks(targets), + ] + + +def validate_catalog(specs: Sequence[CheckSpec]) -> list[str]: + """Return the structural problems that make a catalog unsafe to run.""" + problems: list[str] = [] + seen: set[str] = set() + for spec in specs: + if spec.id in seen: + problems.append(f"{spec.id}: duplicate check id") + seen.add(spec.id) + if spec.rule not in EVALUATORS and spec.scope != EPHEMERAL: + problems.append(f"{spec.id}: unknown rule {spec.rule!r}") + for step in spec.steps: + if step.vantage not in VANTAGE_NAMES: + problems.append(f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}") + if step.kind not in STEP_KINDS: + problems.append(f"{spec.id}: step {step.key} has unknown kind {step.kind!r}") + if step.vantage == OPERATOR and set(step.argv) & set(IMPERSONATION_ARGS): + problems.append(f"{spec.id}: step {step.key} impersonates from the operator vantage") + if spec.rule == "denied" and not any(step.kind == ATTEMPT for step in spec.steps): + problems.append(f"{spec.id}: a deny check needs a real attempt, not only a review") + return problems + + +def resolve_identity(runner: Runner, vantage: Vantage) -> tuple[str, str]: + """Return the authenticated username of a vantage and any failure detail.""" + outcome = runner.run(("kubectl", "auth", "whoami", "-o", "json"), vantage) + if not outcome.ok: + return ("", outcome.error or outcome.combined[:200]) + try: + payload = json.loads(outcome.stdout) + except ValueError as exc: + return ("", f"unparsable identity: {exc}") + username = ((payload.get("status") or {}).get("userInfo") or {}).get("username") + if not isinstance(username, str) or not username: + return ("", "identity response carried no username") + return (username, "") + + +def _first_pod(runner: Runner, operator: Vantage, namespace: str, selector: str) -> tuple[str, str]: + outcome = runner.run( + ( + "kubectl", + "--namespace", + namespace, + "get", + "pods", + "--selector", + selector, + "--field-selector", + "status.phase=Running", + "-o", + "name", + ), + operator, + ) + if not outcome.ok: + return ("", outcome.error or outcome.combined[:200]) + names = [line.strip().removeprefix("pod/") for line in outcome.stdout.splitlines() if line.strip()] + if not names: + return ("", f"no running pod matches {selector}") + return (sorted(names)[0], "") + + +def resolve_vantages( + runner: Runner, targets: Targets, kubeconfig: str | None = None, context: str | None = None +) -> tuple[dict[str, Vantage], list[VantageRecord]]: + """Resolve every vantage the catalog names, recording what each one is.""" + operator = operator_vantage(kubeconfig, context) + username, detail = resolve_identity(runner, operator) + records = [ + VantageRecord( + name=OPERATOR, + description=operator.description or "external read-only operator", + identity=username, + available=bool(username), + detail=detail, + ) + ] + vantages: dict[str, Vantage] = {OPERATOR: operator} + if not username: + return (vantages, records) + + for name, selector, container, description in ( + (SELF, f"app={targets.agent_deployment}", targets.agent_container, "the Hermes agent itself"), + ( + SWITCHYARD, + f"app={targets.switchyard_deployment}", + targets.switchyard_container, + "the Switchyard routing evidence log", + ), + (NODE, f"app={targets.node_daemonset}", targets.node_container, "a node hardening probe"), + ): + pod, failure = _first_pod(runner, operator, targets.namespace, selector) + if not pod: + records.append( + VantageRecord(name=name, description=description, available=False, detail=failure) + ) + continue + vantage = pod_vantage(targets.namespace, pod, container, operator) + identity, identity_detail = resolve_identity(runner, vantage) if name == SELF else ("", "") + records.append( + VantageRecord( + name=name, + description=f"{description} ({targets.namespace}/{pod})", + identity=identity, + available=True, + detail=identity_detail, + ) + ) + vantages[name] = vantage + + chat_pod = f"{targets.chat_statefulset}-{targets.chat_ordinal}" + present = runner.run( + ("kubectl", "--namespace", targets.namespace, "get", f"pod/{chat_pod}", "-o", "name"), + operator, + ) + if present.ok and present.stdout.strip(): + vantages[CHAT] = pod_vantage(targets.namespace, chat_pod, targets.chat_container, operator) + records.append( + VantageRecord( + name=CHAT, + description=f"the chat surface ({targets.namespace}/{chat_pod})", + available=True, + ) + ) + else: + records.append( + VantageRecord( + name=CHAT, + description=f"the chat surface ({targets.namespace}/{chat_pod})", + available=False, + detail=present.error or present.combined[:200] or "chat tenant pod not found", + ) + ) + return (vantages, records) + + +def vantage_problems(records: Iterable[VantageRecord]) -> list[str]: + """Return the identity problems that invalidate dual-vantage evidence.""" + by_name = {record.name: record for record in records} + operator = by_name.get(OPERATOR) + hermes = by_name.get(SELF) + problems: list[str] = [] + if operator is None or not operator.available: + problems.append("the operator vantage is unavailable; no evidence can be collected") + return problems + if hermes is None or not hermes.available: + problems.append("the in-pod Hermes vantage is unavailable; self-probe evidence is missing") + return problems + if not hermes.identity: + problems.append("the in-pod vantage did not report an identity") + elif operator.identity == hermes.identity: + problems.append( + "the operator and in-pod vantages authenticate as the same principal " + f"({operator.identity}); run the harness with a separate operator kubeconfig" + ) + return problems + + +def run_check( + runner: Runner, spec: CheckSpec, vantages: Mapping[str, Vantage] +) -> CheckResult: + """Run one check's steps and classify the result.""" + outcomes: dict[str, Outcome] = {} + recorded: list[Outcome] = [] + for step in spec.steps: + vantage = vantages.get(step.vantage) + if vantage is None: + outcome = Outcome( + argv=step.argv, vantage=step.vantage, error=f"vantage {step.vantage} is unavailable" + ) + else: + outcome = runner.run(step.argv, vantage, step.max_bytes) + outcomes[step.key] = outcome + recorded.append( + outcome + if step.record + else Outcome( + argv=outcome.argv, + vantage=outcome.vantage, + returncode=outcome.returncode, + stdout=UNRECORDED if outcome.stdout else "", + stderr=UNRECORDED if outcome.stderr else "", + truncated=outcome.truncated, + duration_ms=outcome.duration_ms, + error=outcome.error, + ) + ) + evaluation = evaluate(spec, outcomes) + return CheckResult( + spec=spec, + status=evaluation.status, + reason=evaluation.reason, + evidence=evaluation.evidence, + outcomes=recorded, + ) + + +def run_catalog( + runner: Runner, specs: Sequence[CheckSpec], vantages: Mapping[str, Vantage] +) -> list[CheckResult]: + """Run every check in the catalog. + + Nothing is skipped once the deadline passes: the runner returns a + deadline outcome for each remaining command, which classifies as + ``NOT_RUN``, so a truncated sweep is visible in the report rather than + silently shorter. + """ + return [run_check(runner, spec, vantages) for spec in specs] + + +def build_report( + runner: Runner, + targets: Targets, + specs: Sequence[CheckSpec], + vantages: Mapping[str, Vantage], + records: Sequence[VantageRecord], + mode: str, + started_at: str, + extra_results: Sequence[CheckResult] = (), +) -> Report: + """Run the catalog and assemble the report, including structural problems.""" + problems = validate_catalog(specs) + vantage_problems(records) + report = Report( + mode=mode, + started_at=started_at, + baseline={ + "namespace": targets.namespace, + "baseline_commit": targets.baseline_commit, + "dependency_pull_requests": list(targets.dependency_pull_requests), + "expected_suspensions": list(targets.expected_suspensions), + "node_count": targets.node_count, + "pool_replicas": targets.pool_replicas, + }, + vantages=list(records), + harness_errors=problems, + ) + armed_ids = {result.spec.id for result in extra_results} + report.results = [ + *run_catalog(runner, [spec for spec in specs if spec.id not in armed_ids], vantages), + *extra_results, + ] + report.results.sort(key=lambda result: result.spec.id) + if runner.remaining_seconds <= 0: + report.harness_errors.append("the run deadline expired before every check completed") + report.finished_at = utc_now() + return report + + +def unavailable_report(mode: str, started_at: str, reason: str, targets: Targets) -> Report: + """Return a NO_GO report for a run that could not start at all.""" + return Report( + mode=mode, + started_at=started_at, + finished_at=utc_now(), + baseline={"namespace": targets.namespace}, + harness_errors=[reason], + results=[ + CheckResult( + spec=CheckSpec( + id="harness.startup", + title="The harness could start and resolve its vantages", + group="harness", + rule="allowed", + ), + status=NOT_RUN, + reason=reason, + ) + ], + ) diff --git a/testing/tests/test_hermes_handoff_acceptance.py b/testing/tests/test_hermes_handoff_acceptance.py new file mode 100644 index 00000000..87c8adbe --- /dev/null +++ b/testing/tests/test_hermes_handoff_acceptance.py @@ -0,0 +1,395 @@ +"""End-to-end contracts for the Hermes full-handoff acceptance harness CLI.""" + +from __future__ import annotations + +import json +import subprocess + +import pytest + +from testing.tests.test_hermes_handoff_support import FakeClock, load_handoff_module + +acceptance = load_handoff_module("hermes_handoff_acceptance") +catalog = load_handoff_module("hermes_handoff_catalog") +exec_module = load_handoff_module("hermes_handoff_exec") +model = load_handoff_module("hermes_handoff_model") +policy = load_handoff_module("hermes_handoff_policy") +harness_run = load_handoff_module("hermes_handoff_run") + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + + +class ScriptedCluster: + """A ``subprocess.run`` stand-in that answers by command shape. + + Real runs issue a couple of hundred commands; scripting them positionally + would make every test a brittle ordering puzzle. Matching on the command + line keeps each test about the behaviour it is checking. + """ + + def __init__(self, rules: list[tuple[str, object]], default=None) -> None: + self.rules = rules + self.default = default or completed(stderr="not found", returncode=1) + self.calls: list[list[str]] = [] + + def __call__(self, argv, **_kwargs): + self.calls.append(list(argv)) + line = " ".join(argv) + for fragment, response in self.rules: + if fragment in line: + return response + return self.default + + +def whoami(username: str): + return completed(stdout=json.dumps({"status": {"userInfo": {"username": username}}})) + + +HEALTHY_CLUSTER = [ + ("auth whoami", whoami("kubernetes-admin")), + ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), + ("get pods --selector", completed(stdout="pod/other-1\n")), + ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")), +] + + +def run_cli(monkeypatch, argv, rules): + cluster = ScriptedCluster(rules) + clock = FakeClock() + + def build_runner(**kwargs): + return exec_module.Runner(clock=clock, spawn=cluster, environ={}, **kwargs) + + monkeypatch.setattr(acceptance, "Runner", build_runner) + return (acceptance.main(argv), cluster) + + +def read_report(path): + return json.loads(path.read_text(encoding="utf-8")) + + +def test_the_parser_exposes_the_documented_controls() -> None: + arguments = acceptance.build_parser().parse_args( + [ + "--namespace", + "hermes", + "--context", + "atlas-operator", + "--dependency-pr", + "14", + "--dependency-pr", + "15", + "--expected-suspension", + "flux-system/parked", + "--node-count", + "21", + "--pool-worker-env", + "HERMES_POOL_LEASE_SECONDS", + "--expect-telegram-sessions", + ] + ) + + targets = acceptance.targets_from_args(arguments, None) + + assert targets.dependency_pull_requests == (14, 15) + assert targets.expected_suspensions == ("flux-system/parked",) + assert targets.node_count == 21 + assert targets.pool_worker_env == ("HERMES_POOL_LEASE_SECONDS",) + assert targets.expect_telegram_sessions is True + + +def test_defaults_fall_back_to_the_shipped_targets() -> None: + arguments = acceptance.build_parser().parse_args([]) + targets = acceptance.targets_from_args(arguments, None) + + assert targets.dependency_pull_requests == catalog.Targets.dependency_pull_requests + assert targets.expected_suspensions == () + assert targets.repo == catalog.Targets.repo + + +def test_a_default_run_is_read_only_and_never_pushes_or_posts(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + status, cluster = run_cli( + monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert report["mode"] == policy.READ_ONLY + assert not any("push" in call for call in cluster.calls) + assert not any("POST" in call for call in cluster.calls) + assert not any("DELETE" in call for call in cluster.calls) + writes = [ + call + for call in cluster.calls + if {"patch", "create", "delete", "apply", "replace"} & set(call) and "can-i" not in call + ] + assert writes + assert all("--dry-run=server" in call for call in writes) + + +def test_mutating_checks_report_not_run_until_the_mode_is_armed(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) + + ephemeral = { + check["id"]: check + for check in read_report(output)["checks"] + if check["group"] == "ephemeral" + } + + assert len(ephemeral) == 4 + for check in ephemeral.values(): + assert check["status"] == model.NOT_RUN + assert check["mandatory"] is False + assert check["scope"] == model.EPHEMERAL + + +def test_a_zero_state_cluster_is_no_go_with_every_check_accounted_for( + monkeypatch, tmp_path +) -> None: + """Nothing observed must never read as nothing wrong.""" + output = tmp_path / "report.json" + status, _ = run_cli( + monkeypatch, + ["--output", str(output), "--json-only"], + [("auth whoami", whoami("kubernetes-admin"))], + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert report["decision"] == model.NO_GO + assert report["counts"][model.PASS] == 0 + assert sum(report["counts"].values()) == len(report["checks"]) + assert any("in-pod Hermes vantage is unavailable" in error for error in report["harness_errors"]) + + +def test_partial_failure_is_reported_per_check_rather_than_aborting( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + rules = [ + *HEALTHY_CLUSTER, + ("auth can-i", completed(stdout="no\n", returncode=1)), + ("get namespaces", completed(stdout="namespace/hermes\n")), + ("merge-base", completed()), + ] + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) + report = read_report(output) + + statuses = {check["status"] for check in report["checks"]} + + assert status == acceptance.EXIT_NO_GO + assert model.PASS in statuses and model.NOT_RUN in statuses + assert report["blocking"] + + +def test_a_vantage_collision_is_a_no_go_even_with_clean_checks(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + rules = [ + ("exec --namespace hermes hermes-agent-1", whoami("system:serviceaccount:hermes:hermes-agent")), + ("auth whoami", whoami("system:serviceaccount:hermes:hermes-agent")), + ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), + ("get pods --selector", completed(stdout="pod/other-1\n")), + ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")), + ] + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert any("same principal" in error for error in report["harness_errors"]) + + +def test_an_exhausted_deadline_leaves_the_remaining_checks_not_run(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + status, _ = run_cli( + monkeypatch, ["--output", str(output), "--json-only", "--deadline", "0"], HEALTHY_CLUSTER + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert report["counts"][model.NOT_RUN] > 0 + assert any("deadline expired" in error for error in report["harness_errors"]) + + +def test_arming_with_a_wrong_confirmation_stops_before_anything_runs( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + status, cluster = run_cli( + monkeypatch, + [ + "--output", + str(output), + "--json-only", + "--arm-ephemeral-push", + "--confirm", + "please", + "--ephemeral-token", + "acceptance-20260817a", + ], + HEALTHY_CLUSTER, + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert "arming refused" in report["harness_errors"][0] + assert not any("push" in call for call in cluster.calls) + + +def test_arming_against_a_protected_push_target_is_refused(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + status, cluster = run_cli( + monkeypatch, + [ + "--output", + str(output), + "--json-only", + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "main", + ], + HEALTHY_CLUSTER, + ) + + assert status == acceptance.EXIT_NO_GO + assert "arming refused" in read_report(output)["harness_errors"][0] + assert not any("push" in call for call in cluster.calls) + + +def test_an_armed_run_pushes_one_ephemeral_ref_and_verifies_its_removal( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + rules = [ + ("git ls-remote origin refs/heads/ephemeral", completed(stdout="")), + ("git push", completed()), + ("POST /api/v1/repos/atlas/titan-iac/pulls", completed(stdout=json.dumps({"number": 9}))), + ("PATCH", completed()), + ("DELETE", completed()), + ("GET /api/v1/repos/atlas/titan-iac/pulls/9", completed(stdout='{"state": "closed"}')), + *HEALTHY_CLUSTER, + ] + status, cluster = run_cli( + monkeypatch, + [ + "--output", + str(output), + "--json-only", + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817a", + ], + rules, + ) + report = read_report(output) + ephemeral = {check["id"]: check for check in report["checks"] if check["group"] == "ephemeral"} + + assert report["mode"] == policy.ARMED + assert status == acceptance.EXIT_NO_GO # the rest of the cluster is still unproven + assert ephemeral["ephemeral.feature-branch-push"]["status"] == model.PASS + assert ephemeral["ephemeral.draft-pull-request"]["status"] == model.PASS + assert ephemeral["ephemeral.cleanup-verified"]["status"] == model.PASS + assert ephemeral["ephemeral.protected-branch-refusal"]["status"] == model.PASS + + pushes = [call for call in cluster.calls if "push" in call] + assert len(pushes) == 1 + assert pushes[0][-1].startswith("HEAD:refs/heads/ephemeral/hermes-handoff-acceptance/") + assert pushes[0][0] == "kubectl" and "exec" in pushes[0] # pushed as the worker, not the operator + assert any("DELETE" in call for call in cluster.calls) + + +def test_the_report_is_written_to_stdout_when_no_output_path_is_given( + monkeypatch, capsys +) -> None: + status, _ = run_cli(monkeypatch, ["--json-only"], HEALTHY_CLUSTER) + payload = json.loads(capsys.readouterr().out) + + assert status == acceptance.EXIT_NO_GO + assert payload["harness"] == model.HARNESS + + +def test_the_human_summary_accompanies_the_json_unless_suppressed( + monkeypatch, capsys, tmp_path +) -> None: + output = tmp_path / "report.json" + run_cli(monkeypatch, ["--output", str(output)], HEALTHY_CLUSTER) + captured = capsys.readouterr() + + assert model.HARNESS in captured.err + assert "vantage operator" in captured.err + assert captured.out == "" + + +def test_a_report_that_fails_its_own_screening_is_replaced_not_published(tmp_path) -> None: + """The last barrier: never write an artifact nobody re-read.""" + output = tmp_path / "report.json" + arguments = acceptance.build_parser().parse_args(["--output", str(output), "--json-only"]) + report = model.Report(mode="read-only", started_at="2026-08-17T00:00:00Z") + report.harness_errors = ["leaked ghp_ABCDEFGHIJKLMNOPQRSTUV1234"] + + monkey = report.as_dict + report.as_dict = lambda *_args, **_kwargs: { # type: ignore[method-assign] + **monkey(), + "leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234", + } + offenders = acceptance.emit(report, arguments) + payload = read_report(output) + + assert offenders == ["$.leak"] + assert payload["decision"] == model.NO_GO + assert "leak" not in payload + assert "failed its own credential screening" in payload["harness_errors"][0] + + +def test_a_screening_failure_forces_no_go_from_the_entry_point(monkeypatch, tmp_path) -> None: + """The exit status must follow the refusal to publish, not the check tally.""" + output = tmp_path / "report.json" + monkeypatch.setattr(acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"]) + monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) + monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) + + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) + + assert status == acceptance.EXIT_NO_GO + assert read_report(output)["decision"] == model.NO_GO + + +def test_no_recorded_command_output_carries_a_credential_shape(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + leaky = completed(stdout="ANTHROPIC_API_KEY=sk-ant-abcdefghijklmnopqrstuvwxyz\n") + run_cli( + monkeypatch, + ["--output", str(output), "--json-only"], + [*HEALTHY_CLUSTER, ("exec", leaky)], + ) + + assert model.unscreened_fields(read_report(output)) == [] + assert "sk-ant-abcdefghijklmnopqrstuvwxyz" not in output.read_text(encoding="utf-8") + + +@pytest.mark.parametrize("flag", ["--help"]) +def test_the_help_text_describes_the_read_only_default(flag: str, capsys) -> None: + with pytest.raises(SystemExit): + acceptance.main([flag]) + + assert "Read-only by default" in capsys.readouterr().out + + +def test_go_is_reachable_when_nothing_mandatory_is_outstanding(monkeypatch, tmp_path) -> None: + """Exercise the success exit path without pretending the cluster is clean.""" + output = tmp_path / "report.json" + monkeypatch.setattr(harness_run, "build_catalog", lambda _targets: []) + monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) + monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) + + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) + + assert status == acceptance.EXIT_GO + assert read_report(output)["decision"] == model.GO diff --git a/testing/tests/test_hermes_handoff_catalog.py b/testing/tests/test_hermes_handoff_catalog.py new file mode 100644 index 00000000..8f5e2de6 --- /dev/null +++ b/testing/tests/test_hermes_handoff_catalog.py @@ -0,0 +1,194 @@ +"""Structural contracts for the handoff acceptance catalog.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module + +catalog = load_handoff_module("hermes_handoff_catalog") +harness_run = load_handoff_module("hermes_handoff_run") +evaluators = load_handoff_module("hermes_handoff_evaluators") +model = load_handoff_module("hermes_handoff_model") +policy = load_handoff_module("hermes_handoff_policy") +exec_module = load_handoff_module("hermes_handoff_exec") + +NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc) +TARGETS = catalog.Targets(now=NOW) +SPECS = harness_run.build_catalog(TARGETS) +BY_ID = {check.id: check for check in SPECS} + +VANTAGES = { + catalog.OPERATOR: exec_module.operator_vantage(context="atlas-operator"), + catalog.SELF: exec_module.pod_vantage("hermes", "hermes-agent-1", "hermes"), + catalog.SWITCHYARD: exec_module.pod_vantage("hermes", "hermes-switchyard-1", "switchyard"), + catalog.NODE: exec_module.pod_vantage("hermes", "hermes-node-ssh-access-1", "key-reconciler"), + catalog.CHAT: exec_module.pod_vantage("hermes", "hermes-chat-tenant-0", "hermes"), +} + + +def test_the_catalog_is_structurally_valid() -> None: + assert harness_run.validate_catalog(SPECS) == [] + assert len(SPECS) >= 60 + + +def test_every_step_is_accepted_by_the_read_only_policy_from_its_vantage() -> None: + """A catalog entry that cannot run under policy is a bug found before a run.""" + for check in SPECS: + for step in check.steps: + addressed = VANTAGES[step.vantage].wrap(step.argv) + policy.check_argv(addressed, policy.READ_ONLY) + + +def test_no_step_mutates_outside_a_server_dry_run() -> None: + """`auth can-i create ...` is a read; the subcommand is what decides.""" + for check in SPECS: + for step in check.steps: + subcommand = next(policy.positionals(step.argv), "") + if subcommand in policy.KUBECTL_DRY_RUN_SUBCOMMANDS: + assert any(flag in step.argv for flag in policy.DRY_RUN_FLAGS), check.id + + +def test_no_operator_step_impersonates_and_at_least_one_in_pod_step_does() -> None: + operator_steps = [ + step for check in SPECS for step in check.steps if step.vantage == catalog.OPERATOR + ] + assert operator_steps + assert not any(set(step.argv) & set(policy.IMPERSONATION_ARGS) for step in operator_steps) + + impersonation = BY_ID["access.impersonation-is-denied"] + attempts = [step for step in impersonation.steps if step.kind == model.ATTEMPT] + assert any("--as" in step.argv for step in attempts) + + +def test_every_deny_check_pairs_a_review_with_a_live_attempt() -> None: + deny_checks = [check for check in SPECS if check.rule == "denied"] + assert len(deny_checks) >= 6 + for check in deny_checks: + assert any(step.kind == model.ATTEMPT for step in check.steps), check.id + + +def test_both_vantages_are_represented_across_the_catalog() -> None: + used = {step.vantage for check in SPECS for step in check.steps} + assert catalog.OPERATOR in used and catalog.SELF in used + assert used <= set(catalog.VANTAGE_NAMES) + + +def test_every_rule_named_by_a_check_is_registered() -> None: + for check in SPECS: + if check.scope == model.EPHEMERAL: + assert check.rule == "not_armed" + continue + assert check.rule in evaluators.EVALUATORS, check.id + + +def test_every_evidence_step_referenced_by_expect_exists() -> None: + for check in SPECS: + keys = {step.key for step in check.steps} + named = [check.expect.get("step")] + list(check.expect.get("steps", ())) + for key in filter(None, named): + assert key in keys, f"{check.id} names missing step {key}" + + +def test_the_default_catalog_reports_every_mutating_check_as_not_armed() -> None: + ephemeral = [check for check in SPECS if check.scope == model.EPHEMERAL] + assert len(ephemeral) == 4 + for check in ephemeral: + assert check.mandatory is False + assert evaluators.evaluate(check, {}).status == model.NOT_RUN + + +def test_dependency_pull_requests_each_get_their_own_merge_gate() -> None: + for number in TARGETS.dependency_pull_requests: + check = BY_ID[f"baseline.dependency-pr-{number}-merged"] + assert check.mandatory + assert check.expect["fields"] == {"merged": True, "base.ref": "main"} + + +def test_the_baseline_commit_gate_names_the_configured_commit() -> None: + check = BY_ID["baseline.origin-main-descends-merged-work"] + assert TARGETS.baseline_commit in check.steps[0].argv + assert "--is-ancestor" in check.steps[0].argv + + +def test_optional_checks_appear_only_when_a_site_declares_what_to_expect() -> None: + """A names_present over an empty expectation would pass on any workload.""" + assert "pool.assignment-safety-knobs-are-configured" not in BY_ID + assert "surfaces.chat-runs-the-telegram-topic-revision" not in BY_ID + assert "surfaces.chat-telegram-sessions-are-continuous" not in BY_ID + + configured = harness_run.build_catalog( + catalog.Targets( + now=NOW, + pool_worker_env=("HERMES_POOL_LEASE_SECONDS",), + chat_config_revision="20260816-telegram-topics", + expect_telegram_sessions=True, + ) + ) + identifiers = {check.id for check in configured} + assert "pool.assignment-safety-knobs-are-configured" in identifiers + assert "surfaces.chat-runs-the-telegram-topic-revision" in identifiers + assert "surfaces.chat-telegram-sessions-are-continuous" in identifiers + assert harness_run.validate_catalog(configured) == [] + + +def test_bulk_evidence_is_parsed_but_not_pasted_into_the_report() -> None: + routing = BY_ID["routing.provider-effort-and-fallback-evidence"] + assert routing.steps[0].record is False + assert routing.steps[0].max_bytes == TARGETS.routing_tail_bytes + + kanban = BY_ID["surfaces.kanban-activity-is-visible"] + assert kanban.steps[0].record is False + + +def test_freshness_gates_carry_the_run_clock() -> None: + for identifier in ("identity.codex-evidence-is-fresh", "identity.claude-evidence-is-fresh"): + assert BY_ID[identifier].expect["now"] == NOW + routing = BY_ID["routing.provider-effort-and-fallback-evidence"] + assert routing.expect["now"] == NOW + assert routing.expect["max_age_seconds"] == TARGETS.max_evidence_age_seconds + + +def test_check_identifiers_are_unique_and_grouped() -> None: + identifiers = [check.id for check in SPECS] + assert len(identifiers) == len(set(identifiers)) + for check in SPECS: + assert check.id.split(".")[0] + assert check.group + + +@pytest.mark.parametrize( + "identifier", + [ + "identity.no-provider-api-key-in-pod", + "identity.no-provider-api-key-in-manifest", + "forge.worker-holds-no-git-credential-variable", + "access.secrets-are-denied", + "access.service-account-tokens-are-denied", + "access.workload-mutation-is-denied", + "access.pod-exec-is-denied", + "access.required-reads-are-allowed", + "access.pod-logs-are-allowed", + "access.flux-and-helm-status-are-allowed", + "access.namespace-view-agrees-across-vantages", + "nodes.account-password-is-locked", + "nodes.hardening-covers-every-node", + "build.builder-service-account-is-tokenless", + "build.harbor-immutability-rule-is-applied", + "build.deployed-image-is-digest-pinned", + "reliability.finalization-and-replay-patches-are-deployed", + "pool.three-workers-are-ready", + "pool.workers-occupy-distinct-nodes", + "pool.workers-own-distinct-volumes", + "pool.coordinator-retains-sole-state-ownership", + "surfaces.kanban-activity-is-visible", + "surfaces.chat-telegram-topic-state-is-durable", + "gitops.kustomizations-reconcile-cleanly", + "routing.provider-effort-and-fallback-evidence", + "scopes.agent-profile-is-distinct", + ], +) +def test_the_handoff_requirements_each_have_a_mandatory_check(identifier: str) -> None: + assert BY_ID[identifier].mandatory is True diff --git a/testing/tests/test_hermes_handoff_ephemeral.py b/testing/tests/test_hermes_handoff_ephemeral.py new file mode 100644 index 00000000..09cc5d63 --- /dev/null +++ b/testing/tests/test_hermes_handoff_ephemeral.py @@ -0,0 +1,271 @@ +"""Contracts for the separately armed, self-cleaning mutation mode.""" + +from __future__ import annotations + +import json +import subprocess + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + FakeClock, + FakeSpawn, + load_handoff_module, +) + +ephemeral = load_handoff_module("hermes_handoff_ephemeral") +exec_module = load_handoff_module("hermes_handoff_exec") +policy = load_handoff_module("hermes_handoff_policy") +model = load_handoff_module("hermes_handoff_model") + +REPO = "atlas/titan-iac" +TOKEN = "acceptance-20260817a" +VANTAGE = exec_module.operator_vantage() + + +def request(**overrides) -> object: + fields = { + "repo": REPO, + "remote": "origin", + "token": TOKEN, + "confirmation": ephemeral.CONFIRMATION, + "base": "main", + } + fields.update(overrides) + return ephemeral.ArmRequest(**fields) + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + + +def armed_runner(results: list) -> tuple: + spawn = FakeSpawn(results) + runner = exec_module.Runner( + mode=policy.ARMED, clock=FakeClock(), spawn=spawn, environ={}, deadline_seconds=3600 + ) + return (runner, spawn) + + +def by_id(results: list) -> dict: + return {result.spec.id: result for result in results} + + +def test_the_ephemeral_ref_is_derived_from_the_caller_supplied_token() -> None: + assert request().ref == f"{ephemeral.REF_PREFIX}/{TOKEN}" + + +@pytest.mark.parametrize( + "ref", ["main", "master", "refs/heads/main", "HEAD", "Production", "feature/x", ""] +) +def test_protected_and_malformed_push_targets_are_refused(ref: str) -> None: + with pytest.raises(ephemeral.ArmingError): + ephemeral.assert_push_target_allowed(ref) + + +def test_an_ephemeral_ref_is_the_only_accepted_push_target() -> None: + ephemeral.assert_push_target_allowed(f"{ephemeral.REF_PREFIX}/{TOKEN}") + + +@pytest.mark.parametrize( + ("overrides", "fragment"), + [ + ({"confirmation": "yes please"}, "confirmation phrase"), + ({"confirmation": ""}, "confirmation phrase"), + ({"repo": "atlas/other"}, "is not the expected"), + ({"token": "short"}, "ephemeral token"), + ({"token": "Has-Capitals-Here"}, "ephemeral token"), + ({"token": ""}, "ephemeral token"), + ({"base": ""}, "base branch is required"), + ], +) +def test_preflight_refuses_before_any_network_call(overrides: dict, fragment: str) -> None: + with pytest.raises(ephemeral.ArmingError) as caught: + ephemeral.preflight(request(**overrides), REPO) + assert fragment in str(caught.value) + + +def test_preflight_accepts_a_correctly_armed_request() -> None: + ephemeral.preflight(request(), REPO) + + +@pytest.mark.parametrize( + ("raw", "expected"), [("refs/heads/Main", "main"), (" main ", "main"), ("/x/", "x")] +) +def test_branch_names_are_normalised_before_comparison(raw: str, expected: str) -> None: + assert ephemeral.normalise_branch(raw) == expected + + +def test_the_protected_branch_guard_is_exercised_on_every_armed_run() -> None: + """A guard checked once is a guard nobody notices regressing.""" + runner, _ = armed_runner([completed(stdout="", returncode=2)]) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + guard = results["ephemeral.protected-branch-refusal"] + + assert guard.status == model.PASS + assert guard.evidence["accepted"] == [] + assert "main" in guard.evidence["refused"] + + +def test_a_broken_protected_branch_guard_fails_its_own_self_test(monkeypatch) -> None: + """If the guard ever stops refusing, the armed run says so instead of pushing.""" + monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None) + runner, _ = armed_runner([completed(stdout="", returncode=2)]) + + guard = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[ + "ephemeral.protected-branch-refusal" + ] + + assert guard.status == model.FAIL + assert "main" in guard.evidence["accepted"] + assert "the guard accepted" in guard.reason + + +def test_a_full_armed_run_pushes_opens_a_draft_and_removes_both() -> None: + runner, spawn = armed_runner( + [ + completed(stdout=""), # ls-remote: ref unused + completed(stdout=""), # push + completed(stdout=json.dumps({"number": 42, "draft": True})), # create draft + completed(stdout=""), # close + completed(stdout=""), # delete branch + completed(stdout=""), # ls-remote: gone + completed(stdout=json.dumps({"number": 42, "state": "closed"})), + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.feature-branch-push"].status == model.PASS + assert results["ephemeral.draft-pull-request"].status == model.PASS + assert results["ephemeral.draft-pull-request"].evidence["pull_request"] == 42 + assert results["ephemeral.cleanup-verified"].status == model.PASS + assert all(result.spec.mandatory for result in results.values()) + assert all(result.spec.scope == model.EPHEMERAL for result in results.values()) + + pushed = [call for call in spawn.calls if call["argv"][:2] == ["git", "push"]] + assert pushed[0]["argv"][-1] == f"HEAD:refs/heads/{request().ref}" + assert "--force" not in pushed[0]["argv"] + + titles = [arg for call in spawn.calls for arg in call["argv"] if arg.startswith("title=")] + assert titles[0].startswith(f"title={ephemeral.DRAFT_TITLE_PREFIX}") + + +def test_an_existing_ref_stops_the_run_before_it_writes() -> None: + runner, spawn = armed_runner([completed(stdout="abc123\trefs/heads/x\n")]) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert "already exists" in results["ephemeral.feature-branch-push"].reason + assert "ephemeral.draft-pull-request" not in results + assert len(spawn.calls) == 1 + + +def test_an_unverifiable_ref_check_is_not_run_rather_than_assumed_free() -> None: + runner, _ = armed_runner([completed(stderr="network down", returncode=128)]) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.feature-branch-push"].status == model.NOT_RUN + + +def test_a_failed_push_still_reports_cleanup_and_skips_the_draft() -> None: + runner, _ = armed_runner( + [ + completed(stdout=""), + completed(stderr="denied", returncode=1), + completed(stdout=""), # delete branch + completed(stdout=""), # ls-remote: gone + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert results["ephemeral.draft-pull-request"].status == model.NOT_RUN + assert results["ephemeral.cleanup-verified"].status == model.PASS + + +def test_a_draft_response_without_a_number_is_a_failure() -> None: + runner, _ = armed_runner( + [ + completed(stdout=""), + completed(stdout=""), + completed(stdout="{}"), + completed(stdout=""), + completed(stdout=""), + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.draft-pull-request"].status == model.FAIL + + +def test_a_branch_that_survives_cleanup_fails_the_cleanup_check() -> None: + runner, _ = armed_runner( + [ + completed(stdout=""), + completed(stdout=""), + completed(stdout=json.dumps({"number": 7})), + completed(stderr="cannot close", returncode=1), + completed(stderr="cannot delete", returncode=1), + completed(stdout="abc123\trefs/heads/x\n"), + completed(stdout=json.dumps({"number": 7, "state": "open"})), + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + cleanup = results["ephemeral.cleanup-verified"] + + assert cleanup.status == model.FAIL + assert "could not be closed" in cleanup.reason + assert "still exists" in cleanup.reason + assert "still open" in cleanup.reason + + +def test_cleanup_that_cannot_be_verified_is_a_failure_not_a_pass() -> None: + runner, _ = armed_runner( + [ + completed(stdout=""), + completed(stdout=""), + completed(stdout=json.dumps({"number": 7})), + completed(stdout=""), + completed(stdout=""), + subprocess.TimeoutExpired(cmd="git", timeout=1.0), + completed(stderr="gone", returncode=1), + ] + ) + + cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))["ephemeral.cleanup-verified"] + + assert cleanup.status == model.FAIL + assert "could not verify removal" in cleanup.reason + assert "could not confirm pull request #7 is closed" in cleanup.reason + + +def test_the_armed_results_replace_the_default_not_run_placeholders() -> None: + runner, _ = armed_runner([completed(stdout="abc\trefs/heads/x\n")]) + identifiers = {result.spec.id for result in ephemeral.run_armed(runner, VANTAGE, request())} + + assert identifiers <= { + "ephemeral.protected-branch-refusal", + "ephemeral.feature-branch-push", + "ephemeral.draft-pull-request", + "ephemeral.cleanup-verified", + } + + +def test_a_read_only_runner_cannot_execute_the_armed_flow() -> None: + """The mode, not the caller, is what opens the mutations.""" + spawn = FakeSpawn([completed(stdout=""), completed(stdout=""), completed(stdout="")]) + runner = exec_module.Runner(clock=FakeClock(), spawn=spawn, environ={}) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + push = results["ephemeral.feature-branch-push"] + + assert push.status == model.FAIL + assert "policy" in push.reason + assert not any(call["argv"][:2] == ["git", "push"] for call in spawn.calls) diff --git a/testing/tests/test_hermes_handoff_evaluators.py b/testing/tests/test_hermes_handoff_evaluators.py new file mode 100644 index 00000000..2e340c7b --- /dev/null +++ b/testing/tests/test_hermes_handoff_evaluators.py @@ -0,0 +1,201 @@ +"""Contracts for the text-shaped handoff acceptance evaluators.""" + +from __future__ import annotations + +from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step + +evaluators = load_handoff_module("hermes_handoff_evaluators") +model = load_handoff_module("hermes_handoff_model") + +REVIEW = model.REVIEW +ATTEMPT = model.ATTEMPT +FORBIDDEN = outcome(stderr="Error from server (Forbidden): pods is forbidden", returncode=1) + + +def deny_spec() -> object: + return spec( + "denied", + steps=(step("review", kind=REVIEW), step("attempt", kind=ATTEMPT)), + ) + + +def evaluate(built, outcomes): + return evaluators.evaluate(built, outcomes) + + +def test_a_deny_check_passes_only_on_a_review_no_and_a_live_refusal() -> None: + evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN}) + + assert evaluation.status == model.PASS + assert evaluation.evidence == {"review": "no", "attempt": "refused"} + + +def test_a_deny_check_fails_when_the_review_says_the_authority_exists() -> None: + evaluation = evaluate(deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN}) + + assert evaluation.status == model.FAIL + assert "reported yes" in evaluation.reason + + +def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> None: + evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()}) + + assert evaluation.status == model.FAIL + assert "must be refused" in evaluation.reason + + +def test_a_request_that_merely_broke_is_not_evidence_of_a_refusal() -> None: + broken = outcome(stderr="connection refused", returncode=7) + evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": broken}) + + assert evaluation.status == model.NOT_RUN + assert "recognisable refusal" in evaluation.reason + + +def test_a_deny_check_without_a_live_attempt_refuses_to_classify() -> None: + """An authorization review states policy; only an attempt proves enforcement.""" + review_only = spec("denied", steps=(step("review", kind=REVIEW),)) + + evaluation = evaluate(review_only, {"review": outcome(stdout="no")}) + + assert evaluation.status == model.NOT_RUN + assert "require a real attempt" in evaluation.reason + + +def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> None: + built = spec("allowed", steps=(step("review", kind=REVIEW), step("read"))) + + assert evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status == model.PASS + + denied = evaluate(built, {"review": outcome(stdout="no"), "read": outcome()}) + assert denied.status == model.FAIL and "reported no" in denied.reason + + failed = evaluate(built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)}) + assert failed.status == model.FAIL and "step read failed" in failed.reason + + +def test_names_absent_flags_exact_names_and_substrings() -> None: + built = spec( + "names_absent", + {"step": "env", "names": ("GITEA_TOKEN",), "contains": ("API_KEY",)}, + steps=(step("env"),), + ) + + clean = evaluate(built, {"env": outcome(stdout="PATH\nHOME\n")}) + assert clean.status == model.PASS and clean.evidence["observed_count"] == 2 + + leaked = evaluate(built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")}) + assert leaked.status == model.FAIL + assert leaked.evidence["offenders"] == ["GITEA_TOKEN", "OPENAI_API_KEY"] + + unreadable = evaluate(built, {"env": outcome(returncode=1)}) + assert unreadable.status == model.NOT_RUN + + +def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> None: + built = spec( + "names_present", + {"step": "init", "names": ("patch-web-session-activity",), "contains": ("@sha256:",)}, + steps=(step("init"),), + ) + + good = outcome(stdout="patch-web-session-activity\nimage@sha256:abc\n") + assert evaluate(built, {"init": good}).status == model.PASS + + partial = evaluate(built, {"init": outcome(stdout="patch-web-session-activity\n")}) + assert partial.status == model.FAIL and "@sha256:" in partial.reason + + empty = spec("names_present", {"step": "init"}, steps=(step("init"),)) + assert evaluate(empty, {"init": outcome(stdout="")}).status == model.NOT_RUN + + +def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None: + single = spec("stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),)) + assert evaluate(single, {"s": outcome(stdout=" locked \n")}).status == model.PASS + + mismatch = evaluate(single, {"s": outcome(stdout="unlocked")}) + assert mismatch.status == model.FAIL and mismatch.evidence["observed"] == "unlocked" + + several = spec( + "stdout_matches", {"step": "s", "equals": ["600 root:root", "absent"]}, steps=(step("s"),) + ) + assert evaluate(several, {"s": outcome(stdout="absent")}).status == model.PASS + + assert evaluate(single, {"s": outcome(returncode=1)}).status == model.NOT_RUN + + +def test_distinct_count_separates_replica_count_from_actual_spread() -> None: + built = spec( + "distinct_count", {"step": "n", "minimum": 3, "total_equals": 3}, steps=(step("n"),) + ) + + spread = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-06\ntitan-07\n")}) + assert spread.status == model.PASS and spread.evidence["distinct"] == 3 + + stacked = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-05\ntitan-05\n")}) + assert stacked.status == model.FAIL and "distinct values" in stacked.reason + + extra = spec("distinct_count", {"step": "n", "minimum": 1, "total_equals": 2}, steps=(step("n"),)) + over = evaluate(extra, {"n": outcome(stdout="a\nb\nc\n")}) + assert over.status == model.FAIL and "expected exactly 2" in over.reason + + assert evaluate(built, {"n": outcome(stdout="")}).status == model.NOT_RUN + assert evaluate(built, {"n": outcome(returncode=1)}).status == model.NOT_RUN + + +def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None: + built = spec( + "vantages_agree", + {"steps": ("operator", "self")}, + steps=(step("operator", vantage="operator"), step("self")), + ) + + agree = evaluate( + built, + {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/b\nns/a\n")}, + ) + assert agree.status == model.PASS + + differ = evaluate( + built, {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")} + ) + assert differ.status == model.FAIL and "disagree" in differ.reason + + empty = evaluate(built, {"operator": outcome(stdout=""), "self": outcome(stdout="")}) + assert empty.status == model.NOT_RUN + + broken = evaluate(built, {"operator": outcome(returncode=1), "self": outcome()}) + assert broken.status == model.NOT_RUN + + +def test_steps_agree_is_the_same_rule_under_a_neutral_name() -> None: + assert evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"] + + +def test_lines_match_catches_a_blank_leading_column() -> None: + built = spec( + "lines_match", + {"step": "s", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + steps=(step("s"),), + ) + table = "Preview Src ID\n-------\nwork on task cli 2026\n" + + assert evaluate(built, {"s": outcome(stdout=table)}).status == model.PASS + + blank = "Preview Src ID\n-------\n cli 2026\n" + failed = evaluate(built, {"s": outcome(stdout=blank)}) + assert failed.status == model.FAIL and failed.evidence["offenders"] + + assert evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status == model.NOT_RUN + assert evaluate(built, {"s": outcome(returncode=1)}).status == model.NOT_RUN + + +def test_not_armed_is_the_fixed_skip_a_default_run_must_produce() -> None: + built = spec("not_armed", {"reason": "ephemeral mutation mode is not armed"}) + + evaluation = evaluate(built, {}) + + assert evaluation.status == model.NOT_RUN + assert evaluation.evidence == {"armed": False} + assert evaluation.reason == "ephemeral mutation mode is not armed" + assert evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed" diff --git a/testing/tests/test_hermes_handoff_exec.py b/testing/tests/test_hermes_handoff_exec.py new file mode 100644 index 00000000..cd8fc7d2 --- /dev/null +++ b/testing/tests/test_hermes_handoff_exec.py @@ -0,0 +1,173 @@ +"""Contracts for bounded, screened command execution in the handoff harness.""" + +from __future__ import annotations + +import subprocess + +from testing.tests.test_hermes_handoff_support import ( + FakeClock, + FakeSpawn, + load_handoff_module, +) + +runner_module = load_handoff_module("hermes_handoff_exec") +policy = load_handoff_module("hermes_handoff_policy") + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + + +def build(results: list, **kwargs) -> tuple: + clock = FakeClock() + spawn = FakeSpawn(results) + runner = runner_module.Runner(clock=clock, spawn=spawn, environ={}, **kwargs) + return (runner, spawn, clock) + + +OPERATOR = runner_module.operator_vantage() + + +def test_a_successful_command_is_captured_and_screened() -> None: + runner, spawn, _ = build([completed(stdout="password=a-long-enough-secret\n")]) + + outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + + assert outcome.ok and outcome.ran + assert "[redacted]" in outcome.stdout + assert outcome.returncode == 0 + assert spawn.calls[0]["argv"] == ["kubectl", "get", "pods"] + assert runner.commands_run == 1 + + +def test_a_failing_command_is_not_ok_but_still_ran() -> None: + runner, _, _ = build([completed(stderr="Error from server (Forbidden)", returncode=1)]) + + outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + + assert outcome.ran and not outcome.ok + assert "Forbidden" in outcome.combined + + +def test_a_policy_violation_never_reaches_a_subprocess() -> None: + runner, spawn, _ = build([]) + + outcome = runner.run(("kubectl", "delete", "pods", "--all"), OPERATOR) + + assert outcome.error and outcome.error.startswith("policy:") + assert not outcome.ran + assert spawn.calls == [] + assert runner.commands_run == 0 + + +def test_a_timeout_is_recorded_rather_than_raised() -> None: + runner, _, _ = build([subprocess.TimeoutExpired(cmd="kubectl", timeout=1.0)]) + + outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + + assert outcome.error == runner_module.TIMEOUT_ERROR + assert not outcome.ran + + +def test_a_missing_binary_is_recorded_rather_than_raised() -> None: + runner, _, _ = build([FileNotFoundError(2, "No such file or directory")]) + + outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + + assert outcome.error and outcome.error.startswith("spawn:") + + +def test_the_run_deadline_stops_further_probes_without_spawning() -> None: + runner, spawn, clock = build([completed(stdout="first")], deadline_seconds=10.0) + + assert runner.run(("kubectl", "get", "pods"), OPERATOR).ok + clock.advance(11.0) + second = runner.run(("kubectl", "get", "nodes"), OPERATOR) + + assert second.error == runner_module.DEADLINE_ERROR + assert len(spawn.calls) == 1 + assert runner.remaining_seconds < 0 + + +def test_the_command_timeout_is_clamped_by_the_remaining_deadline() -> None: + runner, spawn, clock = build([completed()], command_timeout=30.0, deadline_seconds=10.0) + clock.advance(6.0) + + runner.run(("kubectl", "get", "pods"), OPERATOR) + + assert spawn.calls[0]["timeout"] == 4.0 + + +def test_output_is_bounded_per_command_and_can_be_raised_per_call() -> None: + runner, _, _ = build([completed(stdout="y" * 200), completed(stdout="y" * 200)], max_bytes=50) + + small = runner.run(("kubectl", "get", "pods"), OPERATOR) + large = runner.run(("kubectl", "get", "pods"), OPERATOR, max_bytes=4096) + + assert small.truncated + assert not large.truncated + + +def test_the_child_environment_is_rebuilt_from_an_allowlist() -> None: + source = {"PATH": "/bin", "KUBECONFIG": "/tmp/kc", "ANTHROPIC_API_KEY": "leak", "HOME": "/root"} + environment = runner_module.build_environment(OPERATOR, source) + + assert environment == {"PATH": "/bin", "KUBECONFIG": "/tmp/kc", "HOME": "/root", "LC_ALL": "C"} + assert "ANTHROPIC_API_KEY" not in environment + + +def test_a_vantage_can_override_the_environment_it_needs() -> None: + vantage = runner_module.operator_vantage(kubeconfig="/tmp/operator.yaml") + environment = runner_module.build_environment(vantage, {"KUBECONFIG": "/tmp/other"}) + + assert environment["KUBECONFIG"] == "/tmp/operator.yaml" + assert "kubeconfig-pinned" in vantage.description + + +def test_the_operator_context_is_inserted_only_for_context_aware_tools() -> None: + vantage = runner_module.operator_vantage(context="atlas-operator") + + assert vantage.wrap(("kubectl", "get", "pods"))[:3] == ("kubectl", "--context", "atlas-operator") + assert vantage.wrap(("git", "status")) == ("git", "status") + assert vantage.wrap(()) == () + + +def test_the_pod_vantage_execs_into_the_container_and_drops_the_context() -> None: + operator = runner_module.operator_vantage(context="atlas-operator") + vantage = runner_module.pod_vantage("hermes", "hermes-agent-1", "hermes", operator) + + wrapped = vantage.wrap(("kubectl", "get", "pods")) + + assert wrapped[:3] == ("kubectl", "--context", "atlas-operator") + assert wrapped[3] == "exec" + assert wrapped[-4:] == ("--", "kubectl", "get", "pods") + assert vantage.description == "in-pod hermes/hermes-agent-1[hermes]" + policy.check_argv(wrapped) + + +def test_a_pod_vantage_without_an_operator_still_addresses_the_pod() -> None: + vantage = runner_module.pod_vantage("hermes", "pod-a", "hermes") + + assert vantage.wrap(("kubectl", "version"))[0] == "kubectl" + assert vantage.env == {} + + +def test_outcome_serialisation_screens_the_command_line() -> None: + outcome = runner_module.Outcome( + argv=("git", "clone", "https://user:a-long-enough-secret@scm.example.dev/x"), + vantage="operator", + returncode=0, + stdout="ok", + ) + + payload = outcome.as_dict() + + assert "a-long-enough-secret" not in payload["command"] + assert payload["vantage"] == "operator" + assert payload["stdout"] == "ok" + + +def test_combined_output_joins_only_the_streams_that_carry_text() -> None: + assert runner_module.Outcome(argv=(), vantage="x", stdout="a", stderr="b").combined == "a\nb" + assert runner_module.Outcome(argv=(), vantage="x", stdout="a").combined == "a" + assert runner_module.Outcome(argv=(), vantage="x").combined == "" diff --git a/testing/tests/test_hermes_handoff_json_rules.py b/testing/tests/test_hermes_handoff_json_rules.py new file mode 100644 index 00000000..9bb5c673 --- /dev/null +++ b/testing/tests/test_hermes_handoff_json_rules.py @@ -0,0 +1,279 @@ +"""Contracts for the JSON-shaped handoff acceptance evaluators.""" + +from __future__ import annotations + +import datetime as dt +import json + +from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step + +evaluators = load_handoff_module("hermes_handoff_evaluators") +model = load_handoff_module("hermes_handoff_model") + +NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc) + + +def evaluate(built, payload): + body = payload if isinstance(payload, str) else json.dumps(payload) + return evaluators.evaluate(built, {"s": outcome(stdout=body)}) + + +def one(rule: str, expect: dict): + return spec(rule, {"step": "s", **expect}, steps=(step("s"),)) + + +def test_json_field_compares_paths_against_expected_values() -> None: + built = one("json_field", {"fields": {"authenticated": True, "transport": ["a", "b"]}}) + + assert evaluate(built, {"authenticated": True, "transport": "b"}).status == model.PASS + + wrong = evaluate(built, {"authenticated": False, "transport": "c"}) + assert wrong.status == model.FAIL + assert "authenticated=False" in wrong.reason and "transport='c'" in wrong.reason + + assert evaluate(built, {"authenticated": True}).status == model.NOT_RUN + + +def test_json_field_skips_rather_than_guesses_on_unusable_output() -> None: + built = one("json_field", {"fields": {"a": 1}}) + + assert evaluate(built, "not json").status == model.NOT_RUN + failed = evaluators.evaluate(built, {"s": outcome(returncode=1)}) + assert failed.status == model.NOT_RUN + + +def test_json_numeric_enforces_inclusive_bounds_and_rejects_non_numbers() -> None: + built = one("json_numeric", {"fields": {"latency_ms": {"min": 1, "max": 1000}}}) + + assert evaluate(built, {"latency_ms": 500}).status == model.PASS + assert evaluate(built, {"latency_ms": 1}).status == model.PASS + + low = evaluate(built, {"latency_ms": 0}) + assert low.status == model.FAIL and "below minimum" in low.reason + + high = evaluate(built, {"latency_ms": 5000}) + assert high.status == model.FAIL and "above maximum" in high.reason + + assert evaluate(built, {"latency_ms": True}).status == model.NOT_RUN + assert evaluate(built, {"latency_ms": "fast"}).status == model.NOT_RUN + assert evaluate(built, {}).status == model.NOT_RUN + + +def test_json_recent_treats_stale_evidence_as_a_failure() -> None: + built = one("json_recent", {"now": NOW, "fields": {"checked_at": 3600}}) + + fresh = evaluate(built, {"checked_at": "2026-08-17T11:30:00Z"}) + assert fresh.status == model.PASS and fresh.evidence["checked_at"]["age_seconds"] == 1800 + + stale = evaluate(built, {"checked_at": "2026-08-16T00:00:00Z"}) + assert stale.status == model.FAIL and "limit 3600s" in stale.reason + + naive = evaluate(built, {"checked_at": "2026-08-17T11:30:00"}) + assert naive.status == model.PASS + + assert evaluate(built, {"checked_at": "yesterday"}).status == model.NOT_RUN + assert evaluate(built, {}).status == model.NOT_RUN + + +def test_all_of_field_selects_keys_and_reports_every_mismatch() -> None: + built = one("all_of_field", {"key_field": "number", "keys": [14, 15], "fields": {"merged": True}}) + payload = [ + {"number": 14, "merged": True}, + {"number": 15, "merged": False}, + {"number": 16, "merged": False}, + ] + + evaluation = evaluate(built, payload) + + assert evaluation.status == model.FAIL + assert "15: merged=False" in evaluation.reason + assert "16" not in evaluation.reason + assert evaluation.evidence["examined"] == [14, 15] + + +def test_all_of_field_skips_when_an_expected_key_is_absent_or_nothing_matched() -> None: + keyed = one("all_of_field", {"key_field": "number", "keys": [14], "fields": {"merged": True}}) + absent = evaluate(keyed, [{"number": 99, "merged": True}]) + assert absent.status == model.NOT_RUN and "expected keys absent" in absent.reason + + unkeyed = one("all_of_field", {"fields": {"status": ["done"]}}) + assert evaluate(unkeyed, []).status == model.NOT_RUN + assert evaluate(unkeyed, {"not": "an array"}).status == model.NOT_RUN + + +def test_all_of_field_skips_when_the_named_array_path_is_absent() -> None: + built = spec( + "all_of_field", {"step": "s", "array": "items", "fields": {"a": 1}}, steps=(step("s"),) + ) + + evaluation = evaluate(built, {"other": []}) + + assert evaluation.status == model.NOT_RUN + assert "array path is absent" in evaluation.reason + + +def test_all_of_field_treats_an_absent_field_as_a_mismatch_not_a_crash() -> None: + built = one("all_of_field", {"key_field": "name", "fields": {"suspended": False}}) + + evaluation = evaluate(built, [{"name": "a"}]) + + assert evaluation.status == model.FAIL + assert "suspended=None" in evaluation.reason + + +def test_all_of_field_honours_exemptions_and_non_empty_paths() -> None: + exempted = one( + "all_of_field", + {"key_field": "name", "exempt_keys": ["parked"], "fields": {"suspended": False}}, + ) + payload = [{"name": "parked", "suspended": True}, {"name": "live", "suspended": False}] + assert evaluate(exempted, payload).status == model.PASS + + non_empty = one("all_of_field", {"key_field": "id", "non_empty": ["title"]}) + assert evaluate(non_empty, [{"id": "a", "title": "work"}]).status == model.PASS + blank = evaluate(non_empty, [{"id": "a", "title": ""}, {"id": "b"}]) + assert blank.status == model.FAIL and "is empty" in blank.reason + + +def test_json_record_captures_named_paths_or_skips() -> None: + built = one("json_record", {"record": {"numbers": "[].number", "refs": "[].head.ref"}}) + payload = [{"number": 14, "head": {"ref": "a"}}, {"number": 15, "head": {"ref": "b"}}] + + evaluation = evaluate(built, payload) + assert evaluation.status == model.PASS + assert evaluation.evidence == {"numbers": [14, 15], "refs": ["a", "b"]} + + missing = evaluate(built, [{"number": 14}]) + assert missing.status == model.NOT_RUN and "absent" in missing.reason + + +def routing_log(*records: dict) -> str: + return "\n".join(json.dumps(record) for record in records) + + +def routing_spec(**extra): + return spec( + "routing_evidence", + { + "step": "s", + "providers": ("codex", "claude"), + "efforts": ("high", "xhigh"), + "lanes": ("route", "worker"), + "require_fallback_evidence": True, + **extra, + }, + steps=(step("s"),), + ) + + +FULL_LOG = routing_log( + {"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}, + {"ts": "2026-08-17T11:30:00Z", "model": "route/claude/opus/xhigh"}, + {"ts": "2026-08-17T11:45:00Z", "model": "worker/codex/sol/xhigh", "fallback_reason": "unavailable"}, + {"ts": "2026-08-17T11:50:00Z", "model": "qwen2.5:14b", "tier": "classifier"}, +) + + +def test_routing_evidence_summarises_lanes_providers_efforts_and_fallbacks() -> None: + evaluation = evaluate(routing_spec(), FULL_LOG) + + assert evaluation.status == model.PASS + assert evaluation.evidence["providers"] == ["claude", "codex"] + assert evaluation.evidence["efforts"] == ["high", "xhigh"] + assert evaluation.evidence["lanes"] == ["route", "worker"] + assert evaluation.evidence["fallback_reasons"] == ["unavailable"] + assert evaluation.evidence["records"] == 4 and evaluation.evidence["routed"] == 3 + + +def test_routing_evidence_names_exactly_what_the_window_did_not_show() -> None: + partial = routing_log({"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}) + + evaluation = evaluate(routing_spec(), partial) + + assert evaluation.status == model.FAIL + assert "provider:claude" in evaluation.reason + assert "effort:xhigh" in evaluation.reason + assert "lane:worker" in evaluation.reason + assert "fallback:none-observed" in evaluation.reason + + +def test_routing_evidence_tolerates_a_clipped_tail_but_not_an_empty_one() -> None: + clipped = '{"ts": "2026-08-17T11:0\n' + FULL_LOG + + assert evaluate(routing_spec(), clipped).status == model.PASS + assert evaluate(routing_spec(), "").status == model.NOT_RUN + assert evaluate(routing_spec(), "garbage\n").status == model.NOT_RUN + + +def test_routing_evidence_can_require_freshness() -> None: + fresh = evaluate(routing_spec(max_age_seconds=3600, now=NOW), FULL_LOG) + assert fresh.status == model.PASS and fresh.evidence["age_seconds"] == 600 + + stale = evaluate( + routing_spec(max_age_seconds=60, now=NOW + dt.timedelta(days=2)), FULL_LOG + ) + assert stale.status == model.FAIL and "freshness" in stale.reason + + undated = routing_log({"model": "route/codex/sol/high"}, {"model": "route/claude/opus/xhigh"}) + unknown = evaluate(routing_spec(max_age_seconds=60, now=NOW), undated) + assert unknown.status == model.NOT_RUN + + +def flux_spec(*expected: str): + return spec( + "flux_health", + {"step": "s", "expected_suspensions": expected}, + steps=(step("s"),), + ) + + +def test_flux_health_separates_parked_objects_from_broken_ones() -> None: + rows = ( + "flux-system/core\tfalse\tReady=True,Healthy=True,\n" + "flux-system/parked\ttrue\tReady=False,\n" + "flux-system/broken\tfalse\tReady=False,Healthy=False,\n" + ) + + evaluation = evaluators.evaluate(flux_spec("flux-system/parked"), {"s": outcome(stdout=rows)}) + + assert evaluation.status == model.FAIL + assert evaluation.evidence["unhealthy"] == ["flux-system/broken"] + assert evaluation.evidence["unexpected_suspensions"] == [] + assert "unhealthy: flux-system/broken" in evaluation.reason + + +def test_flux_health_flags_a_suspension_nobody_declared() -> None: + rows = "flux-system/core\tfalse\tReady=True,\nflux-system/parked\ttrue\tReady=False,\n" + + evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) + + assert evaluation.status == model.FAIL + assert evaluation.evidence["unexpected_suspensions"] == ["flux-system/parked"] + + +def test_flux_health_does_not_call_an_in_flight_reconcile_unhealthy() -> None: + """Flux drops Ready to False for the duration of a reconcile.""" + rows = ( + "flux-system/reconciling\tfalse\tReconciling=True,Ready=False,\n" + "flux-system/degraded\tfalse\tReady=False,Healthy=True,\n" + "flux-system/waiting\t\tReady=Unknown,\n" + ) + + evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) + + assert evaluation.status == model.PASS + assert evaluation.evidence["total"] == 3 + + +def test_blank_rows_in_a_projection_are_ignored_by_both_log_parsers() -> None: + padded = "\n\n" + FULL_LOG + "\n\n" + assert evaluate(routing_spec(), padded).status == model.PASS + + rows = "\n\nflux-system/core\tfalse\tReady=True,\n\n" + evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) + assert evaluation.status == model.PASS and evaluation.evidence["total"] == 1 + + +def test_flux_health_skips_on_an_empty_or_failed_projection() -> None: + assert evaluators.evaluate(flux_spec(), {"s": outcome(stdout="")}).status == model.NOT_RUN + assert evaluators.evaluate(flux_spec(), {"s": outcome(returncode=1)}).status == model.NOT_RUN diff --git a/testing/tests/test_hermes_handoff_model.py b/testing/tests/test_hermes_handoff_model.py new file mode 100644 index 00000000..37c3f9b9 --- /dev/null +++ b/testing/tests/test_hermes_handoff_model.py @@ -0,0 +1,180 @@ +"""Contracts for the handoff harness result model, verdict, and rendering.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step + +model = load_handoff_module("hermes_handoff_model") + + +def result(status: str, identifier: str = "test.check", mandatory: bool = True, group: str = "g"): + return model.CheckResult( + spec=model.CheckSpec( + id=identifier, title="t", group=group, rule="allowed", mandatory=mandatory + ), + status=status, + reason=f"{status} reason", + ) + + +def report(*results, errors: tuple[str, ...] = ()) -> model.Report: + return model.Report( + mode="read-only", + started_at="2026-08-17T00:00:00Z", + results=list(results), + harness_errors=list(errors), + ) + + +@pytest.mark.parametrize( + ("status", "mandatory", "blocking"), + [ + (model.PASS, True, False), + (model.NOT_APPLICABLE, True, False), + (model.FAIL, True, True), + (model.NOT_RUN, True, True), + (model.FAIL, False, False), + (model.NOT_RUN, False, False), + ], +) +def test_only_a_mandatory_failure_or_skip_blocks(status, mandatory, blocking) -> None: + assert result(status, mandatory=mandatory).blocking is blocking + + +def test_a_mandatory_skip_is_a_no_go_exactly_like_a_failure() -> None: + """The whole point of the four-valued classification.""" + assert report(result(model.NOT_RUN)).decision == model.NO_GO + assert report(result(model.FAIL)).decision == model.NO_GO + assert report(result(model.PASS), result(model.NOT_APPLICABLE)).decision == model.GO + + +def test_a_harness_error_is_a_no_go_even_when_every_check_passed() -> None: + clean = report(result(model.PASS), errors=("the two vantages are the same principal",)) + assert clean.decision == model.NO_GO + + +def test_a_zero_state_report_is_a_go_only_because_nothing_was_mandatory() -> None: + """An empty catalog has nothing blocking; the harness errors are what stop it.""" + assert report().decision == model.GO + assert report(errors=("no checks were built",)).decision == model.NO_GO + + +def test_blocking_results_are_ordered_failures_before_skips() -> None: + ordered = report( + result(model.NOT_RUN, "b.skip"), + result(model.FAIL, "z.fail"), + result(model.PASS, "a.pass"), + ).blocking + + assert [item.spec.id for item in ordered] == ["z.fail", "b.skip"] + + +def test_counts_and_group_summary_tally_every_status() -> None: + built = report( + result(model.PASS, "a", group="alpha"), + result(model.FAIL, "b", group="alpha"), + result(model.NOT_RUN, "c", group="beta"), + ) + + assert built.counts == { + model.PASS: 1, + model.FAIL: 1, + model.NOT_RUN: 1, + model.NOT_APPLICABLE: 0, + } + assert built.group_summary()["alpha"][model.PASS] == 1 + assert built.group_summary()["beta"][model.NOT_RUN] == 1 + + +def test_the_serialised_report_carries_the_verdict_and_its_reasons() -> None: + built = report(result(model.FAIL, "x.fail"), result(model.PASS, "y.pass")) + built.vantages = [ + model.VantageRecord(name="operator", description="d", identity="u", available=True) + ] + + payload = built.as_dict() + + assert payload["decision"] == model.NO_GO + assert payload["blocking"] == ["x.fail"] + assert payload["harness"] == model.HARNESS + assert payload["schema_version"] == model.SCHEMA_VERSION + assert payload["vantages"][0]["identity"] == "u" + assert {check["id"] for check in payload["checks"]} == {"x.fail", "y.pass"} + + +def test_check_serialisation_screens_evidence_and_can_omit_commands() -> None: + built = model.CheckResult( + spec=spec("allowed", identifier="x", steps=(step("a"),)), + status=model.PASS, + evidence={"token": "a-long-enough-secret", "count": 3}, + outcomes=[outcome(stdout="fine")], + ) + + with_commands = built.as_dict() + without = built.as_dict(include_outcomes=False) + + assert with_commands["evidence"]["token"] == "[redacted]" + assert with_commands["evidence"]["count"] == 3 + assert len(with_commands["commands"]) == 1 + assert "commands" not in without + assert with_commands["vantages"] == ["self"] + + +def test_rationale_is_carried_only_when_a_check_has_one() -> None: + without = model.CheckResult(spec=spec("allowed"), status=model.PASS).as_dict() + assert "rationale" not in without + + annotated = model.CheckSpec( + id="x", title="t", group="g", rule="allowed", rationale="because" + ) + with_reason = model.CheckResult(spec=annotated, status=model.PASS).as_dict() + assert with_reason["rationale"] == "because" + + +def test_distinct_vantages_are_reported_in_first_use_order() -> None: + built = spec( + "allowed", + steps=(step("a", vantage="operator"), step("b", vantage="self"), step("c", vantage="operator")), + ) + assert built.vantages == ("operator", "self") + + +def test_the_human_summary_names_the_verdict_vantages_and_blockers() -> None: + built = report(result(model.FAIL, "x.fail", group="alpha"), errors=("collision",)) + built.vantages = [ + model.VantageRecord(name="operator", description="d", identity="u", available=True), + model.VantageRecord(name="self", description="d", available=False), + ] + + summary = built.render_summary() + + assert summary.splitlines()[0] == f"{model.HARNESS}: {model.NO_GO}" + assert "vantage operator: available identity=u" in summary + assert "vantage self: unavailable identity=unknown" in summary + assert "harness error: collision" in summary + assert "x.fail" in summary + assert "alpha" in summary + + +def test_utc_now_renders_a_zulu_timestamp_and_accepts_an_injected_clock() -> None: + fixed = dt.datetime(2026, 8, 17, 9, 30, 15, 123456, tzinfo=dt.timezone.utc) + assert model.utc_now(lambda: fixed) == "2026-08-17T09:30:15Z" + assert model.utc_now().endswith("Z") + + +def test_unscreened_fields_walks_the_whole_payload() -> None: + offenders = model.unscreened_fields( + { + "clean": "hermes", + "nested": {"leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234"}, + "items": [1, "Authorization: Bearer abcdefghijklmnop"], + } + ) + + assert offenders == ["$.nested.leak", "$.items[1]"] + assert model.unscreened_fields({"clean": "hermes"}) == [] + assert model.unscreened_fields(None) == [] diff --git a/testing/tests/test_hermes_handoff_policy.py b/testing/tests/test_hermes_handoff_policy.py new file mode 100644 index 00000000..13fdc8b4 --- /dev/null +++ b/testing/tests/test_hermes_handoff_policy.py @@ -0,0 +1,159 @@ +"""Contracts for the fail-closed command policy of the handoff harness.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module + +policy = load_handoff_module("hermes_handoff_policy") + + +def check(*argv: str, mode: str | None = None, inner: bool = False) -> None: + policy.check_argv(argv, mode or policy.READ_ONLY, allow_impersonation=inner) + + +@pytest.mark.parametrize( + "argv", + [ + ("kubectl", "--namespace", "hermes", "get", "pods"), + ("kubectl", "auth", "can-i", "get", "secrets", "--all-namespaces"), + ("kubectl", "get", "secrets", "--all-namespaces", "-o", "name"), + ("kubectl", "get", "secrets", "--output=name"), + ("kubectl", "logs", "deploy/x", "--tail", "1"), + ("kubectl", "--context", "atlas", "get", "nodes", "-o", "json"), + ("flux", "get", "kustomizations", "--all-namespaces"), + ("helm", "list", "--all-namespaces"), + ("git", "merge-base", "--is-ancestor", "a", "b"), + ("git", "config", "--get", "remote.origin.url"), + ("hermes", "kanban", "list", "--json"), + ("hermes", "sessions", "list", "--source", "telegram"), + ("hermes", "status"), + (policy.GITEA_CLIENT, "GET", "/api/v1/user"), + ], +) +def test_read_only_commands_are_permitted(argv: tuple[str, ...]) -> None: + check(*argv) + + +@pytest.mark.parametrize( + ("argv", "fragment"), + [ + ((), "empty command"), + (("vault", "read", "kv/x"), "forbidden binary"), + (("curl", "https://example.dev"), "forbidden binary"), + (("rm", "-rf", "/"), "allowlist"), + (("kubectl", "get", "pods", "--token", "abc"), "forbidden argument"), + (("kubectl", "get", "pods", "--as=system:admin"), "forbidden argument"), + (("kubectl", "get", "secrets", "-A"), "-o name"), + (("kubectl", "describe", "secrets", "x"), "-o name"), + (("kubectl", "delete", "-n", "get", "secrets"), "requires a dry run"), + (("kubectl", "port-forward", "svc/x", "80"), "outside the read-only boundary"), + (("kubectl", "exec", "pod"), "-- command boundary"), + (("kubectl", "exec", "pod", "--"), "inner command"), + (("kubectl",), "outside the read-only boundary"), + (("flux", "reconcile", "kustomization", "x"), "outside the read-only boundary"), + (("flux",), "requires a subcommand"), + (("helm", "upgrade", "x"), "outside the read-only boundary"), + (("git", "commit", "-m", "x"), "not permitted"), + (("git", "push", "origin", "HEAD:refs/heads/x"), "not permitted"), + (("git", "config", "user.name", "x"), "may only read"), + (("hermes", "kanban", "complete", "t_1"), "outside the read-only boundary"), + (("hermes",), "requires a subcommand"), + ((policy.GITEA_CLIENT, "POST", "/api/v1/x"), "not permitted"), + ((policy.GITEA_CLIENT, "GET"), "method and a path"), + (("sh", "-c", "rm -rf /"), "frozen template"), + (("sh", "echo"), "frozen template"), + (("git", "log", "/etc/shadow"), "credential path"), + ], +) +def test_unsafe_commands_are_refused(argv: tuple[str, ...], fragment: str) -> None: + with pytest.raises(policy.PolicyError) as caught: + check(*argv) + assert fragment in str(caught.value) + + +def test_unknown_mode_is_refused() -> None: + with pytest.raises(policy.PolicyError, match="unknown mode"): + policy.check_argv(("kubectl", "get", "pods"), "whatever") + + +def test_dry_run_reaches_mutating_subcommands_without_writing() -> None: + check("kubectl", "--namespace", "hermes", "patch", "deploy/x", "--dry-run=server") + check("kubectl", "create", "configmap", "probe", "--dry-run=server") + + +def test_a_resource_named_after_a_subcommand_cannot_smuggle_a_delete() -> None: + """`-n get` must not make a delete look like a read.""" + with pytest.raises(policy.PolicyError, match="requires a dry run"): + check("kubectl", "delete", "-n", "get", "configmap", "x") + + +def test_impersonation_is_operator_forbidden_and_in_pod_permitted() -> None: + with pytest.raises(policy.PolicyError, match="forbidden argument"): + check("kubectl", "get", "namespaces", "--as", "system:admin") + check("kubectl", "get", "namespaces", "--as", "system:admin", inner=True) + + +def test_exec_validates_its_inner_command_and_tolerates_impersonation_inside() -> None: + inner = policy.shell("path_readable", path="/tmp") + check("kubectl", "exec", "--namespace", "hermes", "pod", "--", *inner) + check("kubectl", "exec", "pod", "--", "kubectl", "get", "ns", "--as", "system:admin") + with pytest.raises(policy.PolicyError, match="allowlist"): + check("kubectl", "exec", "pod", "--", "bash", "-c", "id") + + +def test_exec_of_a_shell_template_naming_a_secret_path_is_permitted() -> None: + """The path scan stops at `--`; the tail is a reviewed template, not free text.""" + inner = policy.shell("account_lock_state", account="hermes-agent", path="/host-etc/shadow") + check("kubectl", "exec", "--namespace", "hermes", "pod", "--", *inner) + + +def test_armed_mode_opens_exactly_the_mutations_it_should() -> None: + check("git", "push", "origin", "HEAD:refs/heads/x", mode=policy.ARMED) + check(policy.GITEA_CLIENT, "POST", "/api/v1/x", mode=policy.ARMED) + check(policy.GITEA_CLIENT, "DELETE", "/api/v1/x", mode=policy.ARMED) + with pytest.raises(policy.PolicyError, match="may not force"): + check("git", "push", "--force", "origin", "x", mode=policy.ARMED) + with pytest.raises(policy.PolicyError, match="may not force"): + check("git", "push", "--all", "origin", mode=policy.ARMED) + with pytest.raises(policy.PolicyError, match="not permitted"): + check(policy.GITEA_CLIENT, "PUT", "/api/v1/x", mode=policy.ARMED) + + +def test_every_frozen_template_renders_and_is_accepted() -> None: + parameters = { + "account": "hermes-agent", + "askpass": "/opt/coordinator/gitea_askpass.sh", + "fields": "state,model", + "limit": "200", + "path": "/host-etc/passwd", + "url": "https://scm.example.dev/atlas/repo.git", + } + for name in policy.SHELL_TEMPLATES: + required = policy.template_parameters(name) + argv = policy.shell(name, **{key: parameters[key] for key in required}) + policy.check_argv(argv) + assert argv[0] == "sh" + + +def test_template_rendering_rejects_unknown_names_and_unsafe_parameters() -> None: + with pytest.raises(policy.PolicyError, match="unknown shell template"): + policy.render_shell("nope") + with pytest.raises(policy.PolicyError, match="expects parameters"): + policy.render_shell("path_mode") + with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): + policy.render_shell("path_mode", path="/tmp; rm -rf /") + with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): + policy.render_shell("path_mode", path="$(id)") + + +def test_positionals_skip_flags_and_stop_at_the_boundary() -> None: + argv = ("kubectl", "--namespace", "hermes", "get", "pods", "--", "ignored") + assert list(policy.positionals(argv)) == ["get", "pods"] + assert list(policy.positionals(("kubectl", "--all-namespaces", "get"))) == ["get"] + + +def test_a_subcommand_must_precede_its_resource_arguments() -> None: + with pytest.raises(policy.PolicyError, match="must precede"): + check("kubectl", "pods", "get") diff --git a/testing/tests/test_hermes_handoff_redaction.py b/testing/tests/test_hermes_handoff_redaction.py new file mode 100644 index 00000000..a472d82b --- /dev/null +++ b/testing/tests/test_hermes_handoff_redaction.py @@ -0,0 +1,162 @@ +"""Contracts for the handoff harness credential screen and output bounding.""" + +from __future__ import annotations + +import json + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module + +redaction = load_handoff_module("hermes_handoff_redaction") + + +@pytest.mark.parametrize( + "text", + [ + "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234", + "Authorization: Bearer abcdefghijklmnop", + "cookie: session=abcdefghijklmnop", + "password=hunter2000secret", + "-----BEGIN OPENSSH PRIVATE KEY-----\nabc\n-----END OPENSSH PRIVATE KEY-----", + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI", + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0.SflKxwRJ", + "https://user:supersecretvalue@scm.example.dev/x", + "vault token hvs.CAESIJabcdefghijklmnop", + "aws AKIAIOSFODNN7EXAMPLE", + ], +) +def test_credential_shapes_are_screened(text: str) -> None: + assert redaction.contains_credential_shape(text) + assert redaction.REDACTED in redaction.redact(text) + + +@pytest.mark.parametrize( + "text", + [ + "sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7", + "ab346f55509d584e457fe26cf90be3078f7a375c", + '"automountServiceAccountToken": false', + '"key": "kubernetes.io/arch"', + "namespace: hermes", + "replicas: 3", + "checked_at: 2026-08-17T09:17:07.224400+00:00", + ], +) +def test_ordinary_evidence_survives_screening(text: str) -> None: + assert redaction.redact(text) == text + assert not redaction.contains_credential_shape(text) + + +def test_screened_json_is_still_parsable_json() -> None: + """A screen that corrupts structure would break every downstream evaluator.""" + payload = { + "spec": {"automountServiceAccountToken": True, "key": "kubernetes.io/arch"}, + "data": {"password": "a-long-enough-secret-value"}, + } + screened = redaction.redact(json.dumps(payload, indent=2)) + reparsed = json.loads(screened) + + assert reparsed["spec"]["automountServiceAccountToken"] is True + assert reparsed["spec"]["key"] == "kubernetes.io/arch" + assert reparsed["data"]["password"] == redaction.REDACTED + + +def test_high_entropy_runs_are_screened_but_hex_digests_are_not() -> None: + assert redaction.redact("Zm9vYmFyYmF6cXV4-Quux_1234567890ABCDEFghij") != redaction.REDACTED + long_mixed = "aB3" + "x9Y2z_" * 10 + assert redaction.redact(long_mixed) == redaction.REDACTED + assert redaction.redact("f" * 64) == "f" * 64 + + +@pytest.mark.parametrize( + ("key", "sensitive"), + [ + ("ANTHROPIC_API_KEY", True), + ("gitea_token", True), + ("clientSecret", True), + ("Set-Cookie", True), + ("automountServiceAccountToken", True), + ("key", False), + ("namespace", False), + ("", False), + ], +) +def test_key_sensitivity_classification(key: str, sensitive: bool) -> None: + assert redaction.is_sensitive_key(key) is sensitive + + +@pytest.mark.parametrize( + ("value", "possible"), + [ + ("true", False), + ("False", False), + ("3", False), + ("12.5", False), + ("short", False), + ("", False), + ("a-long-enough-secret", True), + ('"another-long-secret"', True), + ("1234567890123", False), + ("-12.5e9", False), + ], +) +def test_value_shape_gates_screening(value: str, possible: bool) -> None: + assert redaction.could_hold_secret(value) is possible + + +def test_bound_truncates_to_a_byte_budget_and_reports_it() -> None: + text, truncated = redaction.bound("x" * 100, max_bytes=10) + assert truncated + assert text.startswith("x" * 10) + assert redaction.TRUNCATION_NOTE in text + + text, truncated = redaction.bound("short", max_bytes=64) + assert (text, truncated) == ("short", False) + + +def test_bound_with_no_budget_keeps_nothing() -> None: + assert redaction.bound("content", max_bytes=0) == ("", True) + assert redaction.bound("", max_bytes=0) == ("", False) + + +def test_bound_never_splits_a_multibyte_character() -> None: + text, truncated = redaction.bound("é" * 10, max_bytes=5) + assert truncated + assert text.startswith("éé") + + +def test_safe_text_bounds_before_it_screens() -> None: + """Bounding first is what stops a huge dump from outrunning the screen.""" + payload = "password=" + "a" * 200 + text, truncated = redaction.safe_text(payload, max_bytes=20) + assert truncated + assert redaction.REDACTED in text + + +def test_scrub_walks_nested_structures_and_keeps_useful_types() -> None: + scrubbed = redaction.scrub( + { + "token": "a-long-enough-secret", + "automountServiceAccountToken": False, + "items": [{"password": "another-long-secret"}, 3, None], + "note": ("tuple", "values"), + } + ) + + assert scrubbed["token"] == redaction.REDACTED + assert scrubbed["automountServiceAccountToken"] is False + assert scrubbed["items"][0]["password"] == redaction.REDACTED + assert scrubbed["items"][1] == 3 + assert scrubbed["items"][2] is None + assert scrubbed["note"] == ["tuple", "values"] + + +def test_screening_is_idempotent() -> None: + once = redaction.redact("api_key=AKIAIOSFODNN7EXAMPLE") + assert redaction.redact(once) == once + assert redaction.redact("") == "" + + +def test_normalise_key_strips_separators() -> None: + assert redaction.normalise_key("Client-Secret_v2") == "clientsecretv2" diff --git a/testing/tests/test_hermes_handoff_rules.py b/testing/tests/test_hermes_handoff_rules.py new file mode 100644 index 00000000..11f09840 --- /dev/null +++ b/testing/tests/test_hermes_handoff_rules.py @@ -0,0 +1,135 @@ +"""Contracts for the shared evaluator primitives and dispatch.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step + +rules = load_handoff_module("hermes_handoff_rules") +model = load_handoff_module("hermes_handoff_model") +load_handoff_module("hermes_handoff_evaluators") + + +PAYLOAD = { + "a": {"b": [{"c": 1}, {"c": 2}]}, + "items": [{"number": 14, "merged": True}, {"number": 15, "merged": False}], + "flag": True, +} + + +@pytest.mark.parametrize( + ("path", "expected"), + [ + ("", PAYLOAD), + ("flag", True), + ("a.b[0].c", 1), + ("a.b[].c", [1, 2]), + ("a.b[]", [{"c": 1}, {"c": 2}]), + ("items[].number", [14, 15]), + ], +) +def test_dotted_paths_index_and_map(path: str, expected) -> None: + assert rules.dotted(PAYLOAD, path) == expected + + +@pytest.mark.parametrize("path", ["missing", "a.missing", "a.b[9].c", "flag[0]", "flag.b"]) +def test_an_absent_path_raises_rather_than_returning_none(path: str) -> None: + with pytest.raises(KeyError): + rules.dotted(PAYLOAD, path) + + +def test_parse_json_tolerates_control_characters_but_not_empty_output() -> None: + """Kanban bodies routinely carry raw newlines; that is not a broken read.""" + assert rules.parse_json(outcome(stdout='{"title": "a\nb"}'))["title"] == "a\nb" + with pytest.raises(ValueError, match="empty output"): + rules.parse_json(outcome(stdout=" ")) + with pytest.raises(ValueError): + rules.parse_json(outcome(stdout="not json")) + + +def test_lines_drops_blank_rows_and_trims() -> None: + assert rules.lines(outcome(stdout=" a \n\n b\n")) == ["a", "b"] + + +def test_missing_steps_reports_the_first_step_that_produced_nothing() -> None: + built = spec("allowed", steps=(step("a"), step("b"))) + + never_run = rules.missing_steps(built, {}) + assert never_run.status == model.NOT_RUN + assert "never executed" in never_run.reason + + errored = rules.missing_steps(built, {"a": outcome(error="timeout"), "b": outcome()}) + assert errored.status == model.NOT_RUN + assert "did not run" in errored.reason + + assert rules.missing_steps(built, {"a": outcome(), "b": outcome()}) is None + + +def test_an_optional_step_may_be_absent() -> None: + built = spec("allowed", steps=(step("a"), step("b", optional=True))) + assert rules.missing_steps(built, {"a": outcome()}) is None + + +@pytest.mark.parametrize( + ("stderr", "returncode", "denied"), + [ + ("Error from server (Forbidden): pods is forbidden", 1, True), + ("User cannot list resource", 1, True), + ("Gitea API returned HTTP 403", 1, True), + ("connection refused", 1, False), + ("", 0, False), + ], +) +def test_denial_detection_needs_a_refusal_not_merely_a_failure(stderr, returncode, denied) -> None: + assert rules.looks_denied(outcome(stderr=stderr, returncode=returncode)) is denied + + +def test_step_keys_selects_by_kind() -> None: + built = spec( + "denied", + steps=(step("r", kind=model.REVIEW), step("a", kind=model.ATTEMPT), step("x")), + ) + assert rules.step_keys(built, model.REVIEW) == ("r",) + assert rules.step_keys(built, model.ATTEMPT) == ("a",) + assert rules.step_keys(built, model.READ) == ("x",) + + +def test_parsed_step_returns_the_payload_or_the_skip_to_report() -> None: + built = spec("json_field", {"step": "s"}, steps=(step("s"),)) + + payload, key = rules.parsed_step(built, {"s": outcome(stdout='{"a": 1}')}) + assert (payload, key) == ({"a": 1}, "s") + + failed = rules.parsed_step(built, {"s": outcome(returncode=1)}) + assert failed.status == model.NOT_RUN and "failed" in failed.reason + + unparsable = rules.parsed_step(built, {"s": outcome(stdout="nope")}) + assert unparsable.status == model.NOT_RUN and "did not return JSON" in unparsable.reason + + +def test_an_unknown_rule_is_not_run_rather_than_a_crash() -> None: + evaluation = rules.evaluate(spec("no-such-rule"), {}) + assert evaluation.status == model.NOT_RUN + assert "unknown evaluator" in evaluation.reason + + +def test_an_evaluator_that_raises_is_not_run_rather_than_a_pass() -> None: + """A harness bug must never be indistinguishable from a clean release.""" + + @rules.evaluator("explodes-for-test") + def _explode(_spec, _outcomes): + raise RuntimeError("boom") + + evaluation = rules.evaluate(spec("explodes-for-test"), {}) + + assert evaluation.status == model.NOT_RUN + assert "RuntimeError" in evaluation.reason and "boom" in evaluation.reason + del rules.EVALUATORS["explodes-for-test"] + + +def test_is_list_distinguishes_arrays_from_strings() -> None: + assert rules.is_list([1]) and rules.is_list(()) + assert not rules.is_list("text") + assert not rules.is_list(b"bytes") + assert not rules.is_list({"a": 1}) diff --git a/testing/tests/test_hermes_handoff_run.py b/testing/tests/test_hermes_handoff_run.py new file mode 100644 index 00000000..379bb27d --- /dev/null +++ b/testing/tests/test_hermes_handoff_run.py @@ -0,0 +1,320 @@ +"""Contracts for handoff acceptance orchestration: vantages, catalog, report.""" + +from __future__ import annotations + +import datetime as dt +import subprocess + +from testing.tests.test_hermes_handoff_support import ( + FakeClock, + FakeSpawn, + load_handoff_module, + step, +) + +harness_run = load_handoff_module("hermes_handoff_run") +catalog = load_handoff_module("hermes_handoff_catalog") +exec_module = load_handoff_module("hermes_handoff_exec") +model = load_handoff_module("hermes_handoff_model") + +NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc) +TARGETS = catalog.Targets(now=NOW) + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + + +def whoami(username: str) -> object: + return completed(stdout=f'{{"status": {{"userInfo": {{"username": "{username}"}}}}}}') + + +def runner_for(results: list, **kwargs): + return exec_module.Runner( + clock=FakeClock(), spawn=FakeSpawn(results), environ={}, deadline_seconds=3600, **kwargs + ) + + +def check(rule: str, expect: dict | None = None, steps=(), identifier="test.check", mandatory=True): + return model.CheckSpec( + id=identifier, + title="t", + group="g", + rule=rule, + steps=tuple(steps), + expect=expect or {}, + mandatory=mandatory, + ) + + +def test_the_shipped_catalog_is_valid_and_covers_every_group() -> None: + specs = harness_run.build_catalog(TARGETS) + assert harness_run.validate_catalog(specs) == [] + groups = {spec.group for spec in specs} + assert {"baseline", "identity", "routing", "scopes", "access-denied", "gitops"} <= groups + assert {"forge", "nodes", "build", "reliability", "pool", "surfaces", "ephemeral"} <= groups + + +def test_catalog_validation_names_every_structural_problem() -> None: + broken = [ + check("no-such-rule", identifier="a.dup"), + check("allowed", identifier="a.dup"), + check("allowed", steps=(step("s", vantage="nowhere"),), identifier="b.vantage"), + check("allowed", steps=(step("s", kind="guess"),), identifier="c.kind"), + check("denied", steps=(step("s", kind=model.REVIEW),), identifier="d.review-only"), + model.CheckSpec( + id="e.impersonation", + title="t", + group="g", + rule="allowed", + steps=( + model.Step(key="s", vantage=catalog.OPERATOR, argv=("kubectl", "get", "ns", "--as", "x")), + ), + ), + ] + + problems = " | ".join(harness_run.validate_catalog(broken)) + + assert "duplicate check id" in problems + assert "unknown rule" in problems + assert "unknown vantage" in problems + assert "unknown kind" in problems + assert "needs a real attempt" in problems + assert "impersonates from the operator vantage" in problems + + +def test_identity_resolution_extracts_only_the_username() -> None: + runner = runner_for([whoami("system:serviceaccount:hermes:hermes-agent")]) + username, detail = harness_run.resolve_identity(runner, exec_module.operator_vantage()) + assert (username, detail) == ("system:serviceaccount:hermes:hermes-agent", "") + + failed = harness_run.resolve_identity( + runner_for([completed(stderr="no access", returncode=1)]), exec_module.operator_vantage() + ) + assert failed[0] == "" and "no access" in failed[1] + + unparsable = harness_run.resolve_identity( + runner_for([completed(stdout="nope")]), exec_module.operator_vantage() + ) + assert unparsable[0] == "" and "unparsable identity" in unparsable[1] + + nameless = harness_run.resolve_identity( + runner_for([completed(stdout='{"status": {}}')]), exec_module.operator_vantage() + ) + assert nameless[0] == "" and "no username" in nameless[1] + + +def test_vantages_resolve_to_running_pods_and_record_what_is_missing() -> None: + runner = runner_for( + [ + whoami("kubernetes-admin"), + completed(stdout="pod/hermes-agent-2\npod/hermes-agent-1\n"), + whoami("system:serviceaccount:hermes:hermes-agent"), + completed(stdout="pod/hermes-switchyard-1\n"), + completed(stderr="no pods", returncode=1), + completed(stdout="pod/hermes-chat-tenant-0\n"), + ] + ) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + by_name = {record.name: record for record in records} + + assert set(vantages) == {catalog.OPERATOR, catalog.SELF, catalog.SWITCHYARD, catalog.CHAT} + assert by_name[catalog.SELF].identity == "system:serviceaccount:hermes:hermes-agent" + assert "hermes-agent-1" in by_name[catalog.SELF].description # deterministic pick + assert by_name[catalog.NODE].available is False + assert by_name[catalog.CHAT].available is True + + +def test_an_unreachable_operator_stops_vantage_resolution_immediately() -> None: + runner = runner_for([completed(stderr="connection refused", returncode=1)]) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + + assert set(vantages) == {catalog.OPERATOR} + assert records[0].available is False + assert harness_run.vantage_problems(records)[0].startswith("the operator vantage is unavailable") + + +def test_a_missing_chat_pod_is_recorded_rather_than_assumed_present() -> None: + runner = runner_for( + [ + whoami("kubernetes-admin"), + completed(stdout="pod/hermes-agent-1\n"), + whoami("system:serviceaccount:hermes:hermes-agent"), + completed(stdout="pod/hermes-switchyard-1\n"), + completed(stdout="pod/hermes-node-1\n"), + completed(stderr="pods 'hermes-chat-tenant-0' not found", returncode=1), + ] + ) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + chat = next(record for record in records if record.name == catalog.CHAT) + + assert catalog.CHAT not in vantages + assert chat.available is False and "not found" in chat.detail + + +def test_a_selector_that_matches_no_running_pod_leaves_the_vantage_absent() -> None: + runner = runner_for( + [ + whoami("kubernetes-admin"), + completed(stdout="\n"), + completed(stdout="\n"), + completed(stdout="\n"), + completed(stdout="\n"), + ] + ) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + by_name = {record.name: record for record in records} + + assert set(vantages) == {catalog.OPERATOR} + assert "no running pod matches" in by_name[catalog.SELF].detail + + +def test_two_vantages_that_are_the_same_principal_invalidate_the_evidence() -> None: + """Corroboration needs two identities; one identity twice is a restatement.""" + records = [ + model.VantageRecord(name=catalog.OPERATOR, description="d", identity="same", available=True), + model.VantageRecord(name=catalog.SELF, description="d", identity="same", available=True), + ] + + problems = harness_run.vantage_problems(records) + + assert len(problems) == 1 + assert "same principal" in problems[0] + + +def test_vantage_problems_flag_a_missing_or_anonymous_self_probe() -> None: + operator = model.VantageRecord( + name=catalog.OPERATOR, description="d", identity="admin", available=True + ) + + absent = harness_run.vantage_problems([operator]) + assert "in-pod Hermes vantage is unavailable" in absent[0] + + anonymous = harness_run.vantage_problems( + [operator, model.VantageRecord(name=catalog.SELF, description="d", available=True)] + ) + assert "did not report an identity" in anonymous[0] + + clean = harness_run.vantage_problems( + [ + operator, + model.VantageRecord( + name=catalog.SELF, description="d", identity="agent", available=True + ), + ] + ) + assert clean == [] + + +def test_a_check_whose_vantage_is_absent_is_not_run() -> None: + spec = check("allowed", steps=(step("s", vantage=catalog.NODE),)) + + result = harness_run.run_check(runner_for([]), spec, {}) + + assert result.status == model.NOT_RUN + assert "vantage node is unavailable" in result.reason + assert result.outcomes[0].error == "vantage node is unavailable" + + +def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None: + spec = check( + "stdout_matches", + {"step": "s", "equals": "locked"}, + steps=(model.Step(key="s", vantage=catalog.OPERATOR, argv=("kubectl", "version"), record=False),), + ) + runner = runner_for([completed(stdout="locked", stderr="noise")]) + + result = harness_run.run_check(runner, spec, {catalog.OPERATOR: exec_module.operator_vantage()}) + + assert result.status == model.PASS + assert result.outcomes[0].stdout == harness_run.UNRECORDED + assert result.outcomes[0].stderr == harness_run.UNRECORDED + + +def test_an_expired_deadline_leaves_the_report_saying_so() -> None: + clock = FakeClock() + runner = exec_module.Runner(clock=clock, spawn=FakeSpawn([]), environ={}, deadline_seconds=1.0) + clock.advance(5.0) + spec = check("allowed", steps=(step("s", vantage=catalog.OPERATOR),)) + + report = harness_run.build_report( + runner, + TARGETS, + [spec], + {catalog.OPERATOR: exec_module.operator_vantage()}, + [], + "read-only", + "2026-08-17T00:00:00Z", + ) + + assert report.results[0].status == model.NOT_RUN + assert any("deadline expired" in error for error in report.harness_errors) + assert report.decision == model.NO_GO + + +def test_the_report_carries_the_configured_baseline_and_sorts_its_checks() -> None: + runner = runner_for([completed(stdout="ok"), completed(stdout="ok")]) + specs = [ + check("allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="z.last"), + check("allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="a.first"), + ] + records = [ + model.VantageRecord(name=catalog.OPERATOR, description="d", identity="admin", available=True), + model.VantageRecord(name=catalog.SELF, description="d", identity="agent", available=True), + ] + + report = harness_run.build_report( + runner, + TARGETS, + specs, + {catalog.OPERATOR: exec_module.operator_vantage()}, + records, + "read-only", + "2026-08-17T00:00:00Z", + ) + + assert [result.spec.id for result in report.results] == ["a.first", "z.last"] + assert report.baseline["baseline_commit"] == TARGETS.baseline_commit + assert report.baseline["dependency_pull_requests"] == list(TARGETS.dependency_pull_requests) + assert report.decision == model.GO + assert report.finished_at + + +def test_armed_results_replace_the_matching_catalog_placeholders() -> None: + placeholder = check("not_armed", identifier="ephemeral.feature-branch-push", mandatory=False) + armed = model.CheckResult( + spec=model.CheckSpec( + id="ephemeral.feature-branch-push", title="t", group="ephemeral", rule="armed" + ), + status=model.PASS, + reason="pushed", + ) + + report = harness_run.build_report( + runner_for([]), + TARGETS, + [placeholder], + {}, + [], + "ephemeral-armed", + "2026-08-17T00:00:00Z", + [armed], + ) + + assert len(report.results) == 1 + assert report.results[0].status == model.PASS + + +def test_an_unstartable_run_reports_no_go_with_a_named_reason() -> None: + report = harness_run.unavailable_report( + "read-only", "2026-08-17T00:00:00Z", "arming refused: bad phrase", TARGETS + ) + + assert report.decision == model.NO_GO + assert report.results[0].spec.id == "harness.startup" + assert report.results[0].status == model.NOT_RUN + assert report.harness_errors == ["arming refused: bad phrase"] diff --git a/testing/tests/test_hermes_handoff_support.py b/testing/tests/test_hermes_handoff_support.py new file mode 100644 index 00000000..64964a27 --- /dev/null +++ b/testing/tests/test_hermes_handoff_support.py @@ -0,0 +1,110 @@ +"""Shared loader and fixtures for the handoff acceptance harness tests. + +The harness lives in ``scripts/ops`` as sibling modules rather than an installed +package, so tests import it the way the entry point does: by putting that +directory on the path once and importing by name. +""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +OPS = REPO_ROOT / "scripts" / "ops" + +if str(OPS) not in sys.path: + sys.path.insert(0, str(OPS)) + + +def load_handoff_module(name: str) -> Any: + """Import one harness module by name.""" + return importlib.import_module(name) + + +exec_module = load_handoff_module("hermes_handoff_exec") +model = load_handoff_module("hermes_handoff_model") + + +def outcome( + stdout: str = "", + stderr: str = "", + returncode: int | None = 0, + error: str | None = None, + vantage: str = "self", + argv: tuple[str, ...] = ("kubectl", "get", "pods"), +) -> Any: + """Return a synthetic command outcome for evaluator tests.""" + return exec_module.Outcome( + argv=argv, + vantage=vantage, + returncode=returncode, + stdout=stdout, + stderr=stderr, + error=error, + ) + + +def spec( + rule: str, + expect: dict | None = None, + steps: tuple = (), + identifier: str = "test.check", + mandatory: bool = True, + scope: str | None = None, +) -> Any: + """Return a synthetic check specification.""" + return model.CheckSpec( + id=identifier, + title="test check", + group="test", + rule=rule, + steps=steps, + expect=expect or {}, + mandatory=mandatory, + scope=scope or model.ALWAYS, + ) + + +def step(key: str, kind: str | None = None, vantage: str = "self", optional: bool = False) -> Any: + """Return a synthetic step.""" + return model.Step( + key=key, + vantage=vantage, + argv=("kubectl", "get", "pods"), + kind=kind or model.READ, + optional=optional, + ) + + +class FakeClock: + """A monotonic clock the tests advance by hand.""" + + def __init__(self, start: float = 0.0) -> None: + self.now = start + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + """Move the clock forward.""" + self.now += seconds + + +class FakeSpawn: + """A ``subprocess.run`` stand-in driven by a queue of scripted results.""" + + def __init__(self, results: list) -> None: + self.results = list(results) + self.calls: list[dict] = [] + + def __call__(self, argv, **kwargs): + self.calls.append({"argv": list(argv), **kwargs}) + if not self.results: + raise AssertionError(f"unexpected command: {argv}") + outcome_or_error = self.results.pop(0) + if isinstance(outcome_or_error, Exception): + raise outcome_or_error + return outcome_or_error