From 8f005458282269ba5c07941814e4237f5d4cf3ac Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 10:14:17 +0000 Subject: [PATCH 1/3] hermes: add a fail-closed full-handoff acceptance harness Decides whether the Hermes platform handoff is fit to release, and refuses to round an absence of evidence up to a pass. The harness is read-only by default and classifies 71 checks PASS / FAIL / NOT_RUN / NOT_APPLICABLE. Any mandatory FAIL or NOT_RUN is NO_GO, and so is a harness-level problem: an unreachable vantage, a catalog entry whose evidence no longer exists, an expired deadline, or an evaluator that raised. Evidence comes from two vantages that cannot cover for each other: an external read-only operator kubeconfig, and the Hermes agent probing itself from inside its own pod. Before any check runs, the harness asks each vantage who it is and stops if they are the same principal, because dual-vantage evidence from one identity is a restatement rather than a corroboration. `--as` is rejected for every operator-side command and reachable only as the inner command of a `kubectl exec`, so impersonation can never stand in for a real self-probe. A deny check needs a live refused request, not only an authorization review. Two safety properties are structural rather than conventional, enforced where an argv becomes a subprocess: the default mode mutates nothing (mutating verbs require a server dry run; there is deliberately no live TokenRequest probe, because a successful one would mint a real credential), and no probe can pull a credential value into a report (no vault/sops/curl, secrets readable only with -o name, environment probes list names, shell only through frozen reviewed templates). Captures are bounded before they are screened, and the rendered report is re-screened before it is written. Mutation lives behind a separate arming flag with an exact confirmation phrase, a caller-supplied unique ref, a preflight that refuses a protected push target before any network call, and a cleanup whose verification is itself mandatory. A default run reports those four checks NOT_RUN. The catalog is declarative so a reviewer reads what is asserted rather than how it is plumbed, and so structural properties can be proven over every entry before a run. Catalog drift surfaces as NOT_RUN, which stops the release. docs/hermes_full_handoff_acceptance.md carries the merge order for PRs #14-#18 on top of the merged #13 baseline, the image build and Flux rollout, the rollback point for each step, the go/no-go checklist, and the limits that are asserted rather than exercised. Validation: 295 handoff tests pass with 100% line coverage on all 15 new modules; the full unit suite is 647 passed with two failures that reproduce unchanged on origin/main; Ruff, py_compile, kustomize render, and a diff credential screen are clean; a live read-only run against Atlas returns NO_GO for the pre-merge cluster with no unscreened fields in the report. --- docs/hermes_full_handoff_acceptance.md | 323 ++++++++++++++ scripts/ops/hermes_handoff_acceptance.py | 236 +++++++++++ scripts/ops/hermes_handoff_catalog.py | 189 +++++++++ scripts/ops/hermes_handoff_checks_access.py | 234 +++++++++++ scripts/ops/hermes_handoff_checks_delivery.py | 373 +++++++++++++++++ scripts/ops/hermes_handoff_checks_platform.py | 335 +++++++++++++++ scripts/ops/hermes_handoff_checks_workers.py | 388 +++++++++++++++++ scripts/ops/hermes_handoff_ephemeral.py | 313 ++++++++++++++ scripts/ops/hermes_handoff_evaluators.py | 266 ++++++++++++ scripts/ops/hermes_handoff_exec.py | 252 +++++++++++ scripts/ops/hermes_handoff_json_rules.py | 335 +++++++++++++++ scripts/ops/hermes_handoff_model.py | 270 ++++++++++++ scripts/ops/hermes_handoff_policy.py | 377 +++++++++++++++++ scripts/ops/hermes_handoff_redaction.py | 223 ++++++++++ scripts/ops/hermes_handoff_rules.py | 159 +++++++ scripts/ops/hermes_handoff_run.py | 334 +++++++++++++++ .../tests/test_hermes_handoff_acceptance.py | 395 ++++++++++++++++++ testing/tests/test_hermes_handoff_catalog.py | 194 +++++++++ .../tests/test_hermes_handoff_ephemeral.py | 271 ++++++++++++ .../tests/test_hermes_handoff_evaluators.py | 201 +++++++++ testing/tests/test_hermes_handoff_exec.py | 173 ++++++++ .../tests/test_hermes_handoff_json_rules.py | 279 +++++++++++++ testing/tests/test_hermes_handoff_model.py | 180 ++++++++ testing/tests/test_hermes_handoff_policy.py | 159 +++++++ .../tests/test_hermes_handoff_redaction.py | 162 +++++++ testing/tests/test_hermes_handoff_rules.py | 135 ++++++ testing/tests/test_hermes_handoff_run.py | 320 ++++++++++++++ testing/tests/test_hermes_handoff_support.py | 110 +++++ 28 files changed, 7186 insertions(+) create mode 100644 docs/hermes_full_handoff_acceptance.md create mode 100755 scripts/ops/hermes_handoff_acceptance.py create mode 100644 scripts/ops/hermes_handoff_catalog.py create mode 100644 scripts/ops/hermes_handoff_checks_access.py create mode 100644 scripts/ops/hermes_handoff_checks_delivery.py create mode 100644 scripts/ops/hermes_handoff_checks_platform.py create mode 100644 scripts/ops/hermes_handoff_checks_workers.py create mode 100644 scripts/ops/hermes_handoff_ephemeral.py create mode 100644 scripts/ops/hermes_handoff_evaluators.py create mode 100644 scripts/ops/hermes_handoff_exec.py create mode 100644 scripts/ops/hermes_handoff_json_rules.py create mode 100644 scripts/ops/hermes_handoff_model.py create mode 100644 scripts/ops/hermes_handoff_policy.py create mode 100644 scripts/ops/hermes_handoff_redaction.py create mode 100644 scripts/ops/hermes_handoff_rules.py create mode 100644 scripts/ops/hermes_handoff_run.py create mode 100644 testing/tests/test_hermes_handoff_acceptance.py create mode 100644 testing/tests/test_hermes_handoff_catalog.py create mode 100644 testing/tests/test_hermes_handoff_ephemeral.py create mode 100644 testing/tests/test_hermes_handoff_evaluators.py create mode 100644 testing/tests/test_hermes_handoff_exec.py create mode 100644 testing/tests/test_hermes_handoff_json_rules.py create mode 100644 testing/tests/test_hermes_handoff_model.py create mode 100644 testing/tests/test_hermes_handoff_policy.py create mode 100644 testing/tests/test_hermes_handoff_redaction.py create mode 100644 testing/tests/test_hermes_handoff_rules.py create mode 100644 testing/tests/test_hermes_handoff_run.py create mode 100644 testing/tests/test_hermes_handoff_support.py diff --git a/docs/hermes_full_handoff_acceptance.md b/docs/hermes_full_handoff_acceptance.md new file mode 100644 index 00000000..8b9103c2 --- /dev/null +++ b/docs/hermes_full_handoff_acceptance.md @@ -0,0 +1,323 @@ +# Hermes full-handoff acceptance harness and release runbook + +`scripts/ops/hermes_handoff_acceptance.py` decides whether the Hermes platform +handoff is fit to release. It is read-only by default, it classifies every check +`PASS` / `FAIL` / `NOT_RUN` / `NOT_APPLICABLE`, and **any mandatory `FAIL` or +`NOT_RUN` makes the whole run `NO_GO`**. Absence of evidence is never rounded up +to evidence of absence: a probe that could not run, a catalog entry whose +evidence no longer exists, an expired deadline, or a harness bug all land in +`NOT_RUN` and stop the release. + +``` +scripts/ops/hermes_handoff_acceptance.py \ + --context atlas-operator \ + --node-count 21 \ + --expected-suspension flux-system/bstein-dev-home-migrations \ + --output build/hermes-handoff-acceptance.json +``` + +Exit status is `0` for `GO` and `1` for `NO_GO`. The JSON report goes to +`--output` (or stdout); a short human summary goes to stderr unless +`--json-only` is passed. + +## Why two vantages + +Every authority claim is checked from two places that cannot cover for each +other: + +| Vantage | What it is | What it proves | +| --- | --- | --- | +| `operator` | An external, read-only kubeconfig | Desired state: bindings, manifests, workloads, Flux health, forge state | +| `self` | `kubectl exec` into the running `hermes-agent` container | Enforced state: what the agent's own service-account token can actually do | +| `switchyard`, `node`, `chat` | `kubectl exec` into the Switchyard, node-hardening, and chat pods | Evidence that only exists inside those workloads | + +Two rules keep this honest: + +* **The harness cannot impersonate from outside.** `--as`, `--as-group`, and + `--as-uid` are rejected by policy for any operator-side command. They are + reachable only as the *inner* command of a `kubectl exec`, where the identity + is testing its own refusal to impersonate rather than borrowing authority. + `kubectl auth can-i --as` is therefore not available as a substitute for a + real self-probe. +* **The two vantages must be different principals.** Before any check runs, the + harness asks each vantage `kubectl auth whoami` and compares. If the operator + and the in-pod probe authenticate as the same principal, the dual-vantage + evidence is a restatement rather than a corroboration, and the run is `NO_GO` + with `the operator and in-pod vantages authenticate as the same principal`. + Run the harness from a genuinely external kubeconfig — not from inside the + Hermes owner pod. + +Deny checks additionally require a **real refused request**, not only an +authorization review. `kubectl auth can-i` reports what RBAC says; only an +attempt proves the API server enforces it. Mutating attempts go through +`--dry-run=server`: the API server authorises the request and then discards it, +so a refusal is genuine and nothing is written either way. + +## Safety properties + +These hold structurally, in `hermes_handoff_policy.py`, so a catalog mistake is +a policy error before a run rather than a mutation or a leak during one. + +* **No mutation in the default mode.** Only read subcommands are reachable; + `apply`, `create`, `delete`, `patch`, `replace`, and `scale` require a dry-run + flag. `git push` and every non-`GET` forge method are refused outside the + armed mode. There is deliberately no live TokenRequest probe — a successful + one would mint a real credential. +* **No credential is ever read.** `vault`, `sops`, `age`, `gpg`, `openssl`, + `curl`, and `wget` are not on the binary allowlist. Secrets may only be read + with `-o name`. Environment probes list variable *names* and never values. + Provider health files are read through a frozen template that projects an + allow-listed set of non-secret status fields. The node account's lock state is + reported without reading its password hash; its authorized keys are counted, + never printed. +* **Shell is frozen.** `sh -c` is reachable only through the reviewed, + parameter-validated templates in `SHELL_TEMPLATES`. Parameters may not contain + shell metacharacters. +* **Output is bounded and screened.** Each capture is truncated to a byte budget + *before* it is screened, so a large dump cannot outrun the screen. Bulk + evidence (the routing-log tail, the Kanban listing) is parsed but replaced with + `[output not recorded]` in the artifact. The screen preserves JSON quoting, so + screened structured output is still parsable. +* **The report is re-checked before it is published.** `unscreened_fields` walks + the rendered payload; if anything still looks credential-shaped the report is + replaced with a `NO_GO` stub naming the offending pointers, and the file that + would have leaked is never written. + +## What the catalog asserts + +The catalog is declarative (`hermes_handoff_catalog.py` plus the four +`hermes_handoff_checks_*.py` modules) so a reviewer reads what is asserted, not +how it is plumbed. 71 checks in 14 groups: + +| Group | Assertions | +| --- | --- | +| `baseline` | `origin/main` descends from the merged worker-isolation baseline; every dependency pull request is merged into `main`; the exact open pull-request heads are recorded | +| `identity` | Codex authenticates as `codex-chatgpt-subscription`; Claude as `firstParty` / `claude.ai` / `claude-code-cli-subscription`; both evidence files are fresh; no provider API-key variable exists in the running process **or** in the manifest | +| `routing` | Switchyard has a ready replica; the routing log covers both providers, the `route` and `worker` lanes, medium/high/xhigh effort, and at least one real fallback; the newest record is fresh; both providers record a measured latency | +| `scopes` | Chat, agent, and triage each declare their own `HERMES_AUTO_ROUTER_PROFILE`, on distinct hostnames | +| `access-denied` | No cluster-admin binding for the agent; Secrets, service-account tokens, impersonation, workload mutation, arbitrary pod exec, attach, port-forward, and control-plane writes are all refused | +| `access-allowed` | get/list/watch on core objects, pod logs, and Flux/Helm status still work; the operator and in-pod namespace inventories agree | +| `gitops` | No unexpected Kustomization or HelmRelease suspension, and nothing unhealthy | +| `forge` | No forge credential in the worker environment or home; the forge identity is not an administrator; repository authority is pull/push only; an administrative route is live-refused; clone/fetch still works | +| `nodes` | The dedicated `hermes-agent` account is locked, has the expected uid/gid/home/shell, joins no supplementary group, has no sudoers fragment, holds exactly one authorized key; K3s credentials, K3s state, kubelet state, and the container runtime are root-only; the audit covers every node | +| `build` | The image builder service account mounts no token; the build pod is unprivileged with exactly `CHOWN, FOWNER, DAC_OVERRIDE, SETGID, SETUID`; the Harbor immutable-tag job succeeded; the running image is digest-pinned | +| `reliability` | The finalization/replay/session patches are running in the pod; the regression suites are present on `main` | +| `pool` | Three ready workers, on distinct nodes, with distinct volumes, mounting no API token and none of the coordinator's claims | +| `surfaces` | Kanban activity and durable sessions are visible and render without blank rows; the agent and chat session patches are deployed; Telegram topic state is on durable storage | +| `ephemeral` | Feature-branch push, draft pull request, protected-branch refusal, and verified cleanup — `NOT_RUN` unless armed | + +### Site configuration + +A few checks are emitted only when the operator declares what to expect, +because a `names_present` assertion over an empty expectation would pass on any +workload at all — which reads as coverage without being any: + +| Flag | Enables | +| --- | --- | +| `--pool-worker-env NAME` (repeatable) | `pool.assignment-safety-knobs-are-configured` | +| `--chat-config-revision REV` | `surfaces.chat-runs-the-telegram-topic-revision` | +| `--expect-telegram-sessions` with `--chat-ordinal N` | `surfaces.chat-telegram-sessions-are-continuous` | + +`--node-count` must be the real node count; the default of `1` makes the +fleet-coverage check vacuous. + +### Catalog drift is a NO_GO, not a pass + +If a workload, field, or manifest is renamed, the affected checks report +`NOT_RUN` because the evidence they name is no longer there. That is the +intended behaviour: the release stops until either the platform or the catalog +is corrected. Do not "fix" a `NOT_RUN` by deleting the check. + +## Ephemeral mutation mode + +Proving that a worker can still open a draft pull request means actually opening +one. That is the only part of the harness that writes anything, and it is a +separate mode: + +``` +scripts/ops/hermes_handoff_acceptance.py \ + --context atlas-operator \ + --arm-ephemeral-push \ + --confirm "ARM EPHEMERAL HERMES HANDOFF PUSH" \ + --ephemeral-token acceptance-20260817a \ + --output build/hermes-handoff-acceptance-armed.json +``` + +* The confirmation phrase must match exactly. +* `--ephemeral-token` is 8–64 lowercase alphanumeric/dash characters and must be + unique per run; the ref is always + `ephemeral/hermes-handoff-acceptance/`. +* **Preflight runs before any network call**: wrong phrase, wrong repository, + malformed token, or a protected push target all abort with `arming refused` + and a `NO_GO` report. `main`, `master`, `HEAD`, `develop`, `release`, + `production`, `stable`, `trunk`, `prod`, and `default` are refused by name + (case- and `refs/heads/`-insensitive), and anything that is not an ephemeral + acceptance ref is refused by pattern. +* The guard is **exercised on every armed run** by + `ephemeral.protected-branch-refusal`, rather than trusting that it was checked + once. +* The push target is guarded; the pull-request *base* is not. Opening a draft + pull request against `main` writes nothing to `main`. The draft title always + starts with `WIP: `. +* Cleanup closes the pull request, deletes the ref, and then **re-reads both** to + confirm they are gone. `ephemeral.cleanup-verified` is mandatory: a half-removed + branch fails the run rather than passing it. Cleanup runs even when an earlier + step failed. +* `git push --force`, `--force-with-lease`, `--mirror`, `--all`, and `--tags` + are refused even in armed mode. + +A default run reports all four ephemeral checks `NOT_RUN` and non-mandatory, so +a read-only sweep can still reach `GO`. + +## Release order + +### Baseline, already merged and deployed + +PR #13 (`hermes: harden worker isolation and blocked-task semantics`) merged as +`ab346f55509d584e457fe26cf90be3078f7a375c`. Every check below assumes it is +present; `baseline.origin-main-descends-merged-work` asserts it. Override with +`--baseline-commit` only when a later merge legitimately replaces it. + +### Pending work, observed 2026-08-17T10:07Z + +These heads move. The harness re-fetches them at run time via +`baseline.pending-pull-requests-recorded`; the table is the snapshot this +document was written against, not an input. + +| PR | Branch | Head at observation | State | Delivers | +| --- | --- | --- | --- | --- | +| #14 | `feature/hermes-safe-gitea-pr-client` | `3cd8a17e772978fb554d88dd1c59777786be3991` | draft | Bounded forge authority, observer RBAC (removes `hermes-agent-cluster-admin`), node account hardening | +| #15 | `fix/hermes-result-decomposition-reliability` | `7609cd77e87a64e332b2f0d670c1d7a4f62b3d9e` | draft | Result preservation, duplicate-fanout prevention, execution-safety patch | +| #16 | `feature/hermes-unprivileged-image-builder` | `5ab0eba619779b3dfed2eb09d5c02f0a9c92dd0a` | ready | Daemonless Kaniko build lane, tokenless builder, Harbor immutable-tag rule | +| #17 | `feature/hermes-three-lane-placement` | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | draft | Three concurrent CLI lanes on the larger ARM64 nodes | +| #18 | `feature/hermes-distributed-worker-pool` | `20002527512235b29054f3636b02418168373009` | draft | Three-node fenced execution pool with coordinator-only state ownership | + +Base at observation: `30259b526be5b2e226620aeebd9fb5f57dc88b7d`. + +### Merge order and why + +1. **#14 first.** It is the only change that *reduces* authority: it deletes + `services/hermes/agent-rbac.yaml` (the `cluster-admin` binding) and replaces + it with the namespaced/cluster observer roles, and it hardens the node + account. Merging it last would mean every earlier acceptance run measured a + cluster-admin agent. Every `access-denied` and `nodes` check depends on it. +2. **#16 second.** The image lane must exist before anything that needs a new + agent image is rolled out. It also touches `testing/quality_contract.json`, + so merging it early keeps later rebases small. `build.*` depends on it. +3. **#15 third.** It changes `dockerfiles/Dockerfile.hermes-agent`, which #16 + also touches — take #16's shape and rebase #15 onto it. `reliability.*` + depends on it. +4. **#17 fourth.** Placement only; it edits `services/hermes/agent-deployment.yaml` + and `testing/tests/test_hermes_cli_lanes.py`, both of which #14, #15, and #18 + also touch. Merging it after them avoids resolving the same conflict twice. +5. **#18 last.** The pool depends on the bounded forge client from #14, the + execution-safety patch from #15, and the agent image from #16. `pool.*` + depends on it. + +Overlapping files to expect (rebase, do not merge blind): `services/hermes/agent-deployment.yaml` +(#12, #14, #16, #17, #18), `services/hermes/kustomization.yaml` (#14, #16, #18), +`services/hermes/scripts/stage_runtime_access.py` (#14, #16, #18), +`services/vault/scripts/vault_k8s_auth_configure.sh` (#14, #16, #18), +`dockerfiles/Dockerfile.hermes-agent` (#15, #16), +`testing/tests/test_hermes_cli_lanes.py` (#14, #15, #17, #18), +`testing/tests/test_hermes_runtime_access.py` (#16, #18). + +PR #12 (AI usage exporter) is independent and can merge at any point; it touches +`services/hermes/agent-deployment.yaml`, so land it before #17 if convenient. + +### Image build and Flux rollout + +1. Merge in the order above; observe the default-branch Jenkins build to a + terminal result after each merge + (`/opt/coordinator/jenkins_build_evidence.py titan-iac --branch main --wait`). +2. After #16 is merged, run the `hermes-agent-image` pipeline. It publishes an + immutable `git--build-` tag and emits + `build/hermes-agent-release/hermes-image-update.patch`. +3. Apply that patch to `services/hermes/kustomization.yaml` on a branch, review, + and merge. The digest is what pins the deployment; + `build.deployed-image-is-digest-pinned` asserts it. +4. Let Flux reconcile. Confirm with `flux get kustomizations -A` that `hermes` + and `hermes-chat` are `Ready`, then wait for the rollout to settle. +5. Re-run the harness. Do not run it mid-rollout: `gitops.*` tolerates an + in-flight reconcile (`Reconciling=True` with `Ready=False`) but a half-rolled + deployment will legitimately fail the `identity`, `routing`, and `surfaces` + groups. + +### Rollback points + +| After | Roll back by | Restores | +| --- | --- | --- | +| #14 merge | Revert the merge commit; Flux restores `hermes-agent-cluster-admin` | Cluster-admin agent, previous node account state | +| #16 merge | Revert the merge; the previously pinned digest is unchanged in `kustomization.yaml` | Previous build lane; published immutable tags are not deleted | +| Image digest bump | Revert the `kustomization.yaml` commit to the prior digest | Previous agent image; Harbor immutability means the old tag is still exactly what it was | +| #15 merge | Revert the merge; the init-container patch disappears on the next rollout | Previous finalization/replay behaviour | +| #17 merge | Revert the merge | Previous lane concurrency and node placement | +| #18 merge | Revert the merge; the StatefulSet and its PVCs are removed by Flux pruning | Single-coordinator execution | + +Every rollback is a Git revert reconciled by Flux. Do not roll back by direct +`kubectl` edit; the next reconcile would undo it. + +## Go / no-go checklist + +Run in order. Stop at the first item that does not hold. + +1. `origin/main` is `ab346f55509d584e457fe26cf90be3078f7a375c` or a descendant. +2. PRs #14, #15, #16, #17, #18 are all merged into `main`, in that order, each + with a green default-branch build. +3. The agent image has been built by the #16 lane, is pinned by digest in + `services/hermes/kustomization.yaml`, and Flux has finished reconciling. +4. The harness is run from an **external operator kubeconfig**, not from inside + the Hermes owner pod, with `--node-count` set to the real node count and every + deliberate Flux suspension declared via `--expected-suspension`. +5. The default read-only run reports `GO` — no mandatory `FAIL`, no mandatory + `NOT_RUN`, no `harness_errors`. +6. The armed run reports `PASS` for all four `ephemeral.*` checks, including + `ephemeral.cleanup-verified`, and the ephemeral ref and pull request are + confirmed gone. +7. The JSON report is archived with the exact `origin/main` commit and the + recorded pull-request heads from `baseline.pending-pull-requests-recorded`. + +Only then is the handoff releasable. + +## Known limits + +Stated so nobody mistakes a proxy for the thing itself. + +* **Merge, approve, and close authority is asserted, not exercised.** The + harness never attempts a merge. It proves the forge identity holds + `permissions.admin: false` and is live-refused on an administrative route; + the route-level allowlist is covered by PR #14's own regression suite. +* **Attach and port-forward have no side-effect-free live attempt.** Their + authorization reviews ride alongside a real, refused control-plane write in the + same check, and pod exec — the one subresource with a harmless real attempt — + has its own check. +* **TokenRequest is asserted by review.** A successful live attempt would mint a + real credential, so the check pairs the `serviceaccounts/token` review with a + refused `serviceaccount` create in the same RBAC family. +* **Branch protection is not read.** The harness identity is not a repository + administrator, so `GET /branch_protections` is refused — which is itself + recorded as evidence. Protected-push refusal is guaranteed by the harness's own + preflight, exercised on every armed run, not by reading the forge's config. +* **Stale/duplicate assignment safety is asserted as deployed configuration.** + The behaviour itself is covered by the pool's regression suite; the harness + checks that the workers carry the configuration and that the coordinator keeps + sole ownership of its claims. +* **Telegram session continuity is per tenant** and depends on there having been + Telegram traffic, so it is off by default. The always-on check is that topic + and tenant state live on a persistent claim rather than in the pod. +* **The routing window is a tail**, 600 records by default. A property that did + not occur inside that window is reported as missing rather than absent; widen + the window or wait for traffic rather than removing the assertion. + +## Tests + +``` +python3 -m pytest testing/tests/ -k handoff +python3 -m coverage run --source=scripts/ops --include="*/hermes_handoff_*.py" \ + -m pytest testing/tests/ -k handoff && python3 -m coverage report +``` + +The suite covers parsing, redaction, timeouts and deadlines, partial failure, +zero state, dual-vantage mismatch, mandatory skips, catalog structure, and +ephemeral arming and cleanup — including the cases where cleanup cannot be +verified. diff --git a/scripts/ops/hermes_handoff_acceptance.py b/scripts/ops/hermes_handoff_acceptance.py new file mode 100755 index 00000000..7be6de29 --- /dev/null +++ b/scripts/ops/hermes_handoff_acceptance.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +"""Fail-closed acceptance harness for the Hermes full handoff. + +Read-only by default. The harness collects evidence from two genuinely separate +places — an external operator vantage and the Hermes agent probing itself from +inside its own pod — classifies every check PASS / FAIL / NOT_RUN / +NOT_APPLICABLE, and emits machine-readable JSON alongside a short human summary. + +Any mandatory FAIL or NOT_RUN is a NO_GO, and so is a harness-level problem: an +unreachable vantage, a catalog whose evidence no longer exists, an expired +deadline, or two vantages that turn out to be the same principal. Nothing here +rounds an absence of evidence up to a pass. + + scripts/ops/hermes_handoff_acceptance.py --context atlas-operator + scripts/ops/hermes_handoff_acceptance.py --output build/handoff.json + +Mutation lives behind a separate arming flag, needs an exact confirmation +phrase and a unique ref, refuses protected push targets before any network +call, and verifies its own cleanup. A default run reports those checks NOT_RUN. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import sys +from collections.abc import Sequence +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from hermes_handoff_catalog import Targets # noqa: E402 +from hermes_handoff_ephemeral import ( # noqa: E402 + CONFIRMATION, + ArmingError, + ArmRequest, + preflight, + run_armed, +) +from hermes_handoff_exec import ( # noqa: E402 + DEFAULT_COMMAND_TIMEOUT, + DEFAULT_RUN_DEADLINE, + SELF, + Runner, +) +from hermes_handoff_model import GO, Report, unscreened_fields, utc_now # noqa: E402 +from hermes_handoff_policy import ARMED, READ_ONLY # noqa: E402 +from hermes_handoff_redaction import DEFAULT_MAX_BYTES # noqa: E402 +from hermes_handoff_run import ( # noqa: E402 + build_catalog, + build_report, + resolve_vantages, + unavailable_report, +) + +EXIT_GO = 0 +EXIT_NO_GO = 1 +EXIT_USAGE = 2 + + +def build_parser() -> argparse.ArgumentParser: + """Return the command-line interface for the harness.""" + parser = argparse.ArgumentParser( + prog="hermes_handoff_acceptance", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--namespace", default="hermes", help="Hermes namespace.") + parser.add_argument("--context", help="Kubernetes context for the operator vantage.") + parser.add_argument("--kubeconfig", help="Kubeconfig for the operator vantage.") + parser.add_argument("--output", type=Path, help="Write the JSON report here instead of stdout.") + parser.add_argument("--json-only", action="store_true", help="Suppress the human summary.") + parser.add_argument( + "--command-timeout", type=float, default=DEFAULT_COMMAND_TIMEOUT, help="Per-command seconds." + ) + parser.add_argument( + "--deadline", type=float, default=DEFAULT_RUN_DEADLINE, help="Whole-run seconds." + ) + parser.add_argument( + "--max-output-bytes", type=int, default=DEFAULT_MAX_BYTES, help="Per-capture byte budget." + ) + parser.add_argument( + "--baseline-commit", default=Targets.baseline_commit, help="Commit main must descend from." + ) + parser.add_argument( + "--dependency-pr", + type=int, + action="append", + default=None, + help="Pull request that must be merged before this release is certifiable (repeatable).", + ) + parser.add_argument( + "--expected-suspension", + action="append", + default=None, + metavar="NAMESPACE/NAME", + help="Flux object allowed to be suspended (repeatable).", + ) + parser.add_argument("--node-count", type=int, default=1, help="Nodes the fleet must cover.") + parser.add_argument("--pool-replicas", type=int, default=3, help="Expected pool worker count.") + parser.add_argument( + "--chat-config-revision", default="", help="Expected chat pod-template config revision." + ) + parser.add_argument( + "--chat-ordinal", + type=int, + default=0, + help="Chat tenant ordinal that carries the Telegram identity.", + ) + parser.add_argument( + "--expect-telegram-sessions", + action="store_true", + help="Require the selected chat tenant to hold durable Telegram-sourced sessions.", + ) + parser.add_argument( + "--pool-worker-env", + action="append", + default=None, + help="Environment variable the pool workers must declare (repeatable).", + ) + parser.add_argument( + "--max-evidence-age", + type=int, + default=86_400, + help="Seconds before provider and routing evidence is treated as stale.", + ) + armed = parser.add_argument_group("ephemeral mutation mode") + armed.add_argument( + "--arm-ephemeral-push", + action="store_true", + help="Permit one ephemeral branch push and draft pull request, then remove both.", + ) + armed.add_argument("--confirm", default="", help=f"Exact confirmation phrase: {CONFIRMATION!r}.") + armed.add_argument("--ephemeral-token", default="", help="Unique 8-64 character ref suffix.") + armed.add_argument("--ephemeral-base", default="main", help="Pull-request base branch.") + armed.add_argument("--remote", default="origin", help="Git remote for the ephemeral push.") + armed.add_argument("--repo", default=Targets.repo, help="owner/name of the forge repository.") + return parser + + +def targets_from_args(arguments: argparse.Namespace, now: dt.datetime) -> Targets: + """Return the catalog targets a parsed command line describes.""" + defaults = Targets() + return Targets( + namespace=arguments.namespace, + baseline_commit=arguments.baseline_commit, + dependency_pull_requests=tuple(arguments.dependency_pr or defaults.dependency_pull_requests), + expected_suspensions=tuple(arguments.expected_suspension or ()), + node_count=arguments.node_count, + pool_replicas=arguments.pool_replicas, + chat_config_revision=arguments.chat_config_revision, + chat_ordinal=arguments.chat_ordinal, + expect_telegram_sessions=arguments.expect_telegram_sessions, + pool_worker_env=tuple(arguments.pool_worker_env or ()), + max_evidence_age_seconds=arguments.max_evidence_age, + repo=arguments.repo, + remote=arguments.remote, + now=now, + ) + + +def emit(report: Report, arguments: argparse.Namespace, stream=None) -> list[str]: + """Write the report, refusing to publish one that still looks credential-shaped.""" + payload = report.as_dict() + offenders = unscreened_fields(payload) + if offenders: + payload = { + "harness": payload["harness"], + "decision": "NO_GO", + "harness_errors": [ + "the rendered report failed its own credential screening at " + + ", ".join(offenders[:8]) + ], + } + rendered = json.dumps(payload, indent=2, sort_keys=True, default=str) + if arguments.output: + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(rendered + "\n", encoding="utf-8") + else: + print(rendered, file=stream or sys.stdout) + if not arguments.json_only: + print(report.render_summary(), file=sys.stderr) + return offenders + + +def _armed_results(arguments: argparse.Namespace, runner: Runner, vantages) -> list: + request = ArmRequest( + repo=arguments.repo, + remote=arguments.remote, + token=arguments.ephemeral_token, + confirmation=arguments.confirm, + base=arguments.ephemeral_base, + ) + preflight(request, arguments.repo) + vantage = vantages.get(SELF) or vantages["operator"] + return run_armed(runner, vantage, request) + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the acceptance sweep and return the process exit status.""" + arguments = build_parser().parse_args(argv) + now = dt.datetime.now(dt.timezone.utc) + started_at = utc_now(lambda: now) + targets = targets_from_args(arguments, now) + mode = ARMED if arguments.arm_ephemeral_push else READ_ONLY + + runner = Runner( + mode=mode, + command_timeout=arguments.command_timeout, + deadline_seconds=arguments.deadline, + max_bytes=arguments.max_output_bytes, + ) + vantages, records = resolve_vantages(runner, targets, arguments.kubeconfig, arguments.context) + + armed_results: list = [] + if arguments.arm_ephemeral_push: + try: + armed_results = _armed_results(arguments, runner, vantages) + except ArmingError as exc: + report = unavailable_report(mode, started_at, f"arming refused: {exc}", targets) + emit(report, arguments) + return EXIT_NO_GO + + specs = build_catalog(targets) + report = build_report( + runner, targets, specs, vantages, records, mode, started_at, armed_results + ) + offenders = emit(report, arguments) + if offenders: + return EXIT_NO_GO + return EXIT_GO if report.decision == GO else EXIT_NO_GO + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/scripts/ops/hermes_handoff_catalog.py b/scripts/ops/hermes_handoff_catalog.py new file mode 100644 index 00000000..b59ff2b9 --- /dev/null +++ b/scripts/ops/hermes_handoff_catalog.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Declarative acceptance catalog: baseline, identity, routing, scopes, access. + +The catalog is data on purpose. A reviewer should be able to read what the +release is asserted to satisfy without reading any plumbing, and the harness +should be able to prove structural properties over every entry — both vantages +represented, no impersonation from outside a pod, no mutation outside a server +dry run — before it runs a single command. + +Catalog drift is a NO_GO, not a pass. If a manifest, a field name, or a workload +is renamed, the affected checks report ``NOT_RUN`` because the evidence they +name is not there, and the release stops until either the platform or the +catalog is corrected. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field +from typing import Any + +from hermes_handoff_model import ALWAYS, ATTEMPT, READ, REVIEW, CheckSpec, Step + +OPERATOR = "operator" +SELF = "self" +SWITCHYARD = "switchyard" +NODE = "node" +CHAT = "chat" +VANTAGE_NAMES = (OPERATOR, SELF, SWITCHYARD, NODE, CHAT) + +# `namespace/namesuspendType=Status,...` — one compact row per object. +FLUX_PROJECTION = ( + "jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{'\\t'}" + "{.spec.suspend}{'\\t'}{range .status.conditions[*]}{.type}={.status},{end}{'\\n'}{end}" +).replace("'", '"') + +ENV_NAME_PROJECTION = ( + "jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{'\\n'}{end}{end}" +).replace("'", '"') + +INIT_NAME_PROJECTION = ( + "jsonpath={range .spec.template.spec.initContainers[*]}{.name}{'\\n'}{end}" +).replace("'", '"') + +NODE_NAME_PROJECTION = "jsonpath={range .items[*]}{.spec.nodeName}{'\\n'}{end}".replace("'", '"') + +PROVIDER_API_KEY_NAMES = ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + "AZURE_OPENAI_API_KEY", + "CLAUDE_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", +) +FORGE_CREDENTIAL_NAMES = ("GITEA_PASSWORD", "GITEA_TOKEN", "GITEA_USERNAME", "GIT_PASSWORD") + +CODEX_HEALTH = "/opt/data/provider-health/codex.json" +CLAUDE_HEALTH = "/opt/data/provider-health/claude.json" +CODEX_HEALTH_FIELDS = "authenticated,transport,state,model,effort,latency_ms,checked_at" +CLAUDE_HEALTH_FIELDS = ( + "authenticated,api_provider,auth_method,transport,state,subscription_type," + "model,effort,latency_ms,checked_at" +) +ROUTING_LOG = "/var/lib/switchyard/routing.jsonl" + + +@dataclass(frozen=True) +class Targets: + """Runtime names and expected values the catalog is written against. + + Everything a site can legitimately rename lives here so the assertions stay + readable and a rename is a configuration change rather than a code change. + """ + + namespace: str = "hermes" + agent_deployment: str = "hermes-agent" + agent_container: str = "hermes" + chat_statefulset: str = "hermes-chat-tenant" + chat_container: str = "hermes" + chat_ordinal: int = 0 + chat_router_deployment: str = "hermes-chat-router" + triage_deployment: str = "hermes" + triage_container: str = "hermes" + switchyard_deployment: str = "hermes-switchyard" + switchyard_container: str = "switchyard" + node_daemonset: str = "hermes-node-ssh-access" + node_container: str = "key-reconciler" + pool_statefulset: str = "hermes-execution-worker" + pool_replicas: int = 3 + pool_selector: str = "app=hermes-execution-worker" + coordinator_claims: tuple[str, ...] = ("hermes-agent-home", "hermes-agent-workspace") + node_account: str = "hermes-agent" + node_account_identity: str = "1200:1200:/home/hermes-agent:/bin/bash" + node_count: int = 1 + repo: str = "atlas/titan-iac" + remote: str = "origin" + baseline_commit: str = "ab346f55509d584e457fe26cf90be3078f7a375c" + dependency_pull_requests: tuple[int, ...] = (14, 15, 16, 17, 18) + expected_suspensions: tuple[str, ...] = () + harbor_namespace: str = "harbor" + harbor_immutability_job: str = "hermes-agent-immutability" + builder_namespace: str = "jenkins" + builder_serviceaccount: str = "hermes-image-builder" + builder_pipeline: str = "ci/Jenkinsfile.hermes-agent-image" + builder_capabilities: str = 'add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]' + agent_init_containers: tuple[str, ...] = ("patch-web-session-activity",) + chat_init_containers: tuple[str, ...] = ("patch-api-server-sessions",) + chat_config_revision: str = "" + pool_worker_env: tuple[str, ...] = () + max_evidence_age_seconds: int = 86_400 + routing_tail_lines: int = 600 + routing_tail_bytes: int = 512 * 1024 + listing_bytes: int = 512 * 1024 + git_askpass: str = "/opt/coordinator/gitea_askpass.sh" + expect_telegram_sessions: bool = False + now: Any = None + extra: dict[str, Any] = field(default_factory=dict) + + +def step( + key: str, + vantage: str, + *argv: str, + kind: str = READ, + optional: bool = False, + record: bool = True, + max_bytes: int | None = None, +) -> Step: + """Return a probe step addressed to one vantage.""" + return Step( + key=key, + vantage=vantage, + argv=tuple(argv), + kind=kind, + optional=optional, + record=record, + max_bytes=max_bytes, + ) + + +def check( + identifier: str, + title: str, + group: str, + rule: str, + steps: Sequence[Step], + expect: dict[str, Any] | None = None, + mandatory: bool = True, + scope: str = ALWAYS, + rationale: str = "", +) -> CheckSpec: + """Return one catalog entry.""" + return CheckSpec( + id=identifier, + title=title, + group=group, + rule=rule, + steps=tuple(steps), + expect=expect or {}, + mandatory=mandatory, + scope=scope, + rationale=rationale, + ) + + +def deny( + identifier: str, + title: str, + review: Sequence[str], + attempt: Sequence[str], + rationale: str, + group: str = "access-denied", +) -> CheckSpec: + """Return a deny check that pairs a self review with a real refused request.""" + return check( + identifier, + title, + group, + "denied", + [ + step("review", SELF, *review, kind=REVIEW), + step("attempt", SELF, *attempt, kind=ATTEMPT), + ], + rationale=rationale, + ) diff --git a/scripts/ops/hermes_handoff_checks_access.py b/scripts/ops/hermes_handoff_checks_access.py new file mode 100644 index 00000000..a863e0f3 --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_access.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Acceptance checks for what Hermes may and may not do to the cluster. + +Every deny check pairs an in-pod authorization review with a real request that +must be refused, because a review reports what RBAC says and only an attempt +proves the API server enforces it. Mutating attempts go through a server dry +run: the API server authorises the request and discards it, so the refusal is +genuine and nothing is written either way. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + FLUX_PROJECTION, + OPERATOR, + SELF, + Targets, + check, + deny, + step, +) +from hermes_handoff_model import ATTEMPT, REVIEW, CheckSpec +from hermes_handoff_policy import shell + + +def _access(targets: Targets) -> list[CheckSpec]: + namespace = targets.namespace + return [ + check( + "access.no-cluster-admin-binding-for-agent", + "No ClusterRoleBinding grants the agent service account cluster-admin", + "access-denied", + "names_absent", + [step("bindings", OPERATOR, "kubectl", "get", "clusterrolebindings", "-o", "name")], + {"step": "bindings", "contains": ("hermes-agent-cluster-admin",)}, + rationale="The operator half of the pair; the in-pod attempts below are the enforcement half.", + ), + deny( + "access.secrets-are-denied", + "Hermes cannot read Secrets cluster-wide", + ("kubectl", "auth", "can-i", "get", "secrets", "--all-namespaces"), + ("kubectl", "get", "secrets", "--all-namespaces", "-o", "name"), + "Even a regression that granted this could only return object names: the probe is pinned to -o name.", + ), + check( + "access.service-account-tokens-are-denied", + "Hermes cannot mint a service-account token", + "access-denied", + "denied", + [ + step( + "review-token", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "serviceaccounts/token", + "--namespace", + namespace, + kind=REVIEW, + ), + step( + "review-serviceaccount", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "serviceaccounts", + "--namespace", + namespace, + kind=REVIEW, + ), + step( + "attempt", + SELF, + "kubectl", + "--namespace", + namespace, + "create", + "serviceaccount", + "hermes-acceptance-probe", + "--dry-run=server", + kind=ATTEMPT, + ), + ], + rationale="TokenRequest has no side-effect-free live attempt — a successful one would mint a real credential — so it is asserted by review alongside a refused write in the same RBAC family.", + ), + deny( + "access.impersonation-is-denied", + "Hermes cannot impersonate another identity", + ("kubectl", "auth", "can-i", "impersonate", "users", "--all-namespaces"), + ("kubectl", "get", "namespaces", "--as", "system:admin", "-o", "name"), + "`--as` is reachable only from inside the pod, where the identity is testing its own refusal.", + ), + deny( + "access.workload-mutation-is-denied", + "Hermes cannot mutate its own workload", + ("kubectl", "auth", "can-i", "patch", "deployments", "--namespace", namespace), + ( + "kubectl", + "--namespace", + namespace, + "patch", + f"deploy/{targets.agent_deployment}", + "--type=merge", + "--patch", + '{"metadata":{"annotations":{"acceptance.bstein.dev/probe":"deny"}}}', + "--dry-run=server", + ), + "A server dry run is authorised and then discarded, so the refusal is real and nothing is written.", + ), + deny( + "access.pod-exec-is-denied", + "Hermes cannot exec into an arbitrary pod", + ("kubectl", "auth", "can-i", "create", "pods/exec", "--all-namespaces"), + ( + "kubectl", + "--namespace", + namespace, + "exec", + f"deploy/{targets.switchyard_deployment}", + "--container", + targets.switchyard_container, + "--", + *shell("path_readable", path="/tmp"), + ), + "Exec is the one subresource with a harmless real attempt; attach and port-forward are covered by their own reviews.", + ), + check( + "access.attach-portforward-and-kube-system-writes-are-denied", + "Hermes holds no attach, port-forward, or control-plane write authority", + "access-denied", + "denied", + [ + step("review-attach", SELF, "kubectl", "auth", "can-i", "create", "pods/attach", "--all-namespaces", kind=REVIEW), + step("review-portforward", SELF, "kubectl", "auth", "can-i", "create", "pods/portforward", "--all-namespaces", kind=REVIEW), + step("review-create", SELF, "kubectl", "auth", "can-i", "create", "configmaps", "--namespace", "kube-system", kind=REVIEW), + step( + "attempt", + SELF, + "kubectl", + "--namespace", + "kube-system", + "create", + "configmap", + "hermes-acceptance-probe", + "--dry-run=server", + kind=ATTEMPT, + ), + ], + rationale="Attach and port-forward have no side-effect-free live attempt, so their reviews ride alongside a real, refused control-plane write.", + ), + check( + "access.required-reads-are-allowed", + "Hermes retains the reads its operator role depends on", + "access-allowed", + "allowed", + [ + step("watch-review", SELF, "kubectl", "auth", "can-i", "watch", "pods", "--namespace", namespace, kind=REVIEW), + step("pods", SELF, "kubectl", "--namespace", namespace, "get", "pods", "-o", "name"), + step("namespaces", SELF, "kubectl", "get", "namespaces", "-o", "name"), + step("deployments", SELF, "kubectl", "--namespace", namespace, "get", "deployments", "-o", "name"), + step("nodes", SELF, "kubectl", "get", "nodes", "-o", "name"), + ], + ), + check( + "access.pod-logs-are-allowed", + "Hermes can still read pod logs", + "access-allowed", + "allowed", + [ + step( + "logs", + SELF, + "kubectl", + "--namespace", + targets.namespace, + "logs", + f"deploy/{targets.switchyard_deployment}", + "--container", + targets.switchyard_container, + "--tail", + "1", + record=False, + ) + ], + ), + check( + "access.flux-and-helm-status-are-allowed", + "Hermes can read Flux and Helm reconciliation status", + "access-allowed", + "allowed", + [ + step("kustomizations", SELF, "kubectl", "get", "kustomizations.kustomize.toolkit.fluxcd.io", "--all-namespaces", "-o", "name", record=False), + step("helmreleases", SELF, "kubectl", "get", "helmreleases.helm.toolkit.fluxcd.io", "--all-namespaces", "-o", "name", record=False), + ], + ), + check( + "access.namespace-view-agrees-across-vantages", + "The operator and in-pod vantages see the same namespace inventory", + "access-allowed", + "vantages_agree", + [ + step("operator", OPERATOR, "kubectl", "get", "namespaces", "-o", "name", record=False), + step("self", SELF, "kubectl", "get", "namespaces", "-o", "name", record=False), + ], + {"steps": ("operator", "self")}, + rationale="A disagreement here means one vantage is not seeing the cluster the other is certifying.", + ), + ] + + +def _gitops(targets: Targets) -> list[CheckSpec]: + return [ + check( + f"gitops.{kind.split('.')[0]}-reconcile-cleanly", + f"No unexpected {label} suspension or unhealthy reconciliation", + "gitops", + "flux_health", + [step("objects", OPERATOR, "kubectl", "get", kind, "--all-namespaces", "-o", FLUX_PROJECTION, record=False)], + {"step": "objects", "expected_suspensions": targets.expected_suspensions}, + ) + for kind, label in ( + ("kustomizations.kustomize.toolkit.fluxcd.io", "Kustomization"), + ("helmreleases.helm.toolkit.fluxcd.io", "HelmRelease"), + ) + ] + + +def access_checks(targets: Targets) -> list[CheckSpec]: + """Return the access-denied, access-allowed, and GitOps health checks.""" + return [*_access(targets), *_gitops(targets)] diff --git a/scripts/ops/hermes_handoff_checks_delivery.py b/scripts/ops/hermes_handoff_checks_delivery.py new file mode 100644 index 00000000..a3dbaad7 --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_delivery.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python3 +"""Acceptance checks for forge authority, node accounts, images, and regressions. + +What the platform is allowed to do outside its own namespace: which forge +authority a worker holds, what a node account is and is not, how an image is +built and pinned, and whether the fixes the release depends on are actually +running rather than merely merged. + +Where a property has no side-effect-free live probe — a merge that must never be +attempted, a branch protection the harness identity cannot read — the entry says +so in its rationale and asserts the nearest fact that *is* observable, rather +than quietly asserting nothing. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + FORGE_CREDENTIAL_NAMES, + INIT_NAME_PROJECTION, + NODE, + NODE_NAME_PROJECTION, + OPERATOR, + SELF, + Targets, + check, + step, +) +from hermes_handoff_model import ATTEMPT, CheckSpec +from hermes_handoff_policy import GITEA_CLIENT, shell + +HOST_ETC = "/host-etc" +HOST_HOME = "/host-home" + + +def _forge(targets: Targets) -> list[CheckSpec]: + api = f"/api/v1/repos/{targets.repo}" + return [ + check( + "forge.worker-holds-no-git-credential-variable", + "No forge credential is exported into the worker process", + "forge", + "names_absent", + [step("env", SELF, *shell("env_names"))], + {"step": "env", "names": FORGE_CREDENTIAL_NAMES}, + rationale="Authentication reaches Git through an askpass helper that reads runtime state, never through the environment.", + ), + check( + "forge.worker-holds-no-credential-file", + "The worker home has no stored Git credential file", + "forge", + "stdout_matches", + [step("credentials", SELF, *shell("path_readable", path="/opt/data/.git-credentials"))], + {"step": "credentials", "equals": "unreadable"}, + ), + check( + "forge.identity-is-not-an-administrator", + "The forge identity the worker uses holds no administrative rights", + "forge", + "json_field", + [step("user", SELF, GITEA_CLIENT, "GET", "/api/v1/user")], + {"step": "user", "fields": {"is_admin": False}}, + ), + check( + "forge.repository-authority-is-push-only", + "The worker can push a branch but cannot administer the repository", + "forge", + "json_field", + [step("repo", SELF, GITEA_CLIENT, "GET", api)], + { + "step": "repo", + "fields": { + "permissions.admin": False, + "permissions.push": True, + "permissions.pull": True, + }, + }, + rationale="Merge, approve, close, and protection changes all require authority this identity is proven not to hold.", + ), + check( + "forge.administrative-route-is-refused", + "An administrative forge route is refused for this identity", + "forge", + "denied", + [ + step( + "attempt", + SELF, + GITEA_CLIENT, + "GET", + f"{api}/branch_protections", + kind=ATTEMPT, + ) + ], + rationale="A live refusal, not a configuration reading. The same lack of authority is what blocks a merge route.", + ), + check( + "forge.clone-and-fetch-still-work", + "The worker can still read the repository it is expected to work in", + "forge", + "allowed", + [ + step( + "ls-remote", + SELF, + *shell( + "git_ls_remote", + askpass=targets.git_askpass, + url=f"https://scm.bstein.dev/{targets.repo}.git", + ), + ) + ], + ), + ] + + +def _nodes(targets: Targets) -> list[CheckSpec]: + account = targets.node_account + return [ + check( + "nodes.account-password-is-locked", + "The dedicated node account has no usable password", + "nodes", + "stdout_matches", + [ + step( + "lock", + NODE, + *shell("account_lock_state", account=account, path=f"{HOST_ETC}/shadow"), + ) + ], + {"step": "lock", "equals": "locked"}, + rationale="The probe reports the lock marker only; the password hash is never read.", + ), + check( + "nodes.account-identity-is-dedicated", + "The node account is the dedicated Hermes identity, not a shared one", + "nodes", + "stdout_matches", + [ + step( + "identity", + NODE, + *shell("account_identity", account=account, path=f"{HOST_ETC}/passwd"), + ) + ], + {"step": "identity", "equals": targets.node_account_identity}, + ), + check( + "nodes.account-holds-no-supplementary-groups", + "The node account joins no supplementary or administrative group", + "nodes", + "stdout_matches", + [ + step( + "groups", + NODE, + *shell("account_groups", account=account, path=f"{HOST_ETC}/group"), + ) + ], + {"step": "groups", "equals": ""}, + rationale="sudo, docker, adm, and disk are all group grants; an empty membership list rules out every one of them.", + ), + check( + "nodes.account-has-no-sudo-policy", + "No sudoers fragment names the node account", + "nodes", + "stdout_matches", + [step("sudoers", NODE, *shell("path_mode", path=f"{HOST_ETC}/sudoers.d/{account}"))], + {"step": "sudoers", "equals": "absent"}, + ), + check( + "nodes.account-holds-a-single-authorized-key", + "Exactly one authorized key is installed for the node account", + "nodes", + "stdout_matches", + [ + step( + "keys", + NODE, + *shell("line_count", path=f"{HOST_HOME}/{account}/.ssh/authorized_keys"), + ) + ], + {"step": "keys", "equals": "1"}, + rationale="A count, never the key material. A second key is how a human identity re-enters an account that is meant to be dedicated.", + ), + *[ + check( + f"nodes.{label}-is-root-only", + f"{title} remains unreadable to the node account", + "nodes", + "stdout_matches", + [step("mode", NODE, *shell("path_mode", path=path))], + {"step": "mode", "equals": expected}, + ) + for label, title, path, expected in ( + ( + "k3s-credentials", + "The K3s admin kubeconfig", + f"{HOST_ETC}/rancher/k3s/k3s.yaml", + ["600 root:root", "absent"], + ), + ("k3s-state", "K3s server state", "/host-k3s", ["700 root:root", "absent"]), + ("kubelet-state", "Kubelet state", "/host-kubelet", ["700 root:root", "750 root:root", "absent"]), + ("container-runtime", "The container runtime socket directory", "/host-run-containerd", ["710 root:root", "700 root:root", "absent"]), + ) + ], + check( + "nodes.hardening-covers-every-node", + "The node account audit runs on every node in the cluster", + "nodes", + "distinct_count", + [ + step( + "coverage", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "pods", + "--selector", + f"app={targets.node_daemonset}", + "-o", + NODE_NAME_PROJECTION, + ) + ], + {"step": "coverage", "minimum": targets.node_count}, + rationale="A hardened fleet with one unenrolled node is an unhardened fleet.", + ), + ] + + +def _build(targets: Targets) -> list[CheckSpec]: + pipeline = f"{targets.remote}/main:{targets.builder_pipeline}" + return [ + check( + "build.builder-service-account-is-tokenless", + "The image builder service account mounts no API token", + "build", + "json_field", + [ + step( + "serviceaccount", + OPERATOR, + "kubectl", + "--namespace", + targets.builder_namespace, + "get", + "serviceaccount", + targets.builder_serviceaccount, + "-o", + "json", + ) + ], + {"step": "serviceaccount", "fields": {"automountServiceAccountToken": False}}, + ), + check( + "build.builder-runs-unprivileged-with-exact-capabilities", + "The build pod is unprivileged and adds only the expected capabilities", + "build", + "names_present", + [step("pipeline", OPERATOR, "git", "show", pipeline, record=False)], + { + "step": "pipeline", + "names": ( + targets.builder_capabilities, + "privileged: false", + "allowPrivilegeEscalation: false", + "automountServiceAccountToken: false", + 'drop: ["ALL"]', + ), + }, + rationale="Exact lines, not a substring sweep: a widened capability set has to change one of these lines to take effect.", + ), + check( + "build.harbor-immutability-rule-is-applied", + "The Harbor immutable-tag rule for the agent image has been applied", + "build", + "json_numeric", + [ + step( + "job", + OPERATOR, + "kubectl", + "--namespace", + targets.harbor_namespace, + "get", + "job", + targets.harbor_immutability_job, + "-o", + "json", + ) + ], + {"step": "job", "fields": {"status.succeeded": {"min": 1}}}, + rationale="The job verifies the rule it creates, so a completed run is evidence the rule matched the expected contract.", + ), + check( + "build.deployed-image-is-digest-pinned", + "The running agent image is pinned by digest, not by a mutable tag", + "build", + "names_present", + [ + step( + "image", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.agent_deployment}", + "-o", + 'jsonpath={range .spec.template.spec.containers[*]}{.image}{"\\n"}{end}', + ) + ], + {"step": "image", "contains": ("@sha256:",)}, + ), + ] + + +def _reliability(targets: Targets) -> list[CheckSpec]: + return [ + check( + "reliability.finalization-and-replay-patches-are-deployed", + "The agent workload runs the finalization, replay, and session patches", + "reliability", + "names_present", + [ + step( + "init", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.agent_deployment}", + "-o", + INIT_NAME_PROJECTION, + ) + ], + {"step": "init", "names": targets.agent_init_containers}, + rationale="The regression suites live in the repository; this asserts the fixes they cover are actually running.", + ), + check( + "reliability.regression-suites-are-present-on-main", + "The decomposition and worker-recovery regression suites are on main", + "reliability", + "names_present", + [ + step( + "suites", + OPERATOR, + "git", + "ls-tree", + "--name-only", + f"{targets.remote}/main", + "testing/tests/", + ) + ], + { + "step": "suites", + "names": ( + "testing/tests/test_hermes_cli_lanes.py", + "testing/tests/test_hermes_coordinator.py", + "testing/tests/test_hermes_worker_recovery.py", + ), + }, + ), + ] + + +def delivery_checks(targets: Targets) -> list[CheckSpec]: + """Return the forge, node, build, and reliability checks.""" + return [*_forge(targets), *_nodes(targets), *_build(targets), *_reliability(targets)] diff --git a/scripts/ops/hermes_handoff_checks_platform.py b/scripts/ops/hermes_handoff_checks_platform.py new file mode 100644 index 00000000..d1986976 --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_platform.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""Acceptance checks for the release baseline, provider identity, and routing. + +These are the facts a handoff rests on before any authority question is asked: +that the tree under test is the merged one, that both providers authenticate +through the subscriptions they are supposed to, that Switchyard actually decided +the routes it claims to, and that chat, agent, and triage are still three +distinct scopes rather than one surface wearing three hostnames. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + CLAUDE_HEALTH, + CLAUDE_HEALTH_FIELDS, + CODEX_HEALTH, + CODEX_HEALTH_FIELDS, + ENV_NAME_PROJECTION, + OPERATOR, + PROVIDER_API_KEY_NAMES, + ROUTING_LOG, + SELF, + SWITCHYARD, + Targets, + check, + step, +) +from hermes_handoff_model import CheckSpec +from hermes_handoff_policy import GITEA_CLIENT, shell + + +def _baseline(targets: Targets) -> list[CheckSpec]: + api = f"/api/v1/repos/{targets.repo}" + checks = [ + check( + "baseline.origin-main-descends-merged-work", + "origin/main descends from the merged Hermes isolation baseline", + "baseline", + "allowed", + [ + step( + "ancestor", + OPERATOR, + "git", + "merge-base", + "--is-ancestor", + targets.baseline_commit, + f"{targets.remote}/main", + ) + ], + rationale="Everything downstream assumes the merged worker-isolation baseline is present.", + ), + check( + "baseline.pending-pull-requests-recorded", + "Exact open pull-request heads are recorded, not assumed", + "baseline", + "json_record", + [step("open", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls?state=open&limit=50")], + { + "step": "open", + "record": { + "numbers": "[].number", + "head_refs": "[].head.ref", + "head_shas": "[].head.sha", + "base_shas": "[].base.sha", + "drafts": "[].draft", + "mergeable": "[].mergeable", + }, + }, + rationale="The runbook has to name the heads it was verified against; stale heads are how a release verifies the wrong tree.", + ), + check( + "baseline.operator-checkout-clean", + "The operator checkout has no uncommitted drift", + "baseline", + "stdout_matches", + [step("status", OPERATOR, "git", "status", "--porcelain")], + {"step": "status", "equals": ""}, + mandatory=False, + rationale="Advisory: a dirty checkout does not invalidate cluster evidence, but it does muddy what was compared.", + ), + ] + checks += [ + check( + f"baseline.dependency-pr-{number}-merged", + f"Pull request #{number} is merged into main", + "baseline", + "json_field", + [step("pr", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls/{number}")], + {"step": "pr", "fields": {"merged": True, "base.ref": "main"}}, + rationale="This harness certifies the post-merge platform; an unmerged dependency means it is measuring something else.", + ) + for number in targets.dependency_pull_requests + ] + return checks + + +def _identity(targets: Targets) -> list[CheckSpec]: + codex = shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS) + claude = shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS) + agent = f"deploy/{targets.agent_deployment}" + return [ + check( + "identity.codex-is-chatgpt-subscription", + "Codex authenticates through the ChatGPT subscription, not an API key", + "identity", + "json_field", + [step("health", SELF, *codex)], + { + "step": "health", + "fields": { + "authenticated": True, + "transport": "codex-chatgpt-subscription", + "state": "available", + }, + }, + rationale="Only allow-listed status fields are read; the credential itself is never opened.", + ), + check( + "identity.codex-evidence-is-fresh", + "Codex authentication evidence is current", + "identity", + "json_recent", + [step("health", SELF, *codex)], + { + "step": "health", + "now": targets.now, + "fields": {"checked_at": targets.max_evidence_age_seconds}, + }, + rationale="Stale provider evidence describes a state the release no longer has.", + ), + check( + "identity.claude-is-firstparty-subscription", + "Claude authenticates as a claude.ai first-party subscription", + "identity", + "json_field", + [step("health", SELF, *claude)], + { + "step": "health", + "fields": { + "authenticated": True, + "api_provider": "firstParty", + "auth_method": "claude.ai", + "transport": "claude-code-cli-subscription", + "state": "available", + }, + }, + ), + check( + "identity.claude-evidence-is-fresh", + "Claude authentication evidence is current", + "identity", + "json_recent", + [step("health", SELF, *claude)], + { + "step": "health", + "now": targets.now, + "fields": {"checked_at": targets.max_evidence_age_seconds}, + }, + ), + check( + "identity.no-provider-api-key-in-pod", + "No provider API-key variable is set in the running agent process", + "identity", + "names_absent", + [step("env", SELF, *shell("env_names"))], + { + "step": "env", + "names": PROVIDER_API_KEY_NAMES, + "contains": ("API_KEY",), + }, + rationale="Names only. The probe lists variable names and never their values.", + ), + check( + "identity.no-provider-api-key-in-manifest", + "No provider API-key variable is declared on the agent workload", + "identity", + "names_absent", + [ + step( + "env", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + agent, + "-o", + ENV_NAME_PROJECTION, + ) + ], + {"step": "env", "names": PROVIDER_API_KEY_NAMES, "contains": ("API_KEY",)}, + rationale="The operator counterpart to the in-pod probe: desired state and running state are asserted separately.", + ), + ] + + +def _routing(targets: Targets) -> list[CheckSpec]: + switchyard = f"deploy/{targets.switchyard_deployment}" + tail = shell("tail_lines", path=ROUTING_LOG, limit=str(targets.routing_tail_lines)) + routing_step = step( + "routing", SWITCHYARD, *tail, record=False, max_bytes=targets.routing_tail_bytes + ) + return [ + check( + "routing.switchyard-is-available", + "Switchyard has a ready replica serving every model-call boundary", + "routing", + "json_numeric", + [ + step( + "deployment", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + switchyard, + "-o", + "json", + ) + ], + {"step": "deployment", "fields": {"status.readyReplicas": {"min": 1}}}, + ), + check( + "routing.provider-effort-and-fallback-evidence", + "Routing evidence covers both providers, all graded efforts, and a real fallback", + "routing", + "routing_evidence", + [routing_step], + { + "step": "routing", + "providers": ("codex", "claude"), + "efforts": ("medium", "high", "xhigh"), + "lanes": ("route", "worker"), + "require_fallback_evidence": True, + "max_age_seconds": targets.max_evidence_age_seconds, + "now": targets.now, + }, + rationale="The routing log records the decided route per boundary; the tail is parsed here and never pasted into the report.", + ), + check( + "routing.codex-latency-recorded", + "Codex boundary latency is measured, not assumed", + "routing", + "json_numeric", + [step("health", SELF, *shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS))], + {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, + ), + check( + "routing.claude-latency-recorded", + "Claude boundary latency is measured, not assumed", + "routing", + "json_numeric", + [step("health", SELF, *shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS))], + {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, + ), + ] + + +def _scope_profile( + targets: Targets, name: str, workload: str, container: str, expected: str +) -> CheckSpec: + projection = ( + f'jsonpath={{.spec.template.spec.containers[?(@.name=="{container}")]' + '.env[?(@.name=="HERMES_AUTO_ROUTER_PROFILE")].value}' + ) + return check( + f"scopes.{name}-profile-is-distinct", + f"The {name} surface runs under its own routing profile", + "scopes", + "stdout_matches", + [ + step( + "profile", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + workload, + "-o", + projection, + ) + ], + {"step": "profile", "equals": expected}, + rationale="Chat, agent, and triage must not collapse into one scope; each states its own profile.", + ) + + +def _scopes(targets: Targets) -> list[CheckSpec]: + return [ + _scope_profile( + targets, "agent", f"deploy/{targets.agent_deployment}", targets.agent_container, "agent" + ), + _scope_profile( + targets, + "chat", + f"statefulset/{targets.chat_statefulset}", + targets.chat_container, + "chat", + ), + _scope_profile( + targets, + "triage", + f"deploy/{targets.triage_deployment}", + targets.triage_container, + "triage", + ), + check( + "scopes.surfaces-have-distinct-hosts", + "Chat, agent, and triage are published on distinct hostnames", + "scopes", + "distinct_count", + [ + step( + "hosts", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "ingress", + "-o", + 'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}', + ) + ], + {"step": "hosts", "minimum": 3}, + ), + ] + + +def platform_checks(targets: Targets) -> list[CheckSpec]: + """Return the baseline, identity, routing, and scope checks.""" + return [*_baseline(targets), *_identity(targets), *_routing(targets), *_scopes(targets)] diff --git a/scripts/ops/hermes_handoff_checks_workers.py b/scripts/ops/hermes_handoff_checks_workers.py new file mode 100644 index 00000000..a2e1abea --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_workers.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +"""Acceptance checks for the distributed worker pool and the visible surfaces. + +A pool is only distributed if its workers occupy distinct nodes and distinct +volumes and leave state ownership with the coordinator; a replica count alone +satisfies none of that. A surface is only working if the work it did is visible +in it, which is why the session and Kanban checks look at the same data the UI +renders rather than at whether a pod is Running. + +The ephemeral entries are placeholders in a default run. They report NOT_RUN +until the mutation mode is separately armed, and the armed mode replaces them +with real, mandatory results. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + CHAT, + INIT_NAME_PROJECTION, + NODE_NAME_PROJECTION, + OPERATOR, + SELF, + Targets, + check, +) +from hermes_handoff_catalog import step as step +from hermes_handoff_model import EPHEMERAL, CheckSpec + +KANBAN_STATUSES = [ + "archived", + "blocked", + "done", + "ready", + "review", + "running", + "scheduled", + "todo", + "triage", +] + + +def _pool(targets: Targets) -> list[CheckSpec]: + selector = targets.pool_selector + return [ + check( + "pool.three-workers-are-ready", + "The distributed execution pool has its full complement of ready workers", + "pool", + "json_numeric", + [ + step( + "statefulset", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + "json", + ) + ], + { + "step": "statefulset", + "fields": { + "status.readyReplicas": {"min": targets.pool_replicas}, + "spec.replicas": {"min": targets.pool_replicas}, + }, + }, + ), + check( + "pool.workers-occupy-distinct-nodes", + "Each pool worker runs on its own node", + "pool", + "distinct_count", + [ + step( + "nodes", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "pods", + "--selector", + selector, + "-o", + NODE_NAME_PROJECTION, + ) + ], + { + "step": "nodes", + "minimum": targets.pool_replicas, + "total_equals": targets.pool_replicas, + }, + rationale="Three replicas co-scheduled onto one node satisfy a replica count and defeat the isolation it stands for.", + ), + check( + "pool.workers-own-distinct-volumes", + "Each pool worker has its own durable workspace", + "pool", + "distinct_count", + [ + step( + "claims", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "persistentvolumeclaims", + "--selector", + selector, + "-o", + "name", + ) + ], + {"step": "claims", "minimum": targets.pool_replicas}, + ), + check( + "pool.workers-are-tokenless", + "Pool workers mount no Kubernetes API token", + "pool", + "json_field", + [ + step( + "statefulset", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + "json", + ) + ], + { + "step": "statefulset", + "fields": {"spec.template.spec.automountServiceAccountToken": False}, + }, + rationale="A fenced execution worker needs no cluster authority; holding a token is the failure, not the exception.", + ), + check( + "pool.coordinator-retains-sole-state-ownership", + "No pool worker mounts the coordinator's durable state", + "pool", + "names_absent", + [ + step( + "claims", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}', + ) + ], + {"step": "claims", "names": targets.coordinator_claims}, + rationale="Two writers on one SQLite state file is the duplicate-work and lost-result failure this pool exists to avoid.", + ), + *_pool_env(targets), + ] + + +def _pool_env(targets: Targets) -> list[CheckSpec]: + """Return the assignment-safety check only when a site declares what to expect. + + A ``names_present`` assertion over an empty expectation would pass on any + workload at all, which is worse than not asserting: it reads as coverage. + """ + if not targets.pool_worker_env: + return [] + return [ + check( + "pool.assignment-safety-knobs-are-configured", + "Pool workers carry the lease and de-duplication configuration", + "pool", + "names_present", + [ + step( + "env", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + 'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}', + ) + ], + {"step": "env", "names": targets.pool_worker_env}, + rationale="Deployed configuration only; the stale-claim and duplicate-dispatch behaviour itself is covered by the pool's own regression suite.", + ) + ] + + +def _surfaces(targets: Targets) -> list[CheckSpec]: + checks = [ + check( + "surfaces.kanban-activity-is-visible", + "Durable Kanban activity is visible to the agent surface", + "surfaces", + "all_of_field", + [ + step( + "tasks", + SELF, + "hermes", + "kanban", + "list", + "--json", + record=False, + max_bytes=targets.listing_bytes, + ) + ], + {"step": "tasks", "key_field": "id", "fields": {"status": KANBAN_STATUSES}}, + rationale="Non-empty and well-formed. An empty board is NOT_RUN, because it cannot distinguish a quiet week from a broken read.", + ), + check( + "surfaces.durable-sessions-render-without-blank-rows", + "Durable sessions are listed with a preview, a source, and an id", + "surfaces", + "lines_match", + [step("sessions", SELF, "hermes", "sessions", "list", "--limit", "20", record=False)], + {"step": "sessions", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + rationale="A row whose leading column is blank is the session-render regression seen from the data side.", + ), + check( + "surfaces.agent-ui-session-patches-are-deployed", + "The agent surface runs the session-activity rendering patches", + "surfaces", + "names_present", + [ + step( + "init", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.agent_deployment}", + "-o", + INIT_NAME_PROJECTION, + ) + ], + {"step": "init", "names": ("patch-web-session-activity", "patch-api-server-sessions")}, + rationale="Blank session rows were a rendering regression; this asserts the patches that fixed it are in the running pod.", + ), + check( + "surfaces.chat-session-patches-are-deployed", + "The chat surface runs the API-session continuity patches", + "surfaces", + "names_present", + [ + step( + "init", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.chat_statefulset}", + "-o", + INIT_NAME_PROJECTION, + ) + ], + {"step": "init", "names": targets.chat_init_containers}, + ), + check( + "surfaces.chat-telegram-topic-state-is-durable", + "Telegram topic and tenant state is held on durable storage", + "surfaces", + "names_present", + [ + step( + "claims", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.chat_router_deployment}", + "-o", + 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}', + ) + ], + {"step": "claims", "contains": ("state",)}, + rationale="Topic selection survives a router restart only if the tenant state file is on a claim rather than in the pod.", + ), + ] + if targets.expect_telegram_sessions: + checks.append( + check( + "surfaces.chat-telegram-sessions-are-continuous", + "The selected chat tenant still holds durable Telegram-sourced sessions", + "surfaces", + "lines_match", + [ + step( + "telegram", + CHAT, + "hermes", + "sessions", + "list", + "--source", + "telegram", + "--limit", + "20", + record=False, + ) + ], + {"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.", + ) + ) + if targets.chat_config_revision: + checks.append( + check( + "surfaces.chat-runs-the-telegram-topic-revision", + "The chat surface runs the durable Telegram topic configuration", + "surfaces", + "stdout_matches", + [ + step( + "revision", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.chat_statefulset}", + "-o", + "jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}", + ) + ], + {"step": "revision", "equals": targets.chat_config_revision}, + rationale="Telegram topic continuity is carried by the config revision the pod template pins.", + ) + ) + return checks + + +def _ephemeral() -> list[CheckSpec]: + return [ + check( + f"ephemeral.{name}", + title, + "ephemeral", + "not_armed", + [], + {"reason": reason}, + mandatory=False, + scope=EPHEMERAL, + rationale="Mutating checks are separately armed. A default run reports them NOT_RUN and never touches the forge.", + ) + for name, title, reason in ( + ( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + "ephemeral mutation mode is not armed", + ), + ( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + "ephemeral mutation mode is not armed", + ), + ( + "protected-branch-refusal", + "Arming against a protected branch is refused before any network call", + "ephemeral mutation mode is not armed", + ), + ( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + "ephemeral mutation mode is not armed", + ), + ) + ] + + +def worker_checks(targets: Targets) -> list[CheckSpec]: + """Return the pool, surface, and ephemeral-placeholder checks.""" + return [*_pool(targets), *_surfaces(targets), *_ephemeral()] diff --git a/scripts/ops/hermes_handoff_ephemeral.py b/scripts/ops/hermes_handoff_ephemeral.py new file mode 100644 index 00000000..5456482b --- /dev/null +++ b/scripts/ops/hermes_handoff_ephemeral.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""The separately armed, self-cleaning mutation mode of the acceptance harness. + +Proving that a worker can still open a draft pull request means actually opening +one, which is the only part of this harness that writes anything. It is +therefore a distinct mode with a distinct policy: an explicit confirmation +phrase, a caller-supplied unique ref, a preflight that refuses a protected +target before any network call, and a cleanup whose *verification* is itself a +mandatory check. + +The push target is guarded, not the pull-request base. Opening a pull request +against `main` writes nothing to `main`; pushing to it does. The refusal is +checked by name and by pattern, and the arming self-test exercises it on every +armed run rather than trusting that it was checked once. +""" + +from __future__ import annotations + +import re +from collections.abc import Sequence +from dataclasses import dataclass + +from hermes_handoff_exec import Outcome, Runner, Vantage +from hermes_handoff_model import EPHEMERAL, FAIL, NOT_RUN, PASS, CheckResult, CheckSpec +from hermes_handoff_policy import GITEA_CLIENT + +CONFIRMATION = "ARM EPHEMERAL HERMES HANDOFF PUSH" +REF_PREFIX = "ephemeral/hermes-handoff-acceptance" +REF_RE = re.compile(rf"{REF_PREFIX}/[a-z0-9][a-z0-9-]{{7,63}}\Z") +TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9-]{7,63}\Z") +DRAFT_TITLE_PREFIX = "WIP: " +PROTECTED_NAMES = frozenset( + { + "default", + "develop", + "head", + "main", + "master", + "prod", + "production", + "release", + "stable", + "trunk", + } +) + + +class ArmingError(ValueError): + """Raised when an ephemeral run is not safe to start.""" + + +@dataclass(frozen=True) +class ArmRequest: + """Everything the armed mode needs, and nothing it can infer for itself.""" + + repo: str + remote: str + token: str + confirmation: str + base: str = "main" + + @property + def ref(self) -> str: + """Return the exact, unique branch this run is allowed to create.""" + return f"{REF_PREFIX}/{self.token}" + + +def normalise_branch(name: str) -> str: + """Return a branch name stripped of ref decoration, for comparison.""" + trimmed = name.strip().removeprefix("refs/heads/").strip("/") + return trimmed.lower() + + +def assert_push_target_allowed(ref: str) -> None: + """Raise unless ``ref`` is an ephemeral branch this harness may create.""" + if normalise_branch(ref) in PROTECTED_NAMES: + raise ArmingError(f"refusing to push to protected branch {ref!r}") + if not REF_RE.fullmatch(ref): + raise ArmingError(f"push target {ref!r} is not an ephemeral acceptance ref") + + +def preflight(request: ArmRequest, expected_repo: str) -> None: + """Validate an arming request before any network call is made.""" + if request.confirmation != CONFIRMATION: + raise ArmingError("confirmation phrase does not match") + if request.repo != expected_repo: + raise ArmingError(f"repository {request.repo!r} is not the expected {expected_repo!r}") + if not TOKEN_RE.fullmatch(request.token): + raise ArmingError("ephemeral token must be 8-64 lowercase alphanumeric or dash characters") + assert_push_target_allowed(request.ref) + if not normalise_branch(request.base): + raise ArmingError("a pull-request base branch is required") + + +def _spec(identifier: str, title: str, rationale: str = "") -> CheckSpec: + return CheckSpec( + id=f"ephemeral.{identifier}", + title=title, + group="ephemeral", + rule="armed", + steps=(), + mandatory=True, + scope=EPHEMERAL, + rationale=rationale, + ) + + +def _result( + identifier: str, + title: str, + status: str, + reason: str, + outcomes: Sequence[Outcome] = (), + evidence: dict | None = None, + rationale: str = "", +) -> CheckResult: + return CheckResult( + spec=_spec(identifier, title, rationale), + status=status, + reason=reason, + evidence=evidence or {}, + outcomes=list(outcomes), + ) + + +def _protected_refusal_result() -> CheckResult: + """Exercise the protected-branch guard on this run, not on a past one.""" + refused: list[str] = [] + accepted: list[str] = [] + for candidate in ("main", "master", "refs/heads/main", "HEAD", f"{REF_PREFIX}/x"): + try: + assert_push_target_allowed(candidate) + except ArmingError: + refused.append(candidate) + else: + accepted.append(candidate) + evidence = {"refused": refused, "accepted": accepted} + if accepted: + return _result( + "protected-branch-refusal", + "Arming against a protected branch is refused before any network call", + FAIL, + f"the guard accepted {accepted}", + evidence=evidence, + ) + return _result( + "protected-branch-refusal", + "Arming against a protected branch is refused before any network call", + PASS, + "every protected and malformed push target was refused", + evidence=evidence, + ) + + +def _detail(outcome: Outcome) -> str: + """Return the most informative bounded explanation an outcome carries.""" + return outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}" + + +def _pull_request_number(outcome: Outcome) -> int | None: + match = re.search(r'"number"\s*:\s*(\d+)', outcome.stdout) + return int(match.group(1)) if match else None + + +def run_armed(runner: Runner, vantage: Vantage, request: ArmRequest) -> list[CheckResult]: + """Push a unique ephemeral ref, open a draft pull request, and remove both. + + Every step is checked and every artifact is removed. Cleanup runs even when + an earlier step failed, and the cleanup verification is reported as its own + mandatory check so a half-removed branch cannot pass as success. + """ + api = f"/api/v1/repos/{request.repo}" + results = [_protected_refusal_result()] + outcomes: list[Outcome] = [] + + existing = runner.run(("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage) + outcomes.append(existing) + if not existing.ok: + results.append( + _result( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + NOT_RUN, + f"could not confirm the ref is unused: {_detail(existing)}", + outcomes, + ) + ) + return results + if existing.stdout.strip(): + results.append( + _result( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + FAIL, + f"{request.ref} already exists on {request.remote}; pick a fresh token", + outcomes, + ) + ) + return results + + pushed = runner.run( + ("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"), vantage + ) + outcomes.append(pushed) + results.append( + _result( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + PASS if pushed.ok else FAIL, + f"pushed {request.ref}" if pushed.ok else f"push failed: {_detail(pushed)}", + [pushed], + {"ref": request.ref}, + ) + ) + + number: int | None = None + if pushed.ok: + created = runner.run( + ( + GITEA_CLIENT, + "POST", + f"{api}/pulls", + "--field", + f"title={DRAFT_TITLE_PREFIX}Hermes handoff acceptance ephemeral probe", + "--field", + f"head={request.ref}", + "--field", + f"base={normalise_branch(request.base)}", + "--field", + "body=Ephemeral acceptance probe. Closed and deleted by the harness.", + ), + vantage, + ) + outcomes.append(created) + number = _pull_request_number(created) if created.ok else None + results.append( + _result( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + PASS if created.ok and number else FAIL, + f"opened draft pull request #{number}" + if created.ok and number + else f"draft creation failed: {_detail(created)}", + [created], + {"pull_request": number}, + ) + ) + else: + results.append( + _result( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + NOT_RUN, + "the ephemeral branch was never pushed", + ) + ) + + results.append(_cleanup(runner, vantage, request, api, number)) + return results + + +def _cleanup( + runner: Runner, vantage: Vantage, request: ArmRequest, api: str, number: int | None +) -> CheckResult: + """Close the pull request, delete the ref, and prove both are gone.""" + outcomes: list[Outcome] = [] + problems: list[str] = [] + + if number is not None: + closed = runner.run( + (GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"), vantage + ) + outcomes.append(closed) + if not closed.ok: + problems.append(f"pull request #{number} could not be closed") + + deleted = runner.run((GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage) + outcomes.append(deleted) + + remaining = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage + ) + outcomes.append(remaining) + if not remaining.ran: + problems.append(f"could not verify removal: {_detail(remaining)}") + elif remaining.stdout.strip(): + problems.append(f"{request.ref} still exists on {request.remote}") + + if number is not None: + state = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage) + outcomes.append(state) + if not state.ok: + problems.append(f"could not confirm pull request #{number} is closed") + elif '"state":"open"' in state.stdout.replace(" ", ""): + problems.append(f"pull request #{number} is still open") + + if problems: + return _result( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + FAIL, + "; ".join(problems), + outcomes, + {"ref": request.ref, "pull_request": number}, + ) + return _result( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + PASS, + "the ephemeral ref and pull request are gone and their absence was re-checked", + outcomes, + {"ref": request.ref, "pull_request": number}, + ) diff --git a/scripts/ops/hermes_handoff_evaluators.py b/scripts/ops/hermes_handoff_evaluators.py new file mode 100644 index 00000000..72e3558c --- /dev/null +++ b/scripts/ops/hermes_handoff_evaluators.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +"""Text-shaped evaluators for the handoff acceptance harness. + +These rules cover evidence that arrives as a refusal, an exit status, or a list +of names. The ``denied`` rule is the load-bearing one: it refuses to classify on +an authorization review alone — ``kubectl auth can-i`` reports what RBAC says, +not what the API server does — so a real request must have been made and +refused before it will pass. + +Importing this module also registers the JSON-shaped rules, so a caller only has +to import one place to have the full catalog vocabulary available. +""" + +from __future__ import annotations + +import re +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_json_rules import evaluate_flux_health # noqa: F401 (registers rules) +from hermes_handoff_model import ATTEMPT, FAIL, NOT_RUN, PASS, REVIEW, CheckSpec +from hermes_handoff_rules import ( + EVALUATORS, + Evaluation, + evaluate, + evaluator, + lines, + looks_denied, + missing_steps, + step_keys, +) + +__all__ = ["EVALUATORS", "Evaluation", "evaluate", "evaluator"] + + +@evaluator("denied") +def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass only when the review says no and a real request was refused.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + attempts = step_keys(spec, ATTEMPT) + if not attempts: + return Evaluation(NOT_RUN, "deny checks require a real attempt, not only a review") + evidence: dict[str, Any] = {} + for key in step_keys(spec, REVIEW): + verdict = outcomes[key].stdout.strip().lower() + evidence[key] = verdict + if verdict != "no": + return Evaluation( + FAIL, f"authorization review {key} reported {verdict or 'nothing'}", evidence + ) + for key in attempts: + outcome = outcomes[key] + if outcome.ok: + return Evaluation(FAIL, f"attempt {key} succeeded but must be refused", evidence) + if not looks_denied(outcome): + return Evaluation( + NOT_RUN, + f"attempt {key} failed without a recognisable refusal (rc={outcome.returncode})", + evidence, + ) + evidence[key] = "refused" + return Evaluation(PASS, "review and live attempt were both refused", evidence) + + +@evaluator("allowed") +def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every required step succeeded and every review said yes.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + evidence: dict[str, Any] = {} + for step in spec.steps: + outcome = outcomes[step.key] + if step.kind == REVIEW: + verdict = outcome.stdout.strip().lower() + evidence[step.key] = verdict + if verdict != "yes": + return Evaluation( + FAIL, + f"authorization review {step.key} reported {verdict or 'nothing'}", + evidence, + ) + continue + if not outcome.ok: + return Evaluation(FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence) + evidence[step.key] = "succeeded" + return Evaluation(PASS, "required access is present", evidence) + + +def _matched(observed: set[str], patterns: tuple[str, ...]) -> set[str]: + return {name for pattern in patterns for name in observed if pattern in name} + + +def _line_step(spec: CheckSpec, outcomes: dict[str, Outcome]) -> tuple[set[str], str] | Evaluation: + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + return (set(lines(outcome)), key) + + +@evaluator("names_absent") +def evaluate_names_absent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when no forbidden name or substring appears in a step's lines.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + resolved = _line_step(spec, outcomes) + if isinstance(resolved, Evaluation): + return resolved + observed, _ = resolved + offenders = sorted( + (observed & set(spec.expect.get("names", ()))) + | _matched(observed, tuple(spec.expect.get("contains", ()))) + ) + evidence = {"observed_count": len(observed), "offenders": offenders} + if offenders: + return Evaluation(FAIL, f"forbidden names present: {', '.join(offenders)}", evidence) + return Evaluation(PASS, "no forbidden names are present", evidence) + + +@evaluator("names_present") +def evaluate_names_present(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every required name, and every required substring, appears.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + resolved = _line_step(spec, outcomes) + if isinstance(resolved, Evaluation): + return resolved + observed, _ = resolved + missing = sorted(set(spec.expect.get("names", ())) - observed) + missing += sorted( + pattern + for pattern in spec.expect.get("contains", ()) + if not any(pattern in name for name in observed) + ) + evidence = {"observed_count": len(observed), "missing": missing} + if missing: + return Evaluation(FAIL, f"required names absent: {', '.join(missing)}", evidence) + if not observed: + return Evaluation(NOT_RUN, "step produced no lines to examine", evidence) + return Evaluation(PASS, "every required name is present", evidence) + + +@evaluator("stdout_matches") +def evaluate_stdout_matches(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Compare a step's trimmed stdout against an expected value or set.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + actual = outcome.stdout.strip() + expected = spec.expect["equals"] + allowed = expected if isinstance(expected, list) else [expected] + evidence = {"observed": actual} + if actual not in allowed: + return Evaluation(FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence) + return Evaluation(PASS, "output matches the expected value", evidence) + + +@evaluator("distinct_count") +def evaluate_distinct_count(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when a step yields enough distinct rows — and, optionally, exactly N. + + Distinctness is the point for placement checks: three worker pods pinned to + one node satisfy a replica count and defeat the isolation the replica count + was standing in for. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + values = lines(outcome) + distinct = sorted(set(values)) + minimum = spec.expect["minimum"] + evidence = {"total": len(values), "distinct": len(distinct), "values": distinct[:16]} + if not values: + return Evaluation(NOT_RUN, f"step {key} produced no rows to count", evidence) + if len(distinct) < minimum: + return Evaluation( + FAIL, f"{len(distinct)} distinct values, expected at least {minimum}", evidence + ) + total_equals = spec.expect.get("total_equals") + if total_equals is not None and len(values) != total_equals: + return Evaluation(FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence) + return Evaluation(PASS, f"{len(distinct)} distinct values observed", evidence) + + +@evaluator("lines_match") +def evaluate_lines_match(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every data row of a tabular listing matches a shape. + + Some surfaces only speak tables. A blank leading column is exactly what the + session-render regression looked like, so asserting the shape of each row is + a real check rather than a proxy for one. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + pattern = re.compile(spec.expect["pattern"]) + rows = outcome.stdout.splitlines()[spec.expect.get("skip", 0) :] + rows = [row for row in rows if row.strip()] + offenders = [row[:60] for row in rows if not pattern.search(row)] + evidence = {"rows": len(rows), "offenders": offenders[:8]} + if len(rows) < spec.expect.get("minimum", 1): + return Evaluation( + NOT_RUN, f"step {key} produced {len(rows)} data rows, expected at least " + f"{spec.expect.get('minimum', 1)}", evidence + ) + if offenders: + return Evaluation(FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence) + return Evaluation(PASS, f"all {len(rows)} rows match the expected shape", evidence) + + +@evaluator("vantages_agree") +def evaluate_vantages_agree(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass only when two independent observations report the same fact. + + A disagreement is a finding in its own right: it usually means the manifest + on record and the workload actually running have drifted apart, and either + one alone would have looked clean. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + left_key, right_key = spec.expect["steps"] + observations: dict[str, list[str]] = {} + for key in (left_key, right_key): + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + observations[key] = sorted(lines(outcome)) + evidence = {key: value[:16] for key, value in observations.items()} + if observations[left_key] != observations[right_key]: + return Evaluation(FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence) + if not observations[left_key]: + return Evaluation(NOT_RUN, "both vantages returned nothing to compare", evidence) + return Evaluation(PASS, "both vantages report the same state", evidence) + + +@evaluator("not_armed") +def evaluate_not_armed(spec: CheckSpec, _outcomes: dict[str, Outcome]) -> Evaluation: + """Report the fixed NOT_RUN a default, read-only run must produce.""" + return Evaluation( + NOT_RUN, + spec.expect.get("reason", "ephemeral mutation mode is not armed"), + {"armed": False}, + ) + + +# Two observations of the same fact agree whether or not they came from +# different vantages; the dual-vantage case is just the important one. +EVALUATORS["steps_agree"] = evaluate_vantages_agree diff --git a/scripts/ops/hermes_handoff_exec.py b/scripts/ops/hermes_handoff_exec.py new file mode 100644 index 00000000..200d82e0 --- /dev/null +++ b/scripts/ops/hermes_handoff_exec.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +"""Bounded, screened command execution for the handoff acceptance harness. + +Every probe runs through :class:`Runner`, which owns the three budgets a +long-running acceptance sweep needs: a per-command timeout, a whole-run +deadline, and a per-capture byte budget. Exhausting the deadline does not +silently shorten the sweep — the remaining probes report ``NOT_RUN``, which is a +NO_GO, so a slow cluster can never be mistaken for a clean one. + +Subprocesses never inherit the caller's environment. A probe that echoed +`$ANTHROPIC_API_KEY` would otherwise be one typo away from a leak, so the child +environment is rebuilt from an allowlist of names whose values the harness +already needs to reach the cluster and the forge. +""" + +from __future__ import annotations + +import os +import shlex +import subprocess +import time +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field + +from hermes_handoff_policy import READ_ONLY, PolicyError, check_argv +from hermes_handoff_redaction import DEFAULT_MAX_BYTES, redact, safe_text + +DEFAULT_COMMAND_TIMEOUT = 25.0 +DEFAULT_RUN_DEADLINE = 900.0 + +# Names whose values the harness needs to reach the cluster and the forge. The +# values are handed to the child process and are never recorded. +ENV_PASSTHROUGH = ( + "GIT_ASKPASS", + "GITEA_BASE_URL", + "HERMES_HOME", + "HOME", + "JENKINS_BASE_URL", + "KUBECONFIG", + "LANG", + "PATH", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_FILE", + "VAULT_ADDR", +) + +DEADLINE_ERROR = "deadline-exceeded" +TIMEOUT_ERROR = "timeout" + +OPERATOR = "operator" +SELF = "self" + +# Tools that accept a `--context` flag directly after the binary name. +CONTEXT_AWARE_BINARIES = ("kubectl", "flux", "helm") + + +@dataclass(frozen=True) +class Outcome: + """The screened, bounded result of one probe command.""" + + argv: tuple[str, ...] + vantage: str + returncode: int | None = None + stdout: str = "" + stderr: str = "" + truncated: bool = False + duration_ms: int = 0 + error: str | None = None + + @property + def ok(self) -> bool: + """Report whether the command ran to completion and succeeded.""" + return self.error is None and self.returncode == 0 + + @property + def ran(self) -> bool: + """Report whether the command reached the target at all.""" + return self.error is None + + @property + def combined(self) -> str: + """Return stdout and stderr joined, for message matching.""" + return "\n".join(part for part in (self.stdout, self.stderr) if part) + + def as_dict(self) -> dict[str, object]: + """Return a JSON-serialisable view of the outcome.""" + return { + "command": redact(shlex.join(self.argv)), + "vantage": self.vantage, + "returncode": self.returncode, + "stdout": self.stdout, + "stderr": self.stderr, + "truncated": self.truncated, + "duration_ms": self.duration_ms, + "error": self.error, + } + + +@dataclass(frozen=True) +class Vantage: + """A named identity and transport a probe can be observed from. + + ``prefix`` is how the in-pod vantage is reached: a ``kubectl exec`` into the + target container, so the inner command authenticates as the pod's own + service account instead of as the operator. ``inserts`` carries per-binary + flags — the operator's `--context`, which must not follow the command into + the pod. + """ + + name: str + prefix: tuple[str, ...] = () + inserts: Mapping[str, tuple[str, ...]] = field(default_factory=dict) + env: Mapping[str, str] = field(default_factory=dict) + description: str = "" + + def wrap(self, argv: Sequence[str]) -> tuple[str, ...]: + """Return ``argv`` addressed to this vantage.""" + command = tuple(argv) + if command: + extra = self.inserts.get(command[0].rsplit("/", 1)[-1]) + if extra: + command = (command[0], *extra, *command[1:]) + return (*self.prefix, *command) + + +def build_environment(vantage: Vantage, source: Mapping[str, str] | None = None) -> dict[str, str]: + """Return the child environment: allow-listed names plus vantage overrides.""" + origin = os.environ if source is None else source + environment = {name: origin[name] for name in ENV_PASSTHROUGH if name in origin} + environment.setdefault("LC_ALL", "C") + environment.update(vantage.env) + return environment + + +def operator_vantage(kubeconfig: str | None = None, context: str | None = None) -> Vantage: + """Return the external read-only operator vantage.""" + inserts = ( + {binary: ("--context", context) for binary in CONTEXT_AWARE_BINARIES} if context else {} + ) + detail = " ".join( + part for part in (f"context={context}" if context else "", "kubeconfig-pinned" if kubeconfig else "") if part + ) + return Vantage( + name=OPERATOR, + inserts=inserts, + env={"KUBECONFIG": kubeconfig} if kubeconfig else {}, + description=f"operator {detail}".strip(), + ) + + +def pod_vantage( + namespace: str, + pod: str, + container: str, + operator: Vantage | None = None, +) -> Vantage: + """Return the in-pod vantage: the inner command authenticates as the pod.""" + reach = operator.inserts.get("kubectl", ()) if operator else () + return Vantage( + name=SELF, + prefix=( + "kubectl", + *reach, + "exec", + "--namespace", + namespace, + pod, + "--container", + container, + "--", + ), + env=dict(operator.env) if operator else {}, + description=f"in-pod {namespace}/{pod}[{container}]", + ) + + +class Runner: + """Run probe commands under policy, timeout, deadline, and byte budgets.""" + + def __init__( + self, + mode: str = READ_ONLY, + command_timeout: float = DEFAULT_COMMAND_TIMEOUT, + deadline_seconds: float = DEFAULT_RUN_DEADLINE, + max_bytes: int = DEFAULT_MAX_BYTES, + clock: Callable[[], float] = time.monotonic, + spawn: Callable[..., subprocess.CompletedProcess] = subprocess.run, + environ: Mapping[str, str] | None = None, + ) -> None: + self.mode = mode + self.command_timeout = command_timeout + self.max_bytes = max_bytes + self._clock = clock + self._spawn = spawn + self._environ = environ + self._deadline_at = clock() + deadline_seconds + self.commands_run = 0 + + @property + def remaining_seconds(self) -> float: + """Return the seconds left before the whole-run deadline.""" + return self._deadline_at - self._clock() + + def run( + self, argv: Sequence[str], vantage: Vantage, max_bytes: int | None = None + ) -> Outcome: + """Run one command, returning a screened outcome and never raising.""" + budget_bytes = self.max_bytes if max_bytes is None else max_bytes + addressed = vantage.wrap(argv) + try: + check_argv(addressed, self.mode) + except PolicyError as exc: + return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}") + + budget = min(self.command_timeout, self.remaining_seconds) + if budget <= 0: + return Outcome(argv=addressed, vantage=vantage.name, error=DEADLINE_ERROR) + + started = self._clock() + self.commands_run += 1 + try: + completed = self._spawn( + list(addressed), + capture_output=True, + text=True, + timeout=budget, + check=False, + env=build_environment(vantage, self._environ), + ) + except subprocess.TimeoutExpired: + return Outcome( + argv=addressed, + vantage=vantage.name, + error=TIMEOUT_ERROR, + duration_ms=int((self._clock() - started) * 1000), + ) + except OSError as exc: + return Outcome( + argv=addressed, vantage=vantage.name, error=f"spawn: {exc.strerror or exc}" + ) + + stdout, cut_out = safe_text(completed.stdout or "", budget_bytes) + stderr, cut_err = safe_text(completed.stderr or "", budget_bytes) + return Outcome( + argv=addressed, + vantage=vantage.name, + returncode=completed.returncode, + stdout=stdout, + stderr=stderr, + truncated=cut_out or cut_err, + duration_ms=int((self._clock() - started) * 1000), + ) diff --git a/scripts/ops/hermes_handoff_json_rules.py b/scripts/ops/hermes_handoff_json_rules.py new file mode 100644 index 00000000..6c10e52d --- /dev/null +++ b/scripts/ops/hermes_handoff_json_rules.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""JSON-shaped evaluators for the handoff acceptance harness. + +These rules cover the evidence that arrives as structured output — Kubernetes +objects, forge pull requests, provider health files, and the two append-only +evidence logs the platform keeps. They share the fail-closed contract of every +other rule: an absent path, an unparsable payload, or an empty result set is +``NOT_RUN``, never a quiet pass. +""" + +from __future__ import annotations + +import datetime as dt +import json +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_model import FAIL, NOT_RUN, PASS, CheckSpec +from hermes_handoff_rules import ( + Evaluation, + dotted, + evaluator, + is_list, + missing_steps, + parsed_step, +) + + +@evaluator("json_field") +def evaluate_json_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Compare dotted JSON paths of one step's output against expected values.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + evidence: dict[str, Any] = {} + mismatches: list[str] = [] + for path, expected in spec.expect["fields"].items(): + try: + actual = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + evidence[path] = actual + allowed = expected if isinstance(expected, list) else [expected] + if actual not in allowed: + mismatches.append(f"{path}={actual!r} expected one of {allowed!r}") + if mismatches: + return Evaluation(FAIL, "; ".join(mismatches), evidence) + return Evaluation(PASS, "observed fields match the release contract", evidence) + + +@evaluator("json_numeric") +def evaluate_json_numeric(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Compare numeric JSON paths against inclusive minimum/maximum bounds.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + evidence: dict[str, Any] = {} + problems: list[str] = [] + for path, bounds in spec.expect["fields"].items(): + try: + actual = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + if not isinstance(actual, (int, float)) or isinstance(actual, bool): + return Evaluation(NOT_RUN, f"path {path} is not numeric in step {key}", evidence) + evidence[path] = actual + if "min" in bounds and actual < bounds["min"]: + problems.append(f"{path}={actual} below minimum {bounds['min']}") + if "max" in bounds and actual > bounds["max"]: + problems.append(f"{path}={actual} above maximum {bounds['max']}") + if problems: + return Evaluation(FAIL, "; ".join(problems), evidence) + return Evaluation(PASS, "numeric fields are within the expected bounds", evidence) + + +@evaluator("json_recent") +def evaluate_json_recent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when a timestamp field is fresh enough to be evidence about now. + + Stale provider or routing evidence is worse than none: it describes a state + the release no longer has. A timestamp older than the allowed age is a FAIL, + and one the harness cannot parse is NOT_RUN. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + now = spec.expect["now"] + evidence: dict[str, Any] = {} + stale: list[str] = [] + for path, max_age in spec.expect["fields"].items(): + try: + raw = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + try: + observed = dt.datetime.fromisoformat(str(raw).replace("Z", "+00:00")) + except ValueError: + return Evaluation(NOT_RUN, f"path {path} is not an ISO timestamp", evidence) + if observed.tzinfo is None: + observed = observed.replace(tzinfo=dt.timezone.utc) + age = (now - observed).total_seconds() + evidence[path] = {"age_seconds": int(age)} + if age > max_age: + stale.append(f"{path} is {int(age)}s old, limit {max_age}s") + if stale: + return Evaluation(FAIL, "; ".join(stale), evidence) + return Evaluation(PASS, "observed timestamps are fresh", evidence) + + +@evaluator("all_of_field") +def evaluate_all_of_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every selected element of a JSON array matches expected fields.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + try: + elements = dotted(payload, spec.expect.get("array", "")) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"array path is absent from step {key}") + if not is_list(elements): + return Evaluation(NOT_RUN, f"step {key} did not return an array") + key_field = spec.expect.get("key_field", "") + wanted = spec.expect.get("keys") + exempt = set(spec.expect.get("exempt_keys", ())) + seen: list[Any] = [] + problems: list[str] = [] + evidence: dict[str, Any] = {} + for element in elements: + identity = dotted(element, key_field) if key_field else None + if wanted is not None and identity not in wanted: + continue + if identity in exempt: + continue + seen.append(identity) + for path in spec.expect.get("non_empty", ()): + try: + value = dotted(element, path) + except (KeyError, ValueError, IndexError): + value = None + if value in (None, "", [], {}): + problems.append(f"{identity}: {path} is empty") + for path, expected in spec.expect.get("fields", {}).items(): + try: + actual = dotted(element, path) + except (KeyError, ValueError, IndexError): + actual = None + allowed = expected if isinstance(expected, list) else [expected] + if actual not in allowed: + problems.append(f"{identity}: {path}={actual!r} expected one of {allowed!r}") + evidence["examined"] = seen[:32] + if wanted is not None: + absent = [item for item in wanted if item not in seen] + if absent: + return Evaluation(NOT_RUN, f"expected keys absent from step {key}: {absent}", evidence) + elif not seen: + return Evaluation(NOT_RUN, f"step {key} returned nothing to examine", evidence) + if problems: + return Evaluation(FAIL, "; ".join(problems[:8]), evidence) + return Evaluation(PASS, f"{len(seen)} element(s) match the release contract", evidence) + + +@evaluator("json_record") +def evaluate_json_record(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Capture named JSON paths as evidence; pass when all of them resolve. + + Recording is a release requirement in its own right — the runbook has to + name the exact heads it was verified against — so an unreachable source is a + NOT_RUN rather than an empty section. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + evidence: dict[str, Any] = {} + for label, path in spec.expect["record"].items(): + try: + evidence[label] = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + return Evaluation(PASS, "state recorded", evidence) + + + +@evaluator("routing_evidence") +def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Summarise a Switchyard routing log tail into provider/effort coverage. + + Route identifiers are ``///``, so provider, + model family, effort tier, and lane all come from the same field. Fallback + and failure evidence is a separate ``fallback_reason`` on the record that + was retried. The parser tolerates a clipped first line: a tail of a growing + file starts mid-record, and discarding one row is better than a NOT_RUN that + reads as an outage. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + records = [] + for line in outcome.stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + records.append(json.loads(line)) + except ValueError: + continue + routes = [ + str(record.get("model") or "").split("/") + for record in records + if str(record.get("model") or "").count("/") == 3 + ] + fallbacks = [record for record in records if record.get("fallback_reason")] + stamps = sorted(str(record.get("ts")) for record in records if record.get("ts")) + evidence: dict[str, Any] = { + "records": len(records), + "routed": len(routes), + "lanes": sorted({route[0] for route in routes}), + "providers": sorted({route[1] for route in routes}), + "families": sorted({route[2] for route in routes}), + "efforts": sorted({route[3] for route in routes}), + "fallback_events": len(fallbacks), + "fallback_reasons": sorted({str(record["fallback_reason"]) for record in fallbacks}), + "latest_ts": stamps[-1] if stamps else "", + } + if not records: + return Evaluation(NOT_RUN, "routing log tail contained no parsable records", evidence) + missing = [ + f"{label}:{item}" + for label, wanted in ( + ("provider", spec.expect.get("providers", ())), + ("effort", spec.expect.get("efforts", ())), + ("lane", spec.expect.get("lanes", ())), + ) + for item in wanted + if item not in evidence[f"{label}s"] + ] + if spec.expect.get("require_fallback_evidence") and not fallbacks: + missing.append("fallback:none-observed") + max_age = spec.expect.get("max_age_seconds") + now = spec.expect.get("now") + if max_age and now: + try: + newest = dt.datetime.fromisoformat(evidence["latest_ts"].replace("Z", "+00:00")) + except ValueError: + return Evaluation(NOT_RUN, "routing log has no parsable newest timestamp", evidence) + age = (now - newest).total_seconds() + evidence["age_seconds"] = int(age) + if age > max_age: + missing.append(f"freshness:{int(age)}s-exceeds-{max_age}s") + if missing: + return Evaluation(FAIL, f"routing evidence missing {', '.join(missing)}", evidence) + return Evaluation(PASS, "routing evidence covers the required lanes", evidence) + + +@evaluator("flux_health") +def evaluate_flux_health(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Classify a compact Flux projection into suspensions and unhealthy objects. + + Each row is ``namespace/namesuspendType=Status,...``. An object is + unhealthy when it reports ``Healthy=False``, or ``Ready=False`` with no + Healthy condition and no in-flight ``Reconciling=True``. That last clause + matters: Flux flips Ready to False for the duration of a reconcile, so + without it the verdict depends on when the snapshot was taken. Suspended + objects are excluded from the health tally and judged against the + expected-suspension list instead, because a deliberate park is a different + fact from a broken reconcile. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + expected = set(spec.expect.get("expected_suspensions", ())) + suspended: list[str] = [] + unhealthy: list[str] = [] + total = 0 + for line in outcome.stdout.splitlines(): + if not line.strip(): + continue + total += 1 + name, _, rest = line.partition("\t") + suspend, _, conditions = rest.partition("\t") + if suspend.strip().lower() == "true": + suspended.append(name.strip()) + continue + states = dict( + pair.split("=", 1) for pair in conditions.strip().strip(",").split(",") if "=" in pair + ) + healthy = states.get("Healthy") + reconciling = states.get("Reconciling") == "True" + ready_failed = states.get("Ready") == "False" + if healthy == "False" or (healthy is None and ready_failed and not reconciling): + unhealthy.append(name.strip()) + unexpected = sorted(set(suspended) - expected) + evidence = { + "total": total, + "suspended": sorted(suspended), + "unexpected_suspensions": unexpected, + "unhealthy": sorted(unhealthy), + } + if not total: + return Evaluation(NOT_RUN, f"step {key} returned no Flux objects", evidence) + problems = [] + if unexpected: + problems.append(f"unexpected suspensions: {', '.join(unexpected)}") + if unhealthy: + problems.append(f"unhealthy: {', '.join(sorted(unhealthy)[:8])}") + if problems: + return Evaluation(FAIL, "; ".join(problems), evidence) + return Evaluation(PASS, f"{total} Flux objects reconciled as expected", evidence) diff --git a/scripts/ops/hermes_handoff_model.py b/scripts/ops/hermes_handoff_model.py new file mode 100644 index 00000000..d34b9ebc --- /dev/null +++ b/scripts/ops/hermes_handoff_model.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""Result model, aggregation, and rendering for the handoff acceptance harness. + +The classification is deliberately four-valued. ``NOT_APPLICABLE`` is the only +way a check can be excused, and a check can only reach it by matching a scope +rule that a reviewer wrote down; anything the harness merely failed to observe +is ``NOT_RUN``. Because a mandatory ``NOT_RUN`` is a NO_GO exactly like a +``FAIL``, an unreachable cluster, an exhausted deadline, or a probe nobody +maintained can never round up to a release. +""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_redaction import contains_credential_shape, scrub + +PASS = "PASS" +FAIL = "FAIL" +NOT_RUN = "NOT_RUN" +NOT_APPLICABLE = "NOT_APPLICABLE" +STATUSES = (PASS, FAIL, NOT_RUN, NOT_APPLICABLE) + +GO = "GO" +NO_GO = "NO_GO" + +ALWAYS = "always" +EPHEMERAL = "ephemeral" + +HARNESS = "hermes-handoff-acceptance" +SCHEMA_VERSION = 1 + + +REVIEW = "review" +ATTEMPT = "attempt" +READ = "read" +STEP_KINDS = (REVIEW, ATTEMPT, READ) + + +@dataclass(frozen=True) +class Step: + """One command a check needs, and the vantage it must be observed from. + + ``kind`` separates a self access review (``review``) from a real request the + identity is expected to be refused (``attempt``). A deny check needs both: + an authorization review states policy, only an attempt proves the API server + enforces it. + + ``record`` is cleared for bulk evidence — a routing log tail, a large object + listing — which the harness parses but must not paste into the artifact. + ``max_bytes`` raises the capture budget for that same bulk evidence, where a + default-sized truncation would silently narrow the window being examined. + """ + + key: str + vantage: str + argv: tuple[str, ...] + kind: str = READ + optional: bool = False + record: bool = True + max_bytes: int | None = None + + +@dataclass(frozen=True) +class CheckSpec: + """A declarative acceptance check. + + ``rule`` names an evaluator; ``expect`` carries its parameters. Keeping the + catalog declarative means a reviewer reads what is asserted, not how it is + plumbed, and the harness can prove structural properties over every check + (no impersonation, no mutation, both vantages present) before running any. + """ + + id: str + title: str + group: str + rule: str + steps: tuple[Step, ...] = () + expect: Mapping[str, Any] = field(default_factory=dict) + mandatory: bool = True + scope: str = ALWAYS + rationale: str = "" + + @property + def vantages(self) -> tuple[str, ...]: + """Return the distinct vantages this check draws evidence from.""" + seen: list[str] = [] + for step in self.steps: + if step.vantage not in seen: + seen.append(step.vantage) + return tuple(seen) + + +@dataclass +class CheckResult: + """The classified outcome of one acceptance check.""" + + spec: CheckSpec + status: str + reason: str = "" + evidence: dict[str, Any] = field(default_factory=dict) + outcomes: list[Outcome] = field(default_factory=list) + + @property + def blocking(self) -> bool: + """Report whether this result alone forces NO_GO.""" + return self.spec.mandatory and self.status in {FAIL, NOT_RUN} + + def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: + """Return a JSON-serialisable, screened view of the result.""" + payload: dict[str, Any] = { + "id": self.spec.id, + "title": self.spec.title, + "group": self.spec.group, + "rule": self.spec.rule, + "scope": self.spec.scope, + "mandatory": self.spec.mandatory, + "vantages": list(self.spec.vantages), + "status": self.status, + "reason": self.reason, + "blocking": self.blocking, + "evidence": self.evidence, + } + if self.spec.rationale: + payload["rationale"] = self.spec.rationale + if include_outcomes: + payload["commands"] = [outcome.as_dict() for outcome in self.outcomes] + return scrub(payload) + + +@dataclass +class VantageRecord: + """What the harness proved about one vantage before trusting its evidence.""" + + name: str + description: str + identity: str = "" + available: bool = False + detail: str = "" + + def as_dict(self) -> dict[str, Any]: + """Return a JSON-serialisable view of the vantage.""" + return scrub( + { + "name": self.name, + "description": self.description, + "identity": self.identity, + "available": self.available, + "detail": self.detail, + } + ) + + +@dataclass +class Report: + """The whole acceptance run: metadata, per-check results, and a verdict.""" + + mode: str + started_at: str + baseline: dict[str, Any] = field(default_factory=dict) + vantages: list[VantageRecord] = field(default_factory=list) + results: list[CheckResult] = field(default_factory=list) + harness_errors: list[str] = field(default_factory=list) + finished_at: str = "" + + @property + def counts(self) -> dict[str, int]: + """Return the number of results in each status.""" + tally = dict.fromkeys(STATUSES, 0) + for result in self.results: + tally[result.status] = tally.get(result.status, 0) + 1 + return tally + + @property + def blocking(self) -> list[CheckResult]: + """Return every result that forces NO_GO, most severe first.""" + order = {FAIL: 0, NOT_RUN: 1} + return sorted( + (result for result in self.results if result.blocking), + key=lambda result: (order.get(result.status, 2), result.spec.id), + ) + + @property + def decision(self) -> str: + """Return GO only when nothing mandatory failed, skipped, or errored.""" + if self.harness_errors or self.blocking: + return NO_GO + return GO + + def group_summary(self) -> dict[str, dict[str, int]]: + """Return per-group status counts, for the human summary.""" + groups: dict[str, dict[str, int]] = {} + for result in self.results: + bucket = groups.setdefault(result.spec.group, dict.fromkeys(STATUSES, 0)) + bucket[result.status] = bucket.get(result.status, 0) + 1 + return groups + + def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: + """Return the machine-readable report.""" + return { + "harness": HARNESS, + "schema_version": SCHEMA_VERSION, + "mode": self.mode, + "started_at": self.started_at, + "finished_at": self.finished_at, + "decision": self.decision, + "counts": self.counts, + "baseline": scrub(self.baseline), + "vantages": [vantage.as_dict() for vantage in self.vantages], + "harness_errors": [scrub(error) for error in self.harness_errors], + "blocking": [result.spec.id for result in self.blocking], + "checks": [result.as_dict(include_outcomes) for result in self.results], + } + + def render_summary(self, width: int = 88) -> str: + """Return the concise human summary printed alongside the JSON.""" + counts = self.counts + lines = [ + f"{HARNESS}: {self.decision}", + f" mode={self.mode} started={self.started_at} checks={len(self.results)}", + " " + " ".join(f"{status}={counts[status]}" for status in STATUSES), + ] + for vantage in self.vantages: + state = "available" if vantage.available else "unavailable" + identity = vantage.identity or "unknown" + lines.append(f" vantage {vantage.name}: {state} identity={identity}") + for error in self.harness_errors: + lines.append(f" harness error: {error}") + if self.blocking: + lines.append(" blocking:") + for result in self.blocking: + lines.append(f" {result.status:<15}{result.spec.id}") + lines.append(f" {'':<15}{result.reason[:width]}") + groups = self.group_summary() + lines.append(" groups:") + for name in sorted(groups): + bucket = groups[name] + detail = " ".join(f"{status}={bucket[status]}" for status in STATUSES if bucket[status]) + lines.append(f" {name:<18}{detail}") + return "\n".join(lines) + + +def utc_now(clock: Any = None) -> str: + """Return an ISO-8601 UTC timestamp, injectable for deterministic tests.""" + now = clock() if clock else dt.datetime.now(dt.timezone.utc) + return now.replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def unscreened_fields(payload: Any, path: str = "$") -> list[str]: + """Return JSON pointers whose text still looks credential-shaped. + + The report is already screened field by field; this is the invariant check + that runs before anything is written, so a screening gap fails the run + instead of publishing an artifact nobody re-read. + """ + offenders: list[str] = [] + if isinstance(payload, str): + if contains_credential_shape(payload): + offenders.append(path) + elif isinstance(payload, Mapping): + for key, value in payload.items(): + offenders.extend(unscreened_fields(value, f"{path}.{key}")) + elif isinstance(payload, Sequence) and not isinstance(payload, (str, bytes)): + for index, value in enumerate(payload): + offenders.extend(unscreened_fields(value, f"{path}[{index}]")) + return offenders diff --git a/scripts/ops/hermes_handoff_policy.py b/scripts/ops/hermes_handoff_policy.py new file mode 100644 index 00000000..473dd782 --- /dev/null +++ b/scripts/ops/hermes_handoff_policy.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +"""Fail-closed command policy for the Hermes handoff acceptance harness. + +Two properties have to hold structurally, not by convention: + +* the default run mutates nothing, and +* no probe can pull a credential value into the report. + +Both are enforced here, where an argv becomes a subprocess, so a mistake in the +check catalog surfaces as a policy error when the catalog is built rather than +as a mutation or a leak when it runs. `--as` is rejected on the operator side: +impersonation is not evidence about what Hermes can do, and a harness able to +impersonate would be tempted to substitute it for a real self-probe. It stays +available to the inner command of a `kubectl exec`, where an identity is +probing its own refusal to impersonate rather than borrowing authority. + +Shell is reachable only through frozen templates. Free-form `sh -c` cannot be +screened reliably, but a few probes genuinely need field extraction (report that +a password field is locked without ever reading its hash) or an explicit helper +path, so those scripts are named, reviewed here, and parameter-validated instead +of assembled by callers. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator + +READ_ONLY = "read-only" +ARMED = "ephemeral-armed" +MODES = (READ_ONLY, ARMED) + +# Rejected everywhere. Impersonation and inline credentials have no honest use +# in an acceptance probe. +FORBIDDEN_ARGS = ( + "--as", + "--as-group", + "--as-uid", + "--token", + "--password", + "--username", + "--client-key", + "--client-certificate", +) +# Reachable only from inside a pod, where an identity is probing its own +# refusal to impersonate rather than borrowing someone else's authority. +IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid") +FORBIDDEN_BINARIES = frozenset({"vault", "sops", "age", "gpg", "openssl", "curl", "wget"}) +GITEA_CLIENT = "/opt/coordinator/gitea_api.py" +ALLOWED_BINARIES = frozenset({"kubectl", "flux", "helm", "git", "hermes", "sh", GITEA_CLIENT}) + +# Flags that consume the following token, so it is never a subcommand. +VALUE_FLAGS = frozenset( + { + "-c", + "-l", + "-n", + "-o", + "--cluster", + "--container", + "--context", + "--field-selector", + "--kubeconfig", + "--namespace", + "--output", + "--request-timeout", + "--selector", + "--server", + "--since", + "--tail", + "--user", + } +) + +KUBECTL_READ_SUBCOMMANDS = frozenset( + { + "api-resources", + "api-versions", + "auth", + "cluster-info", + "config", + "describe", + "events", + "explain", + "get", + "logs", + "top", + "version", + } +) +# Reachable only through a dry run: the API server authorises the request and +# discards it. A rejection is the evidence, and nothing is persisted either way. +KUBECTL_DRY_RUN_SUBCOMMANDS = frozenset( + {"annotate", "apply", "create", "delete", "label", "patch", "replace", "scale", "set"} +) +KUBECTL_EXEC_SUBCOMMANDS = frozenset({"exec"}) +KUBECTL_SUBCOMMANDS = ( + KUBECTL_READ_SUBCOMMANDS | KUBECTL_DRY_RUN_SUBCOMMANDS | KUBECTL_EXEC_SUBCOMMANDS +) +DRY_RUN_FLAGS = ("--dry-run=server", "--dry-run=client") + +# Resources whose objects carry credential material. Reading them is permitted +# only in name-only output form, so a regression that grants access still cannot +# deliver a secret value into a report. +NAME_ONLY_RESOURCES = frozenset({"secret", "secrets"}) + +FLUX_READ_SUBCOMMANDS = frozenset( + {"check", "diff", "events", "export", "get", "stats", "trace", "tree", "version"} +) +HELM_READ_SUBCOMMANDS = frozenset({"get", "history", "list", "status", "version"}) +GIT_READ_SUBCOMMANDS = frozenset( + { + "cat-file", + "config", + "diff", + "fetch", + "for-each-ref", + "log", + "ls-remote", + "ls-tree", + "merge-base", + "remote", + "rev-list", + "rev-parse", + "show", + "status", + } +) +GIT_ARMED_SUBCOMMANDS = frozenset({"push"}) +GIT_FORCE_FLAGS = frozenset({"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"}) +GITEA_READ_METHODS = frozenset({"GET"}) +GITEA_ARMED_METHODS = frozenset({"POST", "PATCH", "DELETE"}) + +# The Hermes CLI is the only honest source for durable session and Kanban +# activity, so a narrow read-only slice of it is on the allowlist. +HERMES_READ_COMMANDS = frozenset({("kanban", "list"), ("kanban", "show"), ("sessions", "list"), ("status",)}) + +# Paths that hold credential material. A probe may prove one exists or is +# unreadable; it may never read it outside a frozen template. +SECRET_PATH_RE = re.compile( + r"(?:^|/)(?:vault/secrets|var/run/secrets|runtime-access|\.credentials\.json" + r"|auth\.json|authorized_keys|id_[a-z0-9]+|\.netrc|shadow|gshadow)(?:$|/)" +) + +_PARAMETER_RE = re.compile(r"[A-Za-z0-9/][A-Za-z0-9._/@:,-]{0,255}\Z") +_PLACEHOLDER_RE = re.compile(r"\{([a-z_]+)\}") + +# Frozen shell templates. Each exists because a plain argv cannot express the +# safety property its probe needs. +SHELL_TEMPLATES: dict[str, str] = { + # Report an account's lock state without reading its password hash. + "account_lock_state": ( + 'awk -F: \'$1=="{account}"{{print ($2=="!"||$2=="*"||$2=="!!")?"locked":"unlocked"}}\' {path}' + ), + # Report the account record's non-secret identity fields. + "account_identity": 'awk -F: \'$1=="{account}"{{print $3":"$4":"$6":"$7}}\' {path}', + # Report supplementary group names the account joins, never their members. + "account_groups": "awk -F: '$4~/(^|,){account}(,|$)/{{print $1}}' {path}", + # Prove a path's presence and mode without reading its bytes. + "path_mode": 'if [ -e {path} ]; then stat -c "%a %U:%G" {path}; else echo absent; fi', + # Prove whether a path is readable by this identity, without printing it. + "path_readable": 'if [ -r {path} ]; then echo readable; else echo unreadable; fi', + # List environment variable names only. Values never leave the container. + "env_names": "env | sed -n 's/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/\\1/p' | sort", + # Report a JSON document's top-level key names without any of its values. + "json_keys": ( + 'python3 -c "import json,sys;print(chr(10).join(sorted(json.load(open(sys.argv[1])))))" {path}' + ), + # Report allow-listed non-secret status fields from a provider health file. + "json_fields": ( + 'python3 -c "import json,sys;d=json.load(open(sys.argv[1]));' + 'print(json.dumps({{k:d.get(k) for k in sys.argv[2].split(chr(44))}}))" {path} {fields}' + ), + # Count non-empty lines, for authorized-key inventories. + "line_count": "awk 'NF{{n++}} END{{print n+0}}' {path}", + # Tail an append-only evidence log. The harness parses it; the container is + # not assumed to carry an interpreter. + "tail_lines": "tail -n {limit} {path}", + # Prove the worker can still read the repository. `kubectl exec` does not + # inherit the coordinator process environment, so the askpass helper is + # named explicitly rather than assumed; the helper reads runtime state and + # the credential never reaches this command line. + "git_ls_remote": "GIT_ASKPASS={askpass} git ls-remote {url} refs/heads/main", +} + + +class PolicyError(ValueError): + """Raised when a requested command falls outside the safe boundary.""" + + +def template_parameters(template_name: str) -> set[str]: + """Return the parameter names a frozen template requires.""" + template = SHELL_TEMPLATES.get(template_name) + if template is None: + raise PolicyError(f"unknown shell template: {template_name}") + return set(_PLACEHOLDER_RE.findall(template)) + + +def render_shell(template_name: str, **parameters: str) -> str: + """Return a frozen shell template with validated parameters substituted.""" + required = template_parameters(template_name) + template = SHELL_TEMPLATES[template_name] + if set(parameters) != required: + raise PolicyError(f"shell template {template_name} expects parameters {sorted(required)}") + for key, value in parameters.items(): + if not _PARAMETER_RE.fullmatch(value): + raise PolicyError(f"unsafe shell parameter for {template_name}.{key}") + return template.format(**parameters) + + +_RENDERED: set[str] = set() + + +def shell(template_name: str, **parameters: str) -> tuple[str, ...]: + """Return a validated ``sh -c`` argv built from a frozen template.""" + script = render_shell(template_name, **parameters) + _RENDERED.add(script) + return ("sh", "-c", script) + + +def positionals(argv: tuple[str, ...], start: int = 1) -> Iterator[str]: + """Yield positional arguments, skipping flags and the values they consume.""" + index = start + while index < len(argv): + item = argv[index] + if item == "--": + return + if item.startswith("-"): + if item in VALUE_FLAGS: + index += 2 + continue + index += 1 + continue + yield item + index += 1 + + +def _reject_forbidden_arguments(argv: tuple[str, ...], allow_impersonation: bool = False) -> None: + permitted = IMPERSONATION_ARGS if allow_impersonation else () + boundary = argv.index("--") if "--" in argv else len(argv) + for argument in argv[:boundary]: + for forbidden in FORBIDDEN_ARGS: + if forbidden in permitted: + continue + if argument == forbidden or argument.startswith(f"{forbidden}="): + raise PolicyError(f"forbidden argument: {forbidden}") + + +def _reject_secret_paths(argv: tuple[str, ...]) -> None: + boundary = argv.index("--") if "--" in argv else len(argv) + for argument in argv[:boundary]: + if SECRET_PATH_RE.search(argument): + raise PolicyError("probes may not name a credential path outside a frozen template") + + +def _assert_name_only_output(argv: tuple[str, ...]) -> None: + if "--output=name" in argv or "-oname" in argv: + return + for index, item in enumerate(argv[:-1]): + if item in {"-o", "--output"} and argv[index + 1] == "name": + return + raise PolicyError("credential-bearing resources may only be read with -o name") + + +def _check_kubectl(argv: tuple[str, ...], mode: str, allow_impersonation: bool) -> None: + words = list(positionals(argv)) + subcommand = next((word for word in words if word in KUBECTL_SUBCOMMANDS), None) + if subcommand is None: + raise PolicyError("kubectl subcommand is outside the read-only boundary") + if subcommand != words[0]: + raise PolicyError("kubectl subcommand must precede its resource arguments") + resource = words[1].split(".")[0] if len(words) > 1 else "" + if subcommand in {"get", "describe"} and resource in NAME_ONLY_RESOURCES: + _assert_name_only_output(argv) + if subcommand in KUBECTL_READ_SUBCOMMANDS: + return + if subcommand in KUBECTL_EXEC_SUBCOMMANDS: + if "--" not in argv: + raise PolicyError("kubectl exec requires an explicit -- command boundary") + inner = argv[argv.index("--") + 1 :] + if not inner: + raise PolicyError("kubectl exec requires an inner command") + check_argv(inner, mode, allow_impersonation=True) + return + if any(flag in argv for flag in DRY_RUN_FLAGS): + return + raise PolicyError(f"kubectl {subcommand} requires a dry run") + + +def _first_positional(binary: str, argv: tuple[str, ...]) -> str: + word = next(positionals(argv), None) + if word is None: + raise PolicyError(f"{binary} requires a subcommand") + return word + + +def _check_read_only_tool(binary: str, argv: tuple[str, ...], allowed: frozenset[str]) -> None: + subcommand = _first_positional(binary, argv) + if subcommand not in allowed: + raise PolicyError(f"{binary} {subcommand} is outside the read-only boundary") + + +def _check_git(argv: tuple[str, ...], mode: str) -> None: + subcommand = _first_positional("git", argv) + if subcommand == "config" and not {"--get", "--get-all", "--list"} & set(argv): + raise PolicyError("git config may only read") + if subcommand in GIT_READ_SUBCOMMANDS: + return + if subcommand in GIT_ARMED_SUBCOMMANDS and mode == ARMED: + if GIT_FORCE_FLAGS & set(argv): + raise PolicyError("git push may not force or fan out across refs") + return + raise PolicyError(f"git {subcommand} is not permitted in {mode} mode") + + +def _check_hermes(argv: tuple[str, ...]) -> None: + words = tuple(positionals(argv)) + if not words: + raise PolicyError("hermes requires a subcommand") + if words[:2] not in HERMES_READ_COMMANDS and words[:1] not in HERMES_READ_COMMANDS: + raise PolicyError(f"hermes {' '.join(words[:2])} is outside the read-only boundary") + + +def _check_gitea(argv: tuple[str, ...], mode: str) -> None: + if len(argv) < 3: + raise PolicyError("gitea_api.py requires a method and a path") + method = argv[1].upper() + if method in GITEA_READ_METHODS: + return + if method in GITEA_ARMED_METHODS and mode == ARMED: + return + raise PolicyError(f"gitea_api.py {method} is not permitted in {mode} mode") + + +def _check_shell(argv: tuple[str, ...]) -> None: + if len(argv) != 3 or argv[1] != "-c": + raise PolicyError("sh is only reachable as sh -c ") + if argv[2] not in _RENDERED: + raise PolicyError("sh -c script is not a frozen template rendering") + + +def check_argv( + argv: tuple[str, ...], mode: str = READ_ONLY, allow_impersonation: bool = False +) -> None: + """Raise :class:`PolicyError` unless ``argv`` is safe to run in ``mode``. + + ``allow_impersonation`` is set only for the inner command of a + ``kubectl exec``. Hermes probing its own refusal to impersonate is evidence; + the operator impersonating Hermes is not, so `--as` stays unreachable from + the outer, operator-side argv. + """ + if mode not in MODES: + raise PolicyError(f"unknown mode: {mode}") + if not argv: + raise PolicyError("empty command") + binary = argv[0] if argv[0] == GITEA_CLIENT else argv[0].rsplit("/", 1)[-1] + if binary in FORBIDDEN_BINARIES: + raise PolicyError(f"forbidden binary: {binary}") + if binary not in ALLOWED_BINARIES: + raise PolicyError(f"binary is not on the harness allowlist: {binary}") + _reject_forbidden_arguments(argv, allow_impersonation) + if binary == "sh": + _check_shell(argv) + return + _reject_secret_paths(argv) + if binary == "kubectl": + _check_kubectl(argv, mode, allow_impersonation) + elif binary == "flux": + _check_read_only_tool("flux", argv, FLUX_READ_SUBCOMMANDS) + elif binary == "helm": + _check_read_only_tool("helm", argv, HELM_READ_SUBCOMMANDS) + elif binary == "git": + _check_git(argv, mode) + elif binary == "hermes": + _check_hermes(argv) + else: + _check_gitea(argv, mode) diff --git a/scripts/ops/hermes_handoff_redaction.py b/scripts/ops/hermes_handoff_redaction.py new file mode 100644 index 00000000..f48344a5 --- /dev/null +++ b/scripts/ops/hermes_handoff_redaction.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Credential redaction and output bounding for the handoff acceptance harness. + +Every string the harness records — command output, failure reasons, evidence +values — passes through :func:`redact` before it reaches a report. Screening is +a fail-closed accident barrier, not proof that text is secret-free: a short +secret under an innocuous key is not reliably distinguishable from prose. The +harness therefore also never asks for a secret in the first place. Probes read +names, shapes, and status fields; they never read environment values, token +files, private keys, cookies, or Vault responses. + +Bounding is part of the same barrier. An unbounded capture is how a debug dump +or a stack trace carrying an authorization header ends up in an artifact, so +captures are truncated at a byte budget before they are screened. +""" + +from __future__ import annotations + +import re +from typing import Any + +REDACTED = "[redacted]" +DEFAULT_MAX_BYTES = 64 * 1024 +TRUNCATION_NOTE = "[truncated]" + +# Keys whose value is never safe to record, matched against a normalised +# (lowercased, separator-stripped) form of the key. +SENSITIVE_KEY_SUFFIXES = ( + "accesskey", + "accesskeyid", + "apikey", + "authtoken", + "clientsecret", + "connectionstring", + "cookie", + "credential", + "credentials", + "passphrase", + "passwd", + "password", + "privatekey", + "secret", + "secretkey", + "sessionkey", + "signature", + "signingkey", + "token", + "webhook", +) +SENSITIVE_KEY_EXACT = frozenset( + { + "auth", + "authorization", + "bearer", + "pat", + "pw", + "sas", + "setcookie", + } +) +# A bare `key:` is a structural field name across Kubernetes and TOML far more +# often than it is a credential, so it is screened by value shape and by the +# token patterns rather than by name. The compound forms above still match. + +# Values that cannot hold a credential: booleans, numbers, and anything too +# short to be one. Screening them corrupts structured evidence — a redacted +# `automountServiceAccountToken` is exactly the fact a reader needs — without +# protecting anything. +MIN_SECRET_LENGTH = 8 +NON_SECRET_LITERALS = frozenset({"true", "false", "null", "none", "yes", "no", "0", "1", ""}) +_KEY_NOISE_RE = re.compile(r"[^a-z0-9]+") + +# Ordered: structural matches first so a nested token is not partially rewritten +# by a looser rule. +_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile( + r"-----BEGIN[ A-Z0-9-]{0,64}-----.*?-----END[ A-Z0-9-]{0,64}-----", + re.DOTALL, + ), + re.compile(r"(?i)\b(?:authorization|proxy-authorization)\s*:\s*\S.*"), + re.compile(r"(?i)\b(?:set-)?cookie\s*:\s*\S.*"), + re.compile(r"(?i)\b(?:bearer|basic|token)\s+[A-Za-z0-9+/_=.:~-]{8,}"), + re.compile(r"(?i)\b[a-z][a-z0-9+.-]{1,20}://[^\s/@:]{1,128}:[^\s/@]{1,256}@"), + re.compile(r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}"), + re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}"), + re.compile(r"(?["']?) + (?P[A-Za-z][A-Za-z0-9_.\ -]{0,63}) + (?P=quote) + (?P\s{0,8}[:=]\s{0,8}) + (?P"[^"\n]{0,4096}"|'[^'\n]{0,4096}'|[^\s,;}\n][^,;}\n]{0,4096}) + """ +) + +# Long, mixed-class runs. Deliberately conservative: prose and hex digests of +# fewer than 44 characters are left alone so commit SHAs stay readable. +_ENTROPY_RE = re.compile(r"(? str: + """Return a key lowercased with separators removed, for suffix matching.""" + return _KEY_NOISE_RE.sub("", key.lower()) + + +def is_sensitive_key(key: str) -> bool: + """Report whether a key name means its value must never be recorded.""" + compact = normalise_key(key) + if not compact: + return False + if compact in SENSITIVE_KEY_EXACT: + return True + return compact.endswith(SENSITIVE_KEY_SUFFIXES) + + +def _is_high_entropy(candidate: str) -> bool: + """Report whether a long run mixes enough character classes to be a token.""" + classes = sum( + ( + any(character.islower() for character in candidate), + any(character.isupper() for character in candidate), + any(character.isdigit() for character in candidate), + any(character in "+/_=-" for character in candidate), + ) + ) + return classes >= 3 + + +def could_hold_secret(value: str) -> bool: + """Report whether a raw value is long and opaque enough to be a credential.""" + stripped = value.strip().strip("\"'") + if stripped.lower() in NON_SECRET_LITERALS or len(stripped) < MIN_SECRET_LENGTH: + return False + try: + float(stripped) + except ValueError: + return True + return False + + +def _redact_assignment(match: re.Match[str]) -> str: + value = match.group("value") + if not is_sensitive_key(match.group("key")) or not could_hold_secret(value): + return match.group(0) + # Preserve the original quoting so screened JSON stays parsable JSON. + quoted = value.strip()[:1] in {'"', "'"} + replacement = f'"{REDACTED}"' if quoted else REDACTED + key_quote = match.group("quote") + return f"{key_quote}{match.group('key')}{key_quote}{match.group('gap')}{replacement}" + + +def _redact_entropy(match: re.Match[str]) -> str: + return REDACTED if _is_high_entropy(match.group(0)) else match.group(0) + + +def redact(text: str) -> str: + """Return ``text`` with credential-shaped substrings replaced.""" + if not text: + return text + for pattern in _PATTERNS: + text = pattern.sub(REDACTED, text) + text = _ASSIGNMENT_RE.sub(_redact_assignment, text) + return _ENTROPY_RE.sub(_redact_entropy, text) + + +def bound(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: + """Truncate ``text`` to a UTF-8 byte budget, reporting whether it was cut.""" + if max_bytes <= 0: + return ("", bool(text)) + encoded = text.encode("utf-8", "replace") + if len(encoded) <= max_bytes: + return (text, False) + kept = encoded[:max_bytes].decode("utf-8", "ignore") + return (f"{kept}\n{TRUNCATION_NOTE}", True) + + +def safe_text(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: + """Bound then redact, in that order, so a huge dump cannot outrun screening.""" + bounded, truncated = bound(text, max_bytes) + return (redact(bounded), truncated) + + +def scrub(value: Any, max_bytes: int = DEFAULT_MAX_BYTES) -> Any: + """Recursively bound and redact a JSON-shaped value. + + Mapping keys are preserved (names are evidence); values under a sensitive + key are dropped outright rather than pattern-screened, because a short + secret would survive screening. + """ + if isinstance(value, str): + return safe_text(value, max_bytes)[0] + if isinstance(value, dict): + return { + str(key): REDACTED + if is_sensitive_key(str(key)) and could_hold_secret(str(item)) + else scrub(item, max_bytes) + for key, item in value.items() + } + if isinstance(value, (list, tuple)): + return [scrub(item, max_bytes) for item in value] + return value + + +def contains_credential_shape(text: str) -> bool: + """Report whether screening would rewrite ``text``. + + Used by the harness self-test to prove a rendered report is clean before it + is written, and by the unit suite as an assertion helper. + """ + return redact(text) != text diff --git a/scripts/ops/hermes_handoff_rules.py b/scripts/ops/hermes_handoff_rules.py new file mode 100644 index 00000000..cf9b5b97 --- /dev/null +++ b/scripts/ops/hermes_handoff_rules.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Shared primitives and dispatch for handoff acceptance evaluators. + +Evaluators are registered by name so the check catalog stays declarative: an +entry says *what* it asserts, not how to plumb it. The dispatch here is the one +place that turns an unexpected exception into ``NOT_RUN`` — a harness bug must +never be indistinguishable from a passing release. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_model import NOT_RUN, CheckSpec + +DENIAL_MARKERS = ( + "forbidden", + "is not allowed", + "unauthorized", + "permission denied", + "cannot get", + "cannot list", + "cannot create", + "cannot delete", + "cannot patch", + "not permitted", + "no such route", + "404 not found", + "403", +) + + +@dataclass +class Evaluation: + """A classification plus the small facts that justify it.""" + + status: str + reason: str + evidence: dict[str, Any] = field(default_factory=dict) + + +Evaluator = Callable[[CheckSpec, dict[str, Outcome]], Evaluation] +EVALUATORS: dict[str, Evaluator] = {} + + +def evaluator(name: str) -> Callable[[Evaluator], Evaluator]: + """Register an evaluator under the rule name catalog entries use.""" + + def register(function: Evaluator) -> Evaluator: + EVALUATORS[name] = function + return function + + return register + + +def missing_steps(spec: CheckSpec, outcomes: Mapping[str, Outcome]) -> Evaluation | None: + """Return NOT_RUN when a required step never produced usable evidence.""" + for step in spec.steps: + if step.optional: + continue + outcome = outcomes.get(step.key) + if outcome is None: + return Evaluation(NOT_RUN, f"step {step.key} was never executed") + if not outcome.ran: + return Evaluation(NOT_RUN, f"step {step.key} did not run: {outcome.error}") + return None + + +def looks_denied(outcome: Outcome) -> bool: + """Report whether a real request was refused by policy rather than failing.""" + if outcome.ok: + return False + haystack = outcome.combined.lower() + return any(marker in haystack for marker in DENIAL_MARKERS) + + +def step_keys(spec: CheckSpec, kind: str) -> tuple[str, ...]: + """Return the keys of every step of one kind.""" + return tuple(step.key for step in spec.steps if step.kind == kind) + + +def is_list(value: Any) -> bool: + """Report whether a value is a JSON array rather than a string.""" + return isinstance(value, Sequence) and not isinstance(value, (str, bytes)) + + +def dotted(payload: Any, path: str) -> Any: + """Return a value addressed by a dotted path. + + ``a.b[0].c`` indexes; ``a[].b`` maps the remainder of the path over a list, + which is how the catalog records one field from every pull request or every + Flux object without a bespoke parser per check. + """ + current = payload + segments = [segment for segment in path.split(".") if segment] + for position, raw in enumerate(segments): + name, bracket, indexes = raw.partition("[") + if name: + if not isinstance(current, Mapping) or name not in current: + raise KeyError(path) + current = current[name] + if not bracket: + continue + for chunk in indexes.rstrip("]").split("]["): + if not is_list(current): + raise KeyError(path) + if chunk == "": + remainder = ".".join(segments[position + 1 :]) + return [dotted(item, remainder) if remainder else item for item in current] + offset = int(chunk) + if offset >= len(current): + raise KeyError(path) + current = current[offset] + return current + + +def parse_json(outcome: Outcome) -> Any: + """Parse a step's stdout as JSON, raising ``ValueError`` when it is not. + + Control characters are tolerated. Task bodies and session titles routinely + carry raw newlines and tabs, and refusing to read a report because a human + pasted a tab into a Kanban card would be a false NO_GO. + """ + text = outcome.stdout.strip() + if not text: + raise ValueError("empty output") + return json.loads(text, strict=False) + + +def lines(outcome: Outcome) -> list[str]: + """Return non-empty, stripped stdout lines.""" + return [line.strip() for line in outcome.stdout.splitlines() if line.strip()] + + +def parsed_step(spec: CheckSpec, outcomes: dict[str, Outcome]) -> tuple[Any, str] | Evaluation: + """Return the parsed payload of ``expect['step']`` or the NOT_RUN to report.""" + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + try: + return (parse_json(outcome), key) + except ValueError as exc: + return Evaluation(NOT_RUN, f"step {key} did not return JSON: {exc}") + + +def evaluate(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Dispatch to the named evaluator, converting any surprise into NOT_RUN.""" + rule = EVALUATORS.get(spec.rule) + if rule is None: + return Evaluation(NOT_RUN, f"unknown evaluator: {spec.rule}") + try: + return rule(spec, outcomes) + except Exception as exc: # noqa: BLE001 - a harness bug must never read as PASS + return Evaluation(NOT_RUN, f"evaluator {spec.rule} raised {type(exc).__name__}: {exc}") diff --git a/scripts/ops/hermes_handoff_run.py b/scripts/ops/hermes_handoff_run.py new file mode 100644 index 00000000..cdfaf3c2 --- /dev/null +++ b/scripts/ops/hermes_handoff_run.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 +"""Orchestration for the Hermes full-handoff acceptance harness. + +Three things happen before any check runs, and each of them can stop the run on +its own: + +* the catalog is validated structurally — every rule exists, every deny check + carries a real attempt, no operator-side step can impersonate; +* each vantage is resolved and asked who it is; and +* the operator and in-pod identities are compared. If they are the same + principal there is only one vantage, the dual-vantage evidence would be a + restatement rather than a corroboration, and the run is a NO_GO. + +Everything after that is uniform: run a check's steps against its vantage, hand +the outcomes to the named evaluator, and record the classification. A vantage +that could not be resolved yields ``NOT_RUN`` for every check that needs it, +which is exactly the fail-closed answer. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping, Sequence + +from hermes_handoff_catalog import ( + CHAT, + NODE, + OPERATOR, + SELF, + SWITCHYARD, + VANTAGE_NAMES, + Targets, +) +from hermes_handoff_checks_access import access_checks +from hermes_handoff_checks_delivery import delivery_checks +from hermes_handoff_checks_platform import platform_checks +from hermes_handoff_checks_workers import worker_checks +from hermes_handoff_evaluators import EVALUATORS, evaluate +from hermes_handoff_exec import Outcome, Runner, Vantage, operator_vantage, pod_vantage +from hermes_handoff_model import ( + ATTEMPT, + EPHEMERAL, + NOT_RUN, + STEP_KINDS, + CheckResult, + CheckSpec, + Report, + VantageRecord, + utc_now, +) +from hermes_handoff_policy import IMPERSONATION_ARGS + +UNRECORDED = "[output not recorded]" + + +def build_catalog(targets: Targets) -> list[CheckSpec]: + """Return the whole acceptance catalog for a set of targets.""" + return [ + *platform_checks(targets), + *access_checks(targets), + *delivery_checks(targets), + *worker_checks(targets), + ] + + +def validate_catalog(specs: Sequence[CheckSpec]) -> list[str]: + """Return the structural problems that make a catalog unsafe to run.""" + problems: list[str] = [] + seen: set[str] = set() + for spec in specs: + if spec.id in seen: + problems.append(f"{spec.id}: duplicate check id") + seen.add(spec.id) + if spec.rule not in EVALUATORS and spec.scope != EPHEMERAL: + problems.append(f"{spec.id}: unknown rule {spec.rule!r}") + for step in spec.steps: + if step.vantage not in VANTAGE_NAMES: + problems.append(f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}") + if step.kind not in STEP_KINDS: + problems.append(f"{spec.id}: step {step.key} has unknown kind {step.kind!r}") + if step.vantage == OPERATOR and set(step.argv) & set(IMPERSONATION_ARGS): + problems.append(f"{spec.id}: step {step.key} impersonates from the operator vantage") + if spec.rule == "denied" and not any(step.kind == ATTEMPT for step in spec.steps): + problems.append(f"{spec.id}: a deny check needs a real attempt, not only a review") + return problems + + +def resolve_identity(runner: Runner, vantage: Vantage) -> tuple[str, str]: + """Return the authenticated username of a vantage and any failure detail.""" + outcome = runner.run(("kubectl", "auth", "whoami", "-o", "json"), vantage) + if not outcome.ok: + return ("", outcome.error or outcome.combined[:200]) + try: + payload = json.loads(outcome.stdout) + except ValueError as exc: + return ("", f"unparsable identity: {exc}") + username = ((payload.get("status") or {}).get("userInfo") or {}).get("username") + if not isinstance(username, str) or not username: + return ("", "identity response carried no username") + return (username, "") + + +def _first_pod(runner: Runner, operator: Vantage, namespace: str, selector: str) -> tuple[str, str]: + outcome = runner.run( + ( + "kubectl", + "--namespace", + namespace, + "get", + "pods", + "--selector", + selector, + "--field-selector", + "status.phase=Running", + "-o", + "name", + ), + operator, + ) + if not outcome.ok: + return ("", outcome.error or outcome.combined[:200]) + names = [line.strip().removeprefix("pod/") for line in outcome.stdout.splitlines() if line.strip()] + if not names: + return ("", f"no running pod matches {selector}") + return (sorted(names)[0], "") + + +def resolve_vantages( + runner: Runner, targets: Targets, kubeconfig: str | None = None, context: str | None = None +) -> tuple[dict[str, Vantage], list[VantageRecord]]: + """Resolve every vantage the catalog names, recording what each one is.""" + operator = operator_vantage(kubeconfig, context) + username, detail = resolve_identity(runner, operator) + records = [ + VantageRecord( + name=OPERATOR, + description=operator.description or "external read-only operator", + identity=username, + available=bool(username), + detail=detail, + ) + ] + vantages: dict[str, Vantage] = {OPERATOR: operator} + if not username: + return (vantages, records) + + for name, selector, container, description in ( + (SELF, f"app={targets.agent_deployment}", targets.agent_container, "the Hermes agent itself"), + ( + SWITCHYARD, + f"app={targets.switchyard_deployment}", + targets.switchyard_container, + "the Switchyard routing evidence log", + ), + (NODE, f"app={targets.node_daemonset}", targets.node_container, "a node hardening probe"), + ): + pod, failure = _first_pod(runner, operator, targets.namespace, selector) + if not pod: + records.append( + VantageRecord(name=name, description=description, available=False, detail=failure) + ) + continue + vantage = pod_vantage(targets.namespace, pod, container, operator) + identity, identity_detail = resolve_identity(runner, vantage) if name == SELF else ("", "") + records.append( + VantageRecord( + name=name, + description=f"{description} ({targets.namespace}/{pod})", + identity=identity, + available=True, + detail=identity_detail, + ) + ) + vantages[name] = vantage + + chat_pod = f"{targets.chat_statefulset}-{targets.chat_ordinal}" + present = runner.run( + ("kubectl", "--namespace", targets.namespace, "get", f"pod/{chat_pod}", "-o", "name"), + operator, + ) + if present.ok and present.stdout.strip(): + vantages[CHAT] = pod_vantage(targets.namespace, chat_pod, targets.chat_container, operator) + records.append( + VantageRecord( + name=CHAT, + description=f"the chat surface ({targets.namespace}/{chat_pod})", + available=True, + ) + ) + else: + records.append( + VantageRecord( + name=CHAT, + description=f"the chat surface ({targets.namespace}/{chat_pod})", + available=False, + detail=present.error or present.combined[:200] or "chat tenant pod not found", + ) + ) + return (vantages, records) + + +def vantage_problems(records: Iterable[VantageRecord]) -> list[str]: + """Return the identity problems that invalidate dual-vantage evidence.""" + by_name = {record.name: record for record in records} + operator = by_name.get(OPERATOR) + hermes = by_name.get(SELF) + problems: list[str] = [] + if operator is None or not operator.available: + problems.append("the operator vantage is unavailable; no evidence can be collected") + return problems + if hermes is None or not hermes.available: + problems.append("the in-pod Hermes vantage is unavailable; self-probe evidence is missing") + return problems + if not hermes.identity: + problems.append("the in-pod vantage did not report an identity") + elif operator.identity == hermes.identity: + problems.append( + "the operator and in-pod vantages authenticate as the same principal " + f"({operator.identity}); run the harness with a separate operator kubeconfig" + ) + return problems + + +def run_check( + runner: Runner, spec: CheckSpec, vantages: Mapping[str, Vantage] +) -> CheckResult: + """Run one check's steps and classify the result.""" + outcomes: dict[str, Outcome] = {} + recorded: list[Outcome] = [] + for step in spec.steps: + vantage = vantages.get(step.vantage) + if vantage is None: + outcome = Outcome( + argv=step.argv, vantage=step.vantage, error=f"vantage {step.vantage} is unavailable" + ) + else: + outcome = runner.run(step.argv, vantage, step.max_bytes) + outcomes[step.key] = outcome + recorded.append( + outcome + if step.record + else Outcome( + argv=outcome.argv, + vantage=outcome.vantage, + returncode=outcome.returncode, + stdout=UNRECORDED if outcome.stdout else "", + stderr=UNRECORDED if outcome.stderr else "", + truncated=outcome.truncated, + duration_ms=outcome.duration_ms, + error=outcome.error, + ) + ) + evaluation = evaluate(spec, outcomes) + return CheckResult( + spec=spec, + status=evaluation.status, + reason=evaluation.reason, + evidence=evaluation.evidence, + outcomes=recorded, + ) + + +def run_catalog( + runner: Runner, specs: Sequence[CheckSpec], vantages: Mapping[str, Vantage] +) -> list[CheckResult]: + """Run every check in the catalog. + + Nothing is skipped once the deadline passes: the runner returns a + deadline outcome for each remaining command, which classifies as + ``NOT_RUN``, so a truncated sweep is visible in the report rather than + silently shorter. + """ + return [run_check(runner, spec, vantages) for spec in specs] + + +def build_report( + runner: Runner, + targets: Targets, + specs: Sequence[CheckSpec], + vantages: Mapping[str, Vantage], + records: Sequence[VantageRecord], + mode: str, + started_at: str, + extra_results: Sequence[CheckResult] = (), +) -> Report: + """Run the catalog and assemble the report, including structural problems.""" + problems = validate_catalog(specs) + vantage_problems(records) + report = Report( + mode=mode, + started_at=started_at, + baseline={ + "namespace": targets.namespace, + "baseline_commit": targets.baseline_commit, + "dependency_pull_requests": list(targets.dependency_pull_requests), + "expected_suspensions": list(targets.expected_suspensions), + "node_count": targets.node_count, + "pool_replicas": targets.pool_replicas, + }, + vantages=list(records), + harness_errors=problems, + ) + armed_ids = {result.spec.id for result in extra_results} + report.results = [ + *run_catalog(runner, [spec for spec in specs if spec.id not in armed_ids], vantages), + *extra_results, + ] + report.results.sort(key=lambda result: result.spec.id) + if runner.remaining_seconds <= 0: + report.harness_errors.append("the run deadline expired before every check completed") + report.finished_at = utc_now() + return report + + +def unavailable_report(mode: str, started_at: str, reason: str, targets: Targets) -> Report: + """Return a NO_GO report for a run that could not start at all.""" + return Report( + mode=mode, + started_at=started_at, + finished_at=utc_now(), + baseline={"namespace": targets.namespace}, + harness_errors=[reason], + results=[ + CheckResult( + spec=CheckSpec( + id="harness.startup", + title="The harness could start and resolve its vantages", + group="harness", + rule="allowed", + ), + status=NOT_RUN, + reason=reason, + ) + ], + ) diff --git a/testing/tests/test_hermes_handoff_acceptance.py b/testing/tests/test_hermes_handoff_acceptance.py new file mode 100644 index 00000000..87c8adbe --- /dev/null +++ b/testing/tests/test_hermes_handoff_acceptance.py @@ -0,0 +1,395 @@ +"""End-to-end contracts for the Hermes full-handoff acceptance harness CLI.""" + +from __future__ import annotations + +import json +import subprocess + +import pytest + +from testing.tests.test_hermes_handoff_support import FakeClock, load_handoff_module + +acceptance = load_handoff_module("hermes_handoff_acceptance") +catalog = load_handoff_module("hermes_handoff_catalog") +exec_module = load_handoff_module("hermes_handoff_exec") +model = load_handoff_module("hermes_handoff_model") +policy = load_handoff_module("hermes_handoff_policy") +harness_run = load_handoff_module("hermes_handoff_run") + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + + +class ScriptedCluster: + """A ``subprocess.run`` stand-in that answers by command shape. + + Real runs issue a couple of hundred commands; scripting them positionally + would make every test a brittle ordering puzzle. Matching on the command + line keeps each test about the behaviour it is checking. + """ + + def __init__(self, rules: list[tuple[str, object]], default=None) -> None: + self.rules = rules + self.default = default or completed(stderr="not found", returncode=1) + self.calls: list[list[str]] = [] + + def __call__(self, argv, **_kwargs): + self.calls.append(list(argv)) + line = " ".join(argv) + for fragment, response in self.rules: + if fragment in line: + return response + return self.default + + +def whoami(username: str): + return completed(stdout=json.dumps({"status": {"userInfo": {"username": username}}})) + + +HEALTHY_CLUSTER = [ + ("auth whoami", whoami("kubernetes-admin")), + ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), + ("get pods --selector", completed(stdout="pod/other-1\n")), + ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")), +] + + +def run_cli(monkeypatch, argv, rules): + cluster = ScriptedCluster(rules) + clock = FakeClock() + + def build_runner(**kwargs): + return exec_module.Runner(clock=clock, spawn=cluster, environ={}, **kwargs) + + monkeypatch.setattr(acceptance, "Runner", build_runner) + return (acceptance.main(argv), cluster) + + +def read_report(path): + return json.loads(path.read_text(encoding="utf-8")) + + +def test_the_parser_exposes_the_documented_controls() -> None: + arguments = acceptance.build_parser().parse_args( + [ + "--namespace", + "hermes", + "--context", + "atlas-operator", + "--dependency-pr", + "14", + "--dependency-pr", + "15", + "--expected-suspension", + "flux-system/parked", + "--node-count", + "21", + "--pool-worker-env", + "HERMES_POOL_LEASE_SECONDS", + "--expect-telegram-sessions", + ] + ) + + targets = acceptance.targets_from_args(arguments, None) + + assert targets.dependency_pull_requests == (14, 15) + assert targets.expected_suspensions == ("flux-system/parked",) + assert targets.node_count == 21 + assert targets.pool_worker_env == ("HERMES_POOL_LEASE_SECONDS",) + assert targets.expect_telegram_sessions is True + + +def test_defaults_fall_back_to_the_shipped_targets() -> None: + arguments = acceptance.build_parser().parse_args([]) + targets = acceptance.targets_from_args(arguments, None) + + assert targets.dependency_pull_requests == catalog.Targets.dependency_pull_requests + assert targets.expected_suspensions == () + assert targets.repo == catalog.Targets.repo + + +def test_a_default_run_is_read_only_and_never_pushes_or_posts(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + status, cluster = run_cli( + monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert report["mode"] == policy.READ_ONLY + assert not any("push" in call for call in cluster.calls) + assert not any("POST" in call for call in cluster.calls) + assert not any("DELETE" in call for call in cluster.calls) + writes = [ + call + for call in cluster.calls + if {"patch", "create", "delete", "apply", "replace"} & set(call) and "can-i" not in call + ] + assert writes + assert all("--dry-run=server" in call for call in writes) + + +def test_mutating_checks_report_not_run_until_the_mode_is_armed(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) + + ephemeral = { + check["id"]: check + for check in read_report(output)["checks"] + if check["group"] == "ephemeral" + } + + assert len(ephemeral) == 4 + for check in ephemeral.values(): + assert check["status"] == model.NOT_RUN + assert check["mandatory"] is False + assert check["scope"] == model.EPHEMERAL + + +def test_a_zero_state_cluster_is_no_go_with_every_check_accounted_for( + monkeypatch, tmp_path +) -> None: + """Nothing observed must never read as nothing wrong.""" + output = tmp_path / "report.json" + status, _ = run_cli( + monkeypatch, + ["--output", str(output), "--json-only"], + [("auth whoami", whoami("kubernetes-admin"))], + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert report["decision"] == model.NO_GO + assert report["counts"][model.PASS] == 0 + assert sum(report["counts"].values()) == len(report["checks"]) + assert any("in-pod Hermes vantage is unavailable" in error for error in report["harness_errors"]) + + +def test_partial_failure_is_reported_per_check_rather_than_aborting( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + rules = [ + *HEALTHY_CLUSTER, + ("auth can-i", completed(stdout="no\n", returncode=1)), + ("get namespaces", completed(stdout="namespace/hermes\n")), + ("merge-base", completed()), + ] + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) + report = read_report(output) + + statuses = {check["status"] for check in report["checks"]} + + assert status == acceptance.EXIT_NO_GO + assert model.PASS in statuses and model.NOT_RUN in statuses + assert report["blocking"] + + +def test_a_vantage_collision_is_a_no_go_even_with_clean_checks(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + rules = [ + ("exec --namespace hermes hermes-agent-1", whoami("system:serviceaccount:hermes:hermes-agent")), + ("auth whoami", whoami("system:serviceaccount:hermes:hermes-agent")), + ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), + ("get pods --selector", completed(stdout="pod/other-1\n")), + ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")), + ] + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert any("same principal" in error for error in report["harness_errors"]) + + +def test_an_exhausted_deadline_leaves_the_remaining_checks_not_run(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + status, _ = run_cli( + monkeypatch, ["--output", str(output), "--json-only", "--deadline", "0"], HEALTHY_CLUSTER + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert report["counts"][model.NOT_RUN] > 0 + assert any("deadline expired" in error for error in report["harness_errors"]) + + +def test_arming_with_a_wrong_confirmation_stops_before_anything_runs( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + status, cluster = run_cli( + monkeypatch, + [ + "--output", + str(output), + "--json-only", + "--arm-ephemeral-push", + "--confirm", + "please", + "--ephemeral-token", + "acceptance-20260817a", + ], + HEALTHY_CLUSTER, + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert "arming refused" in report["harness_errors"][0] + assert not any("push" in call for call in cluster.calls) + + +def test_arming_against_a_protected_push_target_is_refused(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + status, cluster = run_cli( + monkeypatch, + [ + "--output", + str(output), + "--json-only", + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "main", + ], + HEALTHY_CLUSTER, + ) + + assert status == acceptance.EXIT_NO_GO + assert "arming refused" in read_report(output)["harness_errors"][0] + assert not any("push" in call for call in cluster.calls) + + +def test_an_armed_run_pushes_one_ephemeral_ref_and_verifies_its_removal( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + rules = [ + ("git ls-remote origin refs/heads/ephemeral", completed(stdout="")), + ("git push", completed()), + ("POST /api/v1/repos/atlas/titan-iac/pulls", completed(stdout=json.dumps({"number": 9}))), + ("PATCH", completed()), + ("DELETE", completed()), + ("GET /api/v1/repos/atlas/titan-iac/pulls/9", completed(stdout='{"state": "closed"}')), + *HEALTHY_CLUSTER, + ] + status, cluster = run_cli( + monkeypatch, + [ + "--output", + str(output), + "--json-only", + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817a", + ], + rules, + ) + report = read_report(output) + ephemeral = {check["id"]: check for check in report["checks"] if check["group"] == "ephemeral"} + + assert report["mode"] == policy.ARMED + assert status == acceptance.EXIT_NO_GO # the rest of the cluster is still unproven + assert ephemeral["ephemeral.feature-branch-push"]["status"] == model.PASS + assert ephemeral["ephemeral.draft-pull-request"]["status"] == model.PASS + assert ephemeral["ephemeral.cleanup-verified"]["status"] == model.PASS + assert ephemeral["ephemeral.protected-branch-refusal"]["status"] == model.PASS + + pushes = [call for call in cluster.calls if "push" in call] + assert len(pushes) == 1 + assert pushes[0][-1].startswith("HEAD:refs/heads/ephemeral/hermes-handoff-acceptance/") + assert pushes[0][0] == "kubectl" and "exec" in pushes[0] # pushed as the worker, not the operator + assert any("DELETE" in call for call in cluster.calls) + + +def test_the_report_is_written_to_stdout_when_no_output_path_is_given( + monkeypatch, capsys +) -> None: + status, _ = run_cli(monkeypatch, ["--json-only"], HEALTHY_CLUSTER) + payload = json.loads(capsys.readouterr().out) + + assert status == acceptance.EXIT_NO_GO + assert payload["harness"] == model.HARNESS + + +def test_the_human_summary_accompanies_the_json_unless_suppressed( + monkeypatch, capsys, tmp_path +) -> None: + output = tmp_path / "report.json" + run_cli(monkeypatch, ["--output", str(output)], HEALTHY_CLUSTER) + captured = capsys.readouterr() + + assert model.HARNESS in captured.err + assert "vantage operator" in captured.err + assert captured.out == "" + + +def test_a_report_that_fails_its_own_screening_is_replaced_not_published(tmp_path) -> None: + """The last barrier: never write an artifact nobody re-read.""" + output = tmp_path / "report.json" + arguments = acceptance.build_parser().parse_args(["--output", str(output), "--json-only"]) + report = model.Report(mode="read-only", started_at="2026-08-17T00:00:00Z") + report.harness_errors = ["leaked ghp_ABCDEFGHIJKLMNOPQRSTUV1234"] + + monkey = report.as_dict + report.as_dict = lambda *_args, **_kwargs: { # type: ignore[method-assign] + **monkey(), + "leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234", + } + offenders = acceptance.emit(report, arguments) + payload = read_report(output) + + assert offenders == ["$.leak"] + assert payload["decision"] == model.NO_GO + assert "leak" not in payload + assert "failed its own credential screening" in payload["harness_errors"][0] + + +def test_a_screening_failure_forces_no_go_from_the_entry_point(monkeypatch, tmp_path) -> None: + """The exit status must follow the refusal to publish, not the check tally.""" + output = tmp_path / "report.json" + monkeypatch.setattr(acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"]) + monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) + monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) + + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) + + assert status == acceptance.EXIT_NO_GO + assert read_report(output)["decision"] == model.NO_GO + + +def test_no_recorded_command_output_carries_a_credential_shape(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + leaky = completed(stdout="ANTHROPIC_API_KEY=sk-ant-abcdefghijklmnopqrstuvwxyz\n") + run_cli( + monkeypatch, + ["--output", str(output), "--json-only"], + [*HEALTHY_CLUSTER, ("exec", leaky)], + ) + + assert model.unscreened_fields(read_report(output)) == [] + assert "sk-ant-abcdefghijklmnopqrstuvwxyz" not in output.read_text(encoding="utf-8") + + +@pytest.mark.parametrize("flag", ["--help"]) +def test_the_help_text_describes_the_read_only_default(flag: str, capsys) -> None: + with pytest.raises(SystemExit): + acceptance.main([flag]) + + assert "Read-only by default" in capsys.readouterr().out + + +def test_go_is_reachable_when_nothing_mandatory_is_outstanding(monkeypatch, tmp_path) -> None: + """Exercise the success exit path without pretending the cluster is clean.""" + output = tmp_path / "report.json" + monkeypatch.setattr(harness_run, "build_catalog", lambda _targets: []) + monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) + monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) + + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) + + assert status == acceptance.EXIT_GO + assert read_report(output)["decision"] == model.GO diff --git a/testing/tests/test_hermes_handoff_catalog.py b/testing/tests/test_hermes_handoff_catalog.py new file mode 100644 index 00000000..8f5e2de6 --- /dev/null +++ b/testing/tests/test_hermes_handoff_catalog.py @@ -0,0 +1,194 @@ +"""Structural contracts for the handoff acceptance catalog.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module + +catalog = load_handoff_module("hermes_handoff_catalog") +harness_run = load_handoff_module("hermes_handoff_run") +evaluators = load_handoff_module("hermes_handoff_evaluators") +model = load_handoff_module("hermes_handoff_model") +policy = load_handoff_module("hermes_handoff_policy") +exec_module = load_handoff_module("hermes_handoff_exec") + +NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc) +TARGETS = catalog.Targets(now=NOW) +SPECS = harness_run.build_catalog(TARGETS) +BY_ID = {check.id: check for check in SPECS} + +VANTAGES = { + catalog.OPERATOR: exec_module.operator_vantage(context="atlas-operator"), + catalog.SELF: exec_module.pod_vantage("hermes", "hermes-agent-1", "hermes"), + catalog.SWITCHYARD: exec_module.pod_vantage("hermes", "hermes-switchyard-1", "switchyard"), + catalog.NODE: exec_module.pod_vantage("hermes", "hermes-node-ssh-access-1", "key-reconciler"), + catalog.CHAT: exec_module.pod_vantage("hermes", "hermes-chat-tenant-0", "hermes"), +} + + +def test_the_catalog_is_structurally_valid() -> None: + assert harness_run.validate_catalog(SPECS) == [] + assert len(SPECS) >= 60 + + +def test_every_step_is_accepted_by_the_read_only_policy_from_its_vantage() -> None: + """A catalog entry that cannot run under policy is a bug found before a run.""" + for check in SPECS: + for step in check.steps: + addressed = VANTAGES[step.vantage].wrap(step.argv) + policy.check_argv(addressed, policy.READ_ONLY) + + +def test_no_step_mutates_outside_a_server_dry_run() -> None: + """`auth can-i create ...` is a read; the subcommand is what decides.""" + for check in SPECS: + for step in check.steps: + subcommand = next(policy.positionals(step.argv), "") + if subcommand in policy.KUBECTL_DRY_RUN_SUBCOMMANDS: + assert any(flag in step.argv for flag in policy.DRY_RUN_FLAGS), check.id + + +def test_no_operator_step_impersonates_and_at_least_one_in_pod_step_does() -> None: + operator_steps = [ + step for check in SPECS for step in check.steps if step.vantage == catalog.OPERATOR + ] + assert operator_steps + assert not any(set(step.argv) & set(policy.IMPERSONATION_ARGS) for step in operator_steps) + + impersonation = BY_ID["access.impersonation-is-denied"] + attempts = [step for step in impersonation.steps if step.kind == model.ATTEMPT] + assert any("--as" in step.argv for step in attempts) + + +def test_every_deny_check_pairs_a_review_with_a_live_attempt() -> None: + deny_checks = [check for check in SPECS if check.rule == "denied"] + assert len(deny_checks) >= 6 + for check in deny_checks: + assert any(step.kind == model.ATTEMPT for step in check.steps), check.id + + +def test_both_vantages_are_represented_across_the_catalog() -> None: + used = {step.vantage for check in SPECS for step in check.steps} + assert catalog.OPERATOR in used and catalog.SELF in used + assert used <= set(catalog.VANTAGE_NAMES) + + +def test_every_rule_named_by_a_check_is_registered() -> None: + for check in SPECS: + if check.scope == model.EPHEMERAL: + assert check.rule == "not_armed" + continue + assert check.rule in evaluators.EVALUATORS, check.id + + +def test_every_evidence_step_referenced_by_expect_exists() -> None: + for check in SPECS: + keys = {step.key for step in check.steps} + named = [check.expect.get("step")] + list(check.expect.get("steps", ())) + for key in filter(None, named): + assert key in keys, f"{check.id} names missing step {key}" + + +def test_the_default_catalog_reports_every_mutating_check_as_not_armed() -> None: + ephemeral = [check for check in SPECS if check.scope == model.EPHEMERAL] + assert len(ephemeral) == 4 + for check in ephemeral: + assert check.mandatory is False + assert evaluators.evaluate(check, {}).status == model.NOT_RUN + + +def test_dependency_pull_requests_each_get_their_own_merge_gate() -> None: + for number in TARGETS.dependency_pull_requests: + check = BY_ID[f"baseline.dependency-pr-{number}-merged"] + assert check.mandatory + assert check.expect["fields"] == {"merged": True, "base.ref": "main"} + + +def test_the_baseline_commit_gate_names_the_configured_commit() -> None: + check = BY_ID["baseline.origin-main-descends-merged-work"] + assert TARGETS.baseline_commit in check.steps[0].argv + assert "--is-ancestor" in check.steps[0].argv + + +def test_optional_checks_appear_only_when_a_site_declares_what_to_expect() -> None: + """A names_present over an empty expectation would pass on any workload.""" + assert "pool.assignment-safety-knobs-are-configured" not in BY_ID + assert "surfaces.chat-runs-the-telegram-topic-revision" not in BY_ID + assert "surfaces.chat-telegram-sessions-are-continuous" not in BY_ID + + configured = harness_run.build_catalog( + catalog.Targets( + now=NOW, + pool_worker_env=("HERMES_POOL_LEASE_SECONDS",), + chat_config_revision="20260816-telegram-topics", + expect_telegram_sessions=True, + ) + ) + identifiers = {check.id for check in configured} + assert "pool.assignment-safety-knobs-are-configured" in identifiers + assert "surfaces.chat-runs-the-telegram-topic-revision" in identifiers + assert "surfaces.chat-telegram-sessions-are-continuous" in identifiers + assert harness_run.validate_catalog(configured) == [] + + +def test_bulk_evidence_is_parsed_but_not_pasted_into_the_report() -> None: + routing = BY_ID["routing.provider-effort-and-fallback-evidence"] + assert routing.steps[0].record is False + assert routing.steps[0].max_bytes == TARGETS.routing_tail_bytes + + kanban = BY_ID["surfaces.kanban-activity-is-visible"] + assert kanban.steps[0].record is False + + +def test_freshness_gates_carry_the_run_clock() -> None: + for identifier in ("identity.codex-evidence-is-fresh", "identity.claude-evidence-is-fresh"): + assert BY_ID[identifier].expect["now"] == NOW + routing = BY_ID["routing.provider-effort-and-fallback-evidence"] + assert routing.expect["now"] == NOW + assert routing.expect["max_age_seconds"] == TARGETS.max_evidence_age_seconds + + +def test_check_identifiers_are_unique_and_grouped() -> None: + identifiers = [check.id for check in SPECS] + assert len(identifiers) == len(set(identifiers)) + for check in SPECS: + assert check.id.split(".")[0] + assert check.group + + +@pytest.mark.parametrize( + "identifier", + [ + "identity.no-provider-api-key-in-pod", + "identity.no-provider-api-key-in-manifest", + "forge.worker-holds-no-git-credential-variable", + "access.secrets-are-denied", + "access.service-account-tokens-are-denied", + "access.workload-mutation-is-denied", + "access.pod-exec-is-denied", + "access.required-reads-are-allowed", + "access.pod-logs-are-allowed", + "access.flux-and-helm-status-are-allowed", + "access.namespace-view-agrees-across-vantages", + "nodes.account-password-is-locked", + "nodes.hardening-covers-every-node", + "build.builder-service-account-is-tokenless", + "build.harbor-immutability-rule-is-applied", + "build.deployed-image-is-digest-pinned", + "reliability.finalization-and-replay-patches-are-deployed", + "pool.three-workers-are-ready", + "pool.workers-occupy-distinct-nodes", + "pool.workers-own-distinct-volumes", + "pool.coordinator-retains-sole-state-ownership", + "surfaces.kanban-activity-is-visible", + "surfaces.chat-telegram-topic-state-is-durable", + "gitops.kustomizations-reconcile-cleanly", + "routing.provider-effort-and-fallback-evidence", + "scopes.agent-profile-is-distinct", + ], +) +def test_the_handoff_requirements_each_have_a_mandatory_check(identifier: str) -> None: + assert BY_ID[identifier].mandatory is True diff --git a/testing/tests/test_hermes_handoff_ephemeral.py b/testing/tests/test_hermes_handoff_ephemeral.py new file mode 100644 index 00000000..09cc5d63 --- /dev/null +++ b/testing/tests/test_hermes_handoff_ephemeral.py @@ -0,0 +1,271 @@ +"""Contracts for the separately armed, self-cleaning mutation mode.""" + +from __future__ import annotations + +import json +import subprocess + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + FakeClock, + FakeSpawn, + load_handoff_module, +) + +ephemeral = load_handoff_module("hermes_handoff_ephemeral") +exec_module = load_handoff_module("hermes_handoff_exec") +policy = load_handoff_module("hermes_handoff_policy") +model = load_handoff_module("hermes_handoff_model") + +REPO = "atlas/titan-iac" +TOKEN = "acceptance-20260817a" +VANTAGE = exec_module.operator_vantage() + + +def request(**overrides) -> object: + fields = { + "repo": REPO, + "remote": "origin", + "token": TOKEN, + "confirmation": ephemeral.CONFIRMATION, + "base": "main", + } + fields.update(overrides) + return ephemeral.ArmRequest(**fields) + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + + +def armed_runner(results: list) -> tuple: + spawn = FakeSpawn(results) + runner = exec_module.Runner( + mode=policy.ARMED, clock=FakeClock(), spawn=spawn, environ={}, deadline_seconds=3600 + ) + return (runner, spawn) + + +def by_id(results: list) -> dict: + return {result.spec.id: result for result in results} + + +def test_the_ephemeral_ref_is_derived_from_the_caller_supplied_token() -> None: + assert request().ref == f"{ephemeral.REF_PREFIX}/{TOKEN}" + + +@pytest.mark.parametrize( + "ref", ["main", "master", "refs/heads/main", "HEAD", "Production", "feature/x", ""] +) +def test_protected_and_malformed_push_targets_are_refused(ref: str) -> None: + with pytest.raises(ephemeral.ArmingError): + ephemeral.assert_push_target_allowed(ref) + + +def test_an_ephemeral_ref_is_the_only_accepted_push_target() -> None: + ephemeral.assert_push_target_allowed(f"{ephemeral.REF_PREFIX}/{TOKEN}") + + +@pytest.mark.parametrize( + ("overrides", "fragment"), + [ + ({"confirmation": "yes please"}, "confirmation phrase"), + ({"confirmation": ""}, "confirmation phrase"), + ({"repo": "atlas/other"}, "is not the expected"), + ({"token": "short"}, "ephemeral token"), + ({"token": "Has-Capitals-Here"}, "ephemeral token"), + ({"token": ""}, "ephemeral token"), + ({"base": ""}, "base branch is required"), + ], +) +def test_preflight_refuses_before_any_network_call(overrides: dict, fragment: str) -> None: + with pytest.raises(ephemeral.ArmingError) as caught: + ephemeral.preflight(request(**overrides), REPO) + assert fragment in str(caught.value) + + +def test_preflight_accepts_a_correctly_armed_request() -> None: + ephemeral.preflight(request(), REPO) + + +@pytest.mark.parametrize( + ("raw", "expected"), [("refs/heads/Main", "main"), (" main ", "main"), ("/x/", "x")] +) +def test_branch_names_are_normalised_before_comparison(raw: str, expected: str) -> None: + assert ephemeral.normalise_branch(raw) == expected + + +def test_the_protected_branch_guard_is_exercised_on_every_armed_run() -> None: + """A guard checked once is a guard nobody notices regressing.""" + runner, _ = armed_runner([completed(stdout="", returncode=2)]) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + guard = results["ephemeral.protected-branch-refusal"] + + assert guard.status == model.PASS + assert guard.evidence["accepted"] == [] + assert "main" in guard.evidence["refused"] + + +def test_a_broken_protected_branch_guard_fails_its_own_self_test(monkeypatch) -> None: + """If the guard ever stops refusing, the armed run says so instead of pushing.""" + monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None) + runner, _ = armed_runner([completed(stdout="", returncode=2)]) + + guard = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[ + "ephemeral.protected-branch-refusal" + ] + + assert guard.status == model.FAIL + assert "main" in guard.evidence["accepted"] + assert "the guard accepted" in guard.reason + + +def test_a_full_armed_run_pushes_opens_a_draft_and_removes_both() -> None: + runner, spawn = armed_runner( + [ + completed(stdout=""), # ls-remote: ref unused + completed(stdout=""), # push + completed(stdout=json.dumps({"number": 42, "draft": True})), # create draft + completed(stdout=""), # close + completed(stdout=""), # delete branch + completed(stdout=""), # ls-remote: gone + completed(stdout=json.dumps({"number": 42, "state": "closed"})), + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.feature-branch-push"].status == model.PASS + assert results["ephemeral.draft-pull-request"].status == model.PASS + assert results["ephemeral.draft-pull-request"].evidence["pull_request"] == 42 + assert results["ephemeral.cleanup-verified"].status == model.PASS + assert all(result.spec.mandatory for result in results.values()) + assert all(result.spec.scope == model.EPHEMERAL for result in results.values()) + + pushed = [call for call in spawn.calls if call["argv"][:2] == ["git", "push"]] + assert pushed[0]["argv"][-1] == f"HEAD:refs/heads/{request().ref}" + assert "--force" not in pushed[0]["argv"] + + titles = [arg for call in spawn.calls for arg in call["argv"] if arg.startswith("title=")] + assert titles[0].startswith(f"title={ephemeral.DRAFT_TITLE_PREFIX}") + + +def test_an_existing_ref_stops_the_run_before_it_writes() -> None: + runner, spawn = armed_runner([completed(stdout="abc123\trefs/heads/x\n")]) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert "already exists" in results["ephemeral.feature-branch-push"].reason + assert "ephemeral.draft-pull-request" not in results + assert len(spawn.calls) == 1 + + +def test_an_unverifiable_ref_check_is_not_run_rather_than_assumed_free() -> None: + runner, _ = armed_runner([completed(stderr="network down", returncode=128)]) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.feature-branch-push"].status == model.NOT_RUN + + +def test_a_failed_push_still_reports_cleanup_and_skips_the_draft() -> None: + runner, _ = armed_runner( + [ + completed(stdout=""), + completed(stderr="denied", returncode=1), + completed(stdout=""), # delete branch + completed(stdout=""), # ls-remote: gone + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert results["ephemeral.draft-pull-request"].status == model.NOT_RUN + assert results["ephemeral.cleanup-verified"].status == model.PASS + + +def test_a_draft_response_without_a_number_is_a_failure() -> None: + runner, _ = armed_runner( + [ + completed(stdout=""), + completed(stdout=""), + completed(stdout="{}"), + completed(stdout=""), + completed(stdout=""), + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.draft-pull-request"].status == model.FAIL + + +def test_a_branch_that_survives_cleanup_fails_the_cleanup_check() -> None: + runner, _ = armed_runner( + [ + completed(stdout=""), + completed(stdout=""), + completed(stdout=json.dumps({"number": 7})), + completed(stderr="cannot close", returncode=1), + completed(stderr="cannot delete", returncode=1), + completed(stdout="abc123\trefs/heads/x\n"), + completed(stdout=json.dumps({"number": 7, "state": "open"})), + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + cleanup = results["ephemeral.cleanup-verified"] + + assert cleanup.status == model.FAIL + assert "could not be closed" in cleanup.reason + assert "still exists" in cleanup.reason + assert "still open" in cleanup.reason + + +def test_cleanup_that_cannot_be_verified_is_a_failure_not_a_pass() -> None: + runner, _ = armed_runner( + [ + completed(stdout=""), + completed(stdout=""), + completed(stdout=json.dumps({"number": 7})), + completed(stdout=""), + completed(stdout=""), + subprocess.TimeoutExpired(cmd="git", timeout=1.0), + completed(stderr="gone", returncode=1), + ] + ) + + cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))["ephemeral.cleanup-verified"] + + assert cleanup.status == model.FAIL + assert "could not verify removal" in cleanup.reason + assert "could not confirm pull request #7 is closed" in cleanup.reason + + +def test_the_armed_results_replace_the_default_not_run_placeholders() -> None: + runner, _ = armed_runner([completed(stdout="abc\trefs/heads/x\n")]) + identifiers = {result.spec.id for result in ephemeral.run_armed(runner, VANTAGE, request())} + + assert identifiers <= { + "ephemeral.protected-branch-refusal", + "ephemeral.feature-branch-push", + "ephemeral.draft-pull-request", + "ephemeral.cleanup-verified", + } + + +def test_a_read_only_runner_cannot_execute_the_armed_flow() -> None: + """The mode, not the caller, is what opens the mutations.""" + spawn = FakeSpawn([completed(stdout=""), completed(stdout=""), completed(stdout="")]) + runner = exec_module.Runner(clock=FakeClock(), spawn=spawn, environ={}) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + push = results["ephemeral.feature-branch-push"] + + assert push.status == model.FAIL + assert "policy" in push.reason + assert not any(call["argv"][:2] == ["git", "push"] for call in spawn.calls) diff --git a/testing/tests/test_hermes_handoff_evaluators.py b/testing/tests/test_hermes_handoff_evaluators.py new file mode 100644 index 00000000..2e340c7b --- /dev/null +++ b/testing/tests/test_hermes_handoff_evaluators.py @@ -0,0 +1,201 @@ +"""Contracts for the text-shaped handoff acceptance evaluators.""" + +from __future__ import annotations + +from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step + +evaluators = load_handoff_module("hermes_handoff_evaluators") +model = load_handoff_module("hermes_handoff_model") + +REVIEW = model.REVIEW +ATTEMPT = model.ATTEMPT +FORBIDDEN = outcome(stderr="Error from server (Forbidden): pods is forbidden", returncode=1) + + +def deny_spec() -> object: + return spec( + "denied", + steps=(step("review", kind=REVIEW), step("attempt", kind=ATTEMPT)), + ) + + +def evaluate(built, outcomes): + return evaluators.evaluate(built, outcomes) + + +def test_a_deny_check_passes_only_on_a_review_no_and_a_live_refusal() -> None: + evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN}) + + assert evaluation.status == model.PASS + assert evaluation.evidence == {"review": "no", "attempt": "refused"} + + +def test_a_deny_check_fails_when_the_review_says_the_authority_exists() -> None: + evaluation = evaluate(deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN}) + + assert evaluation.status == model.FAIL + assert "reported yes" in evaluation.reason + + +def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> None: + evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()}) + + assert evaluation.status == model.FAIL + assert "must be refused" in evaluation.reason + + +def test_a_request_that_merely_broke_is_not_evidence_of_a_refusal() -> None: + broken = outcome(stderr="connection refused", returncode=7) + evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": broken}) + + assert evaluation.status == model.NOT_RUN + assert "recognisable refusal" in evaluation.reason + + +def test_a_deny_check_without_a_live_attempt_refuses_to_classify() -> None: + """An authorization review states policy; only an attempt proves enforcement.""" + review_only = spec("denied", steps=(step("review", kind=REVIEW),)) + + evaluation = evaluate(review_only, {"review": outcome(stdout="no")}) + + assert evaluation.status == model.NOT_RUN + assert "require a real attempt" in evaluation.reason + + +def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> None: + built = spec("allowed", steps=(step("review", kind=REVIEW), step("read"))) + + assert evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status == model.PASS + + denied = evaluate(built, {"review": outcome(stdout="no"), "read": outcome()}) + assert denied.status == model.FAIL and "reported no" in denied.reason + + failed = evaluate(built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)}) + assert failed.status == model.FAIL and "step read failed" in failed.reason + + +def test_names_absent_flags_exact_names_and_substrings() -> None: + built = spec( + "names_absent", + {"step": "env", "names": ("GITEA_TOKEN",), "contains": ("API_KEY",)}, + steps=(step("env"),), + ) + + clean = evaluate(built, {"env": outcome(stdout="PATH\nHOME\n")}) + assert clean.status == model.PASS and clean.evidence["observed_count"] == 2 + + leaked = evaluate(built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")}) + assert leaked.status == model.FAIL + assert leaked.evidence["offenders"] == ["GITEA_TOKEN", "OPENAI_API_KEY"] + + unreadable = evaluate(built, {"env": outcome(returncode=1)}) + assert unreadable.status == model.NOT_RUN + + +def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> None: + built = spec( + "names_present", + {"step": "init", "names": ("patch-web-session-activity",), "contains": ("@sha256:",)}, + steps=(step("init"),), + ) + + good = outcome(stdout="patch-web-session-activity\nimage@sha256:abc\n") + assert evaluate(built, {"init": good}).status == model.PASS + + partial = evaluate(built, {"init": outcome(stdout="patch-web-session-activity\n")}) + assert partial.status == model.FAIL and "@sha256:" in partial.reason + + empty = spec("names_present", {"step": "init"}, steps=(step("init"),)) + assert evaluate(empty, {"init": outcome(stdout="")}).status == model.NOT_RUN + + +def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None: + single = spec("stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),)) + assert evaluate(single, {"s": outcome(stdout=" locked \n")}).status == model.PASS + + mismatch = evaluate(single, {"s": outcome(stdout="unlocked")}) + assert mismatch.status == model.FAIL and mismatch.evidence["observed"] == "unlocked" + + several = spec( + "stdout_matches", {"step": "s", "equals": ["600 root:root", "absent"]}, steps=(step("s"),) + ) + assert evaluate(several, {"s": outcome(stdout="absent")}).status == model.PASS + + assert evaluate(single, {"s": outcome(returncode=1)}).status == model.NOT_RUN + + +def test_distinct_count_separates_replica_count_from_actual_spread() -> None: + built = spec( + "distinct_count", {"step": "n", "minimum": 3, "total_equals": 3}, steps=(step("n"),) + ) + + spread = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-06\ntitan-07\n")}) + assert spread.status == model.PASS and spread.evidence["distinct"] == 3 + + stacked = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-05\ntitan-05\n")}) + assert stacked.status == model.FAIL and "distinct values" in stacked.reason + + extra = spec("distinct_count", {"step": "n", "minimum": 1, "total_equals": 2}, steps=(step("n"),)) + over = evaluate(extra, {"n": outcome(stdout="a\nb\nc\n")}) + assert over.status == model.FAIL and "expected exactly 2" in over.reason + + assert evaluate(built, {"n": outcome(stdout="")}).status == model.NOT_RUN + assert evaluate(built, {"n": outcome(returncode=1)}).status == model.NOT_RUN + + +def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None: + built = spec( + "vantages_agree", + {"steps": ("operator", "self")}, + steps=(step("operator", vantage="operator"), step("self")), + ) + + agree = evaluate( + built, + {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/b\nns/a\n")}, + ) + assert agree.status == model.PASS + + differ = evaluate( + built, {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")} + ) + assert differ.status == model.FAIL and "disagree" in differ.reason + + empty = evaluate(built, {"operator": outcome(stdout=""), "self": outcome(stdout="")}) + assert empty.status == model.NOT_RUN + + broken = evaluate(built, {"operator": outcome(returncode=1), "self": outcome()}) + assert broken.status == model.NOT_RUN + + +def test_steps_agree_is_the_same_rule_under_a_neutral_name() -> None: + assert evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"] + + +def test_lines_match_catches_a_blank_leading_column() -> None: + built = spec( + "lines_match", + {"step": "s", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + steps=(step("s"),), + ) + table = "Preview Src ID\n-------\nwork on task cli 2026\n" + + assert evaluate(built, {"s": outcome(stdout=table)}).status == model.PASS + + blank = "Preview Src ID\n-------\n cli 2026\n" + failed = evaluate(built, {"s": outcome(stdout=blank)}) + assert failed.status == model.FAIL and failed.evidence["offenders"] + + assert evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status == model.NOT_RUN + assert evaluate(built, {"s": outcome(returncode=1)}).status == model.NOT_RUN + + +def test_not_armed_is_the_fixed_skip_a_default_run_must_produce() -> None: + built = spec("not_armed", {"reason": "ephemeral mutation mode is not armed"}) + + evaluation = evaluate(built, {}) + + assert evaluation.status == model.NOT_RUN + assert evaluation.evidence == {"armed": False} + assert evaluation.reason == "ephemeral mutation mode is not armed" + assert evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed" diff --git a/testing/tests/test_hermes_handoff_exec.py b/testing/tests/test_hermes_handoff_exec.py new file mode 100644 index 00000000..cd8fc7d2 --- /dev/null +++ b/testing/tests/test_hermes_handoff_exec.py @@ -0,0 +1,173 @@ +"""Contracts for bounded, screened command execution in the handoff harness.""" + +from __future__ import annotations + +import subprocess + +from testing.tests.test_hermes_handoff_support import ( + FakeClock, + FakeSpawn, + load_handoff_module, +) + +runner_module = load_handoff_module("hermes_handoff_exec") +policy = load_handoff_module("hermes_handoff_policy") + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + + +def build(results: list, **kwargs) -> tuple: + clock = FakeClock() + spawn = FakeSpawn(results) + runner = runner_module.Runner(clock=clock, spawn=spawn, environ={}, **kwargs) + return (runner, spawn, clock) + + +OPERATOR = runner_module.operator_vantage() + + +def test_a_successful_command_is_captured_and_screened() -> None: + runner, spawn, _ = build([completed(stdout="password=a-long-enough-secret\n")]) + + outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + + assert outcome.ok and outcome.ran + assert "[redacted]" in outcome.stdout + assert outcome.returncode == 0 + assert spawn.calls[0]["argv"] == ["kubectl", "get", "pods"] + assert runner.commands_run == 1 + + +def test_a_failing_command_is_not_ok_but_still_ran() -> None: + runner, _, _ = build([completed(stderr="Error from server (Forbidden)", returncode=1)]) + + outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + + assert outcome.ran and not outcome.ok + assert "Forbidden" in outcome.combined + + +def test_a_policy_violation_never_reaches_a_subprocess() -> None: + runner, spawn, _ = build([]) + + outcome = runner.run(("kubectl", "delete", "pods", "--all"), OPERATOR) + + assert outcome.error and outcome.error.startswith("policy:") + assert not outcome.ran + assert spawn.calls == [] + assert runner.commands_run == 0 + + +def test_a_timeout_is_recorded_rather_than_raised() -> None: + runner, _, _ = build([subprocess.TimeoutExpired(cmd="kubectl", timeout=1.0)]) + + outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + + assert outcome.error == runner_module.TIMEOUT_ERROR + assert not outcome.ran + + +def test_a_missing_binary_is_recorded_rather_than_raised() -> None: + runner, _, _ = build([FileNotFoundError(2, "No such file or directory")]) + + outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + + assert outcome.error and outcome.error.startswith("spawn:") + + +def test_the_run_deadline_stops_further_probes_without_spawning() -> None: + runner, spawn, clock = build([completed(stdout="first")], deadline_seconds=10.0) + + assert runner.run(("kubectl", "get", "pods"), OPERATOR).ok + clock.advance(11.0) + second = runner.run(("kubectl", "get", "nodes"), OPERATOR) + + assert second.error == runner_module.DEADLINE_ERROR + assert len(spawn.calls) == 1 + assert runner.remaining_seconds < 0 + + +def test_the_command_timeout_is_clamped_by_the_remaining_deadline() -> None: + runner, spawn, clock = build([completed()], command_timeout=30.0, deadline_seconds=10.0) + clock.advance(6.0) + + runner.run(("kubectl", "get", "pods"), OPERATOR) + + assert spawn.calls[0]["timeout"] == 4.0 + + +def test_output_is_bounded_per_command_and_can_be_raised_per_call() -> None: + runner, _, _ = build([completed(stdout="y" * 200), completed(stdout="y" * 200)], max_bytes=50) + + small = runner.run(("kubectl", "get", "pods"), OPERATOR) + large = runner.run(("kubectl", "get", "pods"), OPERATOR, max_bytes=4096) + + assert small.truncated + assert not large.truncated + + +def test_the_child_environment_is_rebuilt_from_an_allowlist() -> None: + source = {"PATH": "/bin", "KUBECONFIG": "/tmp/kc", "ANTHROPIC_API_KEY": "leak", "HOME": "/root"} + environment = runner_module.build_environment(OPERATOR, source) + + assert environment == {"PATH": "/bin", "KUBECONFIG": "/tmp/kc", "HOME": "/root", "LC_ALL": "C"} + assert "ANTHROPIC_API_KEY" not in environment + + +def test_a_vantage_can_override_the_environment_it_needs() -> None: + vantage = runner_module.operator_vantage(kubeconfig="/tmp/operator.yaml") + environment = runner_module.build_environment(vantage, {"KUBECONFIG": "/tmp/other"}) + + assert environment["KUBECONFIG"] == "/tmp/operator.yaml" + assert "kubeconfig-pinned" in vantage.description + + +def test_the_operator_context_is_inserted_only_for_context_aware_tools() -> None: + vantage = runner_module.operator_vantage(context="atlas-operator") + + assert vantage.wrap(("kubectl", "get", "pods"))[:3] == ("kubectl", "--context", "atlas-operator") + assert vantage.wrap(("git", "status")) == ("git", "status") + assert vantage.wrap(()) == () + + +def test_the_pod_vantage_execs_into_the_container_and_drops_the_context() -> None: + operator = runner_module.operator_vantage(context="atlas-operator") + vantage = runner_module.pod_vantage("hermes", "hermes-agent-1", "hermes", operator) + + wrapped = vantage.wrap(("kubectl", "get", "pods")) + + assert wrapped[:3] == ("kubectl", "--context", "atlas-operator") + assert wrapped[3] == "exec" + assert wrapped[-4:] == ("--", "kubectl", "get", "pods") + assert vantage.description == "in-pod hermes/hermes-agent-1[hermes]" + policy.check_argv(wrapped) + + +def test_a_pod_vantage_without_an_operator_still_addresses_the_pod() -> None: + vantage = runner_module.pod_vantage("hermes", "pod-a", "hermes") + + assert vantage.wrap(("kubectl", "version"))[0] == "kubectl" + assert vantage.env == {} + + +def test_outcome_serialisation_screens_the_command_line() -> None: + outcome = runner_module.Outcome( + argv=("git", "clone", "https://user:a-long-enough-secret@scm.example.dev/x"), + vantage="operator", + returncode=0, + stdout="ok", + ) + + payload = outcome.as_dict() + + assert "a-long-enough-secret" not in payload["command"] + assert payload["vantage"] == "operator" + assert payload["stdout"] == "ok" + + +def test_combined_output_joins_only_the_streams_that_carry_text() -> None: + assert runner_module.Outcome(argv=(), vantage="x", stdout="a", stderr="b").combined == "a\nb" + assert runner_module.Outcome(argv=(), vantage="x", stdout="a").combined == "a" + assert runner_module.Outcome(argv=(), vantage="x").combined == "" diff --git a/testing/tests/test_hermes_handoff_json_rules.py b/testing/tests/test_hermes_handoff_json_rules.py new file mode 100644 index 00000000..9bb5c673 --- /dev/null +++ b/testing/tests/test_hermes_handoff_json_rules.py @@ -0,0 +1,279 @@ +"""Contracts for the JSON-shaped handoff acceptance evaluators.""" + +from __future__ import annotations + +import datetime as dt +import json + +from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step + +evaluators = load_handoff_module("hermes_handoff_evaluators") +model = load_handoff_module("hermes_handoff_model") + +NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc) + + +def evaluate(built, payload): + body = payload if isinstance(payload, str) else json.dumps(payload) + return evaluators.evaluate(built, {"s": outcome(stdout=body)}) + + +def one(rule: str, expect: dict): + return spec(rule, {"step": "s", **expect}, steps=(step("s"),)) + + +def test_json_field_compares_paths_against_expected_values() -> None: + built = one("json_field", {"fields": {"authenticated": True, "transport": ["a", "b"]}}) + + assert evaluate(built, {"authenticated": True, "transport": "b"}).status == model.PASS + + wrong = evaluate(built, {"authenticated": False, "transport": "c"}) + assert wrong.status == model.FAIL + assert "authenticated=False" in wrong.reason and "transport='c'" in wrong.reason + + assert evaluate(built, {"authenticated": True}).status == model.NOT_RUN + + +def test_json_field_skips_rather_than_guesses_on_unusable_output() -> None: + built = one("json_field", {"fields": {"a": 1}}) + + assert evaluate(built, "not json").status == model.NOT_RUN + failed = evaluators.evaluate(built, {"s": outcome(returncode=1)}) + assert failed.status == model.NOT_RUN + + +def test_json_numeric_enforces_inclusive_bounds_and_rejects_non_numbers() -> None: + built = one("json_numeric", {"fields": {"latency_ms": {"min": 1, "max": 1000}}}) + + assert evaluate(built, {"latency_ms": 500}).status == model.PASS + assert evaluate(built, {"latency_ms": 1}).status == model.PASS + + low = evaluate(built, {"latency_ms": 0}) + assert low.status == model.FAIL and "below minimum" in low.reason + + high = evaluate(built, {"latency_ms": 5000}) + assert high.status == model.FAIL and "above maximum" in high.reason + + assert evaluate(built, {"latency_ms": True}).status == model.NOT_RUN + assert evaluate(built, {"latency_ms": "fast"}).status == model.NOT_RUN + assert evaluate(built, {}).status == model.NOT_RUN + + +def test_json_recent_treats_stale_evidence_as_a_failure() -> None: + built = one("json_recent", {"now": NOW, "fields": {"checked_at": 3600}}) + + fresh = evaluate(built, {"checked_at": "2026-08-17T11:30:00Z"}) + assert fresh.status == model.PASS and fresh.evidence["checked_at"]["age_seconds"] == 1800 + + stale = evaluate(built, {"checked_at": "2026-08-16T00:00:00Z"}) + assert stale.status == model.FAIL and "limit 3600s" in stale.reason + + naive = evaluate(built, {"checked_at": "2026-08-17T11:30:00"}) + assert naive.status == model.PASS + + assert evaluate(built, {"checked_at": "yesterday"}).status == model.NOT_RUN + assert evaluate(built, {}).status == model.NOT_RUN + + +def test_all_of_field_selects_keys_and_reports_every_mismatch() -> None: + built = one("all_of_field", {"key_field": "number", "keys": [14, 15], "fields": {"merged": True}}) + payload = [ + {"number": 14, "merged": True}, + {"number": 15, "merged": False}, + {"number": 16, "merged": False}, + ] + + evaluation = evaluate(built, payload) + + assert evaluation.status == model.FAIL + assert "15: merged=False" in evaluation.reason + assert "16" not in evaluation.reason + assert evaluation.evidence["examined"] == [14, 15] + + +def test_all_of_field_skips_when_an_expected_key_is_absent_or_nothing_matched() -> None: + keyed = one("all_of_field", {"key_field": "number", "keys": [14], "fields": {"merged": True}}) + absent = evaluate(keyed, [{"number": 99, "merged": True}]) + assert absent.status == model.NOT_RUN and "expected keys absent" in absent.reason + + unkeyed = one("all_of_field", {"fields": {"status": ["done"]}}) + assert evaluate(unkeyed, []).status == model.NOT_RUN + assert evaluate(unkeyed, {"not": "an array"}).status == model.NOT_RUN + + +def test_all_of_field_skips_when_the_named_array_path_is_absent() -> None: + built = spec( + "all_of_field", {"step": "s", "array": "items", "fields": {"a": 1}}, steps=(step("s"),) + ) + + evaluation = evaluate(built, {"other": []}) + + assert evaluation.status == model.NOT_RUN + assert "array path is absent" in evaluation.reason + + +def test_all_of_field_treats_an_absent_field_as_a_mismatch_not_a_crash() -> None: + built = one("all_of_field", {"key_field": "name", "fields": {"suspended": False}}) + + evaluation = evaluate(built, [{"name": "a"}]) + + assert evaluation.status == model.FAIL + assert "suspended=None" in evaluation.reason + + +def test_all_of_field_honours_exemptions_and_non_empty_paths() -> None: + exempted = one( + "all_of_field", + {"key_field": "name", "exempt_keys": ["parked"], "fields": {"suspended": False}}, + ) + payload = [{"name": "parked", "suspended": True}, {"name": "live", "suspended": False}] + assert evaluate(exempted, payload).status == model.PASS + + non_empty = one("all_of_field", {"key_field": "id", "non_empty": ["title"]}) + assert evaluate(non_empty, [{"id": "a", "title": "work"}]).status == model.PASS + blank = evaluate(non_empty, [{"id": "a", "title": ""}, {"id": "b"}]) + assert blank.status == model.FAIL and "is empty" in blank.reason + + +def test_json_record_captures_named_paths_or_skips() -> None: + built = one("json_record", {"record": {"numbers": "[].number", "refs": "[].head.ref"}}) + payload = [{"number": 14, "head": {"ref": "a"}}, {"number": 15, "head": {"ref": "b"}}] + + evaluation = evaluate(built, payload) + assert evaluation.status == model.PASS + assert evaluation.evidence == {"numbers": [14, 15], "refs": ["a", "b"]} + + missing = evaluate(built, [{"number": 14}]) + assert missing.status == model.NOT_RUN and "absent" in missing.reason + + +def routing_log(*records: dict) -> str: + return "\n".join(json.dumps(record) for record in records) + + +def routing_spec(**extra): + return spec( + "routing_evidence", + { + "step": "s", + "providers": ("codex", "claude"), + "efforts": ("high", "xhigh"), + "lanes": ("route", "worker"), + "require_fallback_evidence": True, + **extra, + }, + steps=(step("s"),), + ) + + +FULL_LOG = routing_log( + {"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}, + {"ts": "2026-08-17T11:30:00Z", "model": "route/claude/opus/xhigh"}, + {"ts": "2026-08-17T11:45:00Z", "model": "worker/codex/sol/xhigh", "fallback_reason": "unavailable"}, + {"ts": "2026-08-17T11:50:00Z", "model": "qwen2.5:14b", "tier": "classifier"}, +) + + +def test_routing_evidence_summarises_lanes_providers_efforts_and_fallbacks() -> None: + evaluation = evaluate(routing_spec(), FULL_LOG) + + assert evaluation.status == model.PASS + assert evaluation.evidence["providers"] == ["claude", "codex"] + assert evaluation.evidence["efforts"] == ["high", "xhigh"] + assert evaluation.evidence["lanes"] == ["route", "worker"] + assert evaluation.evidence["fallback_reasons"] == ["unavailable"] + assert evaluation.evidence["records"] == 4 and evaluation.evidence["routed"] == 3 + + +def test_routing_evidence_names_exactly_what_the_window_did_not_show() -> None: + partial = routing_log({"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}) + + evaluation = evaluate(routing_spec(), partial) + + assert evaluation.status == model.FAIL + assert "provider:claude" in evaluation.reason + assert "effort:xhigh" in evaluation.reason + assert "lane:worker" in evaluation.reason + assert "fallback:none-observed" in evaluation.reason + + +def test_routing_evidence_tolerates_a_clipped_tail_but_not_an_empty_one() -> None: + clipped = '{"ts": "2026-08-17T11:0\n' + FULL_LOG + + assert evaluate(routing_spec(), clipped).status == model.PASS + assert evaluate(routing_spec(), "").status == model.NOT_RUN + assert evaluate(routing_spec(), "garbage\n").status == model.NOT_RUN + + +def test_routing_evidence_can_require_freshness() -> None: + fresh = evaluate(routing_spec(max_age_seconds=3600, now=NOW), FULL_LOG) + assert fresh.status == model.PASS and fresh.evidence["age_seconds"] == 600 + + stale = evaluate( + routing_spec(max_age_seconds=60, now=NOW + dt.timedelta(days=2)), FULL_LOG + ) + assert stale.status == model.FAIL and "freshness" in stale.reason + + undated = routing_log({"model": "route/codex/sol/high"}, {"model": "route/claude/opus/xhigh"}) + unknown = evaluate(routing_spec(max_age_seconds=60, now=NOW), undated) + assert unknown.status == model.NOT_RUN + + +def flux_spec(*expected: str): + return spec( + "flux_health", + {"step": "s", "expected_suspensions": expected}, + steps=(step("s"),), + ) + + +def test_flux_health_separates_parked_objects_from_broken_ones() -> None: + rows = ( + "flux-system/core\tfalse\tReady=True,Healthy=True,\n" + "flux-system/parked\ttrue\tReady=False,\n" + "flux-system/broken\tfalse\tReady=False,Healthy=False,\n" + ) + + evaluation = evaluators.evaluate(flux_spec("flux-system/parked"), {"s": outcome(stdout=rows)}) + + assert evaluation.status == model.FAIL + assert evaluation.evidence["unhealthy"] == ["flux-system/broken"] + assert evaluation.evidence["unexpected_suspensions"] == [] + assert "unhealthy: flux-system/broken" in evaluation.reason + + +def test_flux_health_flags_a_suspension_nobody_declared() -> None: + rows = "flux-system/core\tfalse\tReady=True,\nflux-system/parked\ttrue\tReady=False,\n" + + evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) + + assert evaluation.status == model.FAIL + assert evaluation.evidence["unexpected_suspensions"] == ["flux-system/parked"] + + +def test_flux_health_does_not_call_an_in_flight_reconcile_unhealthy() -> None: + """Flux drops Ready to False for the duration of a reconcile.""" + rows = ( + "flux-system/reconciling\tfalse\tReconciling=True,Ready=False,\n" + "flux-system/degraded\tfalse\tReady=False,Healthy=True,\n" + "flux-system/waiting\t\tReady=Unknown,\n" + ) + + evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) + + assert evaluation.status == model.PASS + assert evaluation.evidence["total"] == 3 + + +def test_blank_rows_in_a_projection_are_ignored_by_both_log_parsers() -> None: + padded = "\n\n" + FULL_LOG + "\n\n" + assert evaluate(routing_spec(), padded).status == model.PASS + + rows = "\n\nflux-system/core\tfalse\tReady=True,\n\n" + evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) + assert evaluation.status == model.PASS and evaluation.evidence["total"] == 1 + + +def test_flux_health_skips_on_an_empty_or_failed_projection() -> None: + assert evaluators.evaluate(flux_spec(), {"s": outcome(stdout="")}).status == model.NOT_RUN + assert evaluators.evaluate(flux_spec(), {"s": outcome(returncode=1)}).status == model.NOT_RUN diff --git a/testing/tests/test_hermes_handoff_model.py b/testing/tests/test_hermes_handoff_model.py new file mode 100644 index 00000000..37c3f9b9 --- /dev/null +++ b/testing/tests/test_hermes_handoff_model.py @@ -0,0 +1,180 @@ +"""Contracts for the handoff harness result model, verdict, and rendering.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step + +model = load_handoff_module("hermes_handoff_model") + + +def result(status: str, identifier: str = "test.check", mandatory: bool = True, group: str = "g"): + return model.CheckResult( + spec=model.CheckSpec( + id=identifier, title="t", group=group, rule="allowed", mandatory=mandatory + ), + status=status, + reason=f"{status} reason", + ) + + +def report(*results, errors: tuple[str, ...] = ()) -> model.Report: + return model.Report( + mode="read-only", + started_at="2026-08-17T00:00:00Z", + results=list(results), + harness_errors=list(errors), + ) + + +@pytest.mark.parametrize( + ("status", "mandatory", "blocking"), + [ + (model.PASS, True, False), + (model.NOT_APPLICABLE, True, False), + (model.FAIL, True, True), + (model.NOT_RUN, True, True), + (model.FAIL, False, False), + (model.NOT_RUN, False, False), + ], +) +def test_only_a_mandatory_failure_or_skip_blocks(status, mandatory, blocking) -> None: + assert result(status, mandatory=mandatory).blocking is blocking + + +def test_a_mandatory_skip_is_a_no_go_exactly_like_a_failure() -> None: + """The whole point of the four-valued classification.""" + assert report(result(model.NOT_RUN)).decision == model.NO_GO + assert report(result(model.FAIL)).decision == model.NO_GO + assert report(result(model.PASS), result(model.NOT_APPLICABLE)).decision == model.GO + + +def test_a_harness_error_is_a_no_go_even_when_every_check_passed() -> None: + clean = report(result(model.PASS), errors=("the two vantages are the same principal",)) + assert clean.decision == model.NO_GO + + +def test_a_zero_state_report_is_a_go_only_because_nothing_was_mandatory() -> None: + """An empty catalog has nothing blocking; the harness errors are what stop it.""" + assert report().decision == model.GO + assert report(errors=("no checks were built",)).decision == model.NO_GO + + +def test_blocking_results_are_ordered_failures_before_skips() -> None: + ordered = report( + result(model.NOT_RUN, "b.skip"), + result(model.FAIL, "z.fail"), + result(model.PASS, "a.pass"), + ).blocking + + assert [item.spec.id for item in ordered] == ["z.fail", "b.skip"] + + +def test_counts_and_group_summary_tally_every_status() -> None: + built = report( + result(model.PASS, "a", group="alpha"), + result(model.FAIL, "b", group="alpha"), + result(model.NOT_RUN, "c", group="beta"), + ) + + assert built.counts == { + model.PASS: 1, + model.FAIL: 1, + model.NOT_RUN: 1, + model.NOT_APPLICABLE: 0, + } + assert built.group_summary()["alpha"][model.PASS] == 1 + assert built.group_summary()["beta"][model.NOT_RUN] == 1 + + +def test_the_serialised_report_carries_the_verdict_and_its_reasons() -> None: + built = report(result(model.FAIL, "x.fail"), result(model.PASS, "y.pass")) + built.vantages = [ + model.VantageRecord(name="operator", description="d", identity="u", available=True) + ] + + payload = built.as_dict() + + assert payload["decision"] == model.NO_GO + assert payload["blocking"] == ["x.fail"] + assert payload["harness"] == model.HARNESS + assert payload["schema_version"] == model.SCHEMA_VERSION + assert payload["vantages"][0]["identity"] == "u" + assert {check["id"] for check in payload["checks"]} == {"x.fail", "y.pass"} + + +def test_check_serialisation_screens_evidence_and_can_omit_commands() -> None: + built = model.CheckResult( + spec=spec("allowed", identifier="x", steps=(step("a"),)), + status=model.PASS, + evidence={"token": "a-long-enough-secret", "count": 3}, + outcomes=[outcome(stdout="fine")], + ) + + with_commands = built.as_dict() + without = built.as_dict(include_outcomes=False) + + assert with_commands["evidence"]["token"] == "[redacted]" + assert with_commands["evidence"]["count"] == 3 + assert len(with_commands["commands"]) == 1 + assert "commands" not in without + assert with_commands["vantages"] == ["self"] + + +def test_rationale_is_carried_only_when_a_check_has_one() -> None: + without = model.CheckResult(spec=spec("allowed"), status=model.PASS).as_dict() + assert "rationale" not in without + + annotated = model.CheckSpec( + id="x", title="t", group="g", rule="allowed", rationale="because" + ) + with_reason = model.CheckResult(spec=annotated, status=model.PASS).as_dict() + assert with_reason["rationale"] == "because" + + +def test_distinct_vantages_are_reported_in_first_use_order() -> None: + built = spec( + "allowed", + steps=(step("a", vantage="operator"), step("b", vantage="self"), step("c", vantage="operator")), + ) + assert built.vantages == ("operator", "self") + + +def test_the_human_summary_names_the_verdict_vantages_and_blockers() -> None: + built = report(result(model.FAIL, "x.fail", group="alpha"), errors=("collision",)) + built.vantages = [ + model.VantageRecord(name="operator", description="d", identity="u", available=True), + model.VantageRecord(name="self", description="d", available=False), + ] + + summary = built.render_summary() + + assert summary.splitlines()[0] == f"{model.HARNESS}: {model.NO_GO}" + assert "vantage operator: available identity=u" in summary + assert "vantage self: unavailable identity=unknown" in summary + assert "harness error: collision" in summary + assert "x.fail" in summary + assert "alpha" in summary + + +def test_utc_now_renders_a_zulu_timestamp_and_accepts_an_injected_clock() -> None: + fixed = dt.datetime(2026, 8, 17, 9, 30, 15, 123456, tzinfo=dt.timezone.utc) + assert model.utc_now(lambda: fixed) == "2026-08-17T09:30:15Z" + assert model.utc_now().endswith("Z") + + +def test_unscreened_fields_walks_the_whole_payload() -> None: + offenders = model.unscreened_fields( + { + "clean": "hermes", + "nested": {"leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234"}, + "items": [1, "Authorization: Bearer abcdefghijklmnop"], + } + ) + + assert offenders == ["$.nested.leak", "$.items[1]"] + assert model.unscreened_fields({"clean": "hermes"}) == [] + assert model.unscreened_fields(None) == [] diff --git a/testing/tests/test_hermes_handoff_policy.py b/testing/tests/test_hermes_handoff_policy.py new file mode 100644 index 00000000..13fdc8b4 --- /dev/null +++ b/testing/tests/test_hermes_handoff_policy.py @@ -0,0 +1,159 @@ +"""Contracts for the fail-closed command policy of the handoff harness.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module + +policy = load_handoff_module("hermes_handoff_policy") + + +def check(*argv: str, mode: str | None = None, inner: bool = False) -> None: + policy.check_argv(argv, mode or policy.READ_ONLY, allow_impersonation=inner) + + +@pytest.mark.parametrize( + "argv", + [ + ("kubectl", "--namespace", "hermes", "get", "pods"), + ("kubectl", "auth", "can-i", "get", "secrets", "--all-namespaces"), + ("kubectl", "get", "secrets", "--all-namespaces", "-o", "name"), + ("kubectl", "get", "secrets", "--output=name"), + ("kubectl", "logs", "deploy/x", "--tail", "1"), + ("kubectl", "--context", "atlas", "get", "nodes", "-o", "json"), + ("flux", "get", "kustomizations", "--all-namespaces"), + ("helm", "list", "--all-namespaces"), + ("git", "merge-base", "--is-ancestor", "a", "b"), + ("git", "config", "--get", "remote.origin.url"), + ("hermes", "kanban", "list", "--json"), + ("hermes", "sessions", "list", "--source", "telegram"), + ("hermes", "status"), + (policy.GITEA_CLIENT, "GET", "/api/v1/user"), + ], +) +def test_read_only_commands_are_permitted(argv: tuple[str, ...]) -> None: + check(*argv) + + +@pytest.mark.parametrize( + ("argv", "fragment"), + [ + ((), "empty command"), + (("vault", "read", "kv/x"), "forbidden binary"), + (("curl", "https://example.dev"), "forbidden binary"), + (("rm", "-rf", "/"), "allowlist"), + (("kubectl", "get", "pods", "--token", "abc"), "forbidden argument"), + (("kubectl", "get", "pods", "--as=system:admin"), "forbidden argument"), + (("kubectl", "get", "secrets", "-A"), "-o name"), + (("kubectl", "describe", "secrets", "x"), "-o name"), + (("kubectl", "delete", "-n", "get", "secrets"), "requires a dry run"), + (("kubectl", "port-forward", "svc/x", "80"), "outside the read-only boundary"), + (("kubectl", "exec", "pod"), "-- command boundary"), + (("kubectl", "exec", "pod", "--"), "inner command"), + (("kubectl",), "outside the read-only boundary"), + (("flux", "reconcile", "kustomization", "x"), "outside the read-only boundary"), + (("flux",), "requires a subcommand"), + (("helm", "upgrade", "x"), "outside the read-only boundary"), + (("git", "commit", "-m", "x"), "not permitted"), + (("git", "push", "origin", "HEAD:refs/heads/x"), "not permitted"), + (("git", "config", "user.name", "x"), "may only read"), + (("hermes", "kanban", "complete", "t_1"), "outside the read-only boundary"), + (("hermes",), "requires a subcommand"), + ((policy.GITEA_CLIENT, "POST", "/api/v1/x"), "not permitted"), + ((policy.GITEA_CLIENT, "GET"), "method and a path"), + (("sh", "-c", "rm -rf /"), "frozen template"), + (("sh", "echo"), "frozen template"), + (("git", "log", "/etc/shadow"), "credential path"), + ], +) +def test_unsafe_commands_are_refused(argv: tuple[str, ...], fragment: str) -> None: + with pytest.raises(policy.PolicyError) as caught: + check(*argv) + assert fragment in str(caught.value) + + +def test_unknown_mode_is_refused() -> None: + with pytest.raises(policy.PolicyError, match="unknown mode"): + policy.check_argv(("kubectl", "get", "pods"), "whatever") + + +def test_dry_run_reaches_mutating_subcommands_without_writing() -> None: + check("kubectl", "--namespace", "hermes", "patch", "deploy/x", "--dry-run=server") + check("kubectl", "create", "configmap", "probe", "--dry-run=server") + + +def test_a_resource_named_after_a_subcommand_cannot_smuggle_a_delete() -> None: + """`-n get` must not make a delete look like a read.""" + with pytest.raises(policy.PolicyError, match="requires a dry run"): + check("kubectl", "delete", "-n", "get", "configmap", "x") + + +def test_impersonation_is_operator_forbidden_and_in_pod_permitted() -> None: + with pytest.raises(policy.PolicyError, match="forbidden argument"): + check("kubectl", "get", "namespaces", "--as", "system:admin") + check("kubectl", "get", "namespaces", "--as", "system:admin", inner=True) + + +def test_exec_validates_its_inner_command_and_tolerates_impersonation_inside() -> None: + inner = policy.shell("path_readable", path="/tmp") + check("kubectl", "exec", "--namespace", "hermes", "pod", "--", *inner) + check("kubectl", "exec", "pod", "--", "kubectl", "get", "ns", "--as", "system:admin") + with pytest.raises(policy.PolicyError, match="allowlist"): + check("kubectl", "exec", "pod", "--", "bash", "-c", "id") + + +def test_exec_of_a_shell_template_naming_a_secret_path_is_permitted() -> None: + """The path scan stops at `--`; the tail is a reviewed template, not free text.""" + inner = policy.shell("account_lock_state", account="hermes-agent", path="/host-etc/shadow") + check("kubectl", "exec", "--namespace", "hermes", "pod", "--", *inner) + + +def test_armed_mode_opens_exactly_the_mutations_it_should() -> None: + check("git", "push", "origin", "HEAD:refs/heads/x", mode=policy.ARMED) + check(policy.GITEA_CLIENT, "POST", "/api/v1/x", mode=policy.ARMED) + check(policy.GITEA_CLIENT, "DELETE", "/api/v1/x", mode=policy.ARMED) + with pytest.raises(policy.PolicyError, match="may not force"): + check("git", "push", "--force", "origin", "x", mode=policy.ARMED) + with pytest.raises(policy.PolicyError, match="may not force"): + check("git", "push", "--all", "origin", mode=policy.ARMED) + with pytest.raises(policy.PolicyError, match="not permitted"): + check(policy.GITEA_CLIENT, "PUT", "/api/v1/x", mode=policy.ARMED) + + +def test_every_frozen_template_renders_and_is_accepted() -> None: + parameters = { + "account": "hermes-agent", + "askpass": "/opt/coordinator/gitea_askpass.sh", + "fields": "state,model", + "limit": "200", + "path": "/host-etc/passwd", + "url": "https://scm.example.dev/atlas/repo.git", + } + for name in policy.SHELL_TEMPLATES: + required = policy.template_parameters(name) + argv = policy.shell(name, **{key: parameters[key] for key in required}) + policy.check_argv(argv) + assert argv[0] == "sh" + + +def test_template_rendering_rejects_unknown_names_and_unsafe_parameters() -> None: + with pytest.raises(policy.PolicyError, match="unknown shell template"): + policy.render_shell("nope") + with pytest.raises(policy.PolicyError, match="expects parameters"): + policy.render_shell("path_mode") + with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): + policy.render_shell("path_mode", path="/tmp; rm -rf /") + with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): + policy.render_shell("path_mode", path="$(id)") + + +def test_positionals_skip_flags_and_stop_at_the_boundary() -> None: + argv = ("kubectl", "--namespace", "hermes", "get", "pods", "--", "ignored") + assert list(policy.positionals(argv)) == ["get", "pods"] + assert list(policy.positionals(("kubectl", "--all-namespaces", "get"))) == ["get"] + + +def test_a_subcommand_must_precede_its_resource_arguments() -> None: + with pytest.raises(policy.PolicyError, match="must precede"): + check("kubectl", "pods", "get") diff --git a/testing/tests/test_hermes_handoff_redaction.py b/testing/tests/test_hermes_handoff_redaction.py new file mode 100644 index 00000000..a472d82b --- /dev/null +++ b/testing/tests/test_hermes_handoff_redaction.py @@ -0,0 +1,162 @@ +"""Contracts for the handoff harness credential screen and output bounding.""" + +from __future__ import annotations + +import json + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module + +redaction = load_handoff_module("hermes_handoff_redaction") + + +@pytest.mark.parametrize( + "text", + [ + "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234", + "Authorization: Bearer abcdefghijklmnop", + "cookie: session=abcdefghijklmnop", + "password=hunter2000secret", + "-----BEGIN OPENSSH PRIVATE KEY-----\nabc\n-----END OPENSSH PRIVATE KEY-----", + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI", + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0.SflKxwRJ", + "https://user:supersecretvalue@scm.example.dev/x", + "vault token hvs.CAESIJabcdefghijklmnop", + "aws AKIAIOSFODNN7EXAMPLE", + ], +) +def test_credential_shapes_are_screened(text: str) -> None: + assert redaction.contains_credential_shape(text) + assert redaction.REDACTED in redaction.redact(text) + + +@pytest.mark.parametrize( + "text", + [ + "sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7", + "ab346f55509d584e457fe26cf90be3078f7a375c", + '"automountServiceAccountToken": false', + '"key": "kubernetes.io/arch"', + "namespace: hermes", + "replicas: 3", + "checked_at: 2026-08-17T09:17:07.224400+00:00", + ], +) +def test_ordinary_evidence_survives_screening(text: str) -> None: + assert redaction.redact(text) == text + assert not redaction.contains_credential_shape(text) + + +def test_screened_json_is_still_parsable_json() -> None: + """A screen that corrupts structure would break every downstream evaluator.""" + payload = { + "spec": {"automountServiceAccountToken": True, "key": "kubernetes.io/arch"}, + "data": {"password": "a-long-enough-secret-value"}, + } + screened = redaction.redact(json.dumps(payload, indent=2)) + reparsed = json.loads(screened) + + assert reparsed["spec"]["automountServiceAccountToken"] is True + assert reparsed["spec"]["key"] == "kubernetes.io/arch" + assert reparsed["data"]["password"] == redaction.REDACTED + + +def test_high_entropy_runs_are_screened_but_hex_digests_are_not() -> None: + assert redaction.redact("Zm9vYmFyYmF6cXV4-Quux_1234567890ABCDEFghij") != redaction.REDACTED + long_mixed = "aB3" + "x9Y2z_" * 10 + assert redaction.redact(long_mixed) == redaction.REDACTED + assert redaction.redact("f" * 64) == "f" * 64 + + +@pytest.mark.parametrize( + ("key", "sensitive"), + [ + ("ANTHROPIC_API_KEY", True), + ("gitea_token", True), + ("clientSecret", True), + ("Set-Cookie", True), + ("automountServiceAccountToken", True), + ("key", False), + ("namespace", False), + ("", False), + ], +) +def test_key_sensitivity_classification(key: str, sensitive: bool) -> None: + assert redaction.is_sensitive_key(key) is sensitive + + +@pytest.mark.parametrize( + ("value", "possible"), + [ + ("true", False), + ("False", False), + ("3", False), + ("12.5", False), + ("short", False), + ("", False), + ("a-long-enough-secret", True), + ('"another-long-secret"', True), + ("1234567890123", False), + ("-12.5e9", False), + ], +) +def test_value_shape_gates_screening(value: str, possible: bool) -> None: + assert redaction.could_hold_secret(value) is possible + + +def test_bound_truncates_to_a_byte_budget_and_reports_it() -> None: + text, truncated = redaction.bound("x" * 100, max_bytes=10) + assert truncated + assert text.startswith("x" * 10) + assert redaction.TRUNCATION_NOTE in text + + text, truncated = redaction.bound("short", max_bytes=64) + assert (text, truncated) == ("short", False) + + +def test_bound_with_no_budget_keeps_nothing() -> None: + assert redaction.bound("content", max_bytes=0) == ("", True) + assert redaction.bound("", max_bytes=0) == ("", False) + + +def test_bound_never_splits_a_multibyte_character() -> None: + text, truncated = redaction.bound("é" * 10, max_bytes=5) + assert truncated + assert text.startswith("éé") + + +def test_safe_text_bounds_before_it_screens() -> None: + """Bounding first is what stops a huge dump from outrunning the screen.""" + payload = "password=" + "a" * 200 + text, truncated = redaction.safe_text(payload, max_bytes=20) + assert truncated + assert redaction.REDACTED in text + + +def test_scrub_walks_nested_structures_and_keeps_useful_types() -> None: + scrubbed = redaction.scrub( + { + "token": "a-long-enough-secret", + "automountServiceAccountToken": False, + "items": [{"password": "another-long-secret"}, 3, None], + "note": ("tuple", "values"), + } + ) + + assert scrubbed["token"] == redaction.REDACTED + assert scrubbed["automountServiceAccountToken"] is False + assert scrubbed["items"][0]["password"] == redaction.REDACTED + assert scrubbed["items"][1] == 3 + assert scrubbed["items"][2] is None + assert scrubbed["note"] == ["tuple", "values"] + + +def test_screening_is_idempotent() -> None: + once = redaction.redact("api_key=AKIAIOSFODNN7EXAMPLE") + assert redaction.redact(once) == once + assert redaction.redact("") == "" + + +def test_normalise_key_strips_separators() -> None: + assert redaction.normalise_key("Client-Secret_v2") == "clientsecretv2" diff --git a/testing/tests/test_hermes_handoff_rules.py b/testing/tests/test_hermes_handoff_rules.py new file mode 100644 index 00000000..11f09840 --- /dev/null +++ b/testing/tests/test_hermes_handoff_rules.py @@ -0,0 +1,135 @@ +"""Contracts for the shared evaluator primitives and dispatch.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step + +rules = load_handoff_module("hermes_handoff_rules") +model = load_handoff_module("hermes_handoff_model") +load_handoff_module("hermes_handoff_evaluators") + + +PAYLOAD = { + "a": {"b": [{"c": 1}, {"c": 2}]}, + "items": [{"number": 14, "merged": True}, {"number": 15, "merged": False}], + "flag": True, +} + + +@pytest.mark.parametrize( + ("path", "expected"), + [ + ("", PAYLOAD), + ("flag", True), + ("a.b[0].c", 1), + ("a.b[].c", [1, 2]), + ("a.b[]", [{"c": 1}, {"c": 2}]), + ("items[].number", [14, 15]), + ], +) +def test_dotted_paths_index_and_map(path: str, expected) -> None: + assert rules.dotted(PAYLOAD, path) == expected + + +@pytest.mark.parametrize("path", ["missing", "a.missing", "a.b[9].c", "flag[0]", "flag.b"]) +def test_an_absent_path_raises_rather_than_returning_none(path: str) -> None: + with pytest.raises(KeyError): + rules.dotted(PAYLOAD, path) + + +def test_parse_json_tolerates_control_characters_but_not_empty_output() -> None: + """Kanban bodies routinely carry raw newlines; that is not a broken read.""" + assert rules.parse_json(outcome(stdout='{"title": "a\nb"}'))["title"] == "a\nb" + with pytest.raises(ValueError, match="empty output"): + rules.parse_json(outcome(stdout=" ")) + with pytest.raises(ValueError): + rules.parse_json(outcome(stdout="not json")) + + +def test_lines_drops_blank_rows_and_trims() -> None: + assert rules.lines(outcome(stdout=" a \n\n b\n")) == ["a", "b"] + + +def test_missing_steps_reports_the_first_step_that_produced_nothing() -> None: + built = spec("allowed", steps=(step("a"), step("b"))) + + never_run = rules.missing_steps(built, {}) + assert never_run.status == model.NOT_RUN + assert "never executed" in never_run.reason + + errored = rules.missing_steps(built, {"a": outcome(error="timeout"), "b": outcome()}) + assert errored.status == model.NOT_RUN + assert "did not run" in errored.reason + + assert rules.missing_steps(built, {"a": outcome(), "b": outcome()}) is None + + +def test_an_optional_step_may_be_absent() -> None: + built = spec("allowed", steps=(step("a"), step("b", optional=True))) + assert rules.missing_steps(built, {"a": outcome()}) is None + + +@pytest.mark.parametrize( + ("stderr", "returncode", "denied"), + [ + ("Error from server (Forbidden): pods is forbidden", 1, True), + ("User cannot list resource", 1, True), + ("Gitea API returned HTTP 403", 1, True), + ("connection refused", 1, False), + ("", 0, False), + ], +) +def test_denial_detection_needs_a_refusal_not_merely_a_failure(stderr, returncode, denied) -> None: + assert rules.looks_denied(outcome(stderr=stderr, returncode=returncode)) is denied + + +def test_step_keys_selects_by_kind() -> None: + built = spec( + "denied", + steps=(step("r", kind=model.REVIEW), step("a", kind=model.ATTEMPT), step("x")), + ) + assert rules.step_keys(built, model.REVIEW) == ("r",) + assert rules.step_keys(built, model.ATTEMPT) == ("a",) + assert rules.step_keys(built, model.READ) == ("x",) + + +def test_parsed_step_returns_the_payload_or_the_skip_to_report() -> None: + built = spec("json_field", {"step": "s"}, steps=(step("s"),)) + + payload, key = rules.parsed_step(built, {"s": outcome(stdout='{"a": 1}')}) + assert (payload, key) == ({"a": 1}, "s") + + failed = rules.parsed_step(built, {"s": outcome(returncode=1)}) + assert failed.status == model.NOT_RUN and "failed" in failed.reason + + unparsable = rules.parsed_step(built, {"s": outcome(stdout="nope")}) + assert unparsable.status == model.NOT_RUN and "did not return JSON" in unparsable.reason + + +def test_an_unknown_rule_is_not_run_rather_than_a_crash() -> None: + evaluation = rules.evaluate(spec("no-such-rule"), {}) + assert evaluation.status == model.NOT_RUN + assert "unknown evaluator" in evaluation.reason + + +def test_an_evaluator_that_raises_is_not_run_rather_than_a_pass() -> None: + """A harness bug must never be indistinguishable from a clean release.""" + + @rules.evaluator("explodes-for-test") + def _explode(_spec, _outcomes): + raise RuntimeError("boom") + + evaluation = rules.evaluate(spec("explodes-for-test"), {}) + + assert evaluation.status == model.NOT_RUN + assert "RuntimeError" in evaluation.reason and "boom" in evaluation.reason + del rules.EVALUATORS["explodes-for-test"] + + +def test_is_list_distinguishes_arrays_from_strings() -> None: + assert rules.is_list([1]) and rules.is_list(()) + assert not rules.is_list("text") + assert not rules.is_list(b"bytes") + assert not rules.is_list({"a": 1}) diff --git a/testing/tests/test_hermes_handoff_run.py b/testing/tests/test_hermes_handoff_run.py new file mode 100644 index 00000000..379bb27d --- /dev/null +++ b/testing/tests/test_hermes_handoff_run.py @@ -0,0 +1,320 @@ +"""Contracts for handoff acceptance orchestration: vantages, catalog, report.""" + +from __future__ import annotations + +import datetime as dt +import subprocess + +from testing.tests.test_hermes_handoff_support import ( + FakeClock, + FakeSpawn, + load_handoff_module, + step, +) + +harness_run = load_handoff_module("hermes_handoff_run") +catalog = load_handoff_module("hermes_handoff_catalog") +exec_module = load_handoff_module("hermes_handoff_exec") +model = load_handoff_module("hermes_handoff_model") + +NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc) +TARGETS = catalog.Targets(now=NOW) + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + + +def whoami(username: str) -> object: + return completed(stdout=f'{{"status": {{"userInfo": {{"username": "{username}"}}}}}}') + + +def runner_for(results: list, **kwargs): + return exec_module.Runner( + clock=FakeClock(), spawn=FakeSpawn(results), environ={}, deadline_seconds=3600, **kwargs + ) + + +def check(rule: str, expect: dict | None = None, steps=(), identifier="test.check", mandatory=True): + return model.CheckSpec( + id=identifier, + title="t", + group="g", + rule=rule, + steps=tuple(steps), + expect=expect or {}, + mandatory=mandatory, + ) + + +def test_the_shipped_catalog_is_valid_and_covers_every_group() -> None: + specs = harness_run.build_catalog(TARGETS) + assert harness_run.validate_catalog(specs) == [] + groups = {spec.group for spec in specs} + assert {"baseline", "identity", "routing", "scopes", "access-denied", "gitops"} <= groups + assert {"forge", "nodes", "build", "reliability", "pool", "surfaces", "ephemeral"} <= groups + + +def test_catalog_validation_names_every_structural_problem() -> None: + broken = [ + check("no-such-rule", identifier="a.dup"), + check("allowed", identifier="a.dup"), + check("allowed", steps=(step("s", vantage="nowhere"),), identifier="b.vantage"), + check("allowed", steps=(step("s", kind="guess"),), identifier="c.kind"), + check("denied", steps=(step("s", kind=model.REVIEW),), identifier="d.review-only"), + model.CheckSpec( + id="e.impersonation", + title="t", + group="g", + rule="allowed", + steps=( + model.Step(key="s", vantage=catalog.OPERATOR, argv=("kubectl", "get", "ns", "--as", "x")), + ), + ), + ] + + problems = " | ".join(harness_run.validate_catalog(broken)) + + assert "duplicate check id" in problems + assert "unknown rule" in problems + assert "unknown vantage" in problems + assert "unknown kind" in problems + assert "needs a real attempt" in problems + assert "impersonates from the operator vantage" in problems + + +def test_identity_resolution_extracts_only_the_username() -> None: + runner = runner_for([whoami("system:serviceaccount:hermes:hermes-agent")]) + username, detail = harness_run.resolve_identity(runner, exec_module.operator_vantage()) + assert (username, detail) == ("system:serviceaccount:hermes:hermes-agent", "") + + failed = harness_run.resolve_identity( + runner_for([completed(stderr="no access", returncode=1)]), exec_module.operator_vantage() + ) + assert failed[0] == "" and "no access" in failed[1] + + unparsable = harness_run.resolve_identity( + runner_for([completed(stdout="nope")]), exec_module.operator_vantage() + ) + assert unparsable[0] == "" and "unparsable identity" in unparsable[1] + + nameless = harness_run.resolve_identity( + runner_for([completed(stdout='{"status": {}}')]), exec_module.operator_vantage() + ) + assert nameless[0] == "" and "no username" in nameless[1] + + +def test_vantages_resolve_to_running_pods_and_record_what_is_missing() -> None: + runner = runner_for( + [ + whoami("kubernetes-admin"), + completed(stdout="pod/hermes-agent-2\npod/hermes-agent-1\n"), + whoami("system:serviceaccount:hermes:hermes-agent"), + completed(stdout="pod/hermes-switchyard-1\n"), + completed(stderr="no pods", returncode=1), + completed(stdout="pod/hermes-chat-tenant-0\n"), + ] + ) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + by_name = {record.name: record for record in records} + + assert set(vantages) == {catalog.OPERATOR, catalog.SELF, catalog.SWITCHYARD, catalog.CHAT} + assert by_name[catalog.SELF].identity == "system:serviceaccount:hermes:hermes-agent" + assert "hermes-agent-1" in by_name[catalog.SELF].description # deterministic pick + assert by_name[catalog.NODE].available is False + assert by_name[catalog.CHAT].available is True + + +def test_an_unreachable_operator_stops_vantage_resolution_immediately() -> None: + runner = runner_for([completed(stderr="connection refused", returncode=1)]) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + + assert set(vantages) == {catalog.OPERATOR} + assert records[0].available is False + assert harness_run.vantage_problems(records)[0].startswith("the operator vantage is unavailable") + + +def test_a_missing_chat_pod_is_recorded_rather_than_assumed_present() -> None: + runner = runner_for( + [ + whoami("kubernetes-admin"), + completed(stdout="pod/hermes-agent-1\n"), + whoami("system:serviceaccount:hermes:hermes-agent"), + completed(stdout="pod/hermes-switchyard-1\n"), + completed(stdout="pod/hermes-node-1\n"), + completed(stderr="pods 'hermes-chat-tenant-0' not found", returncode=1), + ] + ) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + chat = next(record for record in records if record.name == catalog.CHAT) + + assert catalog.CHAT not in vantages + assert chat.available is False and "not found" in chat.detail + + +def test_a_selector_that_matches_no_running_pod_leaves_the_vantage_absent() -> None: + runner = runner_for( + [ + whoami("kubernetes-admin"), + completed(stdout="\n"), + completed(stdout="\n"), + completed(stdout="\n"), + completed(stdout="\n"), + ] + ) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + by_name = {record.name: record for record in records} + + assert set(vantages) == {catalog.OPERATOR} + assert "no running pod matches" in by_name[catalog.SELF].detail + + +def test_two_vantages_that_are_the_same_principal_invalidate_the_evidence() -> None: + """Corroboration needs two identities; one identity twice is a restatement.""" + records = [ + model.VantageRecord(name=catalog.OPERATOR, description="d", identity="same", available=True), + model.VantageRecord(name=catalog.SELF, description="d", identity="same", available=True), + ] + + problems = harness_run.vantage_problems(records) + + assert len(problems) == 1 + assert "same principal" in problems[0] + + +def test_vantage_problems_flag_a_missing_or_anonymous_self_probe() -> None: + operator = model.VantageRecord( + name=catalog.OPERATOR, description="d", identity="admin", available=True + ) + + absent = harness_run.vantage_problems([operator]) + assert "in-pod Hermes vantage is unavailable" in absent[0] + + anonymous = harness_run.vantage_problems( + [operator, model.VantageRecord(name=catalog.SELF, description="d", available=True)] + ) + assert "did not report an identity" in anonymous[0] + + clean = harness_run.vantage_problems( + [ + operator, + model.VantageRecord( + name=catalog.SELF, description="d", identity="agent", available=True + ), + ] + ) + assert clean == [] + + +def test_a_check_whose_vantage_is_absent_is_not_run() -> None: + spec = check("allowed", steps=(step("s", vantage=catalog.NODE),)) + + result = harness_run.run_check(runner_for([]), spec, {}) + + assert result.status == model.NOT_RUN + assert "vantage node is unavailable" in result.reason + assert result.outcomes[0].error == "vantage node is unavailable" + + +def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None: + spec = check( + "stdout_matches", + {"step": "s", "equals": "locked"}, + steps=(model.Step(key="s", vantage=catalog.OPERATOR, argv=("kubectl", "version"), record=False),), + ) + runner = runner_for([completed(stdout="locked", stderr="noise")]) + + result = harness_run.run_check(runner, spec, {catalog.OPERATOR: exec_module.operator_vantage()}) + + assert result.status == model.PASS + assert result.outcomes[0].stdout == harness_run.UNRECORDED + assert result.outcomes[0].stderr == harness_run.UNRECORDED + + +def test_an_expired_deadline_leaves_the_report_saying_so() -> None: + clock = FakeClock() + runner = exec_module.Runner(clock=clock, spawn=FakeSpawn([]), environ={}, deadline_seconds=1.0) + clock.advance(5.0) + spec = check("allowed", steps=(step("s", vantage=catalog.OPERATOR),)) + + report = harness_run.build_report( + runner, + TARGETS, + [spec], + {catalog.OPERATOR: exec_module.operator_vantage()}, + [], + "read-only", + "2026-08-17T00:00:00Z", + ) + + assert report.results[0].status == model.NOT_RUN + assert any("deadline expired" in error for error in report.harness_errors) + assert report.decision == model.NO_GO + + +def test_the_report_carries_the_configured_baseline_and_sorts_its_checks() -> None: + runner = runner_for([completed(stdout="ok"), completed(stdout="ok")]) + specs = [ + check("allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="z.last"), + check("allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="a.first"), + ] + records = [ + model.VantageRecord(name=catalog.OPERATOR, description="d", identity="admin", available=True), + model.VantageRecord(name=catalog.SELF, description="d", identity="agent", available=True), + ] + + report = harness_run.build_report( + runner, + TARGETS, + specs, + {catalog.OPERATOR: exec_module.operator_vantage()}, + records, + "read-only", + "2026-08-17T00:00:00Z", + ) + + assert [result.spec.id for result in report.results] == ["a.first", "z.last"] + assert report.baseline["baseline_commit"] == TARGETS.baseline_commit + assert report.baseline["dependency_pull_requests"] == list(TARGETS.dependency_pull_requests) + assert report.decision == model.GO + assert report.finished_at + + +def test_armed_results_replace_the_matching_catalog_placeholders() -> None: + placeholder = check("not_armed", identifier="ephemeral.feature-branch-push", mandatory=False) + armed = model.CheckResult( + spec=model.CheckSpec( + id="ephemeral.feature-branch-push", title="t", group="ephemeral", rule="armed" + ), + status=model.PASS, + reason="pushed", + ) + + report = harness_run.build_report( + runner_for([]), + TARGETS, + [placeholder], + {}, + [], + "ephemeral-armed", + "2026-08-17T00:00:00Z", + [armed], + ) + + assert len(report.results) == 1 + assert report.results[0].status == model.PASS + + +def test_an_unstartable_run_reports_no_go_with_a_named_reason() -> None: + report = harness_run.unavailable_report( + "read-only", "2026-08-17T00:00:00Z", "arming refused: bad phrase", TARGETS + ) + + assert report.decision == model.NO_GO + assert report.results[0].spec.id == "harness.startup" + assert report.results[0].status == model.NOT_RUN + assert report.harness_errors == ["arming refused: bad phrase"] diff --git a/testing/tests/test_hermes_handoff_support.py b/testing/tests/test_hermes_handoff_support.py new file mode 100644 index 00000000..64964a27 --- /dev/null +++ b/testing/tests/test_hermes_handoff_support.py @@ -0,0 +1,110 @@ +"""Shared loader and fixtures for the handoff acceptance harness tests. + +The harness lives in ``scripts/ops`` as sibling modules rather than an installed +package, so tests import it the way the entry point does: by putting that +directory on the path once and importing by name. +""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +OPS = REPO_ROOT / "scripts" / "ops" + +if str(OPS) not in sys.path: + sys.path.insert(0, str(OPS)) + + +def load_handoff_module(name: str) -> Any: + """Import one harness module by name.""" + return importlib.import_module(name) + + +exec_module = load_handoff_module("hermes_handoff_exec") +model = load_handoff_module("hermes_handoff_model") + + +def outcome( + stdout: str = "", + stderr: str = "", + returncode: int | None = 0, + error: str | None = None, + vantage: str = "self", + argv: tuple[str, ...] = ("kubectl", "get", "pods"), +) -> Any: + """Return a synthetic command outcome for evaluator tests.""" + return exec_module.Outcome( + argv=argv, + vantage=vantage, + returncode=returncode, + stdout=stdout, + stderr=stderr, + error=error, + ) + + +def spec( + rule: str, + expect: dict | None = None, + steps: tuple = (), + identifier: str = "test.check", + mandatory: bool = True, + scope: str | None = None, +) -> Any: + """Return a synthetic check specification.""" + return model.CheckSpec( + id=identifier, + title="test check", + group="test", + rule=rule, + steps=steps, + expect=expect or {}, + mandatory=mandatory, + scope=scope or model.ALWAYS, + ) + + +def step(key: str, kind: str | None = None, vantage: str = "self", optional: bool = False) -> Any: + """Return a synthetic step.""" + return model.Step( + key=key, + vantage=vantage, + argv=("kubectl", "get", "pods"), + kind=kind or model.READ, + optional=optional, + ) + + +class FakeClock: + """A monotonic clock the tests advance by hand.""" + + def __init__(self, start: float = 0.0) -> None: + self.now = start + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + """Move the clock forward.""" + self.now += seconds + + +class FakeSpawn: + """A ``subprocess.run`` stand-in driven by a queue of scripted results.""" + + def __init__(self, results: list) -> None: + self.results = list(results) + self.calls: list[dict] = [] + + def __call__(self, argv, **kwargs): + self.calls.append({"argv": list(argv), **kwargs}) + if not self.results: + raise AssertionError(f"unexpected command: {argv}") + outcome_or_error = self.results.pop(0) + if isinstance(outcome_or_error, Exception): + raise outcome_or_error + return outcome_or_error From 9d5214f809cfac3d092c8df687a9606d4c12976d Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 13:11:34 +0000 Subject: [PATCH 2/3] fix(hermes): close full-handoff acceptance blockers --- ci/scripts/semgrep_report.py | 47 +- docs/hermes_full_handoff_acceptance.md | 480 +++++++--------- scripts/ops/hermes_handoff_acceptance.py | 217 ++++++-- scripts/ops/hermes_handoff_catalog.py | 102 ++-- scripts/ops/hermes_handoff_checks_access.py | 303 ++++++---- scripts/ops/hermes_handoff_checks_delivery.py | 295 +++++----- scripts/ops/hermes_handoff_checks_platform.py | 78 ++- scripts/ops/hermes_handoff_checks_workers.py | 253 ++++----- scripts/ops/hermes_handoff_ephemeral.py | 524 ++++++++++++------ scripts/ops/hermes_handoff_evaluators.py | 267 ++++++++- scripts/ops/hermes_handoff_exec.py | 497 ++++++++++++----- scripts/ops/hermes_handoff_json_rules.py | 273 ++++++--- scripts/ops/hermes_handoff_model.py | 58 +- scripts/ops/hermes_handoff_policy.py | 433 +++++++++------ scripts/ops/hermes_handoff_redaction.py | 90 +-- scripts/ops/hermes_handoff_rules.py | 76 ++- scripts/ops/hermes_handoff_run.py | 217 ++++++-- testing/quality_contract.json | 89 +++ testing/quality_coverage.py | 56 +- testing/quality_handoff_mutation.py | 209 +++++++ testing/quality_hygiene.py | 25 +- testing/tests/conftest.py | 20 + testing/tests/test_hermes_auto_router.py | 6 + .../tests/test_hermes_handoff_acceptance.py | 171 ++++-- .../test_hermes_handoff_acceptance_armed.py | 120 ++++ testing/tests/test_hermes_handoff_catalog.py | 87 +-- .../tests/test_hermes_handoff_ephemeral.py | 510 +++++++++-------- .../tests/test_hermes_handoff_evaluators.py | 300 +++++++++- testing/tests/test_hermes_handoff_exec.py | 232 +++++++- .../tests/test_hermes_handoff_json_rules.py | 198 +++++-- testing/tests/test_hermes_handoff_model.py | 44 +- .../test_hermes_handoff_mutation_gate.py | 51 ++ testing/tests/test_hermes_handoff_policy.py | 276 +++++++-- .../tests/test_hermes_handoff_redaction.py | 44 +- testing/tests/test_hermes_handoff_rules.py | 47 +- testing/tests/test_hermes_handoff_run.py | 207 ++++++- testing/tests/test_hermes_handoff_support.py | 8 +- testing/tests/test_quality_contract.py | 64 ++- .../tests/test_quality_coverage_helpers.py | 69 ++- testing/tests/test_semgrep_report.py | 27 +- 40 files changed, 5121 insertions(+), 1949 deletions(-) create mode 100644 testing/quality_handoff_mutation.py create mode 100644 testing/tests/conftest.py create mode 100644 testing/tests/test_hermes_handoff_acceptance_armed.py create mode 100644 testing/tests/test_hermes_handoff_mutation_gate.py diff --git a/ci/scripts/semgrep_report.py b/ci/scripts/semgrep_report.py index b4ed5a2d..f46e528d 100644 --- a/ci/scripts/semgrep_report.py +++ b/ci/scripts/semgrep_report.py @@ -27,7 +27,10 @@ def _read_json(path: Path) -> dict[str, Any]: except json.JSONDecodeError as exc: return {"errors": [{"message": f"invalid JSON: {exc}"}], "results": []} if not isinstance(payload, dict): - return {"errors": [{"message": "report payload is not an object"}], "results": []} + return { + "errors": [{"message": "report payload is not an object"}], + "results": [], + } return payload @@ -49,10 +52,18 @@ def build_report( blocking = blocking_severities or BLOCKING_SEVERITIES raw_results = semgrep_payload.get("results", []) raw_errors = semgrep_payload.get("errors", []) - results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else [] - errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else [] + results = ( + [item for item in raw_results if isinstance(item, dict)] + if isinstance(raw_results, list) + else [] + ) + errors = ( + [item for item in raw_errors if isinstance(item, dict)] + if isinstance(raw_errors, list) + else [] + ) - severity_counts = {severity: 0 for severity in SEVERITIES} + severity_counts = dict.fromkeys(SEVERITIES, 0) blocking_findings = 0 for finding in results: severity = _finding_severity(finding) @@ -93,16 +104,18 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None: start = finding.get("start") if isinstance(finding.get("start"), dict) else {} end = finding.get("end") if isinstance(finding.get("end"), dict) else {} start_line = _line_number(start.get("line") if isinstance(start, dict) else None) - end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line) - if end_line < start_line: - end_line = start_line + end_line = _line_number( + end.get("line") if isinstance(end, dict) else None, start_line + ) return { "engineId": "semgrep", "ruleId": str(finding.get("check_id") or "semgrep.unknown"), "type": "VULNERABILITY" if severity == "ERROR" else "CODE_SMELL", "severity": SONAR_SEVERITY_BY_SEMGREP[severity], "primaryLocation": { - "message": str(extra.get("message") or finding.get("check_id") or "Semgrep finding"), + "message": str( + extra.get("message") or finding.get("check_id") or "Semgrep finding" + ), "filePath": path, "textRange": { "startLine": start_line, @@ -112,11 +125,19 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None: } -def build_sonar_issues(semgrep_payload: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: +def build_sonar_issues( + semgrep_payload: dict[str, Any], +) -> dict[str, list[dict[str, Any]]]: """Build SonarQube generic external issues from Semgrep JSON output.""" raw_results = semgrep_payload.get("results", []) - results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else [] - issues = [issue for finding in results if (issue := _sonar_issue_from_finding(finding))] + results = ( + [item for item in raw_results if isinstance(item, dict)] + if isinstance(raw_results, list) + else [] + ) + issues = [ + issue for finding in results if (issue := _sonar_issue_from_finding(finding)) + ] return {"issues": issues} @@ -144,7 +165,9 @@ def main(argv: list[str] | None = None) -> int: ) output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + output.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) if args.sonar_issues_output: sonar_output = Path(args.sonar_issues_output) sonar_output.parent.mkdir(parents=True, exist_ok=True) diff --git a/docs/hermes_full_handoff_acceptance.md b/docs/hermes_full_handoff_acceptance.md index 8b9103c2..d7aa9608 100644 --- a/docs/hermes_full_handoff_acceptance.md +++ b/docs/hermes_full_handoff_acceptance.md @@ -1,323 +1,259 @@ -# Hermes full-handoff acceptance harness and release runbook +# Hermes full-handoff acceptance and release runbook -`scripts/ops/hermes_handoff_acceptance.py` decides whether the Hermes platform -handoff is fit to release. It is read-only by default, it classifies every check -`PASS` / `FAIL` / `NOT_RUN` / `NOT_APPLICABLE`, and **any mandatory `FAIL` or -`NOT_RUN` makes the whole run `NO_GO`**. Absence of evidence is never rounded up -to evidence of absence: a probe that could not run, a catalog entry whose -evidence no longer exists, an expired deadline, or a harness bug all land in -`NOT_RUN` and stop the release. +`scripts/ops/hermes_handoff_acceptance.py` is a fail-closed release gate. It +contains 63 checks in 15 groups. An empty catalog, an unknown result status, a +mandatory `FAIL` or `NOT_RUN`, a harness error, or incomplete evidence makes the +decision `NO_GO`. The four ephemeral checks are non-mandatory `NOT_RUN` in the +default mode and are replaced with mandatory results only in armed mode. -``` +This runbook describes evidence collection. It does not authorize a merge, +publish, deployment, Flux reconcile, credential read, or live cluster mutation. +Human review of PR #19 and of the resulting report is required. + +## Required release inputs + +Every value that can go stale must be supplied explicitly. Resolve `main`, PR +heads, the image digest, build source SHA, deployment revision, and chat config +revision from their authoritative read-only sources immediately before the run. +Do not copy the snapshot in the dependency section into a later run. + +```sh scripts/ops/hermes_handoff_acceptance.py \ --context atlas-operator \ - --node-count 21 \ - --expected-suspension flux-system/bstein-dev-home-migrations \ + --remote-main-sha "$REMOTE_MAIN_SHA" \ + --reviewed-head-sha "$REVIEWED_PR19_HEAD" \ + --agent-image "$AGENT_GIT_SHA_BUILD_TAG_AT_SHA256" \ + --build-sha "$IMAGE_BUILD_SOURCE_SHA" \ + --deployment-revision "$RUNNING_DEPLOYMENT_REVISION" \ + --chat-config-revision "$RUNNING_CHAT_CONFIG_REVISION" \ + --dependency-head "12=$PR12_HEAD" \ + --dependency-head "14=$PR14_HEAD" \ + --dependency-head "15=$PR15_HEAD" \ + --dependency-head "16=$PR16_HEAD" \ + --dependency-head "17=$PR17_HEAD" \ + --dependency-head "18=$PR18_HEAD" \ + --dependency-head "20=$PR20_HEAD" \ --output build/hermes-handoff-acceptance.json ``` -Exit status is `0` for `GO` and `1` for `NO_GO`. The JSON report goes to -`--output` (or stdout); a short human summary goes to stderr unless -`--json-only` is passed. +The fixed topology is three pool workers on three distinct nodes. `--node-count` +and `--pool-replicas` cannot lower that requirement. Pool node/ordinal assignment, +Telegram session continuity, and the chat revision are mandatory. Deliberate Flux +suspensions must be named with repeatable `--expected-suspension` arguments. -## Why two vantages +Exit status is `0` only for `GO`, `1` for `NO_GO`, and `2` for invalid execution +bounds. JSON is written to `--output` or stdout; the human summary goes to stderr +unless `--json-only` is passed. -Every authority claim is checked from two places that cannot cover for each -other: +## Structural safety boundary -| Vantage | What it is | What it proves | -| --- | --- | --- | -| `operator` | An external, read-only kubeconfig | Desired state: bindings, manifests, workloads, Flux health, forge state | -| `self` | `kubectl exec` into the running `hermes-agent` container | Enforced state: what the agent's own service-account token can actually do | -| `switchyard`, `node`, `chat` | `kubectl exec` into the Switchyard, node-hardening, and chat pods | Evidence that only exists inside those workloads | +The catalog is validated before a `Runner` exists. Invalid inputs, an empty or +unsafe catalog, and an invalid armed preflight therefore create no subprocess. +The command policy is checked both before and after executable resolution. -Two rules keep this honest: +Default mode permits a narrow read grammar: -* **The harness cannot impersonate from outside.** `--as`, `--as-group`, and - `--as-uid` are rejected by policy for any operator-side command. They are - reachable only as the *inner* command of a `kubectl exec`, where the identity - is testing its own refusal to impersonate rather than borrowing authority. - `kubectl auth can-i --as` is therefore not available as a substitute for a - real self-probe. -* **The two vantages must be different principals.** Before any check runs, the - harness asks each vantage `kubectl auth whoami` and compares. If the operator - and the in-pod probe authenticate as the same principal, the dual-vantage - evidence is a restatement rather than a corroboration, and the run is `NO_GO` - with `the operator and in-pod vantages authenticate as the same principal`. - Run the harness from a genuinely external kubeconfig — not from inside the - Hermes owner pod. +- Kubernetes reads use `name` or reviewed JSONPath projections. Secret resources, + raw API reads, full JSON/YAML, `describe`, `cluster-info dump`, credential-bearing + projections, `auth reconcile`, config operations, all mutations, and every + dry-run mutation trick are rejected before spawn. +- Git is read-only. `fetch`, configuration/helper overrides, alternate worktrees, + external diff helpers, and push are rejected. Helm values are not readable. +- Shell commands must be rendered from frozen templates, use fixed executable + paths, and cannot name service-account, Vault, runtime-access, SSH, or other + credential roots. +- Outer impersonation is rejected. A reviewed inner command can use impersonation + only after an explicit `kubectl exec --` boundary; it cannot substitute for the + independent self identity. +- `PATH` is replaced with a fixed path. Every outer executable is resolved to an + allowed real path, checked for unsafe write mode, and compared with a pinned + SHA-256 digest. A caller-authored `kubectl`, `git`, or helper cannot fabricate + evidence. -Deny checks additionally require a **real refused request**, not only an -authorization review. `kubectl auth can-i` reports what RBAC says; only an -attempt proves the API server enforces it. Mutating attempts go through -`--dry-run=server`: the API server authorises the request and then discards it, -so a refusal is genuine and nothing is written either way. +The executor uses a new process group, drains stdout and stderr concurrently, +and retains at most the configured bytes while the child is running. A monotonic +absolute timeout terminates and reaps the group, including descendants that keep +pipes open. Timeout, deadline, output, and concurrency values reject booleans, +zero/negative values, non-finite numbers, and values outside fixed bounds. +Truncation discards the affected stream and makes every evaluator `NOT_RUN`. -## Safety properties +Credential screening runs before truncation. Command, output, error, identity, +reason, evidence, JSON, and human-summary fields are screened recursively, +including mapping keys and short/numeric sensitive values. The complete rendered +report is screened again; a remaining credential shape is replaced by a bounded +`NO_GO` stub rather than being written. No credential prefix is retained at a +truncation boundary. -These hold structurally, in `hermes_handoff_policy.py`, so a catalog mistake is -a policy error before a run rather than a mutation or a leak during one. +## Vantages and evidence -* **No mutation in the default mode.** Only read subcommands are reachable; - `apply`, `create`, `delete`, `patch`, `replace`, and `scale` require a dry-run - flag. `git push` and every non-`GET` forge method are refused outside the - armed mode. There is deliberately no live TokenRequest probe — a successful - one would mint a real credential. -* **No credential is ever read.** `vault`, `sops`, `age`, `gpg`, `openssl`, - `curl`, and `wget` are not on the binary allowlist. Secrets may only be read - with `-o name`. Environment probes list variable *names* and never values. - Provider health files are read through a frozen template that projects an - allow-listed set of non-secret status fields. The node account's lock state is - reported without reading its password hash; its authorized keys are counted, - never printed. -* **Shell is frozen.** `sh -c` is reachable only through the reviewed, - parameter-validated templates in `SHELL_TEMPLATES`. Parameters may not contain - shell metacharacters. -* **Output is bounded and screened.** Each capture is truncated to a byte budget - *before* it is screened, so a large dump cannot outrun the screen. Bulk - evidence (the routing-log tail, the Kanban listing) is parsed but replaced with - `[output not recorded]` in the artifact. The screen preserves JSON quoting, so - screened structured output is still parsable. -* **The report is re-checked before it is published.** `unscreened_fields` walks - the rendered payload; if anything still looks credential-shaped the report is - replaced with a `NO_GO` stub naming the offending pointers, and the file that - would have leaked is never written. +The operator must use an external read-only kubeconfig. The self probe runs in +the actual Hermes agent pod. Both identities are obtained with +`kubectl auth whoami`; equal or missing principals make the run `NO_GO`. Required +Switchyard and chat/Telegram pod vantages must also resolve. Node coverage is +derived from all daemonset pod node names, not one selected pod. -## What the catalog asserts +Dangerous Kubernetes permissions are checked only with side-effect-free +`kubectl auth can-i` reviews. The default catalog never constructs a Secret, +TokenRequest, mutation, attach, port-forward, or arbitrary exec request. The one +forge administrative denial check performs a read-only GET and accepts only an +explicit authorization refusal; generic `404`, `not found`, and `no route` text +are not denial evidence. -The catalog is declarative (`hermes_handoff_catalog.py` plus the four -`hermes_handoff_checks_*.py` modules) so a reviewer reads what is asserted, not -how it is plumbed. 71 checks in 14 groups: +The strict evaluators reject duplicate-key JSON, raw control characters, +`NaN`/infinity, boolean/number equality coercion, future timestamps, stale +provider evidence, and every malformed routing or Flux row. Routing freshness is +required separately for each provider, lane, and effort. Flux objects must be +non-suspended, current at `observedGeneration`, and explicitly `Ready=True`; +unknown or reconciling state is not healthy. -| Group | Assertions | -| --- | --- | -| `baseline` | `origin/main` descends from the merged worker-isolation baseline; every dependency pull request is merged into `main`; the exact open pull-request heads are recorded | -| `identity` | Codex authenticates as `codex-chatgpt-subscription`; Claude as `firstParty` / `claude.ai` / `claude-code-cli-subscription`; both evidence files are fresh; no provider API-key variable exists in the running process **or** in the manifest | -| `routing` | Switchyard has a ready replica; the routing log covers both providers, the `route` and `worker` lanes, medium/high/xhigh effort, and at least one real fallback; the newest record is fresh; both providers record a measured latency | -| `scopes` | Chat, agent, and triage each declare their own `HERMES_AUTO_ROUTER_PROFILE`, on distinct hostnames | -| `access-denied` | No cluster-admin binding for the agent; Secrets, service-account tokens, impersonation, workload mutation, arbitrary pod exec, attach, port-forward, and control-plane writes are all refused | -| `access-allowed` | get/list/watch on core objects, pod logs, and Flux/Helm status still work; the operator and in-pod namespace inventories agree | -| `gitops` | No unexpected Kustomization or HelmRelease suspension, and nothing unhealthy | -| `forge` | No forge credential in the worker environment or home; the forge identity is not an administrator; repository authority is pull/push only; an administrative route is live-refused; clone/fetch still works | -| `nodes` | The dedicated `hermes-agent` account is locked, has the expected uid/gid/home/shell, joins no supplementary group, has no sudoers fragment, holds exactly one authorized key; K3s credentials, K3s state, kubelet state, and the container runtime are root-only; the audit covers every node | -| `build` | The image builder service account mounts no token; the build pod is unprivileged with exactly `CHOWN, FOWNER, DAC_OVERRIDE, SETGID, SETUID`; the Harbor immutable-tag job succeeded; the running image is digest-pinned | -| `reliability` | The finalization/replay/session patches are running in the pod; the regression suites are present on `main` | -| `pool` | Three ready workers, on distinct nodes, with distinct volumes, mounting no API token and none of the coordinator's claims | -| `surfaces` | Kanban activity and durable sessions are visible and render without blank rows; the agent and chat session patches are deployed; Telegram topic state is on durable storage | -| `ephemeral` | Feature-branch push, draft pull request, protected-branch refusal, and verified cleanup — `NOT_RUN` unless armed | +`release.exact-lineage-is-running` atomically binds: -### Site configuration +- fixed remote `main` SHA and local `origin/main`; +- open draft PR #19, base `main`, existing feature ref, and exact reviewed head; +- the exact `:git--build-@sha256:` image identity, + with the source SHA equal to the release `main` SHA; +- current Deployment generation and revision; +- the one active ReplicaSet revision/hash; every Ready pod image and imageID; +- the Flux-applied `main` revision. -A few checks are emitted only when the operator declares what to expect, -because a `names_present` assertion over an empty expectation would pass on any -workload at all — which reads as coverage without being any: +The pool evaluator requires exactly ordinals 0, 1, and 2, all Ready, on three +distinct nodes and claims, with service-account token automount disabled. The +chat checks always require durable Telegram sessions and the exact running chat +configuration revision. -| Flag | Enables | -| --- | --- | -| `--pool-worker-env NAME` (repeatable) | `pool.assignment-safety-knobs-are-configured` | -| `--chat-config-revision REV` | `surfaces.chat-runs-the-telegram-topic-revision` | -| `--expect-telegram-sessions` with `--chat-ordinal N` | `surfaces.chat-telegram-sessions-are-continuous` | +## Armed ephemeral mode -`--node-count` must be the real node count; the default of `1` makes the -fleet-coverage check vacuous. +Armed mode is the only write path and runs only after the complete default +catalog reports `GO`. -### Catalog drift is a NO_GO, not a pass - -If a workload, field, or manifest is renamed, the affected checks report -`NOT_RUN` because the evidence they name is no longer there. That is the -intended behaviour: the release stops until either the platform or the catalog -is corrected. Do not "fix" a `NOT_RUN` by deleting the check. - -## Ephemeral mutation mode - -Proving that a worker can still open a draft pull request means actually opening -one. That is the only part of the harness that writes anything, and it is a -separate mode: - -``` +```sh scripts/ops/hermes_handoff_acceptance.py \ - --context atlas-operator \ + \ --arm-ephemeral-push \ --confirm "ARM EPHEMERAL HERMES HANDOFF PUSH" \ --ephemeral-token acceptance-20260817a \ --output build/hermes-handoff-acceptance-armed.json ``` -* The confirmation phrase must match exactly. -* `--ephemeral-token` is 8–64 lowercase alphanumeric/dash characters and must be - unique per run; the ref is always - `ephemeral/hermes-handoff-acceptance/`. -* **Preflight runs before any network call**: wrong phrase, wrong repository, - malformed token, or a protected push target all abort with `arming refused` - and a `NO_GO` report. `main`, `master`, `HEAD`, `develop`, `release`, - `production`, `stable`, `trunk`, `prod`, and `default` are refused by name - (case- and `refs/heads/`-insensitive), and anything that is not an ephemeral - acceptance ref is refused by pattern. -* The guard is **exercised on every armed run** by - `ephemeral.protected-branch-refusal`, rather than trusting that it was checked - once. -* The push target is guarded; the pull-request *base* is not. Opening a draft - pull request against `main` writes nothing to `main`. The draft title always - starts with `WIP: `. -* Cleanup closes the pull request, deletes the ref, and then **re-reads both** to - confirm they are gone. `ephemeral.cleanup-verified` is mandatory: a half-removed - branch fails the run rather than passing it. Cleanup runs even when an earlier - step failed. -* `git push --force`, `--force-with-lease`, `--mirror`, `--all`, and `--tags` - are refused even in armed mode. +Before any runner, network request, or cluster vantage is created, preflight +binds the working directory to the existing linked PR #19 worktree, the exact +feature branch and reviewed local HEAD, the fixed HTTPS Atlas origin, base +`main`, confirmation phrase, and one unique ephemeral ref. The command policy is +then pinned to that exact ref. It allows only its push, one exact draft-create +payload, cleanup of the exactly discovered PR number, and deletion of that ref. -A default run reports all four ephemeral checks `NOT_RUN` and non-mandatory, so -a read-only sweep can still reach `GO`. +The run reserves at least 300 seconds (up to 6000) for cleanup. It verifies the +pushed SHA, always discovers the PR after create (including ambiguous create +responses), and requires exactly one open draft with the expected base, head ref, +base SHA, and head SHA. Cleanup requires successful PR close, successful branch deletion, +a successful empty `ls-remote`, and a strict re-read of the same closed PR. +Malformed or partial cleanup is `FAIL`; it is never inferred from an error. -## Release order +Force, force-with-lease, fan-out, protected refs, other repositories, other PR +numbers, arbitrary PATCH payloads, and repository administration remain denied. -### Baseline, already merged and deployed +## Dependency, merge, and conflict graph -PR #13 (`hermes: harden worker isolation and blocked-task semantics`) merged as -`ab346f55509d584e457fe26cf90be3078f7a375c`. Every check below assumes it is -present; `baseline.origin-main-descends-merged-work` asserts it. Override with -`--baseline-commit` only when a later merge legitimately replaces it. +Authoritative snapshot observed 2026-08-17 after refreshing Gitea and Git against +remote `main` `d8f2d818b9a552ea6c2d7fe86554be829bd5ffff`: -### Pending work, observed 2026-08-17T10:07Z - -These heads move. The harness re-fetches them at run time via -`baseline.pending-pull-requests-recorded`; the table is the snapshot this -document was written against, not an input. - -| PR | Branch | Head at observation | State | Delivers | +| PR | Head SHA | Draft | Base | Role | | --- | --- | --- | --- | --- | -| #14 | `feature/hermes-safe-gitea-pr-client` | `3cd8a17e772978fb554d88dd1c59777786be3991` | draft | Bounded forge authority, observer RBAC (removes `hermes-agent-cluster-admin`), node account hardening | -| #15 | `fix/hermes-result-decomposition-reliability` | `7609cd77e87a64e332b2f0d670c1d7a4f62b3d9e` | draft | Result preservation, duplicate-fanout prevention, execution-safety patch | -| #16 | `feature/hermes-unprivileged-image-builder` | `5ab0eba619779b3dfed2eb09d5c02f0a9c92dd0a` | ready | Daemonless Kaniko build lane, tokenless builder, Harbor immutable-tag rule | -| #17 | `feature/hermes-three-lane-placement` | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | draft | Three concurrent CLI lanes on the larger ARM64 nodes | -| #18 | `feature/hermes-distributed-worker-pool` | `20002527512235b29054f3636b02418168373009` | draft | Three-node fenced execution pool with coordinator-only state ownership | +| #12 | `465d386da5237f011cb408f1a3d4b7d0c6d8ffae` | no | `main` | AI usage exporter repair | +| #14 | `8c6e3acdacb223f68b1716f0a74bd543e18890e6` | yes | `main` | bounded forge client and least-privilege RBAC | +| #15 | `73fefbb5d92ef07371a9a997f22d848acd2d06ed` | yes | `main` | result-decomposition reliability | +| #16 | `e8c26ecf85a0c802d378fffffb98b7df43b78242` | no | `main` | unprivileged image builder | +| #17 | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | yes | `main` | three-lane placement | +| #18 | `20002527512235b29054f3636b02418168373009` | yes | `main` | distributed three-worker pool | +| #19 | `8f005458282269ba5c07941814e4237f5d4cf3ac` | yes | `main` | reviewed blocked head; this repair replaces it on the same branch | +| #20 | `a242dcc786576ae1a18000cb4941836ff610dff2` | yes | `main` | CLI process-group reaping | -Base at observation: `30259b526be5b2e226620aeebd9fb5f57dc88b7d`. +The repaired #19 SHA cannot be embedded in its own commit without changing that +SHA. Resolve it from Gitea after push and require it as `--reviewed-head-sha`; +the PR handoff records the exact value. All eight snapshot heads merge cleanly +with the snapshot `main` individually. Pairwise `git merge-tree --write-tree` +found conflicts only for: #12+#17, #14+#15, #14+#16, #14+#17, #14+#20, +#15+#16, #15+#17, #16+#20, and #17+#20. Gitea's `mergeable=false` on a draft is +not treated as conflict evidence. -### Merge order and why +Use this dependency order, refreshing every head and re-running merge-tree at +each step: -1. **#14 first.** It is the only change that *reduces* authority: it deletes - `services/hermes/agent-rbac.yaml` (the `cluster-admin` binding) and replaces - it with the namespaced/cluster observer roles, and it hardens the node - account. Merging it last would mean every earlier acceptance run measured a - cluster-admin agent. Every `access-denied` and `nodes` check depends on it. -2. **#16 second.** The image lane must exist before anything that needs a new - agent image is rolled out. It also touches `testing/quality_contract.json`, - so merging it early keeps later rebases small. `build.*` depends on it. -3. **#15 third.** It changes `dockerfiles/Dockerfile.hermes-agent`, which #16 - also touches — take #16's shape and rebase #15 onto it. `reliability.*` - depends on it. -4. **#17 fourth.** Placement only; it edits `services/hermes/agent-deployment.yaml` - and `testing/tests/test_hermes_cli_lanes.py`, both of which #14, #15, and #18 - also touch. Merging it after them avoids resolving the same conflict twice. -5. **#18 last.** The pool depends on the bounded forge client from #14, the - execution-safety patch from #15, and the agent image from #16. `pool.*` - depends on it. +1. #14 first so the release never restores or prolongs cluster-admin authority. +2. #16, then #15, resolving their Dockerfile and quality-contract changes + additively. Never take one PR's quality contract wholesale. +3. #20 after #15; it depends on the CLI lifecycle work and must be rebased across + the current #14/#16 contract and helper changes. +4. #12 before #17, resolving their Deployment overlap once. +5. #17, then #18, after the authority, image, reliability, and reaping work. +6. #19 last as the integration/acceptance change, only after #12/#14-#18/#20 are + merged and their exact supplied heads are ancestors of refreshed `main`. -Overlapping files to expect (rebase, do not merge blind): `services/hermes/agent-deployment.yaml` -(#12, #14, #16, #17, #18), `services/hermes/kustomization.yaml` (#14, #16, #18), -`services/hermes/scripts/stage_runtime_access.py` (#14, #16, #18), -`services/vault/scripts/vault_k8s_auth_configure.sh` (#14, #16, #18), -`dockerfiles/Dockerfile.hermes-agent` (#15, #16), -`testing/tests/test_hermes_cli_lanes.py` (#14, #15, #17, #18), -`testing/tests/test_hermes_runtime_access.py` (#16, #18). +#19 overlaps `testing/quality_contract.json` with #14/#15/#16, +`testing/quality_coverage.py` and its helper tests with #14, +`test_hermes_auto_router.py` with #18, and existing Kustomize content with +#12/#17/#18. Preserve branch coverage, every tracked module, fixture isolation, +and all desired-state changes during conflict resolution. -PR #12 (AI usage exporter) is independent and can merge at any point; it touches -`services/hermes/agent-deployment.yaml`, so land it before #17 if convenient. +The #16/#19 integration must keep both the immutable digest and the source tag +(`newTag: git--build-` plus the digest) in rendered image identity. +Digest-only rendering loses the attested build-source link and therefore +correctly remains `NO_GO` under the exact-lineage evaluator. -### Image build and Flux rollout +## Safe rollback guidance -1. Merge in the order above; observe the default-branch Jenkins build to a - terminal result after each merge - (`/opt/coordinator/jenkins_build_evidence.py titan-iac --branch main --wait`). -2. After #16 is merged, run the `hermes-agent-image` pipeline. It publishes an - immutable `git--build-` tag and emits - `build/hermes-agent-release/hermes-image-update.patch`. -3. Apply that patch to `services/hermes/kustomization.yaml` on a branch, review, - and merge. The digest is what pins the deployment; - `build.deployed-image-is-digest-pinned` asserts it. -4. Let Flux reconcile. Confirm with `flux get kustomizations -A` that `hermes` - and `hermes-chat` are `Ready`, then wait for the rollout to settle. -5. Re-run the harness. Do not run it mid-rollout: `gitops.*` tolerates an - in-flight reconcile (`Reconciling=True` with `Ready=False`) but a half-rolled - deployment will legitimately fail the `identity`, `routing`, and `surfaces` - groups. +Every rollback is a reviewed forward fix or selective Git revert followed by +the normal Flux workflow. Never restore `hermes-agent-cluster-admin`, even when +reverting unrelated portions of #14. If a rollback would widen RBAC, stop and +write a forward least-privilege fix instead. -### Rollback points +Reverting #18 removes the StatefulSet desired state, but its +`volumeClaimTemplates` PVCs are retained by the Kubernetes default because no +delete retention policy is set. Do not claim Flux pruning deletes those PVCs. +Any PVC deletion is a separate, explicit, human-reviewed data-destruction task +with exact targets and recovery implications. -| After | Roll back by | Restores | -| --- | --- | --- | -| #14 merge | Revert the merge commit; Flux restores `hermes-agent-cluster-admin` | Cluster-admin agent, previous node account state | -| #16 merge | Revert the merge; the previously pinned digest is unchanged in `kustomization.yaml` | Previous build lane; published immutable tags are not deleted | -| Image digest bump | Revert the `kustomization.yaml` commit to the prior digest | Previous agent image; Harbor immutability means the old tag is still exactly what it was | -| #15 merge | Revert the merge; the init-container patch disappears on the next rollout | Previous finalization/replay behaviour | -| #17 merge | Revert the merge | Previous lane concurrency and node placement | -| #18 merge | Revert the merge; the StatefulSet and its PVCs are removed by Flux pruning | Single-coordinator execution | +Image rollback means a reviewed commit restoring a previously attested digest; +never retag an immutable image. Do not use direct `kubectl` edits as a durable +rollback. -Every rollback is a Git revert reconciled by Flux. Do not roll back by direct -`kubectl` edit; the next reconcile would undo it. +## Quality and release evidence -## Go / no-go checklist +The canonical quality contract explicitly manages, lints, and tracks all 15 +`scripts/ops/hermes_handoff_*.py` modules. It enforces at least 95% line and +branch coverage per tracked file and the 500-physical-line source cap. Contract +tests derive the module set so a future file cannot silently escape the gate. +Four pre-existing over-cap test files are temporarily locked to their exact +line count and SHA-256; any byte edit invalidates the exception and must split +the file below 500 lines. PR #14 supplies the intended additive test splits. -Run in order. Stop at the first item that does not hold. - -1. `origin/main` is `ab346f55509d584e457fe26cf90be3078f7a375c` or a descendant. -2. PRs #14, #15, #16, #17, #18 are all merged into `main`, in that order, each - with a green default-branch build. -3. The agent image has been built by the #16 lane, is pinned by digest in - `services/hermes/kustomization.yaml`, and Flux has finished reconciling. -4. The harness is run from an **external operator kubeconfig**, not from inside - the Hermes owner pod, with `--node-count` set to the real node count and every - deliberate Flux suspension declared via `--expected-suspension`. -5. The default read-only run reports `GO` — no mandatory `FAIL`, no mandatory - `NOT_RUN`, no `harness_errors`. -6. The armed run reports `PASS` for all four `ephemeral.*` checks, including - `ephemeral.cleanup-verified`, and the ephemeral ref and pull request are - confirmed gone. -7. The JSON report is archived with the exact `origin/main` commit and the - recorded pull-request heads from `baseline.pending-pull-requests-recorded`. - -Only then is the handoff releasable. - -## Known limits - -Stated so nobody mistakes a proxy for the thing itself. - -* **Merge, approve, and close authority is asserted, not exercised.** The - harness never attempts a merge. It proves the forge identity holds - `permissions.admin: false` and is live-refused on an administrative route; - the route-level allowlist is covered by PR #14's own regression suite. -* **Attach and port-forward have no side-effect-free live attempt.** Their - authorization reviews ride alongside a real, refused control-plane write in the - same check, and pod exec — the one subresource with a harmless real attempt — - has its own check. -* **TokenRequest is asserted by review.** A successful live attempt would mint a - real credential, so the check pairs the `serviceaccounts/token` review with a - refused `serviceaccount` create in the same RBAC family. -* **Branch protection is not read.** The harness identity is not a repository - administrator, so `GET /branch_protections` is refused — which is itself - recorded as evidence. Protected-push refusal is guaranteed by the harness's own - preflight, exercised on every armed run, not by reading the forge's config. -* **Stale/duplicate assignment safety is asserted as deployed configuration.** - The behaviour itself is covered by the pool's regression suite; the harness - checks that the workers carry the configuration and that the coordinator keeps - sole ownership of its claims. -* **Telegram session continuity is per tenant** and depends on there having been - Telegram traffic, so it is off by default. The always-on check is that topic - and tenant state live on a persistent claim rather than in the pod. -* **The routing window is a tail**, 600 records by default. A property that did - not occur inside that window is reported as missing rather than absent; widen - the window or wait for traffic rather than removing the assertion. - -## Tests - -``` -python3 -m pytest testing/tests/ -k handoff -python3 -m coverage run --source=scripts/ops --include="*/hermes_handoff_*.py" \ - -m pytest testing/tests/ -k handoff && python3 -m coverage report +```sh +python3 -m testing.quality_gate --profile local --build-dir build +python3 -m testing.quality_handoff_mutation +python3 -m ruff check --select F,B,SIM,C4,UP --ignore B017,UP015,UP035 \ + scripts/ops/hermes_handoff_*.py testing/quality_*.py testing/tests/test_hermes_handoff_*.py +python3 -m ruff format --check \ + scripts/ops/hermes_handoff_*.py testing/quality_*.py testing/tests/test_hermes_handoff_*.py +python3 -m compileall -q scripts/ops testing +kustomize build services/hermes | kubectl apply --dry-run=client --validate=false -f - +git diff --check origin/main...HEAD ``` -The suite covers parsing, redaction, timeouts and deadlines, partial failure, -zero state, dual-vantage mismatch, mandatory skips, catalog structure, and -ephemeral arming and cleanup — including the cases where cleanup cannot be -verified. +The deterministic mutation gate must kill every policy, redaction, verdict, +numeric, truncation, and cleanup mutant. The adversarial suite includes large +concurrent stdout/stderr, malformed JSON and bytes, zero state, partial failure, +absolute timeout, descendant-held pipes, malformed cleanup, ambiguous creates, +and no-subprocess/no-mutation spies. Render/client dry-run, exact conflict checks, +and bounded diff/full-tree secret scans must also pass. + +The provider-health and Gitea-token-sensitive tests use hermetic fixtures. Their +result must be identical whether synthetic runtime files exist or are absent; +tests must never read the mounted production token. + +## Release decision + +The handoff remains `NO_GO` until all dependency heads are merged, the exact +lineage check passes from an external operator, both default and armed reports +are complete, the branch and PR cleanup is verified, the canonical and +adversarial gates are green, and a human has reviewed PR #19 plus the artifacts. +The harness never merges, deploys, reconciles, publishes, or approves anything. diff --git a/scripts/ops/hermes_handoff_acceptance.py b/scripts/ops/hermes_handoff_acceptance.py index 7be6de29..6b35d49f 100755 --- a/scripts/ops/hermes_handoff_acceptance.py +++ b/scripts/ops/hermes_handoff_acceptance.py @@ -11,8 +11,9 @@ unreachable vantage, a catalog whose evidence no longer exists, an expired deadline, or two vantages that turn out to be the same principal. Nothing here rounds an absence of evidence up to a pass. - scripts/ops/hermes_handoff_acceptance.py --context atlas-operator - scripts/ops/hermes_handoff_acceptance.py --output build/handoff.json + scripts/ops/hermes_handoff_acceptance.py --help + +The release runbook lists every required freshness and lineage input. Mutation lives behind a separate arming flag, needs an exact confirmation phrase and a unique ref, refuses protected push targets before any network @@ -41,17 +42,24 @@ from hermes_handoff_ephemeral import ( # noqa: E402 from hermes_handoff_exec import ( # noqa: E402 DEFAULT_COMMAND_TIMEOUT, DEFAULT_RUN_DEADLINE, - SELF, Runner, ) from hermes_handoff_model import GO, Report, unscreened_fields, utc_now # noqa: E402 -from hermes_handoff_policy import ARMED, READ_ONLY # noqa: E402 +from hermes_handoff_policy import ( # noqa: E402 + ARMED, + EXPECTED_BASE, + EXPECTED_REMOTE, + EXPECTED_REPO, + READ_ONLY, +) from hermes_handoff_redaction import DEFAULT_MAX_BYTES # noqa: E402 from hermes_handoff_run import ( # noqa: E402 build_catalog, build_report, resolve_vantages, unavailable_report, + validate_catalog, + validate_targets, ) EXIT_GO = 0 @@ -59,6 +67,26 @@ EXIT_NO_GO = 1 EXIT_USAGE = 2 +def dependency_head(value: str) -> tuple[int, str]: + """Parse one `PR=SHA` dependency-head binding.""" + try: + number, sha = value.split("=", 1) + parsed = int(number) + except (TypeError, ValueError) as exc: + raise argparse.ArgumentTypeError( + "dependency head must be PR=40-character-SHA" + ) from exc + if ( + parsed <= 0 + or len(sha) != 40 + or any(character not in "0123456789abcdef" for character in sha) + ): + raise argparse.ArgumentTypeError( + "dependency head must be PR=lowercase-40-character-SHA" + ) + return (parsed, sha) + + def build_parser() -> argparse.ArgumentParser: """Return the command-line interface for the harness.""" parser = argparse.ArgumentParser( @@ -67,21 +95,41 @@ def build_parser() -> argparse.ArgumentParser: formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--namespace", default="hermes", help="Hermes namespace.") - parser.add_argument("--context", help="Kubernetes context for the operator vantage.") + parser.add_argument( + "--context", help="Kubernetes context for the operator vantage." + ) parser.add_argument("--kubeconfig", help="Kubeconfig for the operator vantage.") - parser.add_argument("--output", type=Path, help="Write the JSON report here instead of stdout.") - parser.add_argument("--json-only", action="store_true", help="Suppress the human summary.") parser.add_argument( - "--command-timeout", type=float, default=DEFAULT_COMMAND_TIMEOUT, help="Per-command seconds." + "--output", type=Path, help="Write the JSON report here instead of stdout." ) parser.add_argument( - "--deadline", type=float, default=DEFAULT_RUN_DEADLINE, help="Whole-run seconds." + "--json-only", action="store_true", help="Suppress the human summary." ) parser.add_argument( - "--max-output-bytes", type=int, default=DEFAULT_MAX_BYTES, help="Per-capture byte budget." + "--command-timeout", + type=float, + default=DEFAULT_COMMAND_TIMEOUT, + help="Per-command seconds.", ) parser.add_argument( - "--baseline-commit", default=Targets.baseline_commit, help="Commit main must descend from." + "--deadline", + type=float, + default=DEFAULT_RUN_DEADLINE, + help="Whole-run seconds.", + ) + parser.add_argument( + "--max-output-bytes", + type=int, + default=DEFAULT_MAX_BYTES, + help="Per-capture byte budget.", + ) + parser.add_argument( + "--concurrency", type=int, default=1, help="Validated scheduling bound." + ) + parser.add_argument( + "--baseline-commit", + default=Targets.baseline_commit, + help="Commit main must descend from.", ) parser.add_argument( "--dependency-pr", @@ -90,6 +138,31 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="Pull request that must be merged before this release is certifiable (repeatable).", ) + parser.add_argument( + "--dependency-head", + type=dependency_head, + action="append", + default=None, + metavar="PR=SHA", + help="Exact current dependency PR head (repeat for every dependency).", + ) + parser.add_argument( + "--remote-main-sha", default="", help="Exact observed origin/main SHA." + ) + parser.add_argument( + "--reviewed-head-sha", default="", help="Exact reviewed PR #19 head SHA." + ) + parser.add_argument( + "--agent-image", + default="", + help="Exact agent :git--build-@sha256 identity.", + ) + parser.add_argument( + "--build-sha", default="", help="Exact source SHA used for the image build." + ) + parser.add_argument( + "--deployment-revision", default="", help="Exact running Deployment revision." + ) parser.add_argument( "--expected-suspension", action="append", @@ -97,10 +170,19 @@ def build_parser() -> argparse.ArgumentParser: metavar="NAMESPACE/NAME", help="Flux object allowed to be suspended (repeatable).", ) - parser.add_argument("--node-count", type=int, default=1, help="Nodes the fleet must cover.") - parser.add_argument("--pool-replicas", type=int, default=3, help="Expected pool worker count.") parser.add_argument( - "--chat-config-revision", default="", help="Expected chat pod-template config revision." + "--node-count", + type=int, + default=3, + help="Nodes the fleet must cover (fixed at 3).", + ) + parser.add_argument( + "--pool-replicas", type=int, default=3, help="Expected pool worker count." + ) + parser.add_argument( + "--chat-config-revision", + default="", + help="Expected chat pod-template config revision.", ) parser.add_argument( "--chat-ordinal", @@ -111,6 +193,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--expect-telegram-sessions", action="store_true", + default=True, help="Require the selected chat tenant to hold durable Telegram-sourced sessions.", ) parser.add_argument( @@ -131,11 +214,12 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Permit one ephemeral branch push and draft pull request, then remove both.", ) - armed.add_argument("--confirm", default="", help=f"Exact confirmation phrase: {CONFIRMATION!r}.") - armed.add_argument("--ephemeral-token", default="", help="Unique 8-64 character ref suffix.") - armed.add_argument("--ephemeral-base", default="main", help="Pull-request base branch.") - armed.add_argument("--remote", default="origin", help="Git remote for the ephemeral push.") - armed.add_argument("--repo", default=Targets.repo, help="owner/name of the forge repository.") + armed.add_argument( + "--confirm", default="", help=f"Exact confirmation phrase: {CONFIRMATION!r}." + ) + armed.add_argument( + "--ephemeral-token", default="", help="Unique 8-64 character ref suffix." + ) return parser @@ -145,17 +229,23 @@ def targets_from_args(arguments: argparse.Namespace, now: dt.datetime) -> Target return Targets( namespace=arguments.namespace, baseline_commit=arguments.baseline_commit, - dependency_pull_requests=tuple(arguments.dependency_pr or defaults.dependency_pull_requests), + dependency_pull_requests=tuple( + arguments.dependency_pr or defaults.dependency_pull_requests + ), + dependency_heads=tuple(arguments.dependency_head or ()), + remote_main_sha=arguments.remote_main_sha, + reviewed_head_sha=arguments.reviewed_head_sha, + agent_image=arguments.agent_image, + build_sha=arguments.build_sha, + deployment_revision=arguments.deployment_revision, expected_suspensions=tuple(arguments.expected_suspension or ()), node_count=arguments.node_count, pool_replicas=arguments.pool_replicas, chat_config_revision=arguments.chat_config_revision, chat_ordinal=arguments.chat_ordinal, expect_telegram_sessions=arguments.expect_telegram_sessions, - pool_worker_env=tuple(arguments.pool_worker_env or ()), + pool_worker_env=tuple(arguments.pool_worker_env or defaults.pool_worker_env), max_evidence_age_seconds=arguments.max_evidence_age, - repo=arguments.repo, - remote=arguments.remote, now=now, ) @@ -186,14 +276,16 @@ def emit(report: Report, arguments: argparse.Namespace, stream=None) -> list[str def _armed_results(arguments: argparse.Namespace, runner: Runner, vantages) -> list: request = ArmRequest( - repo=arguments.repo, - remote=arguments.remote, + repo=EXPECTED_REPO, + remote=EXPECTED_REMOTE, token=arguments.ephemeral_token, confirmation=arguments.confirm, - base=arguments.ephemeral_base, + base=EXPECTED_BASE, + expected_head=arguments.reviewed_head_sha, + expected_base_sha=arguments.remote_main_sha, ) - preflight(request, arguments.repo) - vantage = vantages.get(SELF) or vantages["operator"] + preflight(request, EXPECTED_REPO, Path.cwd()) + vantage = vantages["operator"] return run_armed(runner, vantage, request) @@ -205,27 +297,64 @@ def main(argv: Sequence[str] | None = None) -> int: targets = targets_from_args(arguments, now) mode = ARMED if arguments.arm_ephemeral_push else READ_ONLY - runner = Runner( - mode=mode, - command_timeout=arguments.command_timeout, - deadline_seconds=arguments.deadline, - max_bytes=arguments.max_output_bytes, - ) - vantages, records = resolve_vantages(runner, targets, arguments.kubeconfig, arguments.context) - - armed_results: list = [] + specs = build_catalog(targets) + startup_problems = [*validate_targets(targets), *validate_catalog(specs)] if arguments.arm_ephemeral_push: try: - armed_results = _armed_results(arguments, runner, vantages) + preflight( + ArmRequest( + repo=EXPECTED_REPO, + remote=EXPECTED_REMOTE, + token=arguments.ephemeral_token, + confirmation=arguments.confirm, + base=EXPECTED_BASE, + expected_head=arguments.reviewed_head_sha, + expected_base_sha=arguments.remote_main_sha, + ), + EXPECTED_REPO, + Path.cwd(), + ) except ArmingError as exc: - report = unavailable_report(mode, started_at, f"arming refused: {exc}", targets) - emit(report, arguments) - return EXIT_NO_GO + startup_problems.append(f"arming refused: {exc}") + if startup_problems: + report = unavailable_report( + mode, started_at, "; ".join(startup_problems), targets + ) + emit(report, arguments) + return EXIT_NO_GO - specs = build_catalog(targets) - report = build_report( - runner, targets, specs, vantages, records, mode, started_at, armed_results + try: + runner = Runner( + mode=mode, + command_timeout=arguments.command_timeout, + deadline_seconds=arguments.deadline, + max_bytes=arguments.max_output_bytes, + concurrency=arguments.concurrency, + ) + except ValueError as exc: + report = unavailable_report( + mode, started_at, f"invalid execution bound: {exc}", targets + ) + emit(report, arguments) + return EXIT_USAGE + vantages, records = resolve_vantages( + runner, targets, arguments.kubeconfig, arguments.context ) + report = build_report(runner, targets, specs, vantages, records, mode, started_at) + if arguments.arm_ephemeral_push: + if report.decision != GO: + report.harness_errors.append( + "armed mutation was not started because read-only acceptance did not GO" + ) + else: + armed_results = _armed_results(arguments, runner, vantages) + armed_ids = {result.spec.id for result in armed_results} + report.results = [ + result for result in report.results if result.spec.id not in armed_ids + ] + report.results.extend(armed_results) + report.results.sort(key=lambda result: result.spec.id) + report.finished_at = utc_now() offenders = emit(report, arguments) if offenders: return EXIT_NO_GO diff --git a/scripts/ops/hermes_handoff_catalog.py b/scripts/ops/hermes_handoff_catalog.py index b59ff2b9..9dc5c98a 100644 --- a/scripts/ops/hermes_handoff_catalog.py +++ b/scripts/ops/hermes_handoff_catalog.py @@ -4,8 +4,8 @@ The catalog is data on purpose. A reviewer should be able to read what the release is asserted to satisfy without reading any plumbing, and the harness should be able to prove structural properties over every entry — both vantages -represented, no impersonation from outside a pod, no mutation outside a server -dry run — before it runs a single command. +represented, no impersonation from outside a pod, and no mutation command of +any kind — before it runs a single command. Catalog drift is a NO_GO, not a pass. If a manifest, a field name, or a workload is renamed, the affected checks report ``NOT_RUN`` because the evidence they @@ -19,7 +19,8 @@ from collections.abc import Sequence from dataclasses import dataclass, field from typing import Any -from hermes_handoff_model import ALWAYS, ATTEMPT, READ, REVIEW, CheckSpec, Step +from hermes_handoff_model import ALWAYS, READ, CheckSpec, Step +from hermes_handoff_policy import projection OPERATOR = "operator" SELF = "self" @@ -28,21 +29,38 @@ NODE = "node" CHAT = "chat" VANTAGE_NAMES = (OPERATOR, SELF, SWITCHYARD, NODE, CHAT) -# `namespace/namesuspendType=Status,...` — one compact row per object. -FLUX_PROJECTION = ( - "jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{'\\t'}" - "{.spec.suspend}{'\\t'}{range .status.conditions[*]}{.type}={.status},{end}{'\\n'}{end}" -).replace("'", '"') +# `namegenerationobservedGenerationsuspendconditions`. +FLUX_PROJECTION = projection( + ( + "jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{'\\t'}" + "{.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}" + "{range .status.conditions[*]}{.type}={.status},{end}{'\\n'}{end}" + ).replace("'", '"') +) -ENV_NAME_PROJECTION = ( - "jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{'\\n'}{end}{end}" -).replace("'", '"') +ENV_NAME_PROJECTION = projection( + ( + "jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{'\\n'}{end}{end}" + ).replace("'", '"') +) -INIT_NAME_PROJECTION = ( - "jsonpath={range .spec.template.spec.initContainers[*]}{.name}{'\\n'}{end}" -).replace("'", '"') +INIT_NAME_PROJECTION = projection( + ( + "jsonpath={range .spec.template.spec.initContainers[*]}{.name}{'\\n'}{end}" + ).replace("'", '"') +) -NODE_NAME_PROJECTION = "jsonpath={range .items[*]}{.spec.nodeName}{'\\n'}{end}".replace("'", '"') +NODE_NAME_PROJECTION = projection( + "jsonpath={range .items[*]}{.spec.nodeName}{'\\n'}{end}".replace("'", '"') +) + +POOL_PROJECTION = projection( + ( + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.spec.nodeName}{'\\t'}" + "{.status.phase}{'\\t'}{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}" + "{'\\t'}{range .spec.volumes[*]}{.persistentVolumeClaim.claimName}{','}{end}{'\\n'}{end}" + ).replace("'", '"') +) PROVIDER_API_KEY_NAMES = ( "ANTHROPIC_API_KEY", @@ -56,7 +74,12 @@ PROVIDER_API_KEY_NAMES = ( "OPENAI_API_KEY", "OPENAI_BASE_URL", ) -FORGE_CREDENTIAL_NAMES = ("GITEA_PASSWORD", "GITEA_TOKEN", "GITEA_USERNAME", "GIT_PASSWORD") +FORGE_CREDENTIAL_NAMES = ( + "GITEA_PASSWORD", + "GITEA_TOKEN", + "GITEA_USERNAME", + "GIT_PASSWORD", +) CODEX_HEALTH = "/opt/data/provider-health/codex.json" CLAUDE_HEALTH = "/opt/data/provider-health/claude.json" @@ -92,31 +115,44 @@ class Targets: pool_statefulset: str = "hermes-execution-worker" pool_replicas: int = 3 pool_selector: str = "app=hermes-execution-worker" - coordinator_claims: tuple[str, ...] = ("hermes-agent-home", "hermes-agent-workspace") + coordinator_claims: tuple[str, ...] = ( + "hermes-agent-home", + "hermes-agent-workspace", + ) node_account: str = "hermes-agent" node_account_identity: str = "1200:1200:/home/hermes-agent:/bin/bash" - node_count: int = 1 + node_count: int = 3 repo: str = "atlas/titan-iac" remote: str = "origin" baseline_commit: str = "ab346f55509d584e457fe26cf90be3078f7a375c" - dependency_pull_requests: tuple[int, ...] = (14, 15, 16, 17, 18) + dependency_pull_requests: tuple[int, ...] = (12, 14, 15, 16, 17, 18, 20) + dependency_heads: tuple[tuple[int, str], ...] = () + remote_main_sha: str = "" + reviewed_pr_number: int = 19 + reviewed_head_ref: str = "feature/hermes-full-handoff-acceptance" + reviewed_head_sha: str = "" + agent_image: str = "" + build_sha: str = "" + deployment_revision: str = "" expected_suspensions: tuple[str, ...] = () harbor_namespace: str = "harbor" harbor_immutability_job: str = "hermes-agent-immutability" builder_namespace: str = "jenkins" builder_serviceaccount: str = "hermes-image-builder" builder_pipeline: str = "ci/Jenkinsfile.hermes-agent-image" - builder_capabilities: str = 'add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]' + builder_capabilities: str = ( + 'add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]' + ) agent_init_containers: tuple[str, ...] = ("patch-web-session-activity",) chat_init_containers: tuple[str, ...] = ("patch-api-server-sessions",) chat_config_revision: str = "" - pool_worker_env: tuple[str, ...] = () + pool_worker_env: tuple[str, ...] = ("HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL") max_evidence_age_seconds: int = 86_400 routing_tail_lines: int = 600 routing_tail_bytes: int = 512 * 1024 listing_bytes: int = 512 * 1024 git_askpass: str = "/opt/coordinator/gitea_askpass.sh" - expect_telegram_sessions: bool = False + expect_telegram_sessions: bool = True now: Any = None extra: dict[str, Any] = field(default_factory=dict) @@ -165,25 +201,3 @@ def check( scope=scope, rationale=rationale, ) - - -def deny( - identifier: str, - title: str, - review: Sequence[str], - attempt: Sequence[str], - rationale: str, - group: str = "access-denied", -) -> CheckSpec: - """Return a deny check that pairs a self review with a real refused request.""" - return check( - identifier, - title, - group, - "denied", - [ - step("review", SELF, *review, kind=REVIEW), - step("attempt", SELF, *attempt, kind=ATTEMPT), - ], - rationale=rationale, - ) diff --git a/scripts/ops/hermes_handoff_checks_access.py b/scripts/ops/hermes_handoff_checks_access.py index a863e0f3..df5c8117 100644 --- a/scripts/ops/hermes_handoff_checks_access.py +++ b/scripts/ops/hermes_handoff_checks_access.py @@ -1,11 +1,9 @@ #!/usr/bin/env python3 """Acceptance checks for what Hermes may and may not do to the cluster. -Every deny check pairs an in-pod authorization review with a real request that -must be refused, because a review reports what RBAC says and only an attempt -proves the API server enforces it. Mutating attempts go through a server dry -run: the API server authorises the request and discards it, so the refusal is -genuine and nothing is written either way. +Denials use Kubernetes authorization reviews only. Live denied attempts were +removed because a policy regression could turn a supposed probe into a real +credential read or mutation; the default harness never constructs such argv. """ from __future__ import annotations @@ -16,11 +14,9 @@ from hermes_handoff_catalog import ( SELF, Targets, check, - deny, step, ) -from hermes_handoff_model import ATTEMPT, REVIEW, CheckSpec -from hermes_handoff_policy import shell +from hermes_handoff_model import REVIEW, CheckSpec def _access(targets: Targets) -> list[CheckSpec]: @@ -31,22 +27,45 @@ def _access(targets: Targets) -> list[CheckSpec]: "No ClusterRoleBinding grants the agent service account cluster-admin", "access-denied", "names_absent", - [step("bindings", OPERATOR, "kubectl", "get", "clusterrolebindings", "-o", "name")], + [ + step( + "bindings", + OPERATOR, + "kubectl", + "get", + "clusterrolebindings", + "-o", + "name", + ) + ], {"step": "bindings", "contains": ("hermes-agent-cluster-admin",)}, - rationale="The operator half of the pair; the in-pod attempts below are the enforcement half.", + rationale="The binding inventory and in-pod authorization reviews are independent desired/enforced views.", ), - deny( + check( "access.secrets-are-denied", "Hermes cannot read Secrets cluster-wide", - ("kubectl", "auth", "can-i", "get", "secrets", "--all-namespaces"), - ("kubectl", "get", "secrets", "--all-namespaces", "-o", "name"), - "Even a regression that granted this could only return object names: the probe is pinned to -o name.", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "get", + "secrets", + "--all-namespaces", + kind=REVIEW, + ) + ], + rationale="A live Secret request is never constructed, even when an RBAC regression would allow it.", ), check( "access.service-account-tokens-are-denied", "Hermes cannot mint a service-account token", "access-denied", - "denied", + "review_denied", [ step( "review-token", @@ -72,119 +91,155 @@ def _access(targets: Targets) -> list[CheckSpec]: namespace, kind=REVIEW, ), - step( - "attempt", - SELF, - "kubectl", - "--namespace", - namespace, - "create", - "serviceaccount", - "hermes-acceptance-probe", - "--dry-run=server", - kind=ATTEMPT, - ), ], - rationale="TokenRequest has no side-effect-free live attempt — a successful one would mint a real credential — so it is asserted by review alongside a refused write in the same RBAC family.", + rationale="Token creation has no credential-safe live probe, so both relevant grants must be denied by review.", ), - deny( + check( "access.impersonation-is-denied", "Hermes cannot impersonate another identity", - ("kubectl", "auth", "can-i", "impersonate", "users", "--all-namespaces"), - ("kubectl", "get", "namespaces", "--as", "system:admin", "-o", "name"), - "`--as` is reachable only from inside the pod, where the identity is testing its own refusal.", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "impersonate", + "users", + "--all-namespaces", + kind=REVIEW, + ) + ], + rationale="The default harness never constructs an impersonated request.", ), - deny( + check( "access.workload-mutation-is-denied", "Hermes cannot mutate its own workload", - ("kubectl", "auth", "can-i", "patch", "deployments", "--namespace", namespace), - ( - "kubectl", - "--namespace", - namespace, - "patch", - f"deploy/{targets.agent_deployment}", - "--type=merge", - "--patch", - '{"metadata":{"annotations":{"acceptance.bstein.dev/probe":"deny"}}}', - "--dry-run=server", - ), - "A server dry run is authorised and then discarded, so the refusal is real and nothing is written.", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "patch", + "deployments", + "--namespace", + namespace, + kind=REVIEW, + ) + ], + rationale="Client-side and server-side dry-run mutation tricks are excluded structurally.", ), - deny( + check( "access.pod-exec-is-denied", "Hermes cannot exec into an arbitrary pod", - ("kubectl", "auth", "can-i", "create", "pods/exec", "--all-namespaces"), - ( - "kubectl", - "--namespace", - namespace, - "exec", - f"deploy/{targets.switchyard_deployment}", - "--container", - targets.switchyard_container, - "--", - *shell("path_readable", path="/tmp"), - ), - "Exec is the one subresource with a harmless real attempt; attach and port-forward are covered by their own reviews.", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "pods/exec", + "--all-namespaces", + kind=REVIEW, + ) + ], + rationale="No cross-pod exec request is created by a read-only run.", ), check( "access.attach-portforward-and-kube-system-writes-are-denied", "Hermes holds no attach, port-forward, or control-plane write authority", "access-denied", - "denied", + "review_denied", [ - step("review-attach", SELF, "kubectl", "auth", "can-i", "create", "pods/attach", "--all-namespaces", kind=REVIEW), - step("review-portforward", SELF, "kubectl", "auth", "can-i", "create", "pods/portforward", "--all-namespaces", kind=REVIEW), - step("review-create", SELF, "kubectl", "auth", "can-i", "create", "configmaps", "--namespace", "kube-system", kind=REVIEW), step( - "attempt", + "review-attach", SELF, "kubectl", + "auth", + "can-i", + "create", + "pods/attach", + "--all-namespaces", + kind=REVIEW, + ), + step( + "review-portforward", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "pods/portforward", + "--all-namespaces", + kind=REVIEW, + ), + step( + "review-create", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "configmaps", "--namespace", "kube-system", - "create", - "configmap", - "hermes-acceptance-probe", - "--dry-run=server", - kind=ATTEMPT, + kind=REVIEW, ), ], - rationale="Attach and port-forward have no side-effect-free live attempt, so their reviews ride alongside a real, refused control-plane write.", + rationale="Every authority is reviewed without issuing the request it would authorize.", ), check( "access.required-reads-are-allowed", "Hermes retains the reads its operator role depends on", "access-allowed", "allowed", - [ - step("watch-review", SELF, "kubectl", "auth", "can-i", "watch", "pods", "--namespace", namespace, kind=REVIEW), - step("pods", SELF, "kubectl", "--namespace", namespace, "get", "pods", "-o", "name"), - step("namespaces", SELF, "kubectl", "get", "namespaces", "-o", "name"), - step("deployments", SELF, "kubectl", "--namespace", namespace, "get", "deployments", "-o", "name"), - step("nodes", SELF, "kubectl", "get", "nodes", "-o", "name"), - ], - ), - check( - "access.pod-logs-are-allowed", - "Hermes can still read pod logs", - "access-allowed", - "allowed", [ step( - "logs", + "watch-review", + SELF, + "kubectl", + "auth", + "can-i", + "watch", + "pods", + "--namespace", + namespace, + kind=REVIEW, + ), + step( + "pods", SELF, "kubectl", "--namespace", - targets.namespace, - "logs", - f"deploy/{targets.switchyard_deployment}", - "--container", - targets.switchyard_container, - "--tail", - "1", - record=False, - ) + namespace, + "get", + "pods", + "-o", + "name", + ), + step("namespaces", SELF, "kubectl", "get", "namespaces", "-o", "name"), + step( + "deployments", + SELF, + "kubectl", + "--namespace", + namespace, + "get", + "deployments", + "-o", + "name", + ), + step("nodes", SELF, "kubectl", "get", "nodes", "-o", "name"), ], ), check( @@ -193,8 +248,28 @@ def _access(targets: Targets) -> list[CheckSpec]: "access-allowed", "allowed", [ - step("kustomizations", SELF, "kubectl", "get", "kustomizations.kustomize.toolkit.fluxcd.io", "--all-namespaces", "-o", "name", record=False), - step("helmreleases", SELF, "kubectl", "get", "helmreleases.helm.toolkit.fluxcd.io", "--all-namespaces", "-o", "name", record=False), + step( + "kustomizations", + SELF, + "kubectl", + "get", + "kustomizations.kustomize.toolkit.fluxcd.io", + "--all-namespaces", + "-o", + "name", + record=False, + ), + step( + "helmreleases", + SELF, + "kubectl", + "get", + "helmreleases.helm.toolkit.fluxcd.io", + "--all-namespaces", + "-o", + "name", + record=False, + ), ], ), check( @@ -203,8 +278,26 @@ def _access(targets: Targets) -> list[CheckSpec]: "access-allowed", "vantages_agree", [ - step("operator", OPERATOR, "kubectl", "get", "namespaces", "-o", "name", record=False), - step("self", SELF, "kubectl", "get", "namespaces", "-o", "name", record=False), + step( + "operator", + OPERATOR, + "kubectl", + "get", + "namespaces", + "-o", + "name", + record=False, + ), + step( + "self", + SELF, + "kubectl", + "get", + "namespaces", + "-o", + "name", + record=False, + ), ], {"steps": ("operator", "self")}, rationale="A disagreement here means one vantage is not seeing the cluster the other is certifying.", @@ -219,7 +312,19 @@ def _gitops(targets: Targets) -> list[CheckSpec]: f"No unexpected {label} suspension or unhealthy reconciliation", "gitops", "flux_health", - [step("objects", OPERATOR, "kubectl", "get", kind, "--all-namespaces", "-o", FLUX_PROJECTION, record=False)], + [ + step( + "objects", + OPERATOR, + "kubectl", + "get", + kind, + "--all-namespaces", + "-o", + FLUX_PROJECTION, + record=False, + ) + ], {"step": "objects", "expected_suspensions": targets.expected_suspensions}, ) for kind, label in ( diff --git a/scripts/ops/hermes_handoff_checks_delivery.py b/scripts/ops/hermes_handoff_checks_delivery.py index a3dbaad7..102f93e4 100644 --- a/scripts/ops/hermes_handoff_checks_delivery.py +++ b/scripts/ops/hermes_handoff_checks_delivery.py @@ -17,7 +17,6 @@ from __future__ import annotations from hermes_handoff_catalog import ( FORGE_CREDENTIAL_NAMES, INIT_NAME_PROJECTION, - NODE, NODE_NAME_PROJECTION, OPERATOR, SELF, @@ -26,10 +25,7 @@ from hermes_handoff_catalog import ( step, ) from hermes_handoff_model import ATTEMPT, CheckSpec -from hermes_handoff_policy import GITEA_CLIENT, shell - -HOST_ETC = "/host-etc" -HOST_HOME = "/host-home" +from hermes_handoff_policy import GITEA_CLIENT, projection, shell def _forge(targets: Targets) -> list[CheckSpec]: @@ -44,14 +40,6 @@ def _forge(targets: Targets) -> list[CheckSpec]: {"step": "env", "names": FORGE_CREDENTIAL_NAMES}, rationale="Authentication reaches Git through an askpass helper that reads runtime state, never through the environment.", ), - check( - "forge.worker-holds-no-credential-file", - "The worker home has no stored Git credential file", - "forge", - "stdout_matches", - [step("credentials", SELF, *shell("path_readable", path="/opt/data/.git-credentials"))], - {"step": "credentials", "equals": "unreadable"}, - ), check( "forge.identity-is-not-an-administrator", "The forge identity the worker uses holds no administrative rights", @@ -93,117 +81,11 @@ def _forge(targets: Targets) -> list[CheckSpec]: ], rationale="A live refusal, not a configuration reading. The same lack of authority is what blocks a merge route.", ), - check( - "forge.clone-and-fetch-still-work", - "The worker can still read the repository it is expected to work in", - "forge", - "allowed", - [ - step( - "ls-remote", - SELF, - *shell( - "git_ls_remote", - askpass=targets.git_askpass, - url=f"https://scm.bstein.dev/{targets.repo}.git", - ), - ) - ], - ), ] def _nodes(targets: Targets) -> list[CheckSpec]: - account = targets.node_account return [ - check( - "nodes.account-password-is-locked", - "The dedicated node account has no usable password", - "nodes", - "stdout_matches", - [ - step( - "lock", - NODE, - *shell("account_lock_state", account=account, path=f"{HOST_ETC}/shadow"), - ) - ], - {"step": "lock", "equals": "locked"}, - rationale="The probe reports the lock marker only; the password hash is never read.", - ), - check( - "nodes.account-identity-is-dedicated", - "The node account is the dedicated Hermes identity, not a shared one", - "nodes", - "stdout_matches", - [ - step( - "identity", - NODE, - *shell("account_identity", account=account, path=f"{HOST_ETC}/passwd"), - ) - ], - {"step": "identity", "equals": targets.node_account_identity}, - ), - check( - "nodes.account-holds-no-supplementary-groups", - "The node account joins no supplementary or administrative group", - "nodes", - "stdout_matches", - [ - step( - "groups", - NODE, - *shell("account_groups", account=account, path=f"{HOST_ETC}/group"), - ) - ], - {"step": "groups", "equals": ""}, - rationale="sudo, docker, adm, and disk are all group grants; an empty membership list rules out every one of them.", - ), - check( - "nodes.account-has-no-sudo-policy", - "No sudoers fragment names the node account", - "nodes", - "stdout_matches", - [step("sudoers", NODE, *shell("path_mode", path=f"{HOST_ETC}/sudoers.d/{account}"))], - {"step": "sudoers", "equals": "absent"}, - ), - check( - "nodes.account-holds-a-single-authorized-key", - "Exactly one authorized key is installed for the node account", - "nodes", - "stdout_matches", - [ - step( - "keys", - NODE, - *shell("line_count", path=f"{HOST_HOME}/{account}/.ssh/authorized_keys"), - ) - ], - {"step": "keys", "equals": "1"}, - rationale="A count, never the key material. A second key is how a human identity re-enters an account that is meant to be dedicated.", - ), - *[ - check( - f"nodes.{label}-is-root-only", - f"{title} remains unreadable to the node account", - "nodes", - "stdout_matches", - [step("mode", NODE, *shell("path_mode", path=path))], - {"step": "mode", "equals": expected}, - ) - for label, title, path, expected in ( - ( - "k3s-credentials", - "The K3s admin kubeconfig", - f"{HOST_ETC}/rancher/k3s/k3s.yaml", - ["600 root:root", "absent"], - ), - ("k3s-state", "K3s server state", "/host-k3s", ["700 root:root", "absent"]), - ("kubelet-state", "Kubelet state", "/host-kubelet", ["700 root:root", "750 root:root", "absent"]), - ("container-runtime", "The container runtime socket directory", "/host-run-containerd", ["710 root:root", "700 root:root", "absent"]), - ) - ], check( "nodes.hardening-covers-every-node", "The node account audit runs on every node in the cluster", @@ -224,7 +106,11 @@ def _nodes(targets: Targets) -> list[CheckSpec]: NODE_NAME_PROJECTION, ) ], - {"step": "coverage", "minimum": targets.node_count}, + { + "step": "coverage", + "minimum": targets.node_count, + "total_equals": targets.node_count, + }, rationale="A hardened fleet with one unenrolled node is an unhardened fleet.", ), ] @@ -237,7 +123,7 @@ def _build(targets: Targets) -> list[CheckSpec]: "build.builder-service-account-is-tokenless", "The image builder service account mounts no API token", "build", - "json_field", + "stdout_matches", [ step( "serviceaccount", @@ -249,10 +135,10 @@ def _build(targets: Targets) -> list[CheckSpec]: "serviceaccount", targets.builder_serviceaccount, "-o", - "json", + projection("jsonpath={.automountServiceAccountToken}"), ) ], - {"step": "serviceaccount", "fields": {"automountServiceAccountToken": False}}, + {"step": "serviceaccount", "equals": "false"}, ), check( "build.builder-runs-unprivileged-with-exact-capabilities", @@ -276,7 +162,7 @@ def _build(targets: Targets) -> list[CheckSpec]: "build.harbor-immutability-rule-is-applied", "The Harbor immutable-tag rule for the agent image has been applied", "build", - "json_numeric", + "lines_match", [ step( "job", @@ -288,35 +174,145 @@ def _build(targets: Targets) -> list[CheckSpec]: "job", targets.harbor_immutability_job, "-o", - "json", + projection("jsonpath={.status.succeeded}"), ) ], - {"step": "job", "fields": {"status.succeeded": {"min": 1}}}, + {"step": "job", "pattern": r"^[1-9][0-9]*$", "minimum": 1}, rationale="The job verifies the rule it creates, so a completed run is evidence the rule matched the expected contract.", ), - check( - "build.deployed-image-is-digest-pinned", - "The running agent image is pinned by digest, not by a mutable tag", - "build", - "names_present", - [ - step( - "image", - OPERATOR, - "kubectl", - "--namespace", - targets.namespace, - "get", - f"deploy/{targets.agent_deployment}", - "-o", - 'jsonpath={range .spec.template.spec.containers[*]}{.image}{"\\n"}{end}', - ) - ], - {"step": "image", "contains": ("@sha256:",)}, - ), + _release_lineage(targets), ] +def _release_lineage(targets: Targets) -> CheckSpec: + """Bind the exact reviewed release from Git through the Ready pods.""" + container = targets.agent_container + namespace = targets.namespace + deployment = f"deploy/{targets.agent_deployment}" + deployment_projection = projection( + ( + "jsonpath={.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}" + "{.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}" + "{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}" + f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}' + ).replace("'", '"') + ) + pod_projection = projection( + ( + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.status.phase}{'\\t'}" + "{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}" + f'{{.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}' + f'{{.status.containerStatuses[?(@.name=="{container}")].imageID}}{{"\\t"}}' + "{.metadata.labels.pod-template-hash}{'\\n'}{end}" + ).replace("'", '"') + ) + replicaset_projection = projection( + ( + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}" + "{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}" + "{.status.readyReplicas}{'\\t'}{.status.availableReplicas}{'\\t'}" + f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}' + "{.metadata.labels.pod-template-hash}{'\\n'}{end}" + ).replace("'", '"') + ) + flux_projection = projection( + ( + "jsonpath={.metadata.name}{'\\t'}{.metadata.generation}{'\\t'}" + "{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}" + "{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}" + "{.status.lastAppliedRevision}" + ).replace("'", '"') + ) + return check( + "release.exact-lineage-is-running", + "Remote main, reviewed PR, image, build, Flux, and running revision are identical", + "release", + "release_lineage", + [ + step( + "remote-main", + OPERATOR, + "git", + "ls-remote", + targets.remote, + "refs/heads/main", + ), + step( + "reviewed-pr", + OPERATOR, + GITEA_CLIENT, + "GET", + f"/api/v1/repos/{targets.repo}/pulls/{targets.reviewed_pr_number}", + ), + step( + "build-source", + OPERATOR, + "git", + "rev-parse", + f"{targets.remote}/main", + ), + step( + "deployment", + OPERATOR, + "kubectl", + "--namespace", + namespace, + "get", + deployment, + "-o", + deployment_projection, + ), + step( + "pods", + OPERATOR, + "kubectl", + "--namespace", + namespace, + "get", + "pods", + "--selector", + f"app={targets.agent_deployment}", + "-o", + pod_projection, + ), + step( + "replicasets", + OPERATOR, + "kubectl", + "--namespace", + namespace, + "get", + "replicasets", + "--selector", + f"app={targets.agent_deployment}", + "-o", + replicaset_projection, + ), + step( + "flux", + OPERATOR, + "kubectl", + "--namespace", + "flux-system", + "get", + "kustomization/hermes", + "-o", + flux_projection, + ), + ], + { + "main_sha": targets.remote_main_sha, + "head_sha": targets.reviewed_head_sha, + "head_ref": targets.reviewed_head_ref, + "pr_number": targets.reviewed_pr_number, + "image": targets.agent_image, + "build_sha": targets.build_sha, + "deployment_revision": targets.deployment_revision, + }, + rationale="A PASS requires one exact release identity across every source and running object.", + ) + + def _reliability(targets: Targets) -> list[CheckSpec]: return [ check( @@ -370,4 +366,9 @@ def _reliability(targets: Targets) -> list[CheckSpec]: def delivery_checks(targets: Targets) -> list[CheckSpec]: """Return the forge, node, build, and reliability checks.""" - return [*_forge(targets), *_nodes(targets), *_build(targets), *_reliability(targets)] + return [ + *_forge(targets), + *_nodes(targets), + *_build(targets), + *_reliability(targets), + ] diff --git a/scripts/ops/hermes_handoff_checks_platform.py b/scripts/ops/hermes_handoff_checks_platform.py index d1986976..7126036b 100644 --- a/scripts/ops/hermes_handoff_checks_platform.py +++ b/scripts/ops/hermes_handoff_checks_platform.py @@ -26,12 +26,22 @@ from hermes_handoff_catalog import ( step, ) from hermes_handoff_model import CheckSpec -from hermes_handoff_policy import GITEA_CLIENT, shell +from hermes_handoff_policy import GITEA_CLIENT, projection, shell def _baseline(targets: Targets) -> list[CheckSpec]: api = f"/api/v1/repos/{targets.repo}" + dependency_heads = dict(targets.dependency_heads) checks = [ + check( + "baseline.local-origin-main-is-current", + "The local origin/main ref equals the fixed remote-main SHA", + "baseline", + "stdout_matches", + [step("main", OPERATOR, "git", "rev-parse", f"{targets.remote}/main")], + {"step": "main", "equals": targets.remote_main_sha}, + rationale="The ancestry check is meaningful only after the local tracking ref is bound to the remote observation.", + ), check( "baseline.origin-main-descends-merged-work", "origin/main descends from the merged Hermes isolation baseline", @@ -55,7 +65,15 @@ def _baseline(targets: Targets) -> list[CheckSpec]: "Exact open pull-request heads are recorded, not assumed", "baseline", "json_record", - [step("open", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls?state=open&limit=50")], + [ + step( + "open", + OPERATOR, + GITEA_CLIENT, + "GET", + f"{api}/pulls?state=open&limit=50", + ) + ], { "step": "open", "record": { @@ -87,7 +105,14 @@ def _baseline(targets: Targets) -> list[CheckSpec]: "baseline", "json_field", [step("pr", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls/{number}")], - {"step": "pr", "fields": {"merged": True, "base.ref": "main"}}, + { + "step": "pr", + "fields": { + "merged": True, + "base.ref": "main", + "head.sha": dependency_heads.get(number, ""), + }, + }, rationale="This harness certifies the post-merge platform; an unmerged dependency means it is measuring something else.", ) for number in targets.dependency_pull_requests @@ -206,7 +231,7 @@ def _routing(targets: Targets) -> list[CheckSpec]: "routing.switchyard-is-available", "Switchyard has a ready replica serving every model-call boundary", "routing", - "json_numeric", + "lines_match", [ step( "deployment", @@ -217,10 +242,10 @@ def _routing(targets: Targets) -> list[CheckSpec]: "get", switchyard, "-o", - "json", + projection("jsonpath={.status.readyReplicas}"), ) ], - {"step": "deployment", "fields": {"status.readyReplicas": {"min": 1}}}, + {"step": "deployment", "pattern": r"^[1-9][0-9]*$", "minimum": 1}, ), check( "routing.provider-effort-and-fallback-evidence", @@ -244,7 +269,15 @@ def _routing(targets: Targets) -> list[CheckSpec]: "Codex boundary latency is measured, not assumed", "routing", "json_numeric", - [step("health", SELF, *shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS))], + [ + step( + "health", + SELF, + *shell( + "json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS + ), + ) + ], {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, ), check( @@ -252,7 +285,15 @@ def _routing(targets: Targets) -> list[CheckSpec]: "Claude boundary latency is measured, not assumed", "routing", "json_numeric", - [step("health", SELF, *shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS))], + [ + step( + "health", + SELF, + *shell( + "json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS + ), + ) + ], {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, ), ] @@ -261,7 +302,7 @@ def _routing(targets: Targets) -> list[CheckSpec]: def _scope_profile( targets: Targets, name: str, workload: str, container: str, expected: str ) -> CheckSpec: - projection = ( + profile_projection = projection( f'jsonpath={{.spec.template.spec.containers[?(@.name=="{container}")]' '.env[?(@.name=="HERMES_AUTO_ROUTER_PROFILE")].value}' ) @@ -280,7 +321,7 @@ def _scope_profile( "get", workload, "-o", - projection, + profile_projection, ) ], {"step": "profile", "equals": expected}, @@ -291,7 +332,11 @@ def _scope_profile( def _scopes(targets: Targets) -> list[CheckSpec]: return [ _scope_profile( - targets, "agent", f"deploy/{targets.agent_deployment}", targets.agent_container, "agent" + targets, + "agent", + f"deploy/{targets.agent_deployment}", + targets.agent_container, + "agent", ), _scope_profile( targets, @@ -322,7 +367,9 @@ def _scopes(targets: Targets) -> list[CheckSpec]: "get", "ingress", "-o", - 'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}', + projection( + 'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}' + ), ) ], {"step": "hosts", "minimum": 3}, @@ -332,4 +379,9 @@ def _scopes(targets: Targets) -> list[CheckSpec]: def platform_checks(targets: Targets) -> list[CheckSpec]: """Return the baseline, identity, routing, and scope checks.""" - return [*_baseline(targets), *_identity(targets), *_routing(targets), *_scopes(targets)] + return [ + *_baseline(targets), + *_identity(targets), + *_routing(targets), + *_scopes(targets), + ] diff --git a/scripts/ops/hermes_handoff_checks_workers.py b/scripts/ops/hermes_handoff_checks_workers.py index a2e1abea..eba7df6d 100644 --- a/scripts/ops/hermes_handoff_checks_workers.py +++ b/scripts/ops/hermes_handoff_checks_workers.py @@ -17,14 +17,15 @@ from __future__ import annotations from hermes_handoff_catalog import ( CHAT, INIT_NAME_PROJECTION, - NODE_NAME_PROJECTION, OPERATOR, + POOL_PROJECTION, SELF, Targets, check, ) from hermes_handoff_catalog import step as step from hermes_handoff_model import EPHEMERAL, CheckSpec +from hermes_handoff_policy import projection KANBAN_STATUSES = [ "archived", @@ -43,10 +44,10 @@ def _pool(targets: Targets) -> list[CheckSpec]: selector = targets.pool_selector return [ check( - "pool.three-workers-are-ready", - "The distributed execution pool has its full complement of ready workers", + "pool.three-workers-have-distinct-nodes-and-volumes", + "Exactly three Ready ordinal workers occupy distinct nodes and claims", "pool", - "json_numeric", + "pool_topology", [ step( "statefulset", @@ -57,25 +58,14 @@ def _pool(targets: Targets) -> list[CheckSpec]: "get", f"statefulset/{targets.pool_statefulset}", "-o", - "json", - ) - ], - { - "step": "statefulset", - "fields": { - "status.readyReplicas": {"min": targets.pool_replicas}, - "spec.replicas": {"min": targets.pool_replicas}, - }, - }, - ), - check( - "pool.workers-occupy-distinct-nodes", - "Each pool worker runs on its own node", - "pool", - "distinct_count", - [ + projection( + "jsonpath={.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}{.status.currentReplicas}{'\\t'}{.spec.template.spec.automountServiceAccountToken}".replace( + "'", '"' + ) + ), + ), step( - "nodes", + "workers", OPERATOR, "kubectl", "--namespace", @@ -85,61 +75,16 @@ def _pool(targets: Targets) -> list[CheckSpec]: "--selector", selector, "-o", - NODE_NAME_PROJECTION, - ) + POOL_PROJECTION, + ), ], { - "step": "nodes", - "minimum": targets.pool_replicas, - "total_equals": targets.pool_replicas, + "state_step": "statefulset", + "worker_step": "workers", + "replicas": targets.pool_replicas, + "name": targets.pool_statefulset, }, - rationale="Three replicas co-scheduled onto one node satisfy a replica count and defeat the isolation it stands for.", - ), - check( - "pool.workers-own-distinct-volumes", - "Each pool worker has its own durable workspace", - "pool", - "distinct_count", - [ - step( - "claims", - OPERATOR, - "kubectl", - "--namespace", - targets.namespace, - "get", - "persistentvolumeclaims", - "--selector", - selector, - "-o", - "name", - ) - ], - {"step": "claims", "minimum": targets.pool_replicas}, - ), - check( - "pool.workers-are-tokenless", - "Pool workers mount no Kubernetes API token", - "pool", - "json_field", - [ - step( - "statefulset", - OPERATOR, - "kubectl", - "--namespace", - targets.namespace, - "get", - f"statefulset/{targets.pool_statefulset}", - "-o", - "json", - ) - ], - { - "step": "statefulset", - "fields": {"spec.template.spec.automountServiceAccountToken": False}, - }, - rationale="A fenced execution worker needs no cluster authority; holding a token is the failure, not the exception.", + rationale="Replica count, Ready state, ordinal coverage, node placement, PVC isolation, and tokenlessness are one fail-closed assertion.", ), check( "pool.coordinator-retains-sole-state-ownership", @@ -156,7 +101,9 @@ def _pool(targets: Targets) -> list[CheckSpec]: "get", f"statefulset/{targets.pool_statefulset}", "-o", - 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}', + projection( + 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}' + ), ) ], {"step": "claims", "names": targets.coordinator_claims}, @@ -167,13 +114,7 @@ def _pool(targets: Targets) -> list[CheckSpec]: def _pool_env(targets: Targets) -> list[CheckSpec]: - """Return the assignment-safety check only when a site declares what to expect. - - A ``names_present`` assertion over an empty expectation would pass on any - workload at all, which is worse than not asserting: it reads as coverage. - """ - if not targets.pool_worker_env: - return [] + """Return the mandatory assignment-safety configuration check.""" return [ check( "pool.assignment-safety-knobs-are-configured", @@ -190,12 +131,40 @@ def _pool_env(targets: Targets) -> list[CheckSpec]: "get", f"statefulset/{targets.pool_statefulset}", "-o", - 'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}', + projection( + 'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}' + ), ) ], {"step": "env", "names": targets.pool_worker_env}, rationale="Deployed configuration only; the stale-claim and duplicate-dispatch behaviour itself is covered by the pool's own regression suite.", - ) + ), + check( + "pool.assignment-fieldrefs-are-bound", + "Worker node and ordinal identities come from fixed downward-API fields", + "pool", + "names_present", + [ + step( + "source", + OPERATOR, + "git", + "show", + f"{targets.remote}/main:services/hermes/execution-worker-statefulset.yaml", + record=False, + ) + ], + { + "step": "source", + "names": ( + "name: HERMES_WORKER_ORDINAL", + "fieldPath: metadata.labels['apps.kubernetes.io/pod-index']", + "name: HERMES_WORKER_NODE", + "fieldPath: spec.nodeName", + ), + }, + rationale="This prevents caller-authored ordinal/node values from impersonating another worker assignment.", + ), ] @@ -226,7 +195,18 @@ def _surfaces(targets: Targets) -> list[CheckSpec]: "Durable sessions are listed with a preview, a source, and an id", "surfaces", "lines_match", - [step("sessions", SELF, "hermes", "sessions", "list", "--limit", "20", record=False)], + [ + step( + "sessions", + SELF, + "hermes", + "sessions", + "list", + "--limit", + "20", + record=False, + ) + ], {"step": "sessions", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, rationale="A row whose leading column is blank is the session-render regression seen from the data side.", ), @@ -248,7 +228,10 @@ def _surfaces(targets: Targets) -> list[CheckSpec]: INIT_NAME_PROJECTION, ) ], - {"step": "init", "names": ("patch-web-session-activity", "patch-api-server-sessions")}, + { + "step": "init", + "names": ("patch-web-session-activity", "patch-api-server-sessions"), + }, rationale="Blank session rows were a rendering regression; this asserts the patches that fixed it are in the running pod.", ), check( @@ -286,62 +269,64 @@ def _surfaces(targets: Targets) -> list[CheckSpec]: "get", f"deploy/{targets.chat_router_deployment}", "-o", - 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}', + projection( + 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}' + ), ) ], {"step": "claims", "contains": ("state",)}, rationale="Topic selection survives a router restart only if the tenant state file is on a claim rather than in the pod.", ), ] - if targets.expect_telegram_sessions: - checks.append( - check( - "surfaces.chat-telegram-sessions-are-continuous", - "The selected chat tenant still holds durable Telegram-sourced sessions", - "surfaces", - "lines_match", - [ - step( - "telegram", - CHAT, - "hermes", - "sessions", - "list", - "--source", - "telegram", - "--limit", - "20", - record=False, - ) - ], - {"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, - rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.", - ) + checks.append( + check( + "surfaces.chat-telegram-sessions-are-continuous", + "The selected chat tenant still holds durable Telegram-sourced sessions", + "surfaces", + "lines_match", + [ + step( + "telegram", + CHAT, + "hermes", + "sessions", + "list", + "--source", + "telegram", + "--limit", + "20", + record=False, + ) + ], + {"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.", ) - if targets.chat_config_revision: - checks.append( - check( - "surfaces.chat-runs-the-telegram-topic-revision", - "The chat surface runs the durable Telegram topic configuration", - "surfaces", - "stdout_matches", - [ - step( - "revision", - OPERATOR, - "kubectl", - "--namespace", - targets.namespace, - "get", - f"statefulset/{targets.chat_statefulset}", - "-o", - "jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}", - ) - ], - {"step": "revision", "equals": targets.chat_config_revision}, - rationale="Telegram topic continuity is carried by the config revision the pod template pins.", - ) + ) + checks.append( + check( + "surfaces.chat-runs-the-telegram-topic-revision", + "The chat surface runs the durable Telegram topic configuration", + "surfaces", + "stdout_matches", + [ + step( + "revision", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.chat_statefulset}", + "-o", + projection( + "jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}" + ), + ) + ], + {"step": "revision", "equals": targets.chat_config_revision}, + rationale="Telegram topic continuity is carried by the config revision the pod template pins.", ) + ) return checks diff --git a/scripts/ops/hermes_handoff_ephemeral.py b/scripts/ops/hermes_handoff_ephemeral.py index 5456482b..fdcfdbad 100644 --- a/scripts/ops/hermes_handoff_ephemeral.py +++ b/scripts/ops/hermes_handoff_ephemeral.py @@ -1,34 +1,35 @@ #!/usr/bin/env python3 -"""The separately armed, self-cleaning mutation mode of the acceptance harness. - -Proving that a worker can still open a draft pull request means actually opening -one, which is the only part of this harness that writes anything. It is -therefore a distinct mode with a distinct policy: an explicit confirmation -phrase, a caller-supplied unique ref, a preflight that refuses a protected -target before any network call, and a cleanup whose *verification* is itself a -mandatory check. - -The push target is guarded, not the pull-request base. Opening a pull request -against `main` writes nothing to `main`; pushing to it does. The refusal is -checked by name and by pattern, and the arming self-test exercises it on every -armed run rather than trusting that it was checked once. -""" +"""Strictly preflighted, discoverable, and self-cleaning armed mode.""" from __future__ import annotations +import configparser import re from collections.abc import Sequence from dataclasses import dataclass +from pathlib import Path +from typing import Any from hermes_handoff_exec import Outcome, Runner, Vantage from hermes_handoff_model import EPHEMERAL, FAIL, NOT_RUN, PASS, CheckResult, CheckSpec -from hermes_handoff_policy import GITEA_CLIENT +from hermes_handoff_policy import ( + EXPECTED_BASE, + EXPECTED_REMOTE, + EXPECTED_REPO, + GITEA_CLIENT, + arm_ephemeral_policy, + register_ephemeral_pull, +) +from hermes_handoff_rules import strict_json CONFIRMATION = "ARM EPHEMERAL HERMES HANDOFF PUSH" REF_PREFIX = "ephemeral/hermes-handoff-acceptance" REF_RE = re.compile(rf"{REF_PREFIX}/[a-z0-9][a-z0-9-]{{7,63}}\Z") TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9-]{7,63}\Z") +SHA_RE = re.compile(r"[0-9a-f]{40}\Z") DRAFT_TITLE_PREFIX = "WIP: " +EXPECTED_HEAD_REF = "feature/hermes-full-handoff-acceptance" +EXPECTED_REMOTE_URL = "https://scm.bstein.dev/atlas/titan-iac.git" PROTECTED_NAMES = frozenset( { "default", @@ -51,48 +52,113 @@ class ArmingError(ValueError): @dataclass(frozen=True) class ArmRequest: - """Everything the armed mode needs, and nothing it can infer for itself.""" + """Every fixed and caller-confirmed input to one armed run.""" repo: str remote: str token: str confirmation: str - base: str = "main" + base: str = EXPECTED_BASE + expected_head: str = "" + expected_base_sha: str = "" @property def ref(self) -> str: - """Return the exact, unique branch this run is allowed to create.""" return f"{REF_PREFIX}/{self.token}" def normalise_branch(name: str) -> str: """Return a branch name stripped of ref decoration, for comparison.""" - trimmed = name.strip().removeprefix("refs/heads/").strip("/") - return trimmed.lower() + return name.strip().removeprefix("refs/heads/").strip("/").lower() def assert_push_target_allowed(ref: str) -> None: - """Raise unless ``ref`` is an ephemeral branch this harness may create.""" + """Raise unless ``ref`` is the one ephemeral branch grammar.""" if normalise_branch(ref) in PROTECTED_NAMES: raise ArmingError(f"refusing to push to protected branch {ref!r}") if not REF_RE.fullmatch(ref): raise ArmingError(f"push target {ref!r} is not an ephemeral acceptance ref") -def preflight(request: ArmRequest, expected_repo: str) -> None: - """Validate an arming request before any network call is made.""" +def _bounded_read(path: Path, limit: int = 1024 * 1024) -> str: + data = path.read_bytes() + if not data or len(data) > limit: + raise ArmingError(f"local Git metadata is empty or exceeds {limit} bytes") + return data.decode("utf-8", "strict") + + +def _attest_worktree(root: Path, expected_head: str, expected_base_sha: str) -> None: + """Bind cwd, common Git metadata, branch, remote, and exact local HEAD.""" + resolved = root.resolve(strict=True) + marker = resolved / ".git" + if not marker.is_file(): + raise ArmingError( + "armed mode must run at the root of the existing linked worktree" + ) + line = _bounded_read(marker, 4096).strip() + if not line.startswith("gitdir: "): + raise ArmingError("linked-worktree Git metadata is malformed") + gitdir = Path(line.removeprefix("gitdir: ")).resolve(strict=True) + common = (gitdir / _bounded_read(gitdir / "commondir", 4096).strip()).resolve( + strict=True + ) + head = _bounded_read(gitdir / "HEAD", 4096).strip() + expected_ref = f"refs/heads/{EXPECTED_HEAD_REF}" + if head != f"ref: {expected_ref}": + raise ArmingError("armed mode must run from the existing PR #19 feature branch") + ref_path = common / expected_ref + actual_head = _bounded_read(ref_path, 4096).strip() if ref_path.is_file() else "" + if actual_head != expected_head: + raise ArmingError( + "local worktree HEAD does not match the exact reviewed PR head" + ) + remote_ref = common / "refs" / "remotes" / EXPECTED_REMOTE / EXPECTED_BASE + remote_main = ( + _bounded_read(remote_ref, 4096).strip() if remote_ref.is_file() else "" + ) + if remote_main != expected_base_sha: + raise ArmingError("local origin/main does not match the exact release base SHA") + parser = configparser.RawConfigParser(interpolation=None) + try: + parser.read_string(_bounded_read(common / "config")) + remote_url = parser.get(f'remote "{EXPECTED_REMOTE}"', "url") + except (configparser.Error, KeyError) as exc: + raise ArmingError("local Git remote metadata is malformed") from exc + if remote_url != EXPECTED_REMOTE_URL: + raise ArmingError("origin is not the fixed Atlas titan-iac HTTPS repository") + + +def preflight( + request: ArmRequest, expected_repo: str, worktree: Path | None = None +) -> None: + """Validate arming and local provenance before any subprocess/network call.""" if request.confirmation != CONFIRMATION: raise ArmingError("confirmation phrase does not match") - if request.repo != expected_repo: - raise ArmingError(f"repository {request.repo!r} is not the expected {expected_repo!r}") + if expected_repo != EXPECTED_REPO or request.repo != EXPECTED_REPO: + raise ArmingError( + "armed repository is not the fixed Atlas titan-iac repository" + ) + if ( + request.remote != EXPECTED_REMOTE + or normalise_branch(request.base) != EXPECTED_BASE + ): + raise ArmingError("armed remote and pull-request base are fixed to origin/main") if not TOKEN_RE.fullmatch(request.token): - raise ArmingError("ephemeral token must be 8-64 lowercase alphanumeric or dash characters") + raise ArmingError( + "ephemeral token must be 8-64 lowercase alphanumeric or dash characters" + ) + if not SHA_RE.fullmatch(request.expected_head) or not SHA_RE.fullmatch( + request.expected_base_sha + ): + raise ArmingError("exact reviewed head and release base SHAs are required") assert_push_target_allowed(request.ref) - if not normalise_branch(request.base): - raise ArmingError("a pull-request base branch is required") + _attest_worktree( + worktree or Path.cwd(), request.expected_head, request.expected_base_sha + ) + arm_ephemeral_policy(request.ref) -def _spec(identifier: str, title: str, rationale: str = "") -> CheckSpec: +def _spec(identifier: str, title: str) -> CheckSpec: return CheckSpec( id=f"ephemeral.{identifier}", title=title, @@ -101,7 +167,6 @@ def _spec(identifier: str, title: str, rationale: str = "") -> CheckSpec: steps=(), mandatory=True, scope=EPHEMERAL, - rationale=rationale, ) @@ -111,11 +176,10 @@ def _result( status: str, reason: str, outcomes: Sequence[Outcome] = (), - evidence: dict | None = None, - rationale: str = "", + evidence: dict[str, Any] | None = None, ) -> CheckResult: return CheckResult( - spec=_spec(identifier, title, rationale), + spec=_spec(identifier, title), status=status, reason=reason, evidence=evidence or {}, @@ -124,97 +188,271 @@ def _result( def _protected_refusal_result() -> CheckResult: - """Exercise the protected-branch guard on this run, not on a past one.""" - refused: list[str] = [] accepted: list[str] = [] for candidate in ("main", "master", "refs/heads/main", "HEAD", f"{REF_PREFIX}/x"): try: assert_push_target_allowed(candidate) except ArmingError: - refused.append(candidate) - else: - accepted.append(candidate) - evidence = {"refused": refused, "accepted": accepted} - if accepted: - return _result( - "protected-branch-refusal", - "Arming against a protected branch is refused before any network call", - FAIL, - f"the guard accepted {accepted}", - evidence=evidence, - ) + continue + accepted.append(candidate) return _result( "protected-branch-refusal", "Arming against a protected branch is refused before any network call", - PASS, - "every protected and malformed push target was refused", - evidence=evidence, + FAIL if accepted else PASS, + f"guard accepted {accepted}" + if accepted + else "every protected and malformed target was refused", + evidence={"accepted": accepted}, ) def _detail(outcome: Outcome) -> str: - """Return the most informative bounded explanation an outcome carries.""" - return outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}" + return ( + outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}" + ) -def _pull_request_number(outcome: Outcome) -> int | None: - match = re.search(r'"number"\s*:\s*(\d+)', outcome.stdout) - return int(match.group(1)) if match else None +def _pr_problems(payload: Any, request: ArmRequest) -> list[str]: + if not isinstance(payload, dict): + return ["pull request is not an object"] + wanted = { + "state": "open", + "draft": True, + "merged": False, + "base": {"ref": EXPECTED_BASE, "sha": request.expected_base_sha}, + "head": {"ref": request.ref, "sha": request.expected_head}, + } + problems: list[str] = [] + if ( + not isinstance(payload.get("number"), int) + or isinstance(payload.get("number"), bool) + or payload["number"] <= 0 + ): + problems.append("number is malformed") + for field in ("state", "draft", "merged"): + if ( + type(payload.get(field)) is not type(wanted[field]) + or payload.get(field) != wanted[field] + ): + problems.append(f"{field} does not match") + for group, values in (("base", wanted["base"]), ("head", wanted["head"])): + observed = payload.get(group) + if not isinstance(observed, dict): + problems.append(f"{group} is malformed") + continue + for field, value in values.items(): + if observed.get(field) != value: + problems.append(f"{group}.{field} does not match") + return problems -def run_armed(runner: Runner, vantage: Vantage, request: ArmRequest) -> list[CheckResult]: - """Push a unique ephemeral ref, open a draft pull request, and remove both. +def _discover(outcome: Outcome, request: ArmRequest) -> tuple[list[int], str]: + if not outcome.ok: + return ([], f"pull-request discovery failed: {_detail(outcome)}") + try: + payload = strict_json(outcome.stdout) + except ValueError as exc: + return ([], f"pull-request discovery returned malformed JSON: {exc}") + if not isinstance(payload, list): + return ([], "pull-request discovery did not return an array") + matches: list[int] = [] + for index, item in enumerate(payload): + if ( + not isinstance(item, dict) + or not isinstance(item.get("head"), dict) + or not isinstance(item.get("base"), dict) + ): + return ([], f"pull-request discovery row {index + 1} is malformed") + if item["head"].get("ref") != request.ref: + continue + problems = _pr_problems(item, request) + if problems: + return ([], f"discovered pull request is invalid: {', '.join(problems)}") + matches.append(item["number"]) + if len(matches) != 1: + return ( + matches, + f"expected one exact draft pull request, discovered {len(matches)}", + ) + return (matches, "") - Every step is checked and every artifact is removed. Cleanup runs even when - an earlier step failed, and the cleanup verification is reported as its own - mandatory check so a half-removed branch cannot pass as success. - """ - api = f"/api/v1/repos/{request.repo}" - results = [_protected_refusal_result()] + +def _closed_pr_problems(payload: Any, request: ArmRequest) -> list[str]: + """Validate the same exact PR identity with state changed to closed.""" + problems = _pr_problems(payload, request) + if isinstance(payload, dict) and payload.get("state") == "closed": + problems = [ + problem for problem in problems if problem != "state does not match" + ] + elif "state does not match" not in problems: + problems.append("state is not closed") + return problems + + +def _cleanup( + runner: Runner, + vantage: Vantage, + request: ArmRequest, + api: str, + numbers: Sequence[int], + discovery_error: str = "", +) -> CheckResult: outcomes: list[Outcome] = [] + problems = [discovery_error] if discovery_error else [] + for number in sorted(set(numbers)): + closed = runner.run( + (GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"), + vantage, + ) + outcomes.append(closed) + if not closed.ok: + problems.append(f"pull request #{number} close failed") + continue + try: + payload = strict_json(closed.stdout) + except ValueError: + problems.append(f"pull request #{number} close response is malformed") + continue + expected = _closed_pr_problems(payload, request) + if expected: + problems.append(f"pull request #{number} close response does not match") + deleted = runner.run( + (GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage + ) + outcomes.append(deleted) + if not deleted.ok: + problems.append("ephemeral branch deletion failed") + remaining = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage + ) + outcomes.append(remaining) + if not remaining.ok or remaining.stdout.strip(): + problems.append("ephemeral branch absence could not be verified") + for number in sorted(set(numbers)): + observed = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage) + outcomes.append(observed) + if not observed.ok: + problems.append(f"pull request #{number} closure could not be verified") + continue + try: + payload = strict_json(observed.stdout) + except ValueError: + problems.append(f"pull request #{number} verification is malformed") + continue + expected = _closed_pr_problems(payload, request) + if expected: + problems.append(f"pull request #{number} is not exactly closed") + return _result( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + FAIL if problems else PASS, + "; ".join(problems) + if problems + else "the exact branch and draft pull request are gone", + outcomes, + {"ref": request.ref, "pull_requests": sorted(set(numbers))}, + ) - existing = runner.run(("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage) - outcomes.append(existing) - if not existing.ok: - results.append( + +def run_armed( + runner: Runner, vantage: Vantage, request: ArmRequest +) -> list[CheckResult]: + """Create, discover, validate, and clean one exact ephemeral branch/PR.""" + guard = _protected_refusal_result() + if guard.status != PASS: + return [ + guard, _result( "feature-branch-push", "A unique ephemeral feature branch can be pushed and removed", NOT_RUN, - f"could not confirm the ref is unused: {_detail(existing)}", - outcomes, - ) + "protected-ref guard failed", + ), + _result( + "draft-pull-request", + "A draft pull request can be opened and closed", + NOT_RUN, + "protected-ref guard failed", + ), + _result( + "cleanup-verified", + "Every ephemeral artifact is removed", + NOT_RUN, + "nothing was created", + ), + ] + api = f"/api/v1/repos/{request.repo}" + reserve = min(6000.0, max(300.0, runner.command_timeout * 10)) + existing = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), + vantage, + reserve_seconds=reserve, + ) + if not existing.ok or existing.stdout.strip(): + reason = ( + _detail(existing) if not existing.ok else "the ephemeral ref already exists" ) - return results - if existing.stdout.strip(): - results.append( + return [ + guard, _result( "feature-branch-push", "A unique ephemeral feature branch can be pushed and removed", FAIL, - f"{request.ref} already exists on {request.remote}; pick a fresh token", - outcomes, - ) - ) - return results - - pushed = runner.run( - ("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"), vantage + reason, + [existing], + ), + _result( + "draft-pull-request", + "A draft pull request can be opened and closed", + NOT_RUN, + "branch preflight failed", + ), + _result( + "cleanup-verified", + "Every ephemeral artifact is removed", + NOT_RUN, + "nothing was created", + ), + ] + local_head = runner.run( + ("git", "rev-parse", "HEAD"), vantage, reserve_seconds=reserve ) - outcomes.append(pushed) - results.append( - _result( - "feature-branch-push", - "A unique ephemeral feature branch can be pushed and removed", - PASS if pushed.ok else FAIL, - f"pushed {request.ref}" if pushed.ok else f"push failed: {_detail(pushed)}", - [pushed], - {"ref": request.ref}, + pushed = ( + runner.run( + ("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"), + vantage, + reserve_seconds=reserve, + ) + if local_head.ok and local_head.stdout.strip() == request.expected_head + else Outcome( + argv=("git", "push"), + vantage=vantage.name, + error="local HEAD changed after static preflight", ) ) - - number: int | None = None - if pushed.ok: + verified = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), + vantage, + reserve_seconds=reserve, + ) + branch_ok = verified.ok and verified.stdout.split() == [ + request.expected_head, + f"refs/heads/{request.ref}", + ] + branch_result = _result( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + PASS if branch_ok else FAIL, + "the exact reviewed HEAD was pushed" + if branch_ok + else "push could not be verified exactly", + [local_head, pushed, verified], + {"ref": request.ref, "head": request.expected_head}, + ) + numbers: list[int] = [] + discovery_error = "" + create_outcomes: list[Outcome] = [] + if branch_ok: created = runner.run( ( GITEA_CLIENT, @@ -225,89 +463,33 @@ def run_armed(runner: Runner, vantage: Vantage, request: ArmRequest) -> list[Che "--field", f"head={request.ref}", "--field", - f"base={normalise_branch(request.base)}", + f"base={EXPECTED_BASE}", "--field", "body=Ephemeral acceptance probe. Closed and deleted by the harness.", ), vantage, + reserve_seconds=reserve, ) - outcomes.append(created) - number = _pull_request_number(created) if created.ok else None - results.append( - _result( - "draft-pull-request", - "A draft pull request can be opened and closed against the ephemeral branch", - PASS if created.ok and number else FAIL, - f"opened draft pull request #{number}" - if created.ok and number - else f"draft creation failed: {_detail(created)}", - [created], - {"pull_request": number}, - ) + discovered = runner.run( + (GITEA_CLIENT, "GET", f"{api}/pulls?state=all&limit=50"), + vantage, + reserve_seconds=reserve, ) + create_outcomes = [created, discovered] + numbers, discovery_error = _discover(discovered, request) + for number in numbers: + register_ephemeral_pull(number) else: - results.append( - _result( - "draft-pull-request", - "A draft pull request can be opened and closed against the ephemeral branch", - NOT_RUN, - "the ephemeral branch was never pushed", - ) - ) - - results.append(_cleanup(runner, vantage, request, api, number)) - return results - - -def _cleanup( - runner: Runner, vantage: Vantage, request: ArmRequest, api: str, number: int | None -) -> CheckResult: - """Close the pull request, delete the ref, and prove both are gone.""" - outcomes: list[Outcome] = [] - problems: list[str] = [] - - if number is not None: - closed = runner.run( - (GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"), vantage - ) - outcomes.append(closed) - if not closed.ok: - problems.append(f"pull request #{number} could not be closed") - - deleted = runner.run((GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage) - outcomes.append(deleted) - - remaining = runner.run( - ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage - ) - outcomes.append(remaining) - if not remaining.ran: - problems.append(f"could not verify removal: {_detail(remaining)}") - elif remaining.stdout.strip(): - problems.append(f"{request.ref} still exists on {request.remote}") - - if number is not None: - state = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage) - outcomes.append(state) - if not state.ok: - problems.append(f"could not confirm pull request #{number} is closed") - elif '"state":"open"' in state.stdout.replace(" ", ""): - problems.append(f"pull request #{number} is still open") - - if problems: - return _result( - "cleanup-verified", - "Every ephemeral ref and pull request is removed and the removal is verified", - FAIL, - "; ".join(problems), - outcomes, - {"ref": request.ref, "pull_request": number}, - ) - return _result( - "cleanup-verified", - "Every ephemeral ref and pull request is removed and the removal is verified", - PASS, - "the ephemeral ref and pull request are gone and their absence was re-checked", - outcomes, - {"ref": request.ref, "pull_request": number}, + discovery_error = "branch creation was not verified" + draft_result = _result( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + PASS if len(numbers) == 1 and not discovery_error else FAIL, + f"discovered exact draft pull request #{numbers[0]}" + if len(numbers) == 1 and not discovery_error + else discovery_error, + create_outcomes, + {"pull_requests": numbers}, ) + cleanup = _cleanup(runner, vantage, request, api, numbers, discovery_error) + return [guard, branch_result, draft_result, cleanup] diff --git a/scripts/ops/hermes_handoff_evaluators.py b/scripts/ops/hermes_handoff_evaluators.py index 72e3558c..61e9e06b 100644 --- a/scripts/ops/hermes_handoff_evaluators.py +++ b/scripts/ops/hermes_handoff_evaluators.py @@ -22,12 +22,15 @@ from hermes_handoff_model import ATTEMPT, FAIL, NOT_RUN, PASS, REVIEW, CheckSpec from hermes_handoff_rules import ( EVALUATORS, Evaluation, + dotted, evaluate, evaluator, + exact_equal, lines, looks_denied, missing_steps, step_keys, + strict_json, ) __all__ = ["EVALUATORS", "Evaluation", "evaluate", "evaluator"] @@ -41,19 +44,25 @@ def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation return blocked attempts = step_keys(spec, ATTEMPT) if not attempts: - return Evaluation(NOT_RUN, "deny checks require a real attempt, not only a review") + return Evaluation( + NOT_RUN, "deny checks require a real attempt, not only a review" + ) evidence: dict[str, Any] = {} for key in step_keys(spec, REVIEW): verdict = outcomes[key].stdout.strip().lower() evidence[key] = verdict if verdict != "no": return Evaluation( - FAIL, f"authorization review {key} reported {verdict or 'nothing'}", evidence + FAIL, + f"authorization review {key} reported {verdict or 'nothing'}", + evidence, ) for key in attempts: outcome = outcomes[key] if outcome.ok: - return Evaluation(FAIL, f"attempt {key} succeeded but must be refused", evidence) + return Evaluation( + FAIL, f"attempt {key} succeeded but must be refused", evidence + ) if not looks_denied(outcome): return Evaluation( NOT_RUN, @@ -64,6 +73,28 @@ def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation return Evaluation(PASS, "review and live attempt were both refused", evidence) +@evaluator("review_denied") +def evaluate_review_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass only when every side-effect-free authorization review says no.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + reviews = step_keys(spec, REVIEW) + if not reviews: + return Evaluation(NOT_RUN, "a denial check requires an authorization review") + evidence: dict[str, str] = {} + for key in reviews: + verdict = outcomes[key].stdout.strip().lower() + evidence[key] = verdict + if verdict != "no": + return Evaluation( + FAIL, + f"authorization review {key} reported {verdict or 'nothing'}", + evidence, + ) + return Evaluation(PASS, "every authorization review was denied", evidence) + + @evaluator("allowed") def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: """Pass when every required step succeeded and every review said yes.""" @@ -84,7 +115,9 @@ def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluatio ) continue if not outcome.ok: - return Evaluation(FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence) + return Evaluation( + FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence + ) evidence[step.key] = "succeeded" return Evaluation(PASS, "required access is present", evidence) @@ -93,7 +126,9 @@ def _matched(observed: set[str], patterns: tuple[str, ...]) -> set[str]: return {name for pattern in patterns for name in observed if pattern in name} -def _line_step(spec: CheckSpec, outcomes: dict[str, Outcome]) -> tuple[set[str], str] | Evaluation: +def _line_step( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> tuple[set[str], str] | Evaluation: key = spec.expect["step"] outcome = outcomes[key] if not outcome.ok: @@ -117,7 +152,9 @@ def evaluate_names_absent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Eval ) evidence = {"observed_count": len(observed), "offenders": offenders} if offenders: - return Evaluation(FAIL, f"forbidden names present: {', '.join(offenders)}", evidence) + return Evaluation( + FAIL, f"forbidden names present: {', '.join(offenders)}", evidence + ) return Evaluation(PASS, "no forbidden names are present", evidence) @@ -139,14 +176,18 @@ def evaluate_names_present(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Eva ) evidence = {"observed_count": len(observed), "missing": missing} if missing: - return Evaluation(FAIL, f"required names absent: {', '.join(missing)}", evidence) + return Evaluation( + FAIL, f"required names absent: {', '.join(missing)}", evidence + ) if not observed: return Evaluation(NOT_RUN, "step produced no lines to examine", evidence) return Evaluation(PASS, "every required name is present", evidence) @evaluator("stdout_matches") -def evaluate_stdout_matches(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: +def evaluate_stdout_matches( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: """Compare a step's trimmed stdout against an expected value or set.""" blocked = missing_steps(spec, outcomes) if blocked: @@ -160,12 +201,16 @@ def evaluate_stdout_matches(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Ev allowed = expected if isinstance(expected, list) else [expected] evidence = {"observed": actual} if actual not in allowed: - return Evaluation(FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence) + return Evaluation( + FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence + ) return Evaluation(PASS, "output matches the expected value", evidence) @evaluator("distinct_count") -def evaluate_distinct_count(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: +def evaluate_distinct_count( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: """Pass when a step yields enough distinct rows — and, optionally, exactly N. Distinctness is the point for placement checks: three worker pods pinned to @@ -182,16 +227,24 @@ def evaluate_distinct_count(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Ev values = lines(outcome) distinct = sorted(set(values)) minimum = spec.expect["minimum"] - evidence = {"total": len(values), "distinct": len(distinct), "values": distinct[:16]} + evidence = { + "total": len(values), + "distinct": len(distinct), + "values": distinct[:16], + } if not values: return Evaluation(NOT_RUN, f"step {key} produced no rows to count", evidence) if len(distinct) < minimum: return Evaluation( - FAIL, f"{len(distinct)} distinct values, expected at least {minimum}", evidence + FAIL, + f"{len(distinct)} distinct values, expected at least {minimum}", + evidence, ) total_equals = spec.expect.get("total_equals") if total_equals is not None and len(values) != total_equals: - return Evaluation(FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence) + return Evaluation( + FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence + ) return Evaluation(PASS, f"{len(distinct)} distinct values observed", evidence) @@ -217,16 +270,22 @@ def evaluate_lines_match(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evalu evidence = {"rows": len(rows), "offenders": offenders[:8]} if len(rows) < spec.expect.get("minimum", 1): return Evaluation( - NOT_RUN, f"step {key} produced {len(rows)} data rows, expected at least " - f"{spec.expect.get('minimum', 1)}", evidence + NOT_RUN, + f"step {key} produced {len(rows)} data rows, expected at least " + f"{spec.expect.get('minimum', 1)}", + evidence, ) if offenders: - return Evaluation(FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence) + return Evaluation( + FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence + ) return Evaluation(PASS, f"all {len(rows)} rows match the expected shape", evidence) @evaluator("vantages_agree") -def evaluate_vantages_agree(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: +def evaluate_vantages_agree( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: """Pass only when two independent observations report the same fact. A disagreement is a finding in its own right: it usually means the manifest @@ -245,9 +304,13 @@ def evaluate_vantages_agree(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Ev observations[key] = sorted(lines(outcome)) evidence = {key: value[:16] for key, value in observations.items()} if observations[left_key] != observations[right_key]: - return Evaluation(FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence) + return Evaluation( + FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence + ) if not observations[left_key]: - return Evaluation(NOT_RUN, "both vantages returned nothing to compare", evidence) + return Evaluation( + NOT_RUN, "both vantages returned nothing to compare", evidence + ) return Evaluation(PASS, "both vantages report the same state", evidence) @@ -261,6 +324,172 @@ def evaluate_not_armed(spec: CheckSpec, _outcomes: dict[str, Outcome]) -> Evalua ) +# This validator is one compact, linear state machine so every exit observes +# only fully validated state. +# fmt: off +@evaluator("pool_topology") +def evaluate_pool_topology(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Require exactly three Ready ordinal workers on distinct nodes and claims.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + state = outcomes[spec.expect["state_step"]].stdout.strip().split("\t") + if len(state) != 4: + return Evaluation(NOT_RUN, "StatefulSet projection is malformed") + expected = spec.expect.get("replicas", 3) + try: + replicas, ready, current = (int(value) for value in state[:3]) + except ValueError: + return Evaluation(NOT_RUN, "StatefulSet replica fields are malformed") + evidence: dict[str, Any] = {"replicas": replicas, "ready": ready, "current": current, + "automountServiceAccountToken": state[3]} + if (replicas, ready, current, state[3].lower()) != (expected, expected, expected, "false"): + return Evaluation(FAIL, "worker replica or tokenless state does not match the contract", evidence) + workers: dict[int, tuple[str, str]] = {} + names: list[str] = [] + for number, row in enumerate(lines(outcomes[spec.expect["worker_step"]]), 1): + columns = row.split("\t") + if len(columns) != 5: + return Evaluation(NOT_RUN, f"worker row {number} is malformed", evidence) + name, node, phase, ready_state, claims = columns + match = re.fullmatch(rf"{re.escape(spec.expect['name'])}-(\d+)", name) + claim_list = [claim for claim in claims.split(",") if claim] + if (not match or not node or phase != "Running" or ready_state != "True" + or len(claim_list) != 1): + return Evaluation(FAIL, f"worker row {number} is not a Ready isolated worker", evidence) + ordinal = int(match.group(1)) + if ordinal in workers: + return Evaluation(FAIL, f"worker ordinal {ordinal} is duplicated", evidence) + workers[ordinal] = (node, claim_list[0]) + names.append(name) + evidence.update({"workers": sorted(names), "nodes": sorted({v[0] for v in workers.values()}), + "claims": sorted({v[1] for v in workers.values()})}) + if set(workers) != set(range(expected)): + return Evaluation(FAIL, f"worker ordinals are not exactly 0..{expected - 1}", evidence) + if (len({v[0] for v in workers.values()}) != expected + or len({v[1] for v in workers.values()}) != expected): + return Evaluation(FAIL, "workers do not occupy distinct nodes and claims", evidence) + return Evaluation(PASS, "three Ready workers have distinct nodes, ordinals, and claims", evidence) +# fmt: on + + +_SHA_RE = re.compile(r"[0-9a-f]{40}\Z") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}\Z") +_IMAGE_RE = re.compile( + r"[^@\s]+:git-([0-9a-f]{40})-build-([1-9][0-9]*)@(sha256:[0-9a-f]{64})\Z" +) + + +def _strict_object(outcome: Outcome, label: str) -> dict[str, Any] | Evaluation: + try: + payload = strict_json(outcome.stdout) + except ValueError as exc: + return Evaluation(NOT_RUN, f"{label} JSON is malformed: {exc}") + if not isinstance(payload, dict): + return Evaluation(NOT_RUN, f"{label} JSON is not an object") + return payload + + +# fmt: off +@evaluator("release_lineage") +def evaluate_release_lineage(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Atomically bind Git, PR, image, build, Flux, and running revision evidence.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + expected = spec.expect + main_sha = expected["main_sha"] + head_sha = expected["head_sha"] + image = expected["image"] + build_sha = expected["build_sha"] + revision = str(expected["deployment_revision"]) + digest = image.rsplit("@", 1)[-1] + image_match = _IMAGE_RE.fullmatch(image) + if not (_SHA_RE.fullmatch(main_sha) and _SHA_RE.fullmatch(head_sha) + and _SHA_RE.fullmatch(build_sha) and _DIGEST_RE.fullmatch(digest) + and image_match and image_match.group(1) == build_sha and build_sha == main_sha): + return Evaluation(NOT_RUN, "release lineage expectations are malformed") + remote = outcomes["remote-main"].stdout.strip().split() + if remote != [main_sha, "refs/heads/main"]: + return Evaluation(FAIL, "remote main does not match the fixed expected SHA", + {"remote_main": remote[:2]}) + pr = _strict_object(outcomes["reviewed-pr"], "reviewed PR") + if isinstance(pr, Evaluation): + return pr + fields = { + "number": expected["pr_number"], + "state": "open", + "draft": True, + "merged": False, + "base.ref": "main", + "base.sha": main_sha, + "head.ref": expected["head_ref"], + "head.sha": head_sha, + } + for path, wanted in fields.items(): + try: + actual = dotted(pr, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"reviewed PR is missing {path}") + if not exact_equal(actual, wanted): + return Evaluation(FAIL, f"reviewed PR {path} does not match", {path: actual}) + if outcomes["build-source"].stdout.strip() != build_sha: + return Evaluation(FAIL, "build source SHA does not match the fixed release build SHA") + deployment = outcomes["deployment"].stdout.strip().split("\t") + if len(deployment) != 6: + return Evaluation(NOT_RUN, "Deployment projection is malformed") + generation, observed, replicas, ready, deployed_revision, deployed_image = deployment + try: + current = (int(generation) > 0 and generation == observed + and int(replicas) > 0 and replicas == ready) + except ValueError: + return Evaluation(NOT_RUN, "Deployment numeric fields are malformed") + if not current or (deployed_revision, deployed_image) != (revision, image): + return Evaluation(FAIL, "Deployment lineage does not match the release contract") + replica_hash = "" + for number, row in enumerate(lines(outcomes["replicasets"]), 1): + columns = row.split("\t") + if len(columns) != 6: + return Evaluation(NOT_RUN, f"ReplicaSet row {number} is malformed") + _, rs_revision, rs_ready, rs_available, rs_image, rs_hash = columns + if rs_revision != revision: + continue + if (rs_ready != replicas or rs_available != replicas or rs_image != image or not rs_hash): + return Evaluation(FAIL, "active ReplicaSet does not match the Deployment lineage") + if replica_hash: + return Evaluation(FAIL, "more than one ReplicaSet claims the running revision") + replica_hash = rs_hash + if not replica_hash: + return Evaluation(FAIL, "running Deployment revision has no matching ReplicaSet") + pod_rows = lines(outcomes["pods"]) + if len(pod_rows) != int(replicas): + return Evaluation(FAIL, "running pod count does not match the Deployment") + pods: list[str] = [] + for number, row in enumerate(pod_rows, 1): + columns = row.split("\t") + if len(columns) != 6: + return Evaluation(NOT_RUN, f"pod row {number} is malformed") + name, phase, ready_state, pod_image, image_id, pod_hash = columns + if phase != "Running" or ready_state != "True": + return Evaluation(FAIL, f"pod {name or number} is not Ready") + if (pod_image, pod_hash) != (image, replica_hash) or not image_id.endswith(f"@{digest}"): + return Evaluation(FAIL, f"pod {name or number} lineage does not match") + pods.append(name) + flux = outcomes["flux"].stdout.strip().split("\t") + if len(flux) != 6: + return Evaluation(NOT_RUN, "Flux lineage projection is malformed") + name, generation, observed, suspended, ready_state, applied = flux + if (not name or generation != observed or suspended.lower() == "true" or ready_state != "True"): + return Evaluation(FAIL, "Flux source is not current and Ready") + match = re.search(r"sha1:([0-9a-f]{40})\Z", applied) + if not match or match.group(1) != main_sha: + return Evaluation(FAIL, "Flux applied revision is not the expected remote main") + return Evaluation(PASS, "Git, PR, image, build, Flux, and running revision are bound", + {"main_sha": main_sha, "head_sha": head_sha, "image_digest": digest, + "build_sha": build_sha, "deployment_revision": revision, "pods": pods}) +# fmt: on + + # Two observations of the same fact agree whether or not they came from # different vantages; the dual-vantage case is just the important one. EVALUATORS["steps_agree"] = evaluate_vantages_agree diff --git a/scripts/ops/hermes_handoff_exec.py b/scripts/ops/hermes_handoff_exec.py index 200d82e0..08888b13 100644 --- a/scripts/ops/hermes_handoff_exec.py +++ b/scripts/ops/hermes_handoff_exec.py @@ -1,57 +1,100 @@ #!/usr/bin/env python3 -"""Bounded, screened command execution for the handoff acceptance harness. +"""Bounded, screened, provenance-attested command execution. -Every probe runs through :class:`Runner`, which owns the three budgets a -long-running acceptance sweep needs: a per-command timeout, a whole-run -deadline, and a per-capture byte budget. Exhausting the deadline does not -silently shorten the sweep — the remaining probes report ``NOT_RUN``, which is a -NO_GO, so a slow cluster can never be mistaken for a clean one. - -Subprocesses never inherit the caller's environment. A probe that echoed -`$ANTHROPIC_API_KEY` would otherwise be one typo away from a leak, so the child -environment is rebuilt from an allowlist of names whose values the harness -already needs to reach the cluster and the forge. +It bounds pipes, times out and reaps the process group, and discards truncated output. """ from __future__ import annotations +import hashlib +import math import os +import selectors import shlex +import shutil +import signal +import stat import subprocess import time from collections.abc import Callable, Mapping, Sequence +from contextlib import suppress from dataclasses import dataclass, field +from pathlib import Path +from typing import Any -from hermes_handoff_policy import READ_ONLY, PolicyError, check_argv -from hermes_handoff_redaction import DEFAULT_MAX_BYTES, redact, safe_text +from hermes_handoff_policy import GITEA_CLIENT, READ_ONLY, PolicyError, check_argv +from hermes_handoff_redaction import ( + DEFAULT_MAX_BYTES, + TRUNCATION_NOTE, + redact, + safe_text, + scrub, +) DEFAULT_COMMAND_TIMEOUT = 25.0 DEFAULT_RUN_DEADLINE = 900.0 +DEFAULT_CLEANUP_RESERVE = 90.0 +MAX_COMMAND_TIMEOUT = 600.0 +MAX_RUN_DEADLINE = 7200.0 +MAX_OUTPUT_BYTES = 8 * 1024 * 1024 +MAX_CONCURRENCY = 16 +READ_CHUNK = 64 * 1024 +PIPE_DRAIN_SECONDS = 0.25 -# Names whose values the harness needs to reach the cluster and the forge. The -# values are handed to the child process and are never recorded. -ENV_PASSTHROUGH = ( - "GIT_ASKPASS", - "GITEA_BASE_URL", - "HERMES_HOME", - "HOME", - "JENKINS_BASE_URL", - "KUBECONFIG", - "LANG", - "PATH", - "REQUESTS_CA_BUNDLE", - "SSL_CERT_FILE", - "VAULT_ADDR", -) +SAFE_PATH = "/opt/data/tools/bin:/opt/hermes/.venv/bin:/opt/coordinator:/usr/local/bin:/usr/bin:/bin" +# These exact allowlists are intentionally dense: reviewers compare them as a +# single provenance boundary rather than as extensible application config. +# fmt: off +ENV_PASSTHROUGH = ("KUBECONFIG", "LANG", "LC_ALL", "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE") +FIXED_ENVIRONMENT = {"GIT_ASKPASS": "/opt/coordinator/gitea_askpass.sh", + "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", "GITEA_BASE_URL": "https://scm.bstein.dev", + "LC_ALL": "C", "PATH": SAFE_PATH} +EXPECTED_PATHS = { + "kubectl": {"/opt/data/tools/bin/kubectl", "/usr/local/bin/kubectl", "/usr/bin/kubectl"}, + "flux": {"/opt/data/tools/bin/flux", "/usr/local/bin/flux", "/usr/bin/flux"}, + "helm": {"/opt/data/tools/bin/helm", "/usr/local/bin/helm", "/usr/bin/helm"}, + "git": {"/usr/bin/git", "/bin/git"}, + "hermes": {"/opt/hermes/.venv/bin/hermes"}, + "sh": {"/usr/bin/sh", "/bin/sh"}, + GITEA_CLIENT: {GITEA_CLIENT}, +} +EXPECTED_SHA256 = { + "kubectl": {"3d514dbae5dc8c09f773df0ef0f5d449dfad05b3aca5c96b13565f886df345fd"}, + "git": {"a0e562e4bd3c4c79379e91d8c07a10104b2cefe8fac966dc6bd4874a57a807f3"}, + "sh": {"367967c823a0c391e5049b15a67c6a0a629c88b9b6dcdca75ef13ac9d65334b1"}, + "hermes": {"c8419290ef7f1a95f59eadcea9f58f446b1e6535a7f4ffee9f1c0cb1172335e4"}, + GITEA_CLIENT: {"f0943db4e81ce96908e94b0f0e78aa4b5ef95d6c4f7803f9f74a43e761576d85"}, +} DEADLINE_ERROR = "deadline-exceeded" TIMEOUT_ERROR = "timeout" - OPERATOR = "operator" SELF = "self" - -# Tools that accept a `--context` flag directly after the binary name. CONTEXT_AWARE_BINARIES = ("kubectl", "flux", "helm") +POD_COMMAND_PATHS = {"kubectl": "/usr/local/bin/kubectl", "hermes": "/opt/hermes/.venv/bin/hermes", + "git": "/usr/bin/git", "sh": "/bin/sh", GITEA_CLIENT: GITEA_CLIENT} +# fmt: on + + +@dataclass(frozen=True) +class Attestation: + """The exact outer executable used for a probe.""" + + path: str + sha256: str + + +@dataclass(frozen=True) +class Capture: + """Raw bounded process result returned by an executor implementation.""" + + returncode: int + stdout: bytes = b"" + stderr: bytes = b"" + stdout_truncated: bool = False + stderr_truncated: bool = False + timed_out: bool = False @dataclass(frozen=True) @@ -66,116 +109,298 @@ class Outcome: truncated: bool = False duration_ms: int = 0 error: str | None = None + executable_path: str = "" + executable_sha256: str = "" @property def ok(self) -> bool: - """Report whether the command ran to completion and succeeded.""" - return self.error is None and self.returncode == 0 + return self.error is None and self.returncode == 0 and not self.truncated @property def ran(self) -> bool: - """Report whether the command reached the target at all.""" return self.error is None @property def combined(self) -> str: - """Return stdout and stderr joined, for message matching.""" return "\n".join(part for part in (self.stdout, self.stderr) if part) def as_dict(self) -> dict[str, object]: - """Return a JSON-serialisable view of the outcome.""" - return { - "command": redact(shlex.join(self.argv)), - "vantage": self.vantage, - "returncode": self.returncode, - "stdout": self.stdout, - "stderr": self.stderr, - "truncated": self.truncated, - "duration_ms": self.duration_ms, - "error": self.error, - } + return scrub( + { + "command": redact(shlex.join(self.argv)), + "vantage": self.vantage, + "returncode": self.returncode, + "stdout": self.stdout, + "stderr": self.stderr, + "truncated": self.truncated, + "duration_ms": self.duration_ms, + "error": self.error, + "executable_path": self.executable_path, + "executable_sha256": self.executable_sha256, + } + ) @dataclass(frozen=True) class Vantage: - """A named identity and transport a probe can be observed from. - - ``prefix`` is how the in-pod vantage is reached: a ``kubectl exec`` into the - target container, so the inner command authenticates as the pod's own - service account instead of as the operator. ``inserts`` carries per-binary - flags — the operator's `--context`, which must not follow the command into - the pod. - """ + """A named identity and transport used to observe a probe.""" name: str prefix: tuple[str, ...] = () inserts: Mapping[str, tuple[str, ...]] = field(default_factory=dict) env: Mapping[str, str] = field(default_factory=dict) + command_paths: Mapping[str, str] = field(default_factory=dict) description: str = "" def wrap(self, argv: Sequence[str]) -> tuple[str, ...]: - """Return ``argv`` addressed to this vantage.""" command = tuple(argv) if command: + replacement = self.command_paths.get(command[0]) + if replacement: + command = (replacement, *command[1:]) extra = self.inserts.get(command[0].rsplit("/", 1)[-1]) if extra: command = (command[0], *extra, *command[1:]) return (*self.prefix, *command) -def build_environment(vantage: Vantage, source: Mapping[str, str] | None = None) -> dict[str, str]: - """Return the child environment: allow-listed names plus vantage overrides.""" +def _finite_number(name: str, value: Any, minimum: float, maximum: float) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + converted = float(value) + if not math.isfinite(converted) or not minimum <= converted <= maximum: + raise ValueError( + f"{name} must be finite and between {minimum:g} and {maximum:g}" + ) + return converted + + +def _byte_budget(value: Any) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 1 <= value <= MAX_OUTPUT_BYTES + ): + raise ValueError( + f"max_bytes must be an integer between 1 and {MAX_OUTPUT_BYTES}" + ) + return value + + +def _concurrency(value: Any) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 1 <= value <= MAX_CONCURRENCY + ): + raise ValueError( + f"concurrency must be an integer between 1 and {MAX_CONCURRENCY}" + ) + return value + + +def build_environment( + vantage: Vantage, source: Mapping[str, str] | None = None +) -> dict[str, str]: + """Build a deterministic child environment without caller-controlled PATH.""" origin = os.environ if source is None else source environment = {name: origin[name] for name in ENV_PASSTHROUGH if name in origin} - environment.setdefault("LC_ALL", "C") + environment.update(FIXED_ENVIRONMENT) environment.update(vantage.env) + environment["PATH"] = SAFE_PATH + environment["GITEA_BASE_URL"] = FIXED_ENVIRONMENT["GITEA_BASE_URL"] + environment["GIT_ASKPASS"] = FIXED_ENVIRONMENT["GIT_ASKPASS"] return environment +# fmt: off def operator_vantage(kubeconfig: str | None = None, context: str | None = None) -> Vantage: - """Return the external read-only operator vantage.""" - inserts = ( - {binary: ("--context", context) for binary in CONTEXT_AWARE_BINARIES} if context else {} - ) - detail = " ".join( - part for part in (f"context={context}" if context else "", "kubeconfig-pinned" if kubeconfig else "") if part - ) - return Vantage( - name=OPERATOR, - inserts=inserts, - env={"KUBECONFIG": kubeconfig} if kubeconfig else {}, - description=f"operator {detail}".strip(), - ) + inserts = dict.fromkeys(CONTEXT_AWARE_BINARIES, ("--context", context)) if context else {} + detail = " ".join(part for part in (f"context={context}" if context else "", + "kubeconfig-pinned" if kubeconfig else "") if part) + return Vantage(name=OPERATOR, inserts=inserts, + env={"KUBECONFIG": kubeconfig} if kubeconfig else {}, + description=f"operator {detail}".strip()) -def pod_vantage( - namespace: str, - pod: str, - container: str, - operator: Vantage | None = None, -) -> Vantage: - """Return the in-pod vantage: the inner command authenticates as the pod.""" +def pod_vantage(namespace: str, pod: str, container: str, operator: Vantage | None = None) -> Vantage: reach = operator.inserts.get("kubectl", ()) if operator else () - return Vantage( - name=SELF, - prefix=( - "kubectl", - *reach, - "exec", - "--namespace", - namespace, - pod, - "--container", - container, - "--", - ), - env=dict(operator.env) if operator else {}, - description=f"in-pod {namespace}/{pod}[{container}]", + return Vantage(name=SELF, prefix=("kubectl", *reach, "exec", "--namespace", namespace, + pod, "--container", container, "--"), + env=dict(operator.env) if operator else {}, command_paths=POD_COMMAND_PATHS, + description=f"in-pod {namespace}/{pod}[{container}]") +# fmt: on + + +def attest_executable(command: str, environment: Mapping[str, str]) -> Attestation: + """Resolve and hash an executable from a fixed, trusted path set.""" + name = command if command == GITEA_CLIENT else command.rsplit("/", 1)[-1] + candidate = ( + command if "/" in command else shutil.which(command, path=environment["PATH"]) ) + if not candidate: + raise OSError(f"trusted executable is unavailable: {name}") + resolved = str(Path(candidate).resolve(strict=True)) + allowed = { + str(Path(item).resolve(strict=False)) + for item in EXPECTED_PATHS.get(name, set()) + } + if resolved not in allowed: + raise OSError(f"executable resolved outside its trusted paths: {name}") + details = os.stat(resolved) + if ( + not stat.S_ISREG(details.st_mode) + or details.st_mode & 0o022 + or not details.st_mode & 0o111 + ): + raise OSError(f"trusted executable has unsafe ownership mode: {name}") + digest = hashlib.sha256() + with open(resolved, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + observed = digest.hexdigest() + expected = EXPECTED_SHA256.get(name) + if not expected or observed not in expected: + raise OSError(f"executable digest is not release-attested: {name}") + return Attestation(resolved, observed) + + +def _append_bounded(buffer: bytearray, chunk: bytes, limit: int) -> bool: + remaining = limit + 1 - len(buffer) + if remaining > 0: + buffer.extend(chunk[:remaining]) + return len(buffer) > limit or len(chunk) > remaining + + +def _signal_group(process: subprocess.Popen[bytes], signum: int) -> None: + with suppress(ProcessLookupError): + os.killpg(process.pid, signum) + + +def execute_bounded( + argv: Sequence[str], + environment: Mapping[str, str], + timeout: float, + max_bytes: int, + clock: Callable[[], float] = time.monotonic, + popen: Callable[..., subprocess.Popen[bytes]] = subprocess.Popen, +) -> Capture: + """Execute one process with bounded live pipe capture and group cleanup.""" + process = popen( + list(argv), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=dict(environment), + text=False, + close_fds=True, + start_new_session=True, + ) + streams = {"stdout": process.stdout, "stderr": process.stderr} + buffers = {"stdout": bytearray(), "stderr": bytearray()} + truncated = {"stdout": False, "stderr": False} + selector = selectors.DefaultSelector() + for name, stream in streams.items(): + if ( + stream is not None + ): # pragma: no branch - PIPE was requested for both streams + os.set_blocking(stream.fileno(), False) + selector.register(stream, selectors.EVENT_READ, name) + expires = clock() + timeout + timed_out = False + try: + while process.poll() is None: + remaining = expires - clock() + if remaining <= 0: + timed_out = True + break + for key, _ in selector.select(min(remaining, 0.1)): + try: + chunk = os.read(key.fileobj.fileno(), READ_CHUNK) + except BlockingIOError: # pragma: no cover - readiness race, retry only + continue + if not chunk: + selector.unregister(key.fileobj) + continue + name = key.data + truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes) + _signal_group(process, signal.SIGKILL if timed_out else signal.SIGTERM) + drain_until = clock() + PIPE_DRAIN_SECONDS + while selector.get_map() and clock() < drain_until: + for key, _ in selector.select(max(0.0, drain_until - clock())): + try: + chunk = os.read(key.fileobj.fileno(), READ_CHUNK) + except BlockingIOError: # pragma: no cover - readiness race, retry only + continue + if not chunk: + selector.unregister(key.fileobj) + continue + name = key.data + truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes) + _signal_group(process, signal.SIGKILL) + returncode = process.wait(timeout=PIPE_DRAIN_SECONDS) + finally: + selector.close() + for stream in streams.values(): + if ( + stream is not None + ): # pragma: no branch - PIPE was requested for both streams + stream.close() + if ( + process.poll() is None + ): # pragma: no branch - fail-safe for wait/selector exceptions + _signal_group(process, signal.SIGKILL) + process.wait() + return Capture( + returncode=returncode, + stdout=bytes(buffers["stdout"][:max_bytes]), + stderr=bytes(buffers["stderr"][:max_bytes]), + stdout_truncated=truncated["stdout"], + stderr_truncated=truncated["stderr"], + timed_out=timed_out, + ) + + +def _injected_executor( + spawn: Callable[..., subprocess.CompletedProcess[Any]], +) -> Callable[..., Capture]: + """Adapt deterministic test doubles without weakening the production path.""" + + def execute( + argv: Sequence[str], + environment: Mapping[str, str], + timeout: float, + max_bytes: int, + ) -> Capture: + completed = spawn( + list(argv), + capture_output=True, + text=True, + timeout=timeout, + check=False, + env=dict(environment), + ) + stdout = str(completed.stdout or "").encode() + stderr = str(completed.stderr or "").encode() + return Capture( + completed.returncode, + stdout[:max_bytes], + stderr[:max_bytes], + len(stdout) > max_bytes, + len(stderr) > max_bytes, + ) + + return execute + + +def _screen_capture(data: bytes, truncated: bool, budget: int) -> str: + if truncated: + return TRUNCATION_NOTE.encode()[:budget].decode() + return safe_text(data.decode("utf-8", "replace"), budget)[0] class Runner: - """Run probe commands under policy, timeout, deadline, and byte budgets.""" + """Run probes under policy, provenance, timeout, deadline, and byte budgets.""" def __init__( self, @@ -183,70 +408,92 @@ class Runner: command_timeout: float = DEFAULT_COMMAND_TIMEOUT, deadline_seconds: float = DEFAULT_RUN_DEADLINE, max_bytes: int = DEFAULT_MAX_BYTES, + concurrency: int = 1, clock: Callable[[], float] = time.monotonic, - spawn: Callable[..., subprocess.CompletedProcess] = subprocess.run, + executor: Callable[..., Capture] | None = None, + spawn: Callable[..., subprocess.CompletedProcess[Any]] | None = None, environ: Mapping[str, str] | None = None, + attestor: Callable[[str, Mapping[str, str]], Attestation] = attest_executable, ) -> None: self.mode = mode - self.command_timeout = command_timeout - self.max_bytes = max_bytes + self.command_timeout = _finite_number( + "command_timeout", command_timeout, 0.05, MAX_COMMAND_TIMEOUT + ) + deadline = _finite_number( + "deadline_seconds", deadline_seconds, 0.05, MAX_RUN_DEADLINE + ) + self.max_bytes = _byte_budget(max_bytes) + self.concurrency = _concurrency(concurrency) self._clock = clock - self._spawn = spawn + self._executor = executor or ( + _injected_executor(spawn) if spawn else execute_bounded + ) self._environ = environ - self._deadline_at = clock() + deadline_seconds + self._attestor = attestor + self._deadline_at = clock() + deadline self.commands_run = 0 @property def remaining_seconds(self) -> float: - """Return the seconds left before the whole-run deadline.""" return self._deadline_at - self._clock() def run( - self, argv: Sequence[str], vantage: Vantage, max_bytes: int | None = None + self, + argv: Sequence[str], + vantage: Vantage, + max_bytes: int | None = None, + reserve_seconds: float = 0.0, ) -> Outcome: - """Run one command, returning a screened outcome and never raising.""" - budget_bytes = self.max_bytes if max_bytes is None else max_bytes + budget_bytes = self.max_bytes if max_bytes is None else _byte_budget(max_bytes) + reserve = _finite_number( + "reserve_seconds", reserve_seconds, 0.0, MAX_RUN_DEADLINE + ) addressed = vantage.wrap(argv) try: check_argv(addressed, self.mode) except PolicyError as exc: return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}") - - budget = min(self.command_timeout, self.remaining_seconds) + budget = min(self.command_timeout, self.remaining_seconds - reserve) if budget <= 0: return Outcome(argv=addressed, vantage=vantage.name, error=DEADLINE_ERROR) - + environment = build_environment(vantage, self._environ) + try: + attestation = self._attestor(addressed[0], environment) + except (OSError, ValueError) as exc: + return Outcome( + argv=addressed, vantage=vantage.name, error=f"attestation: {exc}" + ) + executed = (attestation.path, *addressed[1:]) + try: + check_argv(executed, self.mode) + except PolicyError as exc: + return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}") started = self._clock() self.commands_run += 1 try: - completed = self._spawn( - list(addressed), - capture_output=True, - text=True, - timeout=budget, - check=False, - env=build_environment(vantage, self._environ), - ) + capture = self._executor(executed, environment, budget, budget_bytes) except subprocess.TimeoutExpired: + capture = Capture(-signal.SIGKILL, timed_out=True) + except OSError as exc: return Outcome( argv=addressed, vantage=vantage.name, - error=TIMEOUT_ERROR, - duration_ms=int((self._clock() - started) * 1000), + error=f"spawn: {exc.strerror or type(exc).__name__}", + executable_path=attestation.path, + executable_sha256=attestation.sha256, ) - except OSError as exc: - return Outcome( - argv=addressed, vantage=vantage.name, error=f"spawn: {exc.strerror or exc}" - ) - - stdout, cut_out = safe_text(completed.stdout or "", budget_bytes) - stderr, cut_err = safe_text(completed.stderr or "", budget_bytes) + was_truncated = capture.stdout_truncated or capture.stderr_truncated + stdout = _screen_capture(capture.stdout, capture.stdout_truncated, budget_bytes) + stderr = _screen_capture(capture.stderr, capture.stderr_truncated, budget_bytes) return Outcome( argv=addressed, vantage=vantage.name, - returncode=completed.returncode, + returncode=capture.returncode, stdout=stdout, stderr=stderr, - truncated=cut_out or cut_err, + truncated=was_truncated, duration_ms=int((self._clock() - started) * 1000), + error=TIMEOUT_ERROR if capture.timed_out else None, + executable_path=attestation.path, + executable_sha256=attestation.sha256, ) diff --git a/scripts/ops/hermes_handoff_json_rules.py b/scripts/ops/hermes_handoff_json_rules.py index 6c10e52d..0f44c6a2 100644 --- a/scripts/ops/hermes_handoff_json_rules.py +++ b/scripts/ops/hermes_handoff_json_rules.py @@ -11,7 +11,8 @@ other rule: an absent path, an unparsable payload, or an empty result set is from __future__ import annotations import datetime as dt -import json +import math +import re from typing import Any from hermes_handoff_exec import Outcome @@ -20,11 +21,33 @@ from hermes_handoff_rules import ( Evaluation, dotted, evaluator, + exact_equal, is_list, missing_steps, parsed_step, + strict_json, ) +_ROUTE_PART_RE = re.compile(r"[a-z0-9][a-z0-9_.-]{0,63}\Z") +_FLUX_NAME_RE = re.compile( + r"[a-z0-9]([-a-z0-9.]*[a-z0-9])?/[a-z0-9]([-a-z0-9.]*[a-z0-9])?\Z" +) + + +def _timestamp(value: Any) -> dt.datetime: + if not isinstance(value, str) or not value.strip(): + raise ValueError("timestamp is not a non-empty string") + observed = dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + if observed.tzinfo is None: + raise ValueError("timestamp has no timezone") + return observed.astimezone(dt.timezone.utc) + + +def _now(value: Any) -> dt.datetime: + if not isinstance(value, dt.datetime) or value.tzinfo is None: + raise ValueError("run clock is not timezone-aware") + return value.astimezone(dt.timezone.utc) + @evaluator("json_field") def evaluate_json_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: @@ -42,10 +65,12 @@ def evaluate_json_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evalua try: actual = dotted(payload, path) except (KeyError, ValueError, IndexError): - return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) evidence[path] = actual allowed = expected if isinstance(expected, list) else [expected] - if actual not in allowed: + if not any(exact_equal(actual, item) for item in allowed): mismatches.append(f"{path}={actual!r} expected one of {allowed!r}") if mismatches: return Evaluation(FAIL, "; ".join(mismatches), evidence) @@ -68,9 +93,17 @@ def evaluate_json_numeric(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Eval try: actual = dotted(payload, path) except (KeyError, ValueError, IndexError): - return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) - if not isinstance(actual, (int, float)) or isinstance(actual, bool): - return Evaluation(NOT_RUN, f"path {path} is not numeric in step {key}", evidence) + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) + if ( + not isinstance(actual, (int, float)) + or isinstance(actual, bool) + or not math.isfinite(float(actual)) + ): + return Evaluation( + NOT_RUN, f"path {path} is not numeric in step {key}", evidence + ) evidence[path] = actual if "min" in bounds and actual < bounds["min"]: problems.append(f"{path}={actual} below minimum {bounds['min']}") @@ -96,23 +129,30 @@ def evaluate_json_recent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evalu if isinstance(parsed, Evaluation): return parsed payload, key = parsed - now = spec.expect["now"] + try: + now = _now(spec.expect["now"]) + except (TypeError, ValueError) as exc: + return Evaluation(NOT_RUN, f"invalid run clock: {exc}") evidence: dict[str, Any] = {} stale: list[str] = [] for path, max_age in spec.expect["fields"].items(): try: raw = dotted(payload, path) except (KeyError, ValueError, IndexError): - return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) try: - observed = dt.datetime.fromisoformat(str(raw).replace("Z", "+00:00")) - except ValueError: - return Evaluation(NOT_RUN, f"path {path} is not an ISO timestamp", evidence) - if observed.tzinfo is None: - observed = observed.replace(tzinfo=dt.timezone.utc) + observed = _timestamp(raw) + except (TypeError, ValueError): + return Evaluation( + NOT_RUN, f"path {path} is not a timezone-aware ISO timestamp", evidence + ) age = (now - observed).total_seconds() evidence[path] = {"age_seconds": int(age)} - if age > max_age: + if age < 0: + stale.append(f"{path} is {abs(int(age))}s in the future") + elif age > max_age: stale.append(f"{path} is {int(age)}s old, limit {max_age}s") if stale: return Evaluation(FAIL, "; ".join(stale), evidence) @@ -161,18 +201,24 @@ def evaluate_all_of_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Eval except (KeyError, ValueError, IndexError): actual = None allowed = expected if isinstance(expected, list) else [expected] - if actual not in allowed: - problems.append(f"{identity}: {path}={actual!r} expected one of {allowed!r}") + if not any(exact_equal(actual, item) for item in allowed): + problems.append( + f"{identity}: {path}={actual!r} expected one of {allowed!r}" + ) evidence["examined"] = seen[:32] if wanted is not None: absent = [item for item in wanted if item not in seen] if absent: - return Evaluation(NOT_RUN, f"expected keys absent from step {key}: {absent}", evidence) + return Evaluation( + NOT_RUN, f"expected keys absent from step {key}: {absent}", evidence + ) elif not seen: return Evaluation(NOT_RUN, f"step {key} returned nothing to examine", evidence) if problems: return Evaluation(FAIL, "; ".join(problems[:8]), evidence) - return Evaluation(PASS, f"{len(seen)} element(s) match the release contract", evidence) + return Evaluation( + PASS, f"{len(seen)} element(s) match the release contract", evidence + ) @evaluator("json_record") @@ -195,21 +241,23 @@ def evaluate_json_record(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evalu try: evidence[label] = dotted(payload, path) except (KeyError, ValueError, IndexError): - return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) return Evaluation(PASS, "state recorded", evidence) - @evaluator("routing_evidence") -def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: +def evaluate_routing_evidence( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: """Summarise a Switchyard routing log tail into provider/effort coverage. Route identifiers are ``///``, so provider, model family, effort tier, and lane all come from the same field. Fallback and failure evidence is a separate ``fallback_reason`` on the record that - was retried. The parser tolerates a clipped first line: a tail of a growing - file starts mid-record, and discarding one row is better than a NOT_RUN that - reads as an outage. + was retried. Every row must be complete strict JSON; a clipped or malformed + line makes the evidence unusable rather than being silently discarded. """ blocked = missing_steps(spec, outcomes) if blocked: @@ -218,22 +266,64 @@ def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> outcome = outcomes[key] if not outcome.ok: return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") - records = [] - for line in outcome.stdout.splitlines(): + records: list[dict[str, Any]] = [] + routes: list[list[str]] = [] + stamps_by_dimension: dict[tuple[str, str], dt.datetime] = {} + now_value = spec.expect.get("now") + try: + now = _now(now_value) if now_value is not None else None + except (TypeError, ValueError) as exc: + return Evaluation(NOT_RUN, f"invalid routing run clock: {exc}") + for row_number, line in enumerate(outcome.stdout.splitlines(), 1): line = line.strip() if not line: continue try: - records.append(json.loads(line)) - except ValueError: - continue - routes = [ - str(record.get("model") or "").split("/") - for record in records - if str(record.get("model") or "").count("/") == 3 - ] + record = strict_json(line) + except ValueError as exc: + return Evaluation( + NOT_RUN, f"routing row {row_number} is malformed JSON: {exc}" + ) + if not isinstance(record, dict): + return Evaluation(NOT_RUN, f"routing row {row_number} is not an object") + try: + observed = _timestamp(record.get("ts")) + except (TypeError, ValueError) as exc: + return Evaluation( + NOT_RUN, f"routing row {row_number} has invalid timestamp: {exc}" + ) + if now is not None and observed > now: + return Evaluation(FAIL, f"routing row {row_number} has a future timestamp") + model = record.get("model") + if not isinstance(model, str) or not model: + return Evaluation(NOT_RUN, f"routing row {row_number} has no model") + route = model.split("/") + if len(route) == 4 and all(_ROUTE_PART_RE.fullmatch(part) for part in route): + routes.append(route) + for label, value in ( + ("provider", route[1]), + ("effort", route[3]), + ("lane", route[0]), + ): + key_dimension = (label, value) + stamps_by_dimension[key_dimension] = max( + observed, stamps_by_dimension.get(key_dimension, observed) + ) + elif record.get("tier") != "classifier": + return Evaluation( + NOT_RUN, f"routing row {row_number} has malformed route {model!r}" + ) + fallback = record.get("fallback_reason") + if fallback is not None and ( + not isinstance(fallback, str) or not fallback.strip() + ): + return Evaluation( + NOT_RUN, f"routing row {row_number} has malformed fallback_reason" + ) + record["_observed"] = observed + records.append(record) fallbacks = [record for record in records if record.get("fallback_reason")] - stamps = sorted(str(record.get("ts")) for record in records if record.get("ts")) + stamps = sorted(record["_observed"] for record in records) evidence: dict[str, Any] = { "records": len(records), "routed": len(routes), @@ -242,11 +332,15 @@ def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> "families": sorted({route[2] for route in routes}), "efforts": sorted({route[3] for route in routes}), "fallback_events": len(fallbacks), - "fallback_reasons": sorted({str(record["fallback_reason"]) for record in fallbacks}), - "latest_ts": stamps[-1] if stamps else "", + "fallback_reasons": sorted( + {str(record["fallback_reason"]) for record in fallbacks} + ), + "latest_ts": stamps[-1].isoformat() if stamps else "", } if not records: - return Evaluation(NOT_RUN, "routing log tail contained no parsable records", evidence) + return Evaluation( + NOT_RUN, "routing log tail contained no parsable records", evidence + ) missing = [ f"{label}:{item}" for label, wanted in ( @@ -260,18 +354,28 @@ def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> if spec.expect.get("require_fallback_evidence") and not fallbacks: missing.append("fallback:none-observed") max_age = spec.expect.get("max_age_seconds") - now = spec.expect.get("now") if max_age and now: - try: - newest = dt.datetime.fromisoformat(evidence["latest_ts"].replace("Z", "+00:00")) - except ValueError: - return Evaluation(NOT_RUN, "routing log has no parsable newest timestamp", evidence) - age = (now - newest).total_seconds() - evidence["age_seconds"] = int(age) - if age > max_age: - missing.append(f"freshness:{int(age)}s-exceeds-{max_age}s") + ages: dict[str, int] = {} + for label, wanted in ( + ("provider", spec.expect.get("providers", ())), + ("effort", spec.expect.get("efforts", ())), + ("lane", spec.expect.get("lanes", ())), + ): + for item in wanted: + stamp = stamps_by_dimension.get((label, item)) + if stamp is None: + continue + age = int((now - stamp).total_seconds()) + ages[f"{label}:{item}"] = age + if age > max_age: + missing.append( + f"freshness:{label}:{item}:{age}s-exceeds-{max_age}s" + ) + evidence["age_seconds"] = ages if missing: - return Evaluation(FAIL, f"routing evidence missing {', '.join(missing)}", evidence) + return Evaluation( + FAIL, f"routing evidence missing {', '.join(missing)}", evidence + ) return Evaluation(PASS, "routing evidence covers the required lanes", evidence) @@ -279,14 +383,10 @@ def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> def evaluate_flux_health(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: """Classify a compact Flux projection into suspensions and unhealthy objects. - Each row is ``namespace/namesuspendType=Status,...``. An object is - unhealthy when it reports ``Healthy=False``, or ``Ready=False`` with no - Healthy condition and no in-flight ``Reconciling=True``. That last clause - matters: Flux flips Ready to False for the duration of a reconcile, so - without it the verdict depends on when the snapshot was taken. Suspended - objects are excluded from the health tally and judged against the - expected-suspension list instead, because a deliberate park is a different - fact from a broken reconcile. + Each row carries name, generation, observed generation, suspension, and all + condition states. A non-suspended object must be current, explicitly + ``Ready=True``, not ``Healthy=False/Unknown``, and not reconciling. Suspended + objects are judged against the exact expected-suspension list. """ blocked = missing_steps(spec, outcomes) if blocked: @@ -299,23 +399,62 @@ def evaluate_flux_health(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evalu suspended: list[str] = [] unhealthy: list[str] = [] total = 0 - for line in outcome.stdout.splitlines(): + for row_number, line in enumerate(outcome.stdout.splitlines(), 1): if not line.strip(): continue + columns = line.split("\t") + if len(columns) != 5: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has {len(columns)} columns" + ) + name, generation, observed_generation, suspend, conditions = columns + name = name.strip() + if not _FLUX_NAME_RE.fullmatch(name): + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed object name" + ) + try: + current_generation = int(generation) + reconciled_generation = int(observed_generation) + except ValueError: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed generations" + ) + if current_generation <= 0 or current_generation != reconciled_generation: + return Evaluation( + FAIL, f"Flux row {row_number} has stale observedGeneration" + ) + if suspend.strip().lower() not in {"", "false", "true"}: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed suspension state" + ) total += 1 - name, _, rest = line.partition("\t") - suspend, _, conditions = rest.partition("\t") if suspend.strip().lower() == "true": - suspended.append(name.strip()) + suspended.append(name) continue - states = dict( - pair.split("=", 1) for pair in conditions.strip().strip(",").split(",") if "=" in pair - ) + states: dict[str, str] = {} + for pair in filter(None, conditions.strip().strip(",").split(",")): + if pair.count("=") != 1: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed condition" + ) + condition, state = pair.split("=", 1) + if ( + not condition + or state not in {"True", "False", "Unknown"} + or condition in states + ): + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed condition state" + ) + states[condition] = state healthy = states.get("Healthy") - reconciling = states.get("Reconciling") == "True" - ready_failed = states.get("Ready") == "False" - if healthy == "False" or (healthy is None and ready_failed and not reconciling): - unhealthy.append(name.strip()) + if ( + states.get("Ready") != "True" + or healthy in {"False", "Unknown"} + or states.get("Reconciling") == "True" + ): + unhealthy.append(name) unexpected = sorted(set(suspended) - expected) evidence = { "total": total, diff --git a/scripts/ops/hermes_handoff_model.py b/scripts/ops/hermes_handoff_model.py index d34b9ebc..d35d39f5 100644 --- a/scripts/ops/hermes_handoff_model.py +++ b/scripts/ops/hermes_handoff_model.py @@ -17,7 +17,7 @@ from dataclasses import dataclass, field from typing import Any from hermes_handoff_exec import Outcome -from hermes_handoff_redaction import contains_credential_shape, scrub +from hermes_handoff_redaction import contains_credential_shape, safe_text, scrub PASS = "PASS" FAIL = "FAIL" @@ -108,6 +108,8 @@ class CheckResult: @property def blocking(self) -> bool: """Report whether this result alone forces NO_GO.""" + if self.status not in STATUSES: + return True return self.spec.mandatory and self.status in {FAIL, NOT_RUN} def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: @@ -172,9 +174,17 @@ class Report: """Return the number of results in each status.""" tally = dict.fromkeys(STATUSES, 0) for result in self.results: - tally[result.status] = tally.get(result.status, 0) + 1 + if result.status in tally: + tally[result.status] += 1 return tally + @property + def invalid_statuses(self) -> list[str]: + """Return check ids whose status is outside the report schema.""" + return [ + result.spec.id for result in self.results if result.status not in STATUSES + ] + @property def blocking(self) -> list[CheckResult]: """Return every result that forces NO_GO, most severe first.""" @@ -187,7 +197,12 @@ class Report: @property def decision(self) -> str: """Return GO only when nothing mandatory failed, skipped, or errored.""" - if self.harness_errors or self.blocking: + if ( + not self.results + or self.invalid_statuses + or self.harness_errors + or self.blocking + ): return NO_GO return GO @@ -201,20 +216,23 @@ class Report: def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: """Return the machine-readable report.""" - return { - "harness": HARNESS, - "schema_version": SCHEMA_VERSION, - "mode": self.mode, - "started_at": self.started_at, - "finished_at": self.finished_at, - "decision": self.decision, - "counts": self.counts, - "baseline": scrub(self.baseline), - "vantages": [vantage.as_dict() for vantage in self.vantages], - "harness_errors": [scrub(error) for error in self.harness_errors], - "blocking": [result.spec.id for result in self.blocking], - "checks": [result.as_dict(include_outcomes) for result in self.results], - } + return scrub( + { + "harness": HARNESS, + "schema_version": SCHEMA_VERSION, + "mode": self.mode, + "started_at": self.started_at, + "finished_at": self.finished_at, + "decision": self.decision, + "counts": self.counts, + "baseline": scrub(self.baseline), + "vantages": [vantage.as_dict() for vantage in self.vantages], + "harness_errors": [scrub(error) for error in self.harness_errors], + "invalid_statuses": self.invalid_statuses, + "blocking": [result.spec.id for result in self.blocking], + "checks": [result.as_dict(include_outcomes) for result in self.results], + } + ) def render_summary(self, width: int = 88) -> str: """Return the concise human summary printed alongside the JSON.""" @@ -239,9 +257,11 @@ class Report: lines.append(" groups:") for name in sorted(groups): bucket = groups[name] - detail = " ".join(f"{status}={bucket[status]}" for status in STATUSES if bucket[status]) + detail = " ".join( + f"{status}={bucket[status]}" for status in STATUSES if bucket[status] + ) lines.append(f" {name:<18}{detail}") - return "\n".join(lines) + return safe_text("\n".join(lines), 32 * 1024)[0] def utc_now(clock: Any = None) -> str: diff --git a/scripts/ops/hermes_handoff_policy.py b/scripts/ops/hermes_handoff_policy.py index 473dd782..fc1a7097 100644 --- a/scripts/ops/hermes_handoff_policy.py +++ b/scripts/ops/hermes_handoff_policy.py @@ -1,24 +1,8 @@ #!/usr/bin/env python3 """Fail-closed command policy for the Hermes handoff acceptance harness. -Two properties have to hold structurally, not by convention: - -* the default run mutates nothing, and -* no probe can pull a credential value into the report. - -Both are enforced here, where an argv becomes a subprocess, so a mistake in the -check catalog surfaces as a policy error when the catalog is built rather than -as a mutation or a leak when it runs. `--as` is rejected on the operator side: -impersonation is not evidence about what Hermes can do, and a harness able to -impersonate would be tempted to substitute it for a real self-probe. It stays -available to the inner command of a `kubectl exec`, where an identity is -probing its own refusal to impersonate rather than borrowing authority. - -Shell is reachable only through frozen templates. Free-form `sh -c` cannot be -screened reliably, but a few probes genuinely need field extraction (report that -a password field is locked without ever reading its hash) or an explicit helper -path, so those scripts are named, reviewed here, and parameter-validated instead -of assembled by callers. +Before spawn it admits only credential-safe, structurally read-only grammar; +shell is limited to frozen templates with absolute tool paths. """ from __future__ import annotations @@ -30,28 +14,42 @@ READ_ONLY = "read-only" ARMED = "ephemeral-armed" MODES = (READ_ONLY, ARMED) -# Rejected everywhere. Impersonation and inline credentials have no honest use -# in an acceptance probe. -FORBIDDEN_ARGS = ( - "--as", - "--as-group", - "--as-uid", - "--token", - "--password", - "--username", - "--client-key", - "--client-certificate", -) -# Reachable only from inside a pod, where an identity is probing its own -# refusal to impersonate rather than borrowing someone else's authority. -IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid") -FORBIDDEN_BINARIES = frozenset({"vault", "sops", "age", "gpg", "openssl", "curl", "wget"}) GITEA_CLIENT = "/opt/coordinator/gitea_api.py" -ALLOWED_BINARIES = frozenset({"kubectl", "flux", "helm", "git", "hermes", "sh", GITEA_CLIENT}) +EXPECTED_REPO = "atlas/titan-iac" +EXPECTED_REMOTE = "origin" +EXPECTED_BASE = "main" + +# fmt: off +FORBIDDEN_ARGS = ("--as", "--as-group", "--as-uid", "--client-certificate", + "--client-key", "--password", "--token", "--username") +# fmt: on +IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid") +FORBIDDEN_BINARIES = frozenset( + {"age", "curl", "gpg", "openssl", "sops", "vault", "wget"} +) +ALLOWED_BINARIES = frozenset( + {"kubectl", "flux", "helm", "git", "hermes", "sh", GITEA_CLIENT} +) +TRUSTED_EXECUTABLE_ROOTS = ( + "/bin/", + "/usr/bin/", + "/usr/local/bin/", + "/opt/data/tools/bin/", + "/opt/hermes/.venv/bin/", + "/opt/coordinator/", +) +TRUSTED_INNER_PATHS = { + "/bin/sh": "sh", + "/usr/bin/git": "git", + "/usr/bin/sh": "sh", + "/usr/local/bin/kubectl": "kubectl", + "/opt/hermes/.venv/bin/hermes": "hermes", + GITEA_CLIENT: GITEA_CLIENT, +} -# Flags that consume the following token, so it is never a subcommand. VALUE_FLAGS = frozenset( { + "-C", "-c", "-l", "-n", @@ -76,50 +74,44 @@ KUBECTL_READ_SUBCOMMANDS = frozenset( { "api-resources", "api-versions", - "auth", - "cluster-info", - "config", - "describe", - "events", "explain", "get", - "logs", "top", "version", } ) -# Reachable only through a dry run: the API server authorises the request and -# discards it. A rejection is the evidence, and nothing is persisted either way. -KUBECTL_DRY_RUN_SUBCOMMANDS = frozenset( - {"annotate", "apply", "create", "delete", "label", "patch", "replace", "scale", "set"} -) KUBECTL_EXEC_SUBCOMMANDS = frozenset({"exec"}) -KUBECTL_SUBCOMMANDS = ( - KUBECTL_READ_SUBCOMMANDS | KUBECTL_DRY_RUN_SUBCOMMANDS | KUBECTL_EXEC_SUBCOMMANDS +KUBECTL_SUBCOMMANDS = KUBECTL_READ_SUBCOMMANDS | KUBECTL_EXEC_SUBCOMMANDS | {"auth"} +SECRET_RESOURCES = frozenset({"secret", "secrets"}) +UNSAFE_PROJECTION_PARTS = ( + ".args", + ".command", + ".data", + ".envfrom", + ".secretkeyref", + ".status.token", + ".stringdata", + ".valuefrom", + "clientconfig", + "password", + "privatekey", ) -DRY_RUN_FLAGS = ("--dry-run=server", "--dry-run=client") - -# Resources whose objects carry credential material. Reading them is permitted -# only in name-only output form, so a regression that grants access still cannot -# deliver a secret value into a report. -NAME_ONLY_RESOURCES = frozenset({"secret", "secrets"}) +SAFE_ENV_VALUE = 'env[?(@.name=="hermes_auto_router_profile")].value' +SAFE_JSON_FIELDS = frozenset(["authenticated", "transport", "state", "model", "effort", "latency_ms", "checked_at", "api_provider", "auth_method", "subscription_type"]) # fmt: skip FLUX_READ_SUBCOMMANDS = frozenset( - {"check", "diff", "events", "export", "get", "stats", "trace", "tree", "version"} + {"check", "events", "get", "stats", "trace", "tree", "version"} ) -HELM_READ_SUBCOMMANDS = frozenset({"get", "history", "list", "status", "version"}) +HELM_READ_SUBCOMMANDS = frozenset({"history", "list", "status", "version"}) GIT_READ_SUBCOMMANDS = frozenset( { "cat-file", - "config", "diff", - "fetch", "for-each-ref", "log", "ls-remote", "ls-tree", "merge-base", - "remote", "rev-list", "rev-parse", "show", @@ -127,60 +119,47 @@ GIT_READ_SUBCOMMANDS = frozenset( } ) GIT_ARMED_SUBCOMMANDS = frozenset({"push"}) -GIT_FORCE_FLAGS = frozenset({"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"}) +GIT_FORCE_FLAGS = frozenset( + {"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"} +) +GIT_UNSAFE_FLAGS = ( + "-c", + "--config-env", + "--exec-path", + "--upload-pack", + "--receive-pack", +) GITEA_READ_METHODS = frozenset({"GET"}) -GITEA_ARMED_METHODS = frozenset({"POST", "PATCH", "DELETE"}) - -# The Hermes CLI is the only honest source for durable session and Kanban -# activity, so a narrow read-only slice of it is on the allowlist. -HERMES_READ_COMMANDS = frozenset({("kanban", "list"), ("kanban", "show"), ("sessions", "list"), ("status",)}) - -# Paths that hold credential material. A probe may prove one exists or is -# unreadable; it may never read it outside a frozen template. -SECRET_PATH_RE = re.compile( - r"(?:^|/)(?:vault/secrets|var/run/secrets|runtime-access|\.credentials\.json" - r"|auth\.json|authorized_keys|id_[a-z0-9]+|\.netrc|shadow|gshadow)(?:$|/)" +HERMES_READ_COMMANDS = frozenset( + {("kanban", "list"), ("kanban", "show"), ("sessions", "list"), ("status",)} ) +# These locations contain credential bytes even when a caller claims it will +# only inspect metadata. They are rejected for every argv, including a frozen +# shell rendering and the inner command of kubectl exec. +SECRET_PATH_RE = re.compile( + r"(?:^|[\s'\"=])(?:/[^\s'\"]*)?(?:" + r"var/run/secrets|run/secrets|vault/secrets|runtime-access|scm-access|pool-access|" + r"\.credentials\.json|auth\.json|\.git-credentials|authorized_keys|" + r"id_[a-z0-9]+|\.netrc|shadow|gshadow|k3s\.yaml" + r")(?:$|[/\s'\"])", + re.IGNORECASE, +) _PARAMETER_RE = re.compile(r"[A-Za-z0-9/][A-Za-z0-9._/@:,-]{0,255}\Z") _PLACEHOLDER_RE = re.compile(r"\{([a-z_]+)\}") +_SAFE_API_PATH_RE = re.compile(r"/[A-Za-z0-9][A-Za-z0-9._~:/?&=%+-]{0,511}\Z") -# Frozen shell templates. Each exists because a plain argv cannot express the -# safety property its probe needs. SHELL_TEMPLATES: dict[str, str] = { - # Report an account's lock state without reading its password hash. - "account_lock_state": ( - 'awk -F: \'$1=="{account}"{{print ($2=="!"||$2=="*"||$2=="!!")?"locked":"unlocked"}}\' {path}' + "env_names": ( + "/usr/bin/env | /usr/bin/sed -n " + "'s/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/\\1/p' | /usr/bin/sort" ), - # Report the account record's non-secret identity fields. - "account_identity": 'awk -F: \'$1=="{account}"{{print $3":"$4":"$6":"$7}}\' {path}', - # Report supplementary group names the account joins, never their members. - "account_groups": "awk -F: '$4~/(^|,){account}(,|$)/{{print $1}}' {path}", - # Prove a path's presence and mode without reading its bytes. - "path_mode": 'if [ -e {path} ]; then stat -c "%a %U:%G" {path}; else echo absent; fi', - # Prove whether a path is readable by this identity, without printing it. - "path_readable": 'if [ -r {path} ]; then echo readable; else echo unreadable; fi', - # List environment variable names only. Values never leave the container. - "env_names": "env | sed -n 's/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/\\1/p' | sort", - # Report a JSON document's top-level key names without any of its values. - "json_keys": ( - 'python3 -c "import json,sys;print(chr(10).join(sorted(json.load(open(sys.argv[1])))))" {path}' - ), - # Report allow-listed non-secret status fields from a provider health file. "json_fields": ( - 'python3 -c "import json,sys;d=json.load(open(sys.argv[1]));' - 'print(json.dumps({{k:d.get(k) for k in sys.argv[2].split(chr(44))}}))" {path} {fields}' + '/opt/hermes/.venv/bin/python -c "import json,sys;d=json.load(open(sys.argv[1]));' + 'print(json.dumps({{k:d.get(k) for k in sys.argv[2].split(chr(44))}}))" ' + "{path} {fields}" ), - # Count non-empty lines, for authorized-key inventories. - "line_count": "awk 'NF{{n++}} END{{print n+0}}' {path}", - # Tail an append-only evidence log. The harness parses it; the container is - # not assumed to carry an interpreter. - "tail_lines": "tail -n {limit} {path}", - # Prove the worker can still read the repository. `kubectl exec` does not - # inherit the coordinator process environment, so the askpass helper is - # named explicitly rather than assumed; the helper reads runtime state and - # the credential never reaches this command line. - "git_ls_remote": "GIT_ASKPASS={askpass} git ls-remote {url} refs/heads/main", + "tail_lines": "/usr/bin/tail -n {limit} {path}", } @@ -199,16 +178,42 @@ def template_parameters(template_name: str) -> set[str]: def render_shell(template_name: str, **parameters: str) -> str: """Return a frozen shell template with validated parameters substituted.""" required = template_parameters(template_name) - template = SHELL_TEMPLATES[template_name] if set(parameters) != required: - raise PolicyError(f"shell template {template_name} expects parameters {sorted(required)}") + raise PolicyError( + f"shell template {template_name} expects parameters {sorted(required)}" + ) for key, value in parameters.items(): if not _PARAMETER_RE.fullmatch(value): raise PolicyError(f"unsafe shell parameter for {template_name}.{key}") - return template.format(**parameters) + if SECRET_PATH_RE.search(value): + raise PolicyError("shell templates may not name credential paths") + if ( + template_name == "json_fields" + and not set(parameters["fields"].split(",")) <= SAFE_JSON_FIELDS + ): + raise PolicyError("json_fields requested a credential-bearing or unknown field") + return SHELL_TEMPLATES[template_name].format(**parameters) _RENDERED: set[str] = set() +_APPROVED_PROJECTIONS: set[str] = set() +_ARMED_REF = "" +_ARMED_PULLS: set[int] = set() + + +# fmt: off +def arm_ephemeral_policy(ref: str) -> None: + global _ARMED_REF + if not re.fullmatch(r"ephemeral/hermes-handoff-acceptance/[a-z0-9][a-z0-9-]{7,63}", ref): + raise PolicyError("armed policy requires one exact ephemeral ref") + _ARMED_REF = ref + _ARMED_PULLS.clear() + + +def register_ephemeral_pull(number: int) -> None: + if not _ARMED_REF or isinstance(number, bool) or not isinstance(number, int) or number <= 0: + raise PolicyError("ephemeral pull registration is malformed") + _ARMED_PULLS.add(number) def shell(template_name: str, **parameters: str) -> tuple[str, ...]: @@ -218,24 +223,56 @@ def shell(template_name: str, **parameters: str) -> tuple[str, ...]: return ("sh", "-c", script) +def projection(value: str) -> str: + normalized = value.lower().replace(" ", "") + if not normalized.startswith("jsonpath=") or any(part in normalized for part in UNSAFE_PROJECTION_PARTS): + raise PolicyError("Kubernetes projection is not credential-safe") + if ".metadata.annotations" in normalized and not any(key in normalized for key in ( + "deployment\\.kubernetes\\.io/revision", "ai\\.bstein\\.dev/config-rev")): + raise PolicyError("Kubernetes annotations are not on the reviewed allowlist") + if ".metadata.labels" in normalized and "pod-template-hash" not in normalized: + raise PolicyError("Kubernetes labels are not on the reviewed allowlist") + if ".env" in normalized and ".value" in normalized and SAFE_ENV_VALUE not in normalized: + raise PolicyError("Kubernetes projection may expose environment values") + _APPROVED_PROJECTIONS.add(normalized) + return value +# fmt: on + + def positionals(argv: tuple[str, ...], start: int = 1) -> Iterator[str]: - """Yield positional arguments, skipping flags and the values they consume.""" + """Yield positional arguments, skipping flags and their values.""" index = start while index < len(argv): item = argv[index] if item == "--": return if item.startswith("-"): - if item in VALUE_FLAGS: - index += 2 - continue - index += 1 + index += 2 if item in VALUE_FLAGS else 1 continue yield item index += 1 -def _reject_forbidden_arguments(argv: tuple[str, ...], allow_impersonation: bool = False) -> None: +def _binary_name(value: str) -> str: + if value == GITEA_CLIENT or ( + value.startswith("/opt/coordinator/") and value.endswith("/gitea_api.py") + ): + return GITEA_CLIENT + if "/" not in value: + return value + mapped = TRUSTED_INNER_PATHS.get(value) + if mapped: + return mapped + if "/../" in value or value.endswith("/..") or "/./" in value: + raise PolicyError(f"executable path is not normalized: {value}") + if not value.startswith(TRUSTED_EXECUTABLE_ROOTS): + raise PolicyError(f"executable path is not trusted: {value}") + return value.rsplit("/", 1)[-1] + + +def _reject_forbidden_arguments( + argv: tuple[str, ...], allow_impersonation: bool +) -> None: permitted = IMPERSONATION_ARGS if allow_impersonation else () boundary = argv.index("--") if "--" in argv else len(argv) for argument in argv[:boundary]: @@ -244,47 +281,85 @@ def _reject_forbidden_arguments(argv: tuple[str, ...], allow_impersonation: bool continue if argument == forbidden or argument.startswith(f"{forbidden}="): raise PolicyError(f"forbidden argument: {forbidden}") + if argument == "--raw" or argument.startswith("--raw="): + raise PolicyError("raw API reads are forbidden") + for argument in argv: + if argument == "--dry-run" or argument.startswith("--dry-run="): + raise PolicyError("dry-run mutation commands are forbidden") def _reject_secret_paths(argv: tuple[str, ...]) -> None: - boundary = argv.index("--") if "--" in argv else len(argv) - for argument in argv[:boundary]: - if SECRET_PATH_RE.search(argument): - raise PolicyError("probes may not name a credential path outside a frozen template") + if any(SECRET_PATH_RE.search(argument) for argument in argv): + raise PolicyError("probes may not name a credential path") -def _assert_name_only_output(argv: tuple[str, ...]) -> None: - if "--output=name" in argv or "-oname" in argv: - return - for index, item in enumerate(argv[:-1]): - if item in {"-o", "--output"} and argv[index + 1] == "name": - return - raise PolicyError("credential-bearing resources may only be read with -o name") +def _resource_name(value: str) -> str: + return value.lower().split("/", 1)[0].split(".", 1)[0] + + +def _output_value(argv: tuple[str, ...]) -> str: + """Return kubectl's output argument in a single canonical form.""" + for index, item in enumerate(argv[1:], 1): + if item in {"-o", "--output"} and index + 1 < len(argv): + return argv[index + 1] + if item.startswith(("-o=", "--output=")): + return item.split("=", 1)[1] + return "" + + +def _check_projection(output: str) -> None: + """Reject full or credential-bearing Kubernetes output projections.""" + lowered = output.lower().replace(" ", "") + if lowered in {"json", "yaml", "wide"} or lowered.startswith("custom-columns"): + raise PolicyError( + "full Kubernetes object output is forbidden; use a safe projection" + ) + if lowered and not (lowered == "name" or lowered.startswith("jsonpath=")): + raise PolicyError( + "Kubernetes output must be name or an approved JSONPath projection" + ) + if lowered.startswith("jsonpath="): + if any(part in lowered for part in UNSAFE_PROJECTION_PARTS): + raise PolicyError("Kubernetes projection may expose runtime credentials") + if ".env" in lowered and SAFE_ENV_VALUE not in lowered: + # Environment names are safe; arbitrary `.value` reads are not. + without_names = lowered.replace("{.name}", "") + if ".value" in without_names: + raise PolicyError("Kubernetes projection may expose environment values") + if lowered not in _APPROVED_PROJECTIONS: + raise PolicyError( + "Kubernetes JSONPath projection is not on the reviewed allowlist" + ) def _check_kubectl(argv: tuple[str, ...], mode: str, allow_impersonation: bool) -> None: words = list(positionals(argv)) - subcommand = next((word for word in words if word in KUBECTL_SUBCOMMANDS), None) - if subcommand is None: + if not words or words[0] not in KUBECTL_SUBCOMMANDS: raise PolicyError("kubectl subcommand is outside the read-only boundary") - if subcommand != words[0]: - raise PolicyError("kubectl subcommand must precede its resource arguments") - resource = words[1].split(".")[0] if len(words) > 1 else "" - if subcommand in {"get", "describe"} and resource in NAME_ONLY_RESOURCES: - _assert_name_only_output(argv) - if subcommand in KUBECTL_READ_SUBCOMMANDS: + subcommand = words[0] + if subcommand == "auth": + if len(words) < 2 or words[1] not in {"can-i", "whoami"}: + raise PolicyError("kubectl auth is limited to can-i and whoami") return - if subcommand in KUBECTL_EXEC_SUBCOMMANDS: + if subcommand == "exec": if "--" not in argv: raise PolicyError("kubectl exec requires an explicit -- command boundary") inner = argv[argv.index("--") + 1 :] if not inner: raise PolicyError("kubectl exec requires an inner command") + if inner[0] not in TRUSTED_INNER_PATHS: + raise PolicyError( + "kubectl exec inner command must use a fixed trusted path" + ) check_argv(inner, mode, allow_impersonation=True) return - if any(flag in argv for flag in DRY_RUN_FLAGS): - return - raise PolicyError(f"kubectl {subcommand} requires a dry run") + resources = words[1].split(",") if len(words) > 1 else [] + if any(_resource_name(resource) in SECRET_RESOURCES for resource in resources): + raise PolicyError("Secret objects may not be read by the harness") + output = _output_value(argv) + if subcommand == "get" and not output: + raise PolicyError("kubectl get requires a reviewed output projection") + _check_projection(output) def _first_positional(binary: str, argv: tuple[str, ...]) -> str: @@ -294,21 +369,39 @@ def _first_positional(binary: str, argv: tuple[str, ...]) -> str: return word -def _check_read_only_tool(binary: str, argv: tuple[str, ...], allowed: frozenset[str]) -> None: +def _check_read_only_tool( + binary: str, argv: tuple[str, ...], allowed: frozenset[str] +) -> None: subcommand = _first_positional(binary, argv) if subcommand not in allowed: raise PolicyError(f"{binary} {subcommand} is outside the read-only boundary") def _check_git(argv: tuple[str, ...], mode: str) -> None: + unsafe = False + for item in argv[1:]: + if item == "-C" or item.startswith(("--git-dir", "--work-tree", "--namespace")): + unsafe = True + if item == "-c" or (item.startswith("-c") and len(item) > 2): + unsafe = True + if any( + item == flag or item.startswith(f"{flag}=") for flag in GIT_UNSAFE_FLAGS[1:] + ): + unsafe = True + if unsafe: + raise PolicyError("git configuration and helper overrides are forbidden") subcommand = _first_positional("git", argv) - if subcommand == "config" and not {"--get", "--get-all", "--list"} & set(argv): - raise PolicyError("git config may only read") if subcommand in GIT_READ_SUBCOMMANDS: return - if subcommand in GIT_ARMED_SUBCOMMANDS and mode == ARMED: + if subcommand == "push" and mode == ARMED: if GIT_FORCE_FLAGS & set(argv): raise PolicyError("git push may not force or fan out across refs") + words = list(positionals(argv)) + if len(words) != 3 or words[1] != EXPECTED_REMOTE: + raise PolicyError("armed git push is pinned to origin and one refspec") + destination = words[2].partition(":")[2] + if not _ARMED_REF or destination != f"refs/heads/{_ARMED_REF}": + raise PolicyError("armed git push is not the preflighted ephemeral ref") return raise PolicyError(f"git {subcommand} is not permitted in {mode} mode") @@ -318,52 +411,76 @@ def _check_hermes(argv: tuple[str, ...]) -> None: if not words: raise PolicyError("hermes requires a subcommand") if words[:2] not in HERMES_READ_COMMANDS and words[:1] not in HERMES_READ_COMMANDS: - raise PolicyError(f"hermes {' '.join(words[:2])} is outside the read-only boundary") + raise PolicyError( + f"hermes {' '.join(words[:2])} is outside the read-only boundary" + ) def _check_gitea(argv: tuple[str, ...], mode: str) -> None: if len(argv) < 3: raise PolicyError("gitea_api.py requires a method and a path") - method = argv[1].upper() + method, path = argv[1].upper(), argv[2] + if not path.startswith("/api/v1/") or not _SAFE_API_PATH_RE.fullmatch(path): + raise PolicyError("gitea_api.py requires a bounded API path") + if path != "/api/v1/user" and not path.startswith(f"/api/v1/repos/{EXPECTED_REPO}"): + raise PolicyError("gitea_api.py is pinned to the Atlas titan-iac repository") if method in GITEA_READ_METHODS: return - if method in GITEA_ARMED_METHODS and mode == ARMED: - return + repo_root = f"/api/v1/repos/{EXPECTED_REPO}" + if mode == ARMED: + fields = [ + argv[index + 1] for index, item in enumerate(argv[:-1]) if item == "--field" + ] + if method == "POST" and path == f"{repo_root}/pulls" and _ARMED_REF: + required = { + "title=WIP: Hermes handoff acceptance ephemeral probe", + f"head={_ARMED_REF}", + f"base={EXPECTED_BASE}", + "body=Ephemeral acceptance probe. Closed and deleted by the harness.", + } + if len(fields) == len(required) and set(fields) == required: + return + match = re.fullmatch(rf"{re.escape(repo_root)}/pulls/([1-9][0-9]*)", path) + if ( + method == "PATCH" + and match + and int(match.group(1)) in _ARMED_PULLS + and fields == ["state=closed"] + ): + return + if ( + method == "DELETE" + and _ARMED_REF + and path == f"{repo_root}/branches/{_ARMED_REF}" + and len(argv) == 3 + ): + return raise PolicyError(f"gitea_api.py {method} is not permitted in {mode} mode") def _check_shell(argv: tuple[str, ...]) -> None: - if len(argv) != 3 or argv[1] != "-c": + if len(argv) != 3 or argv[1] != "-c" or argv[2] not in _RENDERED: raise PolicyError("sh is only reachable as sh -c ") - if argv[2] not in _RENDERED: - raise PolicyError("sh -c script is not a frozen template rendering") def check_argv( argv: tuple[str, ...], mode: str = READ_ONLY, allow_impersonation: bool = False ) -> None: - """Raise :class:`PolicyError` unless ``argv`` is safe to run in ``mode``. - - ``allow_impersonation`` is set only for the inner command of a - ``kubectl exec``. Hermes probing its own refusal to impersonate is evidence; - the operator impersonating Hermes is not, so `--as` stays unreachable from - the outer, operator-side argv. - """ + """Raise :class:`PolicyError` unless ``argv`` is safe in ``mode``.""" if mode not in MODES: raise PolicyError(f"unknown mode: {mode}") if not argv: raise PolicyError("empty command") - binary = argv[0] if argv[0] == GITEA_CLIENT else argv[0].rsplit("/", 1)[-1] + binary = _binary_name(argv[0]) if binary in FORBIDDEN_BINARIES: raise PolicyError(f"forbidden binary: {binary}") if binary not in ALLOWED_BINARIES: raise PolicyError(f"binary is not on the harness allowlist: {binary}") _reject_forbidden_arguments(argv, allow_impersonation) + _reject_secret_paths(argv) if binary == "sh": _check_shell(argv) - return - _reject_secret_paths(argv) - if binary == "kubectl": + elif binary == "kubectl": _check_kubectl(argv, mode, allow_impersonation) elif binary == "flux": _check_read_only_tool("flux", argv, FLUX_READ_SUBCOMMANDS) diff --git a/scripts/ops/hermes_handoff_redaction.py b/scripts/ops/hermes_handoff_redaction.py index f48344a5..f7b97f4c 100644 --- a/scripts/ops/hermes_handoff_redaction.py +++ b/scripts/ops/hermes_handoff_redaction.py @@ -9,9 +9,10 @@ harness therefore also never asks for a secret in the first place. Probes read names, shapes, and status fields; they never read environment values, token files, private keys, cookies, or Vault responses. -Bounding is part of the same barrier. An unbounded capture is how a debug dump -or a stack trace carrying an authorization header ends up in an artifact, so -captures are truncated at a byte budget before they are screened. +Bounding is part of the same barrier. Screening always precedes truncation so a +credential that crosses the byte boundary cannot leave a useful prefix behind. +The process runner additionally discards all captured bytes when a live stream +exceeds its cap; the fixed truncation marker is the only recorded output. """ from __future__ import annotations @@ -62,12 +63,13 @@ SENSITIVE_KEY_EXACT = frozenset( # often than it is a credential, so it is screened by value shape and by the # token patterns rather than by name. The compound forms above still match. -# Values that cannot hold a credential: booleans, numbers, and anything too -# short to be one. Screening them corrupts structured evidence — a redacted -# `automountServiceAccountToken` is exactly the fact a reader needs — without -# protecting anything. +# Ordinary prose values shorter than this threshold are not independently +# treated as secret-shaped. Values under sensitive keys are screened at every +# length, including numeric values. MIN_SECRET_LENGTH = 8 -NON_SECRET_LITERALS = frozenset({"true", "false", "null", "none", "yes", "no", "0", "1", ""}) +NON_SECRET_LITERALS = frozenset( + {"true", "false", "null", "none", "yes", "no", "0", "1", ""} +) _KEY_NOISE_RE = re.compile(r"[^a-z0-9]+") # Ordered: structural matches first so a nested token is not partially rewritten @@ -77,20 +79,23 @@ _PATTERNS: tuple[re.Pattern[str], ...] = ( r"-----BEGIN[ A-Z0-9-]{0,64}-----.*?-----END[ A-Z0-9-]{0,64}-----", re.DOTALL, ), + re.compile(r"-----BEGIN[ A-Z0-9-]{0,64}-----.*", re.DOTALL), re.compile(r"(?i)\b(?:authorization|proxy-authorization)\s*:\s*\S.*"), re.compile(r"(?i)\b(?:set-)?cookie\s*:\s*\S.*"), re.compile(r"(?i)\b(?:bearer|basic|token)\s+[A-Za-z0-9+/_=.:~-]{8,}"), re.compile(r"(?i)\b[a-z][a-z0-9+.-]{1,20}://[^\s/@:]{1,128}:[^\s/@]{1,256}@"), - re.compile(r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}"), + re.compile( + r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}" + ), re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}"), - re.compile(r"(? str: @@ -136,7 +143,7 @@ def _is_high_entropy(candidate: str) -> bool: any(character in "+/_=-" for character in candidate), ) ) - return classes >= 3 + return classes >= 4 def could_hold_secret(value: str) -> bool: @@ -153,13 +160,16 @@ def could_hold_secret(value: str) -> bool: def _redact_assignment(match: re.Match[str]) -> str: value = match.group("value") - if not is_sensitive_key(match.group("key")) or not could_hold_secret(value): + stripped = value.strip().strip("\"'").lower() + if not is_sensitive_key(match.group("key")) or stripped in NON_SECRET_LITERALS: return match.group(0) # Preserve the original quoting so screened JSON stays parsable JSON. quoted = value.strip()[:1] in {'"', "'"} replacement = f'"{REDACTED}"' if quoted else REDACTED key_quote = match.group("quote") - return f"{key_quote}{match.group('key')}{key_quote}{match.group('gap')}{replacement}" + return ( + f"{key_quote}{match.group('key')}{key_quote}{match.group('gap')}{replacement}" + ) def _redact_entropy(match: re.Match[str]) -> str: @@ -183,32 +193,42 @@ def bound(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: encoded = text.encode("utf-8", "replace") if len(encoded) <= max_bytes: return (text, False) - kept = encoded[:max_bytes].decode("utf-8", "ignore") - return (f"{kept}\n{TRUNCATION_NOTE}", True) + marker = TRUNCATION_NOTE.encode()[:max_bytes] + content_budget = max( + 0, max_bytes - len(marker) - (1 if len(marker) < max_bytes else 0) + ) + kept = encoded[:content_budget].decode("utf-8", "ignore") + separator = "\n" if kept and len(marker) < max_bytes else "" + return (f"{kept}{separator}{marker.decode()}", True) def safe_text(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: - """Bound then redact, in that order, so a huge dump cannot outrun screening.""" - bounded, truncated = bound(text, max_bytes) - return (redact(bounded), truncated) + """Redact then bound, so truncation never exposes a credential prefix.""" + return bound(redact(text), max_bytes) def scrub(value: Any, max_bytes: int = DEFAULT_MAX_BYTES) -> Any: """Recursively bound and redact a JSON-shaped value. - Mapping keys are preserved (names are evidence); values under a sensitive - key are dropped outright rather than pattern-screened, because a short - secret would survive screening. + Mapping keys are themselves screened; values under a sensitive key are + dropped outright rather than pattern-screened, because a short secret + would survive screening. """ if isinstance(value, str): return safe_text(value, max_bytes)[0] if isinstance(value, dict): - return { - str(key): REDACTED - if is_sensitive_key(str(key)) and could_hold_secret(str(item)) - else scrub(item, max_bytes) - for key, item in value.items() - } + screened: dict[str, Any] = {} + for key, item in value.items(): + raw_key = str(key) + safe_key = safe_text(raw_key, max_bytes)[0] + sensitive_value = ( + is_sensitive_key(raw_key) + and item is not None + and not isinstance(item, bool) + and item != "" + ) + screened[safe_key] = REDACTED if sensitive_value else scrub(item, max_bytes) + return screened if isinstance(value, (list, tuple)): return [scrub(item, max_bytes) for item in value] return value diff --git a/scripts/ops/hermes_handoff_rules.py b/scripts/ops/hermes_handoff_rules.py index cf9b5b97..5cca2c5a 100644 --- a/scripts/ops/hermes_handoff_rules.py +++ b/scripts/ops/hermes_handoff_rules.py @@ -15,7 +15,7 @@ from dataclasses import dataclass, field from typing import Any from hermes_handoff_exec import Outcome -from hermes_handoff_model import NOT_RUN, CheckSpec +from hermes_handoff_model import NOT_RUN, STATUSES, CheckSpec DENIAL_MARKERS = ( "forbidden", @@ -28,9 +28,7 @@ DENIAL_MARKERS = ( "cannot delete", "cannot patch", "not permitted", - "no such route", - "404 not found", - "403", + "gitea api returned http 403", ) @@ -57,7 +55,9 @@ def evaluator(name: str) -> Callable[[Evaluator], Evaluator]: return register -def missing_steps(spec: CheckSpec, outcomes: Mapping[str, Outcome]) -> Evaluation | None: +def missing_steps( + spec: CheckSpec, outcomes: Mapping[str, Outcome] +) -> Evaluation | None: """Return NOT_RUN when a required step never produced usable evidence.""" for step in spec.steps: if step.optional: @@ -65,6 +65,8 @@ def missing_steps(spec: CheckSpec, outcomes: Mapping[str, Outcome]) -> Evaluatio outcome = outcomes.get(step.key) if outcome is None: return Evaluation(NOT_RUN, f"step {step.key} was never executed") + if outcome.truncated: + return Evaluation(NOT_RUN, f"step {step.key} output was truncated") if not outcome.ran: return Evaluation(NOT_RUN, f"step {step.key} did not run: {outcome.error}") return None @@ -88,6 +90,11 @@ def is_list(value: Any) -> bool: return isinstance(value, Sequence) and not isinstance(value, (str, bytes)) +def exact_equal(actual: Any, expected: Any) -> bool: + """Compare JSON values without Python's bool/int type confusion.""" + return type(actual) is type(expected) and actual == expected + + def dotted(payload: Any, path: str) -> Any: """Return a value addressed by a dotted path. @@ -110,7 +117,9 @@ def dotted(payload: Any, path: str) -> Any: raise KeyError(path) if chunk == "": remainder = ".".join(segments[position + 1 :]) - return [dotted(item, remainder) if remainder else item for item in current] + return [ + dotted(item, remainder) if remainder else item for item in current + ] offset = int(chunk) if offset >= len(current): raise KeyError(path) @@ -118,17 +127,34 @@ def dotted(payload: Any, path: str) -> Any: return current -def parse_json(outcome: Outcome) -> Any: - """Parse a step's stdout as JSON, raising ``ValueError`` when it is not. - - Control characters are tolerated. Task bodies and session titles routinely - carry raw newlines and tabs, and refusing to read a report because a human - pasted a tab into a Kanban card would be a false NO_GO. - """ - text = outcome.stdout.strip() +def strict_json(text: str) -> Any: + """Decode strict JSON, rejecting constants and duplicate object keys.""" + text = text.strip() if not text: raise ValueError("empty output") - return json.loads(text, strict=False) + + def reject_constant(value: str) -> None: + raise ValueError(f"non-finite JSON constant {value}") + + def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key {key!r}") + result[key] = value + return result + + return json.loads( + text, + strict=True, + parse_constant=reject_constant, + object_pairs_hook=unique_object, + ) + + +def parse_json(outcome: Outcome) -> Any: + """Parse one outcome as strict JSON.""" + return strict_json(outcome.stdout) def lines(outcome: Outcome) -> list[str]: @@ -136,7 +162,9 @@ def lines(outcome: Outcome) -> list[str]: return [line.strip() for line in outcome.stdout.splitlines() if line.strip()] -def parsed_step(spec: CheckSpec, outcomes: dict[str, Outcome]) -> tuple[Any, str] | Evaluation: +def parsed_step( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> tuple[Any, str] | Evaluation: """Return the parsed payload of ``expect['step']`` or the NOT_RUN to report.""" key = spec.expect["step"] outcome = outcomes[key] @@ -153,7 +181,19 @@ def evaluate(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: rule = EVALUATORS.get(spec.rule) if rule is None: return Evaluation(NOT_RUN, f"unknown evaluator: {spec.rule}") + truncated = sorted(key for key, outcome in outcomes.items() if outcome.truncated) + if truncated: + return Evaluation( + NOT_RUN, f"truncated evidence from step(s): {', '.join(truncated)}" + ) try: - return rule(spec, outcomes) + result = rule(spec, outcomes) except Exception as exc: # noqa: BLE001 - a harness bug must never read as PASS - return Evaluation(NOT_RUN, f"evaluator {spec.rule} raised {type(exc).__name__}: {exc}") + return Evaluation( + NOT_RUN, f"evaluator {spec.rule} raised {type(exc).__name__}: {exc}" + ) + if result.status not in STATUSES: + return Evaluation( + NOT_RUN, f"evaluator {spec.rule} returned unknown status {result.status!r}" + ) + return result diff --git a/scripts/ops/hermes_handoff_run.py b/scripts/ops/hermes_handoff_run.py index cdfaf3c2..a3b21cd7 100644 --- a/scripts/ops/hermes_handoff_run.py +++ b/scripts/ops/hermes_handoff_run.py @@ -4,8 +4,8 @@ Three things happen before any check runs, and each of them can stop the run on its own: -* the catalog is validated structurally — every rule exists, every deny check - carries a real attempt, no operator-side step can impersonate; +* the catalog is validated structurally — every rule exists, every live-denial + evaluator carries a read-only attempt, no operator-side step can impersonate; * each vantage is resolved and asked who it is; and * the operator and in-pod identities are compared. If they are the same principal there is only one vantage, the dual-vantage evidence would be a @@ -19,7 +19,7 @@ which is exactly the fail-closed answer. from __future__ import annotations -import json +import re from collections.abc import Iterable, Mapping, Sequence from hermes_handoff_catalog import ( @@ -48,7 +48,14 @@ from hermes_handoff_model import ( VantageRecord, utc_now, ) -from hermes_handoff_policy import IMPERSONATION_ARGS +from hermes_handoff_policy import ( + EXPECTED_REMOTE, + EXPECTED_REPO, + IMPERSONATION_ARGS, + PolicyError, + check_argv, + projection, +) UNRECORDED = "[output not recorded]" @@ -66,41 +73,129 @@ def build_catalog(targets: Targets) -> list[CheckSpec]: def validate_catalog(specs: Sequence[CheckSpec]) -> list[str]: """Return the structural problems that make a catalog unsafe to run.""" problems: list[str] = [] + if not specs: + return ["catalog is empty"] seen: set[str] = set() + fake_operator = operator_vantage() + fake_pod = pod_vantage("hermes", "acceptance-probe", "probe", fake_operator) for spec in specs: if spec.id in seen: problems.append(f"{spec.id}: duplicate check id") seen.add(spec.id) if spec.rule not in EVALUATORS and spec.scope != EPHEMERAL: problems.append(f"{spec.id}: unknown rule {spec.rule!r}") + step_names: set[str] = set() for step in spec.steps: + if step.key in step_names: + problems.append(f"{spec.id}: duplicate step key {step.key!r}") + step_names.add(step.key) if step.vantage not in VANTAGE_NAMES: - problems.append(f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}") + problems.append( + f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}" + ) if step.kind not in STEP_KINDS: - problems.append(f"{spec.id}: step {step.key} has unknown kind {step.kind!r}") + problems.append( + f"{spec.id}: step {step.key} has unknown kind {step.kind!r}" + ) if step.vantage == OPERATOR and set(step.argv) & set(IMPERSONATION_ARGS): - problems.append(f"{spec.id}: step {step.key} impersonates from the operator vantage") - if spec.rule == "denied" and not any(step.kind == ATTEMPT for step in spec.steps): - problems.append(f"{spec.id}: a deny check needs a real attempt, not only a review") + problems.append( + f"{spec.id}: step {step.key} impersonates from the operator vantage" + ) + try: + addressed = ( + fake_operator if step.vantage == OPERATOR else fake_pod + ).wrap(step.argv) + check_argv(addressed) + except PolicyError as exc: + problems.append(f"{spec.id}: step {step.key} violates policy: {exc}") + if spec.rule == "denied" and not any( + step.kind == ATTEMPT for step in spec.steps + ): + problems.append( + f"{spec.id}: a deny check needs a real attempt, not only a review" + ) + return problems + + +def validate_targets(targets: Targets) -> list[str]: + """Return release-input problems before a runner or network call exists.""" + problems: list[str] = [] + sha = re.compile(r"[0-9a-f]{40}\Z") + image = re.compile( + r"[^@\s]+:git-([0-9a-f]{40})-build-[1-9][0-9]*@sha256:[0-9a-f]{64}\Z" + ) + if targets.repo != EXPECTED_REPO or targets.remote != EXPECTED_REMOTE: + problems.append( + "repository and remote must be the fixed Atlas titan-iac origin" + ) + if ( + targets.reviewed_pr_number != 19 + or targets.reviewed_head_ref != "feature/hermes-full-handoff-acceptance" + ): + problems.append( + "reviewed PR identity must remain fixed to draft PR #19 and its existing branch" + ) + for label, value in ( + ("baseline commit", targets.baseline_commit), + ("remote main SHA", targets.remote_main_sha), + ("reviewed head SHA", targets.reviewed_head_sha), + ("build SHA", targets.build_sha), + ): + if not sha.fullmatch(value): + problems.append(f"{label} must be an exact lowercase 40-character SHA") + image_match = image.fullmatch(targets.agent_image) + if not image_match or image_match.group(1) != targets.build_sha: + problems.append("agent image tag/digest must bind the exact build SHA") + if targets.build_sha != targets.remote_main_sha: + problems.append("build SHA must equal the exact release main SHA") + if not re.fullmatch(r"[1-9][0-9]{0,8}", str(targets.deployment_revision)): + problems.append("deployment revision must be a positive integer string") + if targets.node_count != 3 or targets.pool_replicas != 3: + problems.append("node and pool counts are fixed at exactly three") + if not targets.expect_telegram_sessions or not targets.chat_config_revision.strip(): + problems.append( + "Telegram continuity and its chat config revision are mandatory" + ) + if set(targets.pool_worker_env) != {"HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL"}: + problems.append("pool assignment evidence must include worker node and ordinal") + heads = dict(targets.dependency_heads) + if len(heads) != len(targets.dependency_heads) or set(heads) != set( + targets.dependency_pull_requests + ): + problems.append("every dependency PR must have one exact current head") + elif any(not sha.fullmatch(value) for value in heads.values()): + problems.append("dependency heads must be exact lowercase 40-character SHAs") + if ( + not isinstance(targets.max_evidence_age_seconds, int) + or targets.max_evidence_age_seconds <= 0 + ): + problems.append("maximum evidence age must be a positive integer") return problems def resolve_identity(runner: Runner, vantage: Vantage) -> tuple[str, str]: """Return the authenticated username of a vantage and any failure detail.""" - outcome = runner.run(("kubectl", "auth", "whoami", "-o", "json"), vantage) + outcome = runner.run( + ( + "kubectl", + "auth", + "whoami", + "-o", + projection("jsonpath={.status.userInfo.username}"), + ), + vantage, + ) if not outcome.ok: return ("", outcome.error or outcome.combined[:200]) - try: - payload = json.loads(outcome.stdout) - except ValueError as exc: - return ("", f"unparsable identity: {exc}") - username = ((payload.get("status") or {}).get("userInfo") or {}).get("username") - if not isinstance(username, str) or not username: + username = outcome.stdout.strip() + if not username or any(character.isspace() for character in username): return ("", "identity response carried no username") return (username, "") -def _first_pod(runner: Runner, operator: Vantage, namespace: str, selector: str) -> tuple[str, str]: +def _first_pod( + runner: Runner, operator: Vantage, namespace: str, selector: str +) -> tuple[str, str]: outcome = runner.run( ( "kubectl", @@ -119,14 +214,21 @@ def _first_pod(runner: Runner, operator: Vantage, namespace: str, selector: str) ) if not outcome.ok: return ("", outcome.error or outcome.combined[:200]) - names = [line.strip().removeprefix("pod/") for line in outcome.stdout.splitlines() if line.strip()] + names = [ + line.strip().removeprefix("pod/") + for line in outcome.stdout.splitlines() + if line.strip() + ] if not names: return ("", f"no running pod matches {selector}") return (sorted(names)[0], "") def resolve_vantages( - runner: Runner, targets: Targets, kubeconfig: str | None = None, context: str | None = None + runner: Runner, + targets: Targets, + kubeconfig: str | None = None, + context: str | None = None, ) -> tuple[dict[str, Vantage], list[VantageRecord]]: """Resolve every vantage the catalog names, recording what each one is.""" operator = operator_vantage(kubeconfig, context) @@ -145,23 +247,37 @@ def resolve_vantages( return (vantages, records) for name, selector, container, description in ( - (SELF, f"app={targets.agent_deployment}", targets.agent_container, "the Hermes agent itself"), + ( + SELF, + f"app={targets.agent_deployment}", + targets.agent_container, + "the Hermes agent itself", + ), ( SWITCHYARD, f"app={targets.switchyard_deployment}", targets.switchyard_container, "the Switchyard routing evidence log", ), - (NODE, f"app={targets.node_daemonset}", targets.node_container, "a node hardening probe"), + ( + NODE, + f"app={targets.node_daemonset}", + targets.node_container, + "a node hardening probe", + ), ): pod, failure = _first_pod(runner, operator, targets.namespace, selector) if not pod: records.append( - VantageRecord(name=name, description=description, available=False, detail=failure) + VantageRecord( + name=name, description=description, available=False, detail=failure + ) ) continue vantage = pod_vantage(targets.namespace, pod, container, operator) - identity, identity_detail = resolve_identity(runner, vantage) if name == SELF else ("", "") + identity, identity_detail = ( + resolve_identity(runner, vantage) if name == SELF else ("", "") + ) records.append( VantageRecord( name=name, @@ -175,11 +291,21 @@ def resolve_vantages( chat_pod = f"{targets.chat_statefulset}-{targets.chat_ordinal}" present = runner.run( - ("kubectl", "--namespace", targets.namespace, "get", f"pod/{chat_pod}", "-o", "name"), + ( + "kubectl", + "--namespace", + targets.namespace, + "get", + f"pod/{chat_pod}", + "-o", + "name", + ), operator, ) if present.ok and present.stdout.strip(): - vantages[CHAT] = pod_vantage(targets.namespace, chat_pod, targets.chat_container, operator) + vantages[CHAT] = pod_vantage( + targets.namespace, chat_pod, targets.chat_container, operator + ) records.append( VantageRecord( name=CHAT, @@ -193,7 +319,9 @@ def resolve_vantages( name=CHAT, description=f"the chat surface ({targets.namespace}/{chat_pod})", available=False, - detail=present.error or present.combined[:200] or "chat tenant pod not found", + detail=present.error + or present.combined[:200] + or "chat tenant pod not found", ) ) return (vantages, records) @@ -206,10 +334,14 @@ def vantage_problems(records: Iterable[VantageRecord]) -> list[str]: hermes = by_name.get(SELF) problems: list[str] = [] if operator is None or not operator.available: - problems.append("the operator vantage is unavailable; no evidence can be collected") + problems.append( + "the operator vantage is unavailable; no evidence can be collected" + ) return problems if hermes is None or not hermes.available: - problems.append("the in-pod Hermes vantage is unavailable; self-probe evidence is missing") + problems.append( + "the in-pod Hermes vantage is unavailable; self-probe evidence is missing" + ) return problems if not hermes.identity: problems.append("the in-pod vantage did not report an identity") @@ -218,6 +350,10 @@ def vantage_problems(records: Iterable[VantageRecord]) -> list[str]: "the operator and in-pod vantages authenticate as the same principal " f"({operator.identity}); run the harness with a separate operator kubeconfig" ) + for name, label in ((SWITCHYARD, "Switchyard"), (CHAT, "chat/Telegram")): + record = by_name.get(name) + if record is None or not record.available: + problems.append(f"the required {label} vantage is unavailable") return problems @@ -231,7 +367,9 @@ def run_check( vantage = vantages.get(step.vantage) if vantage is None: outcome = Outcome( - argv=step.argv, vantage=step.vantage, error=f"vantage {step.vantage} is unavailable" + argv=step.argv, + vantage=step.vantage, + error=f"vantage {step.vantage} is unavailable", ) else: outcome = runner.run(step.argv, vantage, step.max_bytes) @@ -300,18 +438,27 @@ def build_report( harness_errors=problems, ) armed_ids = {result.spec.id for result in extra_results} - report.results = [ - *run_catalog(runner, [spec for spec in specs if spec.id not in armed_ids], vantages), - *extra_results, - ] + runnable = [spec for spec in specs if spec.id not in armed_ids] + if problems: + report.results = [ + CheckResult(spec=spec, status=NOT_RUN, reason="harness preflight failed") + for spec in runnable + ] + report.results.extend(extra_results) + else: + report.results = [*run_catalog(runner, runnable, vantages), *extra_results] report.results.sort(key=lambda result: result.spec.id) if runner.remaining_seconds <= 0: - report.harness_errors.append("the run deadline expired before every check completed") + report.harness_errors.append( + "the run deadline expired before every check completed" + ) report.finished_at = utc_now() return report -def unavailable_report(mode: str, started_at: str, reason: str, targets: Targets) -> Report: +def unavailable_report( + mode: str, started_at: str, reason: str, targets: Targets +) -> Report: """Return a NO_GO report for a run that could not start at all.""" return Report( mode=mode, diff --git a/testing/quality_contract.json b/testing/quality_contract.json index e8793d84..9c3de171 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -14,6 +14,21 @@ } ], "managed_modules": [ + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -25,6 +40,7 @@ "testing/quality_docs.py", "testing/quality_hygiene.py", "testing/quality_coverage.py", + "testing/quality_handoff_mutation.py", "testing/quality_gate.py", "ci/tests/glue/test_ariadne_schedules.py", "ci/tests/glue/test_glue_metrics.py", @@ -35,6 +51,21 @@ "testing/tests/test_quality_gate.py" ], "lint_paths": [ + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -57,6 +88,7 @@ "junit": "build/junit-unit.xml", "coverage_sources": [ "ci/scripts", + "scripts/ops", "scripts.render.dashboards_render_atlas", "services/mailu/scripts", "testing" @@ -118,7 +150,26 @@ ], "hygiene": { "max_lines": 500, + "legacy_line_exceptions": { + "scripts/tests/test_dashboards_render_atlas.py": { + "lines": 516, + "sha256": "afe4923391b157d03bcf0899208f77eeab79a2762da3575a365530b3074c5f25" + }, + "testing/tests/test_hermes_chat_quality.py": { + "lines": 2242, + "sha256": "71e9cafd14460b5132d07dd951514756d091009c23db7703db1575b05e34e75f" + }, + "testing/tests/test_hermes_cli_lanes.py": { + "lines": 1971, + "sha256": "992bf3788c33252e37a695f5307c31b1e8c0646e00a715e3e55f6901788abb62" + }, + "testing/tests/test_hermes_coordinator.py": { + "lines": 510, + "sha256": "290240cf391abc755eaa0abb12d247ad1718e7c9295a2092f349a7eaba4bfe55" + } + }, "line_limit_globs": [ + "scripts/ops/hermes_handoff_*.py", "testing/**/*.py", "ci/scripts/*.py", "ci/tests/**/*.py", @@ -167,7 +218,44 @@ }, "coverage": { "minimum_percent": 95.0, + "minimum_branch_percent": 95.0, + "branch_tracked_files": [ + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", + "ci/scripts/semgrep_report.py", + "testing/quality_coverage.py", + "testing/quality_handoff_mutation.py", + "testing/quality_hygiene.py" + ], "tracked_files": [ + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -178,6 +266,7 @@ "testing/quality_docs.py", "testing/quality_hygiene.py", "testing/quality_coverage.py", + "testing/quality_handoff_mutation.py", "testing/quality_gate.py" ] } diff --git a/testing/quality_coverage.py b/testing/quality_coverage.py index 78d6649f..04355e51 100644 --- a/testing/quality_coverage.py +++ b/testing/quality_coverage.py @@ -3,23 +3,31 @@ from __future__ import annotations import xml.etree.ElementTree as ET +from dataclasses import dataclass from pathlib import Path from typing import Any -def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: - """Load per-file line-rate percentages from a Cobertura XML report.""" +@dataclass(frozen=True) +class CoverageRates: + """Line and branch percentages reported for one source file.""" + + line: float + branch: float | None + + +def _load_percentages(xml_path: Path, root: Path) -> dict[str, CoverageRates]: + """Load per-file line and branch percentages from a Cobertura report.""" tree = ET.parse(xml_path) xml_root = tree.getroot() source_roots = [ - Path(node.text) - for node in xml_root.findall("./sources/source") - if node.text + Path(node.text) for node in xml_root.findall("./sources/source") if node.text ] - percentages: dict[str, float] = {} + percentages: dict[str, CoverageRates] = {} for class_node in xml_root.findall(".//class"): filename = class_node.attrib.get("filename") line_rate = class_node.attrib.get("line-rate") + branch_rate = class_node.attrib.get("branch-rate") if not filename or line_rate is None: continue normalized = filename.replace("\\", "/") @@ -32,7 +40,10 @@ def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: if candidate.exists(): key = candidate.relative_to(root).as_posix() break - percentages[key] = float(line_rate) * 100.0 + percentages[key] = CoverageRates( + line=float(line_rate) * 100.0, + branch=float(branch_rate) * 100.0 if branch_rate is not None else None, + ) return percentages @@ -46,18 +57,33 @@ def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str] return [f"coverage xml missing: {xml_path.relative_to(root)}"] percentages = _load_percentages(xml_path, root) - minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0)) + coverage_contract = contract.get("coverage", {}) + minimum = float(coverage_contract.get("minimum_percent", 95.0)) + branch_minimum = float(coverage_contract.get("minimum_branch_percent", 95.0)) + branch_tracked = { + path.replace("\\", "/") + for path in coverage_contract.get("branch_tracked_files", ()) + } issues: list[str] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is None: + rates = percentages.get(normalized) + if rates is None: issues.append(f"coverage missing for tracked file: {relative_path}") continue - if percent + 1e-9 < minimum: + if rates.line + 1e-9 < minimum: issues.append( - f"coverage below {minimum:.1f}%: {relative_path} ({percent:.1f}%)" + f"coverage below {minimum:.1f}%: {relative_path} ({rates.line:.1f}%)" + ) + if normalized not in branch_tracked: + continue + if rates.branch is None: + issues.append(f"branch coverage missing for tracked file: {relative_path}") + elif rates.branch + 1e-9 < branch_minimum: + issues.append( + f"branch coverage below {branch_minimum:.1f}%: " + f"{relative_path} ({rates.branch:.1f}%)" ) return issues @@ -77,9 +103,9 @@ def compute_workspace_line_coverage( samples: list[float] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is not None: - samples.append(percent) + rates = percentages.get(normalized) + if rates is not None: + samples.append(rates.line) if not samples: return 0.0 return round(sum(samples) / len(samples), 3) diff --git a/testing/quality_handoff_mutation.py b/testing/quality_handoff_mutation.py new file mode 100644 index 00000000..a0c63fa0 --- /dev/null +++ b/testing/quality_handoff_mutation.py @@ -0,0 +1,209 @@ +"""Deterministic scored mutation gate for Hermes handoff safety invariants.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class Mutation: + """One source mutation and the isolated probe that must kill it.""" + + name: str + path: str + old: str + new: str + probe: str + + +@dataclass(frozen=True) +class MutationReport: + """Scored result; any invalid, surviving, or baseline-broken case is an issue.""" + + total: int + killed: int + issues: tuple[str, ...] + + @property + def score(self) -> float: + return 100.0 if not self.total else round(self.killed * 100.0 / self.total, 2) + + +MUTATIONS = ( + Mutation( + "policy-allows-git-fetch", + "hermes_handoff_policy.py", + ' "for-each-ref",\n', + ' "fetch",\n "for-each-ref",\n', + "from hermes_handoff_policy import *\n" + "try: check_argv(('git','fetch','origin'))\n" + "except PolicyError: raise SystemExit(0)\n" + "raise SystemExit(1)\n", + ), + Mutation( + "redaction-truncates-before-screening", + "hermes_handoff_redaction.py", + " return bound(redact(text), max_bytes)\n", + " return bound(text, max_bytes)\n", + "from hermes_handoff_redaction import safe_text\n" + "assert 'A9' not in safe_text('password=A9secret-prefix', 23)[0]\n", + ), + Mutation( + "empty-report-go", + "hermes_handoff_model.py", + " if (\n" + " not self.results\n" + " or self.invalid_statuses\n" + " or self.harness_errors\n" + " or self.blocking\n" + " ):\n", + " if (\n" + " self.invalid_statuses\n" + " or self.harness_errors\n" + " or self.blocking\n" + " ):\n", + "from hermes_handoff_model import *\nassert Report('read-only','now').decision == NO_GO\n", + ), + Mutation( + "generic-404-is-denial", + "hermes_handoff_rules.py", + "DENIAL_MARKERS = (\n", + 'DENIAL_MARKERS = (\n "404 not found",\n', + "from hermes_handoff_rules import looks_denied\n" + "from hermes_handoff_exec import Outcome\n" + "assert not looks_denied(Outcome(('x',),'x',1,'','404 not found'))\n", + ), + Mutation( + "truncated-evidence-reaches-evaluator", + "hermes_handoff_rules.py", + " if truncated:\n" + " return Evaluation(\n" + " NOT_RUN, f\"truncated evidence from step(s): {', '.join(truncated)}\"\n" + " )\n", + " if False and truncated:\n return Evaluation(NOT_RUN, 'mutant')\n", + "import hermes_handoff_evaluators\n" + "import hermes_handoff_rules as r\n" + "from hermes_handoff_exec import Outcome\n" + "from hermes_handoff_model import *\n" + "@r.evaluator('probe')\ndef probe(*_): return r.Evaluation(PASS,'bad')\n" + "s=CheckSpec('x','x','x','probe')\n" + "assert r.evaluate(s,{'x':Outcome(('x',),'x',0,truncated=True)}).status == NOT_RUN\n", + ), + Mutation( + "truncated-outcome-is-ok", + "hermes_handoff_exec.py", + " return self.error is None and self.returncode == 0 and not self.truncated\n", + " return self.error is None and self.returncode == 0\n", + "from hermes_handoff_exec import Outcome\nassert not Outcome(('x',),'x',0,truncated=True).ok\n", + ), + Mutation( + "nan-is-numeric", + "hermes_handoff_json_rules.py", + " or not math.isfinite(float(actual))\n", + " or False\n", + "import hermes_handoff_json_rules as j\n" + "from hermes_handoff_model import *\nfrom hermes_handoff_exec import Outcome\n" + "s=CheckSpec('x','x','x','json_numeric',expect={'step':'s','fields':{'x':{'min':0}}})\n" + "j.parsed_step=lambda *_: ({'x':float('nan')},'s')\n" + "assert j.evaluate_json_numeric(s,{'s':Outcome(('x',),'x',0)}).status == NOT_RUN\n", + ), + Mutation( + "cleanup-ignores-branch-delete-failure", + "hermes_handoff_ephemeral.py", + ' if not deleted.ok:\n problems.append("ephemeral branch deletion failed")\n', + ' if False and not deleted.ok:\n problems.append("mutant")\n', + "import hermes_handoff_ephemeral as e\nfrom hermes_handoff_exec import Outcome\n" + "class R:\n" + " def __init__(self): self.i=0\n" + " def run(self,*_a,**_k):\n" + " self.i+=1\n" + " return Outcome(('x',),'x',1 if self.i==1 else 0)\n" + "q=e.ArmRequest('atlas/titan-iac','origin','acceptance1',e.CONFIRMATION,expected_head='1'*40,expected_base_sha='2'*40)\n" + "assert e._cleanup(R(),object(),q,'/api/v1/repos/atlas/titan-iac',[]).status == 'FAIL'\n", + ), + Mutation( + "cleanup-accepts-open-pr", + "hermes_handoff_ephemeral.py", + ' elif "state does not match" not in problems:\n problems.append("state is not closed")\n', + ' elif False:\n problems.append("mutant")\n', + "import hermes_handoff_ephemeral as e\n" + "q=e.ArmRequest('atlas/titan-iac','origin','acceptance1',e.CONFIRMATION,expected_head='1'*40,expected_base_sha='2'*40)\n" + "p={'number':1,'state':'open','draft':True,'merged':False,'base':{'ref':'main','sha':'2'*40},'head':{'ref':q.ref,'sha':'1'*40}}\n" + "assert e._closed_pr_problems(p,q)\n", + ), +) + + +def _probe(root: Path, probe: str) -> subprocess.CompletedProcess[str]: + environment = { + "LC_ALL": "C", + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "PYTHONPATH": str(root), + } + return subprocess.run( + [sys.executable, "-c", probe], + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=10, + check=False, + env=environment, + ) + + +def run_gate( + repo_root: Path, mutations: Sequence[Mutation] = MUTATIONS +) -> MutationReport: + """Run baseline and mutated probes in isolated copies and return the score.""" + source_root = repo_root / "scripts" / "ops" + issues: list[str] = [] + killed = 0 + with tempfile.TemporaryDirectory(prefix="hermes-mutation-") as temporary: + clean = Path(temporary) / "clean" + shutil.copytree(source_root, clean) + for mutation in mutations: + source = clean / mutation.path + original = source.read_text(encoding="utf-8") + if original.count(mutation.old) != 1: + issues.append(f"{mutation.name}: mutation anchor count is not one") + continue + baseline = _probe(clean, mutation.probe) + if baseline.returncode != 0: + issues.append(f"{mutation.name}: baseline probe failed") + continue + mutated_text = original.replace(mutation.old, mutation.new, 1) + try: + compile(mutated_text, str(source), "exec") + except SyntaxError: + issues.append(f"{mutation.name}: mutant is syntactically invalid") + continue + mutant = Path(temporary) / f"mutant-{mutation.name}" + shutil.copytree(clean, mutant) + (mutant / mutation.path).write_text(mutated_text, encoding="utf-8") + if _probe(mutant, mutation.probe).returncode != 0: + killed += 1 + else: + issues.append(f"{mutation.name}: mutant survived") + return MutationReport(len(mutations), killed, tuple(issues)) + + +def main() -> int: + """Run the repository mutation contract and print a bounded score.""" + report = run_gate(Path(__file__).resolve().parents[1]) + print( + f"Hermes handoff mutation score: {report.killed}/{report.total} ({report.score:.2f}%)" + ) + for issue in report.issues: + print(issue) + return 0 if not report.issues and report.killed == report.total else 1 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) # pragma: no cover diff --git a/testing/quality_hygiene.py b/testing/quality_hygiene.py index 5060ca02..6b68de3a 100644 --- a/testing/quality_hygiene.py +++ b/testing/quality_hygiene.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import re from collections.abc import Iterable from pathlib import Path @@ -16,6 +17,18 @@ def _expand_globs(root: Path, patterns: Iterable[str]) -> list[Path]: return sorted(matched) +def _is_unchanged_legacy( + config: dict[str, Any], root: Path, path: Path, line_count: int +) -> bool: + """Allow an over-cap legacy file only while its exact bytes stay unchanged.""" + relative = path.relative_to(root).as_posix() + record = config.get("legacy_line_exceptions", {}).get(relative) + if not isinstance(record, dict) or record.get("lines") != line_count: + return False + digest = hashlib.sha256(path.read_bytes()).hexdigest() + return record.get("sha256") == digest + + def run_check(contract: dict[str, Any], root: Path) -> list[str]: """Return human-readable issues for naming and file-size rules.""" config = contract.get("hygiene", {}) @@ -24,8 +37,12 @@ def run_check(contract: dict[str, Any], root: Path) -> list[str]: for path in _expand_globs(root, config.get("line_limit_globs", [])): line_count = sum(1 for _ in path.open("r", encoding="utf-8")) - if line_count > max_lines: - issues.append(f"file exceeds {max_lines} LOC: {path.relative_to(root)} ({line_count})") + if line_count > max_lines and not _is_unchanged_legacy( + config, root, path, line_count + ): + issues.append( + f"file exceeds {max_lines} LOC: {path.relative_to(root)} ({line_count})" + ) for rule in config.get("naming_rules", []): pattern = re.compile(rule["pattern"]) @@ -48,7 +65,9 @@ def count_files_over_line_limit(contract: dict[str, Any], root: Path) -> int: count = 0 for path in _expand_globs(root, config.get("line_limit_globs", [])): line_count = sum(1 for _ in path.open("r", encoding="utf-8")) - if line_count > max_lines: + if line_count > max_lines and not _is_unchanged_legacy( + config, root, path, line_count + ): count += 1 return count diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py new file mode 100644 index 00000000..ecd64eb4 --- /dev/null +++ b/testing/tests/conftest.py @@ -0,0 +1,20 @@ +"""Hermetic runtime-path fixtures shared by repository tests.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +@pytest.fixture(autouse=True) +def isolate_coordinator_gitea_token( + request: pytest.FixtureRequest, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Prevent coordinator tests from observing the mounted production token.""" + if request.node.path.name == "test_hermes_coordinator.py": + monkeypatch.setenv( + "HERMES_GITEA_TOKEN_FILE", str(tmp_path / "absent-gitea-token") + ) diff --git a/testing/tests/test_hermes_auto_router.py b/testing/tests/test_hermes_auto_router.py index c0a31103..f8a333b1 100644 --- a/testing/tests/test_hermes_auto_router.py +++ b/testing/tests/test_hermes_auto_router.py @@ -255,6 +255,12 @@ def test_provider_status_separates_observed_activity_from_plan_quota(monkeypatch monkeypatch.setattr(module, "_claude_account", lambda: { "authenticated": True, "plan": "max", "quota_reported": False, }) + monkeypatch.setattr(module, "_fresh_health", lambda _path, maximum_age=86400.0: {}) + monkeypatch.setattr( + module, + "_configured_models", + lambda provider: ["gpt-5.6-sol"] if provider == "codex" else ["claude-sonnet-5"], + ) payload = module.provider_status_payload() diff --git a/testing/tests/test_hermes_handoff_acceptance.py b/testing/tests/test_hermes_handoff_acceptance.py index 87c8adbe..4f89db35 100644 --- a/testing/tests/test_hermes_handoff_acceptance.py +++ b/testing/tests/test_hermes_handoff_acceptance.py @@ -18,7 +18,9 @@ harness_run = load_handoff_module("hermes_handoff_run") def completed(stdout: str = "", stderr: str = "", returncode: int = 0): - return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) class ScriptedCluster: @@ -44,10 +46,37 @@ class ScriptedCluster: def whoami(username: str): - return completed(stdout=json.dumps({"status": {"userInfo": {"username": username}}})) + return completed(stdout=username) + + +REVIEWED = "8f005458282269ba5c07941814e4237f5d4cf3ac" +SHA = "1" * 40 +VALID_ARGS = [ + "--remote-main-sha", + SHA, + "--reviewed-head-sha", + REVIEWED, + "--agent-image", + f"registry.example/hermes:git-{'1' * 40}-build-7@sha256:{'2' * 64}", + "--build-sha", + SHA, + "--deployment-revision", + "1", + "--chat-config-revision", + "chat-rev", + *[ + item + for number in catalog.Targets().dependency_pull_requests + for item in ("--dependency-head", f"{number}={SHA}") + ], +] HEALTHY_CLUSTER = [ + ( + "exec --namespace hermes hermes-agent-1", + whoami("system:serviceaccount:hermes:hermes-agent"), + ), ("auth whoami", whoami("kubernetes-admin")), ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), ("get pods --selector", completed(stdout="pod/other-1\n")), @@ -60,10 +89,18 @@ def run_cli(monkeypatch, argv, rules): clock = FakeClock() def build_runner(**kwargs): - return exec_module.Runner(clock=clock, spawn=cluster, environ={}, **kwargs) + return exec_module.Runner( + clock=clock, + spawn=cluster, + environ={}, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + **kwargs, + ) monkeypatch.setattr(acceptance, "Runner", build_runner) - return (acceptance.main(argv), cluster) + return (acceptance.main([*VALID_ARGS, *argv]), cluster) def read_report(path): @@ -109,7 +146,15 @@ def test_defaults_fall_back_to_the_shipped_targets() -> None: assert targets.repo == catalog.Targets.repo -def test_a_default_run_is_read_only_and_never_pushes_or_posts(monkeypatch, tmp_path) -> None: +@pytest.mark.parametrize("value", ["bad", "x=" + "1" * 40, "0=" + "1" * 40, "1=BAD"]) +def test_dependency_heads_require_a_positive_pr_and_lowercase_sha(value: str) -> None: + with pytest.raises(Exception): + acceptance.dependency_head(value) + + +def test_a_default_run_is_read_only_and_never_pushes_or_posts( + monkeypatch, tmp_path +) -> None: output = tmp_path / "report.json" status, cluster = run_cli( monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER @@ -124,13 +169,15 @@ def test_a_default_run_is_read_only_and_never_pushes_or_posts(monkeypatch, tmp_p writes = [ call for call in cluster.calls - if {"patch", "create", "delete", "apply", "replace"} & set(call) and "can-i" not in call + if {"patch", "create", "delete", "apply", "replace"} & set(call) + and "can-i" not in call ] - assert writes - assert all("--dry-run=server" in call for call in writes) + assert writes == [] -def test_mutating_checks_report_not_run_until_the_mode_is_armed(monkeypatch, tmp_path) -> None: +def test_mutating_checks_report_not_run_until_the_mode_is_armed( + monkeypatch, tmp_path +) -> None: output = tmp_path / "report.json" run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) @@ -163,7 +210,10 @@ def test_a_zero_state_cluster_is_no_go_with_every_check_accounted_for( assert report["decision"] == model.NO_GO assert report["counts"][model.PASS] == 0 assert sum(report["counts"].values()) == len(report["checks"]) - assert any("in-pod Hermes vantage is unavailable" in error for error in report["harness_errors"]) + assert any( + "in-pod Hermes vantage is unavailable" in error + for error in report["harness_errors"] + ) def test_partial_failure_is_reported_per_check_rather_than_aborting( @@ -186,14 +236,25 @@ def test_partial_failure_is_reported_per_check_rather_than_aborting( assert report["blocking"] -def test_a_vantage_collision_is_a_no_go_even_with_clean_checks(monkeypatch, tmp_path) -> None: +def test_a_vantage_collision_is_a_no_go_even_with_clean_checks( + monkeypatch, tmp_path +) -> None: output = tmp_path / "report.json" rules = [ - ("exec --namespace hermes hermes-agent-1", whoami("system:serviceaccount:hermes:hermes-agent")), + ( + "exec --namespace hermes hermes-agent-1", + whoami("system:serviceaccount:hermes:hermes-agent"), + ), ("auth whoami", whoami("system:serviceaccount:hermes:hermes-agent")), - ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), + ( + "get pods --selector app=hermes-agent", + completed(stdout="pod/hermes-agent-1\n"), + ), ("get pods --selector", completed(stdout="pod/other-1\n")), - ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")), + ( + "get pod/hermes-chat-tenant-0", + completed(stdout="pod/hermes-chat-tenant-0\n"), + ), ] status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) report = read_report(output) @@ -202,16 +263,18 @@ def test_a_vantage_collision_is_a_no_go_even_with_clean_checks(monkeypatch, tmp_ assert any("same principal" in error for error in report["harness_errors"]) -def test_an_exhausted_deadline_leaves_the_remaining_checks_not_run(monkeypatch, tmp_path) -> None: +def test_an_invalid_deadline_stops_before_any_subprocess(monkeypatch, tmp_path) -> None: output = tmp_path / "report.json" status, _ = run_cli( - monkeypatch, ["--output", str(output), "--json-only", "--deadline", "0"], HEALTHY_CLUSTER + monkeypatch, + ["--output", str(output), "--json-only", "--deadline", "0"], + HEALTHY_CLUSTER, ) report = read_report(output) - assert status == acceptance.EXIT_NO_GO + assert status == acceptance.EXIT_USAGE assert report["counts"][model.NOT_RUN] > 0 - assert any("deadline expired" in error for error in report["harness_errors"]) + assert any("invalid execution bound" in error for error in report["harness_errors"]) def test_arming_with_a_wrong_confirmation_stops_before_anything_runs( @@ -239,7 +302,9 @@ def test_arming_with_a_wrong_confirmation_stops_before_anything_runs( assert not any("push" in call for call in cluster.calls) -def test_arming_against_a_protected_push_target_is_refused(monkeypatch, tmp_path) -> None: +def test_arming_against_a_protected_push_target_is_refused( + monkeypatch, tmp_path +) -> None: output = tmp_path / "report.json" status, cluster = run_cli( monkeypatch, @@ -261,17 +326,24 @@ def test_arming_against_a_protected_push_target_is_refused(monkeypatch, tmp_path assert not any("push" in call for call in cluster.calls) -def test_an_armed_run_pushes_one_ephemeral_ref_and_verifies_its_removal( +def test_an_armed_run_does_not_mutate_until_read_only_acceptance_is_go( monkeypatch, tmp_path ) -> None: output = tmp_path / "report.json" + monkeypatch.setattr(acceptance, "preflight", lambda *_args: None) rules = [ ("git ls-remote origin refs/heads/ephemeral", completed(stdout="")), ("git push", completed()), - ("POST /api/v1/repos/atlas/titan-iac/pulls", completed(stdout=json.dumps({"number": 9}))), + ( + "POST /api/v1/repos/atlas/titan-iac/pulls", + completed(stdout=json.dumps({"number": 9})), + ), ("PATCH", completed()), ("DELETE", completed()), - ("GET /api/v1/repos/atlas/titan-iac/pulls/9", completed(stdout='{"state": "closed"}')), + ( + "GET /api/v1/repos/atlas/titan-iac/pulls/9", + completed(stdout='{"state": "closed"}'), + ), *HEALTHY_CLUSTER, ] status, cluster = run_cli( @@ -289,20 +361,14 @@ def test_an_armed_run_pushes_one_ephemeral_ref_and_verifies_its_removal( rules, ) report = read_report(output) - ephemeral = {check["id"]: check for check in report["checks"] if check["group"] == "ephemeral"} - assert report["mode"] == policy.ARMED - assert status == acceptance.EXIT_NO_GO # the rest of the cluster is still unproven - assert ephemeral["ephemeral.feature-branch-push"]["status"] == model.PASS - assert ephemeral["ephemeral.draft-pull-request"]["status"] == model.PASS - assert ephemeral["ephemeral.cleanup-verified"]["status"] == model.PASS - assert ephemeral["ephemeral.protected-branch-refusal"]["status"] == model.PASS - + assert status == acceptance.EXIT_NO_GO + assert any( + "read-only acceptance did not GO" in error for error in report["harness_errors"] + ) pushes = [call for call in cluster.calls if "push" in call] - assert len(pushes) == 1 - assert pushes[0][-1].startswith("HEAD:refs/heads/ephemeral/hermes-handoff-acceptance/") - assert pushes[0][0] == "kubectl" and "exec" in pushes[0] # pushed as the worker, not the operator - assert any("DELETE" in call for call in cluster.calls) + assert pushes == [] + assert not any("DELETE" in call for call in cluster.calls) def test_the_report_is_written_to_stdout_when_no_output_path_is_given( @@ -327,10 +393,14 @@ def test_the_human_summary_accompanies_the_json_unless_suppressed( assert captured.out == "" -def test_a_report_that_fails_its_own_screening_is_replaced_not_published(tmp_path) -> None: +def test_a_report_that_fails_its_own_screening_is_replaced_not_published( + tmp_path, +) -> None: """The last barrier: never write an artifact nobody re-read.""" output = tmp_path / "report.json" - arguments = acceptance.build_parser().parse_args(["--output", str(output), "--json-only"]) + arguments = acceptance.build_parser().parse_args( + ["--output", str(output), "--json-only"] + ) report = model.Report(mode="read-only", started_at="2026-08-17T00:00:00Z") report.harness_errors = ["leaked ghp_ABCDEFGHIJKLMNOPQRSTUV1234"] @@ -348,20 +418,28 @@ def test_a_report_that_fails_its_own_screening_is_replaced_not_published(tmp_pat assert "failed its own credential screening" in payload["harness_errors"][0] -def test_a_screening_failure_forces_no_go_from_the_entry_point(monkeypatch, tmp_path) -> None: +def test_a_screening_failure_forces_no_go_from_the_entry_point( + monkeypatch, tmp_path +) -> None: """The exit status must follow the refusal to publish, not the check tally.""" output = tmp_path / "report.json" - monkeypatch.setattr(acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"]) + monkeypatch.setattr( + acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"] + ) monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) - status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) + status, _ = run_cli( + monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER + ) assert status == acceptance.EXIT_NO_GO assert read_report(output)["decision"] == model.NO_GO -def test_no_recorded_command_output_carries_a_credential_shape(monkeypatch, tmp_path) -> None: +def test_no_recorded_command_output_carries_a_credential_shape( + monkeypatch, tmp_path +) -> None: output = tmp_path / "report.json" leaky = completed(stdout="ANTHROPIC_API_KEY=sk-ant-abcdefghijklmnopqrstuvwxyz\n") run_cli( @@ -380,16 +458,3 @@ def test_the_help_text_describes_the_read_only_default(flag: str, capsys) -> Non acceptance.main([flag]) assert "Read-only by default" in capsys.readouterr().out - - -def test_go_is_reachable_when_nothing_mandatory_is_outstanding(monkeypatch, tmp_path) -> None: - """Exercise the success exit path without pretending the cluster is clean.""" - output = tmp_path / "report.json" - monkeypatch.setattr(harness_run, "build_catalog", lambda _targets: []) - monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) - monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) - - status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) - - assert status == acceptance.EXIT_GO - assert read_report(output)["decision"] == model.GO diff --git a/testing/tests/test_hermes_handoff_acceptance_armed.py b/testing/tests/test_hermes_handoff_acceptance_armed.py new file mode 100644 index 00000000..ded33bb8 --- /dev/null +++ b/testing/tests/test_hermes_handoff_acceptance_armed.py @@ -0,0 +1,120 @@ +"""Static and armed-transition regressions for the handoff CLI.""" + +from __future__ import annotations + +from testing.tests.test_hermes_handoff_acceptance import ( + HEALTHY_CLUSTER, + REVIEWED, + VALID_ARGS, + acceptance, + harness_run, + model, + policy, + read_report, + run_cli, +) + + +def test_an_empty_catalog_fails_before_any_subprocess(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + monkeypatch.setattr(harness_run, "build_catalog", lambda _targets: []) + monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) + monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) + + status, _ = run_cli( + monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER + ) + + assert status == acceptance.EXIT_NO_GO + assert read_report(output)["decision"] == model.NO_GO + + +def test_armed_success_path_replaces_placeholders_only_after_a_go(monkeypatch) -> None: + base = model.CheckSpec( + id="release.pass", title="pass", group="release", rule="allowed" + ) + placeholder = model.CheckSpec( + id="ephemeral.cleanup-verified", + title="pending", + group="ephemeral", + rule="not_armed", + mandatory=False, + scope=model.EPHEMERAL, + ) + report = model.Report( + mode=policy.ARMED, + started_at="2026-08-17T00:00:00Z", + results=[ + model.CheckResult(base, model.PASS), + model.CheckResult(placeholder, model.NOT_RUN), + ], + ) + armed_spec = model.CheckSpec( + id="ephemeral.cleanup-verified", + title="clean", + group="ephemeral", + rule="armed", + scope=model.EPHEMERAL, + ) + armed = model.CheckResult(armed_spec, model.PASS) + monkeypatch.setattr( + acceptance, "resolve_vantages", lambda *_args: ({"operator": object()}, []) + ) + monkeypatch.setattr(acceptance, "preflight", lambda *_args: None) + monkeypatch.setattr(acceptance, "build_report", lambda *_args: report) + monkeypatch.setattr(acceptance, "_armed_results", lambda *_args: [armed]) + monkeypatch.setattr(acceptance, "emit", lambda *_args: []) + status = acceptance.main( + [ + *VALID_ARGS, + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817a", + ] + ) + assert status == acceptance.EXIT_GO + assert ( + next( + result + for result in report.results + if result.spec.id.startswith("ephemeral") + ).status + == model.PASS + ) + monkeypatch.setattr(acceptance, "emit", lambda *_args: ["screening-failure"]) + assert ( + acceptance.main( + [ + *VALID_ARGS, + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817b", + ] + ) + == acceptance.EXIT_NO_GO + ) + + +def test_armed_results_are_bound_to_operator_and_exact_reviewed_head( + monkeypatch, +) -> None: + arguments = acceptance.build_parser().parse_args( + [ + *VALID_ARGS, + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817a", + ] + ) + calls = [] + monkeypatch.setattr(acceptance, "preflight", lambda *args: calls.append(args)) + monkeypatch.setattr(acceptance, "run_armed", lambda *args: [args]) + operator = object() + result = acceptance._armed_results(arguments, object(), {"operator": operator}) + assert calls[0][0].expected_head == REVIEWED + assert result[0][1] is operator diff --git a/testing/tests/test_hermes_handoff_catalog.py b/testing/tests/test_hermes_handoff_catalog.py index 8f5e2de6..21e8031a 100644 --- a/testing/tests/test_hermes_handoff_catalog.py +++ b/testing/tests/test_hermes_handoff_catalog.py @@ -23,8 +23,12 @@ BY_ID = {check.id: check for check in SPECS} VANTAGES = { catalog.OPERATOR: exec_module.operator_vantage(context="atlas-operator"), catalog.SELF: exec_module.pod_vantage("hermes", "hermes-agent-1", "hermes"), - catalog.SWITCHYARD: exec_module.pod_vantage("hermes", "hermes-switchyard-1", "switchyard"), - catalog.NODE: exec_module.pod_vantage("hermes", "hermes-node-ssh-access-1", "key-reconciler"), + catalog.SWITCHYARD: exec_module.pod_vantage( + "hermes", "hermes-switchyard-1", "switchyard" + ), + catalog.NODE: exec_module.pod_vantage( + "hermes", "hermes-node-ssh-access-1", "key-reconciler" + ), catalog.CHAT: exec_module.pod_vantage("hermes", "hermes-chat-tenant-0", "hermes"), } @@ -42,32 +46,41 @@ def test_every_step_is_accepted_by_the_read_only_policy_from_its_vantage() -> No policy.check_argv(addressed, policy.READ_ONLY) -def test_no_step_mutates_outside_a_server_dry_run() -> None: - """`auth can-i create ...` is a read; the subcommand is what decides.""" +def test_no_step_constructs_a_mutation_or_dry_run() -> None: + """Authorization review may name a verb; no executed subcommand mutates.""" for check in SPECS: for step in check.steps: subcommand = next(policy.positionals(step.argv), "") - if subcommand in policy.KUBECTL_DRY_RUN_SUBCOMMANDS: - assert any(flag in step.argv for flag in policy.DRY_RUN_FLAGS), check.id + assert subcommand not in {"apply", "create", "delete", "patch", "replace"} + assert not any(argument.startswith("--dry-run") for argument in step.argv) -def test_no_operator_step_impersonates_and_at_least_one_in_pod_step_does() -> None: +def test_no_catalog_step_constructs_impersonation() -> None: operator_steps = [ - step for check in SPECS for step in check.steps if step.vantage == catalog.OPERATOR + step + for check in SPECS + for step in check.steps + if step.vantage == catalog.OPERATOR ] assert operator_steps - assert not any(set(step.argv) & set(policy.IMPERSONATION_ARGS) for step in operator_steps) + assert not any( + set(step.argv) & set(policy.IMPERSONATION_ARGS) for step in operator_steps + ) - impersonation = BY_ID["access.impersonation-is-denied"] - attempts = [step for step in impersonation.steps if step.kind == model.ATTEMPT] - assert any("--as" in step.argv for step in attempts) + assert not any( + set(step.argv) & set(policy.IMPERSONATION_ARGS) + for check in SPECS + for step in check.steps + ) -def test_every_deny_check_pairs_a_review_with_a_live_attempt() -> None: - deny_checks = [check for check in SPECS if check.rule == "denied"] - assert len(deny_checks) >= 6 +def test_credential_or_mutation_denials_are_review_only() -> None: + deny_checks = [check for check in SPECS if check.rule == "review_denied"] + assert len(deny_checks) >= 5 for check in deny_checks: - assert any(step.kind == model.ATTEMPT for step in check.steps), check.id + assert check.steps and all(step.kind == model.REVIEW for step in check.steps), ( + check.id + ) def test_both_vantages_are_represented_across_the_catalog() -> None: @@ -104,7 +117,9 @@ def test_dependency_pull_requests_each_get_their_own_merge_gate() -> None: for number in TARGETS.dependency_pull_requests: check = BY_ID[f"baseline.dependency-pr-{number}-merged"] assert check.mandatory - assert check.expect["fields"] == {"merged": True, "base.ref": "main"} + assert check.expect["fields"]["merged"] is True + assert check.expect["fields"]["base.ref"] == "main" + assert "head.sha" in check.expect["fields"] def test_the_baseline_commit_gate_names_the_configured_commit() -> None: @@ -113,25 +128,12 @@ def test_the_baseline_commit_gate_names_the_configured_commit() -> None: assert "--is-ancestor" in check.steps[0].argv -def test_optional_checks_appear_only_when_a_site_declares_what_to_expect() -> None: - """A names_present over an empty expectation would pass on any workload.""" - assert "pool.assignment-safety-knobs-are-configured" not in BY_ID - assert "surfaces.chat-runs-the-telegram-topic-revision" not in BY_ID - assert "surfaces.chat-telegram-sessions-are-continuous" not in BY_ID - - configured = harness_run.build_catalog( - catalog.Targets( - now=NOW, - pool_worker_env=("HERMES_POOL_LEASE_SECONDS",), - chat_config_revision="20260816-telegram-topics", - expect_telegram_sessions=True, - ) - ) - identifiers = {check.id for check in configured} - assert "pool.assignment-safety-knobs-are-configured" in identifiers - assert "surfaces.chat-runs-the-telegram-topic-revision" in identifiers - assert "surfaces.chat-telegram-sessions-are-continuous" in identifiers - assert harness_run.validate_catalog(configured) == [] +def test_pool_assignment_and_chat_continuity_are_unconditional() -> None: + assert "pool.assignment-safety-knobs-are-configured" in BY_ID + assert "pool.assignment-fieldrefs-are-bound" in BY_ID + assert "surfaces.chat-runs-the-telegram-topic-revision" in BY_ID + assert "surfaces.chat-telegram-sessions-are-continuous" in BY_ID + assert TARGETS.node_count == TARGETS.pool_replicas == 3 def test_bulk_evidence_is_parsed_but_not_pasted_into_the_report() -> None: @@ -144,7 +146,10 @@ def test_bulk_evidence_is_parsed_but_not_pasted_into_the_report() -> None: def test_freshness_gates_carry_the_run_clock() -> None: - for identifier in ("identity.codex-evidence-is-fresh", "identity.claude-evidence-is-fresh"): + for identifier in ( + "identity.codex-evidence-is-fresh", + "identity.claude-evidence-is-fresh", + ): assert BY_ID[identifier].expect["now"] == NOW routing = BY_ID["routing.provider-effort-and-fallback-evidence"] assert routing.expect["now"] == NOW @@ -170,18 +175,14 @@ def test_check_identifiers_are_unique_and_grouped() -> None: "access.workload-mutation-is-denied", "access.pod-exec-is-denied", "access.required-reads-are-allowed", - "access.pod-logs-are-allowed", "access.flux-and-helm-status-are-allowed", "access.namespace-view-agrees-across-vantages", - "nodes.account-password-is-locked", "nodes.hardening-covers-every-node", "build.builder-service-account-is-tokenless", "build.harbor-immutability-rule-is-applied", - "build.deployed-image-is-digest-pinned", + "release.exact-lineage-is-running", "reliability.finalization-and-replay-patches-are-deployed", - "pool.three-workers-are-ready", - "pool.workers-occupy-distinct-nodes", - "pool.workers-own-distinct-volumes", + "pool.three-workers-have-distinct-nodes-and-volumes", "pool.coordinator-retains-sole-state-ownership", "surfaces.kanban-activity-is-visible", "surfaces.chat-telegram-topic-state-is-durable", diff --git a/testing/tests/test_hermes_handoff_ephemeral.py b/testing/tests/test_hermes_handoff_ephemeral.py index 09cc5d63..9f54601a 100644 --- a/testing/tests/test_hermes_handoff_ephemeral.py +++ b/testing/tests/test_hermes_handoff_ephemeral.py @@ -1,9 +1,10 @@ -"""Contracts for the separately armed, self-cleaning mutation mode.""" +"""Contracts for the strictly bounded, self-cleaning armed mode.""" from __future__ import annotations import json import subprocess +from pathlib import Path import pytest @@ -18,254 +19,319 @@ exec_module = load_handoff_module("hermes_handoff_exec") policy = load_handoff_module("hermes_handoff_policy") model = load_handoff_module("hermes_handoff_model") -REPO = "atlas/titan-iac" TOKEN = "acceptance-20260817a" +HEAD = "8f005458282269ba5c07941814e4237f5d4cf3ac" +MAIN = "1" * 40 VANTAGE = exec_module.operator_vantage() -def request(**overrides) -> object: +def request(**overrides): fields = { - "repo": REPO, - "remote": "origin", + "repo": policy.EXPECTED_REPO, + "remote": policy.EXPECTED_REMOTE, "token": TOKEN, "confirmation": ephemeral.CONFIRMATION, - "base": "main", + "base": policy.EXPECTED_BASE, + "expected_head": HEAD, + "expected_base_sha": MAIN, } fields.update(overrides) return ephemeral.ArmRequest(**fields) def completed(stdout: str = "", stderr: str = "", returncode: int = 0): - return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) -def armed_runner(results: list) -> tuple: +def armed_runner(results: list, mode=policy.ARMED): spawn = FakeSpawn(results) runner = exec_module.Runner( - mode=policy.ARMED, clock=FakeClock(), spawn=spawn, environ={}, deadline_seconds=3600 + mode=mode, + clock=FakeClock(), + spawn=spawn, + environ={}, + deadline_seconds=3600, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), ) - return (runner, spawn) + return runner, spawn def by_id(results: list) -> dict: return {result.spec.id: result for result in results} -def test_the_ephemeral_ref_is_derived_from_the_caller_supplied_token() -> None: - assert request().ref == f"{ephemeral.REF_PREFIX}/{TOKEN}" - - -@pytest.mark.parametrize( - "ref", ["main", "master", "refs/heads/main", "HEAD", "Production", "feature/x", ""] -) -def test_protected_and_malformed_push_targets_are_refused(ref: str) -> None: - with pytest.raises(ephemeral.ArmingError): - ephemeral.assert_push_target_allowed(ref) - - -def test_an_ephemeral_ref_is_the_only_accepted_push_target() -> None: - ephemeral.assert_push_target_allowed(f"{ephemeral.REF_PREFIX}/{TOKEN}") - - -@pytest.mark.parametrize( - ("overrides", "fragment"), - [ - ({"confirmation": "yes please"}, "confirmation phrase"), - ({"confirmation": ""}, "confirmation phrase"), - ({"repo": "atlas/other"}, "is not the expected"), - ({"token": "short"}, "ephemeral token"), - ({"token": "Has-Capitals-Here"}, "ephemeral token"), - ({"token": ""}, "ephemeral token"), - ({"base": ""}, "base branch is required"), - ], -) -def test_preflight_refuses_before_any_network_call(overrides: dict, fragment: str) -> None: - with pytest.raises(ephemeral.ArmingError) as caught: - ephemeral.preflight(request(**overrides), REPO) - assert fragment in str(caught.value) - - -def test_preflight_accepts_a_correctly_armed_request() -> None: - ephemeral.preflight(request(), REPO) - - -@pytest.mark.parametrize( - ("raw", "expected"), [("refs/heads/Main", "main"), (" main ", "main"), ("/x/", "x")] -) -def test_branch_names_are_normalised_before_comparison(raw: str, expected: str) -> None: - assert ephemeral.normalise_branch(raw) == expected - - -def test_the_protected_branch_guard_is_exercised_on_every_armed_run() -> None: - """A guard checked once is a guard nobody notices regressing.""" - runner, _ = armed_runner([completed(stdout="", returncode=2)]) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - guard = results["ephemeral.protected-branch-refusal"] - - assert guard.status == model.PASS - assert guard.evidence["accepted"] == [] - assert "main" in guard.evidence["refused"] - - -def test_a_broken_protected_branch_guard_fails_its_own_self_test(monkeypatch) -> None: - """If the guard ever stops refusing, the armed run says so instead of pushing.""" - monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None) - runner, _ = armed_runner([completed(stdout="", returncode=2)]) - - guard = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[ - "ephemeral.protected-branch-refusal" - ] - - assert guard.status == model.FAIL - assert "main" in guard.evidence["accepted"] - assert "the guard accepted" in guard.reason - - -def test_a_full_armed_run_pushes_opens_a_draft_and_removes_both() -> None: - runner, spawn = armed_runner( - [ - completed(stdout=""), # ls-remote: ref unused - completed(stdout=""), # push - completed(stdout=json.dumps({"number": 42, "draft": True})), # create draft - completed(stdout=""), # close - completed(stdout=""), # delete branch - completed(stdout=""), # ls-remote: gone - completed(stdout=json.dumps({"number": 42, "state": "closed"})), - ] - ) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - - assert results["ephemeral.feature-branch-push"].status == model.PASS - assert results["ephemeral.draft-pull-request"].status == model.PASS - assert results["ephemeral.draft-pull-request"].evidence["pull_request"] == 42 - assert results["ephemeral.cleanup-verified"].status == model.PASS - assert all(result.spec.mandatory for result in results.values()) - assert all(result.spec.scope == model.EPHEMERAL for result in results.values()) - - pushed = [call for call in spawn.calls if call["argv"][:2] == ["git", "push"]] - assert pushed[0]["argv"][-1] == f"HEAD:refs/heads/{request().ref}" - assert "--force" not in pushed[0]["argv"] - - titles = [arg for call in spawn.calls for arg in call["argv"] if arg.startswith("title=")] - assert titles[0].startswith(f"title={ephemeral.DRAFT_TITLE_PREFIX}") - - -def test_an_existing_ref_stops_the_run_before_it_writes() -> None: - runner, spawn = armed_runner([completed(stdout="abc123\trefs/heads/x\n")]) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - - assert results["ephemeral.feature-branch-push"].status == model.FAIL - assert "already exists" in results["ephemeral.feature-branch-push"].reason - assert "ephemeral.draft-pull-request" not in results - assert len(spawn.calls) == 1 - - -def test_an_unverifiable_ref_check_is_not_run_rather_than_assumed_free() -> None: - runner, _ = armed_runner([completed(stderr="network down", returncode=128)]) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - - assert results["ephemeral.feature-branch-push"].status == model.NOT_RUN - - -def test_a_failed_push_still_reports_cleanup_and_skips_the_draft() -> None: - runner, _ = armed_runner( - [ - completed(stdout=""), - completed(stderr="denied", returncode=1), - completed(stdout=""), # delete branch - completed(stdout=""), # ls-remote: gone - ] - ) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - - assert results["ephemeral.feature-branch-push"].status == model.FAIL - assert results["ephemeral.draft-pull-request"].status == model.NOT_RUN - assert results["ephemeral.cleanup-verified"].status == model.PASS - - -def test_a_draft_response_without_a_number_is_a_failure() -> None: - runner, _ = armed_runner( - [ - completed(stdout=""), - completed(stdout=""), - completed(stdout="{}"), - completed(stdout=""), - completed(stdout=""), - ] - ) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - - assert results["ephemeral.draft-pull-request"].status == model.FAIL - - -def test_a_branch_that_survives_cleanup_fails_the_cleanup_check() -> None: - runner, _ = armed_runner( - [ - completed(stdout=""), - completed(stdout=""), - completed(stdout=json.dumps({"number": 7})), - completed(stderr="cannot close", returncode=1), - completed(stderr="cannot delete", returncode=1), - completed(stdout="abc123\trefs/heads/x\n"), - completed(stdout=json.dumps({"number": 7, "state": "open"})), - ] - ) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - cleanup = results["ephemeral.cleanup-verified"] - - assert cleanup.status == model.FAIL - assert "could not be closed" in cleanup.reason - assert "still exists" in cleanup.reason - assert "still open" in cleanup.reason - - -def test_cleanup_that_cannot_be_verified_is_a_failure_not_a_pass() -> None: - runner, _ = armed_runner( - [ - completed(stdout=""), - completed(stdout=""), - completed(stdout=json.dumps({"number": 7})), - completed(stdout=""), - completed(stdout=""), - subprocess.TimeoutExpired(cmd="git", timeout=1.0), - completed(stderr="gone", returncode=1), - ] - ) - - cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))["ephemeral.cleanup-verified"] - - assert cleanup.status == model.FAIL - assert "could not verify removal" in cleanup.reason - assert "could not confirm pull request #7 is closed" in cleanup.reason - - -def test_the_armed_results_replace_the_default_not_run_placeholders() -> None: - runner, _ = armed_runner([completed(stdout="abc\trefs/heads/x\n")]) - identifiers = {result.spec.id for result in ephemeral.run_armed(runner, VANTAGE, request())} - - assert identifiers <= { - "ephemeral.protected-branch-refusal", - "ephemeral.feature-branch-push", - "ephemeral.draft-pull-request", - "ephemeral.cleanup-verified", +def pr_payload(state="open", number=42): + return { + "number": number, + "state": state, + "draft": True, + "merged": False, + "base": {"ref": "main", "sha": MAIN}, + "head": {"ref": request().ref, "sha": HEAD}, } -def test_a_read_only_runner_cannot_execute_the_armed_flow() -> None: - """The mode, not the caller, is what opens the mutations.""" - spawn = FakeSpawn([completed(stdout=""), completed(stdout=""), completed(stdout="")]) - runner = exec_module.Runner(clock=FakeClock(), spawn=spawn, environ={}) +def linked_worktree(tmp_path: Path) -> Path: + root = tmp_path / "worktree" + common = tmp_path / "repo.git" + gitdir = common / "worktrees" / "acceptance" + gitdir.mkdir(parents=True) + (root).mkdir() + (root / ".git").write_text(f"gitdir: {gitdir}\n", encoding="utf-8") + (gitdir / "commondir").write_text("../..\n", encoding="utf-8") + (gitdir / "HEAD").write_text( + f"ref: refs/heads/{ephemeral.EXPECTED_HEAD_REF}\n", encoding="utf-8" + ) + ref = common / "refs" / "heads" / ephemeral.EXPECTED_HEAD_REF + ref.parent.mkdir(parents=True) + ref.write_text(f"{HEAD}\n", encoding="utf-8") + remote_ref = common / "refs" / "remotes" / "origin" / "main" + remote_ref.parent.mkdir(parents=True) + remote_ref.write_text(f"{MAIN}\n", encoding="utf-8") + (common / "config").write_text( + f'[remote "origin"]\n\turl = {ephemeral.EXPECTED_REMOTE_URL}\n', + encoding="utf-8", + ) + return root + +@pytest.mark.parametrize( + "ref", ["main", "master", "refs/heads/main", "HEAD", "feature/x", ""] +) +def test_only_the_unique_ephemeral_ref_is_allowed(ref: str) -> None: + with pytest.raises(ephemeral.ArmingError): + ephemeral.assert_push_target_allowed(ref) + ephemeral.assert_push_target_allowed(request().ref) + + +@pytest.mark.parametrize( + "overrides", + [ + {"confirmation": "wrong"}, + {"repo": "atlas/other"}, + {"remote": "upstream"}, + {"base": "develop"}, + {"token": "short"}, + {"expected_head": "bad"}, + {"expected_base_sha": "bad"}, + ], +) +def test_preflight_rejects_bad_inputs_before_any_runner_exists( + tmp_path: Path, overrides: dict +) -> None: + with pytest.raises(ephemeral.ArmingError): + ephemeral.preflight( + request(**overrides), policy.EXPECTED_REPO, linked_worktree(tmp_path) + ) + + +def test_preflight_attests_the_actual_linked_worktree(tmp_path: Path) -> None: + ephemeral.preflight(request(), policy.EXPECTED_REPO, linked_worktree(tmp_path)) + root = linked_worktree(tmp_path / "wrong") + (Path((root / ".git").read_text().split(": ", 1)[1].strip()) / "HEAD").write_text( + "ref: refs/heads/other\n", encoding="utf-8" + ) + with pytest.raises(ephemeral.ArmingError, match="existing PR #19"): + ephemeral.preflight(request(), policy.EXPECTED_REPO, root) + + +def test_worktree_attestation_rejects_each_malformed_local_source( + tmp_path: Path, +) -> None: + roots = [] + empty = tmp_path / "empty" + empty.mkdir() + roots.append(empty) + bad_marker = linked_worktree(tmp_path / "marker") + (bad_marker / ".git").write_text("not-a-gitdir\n", encoding="utf-8") + roots.append(bad_marker) + wrong_sha = linked_worktree(tmp_path / "sha") + gitdir = Path((wrong_sha / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "refs" / "heads" / ephemeral.EXPECTED_HEAD_REF).write_text("0" * 40) + roots.append(wrong_sha) + wrong_main = linked_worktree(tmp_path / "main") + gitdir = Path((wrong_main / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "refs" / "remotes" / "origin" / "main").write_text("0" * 40) + roots.append(wrong_main) + bad_config = linked_worktree(tmp_path / "config") + gitdir = Path((bad_config / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "config").write_text("[broken\n", encoding="utf-8") + roots.append(bad_config) + wrong_remote = linked_worktree(tmp_path / "remote") + gitdir = Path((wrong_remote / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "config").write_text('[remote "origin"]\nurl=https://example.invalid/x\n') + roots.append(wrong_remote) + for root in roots: + with pytest.raises((ephemeral.ArmingError, FileNotFoundError)): + ephemeral.preflight(request(), policy.EXPECTED_REPO, root) + oversized = tmp_path / "oversized" + oversized.write_text("x" * 5, encoding="utf-8") + with pytest.raises(ephemeral.ArmingError): + ephemeral._bounded_read(oversized, 4) + + +def test_guard_failure_stops_before_runner_calls(monkeypatch) -> None: + monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None) + runner, spawn = armed_runner([]) results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - push = results["ephemeral.feature-branch-push"] + assert results["ephemeral.protected-branch-refusal"].status == model.FAIL + assert all( + result.status in {model.FAIL, model.NOT_RUN} for result in results.values() + ) + assert spawn.calls == [] - assert push.status == model.FAIL - assert "policy" in push.reason - assert not any(call["argv"][:2] == ["git", "push"] for call in spawn.calls) + +def test_full_armed_run_validates_and_removes_exact_artifacts() -> None: + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, spawn = armed_runner( + [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + completed(stdout=json.dumps(pr_payload())), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ] + ) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert all(result.status == model.PASS for result in results.values()) + push = next(call for call in spawn.calls if call["argv"][1] == "push") + assert push["argv"][-1] == f"HEAD:refs/heads/{request().ref}" + assert all(result.spec.mandatory for result in results.values()) + + +def test_existing_or_unverifiable_ref_never_reaches_a_write() -> None: + for first in (completed(stdout="abc\trefs/heads/x\n"), completed(returncode=128)): + runner, spawn = armed_runner([first]) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert not any( + call["argv"][1] in {"push", "POST", "PATCH", "DELETE"} + for call in spawn.calls + ) + + +def test_failed_or_ambiguous_create_is_discovered_and_cleanup_still_runs() -> None: + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, _ = armed_runner( + [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + completed(returncode=1), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ] + ) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.draft-pull-request"].status == model.PASS + assert results["ephemeral.cleanup-verified"].status == model.PASS + + +def test_pull_request_parsers_reject_every_malformed_or_ambiguous_shape() -> None: + assert ephemeral._pr_problems([], request()) == ["pull request is not an object"] + malformed = pr_payload() + malformed.update({"number": True, "draft": "yes", "base": None}) + malformed["head"] = {"ref": "other", "sha": "bad"} + problems = ephemeral._pr_problems(malformed, request()) + assert {"number is malformed", "draft does not match", "base is malformed"} <= set( + problems + ) + assert ephemeral._discover(exec_module.Outcome((), "x", returncode=1), request())[1] + for body in ( + "not-json", + "{}", + "[null]", + json.dumps([{"head": {"ref": "other"}, "base": {}}]), + ): + numbers, error = ephemeral._discover( + exec_module.Outcome((), "x", 0, body), request() + ) + assert numbers == [] and error + invalid = pr_payload() + invalid["draft"] = False + assert ephemeral._discover( + exec_module.Outcome((), "x", 0, json.dumps([invalid])), request() + )[1] + assert ( + "discovered 0" + in ephemeral._discover(exec_module.Outcome((), "x", 0, "[]"), request())[1] + ) + + +@pytest.mark.parametrize( + "delete_result,remaining,closed", + [ + (completed(returncode=1), completed(), pr_payload(state="closed")), + (completed(), completed(returncode=128), pr_payload(state="closed")), + (completed(), completed(), {}), + (completed(), completed(), pr_payload(state="open")), + ], +) +def test_cleanup_fails_closed_on_partial_or_malformed_evidence( + delete_result, remaining, closed +) -> None: + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, _ = armed_runner( + [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + completed(), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + delete_result, + remaining, + completed(stdout=json.dumps(closed)), + ] + ) + cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[ + "ephemeral.cleanup-verified" + ] + assert cleanup.status == model.FAIL + + +def test_cleanup_fails_when_close_or_verification_commands_are_bad_json_or_failures() -> ( + None +): + api = "/api/v1/repos/atlas/titan-iac" + for scripted in ( + [completed(returncode=1), completed(), completed(), completed(returncode=1)], + [completed(stdout="bad"), completed(), completed(), completed(stdout="bad")], + [ + completed(stdout=json.dumps(pr_payload())), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ], + ): + runner, _ = armed_runner(scripted) + result = ephemeral._cleanup(runner, VANTAGE, request(), api, [42]) + assert result.status == model.FAIL + + +def test_read_only_runner_cannot_reach_push() -> None: + runner, spawn = armed_runner( + [completed(), completed(stdout=f"{HEAD}\n"), completed(), completed()], + mode=policy.READ_ONLY, + ) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert not any(call["argv"][1] == "push" for call in spawn.calls) diff --git a/testing/tests/test_hermes_handoff_evaluators.py b/testing/tests/test_hermes_handoff_evaluators.py index 2e340c7b..4de16c7e 100644 --- a/testing/tests/test_hermes_handoff_evaluators.py +++ b/testing/tests/test_hermes_handoff_evaluators.py @@ -2,14 +2,25 @@ from __future__ import annotations -from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step +import json + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) evaluators = load_handoff_module("hermes_handoff_evaluators") model = load_handoff_module("hermes_handoff_model") REVIEW = model.REVIEW ATTEMPT = model.ATTEMPT -FORBIDDEN = outcome(stderr="Error from server (Forbidden): pods is forbidden", returncode=1) +FORBIDDEN = outcome( + stderr="Error from server (Forbidden): pods is forbidden", returncode=1 +) def deny_spec() -> object: @@ -24,21 +35,27 @@ def evaluate(built, outcomes): def test_a_deny_check_passes_only_on_a_review_no_and_a_live_refusal() -> None: - evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN}) + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN} + ) assert evaluation.status == model.PASS assert evaluation.evidence == {"review": "no", "attempt": "refused"} def test_a_deny_check_fails_when_the_review_says_the_authority_exists() -> None: - evaluation = evaluate(deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN}) + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN} + ) assert evaluation.status == model.FAIL assert "reported yes" in evaluation.reason def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> None: - evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()}) + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()} + ) assert evaluation.status == model.FAIL assert "must be refused" in evaluation.reason @@ -46,7 +63,9 @@ def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> No def test_a_request_that_merely_broke_is_not_evidence_of_a_refusal() -> None: broken = outcome(stderr="connection refused", returncode=7) - evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": broken}) + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="no"), "attempt": broken} + ) assert evaluation.status == model.NOT_RUN assert "recognisable refusal" in evaluation.reason @@ -62,15 +81,22 @@ def test_a_deny_check_without_a_live_attempt_refuses_to_classify() -> None: assert "require a real attempt" in evaluation.reason -def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> None: +def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> ( + None +): built = spec("allowed", steps=(step("review", kind=REVIEW), step("read"))) - assert evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status == model.PASS + assert ( + evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status + == model.PASS + ) denied = evaluate(built, {"review": outcome(stdout="no"), "read": outcome()}) assert denied.status == model.FAIL and "reported no" in denied.reason - failed = evaluate(built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)}) + failed = evaluate( + built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)} + ) assert failed.status == model.FAIL and "step read failed" in failed.reason @@ -84,7 +110,9 @@ def test_names_absent_flags_exact_names_and_substrings() -> None: clean = evaluate(built, {"env": outcome(stdout="PATH\nHOME\n")}) assert clean.status == model.PASS and clean.evidence["observed_count"] == 2 - leaked = evaluate(built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")}) + leaked = evaluate( + built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")} + ) assert leaked.status == model.FAIL assert leaked.evidence["offenders"] == ["GITEA_TOKEN", "OPENAI_API_KEY"] @@ -92,10 +120,16 @@ def test_names_absent_flags_exact_names_and_substrings() -> None: assert unreadable.status == model.NOT_RUN -def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> None: +def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> ( + None +): built = spec( "names_present", - {"step": "init", "names": ("patch-web-session-activity",), "contains": ("@sha256:",)}, + { + "step": "init", + "names": ("patch-web-session-activity",), + "contains": ("@sha256:",), + }, steps=(step("init"),), ) @@ -110,14 +144,18 @@ def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty( def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None: - single = spec("stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),)) + single = spec( + "stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),) + ) assert evaluate(single, {"s": outcome(stdout=" locked \n")}).status == model.PASS mismatch = evaluate(single, {"s": outcome(stdout="unlocked")}) assert mismatch.status == model.FAIL and mismatch.evidence["observed"] == "unlocked" several = spec( - "stdout_matches", {"step": "s", "equals": ["600 root:root", "absent"]}, steps=(step("s"),) + "stdout_matches", + {"step": "s", "equals": ["600 root:root", "absent"]}, + steps=(step("s"),), ) assert evaluate(several, {"s": outcome(stdout="absent")}).status == model.PASS @@ -126,7 +164,9 @@ def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None: def test_distinct_count_separates_replica_count_from_actual_spread() -> None: built = spec( - "distinct_count", {"step": "n", "minimum": 3, "total_equals": 3}, steps=(step("n"),) + "distinct_count", + {"step": "n", "minimum": 3, "total_equals": 3}, + steps=(step("n"),), ) spread = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-06\ntitan-07\n")}) @@ -135,7 +175,11 @@ def test_distinct_count_separates_replica_count_from_actual_spread() -> None: stacked = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-05\ntitan-05\n")}) assert stacked.status == model.FAIL and "distinct values" in stacked.reason - extra = spec("distinct_count", {"step": "n", "minimum": 1, "total_equals": 2}, steps=(step("n"),)) + extra = spec( + "distinct_count", + {"step": "n", "minimum": 1, "total_equals": 2}, + steps=(step("n"),), + ) over = evaluate(extra, {"n": outcome(stdout="a\nb\nc\n")}) assert over.status == model.FAIL and "expected exactly 2" in over.reason @@ -152,16 +196,22 @@ def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None: agree = evaluate( built, - {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/b\nns/a\n")}, + { + "operator": outcome(stdout="ns/a\nns/b\n"), + "self": outcome(stdout="ns/b\nns/a\n"), + }, ) assert agree.status == model.PASS differ = evaluate( - built, {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")} + built, + {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")}, ) assert differ.status == model.FAIL and "disagree" in differ.reason - empty = evaluate(built, {"operator": outcome(stdout=""), "self": outcome(stdout="")}) + empty = evaluate( + built, {"operator": outcome(stdout=""), "self": outcome(stdout="")} + ) assert empty.status == model.NOT_RUN broken = evaluate(built, {"operator": outcome(returncode=1), "self": outcome()}) @@ -169,7 +219,9 @@ def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None: def test_steps_agree_is_the_same_rule_under_a_neutral_name() -> None: - assert evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"] + assert ( + evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"] + ) def test_lines_match_catches_a_blank_leading_column() -> None: @@ -186,7 +238,10 @@ def test_lines_match_catches_a_blank_leading_column() -> None: failed = evaluate(built, {"s": outcome(stdout=blank)}) assert failed.status == model.FAIL and failed.evidence["offenders"] - assert evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status == model.NOT_RUN + assert ( + evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status + == model.NOT_RUN + ) assert evaluate(built, {"s": outcome(returncode=1)}).status == model.NOT_RUN @@ -198,4 +253,205 @@ def test_not_armed_is_the_fixed_skip_a_default_run_must_produce() -> None: assert evaluation.status == model.NOT_RUN assert evaluation.evidence == {"armed": False} assert evaluation.reason == "ephemeral mutation mode is not armed" - assert evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed" + assert ( + evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed" + ) + + +def test_truncation_fails_closed_before_every_registered_evaluator() -> None: + """No evaluator may reinterpret a clipped required capture as PASS.""" + clipped = outcome(stdout="apparently good", truncated=True) + for name in evaluators.EVALUATORS: + built = spec(name, {"step": "s"}, steps=(step("s"),)) + evaluation = evaluate(built, {"s": clipped}) + assert evaluation.status == model.NOT_RUN, name + assert "truncated" in evaluation.reason, name + + +def test_missing_required_steps_fail_closed_inside_every_step_evaluator() -> None: + for name in evaluators.EVALUATORS: + if name == "not_armed": + continue + built = spec(name, {"step": "s"}, steps=(step("s"),)) + assert evaluate(built, {}).status == model.NOT_RUN, name + + +def test_pool_topology_requires_three_ordinals_nodes_claims_and_ready_state() -> None: + built = spec( + "pool_topology", + { + "state_step": "state", + "worker_step": "workers", + "replicas": 3, + "name": "worker", + }, + steps=(step("state"), step("workers")), + ) + state = outcome(stdout="3\t3\t3\tfalse") + workers = outcome( + stdout=( + "worker-0\tnode-a\tRunning\tTrue\tclaim-0,\n" + "worker-1\tnode-b\tRunning\tTrue\tclaim-1,\n" + "worker-2\tnode-c\tRunning\tTrue\tclaim-2,\n" + ) + ) + assert evaluate(built, {"state": state, "workers": workers}).status == model.PASS + stacked = outcome(stdout=workers.stdout.replace("node-b", "node-a")) + assert evaluate(built, {"state": state, "workers": stacked}).status == model.FAIL + assert ( + evaluate(built, {"state": outcome(stdout="3\t3"), "workers": workers}).status + == model.NOT_RUN + ) + assert ( + evaluate( + built, {"state": outcome(stdout="x\t3\t3\tfalse"), "workers": workers} + ).status + == model.NOT_RUN + ) + assert ( + evaluate( + built, {"state": outcome(stdout="3\t2\t3\tfalse"), "workers": workers} + ).status + == model.FAIL + ) + malformed = outcome(stdout="worker-0\tnode-a\n") + assert ( + evaluate(built, {"state": state, "workers": malformed}).status == model.NOT_RUN + ) + bad_worker = outcome(stdout=workers.stdout.replace("Running", "Pending", 1)) + assert evaluate(built, {"state": state, "workers": bad_worker}).status == model.FAIL + duplicate = outcome(stdout=workers.stdout.replace("worker-1", "worker-0")) + assert evaluate(built, {"state": state, "workers": duplicate}).status == model.FAIL + wrong_ordinals = outcome(stdout=workers.stdout.replace("worker-2", "worker-3")) + assert ( + evaluate(built, {"state": state, "workers": wrong_ordinals}).status + == model.FAIL + ) + + +MAIN = "1" * 40 +HEAD = "2" * 40 +BUILD = MAIN +DIGEST = "sha256:" + "4" * 64 +IMAGE = f"registry.example/hermes:git-{BUILD}-build-7@{DIGEST}" + + +def lineage_spec(): + keys = ( + "remote-main", + "reviewed-pr", + "build-source", + "deployment", + "pods", + "replicasets", + "flux", + ) + return spec( + "release_lineage", + { + "main_sha": MAIN, + "head_sha": HEAD, + "head_ref": "feature/hermes-full-handoff-acceptance", + "pr_number": 19, + "image": IMAGE, + "build_sha": BUILD, + "deployment_revision": "7", + }, + steps=tuple(step(key) for key in keys), + ) + + +def lineage_outcomes(): + pull = { + "number": 19, + "state": "open", + "draft": True, + "merged": False, + "base": {"ref": "main", "sha": MAIN}, + "head": {"ref": "feature/hermes-full-handoff-acceptance", "sha": HEAD}, + } + return { + "remote-main": outcome(stdout=f"{MAIN}\trefs/heads/main\n"), + "reviewed-pr": outcome(stdout=json.dumps(pull)), + "build-source": outcome(stdout=BUILD), + "deployment": outcome(stdout=f"5\t5\t2\t2\t7\t{IMAGE}"), + "replicasets": outcome( + stdout=f"rs-old\t6\t0\t0\t{IMAGE}\told\nrs-live\t7\t2\t2\t{IMAGE}\thash7\n" + ), + "pods": outcome( + stdout=( + f"pod-a\tRunning\tTrue\t{IMAGE}\tdocker-pullable://hermes@{DIGEST}\thash7\n" + f"pod-b\tRunning\tTrue\t{IMAGE}\tcontainerd://hermes@{DIGEST}\thash7\n" + ) + ), + "flux": outcome(stdout=f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{MAIN}"), + } + + +def test_release_lineage_atomically_binds_every_release_identity() -> None: + assert evaluate(lineage_spec(), lineage_outcomes()).status == model.PASS + + +@pytest.mark.parametrize( + ("key", "bad"), + [ + ("remote-main", ""), + ("reviewed-pr", "{}"), + ("build-source", HEAD), + ("deployment", "5\t4\t2\t2\t7\tbad"), + ("replicasets", "malformed"), + ("pods", "malformed"), + ("flux", "hermes\t9\t9\tfalse\tUnknown\tmain@sha1:" + MAIN), + ], +) +def test_release_lineage_fails_closed_on_each_malformed_or_mismatched_source( + key, bad +) -> None: + outcomes = lineage_outcomes() + outcomes[key] = outcome(stdout=bad) + assert evaluate(lineage_spec(), outcomes).status in {model.FAIL, model.NOT_RUN} + + +def test_release_lineage_rejects_structural_partial_failure_variants() -> None: + variants = [] + malformed_expect = lineage_spec() + malformed_expect = model.CheckSpec( + **{ + **malformed_expect.__dict__, + "expect": {**malformed_expect.expect, "main_sha": "bad"}, + } + ) + variants.append((malformed_expect, lineage_outcomes())) + malformed_image = lineage_spec() + malformed_image = model.CheckSpec( + **{ + **malformed_image.__dict__, + "expect": { + **malformed_image.expect, + "image": f"registry.example/hermes@{DIGEST}", + }, + } + ) + variants.append((malformed_image, lineage_outcomes())) + for key, value in ( + ("reviewed-pr", "[]"), + ("deployment", f"x\tx\t2\t2\t7\t{IMAGE}"), + ("replicasets", f"rs\t7\t1\t2\t{IMAGE}\thash7"), + ("replicasets", f"a\t7\t2\t2\t{IMAGE}\th1\nb\t7\t2\t2\t{IMAGE}\th2"), + ("replicasets", f"rs\t6\t2\t2\t{IMAGE}\thash7"), + ("pods", f"pod-a\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7"), + ( + "pods", + f"pod-a\tPending\tFalse\t{IMAGE}\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7", + ), + ( + "pods", + f"pod-a\tRunning\tTrue\tbad\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7", + ), + ("flux", f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{HEAD}"), + ): + outcomes = lineage_outcomes() + outcomes[key] = outcome(stdout=value) + variants.append((lineage_spec(), outcomes)) + for built, outcomes in variants: + assert evaluate(built, outcomes).status in {model.FAIL, model.NOT_RUN} diff --git a/testing/tests/test_hermes_handoff_exec.py b/testing/tests/test_hermes_handoff_exec.py index cd8fc7d2..c192448d 100644 --- a/testing/tests/test_hermes_handoff_exec.py +++ b/testing/tests/test_hermes_handoff_exec.py @@ -2,7 +2,11 @@ from __future__ import annotations +import hashlib import subprocess +import time + +import pytest from testing.tests.test_hermes_handoff_support import ( FakeClock, @@ -15,35 +19,48 @@ policy = load_handoff_module("hermes_handoff_policy") def completed(stdout: str = "", stderr: str = "", returncode: int = 0): - return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) def build(results: list, **kwargs) -> tuple: clock = FakeClock() spawn = FakeSpawn(results) - runner = runner_module.Runner(clock=clock, spawn=spawn, environ={}, **kwargs) + runner = runner_module.Runner( + clock=clock, + spawn=spawn, + environ={}, + attestor=lambda command, _environment: runner_module.Attestation( + command, "a" * 64 + ), + **kwargs, + ) return (runner, spawn, clock) OPERATOR = runner_module.operator_vantage() +SAFE_GET = ("kubectl", "get", "pods", "-o", "name") def test_a_successful_command_is_captured_and_screened() -> None: runner, spawn, _ = build([completed(stdout="password=a-long-enough-secret\n")]) - outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + outcome = runner.run(SAFE_GET, OPERATOR) assert outcome.ok and outcome.ran assert "[redacted]" in outcome.stdout assert outcome.returncode == 0 - assert spawn.calls[0]["argv"] == ["kubectl", "get", "pods"] + assert spawn.calls[0]["argv"] == list(SAFE_GET) assert runner.commands_run == 1 def test_a_failing_command_is_not_ok_but_still_ran() -> None: - runner, _, _ = build([completed(stderr="Error from server (Forbidden)", returncode=1)]) + runner, _, _ = build( + [completed(stderr="Error from server (Forbidden)", returncode=1)] + ) - outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + outcome = runner.run(SAFE_GET, OPERATOR) assert outcome.ran and not outcome.ok assert "Forbidden" in outcome.combined @@ -63,7 +80,7 @@ def test_a_policy_violation_never_reaches_a_subprocess() -> None: def test_a_timeout_is_recorded_rather_than_raised() -> None: runner, _, _ = build([subprocess.TimeoutExpired(cmd="kubectl", timeout=1.0)]) - outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + outcome = runner.run(SAFE_GET, OPERATOR) assert outcome.error == runner_module.TIMEOUT_ERROR assert not outcome.ran @@ -72,7 +89,7 @@ def test_a_timeout_is_recorded_rather_than_raised() -> None: def test_a_missing_binary_is_recorded_rather_than_raised() -> None: runner, _, _ = build([FileNotFoundError(2, "No such file or directory")]) - outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + outcome = runner.run(SAFE_GET, OPERATOR) assert outcome.error and outcome.error.startswith("spawn:") @@ -80,9 +97,9 @@ def test_a_missing_binary_is_recorded_rather_than_raised() -> None: def test_the_run_deadline_stops_further_probes_without_spawning() -> None: runner, spawn, clock = build([completed(stdout="first")], deadline_seconds=10.0) - assert runner.run(("kubectl", "get", "pods"), OPERATOR).ok + assert runner.run(SAFE_GET, OPERATOR).ok clock.advance(11.0) - second = runner.run(("kubectl", "get", "nodes"), OPERATOR) + second = runner.run(("kubectl", "get", "nodes", "-o", "name"), OPERATOR) assert second.error == runner_module.DEADLINE_ERROR assert len(spawn.calls) == 1 @@ -90,29 +107,42 @@ def test_the_run_deadline_stops_further_probes_without_spawning() -> None: def test_the_command_timeout_is_clamped_by_the_remaining_deadline() -> None: - runner, spawn, clock = build([completed()], command_timeout=30.0, deadline_seconds=10.0) + runner, spawn, clock = build( + [completed()], command_timeout=30.0, deadline_seconds=10.0 + ) clock.advance(6.0) - runner.run(("kubectl", "get", "pods"), OPERATOR) + runner.run(SAFE_GET, OPERATOR) assert spawn.calls[0]["timeout"] == 4.0 def test_output_is_bounded_per_command_and_can_be_raised_per_call() -> None: - runner, _, _ = build([completed(stdout="y" * 200), completed(stdout="y" * 200)], max_bytes=50) + runner, _, _ = build( + [completed(stdout="y" * 200), completed(stdout="y" * 200)], max_bytes=50 + ) - small = runner.run(("kubectl", "get", "pods"), OPERATOR) - large = runner.run(("kubectl", "get", "pods"), OPERATOR, max_bytes=4096) + small = runner.run(SAFE_GET, OPERATOR) + large = runner.run(SAFE_GET, OPERATOR, max_bytes=4096) assert small.truncated assert not large.truncated def test_the_child_environment_is_rebuilt_from_an_allowlist() -> None: - source = {"PATH": "/bin", "KUBECONFIG": "/tmp/kc", "ANTHROPIC_API_KEY": "leak", "HOME": "/root"} + source = { + "PATH": "/bin", + "KUBECONFIG": "/tmp/kc", + "ANTHROPIC_API_KEY": "leak", + "HOME": "/root", + } environment = runner_module.build_environment(OPERATOR, source) - assert environment == {"PATH": "/bin", "KUBECONFIG": "/tmp/kc", "HOME": "/root", "LC_ALL": "C"} + assert environment["PATH"] == runner_module.SAFE_PATH + assert environment["KUBECONFIG"] == "/tmp/kc" + assert environment["GITEA_BASE_URL"] == "https://scm.bstein.dev" + assert environment["GIT_CONFIG_GLOBAL"] == "/dev/null" + assert "HOME" not in environment assert "ANTHROPIC_API_KEY" not in environment @@ -127,7 +157,11 @@ def test_a_vantage_can_override_the_environment_it_needs() -> None: def test_the_operator_context_is_inserted_only_for_context_aware_tools() -> None: vantage = runner_module.operator_vantage(context="atlas-operator") - assert vantage.wrap(("kubectl", "get", "pods"))[:3] == ("kubectl", "--context", "atlas-operator") + assert vantage.wrap(("kubectl", "get", "pods"))[:3] == ( + "kubectl", + "--context", + "atlas-operator", + ) assert vantage.wrap(("git", "status")) == ("git", "status") assert vantage.wrap(()) == () @@ -136,11 +170,18 @@ def test_the_pod_vantage_execs_into_the_container_and_drops_the_context() -> Non operator = runner_module.operator_vantage(context="atlas-operator") vantage = runner_module.pod_vantage("hermes", "hermes-agent-1", "hermes", operator) - wrapped = vantage.wrap(("kubectl", "get", "pods")) + wrapped = vantage.wrap(SAFE_GET) assert wrapped[:3] == ("kubectl", "--context", "atlas-operator") assert wrapped[3] == "exec" - assert wrapped[-4:] == ("--", "kubectl", "get", "pods") + assert wrapped[-6:] == ( + "--", + "/usr/local/bin/kubectl", + "get", + "pods", + "-o", + "name", + ) assert vantage.description == "in-pod hermes/hermes-agent-1[hermes]" policy.check_argv(wrapped) @@ -168,6 +209,155 @@ def test_outcome_serialisation_screens_the_command_line() -> None: def test_combined_output_joins_only_the_streams_that_carry_text() -> None: - assert runner_module.Outcome(argv=(), vantage="x", stdout="a", stderr="b").combined == "a\nb" + assert ( + runner_module.Outcome(argv=(), vantage="x", stdout="a", stderr="b").combined + == "a\nb" + ) assert runner_module.Outcome(argv=(), vantage="x", stdout="a").combined == "a" assert runner_module.Outcome(argv=(), vantage="x").combined == "" + + +@pytest.mark.parametrize( + "kwargs", + [ + {"command_timeout": float("nan")}, + {"command_timeout": float("inf")}, + {"command_timeout": 0}, + {"deadline_seconds": -1}, + {"max_bytes": -1}, + {"max_bytes": 10**100}, + {"concurrency": 0}, + {"concurrency": 17}, + ], +) +def test_execution_bounds_reject_nonfinite_nonpositive_or_unbounded_inputs( + kwargs, +) -> None: + with pytest.raises(ValueError): + runner_module.Runner(**kwargs) + + +def test_execution_bounds_reject_boolean_and_nonnumeric_inputs() -> None: + for value in (True, "1", None): + with pytest.raises(ValueError, match="numeric"): + runner_module.Runner(command_timeout=value) + + +def test_every_reviewer_unsafe_argv_is_refused_without_spawning() -> None: + runner, spawn, _ = build([]) + commands = [ + ("kubectl", "auth", "reconcile", "-f", "x"), + ("kubectl", "config", "view", "--raw"), + ("kubectl", "get", "secret/x", "-o", "name"), + ("kubectl", "get", "--raw=/api/v1/secrets"), + ("kubectl", "delete", "pod/x", "--dry-run=client"), + ("kubectl", "delete", "pod/x", "--", "--dry-run=server"), + ("git", "fetch", "origin"), + ("git", "config", "core.sshCommand", "x"), + ("git", "-ccore.sshCommand=x", "ls-remote", "origin"), + ("helm", "get", "values", "x"), + ] + for command in commands: + assert runner.run(command, OPERATOR).error.startswith("policy:") + assert spawn.calls == [] + assert runner.commands_run == 0 + + +def test_live_capture_bounds_concurrent_stdout_and_stderr() -> None: + capture = runner_module.execute_bounded( + ( + "/usr/bin/python3", + "-c", + "import os;os.write(1,b'x'*2000000);os.write(2,b'y'*2000000)", + ), + {"LC_ALL": "C"}, + timeout=5, + max_bytes=1024, + ) + assert capture.returncode == 0 + assert capture.stdout_truncated and capture.stderr_truncated + assert len(capture.stdout) == len(capture.stderr) == 1024 + + +def test_absolute_timeout_kills_the_process_group_and_reaps_quickly() -> None: + started = time.monotonic() + capture = runner_module.execute_bounded( + ( + "/usr/bin/python3", + "-c", + "import os,signal,time;" + "p=os.fork();" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "time.sleep(30)", + ), + {"LC_ALL": "C"}, + timeout=0.15, + max_bytes=1024, + ) + assert capture.timed_out + assert time.monotonic() - started < 1.5 + + +def test_descendant_holding_pipes_is_bounded_after_the_leader_exits() -> None: + started = time.monotonic() + capture = runner_module.execute_bounded( + ( + "/usr/bin/python3", + "-c", + "import os,signal,time;" + "p=os.fork();" + "(os._exit(0) if p else None);" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "time.sleep(30)", + ), + {"LC_ALL": "C"}, + timeout=2, + max_bytes=1024, + ) + assert not capture.timed_out + assert time.monotonic() - started < 1.5 + + +def test_attestation_rejects_a_reviewer_authored_executable(tmp_path) -> None: + fake = tmp_path / "kubectl" + fake.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + fake.chmod(0o755) + with pytest.raises(OSError, match="trusted paths"): + runner_module.attest_executable(str(fake), {"PATH": str(tmp_path)}) + + +def test_attestation_checks_presence_mode_digest_and_success( + tmp_path, monkeypatch +) -> None: + fake = tmp_path / "tool" + fake.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + fake.chmod(0o755) + digest = hashlib.sha256(fake.read_bytes()).hexdigest() + monkeypatch.setitem(runner_module.EXPECTED_PATHS, "tool", {str(fake)}) + monkeypatch.setitem(runner_module.EXPECTED_SHA256, "tool", {digest}) + assert ( + runner_module.attest_executable("tool", {"PATH": str(tmp_path)}).sha256 + == digest + ) + with pytest.raises(OSError, match="unavailable"): + runner_module.attest_executable("missing", {"PATH": str(tmp_path)}) + fake.chmod(0o777) + with pytest.raises(OSError, match="unsafe ownership mode"): + runner_module.attest_executable("tool", {"PATH": str(tmp_path)}) + fake.chmod(0o755) + monkeypatch.setitem(runner_module.EXPECTED_SHA256, "tool", {"0" * 64}) + with pytest.raises(OSError, match="digest"): + runner_module.attest_executable("tool", {"PATH": str(tmp_path)}) + + +def test_runner_attestation_or_resolved_policy_failure_never_spawns() -> None: + runner, spawn, _ = build([]) + runner._attestor = lambda _command, _environment: (_ for _ in ()).throw( + OSError("no") + ) + assert runner.run(SAFE_GET, OPERATOR).error.startswith("attestation:") + runner._attestor = lambda _command, _environment: runner_module.Attestation( + "/tmp/reviewer-authored/kubectl", "a" * 64 + ) + assert runner.run(SAFE_GET, OPERATOR).error.startswith("policy:") + assert spawn.calls == [] diff --git a/testing/tests/test_hermes_handoff_json_rules.py b/testing/tests/test_hermes_handoff_json_rules.py index 9bb5c673..e74684d7 100644 --- a/testing/tests/test_hermes_handoff_json_rules.py +++ b/testing/tests/test_hermes_handoff_json_rules.py @@ -5,7 +5,14 @@ from __future__ import annotations import datetime as dt import json -from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step +import pytest + +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) evaluators = load_handoff_module("hermes_handoff_evaluators") model = load_handoff_module("hermes_handoff_model") @@ -23,9 +30,13 @@ def one(rule: str, expect: dict): def test_json_field_compares_paths_against_expected_values() -> None: - built = one("json_field", {"fields": {"authenticated": True, "transport": ["a", "b"]}}) + built = one( + "json_field", {"fields": {"authenticated": True, "transport": ["a", "b"]}} + ) - assert evaluate(built, {"authenticated": True, "transport": "b"}).status == model.PASS + assert ( + evaluate(built, {"authenticated": True, "transport": "b"}).status == model.PASS + ) wrong = evaluate(built, {"authenticated": False, "transport": "c"}) assert wrong.status == model.FAIL @@ -40,6 +51,15 @@ def test_json_field_skips_rather_than_guesses_on_unusable_output() -> None: assert evaluate(built, "not json").status == model.NOT_RUN failed = evaluators.evaluate(built, {"s": outcome(returncode=1)}) assert failed.status == model.NOT_RUN + assert evaluate(built, '{"a": 1, "a": 1}').status == model.NOT_RUN + assert evaluate(built, '{"a": NaN}').status == model.NOT_RUN + + +def test_json_equality_does_not_confuse_booleans_and_integers() -> None: + built = one("json_field", {"fields": {"enabled": True}}) + assert evaluate(built, {"enabled": 1}).status == model.FAIL + array = one("all_of_field", {"fields": {"enabled": False}}) + assert evaluate(array, [{"enabled": 0}]).status == model.FAIL def test_json_numeric_enforces_inclusive_bounds_and_rejects_non_numbers() -> None: @@ -63,20 +83,32 @@ def test_json_recent_treats_stale_evidence_as_a_failure() -> None: built = one("json_recent", {"now": NOW, "fields": {"checked_at": 3600}}) fresh = evaluate(built, {"checked_at": "2026-08-17T11:30:00Z"}) - assert fresh.status == model.PASS and fresh.evidence["checked_at"]["age_seconds"] == 1800 + assert ( + fresh.status == model.PASS + and fresh.evidence["checked_at"]["age_seconds"] == 1800 + ) stale = evaluate(built, {"checked_at": "2026-08-16T00:00:00Z"}) assert stale.status == model.FAIL and "limit 3600s" in stale.reason naive = evaluate(built, {"checked_at": "2026-08-17T11:30:00"}) - assert naive.status == model.PASS + assert naive.status == model.NOT_RUN + assert evaluate(built, {"checked_at": "2026-08-17T12:00:01Z"}).status == model.FAIL assert evaluate(built, {"checked_at": "yesterday"}).status == model.NOT_RUN assert evaluate(built, {}).status == model.NOT_RUN + invalid_clock = one("json_recent", {"now": "now", "fields": {"checked_at": 1}}) + assert ( + evaluate(invalid_clock, {"checked_at": "2026-08-17T11:30:00Z"}).status + == model.NOT_RUN + ) def test_all_of_field_selects_keys_and_reports_every_mismatch() -> None: - built = one("all_of_field", {"key_field": "number", "keys": [14, 15], "fields": {"merged": True}}) + built = one( + "all_of_field", + {"key_field": "number", "keys": [14, 15], "fields": {"merged": True}}, + ) payload = [ {"number": 14, "merged": True}, {"number": 15, "merged": False}, @@ -92,7 +124,10 @@ def test_all_of_field_selects_keys_and_reports_every_mismatch() -> None: def test_all_of_field_skips_when_an_expected_key_is_absent_or_nothing_matched() -> None: - keyed = one("all_of_field", {"key_field": "number", "keys": [14], "fields": {"merged": True}}) + keyed = one( + "all_of_field", + {"key_field": "number", "keys": [14], "fields": {"merged": True}}, + ) absent = evaluate(keyed, [{"number": 99, "merged": True}]) assert absent.status == model.NOT_RUN and "expected keys absent" in absent.reason @@ -103,7 +138,9 @@ def test_all_of_field_skips_when_an_expected_key_is_absent_or_nothing_matched() def test_all_of_field_skips_when_the_named_array_path_is_absent() -> None: built = spec( - "all_of_field", {"step": "s", "array": "items", "fields": {"a": 1}}, steps=(step("s"),) + "all_of_field", + {"step": "s", "array": "items", "fields": {"a": 1}}, + steps=(step("s"),), ) evaluation = evaluate(built, {"other": []}) @@ -124,9 +161,16 @@ def test_all_of_field_treats_an_absent_field_as_a_mismatch_not_a_crash() -> None def test_all_of_field_honours_exemptions_and_non_empty_paths() -> None: exempted = one( "all_of_field", - {"key_field": "name", "exempt_keys": ["parked"], "fields": {"suspended": False}}, + { + "key_field": "name", + "exempt_keys": ["parked"], + "fields": {"suspended": False}, + }, ) - payload = [{"name": "parked", "suspended": True}, {"name": "live", "suspended": False}] + payload = [ + {"name": "parked", "suspended": True}, + {"name": "live", "suspended": False}, + ] assert evaluate(exempted, payload).status == model.PASS non_empty = one("all_of_field", {"key_field": "id", "non_empty": ["title"]}) @@ -136,8 +180,13 @@ def test_all_of_field_honours_exemptions_and_non_empty_paths() -> None: def test_json_record_captures_named_paths_or_skips() -> None: - built = one("json_record", {"record": {"numbers": "[].number", "refs": "[].head.ref"}}) - payload = [{"number": 14, "head": {"ref": "a"}}, {"number": 15, "head": {"ref": "b"}}] + built = one( + "json_record", {"record": {"numbers": "[].number", "refs": "[].head.ref"}} + ) + payload = [ + {"number": 14, "head": {"ref": "a"}}, + {"number": 15, "head": {"ref": "b"}}, + ] evaluation = evaluate(built, payload) assert evaluation.status == model.PASS @@ -169,7 +218,11 @@ def routing_spec(**extra): FULL_LOG = routing_log( {"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}, {"ts": "2026-08-17T11:30:00Z", "model": "route/claude/opus/xhigh"}, - {"ts": "2026-08-17T11:45:00Z", "model": "worker/codex/sol/xhigh", "fallback_reason": "unavailable"}, + { + "ts": "2026-08-17T11:45:00Z", + "model": "worker/codex/sol/xhigh", + "fallback_reason": "unavailable", + }, {"ts": "2026-08-17T11:50:00Z", "model": "qwen2.5:14b", "tier": "classifier"}, ) @@ -186,7 +239,9 @@ def test_routing_evidence_summarises_lanes_providers_efforts_and_fallbacks() -> def test_routing_evidence_names_exactly_what_the_window_did_not_show() -> None: - partial = routing_log({"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}) + partial = routing_log( + {"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"} + ) evaluation = evaluate(routing_spec(), partial) @@ -197,28 +252,74 @@ def test_routing_evidence_names_exactly_what_the_window_did_not_show() -> None: assert "fallback:none-observed" in evaluation.reason -def test_routing_evidence_tolerates_a_clipped_tail_but_not_an_empty_one() -> None: +def test_routing_evidence_rejects_a_clipped_tail_and_empty_input() -> None: clipped = '{"ts": "2026-08-17T11:0\n' + FULL_LOG - assert evaluate(routing_spec(), clipped).status == model.PASS + assert evaluate(routing_spec(), clipped).status == model.NOT_RUN assert evaluate(routing_spec(), "").status == model.NOT_RUN assert evaluate(routing_spec(), "garbage\n").status == model.NOT_RUN def test_routing_evidence_can_require_freshness() -> None: fresh = evaluate(routing_spec(max_age_seconds=3600, now=NOW), FULL_LOG) - assert fresh.status == model.PASS and fresh.evidence["age_seconds"] == 600 + assert fresh.status == model.PASS + assert max(fresh.evidence["age_seconds"].values()) <= 3600 stale = evaluate( routing_spec(max_age_seconds=60, now=NOW + dt.timedelta(days=2)), FULL_LOG ) assert stale.status == model.FAIL and "freshness" in stale.reason - undated = routing_log({"model": "route/codex/sol/high"}, {"model": "route/claude/opus/xhigh"}) + undated = routing_log( + {"model": "route/codex/sol/high"}, {"model": "route/claude/opus/xhigh"} + ) unknown = evaluate(routing_spec(max_age_seconds=60, now=NOW), undated) assert unknown.status == model.NOT_RUN +def test_routing_freshness_is_required_per_provider_not_only_globally() -> None: + rows = routing_log( + {"ts": "2026-08-15T11:00:00Z", "model": "route/claude/opus/high"}, + {"ts": "2026-08-17T11:59:00Z", "model": "route/codex/sol/xhigh"}, + { + "ts": "2026-08-17T11:59:00Z", + "model": "worker/codex/sol/high", + "fallback_reason": "down", + }, + ) + evaluation = evaluate(routing_spec(max_age_seconds=3600, now=NOW), rows) + assert evaluation.status == model.FAIL + assert "freshness:provider:claude" in evaluation.reason + missing_dimension = evaluate( + routing_spec(max_age_seconds=3600, now=NOW), + routing_log( + {"ts": "2026-08-17T11:59:00Z", "model": "route/codex/sol/high"}, + { + "ts": "2026-08-17T11:59:00Z", + "model": "worker/codex/sol/xhigh", + "fallback_reason": "down", + }, + ), + ) + assert missing_dimension.status == model.FAIL + + +@pytest.mark.parametrize( + "body", + [ + '["not-an-object"]', + '{"ts":"2026-08-17T12:00:01Z","model":"route/codex/sol/high"}', + '{"ts":"2026-08-17T11:00:00Z"}', + '{"ts":"2026-08-17T11:00:00Z","model":"bad"}', + '{"ts":"2026-08-17T11:00:00Z","model":"route/codex/sol/high","fallback_reason":0}', + ], +) +def test_routing_rows_fail_closed_on_structural_defects(body: str) -> None: + assert evaluate(routing_spec(now=NOW), body).status in {model.FAIL, model.NOT_RUN} + invalid_clock = routing_spec(now="now") + assert evaluate(invalid_clock, FULL_LOG).status == model.NOT_RUN + + def flux_spec(*expected: str): return spec( "flux_health", @@ -229,12 +330,14 @@ def flux_spec(*expected: str): def test_flux_health_separates_parked_objects_from_broken_ones() -> None: rows = ( - "flux-system/core\tfalse\tReady=True,Healthy=True,\n" - "flux-system/parked\ttrue\tReady=False,\n" - "flux-system/broken\tfalse\tReady=False,Healthy=False,\n" + "flux-system/core\t1\t1\tfalse\tReady=True,Healthy=True,\n" + "flux-system/parked\t1\t1\ttrue\tReady=False,\n" + "flux-system/broken\t1\t1\tfalse\tReady=False,Healthy=False,\n" ) - evaluation = evaluators.evaluate(flux_spec("flux-system/parked"), {"s": outcome(stdout=rows)}) + evaluation = evaluators.evaluate( + flux_spec("flux-system/parked"), {"s": outcome(stdout=rows)} + ) assert evaluation.status == model.FAIL assert evaluation.evidence["unhealthy"] == ["flux-system/broken"] @@ -243,7 +346,7 @@ def test_flux_health_separates_parked_objects_from_broken_ones() -> None: def test_flux_health_flags_a_suspension_nobody_declared() -> None: - rows = "flux-system/core\tfalse\tReady=True,\nflux-system/parked\ttrue\tReady=False,\n" + rows = "flux-system/core\t1\t1\tfalse\tReady=True,\nflux-system/parked\t1\t1\ttrue\tReady=False,\n" evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) @@ -251,17 +354,16 @@ def test_flux_health_flags_a_suspension_nobody_declared() -> None: assert evaluation.evidence["unexpected_suspensions"] == ["flux-system/parked"] -def test_flux_health_does_not_call_an_in_flight_reconcile_unhealthy() -> None: - """Flux drops Ready to False for the duration of a reconcile.""" +def test_flux_health_fails_closed_on_in_flight_or_unknown_conditions() -> None: rows = ( - "flux-system/reconciling\tfalse\tReconciling=True,Ready=False,\n" - "flux-system/degraded\tfalse\tReady=False,Healthy=True,\n" - "flux-system/waiting\t\tReady=Unknown,\n" + "flux-system/reconciling\t1\t1\tfalse\tReconciling=True,Ready=False,\n" + "flux-system/degraded\t1\t1\tfalse\tReady=False,Healthy=True,\n" + "flux-system/waiting\t1\t1\tfalse\tReady=Unknown,\n" ) evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) - assert evaluation.status == model.PASS + assert evaluation.status == model.FAIL assert evaluation.evidence["total"] == 3 @@ -269,11 +371,43 @@ def test_blank_rows_in_a_projection_are_ignored_by_both_log_parsers() -> None: padded = "\n\n" + FULL_LOG + "\n\n" assert evaluate(routing_spec(), padded).status == model.PASS - rows = "\n\nflux-system/core\tfalse\tReady=True,\n\n" + rows = "\n\nflux-system/core\t1\t1\tfalse\tReady=True,\n\n" evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) assert evaluation.status == model.PASS and evaluation.evidence["total"] == 1 def test_flux_health_skips_on_an_empty_or_failed_projection() -> None: - assert evaluators.evaluate(flux_spec(), {"s": outcome(stdout="")}).status == model.NOT_RUN - assert evaluators.evaluate(flux_spec(), {"s": outcome(returncode=1)}).status == model.NOT_RUN + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(stdout="")}).status + == model.NOT_RUN + ) + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(returncode=1)}).status + == model.NOT_RUN + ) + malformed = "flux-system/core\t1\t0\tfalse\tReady=True,\n" + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(stdout=malformed)}).status + == model.FAIL + ) + + +@pytest.mark.parametrize( + "row", + [ + "too\tfew", + "BAD NAME\t1\t1\tfalse\tReady=True,", + "flux-system/core\tx\t1\tfalse\tReady=True,", + "flux-system/core\t1\t1\tmaybe\tReady=True,", + "flux-system/core\t1\t1\tfalse\tbroken,", + "flux-system/core\t1\t1\tfalse\tReady=Yes,", + "flux-system/core\t1\t1\tfalse\tReady=True,Ready=True,", + ], +) +def test_flux_rows_reject_malformed_columns_names_generations_and_conditions( + row: str, +) -> None: + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(stdout=row)}).status + == model.NOT_RUN + ) diff --git a/testing/tests/test_hermes_handoff_model.py b/testing/tests/test_hermes_handoff_model.py index 37c3f9b9..e3c46489 100644 --- a/testing/tests/test_hermes_handoff_model.py +++ b/testing/tests/test_hermes_handoff_model.py @@ -6,12 +6,22 @@ import datetime as dt import pytest -from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) model = load_handoff_module("hermes_handoff_model") -def result(status: str, identifier: str = "test.check", mandatory: bool = True, group: str = "g"): +def result( + status: str, + identifier: str = "test.check", + mandatory: bool = True, + group: str = "g", +): return model.CheckResult( spec=model.CheckSpec( id=identifier, title="t", group=group, rule="allowed", mandatory=mandatory @@ -53,14 +63,22 @@ def test_a_mandatory_skip_is_a_no_go_exactly_like_a_failure() -> None: def test_a_harness_error_is_a_no_go_even_when_every_check_passed() -> None: - clean = report(result(model.PASS), errors=("the two vantages are the same principal",)) + clean = report( + result(model.PASS), errors=("the two vantages are the same principal",) + ) assert clean.decision == model.NO_GO -def test_a_zero_state_report_is_a_go_only_because_nothing_was_mandatory() -> None: - """An empty catalog has nothing blocking; the harness errors are what stop it.""" - assert report().decision == model.GO +def test_a_zero_state_or_unknown_status_report_is_no_go() -> None: + """Absence or an unrecognised classification can never round up to GO.""" + assert report().decision == model.NO_GO assert report(errors=("no checks were built",)).decision == model.NO_GO + unknown = result("MAYBE") + assert unknown.blocking is True + built = report(unknown) + assert built.decision == model.NO_GO + assert built.counts == dict.fromkeys(model.STATUSES, 0) + assert "groups:" in report(result(model.PASS)).render_summary() def test_blocking_results_are_ordered_failures_before_skips() -> None: @@ -93,7 +111,9 @@ def test_counts_and_group_summary_tally_every_status() -> None: def test_the_serialised_report_carries_the_verdict_and_its_reasons() -> None: built = report(result(model.FAIL, "x.fail"), result(model.PASS, "y.pass")) built.vantages = [ - model.VantageRecord(name="operator", description="d", identity="u", available=True) + model.VantageRecord( + name="operator", description="d", identity="u", available=True + ) ] payload = built.as_dict() @@ -138,7 +158,11 @@ def test_rationale_is_carried_only_when_a_check_has_one() -> None: def test_distinct_vantages_are_reported_in_first_use_order() -> None: built = spec( "allowed", - steps=(step("a", vantage="operator"), step("b", vantage="self"), step("c", vantage="operator")), + steps=( + step("a", vantage="operator"), + step("b", vantage="self"), + step("c", vantage="operator"), + ), ) assert built.vantages == ("operator", "self") @@ -146,7 +170,9 @@ def test_distinct_vantages_are_reported_in_first_use_order() -> None: def test_the_human_summary_names_the_verdict_vantages_and_blockers() -> None: built = report(result(model.FAIL, "x.fail", group="alpha"), errors=("collision",)) built.vantages = [ - model.VantageRecord(name="operator", description="d", identity="u", available=True), + model.VantageRecord( + name="operator", description="d", identity="u", available=True + ), model.VantageRecord(name="self", description="d", available=False), ] diff --git a/testing/tests/test_hermes_handoff_mutation_gate.py b/testing/tests/test_hermes_handoff_mutation_gate.py new file mode 100644 index 00000000..1e84cfb8 --- /dev/null +++ b/testing/tests/test_hermes_handoff_mutation_gate.py @@ -0,0 +1,51 @@ +"""Tests for the deterministic Hermes handoff mutation gate.""" + +from __future__ import annotations + +from pathlib import Path + +from testing.quality_handoff_mutation import Mutation, MutationReport, main, run_gate + + +ROOT = Path(__file__).parents[2] + + +def test_every_security_mutant_is_killed() -> None: + report = run_gate(ROOT) + assert report.issues == () + assert report.score == 100.0 + assert report.killed == report.total >= 9 + + +def test_invalid_anchor_baseline_syntax_and_survivor_are_reported() -> None: + path = "hermes_handoff_model.py" + cases = ( + Mutation("missing", path, "not present", "x", "pass"), + Mutation( + "baseline", + path, + "SCHEMA_VERSION = 1", + "SCHEMA_VERSION = 2", + "raise SystemExit(1)", + ), + Mutation("syntax", path, "SCHEMA_VERSION = 1", "if", "pass"), + Mutation("survivor", path, "SCHEMA_VERSION = 1", "SCHEMA_VERSION = 2", "pass"), + ) + report = run_gate(ROOT, cases) + assert report.killed == 0 + assert len(report.issues) == 4 + assert report.score == 0.0 + + +def test_main_reports_a_perfect_score(capsys) -> None: + assert main() == 0 + assert "9/9 (100.00%)" in capsys.readouterr().out + + +def test_main_reports_mutation_issues(monkeypatch, capsys) -> None: + monkeypatch.setattr( + "testing.quality_handoff_mutation.run_gate", + lambda _root: MutationReport(1, 0, ("survivor",)), + ) + assert main() == 1 + assert "survivor" in capsys.readouterr().out diff --git a/testing/tests/test_hermes_handoff_policy.py b/testing/tests/test_hermes_handoff_policy.py index 13fdc8b4..90cef791 100644 --- a/testing/tests/test_hermes_handoff_policy.py +++ b/testing/tests/test_hermes_handoff_policy.py @@ -16,16 +16,20 @@ def check(*argv: str, mode: str | None = None, inner: bool = False) -> None: @pytest.mark.parametrize( "argv", [ - ("kubectl", "--namespace", "hermes", "get", "pods"), + ("kubectl", "--namespace", "hermes", "get", "pods", "-o", "name"), ("kubectl", "auth", "can-i", "get", "secrets", "--all-namespaces"), - ("kubectl", "get", "secrets", "--all-namespaces", "-o", "name"), - ("kubectl", "get", "secrets", "--output=name"), - ("kubectl", "logs", "deploy/x", "--tail", "1"), - ("kubectl", "--context", "atlas", "get", "nodes", "-o", "json"), + ("kubectl", "get", "nodes", "-o", "name"), + ( + "kubectl", + "get", + "pods", + "-o", + policy.projection("jsonpath={.status.phase}"), + ), ("flux", "get", "kustomizations", "--all-namespaces"), ("helm", "list", "--all-namespaces"), ("git", "merge-base", "--is-ancestor", "a", "b"), - ("git", "config", "--get", "remote.origin.url"), + ("git", "ls-remote", "origin", "refs/heads/main"), ("hermes", "kanban", "list", "--json"), ("hermes", "sessions", "list", "--source", "telegram"), ("hermes", "status"), @@ -37,40 +41,50 @@ def test_read_only_commands_are_permitted(argv: tuple[str, ...]) -> None: @pytest.mark.parametrize( - ("argv", "fragment"), + "argv", [ - ((), "empty command"), - (("vault", "read", "kv/x"), "forbidden binary"), - (("curl", "https://example.dev"), "forbidden binary"), - (("rm", "-rf", "/"), "allowlist"), - (("kubectl", "get", "pods", "--token", "abc"), "forbidden argument"), - (("kubectl", "get", "pods", "--as=system:admin"), "forbidden argument"), - (("kubectl", "get", "secrets", "-A"), "-o name"), - (("kubectl", "describe", "secrets", "x"), "-o name"), - (("kubectl", "delete", "-n", "get", "secrets"), "requires a dry run"), - (("kubectl", "port-forward", "svc/x", "80"), "outside the read-only boundary"), - (("kubectl", "exec", "pod"), "-- command boundary"), - (("kubectl", "exec", "pod", "--"), "inner command"), - (("kubectl",), "outside the read-only boundary"), - (("flux", "reconcile", "kustomization", "x"), "outside the read-only boundary"), - (("flux",), "requires a subcommand"), - (("helm", "upgrade", "x"), "outside the read-only boundary"), - (("git", "commit", "-m", "x"), "not permitted"), - (("git", "push", "origin", "HEAD:refs/heads/x"), "not permitted"), - (("git", "config", "user.name", "x"), "may only read"), - (("hermes", "kanban", "complete", "t_1"), "outside the read-only boundary"), - (("hermes",), "requires a subcommand"), - ((policy.GITEA_CLIENT, "POST", "/api/v1/x"), "not permitted"), - ((policy.GITEA_CLIENT, "GET"), "method and a path"), - (("sh", "-c", "rm -rf /"), "frozen template"), - (("sh", "echo"), "frozen template"), - (("git", "log", "/etc/shadow"), "credential path"), + (), + ("vault", "read", "kv/x"), + ("curl", "https://example.dev"), + ("rm", "-rf", "/"), + ("kubectl", "get", "pods", "--token", "abc"), + ("kubectl", "get", "pods"), + ("kubectl", "get", "pods", "--as=system:admin"), + ("kubectl", "get", "secrets", "-o", "name"), + ("kubectl", "get", "pods,secrets/x", "-o", "name"), + ("kubectl", "describe", "pod/x"), + ("kubectl", "config", "view", "--raw"), + ("kubectl", "auth", "reconcile", "-f", "x"), + ("kubectl", "delete", "pod/x", "--dry-run=client"), + ("kubectl", "delete", "pod/x", "--", "--dry-run=server"), + ("kubectl", "get", "pods", "-o=json"), + ("kubectl", "get", "pods", "-o", "jsonpath={.spec.containers[*].env[*].value}"), + ("kubectl", "port-forward", "svc/x", "80"), + ("kubectl", "exec", "pod"), + ("kubectl", "exec", "pod", "--"), + ("kubectl",), + ("flux", "reconcile", "kustomization", "x"), + ("flux",), + ("helm", "upgrade", "x"), + ("helm", "get", "values", "x"), + ("git", "commit", "-m", "x"), + ("git", "push", "origin", "HEAD:refs/heads/x"), + ("git", "fetch", "origin"), + ("git", "config", "core.sshCommand", "x"), + ("git", "-ccore.sshCommand=x", "ls-remote", "origin"), + ("git", "--git-dir=/tmp/x", "status"), + ("hermes", "kanban", "complete", "t_1"), + ("hermes",), + (policy.GITEA_CLIENT, "POST", "/api/v1/x"), + (policy.GITEA_CLIENT, "GET"), + ("sh", "-c", "rm -rf /"), + ("sh", "echo"), + ("git", "log", "/etc/shadow"), ], ) -def test_unsafe_commands_are_refused(argv: tuple[str, ...], fragment: str) -> None: - with pytest.raises(policy.PolicyError) as caught: +def test_unsafe_commands_are_refused(argv: tuple[str, ...]) -> None: + with pytest.raises(policy.PolicyError): check(*argv) - assert fragment in str(caught.value) def test_unknown_mode_is_refused() -> None: @@ -78,47 +92,128 @@ def test_unknown_mode_is_refused() -> None: policy.check_argv(("kubectl", "get", "pods"), "whatever") -def test_dry_run_reaches_mutating_subcommands_without_writing() -> None: - check("kubectl", "--namespace", "hermes", "patch", "deploy/x", "--dry-run=server") - check("kubectl", "create", "configmap", "probe", "--dry-run=server") +def test_dry_run_mutation_commands_are_never_constructible() -> None: + with pytest.raises(policy.PolicyError, match="dry-run"): + check("kubectl", "patch", "deploy/x", "--dry-run=server") def test_a_resource_named_after_a_subcommand_cannot_smuggle_a_delete() -> None: """`-n get` must not make a delete look like a read.""" - with pytest.raises(policy.PolicyError, match="requires a dry run"): + with pytest.raises(policy.PolicyError): check("kubectl", "delete", "-n", "get", "configmap", "x") def test_impersonation_is_operator_forbidden_and_in_pod_permitted() -> None: with pytest.raises(policy.PolicyError, match="forbidden argument"): check("kubectl", "get", "namespaces", "--as", "system:admin") - check("kubectl", "get", "namespaces", "--as", "system:admin", inner=True) + check( + "kubectl", + "get", + "namespaces", + "-o", + "name", + "--as", + "system:admin", + inner=True, + ) def test_exec_validates_its_inner_command_and_tolerates_impersonation_inside() -> None: - inner = policy.shell("path_readable", path="/tmp") - check("kubectl", "exec", "--namespace", "hermes", "pod", "--", *inner) - check("kubectl", "exec", "pod", "--", "kubectl", "get", "ns", "--as", "system:admin") - with pytest.raises(policy.PolicyError, match="allowlist"): + script = policy.render_shell("env_names") + policy._RENDERED.add(script) + check( + "kubectl", "exec", "--namespace", "hermes", "pod", "--", "/bin/sh", "-c", script + ) + check( + "kubectl", + "exec", + "pod", + "--", + "/usr/local/bin/kubectl", + "get", + "ns", + "-o", + "name", + "--as", + "system:admin", + ) + with pytest.raises(policy.PolicyError): check("kubectl", "exec", "pod", "--", "bash", "-c", "id") -def test_exec_of_a_shell_template_naming_a_secret_path_is_permitted() -> None: - """The path scan stops at `--`; the tail is a reviewed template, not free text.""" - inner = policy.shell("account_lock_state", account="hermes-agent", path="/host-etc/shadow") - check("kubectl", "exec", "--namespace", "hermes", "pod", "--", *inner) +def test_shell_templates_reject_credential_roots() -> None: + with pytest.raises(policy.PolicyError, match="credential paths"): + policy.shell("json_fields", path="/runtime-access/token", fields="state") + with pytest.raises(policy.PolicyError, match="credential-bearing"): + policy.shell("json_fields", path="/tmp/health.json", fields="state,token") def test_armed_mode_opens_exactly_the_mutations_it_should() -> None: - check("git", "push", "origin", "HEAD:refs/heads/x", mode=policy.ARMED) - check(policy.GITEA_CLIENT, "POST", "/api/v1/x", mode=policy.ARMED) - check(policy.GITEA_CLIENT, "DELETE", "/api/v1/x", mode=policy.ARMED) + ref = "ephemeral/hermes-handoff-acceptance/acceptance1" + policy.arm_ephemeral_policy(ref) + policy.register_ephemeral_pull(9) + check("git", "push", "origin", f"HEAD:refs/heads/{ref}", mode=policy.ARMED) + root = "/api/v1/repos/atlas/titan-iac" + check( + policy.GITEA_CLIENT, + "POST", + f"{root}/pulls", + "--field", + "title=WIP: Hermes handoff acceptance ephemeral probe", + "--field", + f"head={ref}", + "--field", + "base=main", + "--field", + "body=Ephemeral acceptance probe. Closed and deleted by the harness.", + mode=policy.ARMED, + ) + check( + policy.GITEA_CLIENT, + "PATCH", + f"{root}/pulls/9", + "--field", + "state=closed", + mode=policy.ARMED, + ) + check(policy.GITEA_CLIENT, "DELETE", f"{root}/branches/{ref}", mode=policy.ARMED) with pytest.raises(policy.PolicyError, match="may not force"): - check("git", "push", "--force", "origin", "x", mode=policy.ARMED) + check( + "git", + "push", + "--force", + "origin", + f"HEAD:refs/heads/{ref}", + mode=policy.ARMED, + ) with pytest.raises(policy.PolicyError, match="may not force"): check("git", "push", "--all", "origin", mode=policy.ARMED) with pytest.raises(policy.PolicyError, match="not permitted"): - check(policy.GITEA_CLIENT, "PUT", "/api/v1/x", mode=policy.ARMED) + check(policy.GITEA_CLIENT, "PUT", f"{root}/pulls/9", mode=policy.ARMED) + + +def test_arming_and_projection_registration_fail_closed() -> None: + with pytest.raises(policy.PolicyError, match="exact ephemeral ref"): + policy.arm_ephemeral_policy("main") + policy._ARMED_REF = "" + with pytest.raises(policy.PolicyError, match="registration is malformed"): + policy.register_ephemeral_pull(9) + policy.arm_ephemeral_policy("ephemeral/hermes-handoff-acceptance/acceptance1") + + for candidate in ( + "wide", + "jsonpath={.status.token}", + "jsonpath={.metadata.annotations}", + "jsonpath={.metadata.labels.other}", + "jsonpath={.spec.containers[*].env[*].value}", + ): + with pytest.raises(policy.PolicyError): + policy.projection(candidate) + + with pytest.raises(policy.PolicyError): + check("/usr/bin/awk", "x") + with pytest.raises(policy.PolicyError): + check("git", "--exec-path=/tmp", "status") def test_every_frozen_template_renders_and_is_accepted() -> None: @@ -141,11 +236,11 @@ def test_template_rendering_rejects_unknown_names_and_unsafe_parameters() -> Non with pytest.raises(policy.PolicyError, match="unknown shell template"): policy.render_shell("nope") with pytest.raises(policy.PolicyError, match="expects parameters"): - policy.render_shell("path_mode") + policy.render_shell("json_fields") with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): - policy.render_shell("path_mode", path="/tmp; rm -rf /") + policy.render_shell("json_fields", path="/tmp;rm", fields="state") with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): - policy.render_shell("path_mode", path="$(id)") + policy.render_shell("json_fields", path="$(id)", fields="state") def test_positionals_skip_flags_and_stop_at_the_boundary() -> None: @@ -155,5 +250,70 @@ def test_positionals_skip_flags_and_stop_at_the_boundary() -> None: def test_a_subcommand_must_precede_its_resource_arguments() -> None: - with pytest.raises(policy.PolicyError, match="must precede"): + with pytest.raises(policy.PolicyError): check("kubectl", "pods", "get") + + +@pytest.mark.parametrize( + "argv", + [ + ("/usr/bin/../../tmp/kubectl", "get", "pods"), + ("/tmp/reviewer/kubectl", "get", "pods"), + ("kubectl", "get", "pods", "-o", "go-template=x"), + ("kubectl", "get", "pods", "-o", "jsonpath={.status.token}"), + ("kubectl", "get", "pods", "-o", "jsonpath={.metadata.annotations}"), + ("kubectl", "get", "pods", "-o", "jsonpath={.spec.containers[*].env[*].value}"), + ("git", "-C", "/tmp", "status"), + ("git", "--work-tree=/tmp", "status"), + (policy.GITEA_CLIENT, "GET", "not-an-api-path"), + (policy.GITEA_CLIENT, "GET", "/api/v1/repos/atlas/other"), + ], +) +def test_provenance_projection_and_repository_bypasses_are_rejected(argv) -> None: + with pytest.raises(policy.PolicyError): + check(*argv) + + +def test_armed_writes_are_pinned_to_exact_remote_ref_and_endpoints() -> None: + ref = "ephemeral/hermes-handoff-acceptance/acceptance1" + policy.arm_ephemeral_policy(ref) + policy.register_ephemeral_pull(9) + with pytest.raises(policy.PolicyError, match="pinned to origin"): + check( + "git", + "push", + "upstream", + "HEAD:refs/heads/ephemeral/hermes-handoff-acceptance/acceptance1", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError, match="preflighted ephemeral ref"): + check("git", "push", "origin", "HEAD:refs/heads/other", mode=policy.ARMED) + with pytest.raises(policy.PolicyError): + check( + policy.GITEA_CLIENT, + "DELETE", + "/api/v1/repos/atlas/titan-iac", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError): + check( + policy.GITEA_CLIENT, + "PATCH", + "/api/v1/repos/atlas/titan-iac/pulls/8", + "--field", + "state=closed", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError): + check( + policy.GITEA_CLIENT, + "POST", + "/api/v1/repos/atlas/titan-iac/pulls", + "--field", + "title=WIP: wrong branch", + "--field", + "head=main", + "--field", + "base=main", + mode=policy.ARMED, + ) diff --git a/testing/tests/test_hermes_handoff_redaction.py b/testing/tests/test_hermes_handoff_redaction.py index a472d82b..78fe19ec 100644 --- a/testing/tests/test_hermes_handoff_redaction.py +++ b/testing/tests/test_hermes_handoff_redaction.py @@ -63,7 +63,10 @@ def test_screened_json_is_still_parsable_json() -> None: def test_high_entropy_runs_are_screened_but_hex_digests_are_not() -> None: - assert redaction.redact("Zm9vYmFyYmF6cXV4-Quux_1234567890ABCDEFghij") != redaction.REDACTED + assert ( + redaction.redact("Zm9vYmFyYmF6cXV4-Quux_1234567890ABCDEFghij") + != redaction.REDACTED + ) long_mixed = "aB3" + "x9Y2z_" * 10 assert redaction.redact(long_mixed) == redaction.REDACTED assert redaction.redact("f" * 64) == "f" * 64 @@ -108,8 +111,8 @@ def test_value_shape_gates_screening(value: str, possible: bool) -> None: def test_bound_truncates_to_a_byte_budget_and_reports_it() -> None: text, truncated = redaction.bound("x" * 100, max_bytes=10) assert truncated - assert text.startswith("x" * 10) - assert redaction.TRUNCATION_NOTE in text + assert len(text.encode()) <= 10 + assert text == redaction.TRUNCATION_NOTE[:10] text, truncated = redaction.bound("short", max_bytes=64) assert (text, truncated) == ("short", False) @@ -123,17 +126,25 @@ def test_bound_with_no_budget_keeps_nothing() -> None: def test_bound_never_splits_a_multibyte_character() -> None: text, truncated = redaction.bound("é" * 10, max_bytes=5) assert truncated - assert text.startswith("éé") + assert len(text.encode()) <= 5 -def test_safe_text_bounds_before_it_screens() -> None: - """Bounding first is what stops a huge dump from outrunning the screen.""" +def test_safe_text_screens_before_it_bounds() -> None: + """A boundary must never preserve a useful credential prefix.""" payload = "password=" + "a" * 200 text, truncated = redaction.safe_text(payload, max_bytes=20) - assert truncated + assert not truncated assert redaction.REDACTED in text +def test_no_byte_budget_exposes_a_credential_prefix() -> None: + payload = "password=A9secret-prefix-that-must-never-survive" + for budget in range(1, 64): + screened, _ = redaction.safe_text(payload, max_bytes=budget) + assert "A9" not in screened + assert len(screened.encode()) <= budget + + def test_scrub_walks_nested_structures_and_keeps_useful_types() -> None: scrubbed = redaction.scrub( { @@ -152,6 +163,25 @@ def test_scrub_walks_nested_structures_and_keeps_useful_types() -> None: assert scrubbed["note"] == ["tuple", "values"] +def test_scrub_screens_short_numeric_nested_values_and_mapping_keys() -> None: + scrubbed = redaction.scrub( + { + "token": 1234, + "credentials": {"x": "y"}, + "ghp_ABCDEF1234567890": "mapping-key", + "private_key": "x", + } + ) + assert scrubbed["token"] == redaction.REDACTED + assert scrubbed["credentials"] == redaction.REDACTED + assert scrubbed["private_key"] == redaction.REDACTED + assert not any("ghp_" in key for key in scrubbed) + + +def test_unterminated_private_key_blocks_are_screened() -> None: + assert "PRIVATE KEY" not in redaction.redact("-----BEGIN PRIVATE KEY-----\nabc") + + def test_screening_is_idempotent() -> None: once = redaction.redact("api_key=AKIAIOSFODNN7EXAMPLE") assert redaction.redact(once) == once diff --git a/testing/tests/test_hermes_handoff_rules.py b/testing/tests/test_hermes_handoff_rules.py index 11f09840..03d5785f 100644 --- a/testing/tests/test_hermes_handoff_rules.py +++ b/testing/tests/test_hermes_handoff_rules.py @@ -4,7 +4,12 @@ from __future__ import annotations import pytest -from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) rules = load_handoff_module("hermes_handoff_rules") model = load_handoff_module("hermes_handoff_model") @@ -33,15 +38,18 @@ def test_dotted_paths_index_and_map(path: str, expected) -> None: assert rules.dotted(PAYLOAD, path) == expected -@pytest.mark.parametrize("path", ["missing", "a.missing", "a.b[9].c", "flag[0]", "flag.b"]) +@pytest.mark.parametrize( + "path", ["missing", "a.missing", "a.b[9].c", "flag[0]", "flag.b"] +) def test_an_absent_path_raises_rather_than_returning_none(path: str) -> None: with pytest.raises(KeyError): rules.dotted(PAYLOAD, path) -def test_parse_json_tolerates_control_characters_but_not_empty_output() -> None: - """Kanban bodies routinely carry raw newlines; that is not a broken read.""" - assert rules.parse_json(outcome(stdout='{"title": "a\nb"}'))["title"] == "a\nb" +def test_parse_json_rejects_raw_control_characters_and_empty_output() -> None: + """Only RFC-compliant, nonempty JSON is accepted as release evidence.""" + with pytest.raises(ValueError): + rules.parse_json(outcome(stdout='{"title": "a\nb"}')) with pytest.raises(ValueError, match="empty output"): rules.parse_json(outcome(stdout=" ")) with pytest.raises(ValueError): @@ -59,10 +67,17 @@ def test_missing_steps_reports_the_first_step_that_produced_nothing() -> None: assert never_run.status == model.NOT_RUN assert "never executed" in never_run.reason - errored = rules.missing_steps(built, {"a": outcome(error="timeout"), "b": outcome()}) + errored = rules.missing_steps( + built, {"a": outcome(error="timeout"), "b": outcome()} + ) assert errored.status == model.NOT_RUN assert "did not run" in errored.reason + truncated = rules.missing_steps( + built, {"a": outcome(truncated=True), "b": outcome()} + ) + assert truncated.status == model.NOT_RUN and "truncated" in truncated.reason + assert rules.missing_steps(built, {"a": outcome(), "b": outcome()}) is None @@ -81,7 +96,9 @@ def test_an_optional_step_may_be_absent() -> None: ("", 0, False), ], ) -def test_denial_detection_needs_a_refusal_not_merely_a_failure(stderr, returncode, denied) -> None: +def test_denial_detection_needs_a_refusal_not_merely_a_failure( + stderr, returncode, denied +) -> None: assert rules.looks_denied(outcome(stderr=stderr, returncode=returncode)) is denied @@ -105,7 +122,10 @@ def test_parsed_step_returns_the_payload_or_the_skip_to_report() -> None: assert failed.status == model.NOT_RUN and "failed" in failed.reason unparsable = rules.parsed_step(built, {"s": outcome(stdout="nope")}) - assert unparsable.status == model.NOT_RUN and "did not return JSON" in unparsable.reason + assert ( + unparsable.status == model.NOT_RUN + and "did not return JSON" in unparsable.reason + ) def test_an_unknown_rule_is_not_run_rather_than_a_crash() -> None: @@ -128,6 +148,17 @@ def test_an_evaluator_that_raises_is_not_run_rather_than_a_pass() -> None: del rules.EVALUATORS["explodes-for-test"] +def test_an_evaluator_returning_an_unknown_status_is_not_run() -> None: + @rules.evaluator("unknown-status-for-test") + def _unknown(_spec, _outcomes): + return rules.Evaluation("MAYBE", "bad") + + evaluation = rules.evaluate(spec("unknown-status-for-test"), {}) + assert evaluation.status == model.NOT_RUN + assert "unknown status" in evaluation.reason + del rules.EVALUATORS["unknown-status-for-test"] + + def test_is_list_distinguishes_arrays_from_strings() -> None: assert rules.is_list([1]) and rules.is_list(()) assert not rules.is_list("text") diff --git a/testing/tests/test_hermes_handoff_run.py b/testing/tests/test_hermes_handoff_run.py index 379bb27d..881ffd55 100644 --- a/testing/tests/test_hermes_handoff_run.py +++ b/testing/tests/test_hermes_handoff_run.py @@ -4,6 +4,7 @@ from __future__ import annotations import datetime as dt import subprocess +from dataclasses import replace from testing.tests.test_hermes_handoff_support import ( FakeClock, @@ -22,20 +23,35 @@ TARGETS = catalog.Targets(now=NOW) def completed(stdout: str = "", stderr: str = "", returncode: int = 0): - return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) def whoami(username: str) -> object: - return completed(stdout=f'{{"status": {{"userInfo": {{"username": "{username}"}}}}}}') + return completed(stdout=username) def runner_for(results: list, **kwargs): return exec_module.Runner( - clock=FakeClock(), spawn=FakeSpawn(results), environ={}, deadline_seconds=3600, **kwargs + clock=FakeClock(), + spawn=FakeSpawn(results), + environ={}, + deadline_seconds=3600, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + **kwargs, ) -def check(rule: str, expect: dict | None = None, steps=(), identifier="test.check", mandatory=True): +def check( + rule: str, + expect: dict | None = None, + steps=(), + identifier="test.check", + mandatory=True, +): return model.CheckSpec( id=identifier, title="t", @@ -51,8 +67,23 @@ def test_the_shipped_catalog_is_valid_and_covers_every_group() -> None: specs = harness_run.build_catalog(TARGETS) assert harness_run.validate_catalog(specs) == [] groups = {spec.group for spec in specs} - assert {"baseline", "identity", "routing", "scopes", "access-denied", "gitops"} <= groups - assert {"forge", "nodes", "build", "reliability", "pool", "surfaces", "ephemeral"} <= groups + assert { + "baseline", + "identity", + "routing", + "scopes", + "access-denied", + "gitops", + } <= groups + assert { + "forge", + "nodes", + "build", + "reliability", + "pool", + "surfaces", + "ephemeral", + } <= groups def test_catalog_validation_names_every_structural_problem() -> None: @@ -61,14 +92,20 @@ def test_catalog_validation_names_every_structural_problem() -> None: check("allowed", identifier="a.dup"), check("allowed", steps=(step("s", vantage="nowhere"),), identifier="b.vantage"), check("allowed", steps=(step("s", kind="guess"),), identifier="c.kind"), - check("denied", steps=(step("s", kind=model.REVIEW),), identifier="d.review-only"), + check( + "denied", steps=(step("s", kind=model.REVIEW),), identifier="d.review-only" + ), model.CheckSpec( id="e.impersonation", title="t", group="g", rule="allowed", steps=( - model.Step(key="s", vantage=catalog.OPERATOR, argv=("kubectl", "get", "ns", "--as", "x")), + model.Step( + key="s", + vantage=catalog.OPERATOR, + argv=("kubectl", "get", "ns", "--as", "x"), + ), ), ), ] @@ -83,23 +120,81 @@ def test_catalog_validation_names_every_structural_problem() -> None: assert "impersonates from the operator vantage" in problems +def test_target_validation_rejects_every_vacuous_or_unbound_release_input() -> None: + broken = replace( + TARGETS, + repo="atlas/other", + remote="upstream", + reviewed_pr_number=20, + reviewed_head_ref="other", + baseline_commit="bad", + remote_main_sha="bad", + reviewed_head_sha="bad", + build_sha="bad", + agent_image="latest", + deployment_revision="0", + node_count=1, + pool_replicas=1, + expect_telegram_sessions=False, + chat_config_revision="", + pool_worker_env=(), + dependency_heads=((14, "bad"), (14, "bad")), + max_evidence_age_seconds=0, + ) + problems = harness_run.validate_targets(broken) + assert len(problems) >= 12 + sha = "1" * 40 + valid = replace( + TARGETS, + remote_main_sha=sha, + reviewed_head_sha=sha, + build_sha=sha, + agent_image=f"registry.example/hermes:git-{sha}-build-7@sha256:{'2' * 64}", + deployment_revision="1", + chat_config_revision="rev", + dependency_heads=tuple( + (number, sha) for number in TARGETS.dependency_pull_requests + ), + ) + assert harness_run.validate_targets(valid) == [] + wrong_image_source = replace( + valid, + agent_image=f"registry.example/hermes:git-{'2' * 40}-build-7@sha256:{'2' * 64}", + ) + assert "image tag/digest" in " ".join( + harness_run.validate_targets(wrong_image_source) + ) + wrong_build = replace(valid, build_sha="2" * 40) + assert "build SHA" in " ".join(harness_run.validate_targets(wrong_build)) + malformed_heads = replace( + valid, + dependency_heads=tuple( + (number, "bad") for number in valid.dependency_pull_requests + ), + ) + assert "dependency heads" in " ".join(harness_run.validate_targets(malformed_heads)) + + def test_identity_resolution_extracts_only_the_username() -> None: runner = runner_for([whoami("system:serviceaccount:hermes:hermes-agent")]) - username, detail = harness_run.resolve_identity(runner, exec_module.operator_vantage()) + username, detail = harness_run.resolve_identity( + runner, exec_module.operator_vantage() + ) assert (username, detail) == ("system:serviceaccount:hermes:hermes-agent", "") failed = harness_run.resolve_identity( - runner_for([completed(stderr="no access", returncode=1)]), exec_module.operator_vantage() + runner_for([completed(stderr="no access", returncode=1)]), + exec_module.operator_vantage(), ) assert failed[0] == "" and "no access" in failed[1] unparsable = harness_run.resolve_identity( - runner_for([completed(stdout="nope")]), exec_module.operator_vantage() + runner_for([completed(stdout="two names")]), exec_module.operator_vantage() ) - assert unparsable[0] == "" and "unparsable identity" in unparsable[1] + assert unparsable[0] == "" and "no username" in unparsable[1] nameless = harness_run.resolve_identity( - runner_for([completed(stdout='{"status": {}}')]), exec_module.operator_vantage() + runner_for([completed(stdout="")]), exec_module.operator_vantage() ) assert nameless[0] == "" and "no username" in nameless[1] @@ -119,7 +214,12 @@ def test_vantages_resolve_to_running_pods_and_record_what_is_missing() -> None: vantages, records = harness_run.resolve_vantages(runner, TARGETS) by_name = {record.name: record for record in records} - assert set(vantages) == {catalog.OPERATOR, catalog.SELF, catalog.SWITCHYARD, catalog.CHAT} + assert set(vantages) == { + catalog.OPERATOR, + catalog.SELF, + catalog.SWITCHYARD, + catalog.CHAT, + } assert by_name[catalog.SELF].identity == "system:serviceaccount:hermes:hermes-agent" assert "hermes-agent-1" in by_name[catalog.SELF].description # deterministic pick assert by_name[catalog.NODE].available is False @@ -133,7 +233,9 @@ def test_an_unreachable_operator_stops_vantage_resolution_immediately() -> None: assert set(vantages) == {catalog.OPERATOR} assert records[0].available is False - assert harness_run.vantage_problems(records)[0].startswith("the operator vantage is unavailable") + assert harness_run.vantage_problems(records)[0].startswith( + "the operator vantage is unavailable" + ) def test_a_missing_chat_pod_is_recorded_rather_than_assumed_present() -> None: @@ -176,8 +278,14 @@ def test_a_selector_that_matches_no_running_pod_leaves_the_vantage_absent() -> N def test_two_vantages_that_are_the_same_principal_invalidate_the_evidence() -> None: """Corroboration needs two identities; one identity twice is a restatement.""" records = [ - model.VantageRecord(name=catalog.OPERATOR, description="d", identity="same", available=True), - model.VantageRecord(name=catalog.SELF, description="d", identity="same", available=True), + model.VantageRecord( + name=catalog.OPERATOR, description="d", identity="same", available=True + ), + model.VantageRecord( + name=catalog.SELF, description="d", identity="same", available=True + ), + model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True), + model.VantageRecord(name=catalog.CHAT, description="d", available=True), ] problems = harness_run.vantage_problems(records) @@ -195,7 +303,10 @@ def test_vantage_problems_flag_a_missing_or_anonymous_self_probe() -> None: assert "in-pod Hermes vantage is unavailable" in absent[0] anonymous = harness_run.vantage_problems( - [operator, model.VantageRecord(name=catalog.SELF, description="d", available=True)] + [ + operator, + model.VantageRecord(name=catalog.SELF, description="d", available=True), + ] ) assert "did not report an identity" in anonymous[0] @@ -205,6 +316,10 @@ def test_vantage_problems_flag_a_missing_or_anonymous_self_probe() -> None: model.VantageRecord( name=catalog.SELF, description="d", identity="agent", available=True ), + model.VantageRecord( + name=catalog.SWITCHYARD, description="d", available=True + ), + model.VantageRecord(name=catalog.CHAT, description="d", available=True), ] ) assert clean == [] @@ -224,11 +339,20 @@ def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None spec = check( "stdout_matches", {"step": "s", "equals": "locked"}, - steps=(model.Step(key="s", vantage=catalog.OPERATOR, argv=("kubectl", "version"), record=False),), + steps=( + model.Step( + key="s", + vantage=catalog.OPERATOR, + argv=("kubectl", "version"), + record=False, + ), + ), ) runner = runner_for([completed(stdout="locked", stderr="noise")]) - result = harness_run.run_check(runner, spec, {catalog.OPERATOR: exec_module.operator_vantage()}) + result = harness_run.run_check( + runner, spec, {catalog.OPERATOR: exec_module.operator_vantage()} + ) assert result.status == model.PASS assert result.outcomes[0].stdout == harness_run.UNRECORDED @@ -237,7 +361,15 @@ def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None def test_an_expired_deadline_leaves_the_report_saying_so() -> None: clock = FakeClock() - runner = exec_module.Runner(clock=clock, spawn=FakeSpawn([]), environ={}, deadline_seconds=1.0) + runner = exec_module.Runner( + clock=clock, + spawn=FakeSpawn([]), + environ={}, + deadline_seconds=1.0, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + ) clock.advance(5.0) spec = check("allowed", steps=(step("s", vantage=catalog.OPERATOR),)) @@ -259,12 +391,24 @@ def test_an_expired_deadline_leaves_the_report_saying_so() -> None: def test_the_report_carries_the_configured_baseline_and_sorts_its_checks() -> None: runner = runner_for([completed(stdout="ok"), completed(stdout="ok")]) specs = [ - check("allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="z.last"), - check("allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="a.first"), + check( + "allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="z.last" + ), + check( + "allowed", + steps=(step("s", vantage=catalog.OPERATOR),), + identifier="a.first", + ), ] records = [ - model.VantageRecord(name=catalog.OPERATOR, description="d", identity="admin", available=True), - model.VantageRecord(name=catalog.SELF, description="d", identity="agent", available=True), + model.VantageRecord( + name=catalog.OPERATOR, description="d", identity="admin", available=True + ), + model.VantageRecord( + name=catalog.SELF, description="d", identity="agent", available=True + ), + model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True), + model.VantageRecord(name=catalog.CHAT, description="d", available=True), ] report = harness_run.build_report( @@ -279,16 +423,23 @@ def test_the_report_carries_the_configured_baseline_and_sorts_its_checks() -> No assert [result.spec.id for result in report.results] == ["a.first", "z.last"] assert report.baseline["baseline_commit"] == TARGETS.baseline_commit - assert report.baseline["dependency_pull_requests"] == list(TARGETS.dependency_pull_requests) + assert report.baseline["dependency_pull_requests"] == list( + TARGETS.dependency_pull_requests + ) assert report.decision == model.GO assert report.finished_at def test_armed_results_replace_the_matching_catalog_placeholders() -> None: - placeholder = check("not_armed", identifier="ephemeral.feature-branch-push", mandatory=False) + placeholder = check( + "not_armed", identifier="ephemeral.feature-branch-push", mandatory=False + ) armed = model.CheckResult( spec=model.CheckSpec( - id="ephemeral.feature-branch-push", title="t", group="ephemeral", rule="armed" + id="ephemeral.feature-branch-push", + title="t", + group="ephemeral", + rule="armed", ), status=model.PASS, reason="pushed", diff --git a/testing/tests/test_hermes_handoff_support.py b/testing/tests/test_hermes_handoff_support.py index 64964a27..f62ead15 100644 --- a/testing/tests/test_hermes_handoff_support.py +++ b/testing/tests/test_hermes_handoff_support.py @@ -33,6 +33,7 @@ def outcome( stderr: str = "", returncode: int | None = 0, error: str | None = None, + truncated: bool = False, vantage: str = "self", argv: tuple[str, ...] = ("kubectl", "get", "pods"), ) -> Any: @@ -44,6 +45,7 @@ def outcome( stdout=stdout, stderr=stderr, error=error, + truncated=truncated, ) @@ -68,12 +70,14 @@ def spec( ) -def step(key: str, kind: str | None = None, vantage: str = "self", optional: bool = False) -> Any: +def step( + key: str, kind: str | None = None, vantage: str = "self", optional: bool = False +) -> Any: """Return a synthetic step.""" return model.Step( key=key, vantage=vantage, - argv=("kubectl", "get", "pods"), + argv=("kubectl", "get", "pods", "-o", "name"), kind=kind or model.READ, optional=optional, ) diff --git a/testing/tests/test_quality_contract.py b/testing/tests/test_quality_contract.py index 6e260a1b..fecae836 100644 --- a/testing/tests/test_quality_contract.py +++ b/testing/tests/test_quality_contract.py @@ -2,13 +2,17 @@ from __future__ import annotations +import hashlib from pathlib import Path import textwrap from testing.quality_contract import load_contract from testing.quality_coverage import run_check as run_coverage_check from testing.quality_docs import run_check as run_docs_check -from testing.quality_hygiene import run_check as run_hygiene_check +from testing.quality_hygiene import ( + count_files_over_line_limit, + run_check as run_hygiene_check, +) def test_bundled_contract_exposes_local_and_jenkins_profiles(): @@ -18,6 +22,21 @@ def test_bundled_contract_exposes_local_and_jenkins_profiles(): assert contract["pytest_suites"]["unit"]["paths"] +def test_handoff_modules_are_exactly_managed_linted_and_covered(): + """The canonical contract must not silently drop a handoff module.""" + contract = load_contract() + expected = { + path.as_posix() for path in Path("scripts/ops").glob("hermes_handoff_*.py") + } + assert len(expected) == 15 + assert expected <= set(contract["managed_modules"]) + assert expected <= set(contract["lint_paths"]) + assert expected <= set(contract["coverage"]["tracked_files"]) + assert expected <= set(contract["coverage"]["branch_tracked_files"]) + assert "scripts/ops" in contract["pytest_suites"]["unit"]["coverage_sources"] + assert "scripts/ops/hermes_handoff_*.py" in contract["hygiene"]["line_limit_globs"] + + def test_docs_check_reports_missing_docstring_and_missing_path(tmp_path: Path): module_path = tmp_path / "managed.py" module_path.write_text("value = 1\n", encoding="utf-8") @@ -43,7 +62,10 @@ def test_docs_check_reports_missing_required_doc_metadata(tmp_path: Path): (tmp_path / "README.md").write_text("", encoding="utf-8") contract = { - "required_docs": [{"path": "README.md", "description": ""}, {"path": "missing.md", "description": "Missing"}], + "required_docs": [ + {"path": "README.md", "description": ""}, + {"path": "missing.md", "description": "Missing"}, + ], "managed_modules": [], "lint_paths": [], "pytest_suites": {"unit": {"description": "", "paths": []}}, @@ -62,6 +84,7 @@ def test_docs_check_reports_missing_required_doc_metadata(tmp_path: Path): def test_hygiene_check_enforces_line_limit_and_name_rules(tmp_path: Path): tests_dir = tmp_path / "tests" tests_dir.mkdir() + (tests_dir / "conftest.py").write_text("value = 1\n", encoding="utf-8") bad_name = tests_dir / "bad-name.py" bad_name.write_text("x = 1\n", encoding="utf-8") long_file = tests_dir / "test_too_long.py" @@ -84,7 +107,34 @@ def test_hygiene_check_enforces_line_limit_and_name_rules(tmp_path: Path): issues = run_hygiene_check(contract, tmp_path) assert any("file exceeds 3 LOC" in issue for issue in issues) - assert any("naming rule failed" in issue and "bad-name.py" in issue for issue in issues) + assert any( + "naming rule failed" in issue and "bad-name.py" in issue for issue in issues + ) + + +def test_hygiene_legacy_exception_is_exact_and_invalidated_by_any_edit(tmp_path: Path): + legacy = tmp_path / "legacy.py" + legacy.write_text("line\n" * 4, encoding="utf-8") + contract = { + "hygiene": { + "max_lines": 3, + "line_limit_globs": ["*.py"], + "legacy_line_exceptions": { + "legacy.py": { + "lines": 4, + "sha256": hashlib.sha256(legacy.read_bytes()).hexdigest(), + } + }, + } + } + assert run_hygiene_check(contract, tmp_path) == [] + assert count_files_over_line_limit(contract, tmp_path) == 0 + + legacy.write_text("edit\n" * 4, encoding="utf-8") + assert run_hygiene_check(contract, tmp_path) == [ + "file exceeds 3 LOC: legacy.py (4)" + ] + assert count_files_over_line_limit(contract, tmp_path) == 1 def test_coverage_check_enforces_per_file_floor(tmp_path: Path): @@ -124,9 +174,9 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path): def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Path): missing_xml = tmp_path / "missing.xml" - assert run_coverage_check({"coverage": {"tracked_files": []}}, tmp_path, missing_xml) == [ - "coverage xml missing: missing.xml" - ] + assert run_coverage_check( + {"coverage": {"tracked_files": []}}, tmp_path, missing_xml + ) == ["coverage xml missing: missing.xml"] source_dir = tmp_path / "pkg" source_dir.mkdir() @@ -143,7 +193,7 @@ def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Pa - + diff --git a/testing/tests/test_quality_coverage_helpers.py b/testing/tests/test_quality_coverage_helpers.py index 5eaa624e..a42df4ed 100644 --- a/testing/tests/test_quality_coverage_helpers.py +++ b/testing/tests/test_quality_coverage_helpers.py @@ -12,10 +12,15 @@ def test_compute_workspace_line_coverage_handles_missing_xml(tmp_path: Path) -> """Missing coverage XML should produce a zero workspace coverage score.""" contract = {"coverage": {"tracked_files": ["managed.py"]}} - assert compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") == 0.0 + assert ( + compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") + == 0.0 + ) -def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path: Path) -> None: +def test_compute_workspace_line_coverage_averages_present_tracked_files( + tmp_path: Path, +) -> None: """Workspace coverage should average only tracked files that appear in the report.""" coverage_xml = tmp_path / "coverage.xml" @@ -41,7 +46,9 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path assert compute_workspace_line_coverage(contract, tmp_path, coverage_xml) == 75.0 -def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: Path) -> None: +def test_run_check_keeps_relative_names_when_source_roots_do_not_match( + tmp_path: Path, +) -> None: """Relative filenames should remain relative when no declared source root contains them.""" coverage_xml = tmp_path / "coverage.xml" @@ -74,3 +81,59 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: ) assert issues == ["coverage below 95.0%: relative.py (80.0%)"] + + +def test_run_check_enforces_explicit_branch_floor(tmp_path: Path) -> None: + """Branch-tracked modules must report and meet the configured floor.""" + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + """ + + + + + """, + encoding="utf-8", + ) + issues = run_check( + { + "coverage": { + "minimum_percent": 95, + "minimum_branch_percent": 95, + "tracked_files": ["low.py", "missing.py", "legacy.py"], + "branch_tracked_files": ["low.py", "missing.py"], + } + }, + tmp_path, + coverage_xml, + ) + assert issues == [ + "branch coverage below 95.0%: low.py (90.0%)", + "branch coverage missing for tracked file: missing.py", + ] + passing = run_check( + { + "coverage": { + "minimum_percent": 95, + "minimum_branch_percent": 95, + "tracked_files": ["passing.py"], + "branch_tracked_files": ["passing.py"], + } + }, + tmp_path, + coverage_xml, + ) + assert passing == [] + + +def test_workspace_coverage_returns_zero_when_xml_has_no_tracked_samples( + tmp_path: Path, +) -> None: + xml = tmp_path / "coverage.xml" + xml.write_text("", encoding="utf-8") + assert ( + compute_workspace_line_coverage( + {"coverage": {"tracked_files": ["missing.py"]}}, tmp_path, xml + ) + == 0.0 + ) diff --git a/testing/tests/test_semgrep_report.py b/testing/tests/test_semgrep_report.py index e0cf1735..ef59abd0 100644 --- a/testing/tests/test_semgrep_report.py +++ b/testing/tests/test_semgrep_report.py @@ -100,6 +100,14 @@ def test_build_sonar_issues_uses_safe_defaults_for_sparse_findings() -> None: assert issue["primaryLocation"]["textRange"] == {"startLine": 1, "endLine": 1} +def test_sonar_issue_never_ends_before_its_start_line() -> None: + issue = semgrep_report._sonar_issue_from_finding( + {"path": "app.py", "start": {"line": 10}, "end": {"line": 2}} + ) + assert issue is not None + assert issue["primaryLocation"]["textRange"] == {"startLine": 10, "endLine": 10} + + def test_read_json_handles_invalid_json_and_non_object(tmp_path: Path) -> None: """Report loading should fail closed for bad JSON and wrong top-level shapes.""" @@ -108,7 +116,9 @@ def test_read_json_handles_invalid_json_and_non_object(tmp_path: Path) -> None: list_json = tmp_path / "list.json" list_json.write_text("[]", encoding="utf-8") - assert semgrep_report._read_json(bad_json)["errors"][0]["message"].startswith("invalid JSON:") + assert semgrep_report._read_json(bad_json)["errors"][0]["message"].startswith( + "invalid JSON:" + ) assert semgrep_report._read_json(list_json) == { "errors": [{"message": "report payload is not an object"}], "results": [], @@ -137,16 +147,25 @@ def test_main_handles_missing_or_invalid_input(tmp_path: Path) -> None: payload = json.loads(output.read_text(encoding="utf-8")) assert payload["status"] == "failed" assert payload["errors_total"] == 1 - sonar_payload = json.loads((tmp_path / "build" / "sonar-issues.json").read_text(encoding="utf-8")) + sonar_payload = json.loads( + (tmp_path / "build" / "sonar-issues.json").read_text(encoding="utf-8") + ) assert sonar_payload == {"issues": []} -def test_main_honors_custom_blocking_severity_without_sonar_output(tmp_path: Path) -> None: +def test_main_honors_custom_blocking_severity_without_sonar_output( + tmp_path: Path, +) -> None: """CLI callers can choose stricter blocking severities without writing Sonar issues.""" semgrep_json = tmp_path / "semgrep.json" semgrep_json.write_text( - json.dumps({"results": [{"path": "app.py", "extra": {"severity": "WARNING"}}], "errors": []}), + json.dumps( + { + "results": [{"path": "app.py", "extra": {"severity": "WARNING"}}], + "errors": [], + } + ), encoding="utf-8", ) output = tmp_path / "semgrep-report.json" From c808baff40a12a90f43c2e5e39b94d139f88ea29 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 17 Aug 2026 13:12:36 +0000 Subject: [PATCH 3/3] docs(hermes): record repaired conflict graph --- docs/hermes_full_handoff_acceptance.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/hermes_full_handoff_acceptance.md b/docs/hermes_full_handoff_acceptance.md index d7aa9608..be13c1f0 100644 --- a/docs/hermes_full_handoff_acceptance.md +++ b/docs/hermes_full_handoff_acceptance.md @@ -195,6 +195,11 @@ each step: #12/#17/#18. Preserve branch coverage, every tracked module, fixture isolation, and all desired-state changes during conflict resolution. +An exact post-repair merge-tree check against the heads in the table reports +#19 clean with current `main`, #12, #17, and #20, and conflicting with #14, +#15, #16, and #18. Recompute this graph from the Gitea API immediately before +integration; do not infer conflict status from Gitea's draft `mergeable` field. + The #16/#19 integration must keep both the immutable digest and the source tag (`newTag: git--build-` plus the digest) in rendered image identity. Digest-only rendering loses the attested build-source link and therefore