diff --git a/ci/scripts/semgrep_report.py b/ci/scripts/semgrep_report.py index b4ed5a2d..f46e528d 100644 --- a/ci/scripts/semgrep_report.py +++ b/ci/scripts/semgrep_report.py @@ -27,7 +27,10 @@ def _read_json(path: Path) -> dict[str, Any]: except json.JSONDecodeError as exc: return {"errors": [{"message": f"invalid JSON: {exc}"}], "results": []} if not isinstance(payload, dict): - return {"errors": [{"message": "report payload is not an object"}], "results": []} + return { + "errors": [{"message": "report payload is not an object"}], + "results": [], + } return payload @@ -49,10 +52,18 @@ def build_report( blocking = blocking_severities or BLOCKING_SEVERITIES raw_results = semgrep_payload.get("results", []) raw_errors = semgrep_payload.get("errors", []) - results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else [] - errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else [] + results = ( + [item for item in raw_results if isinstance(item, dict)] + if isinstance(raw_results, list) + else [] + ) + errors = ( + [item for item in raw_errors if isinstance(item, dict)] + if isinstance(raw_errors, list) + else [] + ) - severity_counts = {severity: 0 for severity in SEVERITIES} + severity_counts = dict.fromkeys(SEVERITIES, 0) blocking_findings = 0 for finding in results: severity = _finding_severity(finding) @@ -93,16 +104,18 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None: start = finding.get("start") if isinstance(finding.get("start"), dict) else {} end = finding.get("end") if isinstance(finding.get("end"), dict) else {} start_line = _line_number(start.get("line") if isinstance(start, dict) else None) - end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line) - if end_line < start_line: - end_line = start_line + end_line = _line_number( + end.get("line") if isinstance(end, dict) else None, start_line + ) return { "engineId": "semgrep", "ruleId": str(finding.get("check_id") or "semgrep.unknown"), "type": "VULNERABILITY" if severity == "ERROR" else "CODE_SMELL", "severity": SONAR_SEVERITY_BY_SEMGREP[severity], "primaryLocation": { - "message": str(extra.get("message") or finding.get("check_id") or "Semgrep finding"), + "message": str( + extra.get("message") or finding.get("check_id") or "Semgrep finding" + ), "filePath": path, "textRange": { "startLine": start_line, @@ -112,11 +125,19 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None: } -def build_sonar_issues(semgrep_payload: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: +def build_sonar_issues( + semgrep_payload: dict[str, Any], +) -> dict[str, list[dict[str, Any]]]: """Build SonarQube generic external issues from Semgrep JSON output.""" raw_results = semgrep_payload.get("results", []) - results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else [] - issues = [issue for finding in results if (issue := _sonar_issue_from_finding(finding))] + results = ( + [item for item in raw_results if isinstance(item, dict)] + if isinstance(raw_results, list) + else [] + ) + issues = [ + issue for finding in results if (issue := _sonar_issue_from_finding(finding)) + ] return {"issues": issues} @@ -144,7 +165,9 @@ def main(argv: list[str] | None = None) -> int: ) output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + output.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) if args.sonar_issues_output: sonar_output = Path(args.sonar_issues_output) sonar_output.parent.mkdir(parents=True, exist_ok=True) diff --git a/docs/hermes_full_handoff_acceptance.md b/docs/hermes_full_handoff_acceptance.md index 8b9103c2..d7aa9608 100644 --- a/docs/hermes_full_handoff_acceptance.md +++ b/docs/hermes_full_handoff_acceptance.md @@ -1,323 +1,259 @@ -# Hermes full-handoff acceptance harness and release runbook +# Hermes full-handoff acceptance and release runbook -`scripts/ops/hermes_handoff_acceptance.py` decides whether the Hermes platform -handoff is fit to release. It is read-only by default, it classifies every check -`PASS` / `FAIL` / `NOT_RUN` / `NOT_APPLICABLE`, and **any mandatory `FAIL` or -`NOT_RUN` makes the whole run `NO_GO`**. Absence of evidence is never rounded up -to evidence of absence: a probe that could not run, a catalog entry whose -evidence no longer exists, an expired deadline, or a harness bug all land in -`NOT_RUN` and stop the release. +`scripts/ops/hermes_handoff_acceptance.py` is a fail-closed release gate. It +contains 63 checks in 15 groups. An empty catalog, an unknown result status, a +mandatory `FAIL` or `NOT_RUN`, a harness error, or incomplete evidence makes the +decision `NO_GO`. The four ephemeral checks are non-mandatory `NOT_RUN` in the +default mode and are replaced with mandatory results only in armed mode. -``` +This runbook describes evidence collection. It does not authorize a merge, +publish, deployment, Flux reconcile, credential read, or live cluster mutation. +Human review of PR #19 and of the resulting report is required. + +## Required release inputs + +Every value that can go stale must be supplied explicitly. Resolve `main`, PR +heads, the image digest, build source SHA, deployment revision, and chat config +revision from their authoritative read-only sources immediately before the run. +Do not copy the snapshot in the dependency section into a later run. + +```sh scripts/ops/hermes_handoff_acceptance.py \ --context atlas-operator \ - --node-count 21 \ - --expected-suspension flux-system/bstein-dev-home-migrations \ + --remote-main-sha "$REMOTE_MAIN_SHA" \ + --reviewed-head-sha "$REVIEWED_PR19_HEAD" \ + --agent-image "$AGENT_GIT_SHA_BUILD_TAG_AT_SHA256" \ + --build-sha "$IMAGE_BUILD_SOURCE_SHA" \ + --deployment-revision "$RUNNING_DEPLOYMENT_REVISION" \ + --chat-config-revision "$RUNNING_CHAT_CONFIG_REVISION" \ + --dependency-head "12=$PR12_HEAD" \ + --dependency-head "14=$PR14_HEAD" \ + --dependency-head "15=$PR15_HEAD" \ + --dependency-head "16=$PR16_HEAD" \ + --dependency-head "17=$PR17_HEAD" \ + --dependency-head "18=$PR18_HEAD" \ + --dependency-head "20=$PR20_HEAD" \ --output build/hermes-handoff-acceptance.json ``` -Exit status is `0` for `GO` and `1` for `NO_GO`. The JSON report goes to -`--output` (or stdout); a short human summary goes to stderr unless -`--json-only` is passed. +The fixed topology is three pool workers on three distinct nodes. `--node-count` +and `--pool-replicas` cannot lower that requirement. Pool node/ordinal assignment, +Telegram session continuity, and the chat revision are mandatory. Deliberate Flux +suspensions must be named with repeatable `--expected-suspension` arguments. -## Why two vantages +Exit status is `0` only for `GO`, `1` for `NO_GO`, and `2` for invalid execution +bounds. JSON is written to `--output` or stdout; the human summary goes to stderr +unless `--json-only` is passed. -Every authority claim is checked from two places that cannot cover for each -other: +## Structural safety boundary -| Vantage | What it is | What it proves | -| --- | --- | --- | -| `operator` | An external, read-only kubeconfig | Desired state: bindings, manifests, workloads, Flux health, forge state | -| `self` | `kubectl exec` into the running `hermes-agent` container | Enforced state: what the agent's own service-account token can actually do | -| `switchyard`, `node`, `chat` | `kubectl exec` into the Switchyard, node-hardening, and chat pods | Evidence that only exists inside those workloads | +The catalog is validated before a `Runner` exists. Invalid inputs, an empty or +unsafe catalog, and an invalid armed preflight therefore create no subprocess. +The command policy is checked both before and after executable resolution. -Two rules keep this honest: +Default mode permits a narrow read grammar: -* **The harness cannot impersonate from outside.** `--as`, `--as-group`, and - `--as-uid` are rejected by policy for any operator-side command. They are - reachable only as the *inner* command of a `kubectl exec`, where the identity - is testing its own refusal to impersonate rather than borrowing authority. - `kubectl auth can-i --as` is therefore not available as a substitute for a - real self-probe. -* **The two vantages must be different principals.** Before any check runs, the - harness asks each vantage `kubectl auth whoami` and compares. If the operator - and the in-pod probe authenticate as the same principal, the dual-vantage - evidence is a restatement rather than a corroboration, and the run is `NO_GO` - with `the operator and in-pod vantages authenticate as the same principal`. - Run the harness from a genuinely external kubeconfig — not from inside the - Hermes owner pod. +- Kubernetes reads use `name` or reviewed JSONPath projections. Secret resources, + raw API reads, full JSON/YAML, `describe`, `cluster-info dump`, credential-bearing + projections, `auth reconcile`, config operations, all mutations, and every + dry-run mutation trick are rejected before spawn. +- Git is read-only. `fetch`, configuration/helper overrides, alternate worktrees, + external diff helpers, and push are rejected. Helm values are not readable. +- Shell commands must be rendered from frozen templates, use fixed executable + paths, and cannot name service-account, Vault, runtime-access, SSH, or other + credential roots. +- Outer impersonation is rejected. A reviewed inner command can use impersonation + only after an explicit `kubectl exec --` boundary; it cannot substitute for the + independent self identity. +- `PATH` is replaced with a fixed path. Every outer executable is resolved to an + allowed real path, checked for unsafe write mode, and compared with a pinned + SHA-256 digest. A caller-authored `kubectl`, `git`, or helper cannot fabricate + evidence. -Deny checks additionally require a **real refused request**, not only an -authorization review. `kubectl auth can-i` reports what RBAC says; only an -attempt proves the API server enforces it. Mutating attempts go through -`--dry-run=server`: the API server authorises the request and then discards it, -so a refusal is genuine and nothing is written either way. +The executor uses a new process group, drains stdout and stderr concurrently, +and retains at most the configured bytes while the child is running. A monotonic +absolute timeout terminates and reaps the group, including descendants that keep +pipes open. Timeout, deadline, output, and concurrency values reject booleans, +zero/negative values, non-finite numbers, and values outside fixed bounds. +Truncation discards the affected stream and makes every evaluator `NOT_RUN`. -## Safety properties +Credential screening runs before truncation. Command, output, error, identity, +reason, evidence, JSON, and human-summary fields are screened recursively, +including mapping keys and short/numeric sensitive values. The complete rendered +report is screened again; a remaining credential shape is replaced by a bounded +`NO_GO` stub rather than being written. No credential prefix is retained at a +truncation boundary. -These hold structurally, in `hermes_handoff_policy.py`, so a catalog mistake is -a policy error before a run rather than a mutation or a leak during one. +## Vantages and evidence -* **No mutation in the default mode.** Only read subcommands are reachable; - `apply`, `create`, `delete`, `patch`, `replace`, and `scale` require a dry-run - flag. `git push` and every non-`GET` forge method are refused outside the - armed mode. There is deliberately no live TokenRequest probe — a successful - one would mint a real credential. -* **No credential is ever read.** `vault`, `sops`, `age`, `gpg`, `openssl`, - `curl`, and `wget` are not on the binary allowlist. Secrets may only be read - with `-o name`. Environment probes list variable *names* and never values. - Provider health files are read through a frozen template that projects an - allow-listed set of non-secret status fields. The node account's lock state is - reported without reading its password hash; its authorized keys are counted, - never printed. -* **Shell is frozen.** `sh -c` is reachable only through the reviewed, - parameter-validated templates in `SHELL_TEMPLATES`. Parameters may not contain - shell metacharacters. -* **Output is bounded and screened.** Each capture is truncated to a byte budget - *before* it is screened, so a large dump cannot outrun the screen. Bulk - evidence (the routing-log tail, the Kanban listing) is parsed but replaced with - `[output not recorded]` in the artifact. The screen preserves JSON quoting, so - screened structured output is still parsable. -* **The report is re-checked before it is published.** `unscreened_fields` walks - the rendered payload; if anything still looks credential-shaped the report is - replaced with a `NO_GO` stub naming the offending pointers, and the file that - would have leaked is never written. +The operator must use an external read-only kubeconfig. The self probe runs in +the actual Hermes agent pod. Both identities are obtained with +`kubectl auth whoami`; equal or missing principals make the run `NO_GO`. Required +Switchyard and chat/Telegram pod vantages must also resolve. Node coverage is +derived from all daemonset pod node names, not one selected pod. -## What the catalog asserts +Dangerous Kubernetes permissions are checked only with side-effect-free +`kubectl auth can-i` reviews. The default catalog never constructs a Secret, +TokenRequest, mutation, attach, port-forward, or arbitrary exec request. The one +forge administrative denial check performs a read-only GET and accepts only an +explicit authorization refusal; generic `404`, `not found`, and `no route` text +are not denial evidence. -The catalog is declarative (`hermes_handoff_catalog.py` plus the four -`hermes_handoff_checks_*.py` modules) so a reviewer reads what is asserted, not -how it is plumbed. 71 checks in 14 groups: +The strict evaluators reject duplicate-key JSON, raw control characters, +`NaN`/infinity, boolean/number equality coercion, future timestamps, stale +provider evidence, and every malformed routing or Flux row. Routing freshness is +required separately for each provider, lane, and effort. Flux objects must be +non-suspended, current at `observedGeneration`, and explicitly `Ready=True`; +unknown or reconciling state is not healthy. -| Group | Assertions | -| --- | --- | -| `baseline` | `origin/main` descends from the merged worker-isolation baseline; every dependency pull request is merged into `main`; the exact open pull-request heads are recorded | -| `identity` | Codex authenticates as `codex-chatgpt-subscription`; Claude as `firstParty` / `claude.ai` / `claude-code-cli-subscription`; both evidence files are fresh; no provider API-key variable exists in the running process **or** in the manifest | -| `routing` | Switchyard has a ready replica; the routing log covers both providers, the `route` and `worker` lanes, medium/high/xhigh effort, and at least one real fallback; the newest record is fresh; both providers record a measured latency | -| `scopes` | Chat, agent, and triage each declare their own `HERMES_AUTO_ROUTER_PROFILE`, on distinct hostnames | -| `access-denied` | No cluster-admin binding for the agent; Secrets, service-account tokens, impersonation, workload mutation, arbitrary pod exec, attach, port-forward, and control-plane writes are all refused | -| `access-allowed` | get/list/watch on core objects, pod logs, and Flux/Helm status still work; the operator and in-pod namespace inventories agree | -| `gitops` | No unexpected Kustomization or HelmRelease suspension, and nothing unhealthy | -| `forge` | No forge credential in the worker environment or home; the forge identity is not an administrator; repository authority is pull/push only; an administrative route is live-refused; clone/fetch still works | -| `nodes` | The dedicated `hermes-agent` account is locked, has the expected uid/gid/home/shell, joins no supplementary group, has no sudoers fragment, holds exactly one authorized key; K3s credentials, K3s state, kubelet state, and the container runtime are root-only; the audit covers every node | -| `build` | The image builder service account mounts no token; the build pod is unprivileged with exactly `CHOWN, FOWNER, DAC_OVERRIDE, SETGID, SETUID`; the Harbor immutable-tag job succeeded; the running image is digest-pinned | -| `reliability` | The finalization/replay/session patches are running in the pod; the regression suites are present on `main` | -| `pool` | Three ready workers, on distinct nodes, with distinct volumes, mounting no API token and none of the coordinator's claims | -| `surfaces` | Kanban activity and durable sessions are visible and render without blank rows; the agent and chat session patches are deployed; Telegram topic state is on durable storage | -| `ephemeral` | Feature-branch push, draft pull request, protected-branch refusal, and verified cleanup — `NOT_RUN` unless armed | +`release.exact-lineage-is-running` atomically binds: -### Site configuration +- fixed remote `main` SHA and local `origin/main`; +- open draft PR #19, base `main`, existing feature ref, and exact reviewed head; +- the exact `:git--build-@sha256:` image identity, + with the source SHA equal to the release `main` SHA; +- current Deployment generation and revision; +- the one active ReplicaSet revision/hash; every Ready pod image and imageID; +- the Flux-applied `main` revision. -A few checks are emitted only when the operator declares what to expect, -because a `names_present` assertion over an empty expectation would pass on any -workload at all — which reads as coverage without being any: +The pool evaluator requires exactly ordinals 0, 1, and 2, all Ready, on three +distinct nodes and claims, with service-account token automount disabled. The +chat checks always require durable Telegram sessions and the exact running chat +configuration revision. -| Flag | Enables | -| --- | --- | -| `--pool-worker-env NAME` (repeatable) | `pool.assignment-safety-knobs-are-configured` | -| `--chat-config-revision REV` | `surfaces.chat-runs-the-telegram-topic-revision` | -| `--expect-telegram-sessions` with `--chat-ordinal N` | `surfaces.chat-telegram-sessions-are-continuous` | +## Armed ephemeral mode -`--node-count` must be the real node count; the default of `1` makes the -fleet-coverage check vacuous. +Armed mode is the only write path and runs only after the complete default +catalog reports `GO`. -### Catalog drift is a NO_GO, not a pass - -If a workload, field, or manifest is renamed, the affected checks report -`NOT_RUN` because the evidence they name is no longer there. That is the -intended behaviour: the release stops until either the platform or the catalog -is corrected. Do not "fix" a `NOT_RUN` by deleting the check. - -## Ephemeral mutation mode - -Proving that a worker can still open a draft pull request means actually opening -one. That is the only part of the harness that writes anything, and it is a -separate mode: - -``` +```sh scripts/ops/hermes_handoff_acceptance.py \ - --context atlas-operator \ + \ --arm-ephemeral-push \ --confirm "ARM EPHEMERAL HERMES HANDOFF PUSH" \ --ephemeral-token acceptance-20260817a \ --output build/hermes-handoff-acceptance-armed.json ``` -* The confirmation phrase must match exactly. -* `--ephemeral-token` is 8–64 lowercase alphanumeric/dash characters and must be - unique per run; the ref is always - `ephemeral/hermes-handoff-acceptance/`. -* **Preflight runs before any network call**: wrong phrase, wrong repository, - malformed token, or a protected push target all abort with `arming refused` - and a `NO_GO` report. `main`, `master`, `HEAD`, `develop`, `release`, - `production`, `stable`, `trunk`, `prod`, and `default` are refused by name - (case- and `refs/heads/`-insensitive), and anything that is not an ephemeral - acceptance ref is refused by pattern. -* The guard is **exercised on every armed run** by - `ephemeral.protected-branch-refusal`, rather than trusting that it was checked - once. -* The push target is guarded; the pull-request *base* is not. Opening a draft - pull request against `main` writes nothing to `main`. The draft title always - starts with `WIP: `. -* Cleanup closes the pull request, deletes the ref, and then **re-reads both** to - confirm they are gone. `ephemeral.cleanup-verified` is mandatory: a half-removed - branch fails the run rather than passing it. Cleanup runs even when an earlier - step failed. -* `git push --force`, `--force-with-lease`, `--mirror`, `--all`, and `--tags` - are refused even in armed mode. +Before any runner, network request, or cluster vantage is created, preflight +binds the working directory to the existing linked PR #19 worktree, the exact +feature branch and reviewed local HEAD, the fixed HTTPS Atlas origin, base +`main`, confirmation phrase, and one unique ephemeral ref. The command policy is +then pinned to that exact ref. It allows only its push, one exact draft-create +payload, cleanup of the exactly discovered PR number, and deletion of that ref. -A default run reports all four ephemeral checks `NOT_RUN` and non-mandatory, so -a read-only sweep can still reach `GO`. +The run reserves at least 300 seconds (up to 6000) for cleanup. It verifies the +pushed SHA, always discovers the PR after create (including ambiguous create +responses), and requires exactly one open draft with the expected base, head ref, +base SHA, and head SHA. Cleanup requires successful PR close, successful branch deletion, +a successful empty `ls-remote`, and a strict re-read of the same closed PR. +Malformed or partial cleanup is `FAIL`; it is never inferred from an error. -## Release order +Force, force-with-lease, fan-out, protected refs, other repositories, other PR +numbers, arbitrary PATCH payloads, and repository administration remain denied. -### Baseline, already merged and deployed +## Dependency, merge, and conflict graph -PR #13 (`hermes: harden worker isolation and blocked-task semantics`) merged as -`ab346f55509d584e457fe26cf90be3078f7a375c`. Every check below assumes it is -present; `baseline.origin-main-descends-merged-work` asserts it. Override with -`--baseline-commit` only when a later merge legitimately replaces it. +Authoritative snapshot observed 2026-08-17 after refreshing Gitea and Git against +remote `main` `d8f2d818b9a552ea6c2d7fe86554be829bd5ffff`: -### Pending work, observed 2026-08-17T10:07Z - -These heads move. The harness re-fetches them at run time via -`baseline.pending-pull-requests-recorded`; the table is the snapshot this -document was written against, not an input. - -| PR | Branch | Head at observation | State | Delivers | +| PR | Head SHA | Draft | Base | Role | | --- | --- | --- | --- | --- | -| #14 | `feature/hermes-safe-gitea-pr-client` | `3cd8a17e772978fb554d88dd1c59777786be3991` | draft | Bounded forge authority, observer RBAC (removes `hermes-agent-cluster-admin`), node account hardening | -| #15 | `fix/hermes-result-decomposition-reliability` | `7609cd77e87a64e332b2f0d670c1d7a4f62b3d9e` | draft | Result preservation, duplicate-fanout prevention, execution-safety patch | -| #16 | `feature/hermes-unprivileged-image-builder` | `5ab0eba619779b3dfed2eb09d5c02f0a9c92dd0a` | ready | Daemonless Kaniko build lane, tokenless builder, Harbor immutable-tag rule | -| #17 | `feature/hermes-three-lane-placement` | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | draft | Three concurrent CLI lanes on the larger ARM64 nodes | -| #18 | `feature/hermes-distributed-worker-pool` | `20002527512235b29054f3636b02418168373009` | draft | Three-node fenced execution pool with coordinator-only state ownership | +| #12 | `465d386da5237f011cb408f1a3d4b7d0c6d8ffae` | no | `main` | AI usage exporter repair | +| #14 | `8c6e3acdacb223f68b1716f0a74bd543e18890e6` | yes | `main` | bounded forge client and least-privilege RBAC | +| #15 | `73fefbb5d92ef07371a9a997f22d848acd2d06ed` | yes | `main` | result-decomposition reliability | +| #16 | `e8c26ecf85a0c802d378fffffb98b7df43b78242` | no | `main` | unprivileged image builder | +| #17 | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | yes | `main` | three-lane placement | +| #18 | `20002527512235b29054f3636b02418168373009` | yes | `main` | distributed three-worker pool | +| #19 | `8f005458282269ba5c07941814e4237f5d4cf3ac` | yes | `main` | reviewed blocked head; this repair replaces it on the same branch | +| #20 | `a242dcc786576ae1a18000cb4941836ff610dff2` | yes | `main` | CLI process-group reaping | -Base at observation: `30259b526be5b2e226620aeebd9fb5f57dc88b7d`. +The repaired #19 SHA cannot be embedded in its own commit without changing that +SHA. Resolve it from Gitea after push and require it as `--reviewed-head-sha`; +the PR handoff records the exact value. All eight snapshot heads merge cleanly +with the snapshot `main` individually. Pairwise `git merge-tree --write-tree` +found conflicts only for: #12+#17, #14+#15, #14+#16, #14+#17, #14+#20, +#15+#16, #15+#17, #16+#20, and #17+#20. Gitea's `mergeable=false` on a draft is +not treated as conflict evidence. -### Merge order and why +Use this dependency order, refreshing every head and re-running merge-tree at +each step: -1. **#14 first.** It is the only change that *reduces* authority: it deletes - `services/hermes/agent-rbac.yaml` (the `cluster-admin` binding) and replaces - it with the namespaced/cluster observer roles, and it hardens the node - account. Merging it last would mean every earlier acceptance run measured a - cluster-admin agent. Every `access-denied` and `nodes` check depends on it. -2. **#16 second.** The image lane must exist before anything that needs a new - agent image is rolled out. It also touches `testing/quality_contract.json`, - so merging it early keeps later rebases small. `build.*` depends on it. -3. **#15 third.** It changes `dockerfiles/Dockerfile.hermes-agent`, which #16 - also touches — take #16's shape and rebase #15 onto it. `reliability.*` - depends on it. -4. **#17 fourth.** Placement only; it edits `services/hermes/agent-deployment.yaml` - and `testing/tests/test_hermes_cli_lanes.py`, both of which #14, #15, and #18 - also touch. Merging it after them avoids resolving the same conflict twice. -5. **#18 last.** The pool depends on the bounded forge client from #14, the - execution-safety patch from #15, and the agent image from #16. `pool.*` - depends on it. +1. #14 first so the release never restores or prolongs cluster-admin authority. +2. #16, then #15, resolving their Dockerfile and quality-contract changes + additively. Never take one PR's quality contract wholesale. +3. #20 after #15; it depends on the CLI lifecycle work and must be rebased across + the current #14/#16 contract and helper changes. +4. #12 before #17, resolving their Deployment overlap once. +5. #17, then #18, after the authority, image, reliability, and reaping work. +6. #19 last as the integration/acceptance change, only after #12/#14-#18/#20 are + merged and their exact supplied heads are ancestors of refreshed `main`. -Overlapping files to expect (rebase, do not merge blind): `services/hermes/agent-deployment.yaml` -(#12, #14, #16, #17, #18), `services/hermes/kustomization.yaml` (#14, #16, #18), -`services/hermes/scripts/stage_runtime_access.py` (#14, #16, #18), -`services/vault/scripts/vault_k8s_auth_configure.sh` (#14, #16, #18), -`dockerfiles/Dockerfile.hermes-agent` (#15, #16), -`testing/tests/test_hermes_cli_lanes.py` (#14, #15, #17, #18), -`testing/tests/test_hermes_runtime_access.py` (#16, #18). +#19 overlaps `testing/quality_contract.json` with #14/#15/#16, +`testing/quality_coverage.py` and its helper tests with #14, +`test_hermes_auto_router.py` with #18, and existing Kustomize content with +#12/#17/#18. Preserve branch coverage, every tracked module, fixture isolation, +and all desired-state changes during conflict resolution. -PR #12 (AI usage exporter) is independent and can merge at any point; it touches -`services/hermes/agent-deployment.yaml`, so land it before #17 if convenient. +The #16/#19 integration must keep both the immutable digest and the source tag +(`newTag: git--build-` plus the digest) in rendered image identity. +Digest-only rendering loses the attested build-source link and therefore +correctly remains `NO_GO` under the exact-lineage evaluator. -### Image build and Flux rollout +## Safe rollback guidance -1. Merge in the order above; observe the default-branch Jenkins build to a - terminal result after each merge - (`/opt/coordinator/jenkins_build_evidence.py titan-iac --branch main --wait`). -2. After #16 is merged, run the `hermes-agent-image` pipeline. It publishes an - immutable `git--build-` tag and emits - `build/hermes-agent-release/hermes-image-update.patch`. -3. Apply that patch to `services/hermes/kustomization.yaml` on a branch, review, - and merge. The digest is what pins the deployment; - `build.deployed-image-is-digest-pinned` asserts it. -4. Let Flux reconcile. Confirm with `flux get kustomizations -A` that `hermes` - and `hermes-chat` are `Ready`, then wait for the rollout to settle. -5. Re-run the harness. Do not run it mid-rollout: `gitops.*` tolerates an - in-flight reconcile (`Reconciling=True` with `Ready=False`) but a half-rolled - deployment will legitimately fail the `identity`, `routing`, and `surfaces` - groups. +Every rollback is a reviewed forward fix or selective Git revert followed by +the normal Flux workflow. Never restore `hermes-agent-cluster-admin`, even when +reverting unrelated portions of #14. If a rollback would widen RBAC, stop and +write a forward least-privilege fix instead. -### Rollback points +Reverting #18 removes the StatefulSet desired state, but its +`volumeClaimTemplates` PVCs are retained by the Kubernetes default because no +delete retention policy is set. Do not claim Flux pruning deletes those PVCs. +Any PVC deletion is a separate, explicit, human-reviewed data-destruction task +with exact targets and recovery implications. -| After | Roll back by | Restores | -| --- | --- | --- | -| #14 merge | Revert the merge commit; Flux restores `hermes-agent-cluster-admin` | Cluster-admin agent, previous node account state | -| #16 merge | Revert the merge; the previously pinned digest is unchanged in `kustomization.yaml` | Previous build lane; published immutable tags are not deleted | -| Image digest bump | Revert the `kustomization.yaml` commit to the prior digest | Previous agent image; Harbor immutability means the old tag is still exactly what it was | -| #15 merge | Revert the merge; the init-container patch disappears on the next rollout | Previous finalization/replay behaviour | -| #17 merge | Revert the merge | Previous lane concurrency and node placement | -| #18 merge | Revert the merge; the StatefulSet and its PVCs are removed by Flux pruning | Single-coordinator execution | +Image rollback means a reviewed commit restoring a previously attested digest; +never retag an immutable image. Do not use direct `kubectl` edits as a durable +rollback. -Every rollback is a Git revert reconciled by Flux. Do not roll back by direct -`kubectl` edit; the next reconcile would undo it. +## Quality and release evidence -## Go / no-go checklist +The canonical quality contract explicitly manages, lints, and tracks all 15 +`scripts/ops/hermes_handoff_*.py` modules. It enforces at least 95% line and +branch coverage per tracked file and the 500-physical-line source cap. Contract +tests derive the module set so a future file cannot silently escape the gate. +Four pre-existing over-cap test files are temporarily locked to their exact +line count and SHA-256; any byte edit invalidates the exception and must split +the file below 500 lines. PR #14 supplies the intended additive test splits. -Run in order. Stop at the first item that does not hold. - -1. `origin/main` is `ab346f55509d584e457fe26cf90be3078f7a375c` or a descendant. -2. PRs #14, #15, #16, #17, #18 are all merged into `main`, in that order, each - with a green default-branch build. -3. The agent image has been built by the #16 lane, is pinned by digest in - `services/hermes/kustomization.yaml`, and Flux has finished reconciling. -4. The harness is run from an **external operator kubeconfig**, not from inside - the Hermes owner pod, with `--node-count` set to the real node count and every - deliberate Flux suspension declared via `--expected-suspension`. -5. The default read-only run reports `GO` — no mandatory `FAIL`, no mandatory - `NOT_RUN`, no `harness_errors`. -6. The armed run reports `PASS` for all four `ephemeral.*` checks, including - `ephemeral.cleanup-verified`, and the ephemeral ref and pull request are - confirmed gone. -7. The JSON report is archived with the exact `origin/main` commit and the - recorded pull-request heads from `baseline.pending-pull-requests-recorded`. - -Only then is the handoff releasable. - -## Known limits - -Stated so nobody mistakes a proxy for the thing itself. - -* **Merge, approve, and close authority is asserted, not exercised.** The - harness never attempts a merge. It proves the forge identity holds - `permissions.admin: false` and is live-refused on an administrative route; - the route-level allowlist is covered by PR #14's own regression suite. -* **Attach and port-forward have no side-effect-free live attempt.** Their - authorization reviews ride alongside a real, refused control-plane write in the - same check, and pod exec — the one subresource with a harmless real attempt — - has its own check. -* **TokenRequest is asserted by review.** A successful live attempt would mint a - real credential, so the check pairs the `serviceaccounts/token` review with a - refused `serviceaccount` create in the same RBAC family. -* **Branch protection is not read.** The harness identity is not a repository - administrator, so `GET /branch_protections` is refused — which is itself - recorded as evidence. Protected-push refusal is guaranteed by the harness's own - preflight, exercised on every armed run, not by reading the forge's config. -* **Stale/duplicate assignment safety is asserted as deployed configuration.** - The behaviour itself is covered by the pool's regression suite; the harness - checks that the workers carry the configuration and that the coordinator keeps - sole ownership of its claims. -* **Telegram session continuity is per tenant** and depends on there having been - Telegram traffic, so it is off by default. The always-on check is that topic - and tenant state live on a persistent claim rather than in the pod. -* **The routing window is a tail**, 600 records by default. A property that did - not occur inside that window is reported as missing rather than absent; widen - the window or wait for traffic rather than removing the assertion. - -## Tests - -``` -python3 -m pytest testing/tests/ -k handoff -python3 -m coverage run --source=scripts/ops --include="*/hermes_handoff_*.py" \ - -m pytest testing/tests/ -k handoff && python3 -m coverage report +```sh +python3 -m testing.quality_gate --profile local --build-dir build +python3 -m testing.quality_handoff_mutation +python3 -m ruff check --select F,B,SIM,C4,UP --ignore B017,UP015,UP035 \ + scripts/ops/hermes_handoff_*.py testing/quality_*.py testing/tests/test_hermes_handoff_*.py +python3 -m ruff format --check \ + scripts/ops/hermes_handoff_*.py testing/quality_*.py testing/tests/test_hermes_handoff_*.py +python3 -m compileall -q scripts/ops testing +kustomize build services/hermes | kubectl apply --dry-run=client --validate=false -f - +git diff --check origin/main...HEAD ``` -The suite covers parsing, redaction, timeouts and deadlines, partial failure, -zero state, dual-vantage mismatch, mandatory skips, catalog structure, and -ephemeral arming and cleanup — including the cases where cleanup cannot be -verified. +The deterministic mutation gate must kill every policy, redaction, verdict, +numeric, truncation, and cleanup mutant. The adversarial suite includes large +concurrent stdout/stderr, malformed JSON and bytes, zero state, partial failure, +absolute timeout, descendant-held pipes, malformed cleanup, ambiguous creates, +and no-subprocess/no-mutation spies. Render/client dry-run, exact conflict checks, +and bounded diff/full-tree secret scans must also pass. + +The provider-health and Gitea-token-sensitive tests use hermetic fixtures. Their +result must be identical whether synthetic runtime files exist or are absent; +tests must never read the mounted production token. + +## Release decision + +The handoff remains `NO_GO` until all dependency heads are merged, the exact +lineage check passes from an external operator, both default and armed reports +are complete, the branch and PR cleanup is verified, the canonical and +adversarial gates are green, and a human has reviewed PR #19 plus the artifacts. +The harness never merges, deploys, reconciles, publishes, or approves anything. diff --git a/scripts/ops/hermes_handoff_acceptance.py b/scripts/ops/hermes_handoff_acceptance.py index 7be6de29..6b35d49f 100755 --- a/scripts/ops/hermes_handoff_acceptance.py +++ b/scripts/ops/hermes_handoff_acceptance.py @@ -11,8 +11,9 @@ unreachable vantage, a catalog whose evidence no longer exists, an expired deadline, or two vantages that turn out to be the same principal. Nothing here rounds an absence of evidence up to a pass. - scripts/ops/hermes_handoff_acceptance.py --context atlas-operator - scripts/ops/hermes_handoff_acceptance.py --output build/handoff.json + scripts/ops/hermes_handoff_acceptance.py --help + +The release runbook lists every required freshness and lineage input. Mutation lives behind a separate arming flag, needs an exact confirmation phrase and a unique ref, refuses protected push targets before any network @@ -41,17 +42,24 @@ from hermes_handoff_ephemeral import ( # noqa: E402 from hermes_handoff_exec import ( # noqa: E402 DEFAULT_COMMAND_TIMEOUT, DEFAULT_RUN_DEADLINE, - SELF, Runner, ) from hermes_handoff_model import GO, Report, unscreened_fields, utc_now # noqa: E402 -from hermes_handoff_policy import ARMED, READ_ONLY # noqa: E402 +from hermes_handoff_policy import ( # noqa: E402 + ARMED, + EXPECTED_BASE, + EXPECTED_REMOTE, + EXPECTED_REPO, + READ_ONLY, +) from hermes_handoff_redaction import DEFAULT_MAX_BYTES # noqa: E402 from hermes_handoff_run import ( # noqa: E402 build_catalog, build_report, resolve_vantages, unavailable_report, + validate_catalog, + validate_targets, ) EXIT_GO = 0 @@ -59,6 +67,26 @@ EXIT_NO_GO = 1 EXIT_USAGE = 2 +def dependency_head(value: str) -> tuple[int, str]: + """Parse one `PR=SHA` dependency-head binding.""" + try: + number, sha = value.split("=", 1) + parsed = int(number) + except (TypeError, ValueError) as exc: + raise argparse.ArgumentTypeError( + "dependency head must be PR=40-character-SHA" + ) from exc + if ( + parsed <= 0 + or len(sha) != 40 + or any(character not in "0123456789abcdef" for character in sha) + ): + raise argparse.ArgumentTypeError( + "dependency head must be PR=lowercase-40-character-SHA" + ) + return (parsed, sha) + + def build_parser() -> argparse.ArgumentParser: """Return the command-line interface for the harness.""" parser = argparse.ArgumentParser( @@ -67,21 +95,41 @@ def build_parser() -> argparse.ArgumentParser: formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--namespace", default="hermes", help="Hermes namespace.") - parser.add_argument("--context", help="Kubernetes context for the operator vantage.") + parser.add_argument( + "--context", help="Kubernetes context for the operator vantage." + ) parser.add_argument("--kubeconfig", help="Kubeconfig for the operator vantage.") - parser.add_argument("--output", type=Path, help="Write the JSON report here instead of stdout.") - parser.add_argument("--json-only", action="store_true", help="Suppress the human summary.") parser.add_argument( - "--command-timeout", type=float, default=DEFAULT_COMMAND_TIMEOUT, help="Per-command seconds." + "--output", type=Path, help="Write the JSON report here instead of stdout." ) parser.add_argument( - "--deadline", type=float, default=DEFAULT_RUN_DEADLINE, help="Whole-run seconds." + "--json-only", action="store_true", help="Suppress the human summary." ) parser.add_argument( - "--max-output-bytes", type=int, default=DEFAULT_MAX_BYTES, help="Per-capture byte budget." + "--command-timeout", + type=float, + default=DEFAULT_COMMAND_TIMEOUT, + help="Per-command seconds.", ) parser.add_argument( - "--baseline-commit", default=Targets.baseline_commit, help="Commit main must descend from." + "--deadline", + type=float, + default=DEFAULT_RUN_DEADLINE, + help="Whole-run seconds.", + ) + parser.add_argument( + "--max-output-bytes", + type=int, + default=DEFAULT_MAX_BYTES, + help="Per-capture byte budget.", + ) + parser.add_argument( + "--concurrency", type=int, default=1, help="Validated scheduling bound." + ) + parser.add_argument( + "--baseline-commit", + default=Targets.baseline_commit, + help="Commit main must descend from.", ) parser.add_argument( "--dependency-pr", @@ -90,6 +138,31 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="Pull request that must be merged before this release is certifiable (repeatable).", ) + parser.add_argument( + "--dependency-head", + type=dependency_head, + action="append", + default=None, + metavar="PR=SHA", + help="Exact current dependency PR head (repeat for every dependency).", + ) + parser.add_argument( + "--remote-main-sha", default="", help="Exact observed origin/main SHA." + ) + parser.add_argument( + "--reviewed-head-sha", default="", help="Exact reviewed PR #19 head SHA." + ) + parser.add_argument( + "--agent-image", + default="", + help="Exact agent :git--build-@sha256 identity.", + ) + parser.add_argument( + "--build-sha", default="", help="Exact source SHA used for the image build." + ) + parser.add_argument( + "--deployment-revision", default="", help="Exact running Deployment revision." + ) parser.add_argument( "--expected-suspension", action="append", @@ -97,10 +170,19 @@ def build_parser() -> argparse.ArgumentParser: metavar="NAMESPACE/NAME", help="Flux object allowed to be suspended (repeatable).", ) - parser.add_argument("--node-count", type=int, default=1, help="Nodes the fleet must cover.") - parser.add_argument("--pool-replicas", type=int, default=3, help="Expected pool worker count.") parser.add_argument( - "--chat-config-revision", default="", help="Expected chat pod-template config revision." + "--node-count", + type=int, + default=3, + help="Nodes the fleet must cover (fixed at 3).", + ) + parser.add_argument( + "--pool-replicas", type=int, default=3, help="Expected pool worker count." + ) + parser.add_argument( + "--chat-config-revision", + default="", + help="Expected chat pod-template config revision.", ) parser.add_argument( "--chat-ordinal", @@ -111,6 +193,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--expect-telegram-sessions", action="store_true", + default=True, help="Require the selected chat tenant to hold durable Telegram-sourced sessions.", ) parser.add_argument( @@ -131,11 +214,12 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Permit one ephemeral branch push and draft pull request, then remove both.", ) - armed.add_argument("--confirm", default="", help=f"Exact confirmation phrase: {CONFIRMATION!r}.") - armed.add_argument("--ephemeral-token", default="", help="Unique 8-64 character ref suffix.") - armed.add_argument("--ephemeral-base", default="main", help="Pull-request base branch.") - armed.add_argument("--remote", default="origin", help="Git remote for the ephemeral push.") - armed.add_argument("--repo", default=Targets.repo, help="owner/name of the forge repository.") + armed.add_argument( + "--confirm", default="", help=f"Exact confirmation phrase: {CONFIRMATION!r}." + ) + armed.add_argument( + "--ephemeral-token", default="", help="Unique 8-64 character ref suffix." + ) return parser @@ -145,17 +229,23 @@ def targets_from_args(arguments: argparse.Namespace, now: dt.datetime) -> Target return Targets( namespace=arguments.namespace, baseline_commit=arguments.baseline_commit, - dependency_pull_requests=tuple(arguments.dependency_pr or defaults.dependency_pull_requests), + dependency_pull_requests=tuple( + arguments.dependency_pr or defaults.dependency_pull_requests + ), + dependency_heads=tuple(arguments.dependency_head or ()), + remote_main_sha=arguments.remote_main_sha, + reviewed_head_sha=arguments.reviewed_head_sha, + agent_image=arguments.agent_image, + build_sha=arguments.build_sha, + deployment_revision=arguments.deployment_revision, expected_suspensions=tuple(arguments.expected_suspension or ()), node_count=arguments.node_count, pool_replicas=arguments.pool_replicas, chat_config_revision=arguments.chat_config_revision, chat_ordinal=arguments.chat_ordinal, expect_telegram_sessions=arguments.expect_telegram_sessions, - pool_worker_env=tuple(arguments.pool_worker_env or ()), + pool_worker_env=tuple(arguments.pool_worker_env or defaults.pool_worker_env), max_evidence_age_seconds=arguments.max_evidence_age, - repo=arguments.repo, - remote=arguments.remote, now=now, ) @@ -186,14 +276,16 @@ def emit(report: Report, arguments: argparse.Namespace, stream=None) -> list[str def _armed_results(arguments: argparse.Namespace, runner: Runner, vantages) -> list: request = ArmRequest( - repo=arguments.repo, - remote=arguments.remote, + repo=EXPECTED_REPO, + remote=EXPECTED_REMOTE, token=arguments.ephemeral_token, confirmation=arguments.confirm, - base=arguments.ephemeral_base, + base=EXPECTED_BASE, + expected_head=arguments.reviewed_head_sha, + expected_base_sha=arguments.remote_main_sha, ) - preflight(request, arguments.repo) - vantage = vantages.get(SELF) or vantages["operator"] + preflight(request, EXPECTED_REPO, Path.cwd()) + vantage = vantages["operator"] return run_armed(runner, vantage, request) @@ -205,27 +297,64 @@ def main(argv: Sequence[str] | None = None) -> int: targets = targets_from_args(arguments, now) mode = ARMED if arguments.arm_ephemeral_push else READ_ONLY - runner = Runner( - mode=mode, - command_timeout=arguments.command_timeout, - deadline_seconds=arguments.deadline, - max_bytes=arguments.max_output_bytes, - ) - vantages, records = resolve_vantages(runner, targets, arguments.kubeconfig, arguments.context) - - armed_results: list = [] + specs = build_catalog(targets) + startup_problems = [*validate_targets(targets), *validate_catalog(specs)] if arguments.arm_ephemeral_push: try: - armed_results = _armed_results(arguments, runner, vantages) + preflight( + ArmRequest( + repo=EXPECTED_REPO, + remote=EXPECTED_REMOTE, + token=arguments.ephemeral_token, + confirmation=arguments.confirm, + base=EXPECTED_BASE, + expected_head=arguments.reviewed_head_sha, + expected_base_sha=arguments.remote_main_sha, + ), + EXPECTED_REPO, + Path.cwd(), + ) except ArmingError as exc: - report = unavailable_report(mode, started_at, f"arming refused: {exc}", targets) - emit(report, arguments) - return EXIT_NO_GO + startup_problems.append(f"arming refused: {exc}") + if startup_problems: + report = unavailable_report( + mode, started_at, "; ".join(startup_problems), targets + ) + emit(report, arguments) + return EXIT_NO_GO - specs = build_catalog(targets) - report = build_report( - runner, targets, specs, vantages, records, mode, started_at, armed_results + try: + runner = Runner( + mode=mode, + command_timeout=arguments.command_timeout, + deadline_seconds=arguments.deadline, + max_bytes=arguments.max_output_bytes, + concurrency=arguments.concurrency, + ) + except ValueError as exc: + report = unavailable_report( + mode, started_at, f"invalid execution bound: {exc}", targets + ) + emit(report, arguments) + return EXIT_USAGE + vantages, records = resolve_vantages( + runner, targets, arguments.kubeconfig, arguments.context ) + report = build_report(runner, targets, specs, vantages, records, mode, started_at) + if arguments.arm_ephemeral_push: + if report.decision != GO: + report.harness_errors.append( + "armed mutation was not started because read-only acceptance did not GO" + ) + else: + armed_results = _armed_results(arguments, runner, vantages) + armed_ids = {result.spec.id for result in armed_results} + report.results = [ + result for result in report.results if result.spec.id not in armed_ids + ] + report.results.extend(armed_results) + report.results.sort(key=lambda result: result.spec.id) + report.finished_at = utc_now() offenders = emit(report, arguments) if offenders: return EXIT_NO_GO diff --git a/scripts/ops/hermes_handoff_catalog.py b/scripts/ops/hermes_handoff_catalog.py index b59ff2b9..9dc5c98a 100644 --- a/scripts/ops/hermes_handoff_catalog.py +++ b/scripts/ops/hermes_handoff_catalog.py @@ -4,8 +4,8 @@ The catalog is data on purpose. A reviewer should be able to read what the release is asserted to satisfy without reading any plumbing, and the harness should be able to prove structural properties over every entry — both vantages -represented, no impersonation from outside a pod, no mutation outside a server -dry run — before it runs a single command. +represented, no impersonation from outside a pod, and no mutation command of +any kind — before it runs a single command. Catalog drift is a NO_GO, not a pass. If a manifest, a field name, or a workload is renamed, the affected checks report ``NOT_RUN`` because the evidence they @@ -19,7 +19,8 @@ from collections.abc import Sequence from dataclasses import dataclass, field from typing import Any -from hermes_handoff_model import ALWAYS, ATTEMPT, READ, REVIEW, CheckSpec, Step +from hermes_handoff_model import ALWAYS, READ, CheckSpec, Step +from hermes_handoff_policy import projection OPERATOR = "operator" SELF = "self" @@ -28,21 +29,38 @@ NODE = "node" CHAT = "chat" VANTAGE_NAMES = (OPERATOR, SELF, SWITCHYARD, NODE, CHAT) -# `namespace/namesuspendType=Status,...` — one compact row per object. -FLUX_PROJECTION = ( - "jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{'\\t'}" - "{.spec.suspend}{'\\t'}{range .status.conditions[*]}{.type}={.status},{end}{'\\n'}{end}" -).replace("'", '"') +# `namegenerationobservedGenerationsuspendconditions`. +FLUX_PROJECTION = projection( + ( + "jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{'\\t'}" + "{.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}" + "{range .status.conditions[*]}{.type}={.status},{end}{'\\n'}{end}" + ).replace("'", '"') +) -ENV_NAME_PROJECTION = ( - "jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{'\\n'}{end}{end}" -).replace("'", '"') +ENV_NAME_PROJECTION = projection( + ( + "jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{'\\n'}{end}{end}" + ).replace("'", '"') +) -INIT_NAME_PROJECTION = ( - "jsonpath={range .spec.template.spec.initContainers[*]}{.name}{'\\n'}{end}" -).replace("'", '"') +INIT_NAME_PROJECTION = projection( + ( + "jsonpath={range .spec.template.spec.initContainers[*]}{.name}{'\\n'}{end}" + ).replace("'", '"') +) -NODE_NAME_PROJECTION = "jsonpath={range .items[*]}{.spec.nodeName}{'\\n'}{end}".replace("'", '"') +NODE_NAME_PROJECTION = projection( + "jsonpath={range .items[*]}{.spec.nodeName}{'\\n'}{end}".replace("'", '"') +) + +POOL_PROJECTION = projection( + ( + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.spec.nodeName}{'\\t'}" + "{.status.phase}{'\\t'}{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}" + "{'\\t'}{range .spec.volumes[*]}{.persistentVolumeClaim.claimName}{','}{end}{'\\n'}{end}" + ).replace("'", '"') +) PROVIDER_API_KEY_NAMES = ( "ANTHROPIC_API_KEY", @@ -56,7 +74,12 @@ PROVIDER_API_KEY_NAMES = ( "OPENAI_API_KEY", "OPENAI_BASE_URL", ) -FORGE_CREDENTIAL_NAMES = ("GITEA_PASSWORD", "GITEA_TOKEN", "GITEA_USERNAME", "GIT_PASSWORD") +FORGE_CREDENTIAL_NAMES = ( + "GITEA_PASSWORD", + "GITEA_TOKEN", + "GITEA_USERNAME", + "GIT_PASSWORD", +) CODEX_HEALTH = "/opt/data/provider-health/codex.json" CLAUDE_HEALTH = "/opt/data/provider-health/claude.json" @@ -92,31 +115,44 @@ class Targets: pool_statefulset: str = "hermes-execution-worker" pool_replicas: int = 3 pool_selector: str = "app=hermes-execution-worker" - coordinator_claims: tuple[str, ...] = ("hermes-agent-home", "hermes-agent-workspace") + coordinator_claims: tuple[str, ...] = ( + "hermes-agent-home", + "hermes-agent-workspace", + ) node_account: str = "hermes-agent" node_account_identity: str = "1200:1200:/home/hermes-agent:/bin/bash" - node_count: int = 1 + node_count: int = 3 repo: str = "atlas/titan-iac" remote: str = "origin" baseline_commit: str = "ab346f55509d584e457fe26cf90be3078f7a375c" - dependency_pull_requests: tuple[int, ...] = (14, 15, 16, 17, 18) + dependency_pull_requests: tuple[int, ...] = (12, 14, 15, 16, 17, 18, 20) + dependency_heads: tuple[tuple[int, str], ...] = () + remote_main_sha: str = "" + reviewed_pr_number: int = 19 + reviewed_head_ref: str = "feature/hermes-full-handoff-acceptance" + reviewed_head_sha: str = "" + agent_image: str = "" + build_sha: str = "" + deployment_revision: str = "" expected_suspensions: tuple[str, ...] = () harbor_namespace: str = "harbor" harbor_immutability_job: str = "hermes-agent-immutability" builder_namespace: str = "jenkins" builder_serviceaccount: str = "hermes-image-builder" builder_pipeline: str = "ci/Jenkinsfile.hermes-agent-image" - builder_capabilities: str = 'add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]' + builder_capabilities: str = ( + 'add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]' + ) agent_init_containers: tuple[str, ...] = ("patch-web-session-activity",) chat_init_containers: tuple[str, ...] = ("patch-api-server-sessions",) chat_config_revision: str = "" - pool_worker_env: tuple[str, ...] = () + pool_worker_env: tuple[str, ...] = ("HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL") max_evidence_age_seconds: int = 86_400 routing_tail_lines: int = 600 routing_tail_bytes: int = 512 * 1024 listing_bytes: int = 512 * 1024 git_askpass: str = "/opt/coordinator/gitea_askpass.sh" - expect_telegram_sessions: bool = False + expect_telegram_sessions: bool = True now: Any = None extra: dict[str, Any] = field(default_factory=dict) @@ -165,25 +201,3 @@ def check( scope=scope, rationale=rationale, ) - - -def deny( - identifier: str, - title: str, - review: Sequence[str], - attempt: Sequence[str], - rationale: str, - group: str = "access-denied", -) -> CheckSpec: - """Return a deny check that pairs a self review with a real refused request.""" - return check( - identifier, - title, - group, - "denied", - [ - step("review", SELF, *review, kind=REVIEW), - step("attempt", SELF, *attempt, kind=ATTEMPT), - ], - rationale=rationale, - ) diff --git a/scripts/ops/hermes_handoff_checks_access.py b/scripts/ops/hermes_handoff_checks_access.py index a863e0f3..df5c8117 100644 --- a/scripts/ops/hermes_handoff_checks_access.py +++ b/scripts/ops/hermes_handoff_checks_access.py @@ -1,11 +1,9 @@ #!/usr/bin/env python3 """Acceptance checks for what Hermes may and may not do to the cluster. -Every deny check pairs an in-pod authorization review with a real request that -must be refused, because a review reports what RBAC says and only an attempt -proves the API server enforces it. Mutating attempts go through a server dry -run: the API server authorises the request and discards it, so the refusal is -genuine and nothing is written either way. +Denials use Kubernetes authorization reviews only. Live denied attempts were +removed because a policy regression could turn a supposed probe into a real +credential read or mutation; the default harness never constructs such argv. """ from __future__ import annotations @@ -16,11 +14,9 @@ from hermes_handoff_catalog import ( SELF, Targets, check, - deny, step, ) -from hermes_handoff_model import ATTEMPT, REVIEW, CheckSpec -from hermes_handoff_policy import shell +from hermes_handoff_model import REVIEW, CheckSpec def _access(targets: Targets) -> list[CheckSpec]: @@ -31,22 +27,45 @@ def _access(targets: Targets) -> list[CheckSpec]: "No ClusterRoleBinding grants the agent service account cluster-admin", "access-denied", "names_absent", - [step("bindings", OPERATOR, "kubectl", "get", "clusterrolebindings", "-o", "name")], + [ + step( + "bindings", + OPERATOR, + "kubectl", + "get", + "clusterrolebindings", + "-o", + "name", + ) + ], {"step": "bindings", "contains": ("hermes-agent-cluster-admin",)}, - rationale="The operator half of the pair; the in-pod attempts below are the enforcement half.", + rationale="The binding inventory and in-pod authorization reviews are independent desired/enforced views.", ), - deny( + check( "access.secrets-are-denied", "Hermes cannot read Secrets cluster-wide", - ("kubectl", "auth", "can-i", "get", "secrets", "--all-namespaces"), - ("kubectl", "get", "secrets", "--all-namespaces", "-o", "name"), - "Even a regression that granted this could only return object names: the probe is pinned to -o name.", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "get", + "secrets", + "--all-namespaces", + kind=REVIEW, + ) + ], + rationale="A live Secret request is never constructed, even when an RBAC regression would allow it.", ), check( "access.service-account-tokens-are-denied", "Hermes cannot mint a service-account token", "access-denied", - "denied", + "review_denied", [ step( "review-token", @@ -72,119 +91,155 @@ def _access(targets: Targets) -> list[CheckSpec]: namespace, kind=REVIEW, ), - step( - "attempt", - SELF, - "kubectl", - "--namespace", - namespace, - "create", - "serviceaccount", - "hermes-acceptance-probe", - "--dry-run=server", - kind=ATTEMPT, - ), ], - rationale="TokenRequest has no side-effect-free live attempt — a successful one would mint a real credential — so it is asserted by review alongside a refused write in the same RBAC family.", + rationale="Token creation has no credential-safe live probe, so both relevant grants must be denied by review.", ), - deny( + check( "access.impersonation-is-denied", "Hermes cannot impersonate another identity", - ("kubectl", "auth", "can-i", "impersonate", "users", "--all-namespaces"), - ("kubectl", "get", "namespaces", "--as", "system:admin", "-o", "name"), - "`--as` is reachable only from inside the pod, where the identity is testing its own refusal.", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "impersonate", + "users", + "--all-namespaces", + kind=REVIEW, + ) + ], + rationale="The default harness never constructs an impersonated request.", ), - deny( + check( "access.workload-mutation-is-denied", "Hermes cannot mutate its own workload", - ("kubectl", "auth", "can-i", "patch", "deployments", "--namespace", namespace), - ( - "kubectl", - "--namespace", - namespace, - "patch", - f"deploy/{targets.agent_deployment}", - "--type=merge", - "--patch", - '{"metadata":{"annotations":{"acceptance.bstein.dev/probe":"deny"}}}', - "--dry-run=server", - ), - "A server dry run is authorised and then discarded, so the refusal is real and nothing is written.", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "patch", + "deployments", + "--namespace", + namespace, + kind=REVIEW, + ) + ], + rationale="Client-side and server-side dry-run mutation tricks are excluded structurally.", ), - deny( + check( "access.pod-exec-is-denied", "Hermes cannot exec into an arbitrary pod", - ("kubectl", "auth", "can-i", "create", "pods/exec", "--all-namespaces"), - ( - "kubectl", - "--namespace", - namespace, - "exec", - f"deploy/{targets.switchyard_deployment}", - "--container", - targets.switchyard_container, - "--", - *shell("path_readable", path="/tmp"), - ), - "Exec is the one subresource with a harmless real attempt; attach and port-forward are covered by their own reviews.", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "pods/exec", + "--all-namespaces", + kind=REVIEW, + ) + ], + rationale="No cross-pod exec request is created by a read-only run.", ), check( "access.attach-portforward-and-kube-system-writes-are-denied", "Hermes holds no attach, port-forward, or control-plane write authority", "access-denied", - "denied", + "review_denied", [ - step("review-attach", SELF, "kubectl", "auth", "can-i", "create", "pods/attach", "--all-namespaces", kind=REVIEW), - step("review-portforward", SELF, "kubectl", "auth", "can-i", "create", "pods/portforward", "--all-namespaces", kind=REVIEW), - step("review-create", SELF, "kubectl", "auth", "can-i", "create", "configmaps", "--namespace", "kube-system", kind=REVIEW), step( - "attempt", + "review-attach", SELF, "kubectl", + "auth", + "can-i", + "create", + "pods/attach", + "--all-namespaces", + kind=REVIEW, + ), + step( + "review-portforward", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "pods/portforward", + "--all-namespaces", + kind=REVIEW, + ), + step( + "review-create", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "configmaps", "--namespace", "kube-system", - "create", - "configmap", - "hermes-acceptance-probe", - "--dry-run=server", - kind=ATTEMPT, + kind=REVIEW, ), ], - rationale="Attach and port-forward have no side-effect-free live attempt, so their reviews ride alongside a real, refused control-plane write.", + rationale="Every authority is reviewed without issuing the request it would authorize.", ), check( "access.required-reads-are-allowed", "Hermes retains the reads its operator role depends on", "access-allowed", "allowed", - [ - step("watch-review", SELF, "kubectl", "auth", "can-i", "watch", "pods", "--namespace", namespace, kind=REVIEW), - step("pods", SELF, "kubectl", "--namespace", namespace, "get", "pods", "-o", "name"), - step("namespaces", SELF, "kubectl", "get", "namespaces", "-o", "name"), - step("deployments", SELF, "kubectl", "--namespace", namespace, "get", "deployments", "-o", "name"), - step("nodes", SELF, "kubectl", "get", "nodes", "-o", "name"), - ], - ), - check( - "access.pod-logs-are-allowed", - "Hermes can still read pod logs", - "access-allowed", - "allowed", [ step( - "logs", + "watch-review", + SELF, + "kubectl", + "auth", + "can-i", + "watch", + "pods", + "--namespace", + namespace, + kind=REVIEW, + ), + step( + "pods", SELF, "kubectl", "--namespace", - targets.namespace, - "logs", - f"deploy/{targets.switchyard_deployment}", - "--container", - targets.switchyard_container, - "--tail", - "1", - record=False, - ) + namespace, + "get", + "pods", + "-o", + "name", + ), + step("namespaces", SELF, "kubectl", "get", "namespaces", "-o", "name"), + step( + "deployments", + SELF, + "kubectl", + "--namespace", + namespace, + "get", + "deployments", + "-o", + "name", + ), + step("nodes", SELF, "kubectl", "get", "nodes", "-o", "name"), ], ), check( @@ -193,8 +248,28 @@ def _access(targets: Targets) -> list[CheckSpec]: "access-allowed", "allowed", [ - step("kustomizations", SELF, "kubectl", "get", "kustomizations.kustomize.toolkit.fluxcd.io", "--all-namespaces", "-o", "name", record=False), - step("helmreleases", SELF, "kubectl", "get", "helmreleases.helm.toolkit.fluxcd.io", "--all-namespaces", "-o", "name", record=False), + step( + "kustomizations", + SELF, + "kubectl", + "get", + "kustomizations.kustomize.toolkit.fluxcd.io", + "--all-namespaces", + "-o", + "name", + record=False, + ), + step( + "helmreleases", + SELF, + "kubectl", + "get", + "helmreleases.helm.toolkit.fluxcd.io", + "--all-namespaces", + "-o", + "name", + record=False, + ), ], ), check( @@ -203,8 +278,26 @@ def _access(targets: Targets) -> list[CheckSpec]: "access-allowed", "vantages_agree", [ - step("operator", OPERATOR, "kubectl", "get", "namespaces", "-o", "name", record=False), - step("self", SELF, "kubectl", "get", "namespaces", "-o", "name", record=False), + step( + "operator", + OPERATOR, + "kubectl", + "get", + "namespaces", + "-o", + "name", + record=False, + ), + step( + "self", + SELF, + "kubectl", + "get", + "namespaces", + "-o", + "name", + record=False, + ), ], {"steps": ("operator", "self")}, rationale="A disagreement here means one vantage is not seeing the cluster the other is certifying.", @@ -219,7 +312,19 @@ def _gitops(targets: Targets) -> list[CheckSpec]: f"No unexpected {label} suspension or unhealthy reconciliation", "gitops", "flux_health", - [step("objects", OPERATOR, "kubectl", "get", kind, "--all-namespaces", "-o", FLUX_PROJECTION, record=False)], + [ + step( + "objects", + OPERATOR, + "kubectl", + "get", + kind, + "--all-namespaces", + "-o", + FLUX_PROJECTION, + record=False, + ) + ], {"step": "objects", "expected_suspensions": targets.expected_suspensions}, ) for kind, label in ( diff --git a/scripts/ops/hermes_handoff_checks_delivery.py b/scripts/ops/hermes_handoff_checks_delivery.py index a3dbaad7..102f93e4 100644 --- a/scripts/ops/hermes_handoff_checks_delivery.py +++ b/scripts/ops/hermes_handoff_checks_delivery.py @@ -17,7 +17,6 @@ from __future__ import annotations from hermes_handoff_catalog import ( FORGE_CREDENTIAL_NAMES, INIT_NAME_PROJECTION, - NODE, NODE_NAME_PROJECTION, OPERATOR, SELF, @@ -26,10 +25,7 @@ from hermes_handoff_catalog import ( step, ) from hermes_handoff_model import ATTEMPT, CheckSpec -from hermes_handoff_policy import GITEA_CLIENT, shell - -HOST_ETC = "/host-etc" -HOST_HOME = "/host-home" +from hermes_handoff_policy import GITEA_CLIENT, projection, shell def _forge(targets: Targets) -> list[CheckSpec]: @@ -44,14 +40,6 @@ def _forge(targets: Targets) -> list[CheckSpec]: {"step": "env", "names": FORGE_CREDENTIAL_NAMES}, rationale="Authentication reaches Git through an askpass helper that reads runtime state, never through the environment.", ), - check( - "forge.worker-holds-no-credential-file", - "The worker home has no stored Git credential file", - "forge", - "stdout_matches", - [step("credentials", SELF, *shell("path_readable", path="/opt/data/.git-credentials"))], - {"step": "credentials", "equals": "unreadable"}, - ), check( "forge.identity-is-not-an-administrator", "The forge identity the worker uses holds no administrative rights", @@ -93,117 +81,11 @@ def _forge(targets: Targets) -> list[CheckSpec]: ], rationale="A live refusal, not a configuration reading. The same lack of authority is what blocks a merge route.", ), - check( - "forge.clone-and-fetch-still-work", - "The worker can still read the repository it is expected to work in", - "forge", - "allowed", - [ - step( - "ls-remote", - SELF, - *shell( - "git_ls_remote", - askpass=targets.git_askpass, - url=f"https://scm.bstein.dev/{targets.repo}.git", - ), - ) - ], - ), ] def _nodes(targets: Targets) -> list[CheckSpec]: - account = targets.node_account return [ - check( - "nodes.account-password-is-locked", - "The dedicated node account has no usable password", - "nodes", - "stdout_matches", - [ - step( - "lock", - NODE, - *shell("account_lock_state", account=account, path=f"{HOST_ETC}/shadow"), - ) - ], - {"step": "lock", "equals": "locked"}, - rationale="The probe reports the lock marker only; the password hash is never read.", - ), - check( - "nodes.account-identity-is-dedicated", - "The node account is the dedicated Hermes identity, not a shared one", - "nodes", - "stdout_matches", - [ - step( - "identity", - NODE, - *shell("account_identity", account=account, path=f"{HOST_ETC}/passwd"), - ) - ], - {"step": "identity", "equals": targets.node_account_identity}, - ), - check( - "nodes.account-holds-no-supplementary-groups", - "The node account joins no supplementary or administrative group", - "nodes", - "stdout_matches", - [ - step( - "groups", - NODE, - *shell("account_groups", account=account, path=f"{HOST_ETC}/group"), - ) - ], - {"step": "groups", "equals": ""}, - rationale="sudo, docker, adm, and disk are all group grants; an empty membership list rules out every one of them.", - ), - check( - "nodes.account-has-no-sudo-policy", - "No sudoers fragment names the node account", - "nodes", - "stdout_matches", - [step("sudoers", NODE, *shell("path_mode", path=f"{HOST_ETC}/sudoers.d/{account}"))], - {"step": "sudoers", "equals": "absent"}, - ), - check( - "nodes.account-holds-a-single-authorized-key", - "Exactly one authorized key is installed for the node account", - "nodes", - "stdout_matches", - [ - step( - "keys", - NODE, - *shell("line_count", path=f"{HOST_HOME}/{account}/.ssh/authorized_keys"), - ) - ], - {"step": "keys", "equals": "1"}, - rationale="A count, never the key material. A second key is how a human identity re-enters an account that is meant to be dedicated.", - ), - *[ - check( - f"nodes.{label}-is-root-only", - f"{title} remains unreadable to the node account", - "nodes", - "stdout_matches", - [step("mode", NODE, *shell("path_mode", path=path))], - {"step": "mode", "equals": expected}, - ) - for label, title, path, expected in ( - ( - "k3s-credentials", - "The K3s admin kubeconfig", - f"{HOST_ETC}/rancher/k3s/k3s.yaml", - ["600 root:root", "absent"], - ), - ("k3s-state", "K3s server state", "/host-k3s", ["700 root:root", "absent"]), - ("kubelet-state", "Kubelet state", "/host-kubelet", ["700 root:root", "750 root:root", "absent"]), - ("container-runtime", "The container runtime socket directory", "/host-run-containerd", ["710 root:root", "700 root:root", "absent"]), - ) - ], check( "nodes.hardening-covers-every-node", "The node account audit runs on every node in the cluster", @@ -224,7 +106,11 @@ def _nodes(targets: Targets) -> list[CheckSpec]: NODE_NAME_PROJECTION, ) ], - {"step": "coverage", "minimum": targets.node_count}, + { + "step": "coverage", + "minimum": targets.node_count, + "total_equals": targets.node_count, + }, rationale="A hardened fleet with one unenrolled node is an unhardened fleet.", ), ] @@ -237,7 +123,7 @@ def _build(targets: Targets) -> list[CheckSpec]: "build.builder-service-account-is-tokenless", "The image builder service account mounts no API token", "build", - "json_field", + "stdout_matches", [ step( "serviceaccount", @@ -249,10 +135,10 @@ def _build(targets: Targets) -> list[CheckSpec]: "serviceaccount", targets.builder_serviceaccount, "-o", - "json", + projection("jsonpath={.automountServiceAccountToken}"), ) ], - {"step": "serviceaccount", "fields": {"automountServiceAccountToken": False}}, + {"step": "serviceaccount", "equals": "false"}, ), check( "build.builder-runs-unprivileged-with-exact-capabilities", @@ -276,7 +162,7 @@ def _build(targets: Targets) -> list[CheckSpec]: "build.harbor-immutability-rule-is-applied", "The Harbor immutable-tag rule for the agent image has been applied", "build", - "json_numeric", + "lines_match", [ step( "job", @@ -288,35 +174,145 @@ def _build(targets: Targets) -> list[CheckSpec]: "job", targets.harbor_immutability_job, "-o", - "json", + projection("jsonpath={.status.succeeded}"), ) ], - {"step": "job", "fields": {"status.succeeded": {"min": 1}}}, + {"step": "job", "pattern": r"^[1-9][0-9]*$", "minimum": 1}, rationale="The job verifies the rule it creates, so a completed run is evidence the rule matched the expected contract.", ), - check( - "build.deployed-image-is-digest-pinned", - "The running agent image is pinned by digest, not by a mutable tag", - "build", - "names_present", - [ - step( - "image", - OPERATOR, - "kubectl", - "--namespace", - targets.namespace, - "get", - f"deploy/{targets.agent_deployment}", - "-o", - 'jsonpath={range .spec.template.spec.containers[*]}{.image}{"\\n"}{end}', - ) - ], - {"step": "image", "contains": ("@sha256:",)}, - ), + _release_lineage(targets), ] +def _release_lineage(targets: Targets) -> CheckSpec: + """Bind the exact reviewed release from Git through the Ready pods.""" + container = targets.agent_container + namespace = targets.namespace + deployment = f"deploy/{targets.agent_deployment}" + deployment_projection = projection( + ( + "jsonpath={.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}" + "{.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}" + "{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}" + f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}' + ).replace("'", '"') + ) + pod_projection = projection( + ( + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.status.phase}{'\\t'}" + "{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}" + f'{{.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}' + f'{{.status.containerStatuses[?(@.name=="{container}")].imageID}}{{"\\t"}}' + "{.metadata.labels.pod-template-hash}{'\\n'}{end}" + ).replace("'", '"') + ) + replicaset_projection = projection( + ( + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}" + "{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}" + "{.status.readyReplicas}{'\\t'}{.status.availableReplicas}{'\\t'}" + f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}' + "{.metadata.labels.pod-template-hash}{'\\n'}{end}" + ).replace("'", '"') + ) + flux_projection = projection( + ( + "jsonpath={.metadata.name}{'\\t'}{.metadata.generation}{'\\t'}" + "{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}" + "{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}" + "{.status.lastAppliedRevision}" + ).replace("'", '"') + ) + return check( + "release.exact-lineage-is-running", + "Remote main, reviewed PR, image, build, Flux, and running revision are identical", + "release", + "release_lineage", + [ + step( + "remote-main", + OPERATOR, + "git", + "ls-remote", + targets.remote, + "refs/heads/main", + ), + step( + "reviewed-pr", + OPERATOR, + GITEA_CLIENT, + "GET", + f"/api/v1/repos/{targets.repo}/pulls/{targets.reviewed_pr_number}", + ), + step( + "build-source", + OPERATOR, + "git", + "rev-parse", + f"{targets.remote}/main", + ), + step( + "deployment", + OPERATOR, + "kubectl", + "--namespace", + namespace, + "get", + deployment, + "-o", + deployment_projection, + ), + step( + "pods", + OPERATOR, + "kubectl", + "--namespace", + namespace, + "get", + "pods", + "--selector", + f"app={targets.agent_deployment}", + "-o", + pod_projection, + ), + step( + "replicasets", + OPERATOR, + "kubectl", + "--namespace", + namespace, + "get", + "replicasets", + "--selector", + f"app={targets.agent_deployment}", + "-o", + replicaset_projection, + ), + step( + "flux", + OPERATOR, + "kubectl", + "--namespace", + "flux-system", + "get", + "kustomization/hermes", + "-o", + flux_projection, + ), + ], + { + "main_sha": targets.remote_main_sha, + "head_sha": targets.reviewed_head_sha, + "head_ref": targets.reviewed_head_ref, + "pr_number": targets.reviewed_pr_number, + "image": targets.agent_image, + "build_sha": targets.build_sha, + "deployment_revision": targets.deployment_revision, + }, + rationale="A PASS requires one exact release identity across every source and running object.", + ) + + def _reliability(targets: Targets) -> list[CheckSpec]: return [ check( @@ -370,4 +366,9 @@ def _reliability(targets: Targets) -> list[CheckSpec]: def delivery_checks(targets: Targets) -> list[CheckSpec]: """Return the forge, node, build, and reliability checks.""" - return [*_forge(targets), *_nodes(targets), *_build(targets), *_reliability(targets)] + return [ + *_forge(targets), + *_nodes(targets), + *_build(targets), + *_reliability(targets), + ] diff --git a/scripts/ops/hermes_handoff_checks_platform.py b/scripts/ops/hermes_handoff_checks_platform.py index d1986976..7126036b 100644 --- a/scripts/ops/hermes_handoff_checks_platform.py +++ b/scripts/ops/hermes_handoff_checks_platform.py @@ -26,12 +26,22 @@ from hermes_handoff_catalog import ( step, ) from hermes_handoff_model import CheckSpec -from hermes_handoff_policy import GITEA_CLIENT, shell +from hermes_handoff_policy import GITEA_CLIENT, projection, shell def _baseline(targets: Targets) -> list[CheckSpec]: api = f"/api/v1/repos/{targets.repo}" + dependency_heads = dict(targets.dependency_heads) checks = [ + check( + "baseline.local-origin-main-is-current", + "The local origin/main ref equals the fixed remote-main SHA", + "baseline", + "stdout_matches", + [step("main", OPERATOR, "git", "rev-parse", f"{targets.remote}/main")], + {"step": "main", "equals": targets.remote_main_sha}, + rationale="The ancestry check is meaningful only after the local tracking ref is bound to the remote observation.", + ), check( "baseline.origin-main-descends-merged-work", "origin/main descends from the merged Hermes isolation baseline", @@ -55,7 +65,15 @@ def _baseline(targets: Targets) -> list[CheckSpec]: "Exact open pull-request heads are recorded, not assumed", "baseline", "json_record", - [step("open", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls?state=open&limit=50")], + [ + step( + "open", + OPERATOR, + GITEA_CLIENT, + "GET", + f"{api}/pulls?state=open&limit=50", + ) + ], { "step": "open", "record": { @@ -87,7 +105,14 @@ def _baseline(targets: Targets) -> list[CheckSpec]: "baseline", "json_field", [step("pr", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls/{number}")], - {"step": "pr", "fields": {"merged": True, "base.ref": "main"}}, + { + "step": "pr", + "fields": { + "merged": True, + "base.ref": "main", + "head.sha": dependency_heads.get(number, ""), + }, + }, rationale="This harness certifies the post-merge platform; an unmerged dependency means it is measuring something else.", ) for number in targets.dependency_pull_requests @@ -206,7 +231,7 @@ def _routing(targets: Targets) -> list[CheckSpec]: "routing.switchyard-is-available", "Switchyard has a ready replica serving every model-call boundary", "routing", - "json_numeric", + "lines_match", [ step( "deployment", @@ -217,10 +242,10 @@ def _routing(targets: Targets) -> list[CheckSpec]: "get", switchyard, "-o", - "json", + projection("jsonpath={.status.readyReplicas}"), ) ], - {"step": "deployment", "fields": {"status.readyReplicas": {"min": 1}}}, + {"step": "deployment", "pattern": r"^[1-9][0-9]*$", "minimum": 1}, ), check( "routing.provider-effort-and-fallback-evidence", @@ -244,7 +269,15 @@ def _routing(targets: Targets) -> list[CheckSpec]: "Codex boundary latency is measured, not assumed", "routing", "json_numeric", - [step("health", SELF, *shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS))], + [ + step( + "health", + SELF, + *shell( + "json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS + ), + ) + ], {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, ), check( @@ -252,7 +285,15 @@ def _routing(targets: Targets) -> list[CheckSpec]: "Claude boundary latency is measured, not assumed", "routing", "json_numeric", - [step("health", SELF, *shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS))], + [ + step( + "health", + SELF, + *shell( + "json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS + ), + ) + ], {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, ), ] @@ -261,7 +302,7 @@ def _routing(targets: Targets) -> list[CheckSpec]: def _scope_profile( targets: Targets, name: str, workload: str, container: str, expected: str ) -> CheckSpec: - projection = ( + profile_projection = projection( f'jsonpath={{.spec.template.spec.containers[?(@.name=="{container}")]' '.env[?(@.name=="HERMES_AUTO_ROUTER_PROFILE")].value}' ) @@ -280,7 +321,7 @@ def _scope_profile( "get", workload, "-o", - projection, + profile_projection, ) ], {"step": "profile", "equals": expected}, @@ -291,7 +332,11 @@ def _scope_profile( def _scopes(targets: Targets) -> list[CheckSpec]: return [ _scope_profile( - targets, "agent", f"deploy/{targets.agent_deployment}", targets.agent_container, "agent" + targets, + "agent", + f"deploy/{targets.agent_deployment}", + targets.agent_container, + "agent", ), _scope_profile( targets, @@ -322,7 +367,9 @@ def _scopes(targets: Targets) -> list[CheckSpec]: "get", "ingress", "-o", - 'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}', + projection( + 'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}' + ), ) ], {"step": "hosts", "minimum": 3}, @@ -332,4 +379,9 @@ def _scopes(targets: Targets) -> list[CheckSpec]: def platform_checks(targets: Targets) -> list[CheckSpec]: """Return the baseline, identity, routing, and scope checks.""" - return [*_baseline(targets), *_identity(targets), *_routing(targets), *_scopes(targets)] + return [ + *_baseline(targets), + *_identity(targets), + *_routing(targets), + *_scopes(targets), + ] diff --git a/scripts/ops/hermes_handoff_checks_workers.py b/scripts/ops/hermes_handoff_checks_workers.py index a2e1abea..eba7df6d 100644 --- a/scripts/ops/hermes_handoff_checks_workers.py +++ b/scripts/ops/hermes_handoff_checks_workers.py @@ -17,14 +17,15 @@ from __future__ import annotations from hermes_handoff_catalog import ( CHAT, INIT_NAME_PROJECTION, - NODE_NAME_PROJECTION, OPERATOR, + POOL_PROJECTION, SELF, Targets, check, ) from hermes_handoff_catalog import step as step from hermes_handoff_model import EPHEMERAL, CheckSpec +from hermes_handoff_policy import projection KANBAN_STATUSES = [ "archived", @@ -43,10 +44,10 @@ def _pool(targets: Targets) -> list[CheckSpec]: selector = targets.pool_selector return [ check( - "pool.three-workers-are-ready", - "The distributed execution pool has its full complement of ready workers", + "pool.three-workers-have-distinct-nodes-and-volumes", + "Exactly three Ready ordinal workers occupy distinct nodes and claims", "pool", - "json_numeric", + "pool_topology", [ step( "statefulset", @@ -57,25 +58,14 @@ def _pool(targets: Targets) -> list[CheckSpec]: "get", f"statefulset/{targets.pool_statefulset}", "-o", - "json", - ) - ], - { - "step": "statefulset", - "fields": { - "status.readyReplicas": {"min": targets.pool_replicas}, - "spec.replicas": {"min": targets.pool_replicas}, - }, - }, - ), - check( - "pool.workers-occupy-distinct-nodes", - "Each pool worker runs on its own node", - "pool", - "distinct_count", - [ + projection( + "jsonpath={.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}{.status.currentReplicas}{'\\t'}{.spec.template.spec.automountServiceAccountToken}".replace( + "'", '"' + ) + ), + ), step( - "nodes", + "workers", OPERATOR, "kubectl", "--namespace", @@ -85,61 +75,16 @@ def _pool(targets: Targets) -> list[CheckSpec]: "--selector", selector, "-o", - NODE_NAME_PROJECTION, - ) + POOL_PROJECTION, + ), ], { - "step": "nodes", - "minimum": targets.pool_replicas, - "total_equals": targets.pool_replicas, + "state_step": "statefulset", + "worker_step": "workers", + "replicas": targets.pool_replicas, + "name": targets.pool_statefulset, }, - rationale="Three replicas co-scheduled onto one node satisfy a replica count and defeat the isolation it stands for.", - ), - check( - "pool.workers-own-distinct-volumes", - "Each pool worker has its own durable workspace", - "pool", - "distinct_count", - [ - step( - "claims", - OPERATOR, - "kubectl", - "--namespace", - targets.namespace, - "get", - "persistentvolumeclaims", - "--selector", - selector, - "-o", - "name", - ) - ], - {"step": "claims", "minimum": targets.pool_replicas}, - ), - check( - "pool.workers-are-tokenless", - "Pool workers mount no Kubernetes API token", - "pool", - "json_field", - [ - step( - "statefulset", - OPERATOR, - "kubectl", - "--namespace", - targets.namespace, - "get", - f"statefulset/{targets.pool_statefulset}", - "-o", - "json", - ) - ], - { - "step": "statefulset", - "fields": {"spec.template.spec.automountServiceAccountToken": False}, - }, - rationale="A fenced execution worker needs no cluster authority; holding a token is the failure, not the exception.", + rationale="Replica count, Ready state, ordinal coverage, node placement, PVC isolation, and tokenlessness are one fail-closed assertion.", ), check( "pool.coordinator-retains-sole-state-ownership", @@ -156,7 +101,9 @@ def _pool(targets: Targets) -> list[CheckSpec]: "get", f"statefulset/{targets.pool_statefulset}", "-o", - 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}', + projection( + 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}' + ), ) ], {"step": "claims", "names": targets.coordinator_claims}, @@ -167,13 +114,7 @@ def _pool(targets: Targets) -> list[CheckSpec]: def _pool_env(targets: Targets) -> list[CheckSpec]: - """Return the assignment-safety check only when a site declares what to expect. - - A ``names_present`` assertion over an empty expectation would pass on any - workload at all, which is worse than not asserting: it reads as coverage. - """ - if not targets.pool_worker_env: - return [] + """Return the mandatory assignment-safety configuration check.""" return [ check( "pool.assignment-safety-knobs-are-configured", @@ -190,12 +131,40 @@ def _pool_env(targets: Targets) -> list[CheckSpec]: "get", f"statefulset/{targets.pool_statefulset}", "-o", - 'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}', + projection( + 'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}' + ), ) ], {"step": "env", "names": targets.pool_worker_env}, rationale="Deployed configuration only; the stale-claim and duplicate-dispatch behaviour itself is covered by the pool's own regression suite.", - ) + ), + check( + "pool.assignment-fieldrefs-are-bound", + "Worker node and ordinal identities come from fixed downward-API fields", + "pool", + "names_present", + [ + step( + "source", + OPERATOR, + "git", + "show", + f"{targets.remote}/main:services/hermes/execution-worker-statefulset.yaml", + record=False, + ) + ], + { + "step": "source", + "names": ( + "name: HERMES_WORKER_ORDINAL", + "fieldPath: metadata.labels['apps.kubernetes.io/pod-index']", + "name: HERMES_WORKER_NODE", + "fieldPath: spec.nodeName", + ), + }, + rationale="This prevents caller-authored ordinal/node values from impersonating another worker assignment.", + ), ] @@ -226,7 +195,18 @@ def _surfaces(targets: Targets) -> list[CheckSpec]: "Durable sessions are listed with a preview, a source, and an id", "surfaces", "lines_match", - [step("sessions", SELF, "hermes", "sessions", "list", "--limit", "20", record=False)], + [ + step( + "sessions", + SELF, + "hermes", + "sessions", + "list", + "--limit", + "20", + record=False, + ) + ], {"step": "sessions", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, rationale="A row whose leading column is blank is the session-render regression seen from the data side.", ), @@ -248,7 +228,10 @@ def _surfaces(targets: Targets) -> list[CheckSpec]: INIT_NAME_PROJECTION, ) ], - {"step": "init", "names": ("patch-web-session-activity", "patch-api-server-sessions")}, + { + "step": "init", + "names": ("patch-web-session-activity", "patch-api-server-sessions"), + }, rationale="Blank session rows were a rendering regression; this asserts the patches that fixed it are in the running pod.", ), check( @@ -286,62 +269,64 @@ def _surfaces(targets: Targets) -> list[CheckSpec]: "get", f"deploy/{targets.chat_router_deployment}", "-o", - 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}', + projection( + 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}' + ), ) ], {"step": "claims", "contains": ("state",)}, rationale="Topic selection survives a router restart only if the tenant state file is on a claim rather than in the pod.", ), ] - if targets.expect_telegram_sessions: - checks.append( - check( - "surfaces.chat-telegram-sessions-are-continuous", - "The selected chat tenant still holds durable Telegram-sourced sessions", - "surfaces", - "lines_match", - [ - step( - "telegram", - CHAT, - "hermes", - "sessions", - "list", - "--source", - "telegram", - "--limit", - "20", - record=False, - ) - ], - {"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, - rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.", - ) + checks.append( + check( + "surfaces.chat-telegram-sessions-are-continuous", + "The selected chat tenant still holds durable Telegram-sourced sessions", + "surfaces", + "lines_match", + [ + step( + "telegram", + CHAT, + "hermes", + "sessions", + "list", + "--source", + "telegram", + "--limit", + "20", + record=False, + ) + ], + {"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.", ) - if targets.chat_config_revision: - checks.append( - check( - "surfaces.chat-runs-the-telegram-topic-revision", - "The chat surface runs the durable Telegram topic configuration", - "surfaces", - "stdout_matches", - [ - step( - "revision", - OPERATOR, - "kubectl", - "--namespace", - targets.namespace, - "get", - f"statefulset/{targets.chat_statefulset}", - "-o", - "jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}", - ) - ], - {"step": "revision", "equals": targets.chat_config_revision}, - rationale="Telegram topic continuity is carried by the config revision the pod template pins.", - ) + ) + checks.append( + check( + "surfaces.chat-runs-the-telegram-topic-revision", + "The chat surface runs the durable Telegram topic configuration", + "surfaces", + "stdout_matches", + [ + step( + "revision", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.chat_statefulset}", + "-o", + projection( + "jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}" + ), + ) + ], + {"step": "revision", "equals": targets.chat_config_revision}, + rationale="Telegram topic continuity is carried by the config revision the pod template pins.", ) + ) return checks diff --git a/scripts/ops/hermes_handoff_ephemeral.py b/scripts/ops/hermes_handoff_ephemeral.py index 5456482b..fdcfdbad 100644 --- a/scripts/ops/hermes_handoff_ephemeral.py +++ b/scripts/ops/hermes_handoff_ephemeral.py @@ -1,34 +1,35 @@ #!/usr/bin/env python3 -"""The separately armed, self-cleaning mutation mode of the acceptance harness. - -Proving that a worker can still open a draft pull request means actually opening -one, which is the only part of this harness that writes anything. It is -therefore a distinct mode with a distinct policy: an explicit confirmation -phrase, a caller-supplied unique ref, a preflight that refuses a protected -target before any network call, and a cleanup whose *verification* is itself a -mandatory check. - -The push target is guarded, not the pull-request base. Opening a pull request -against `main` writes nothing to `main`; pushing to it does. The refusal is -checked by name and by pattern, and the arming self-test exercises it on every -armed run rather than trusting that it was checked once. -""" +"""Strictly preflighted, discoverable, and self-cleaning armed mode.""" from __future__ import annotations +import configparser import re from collections.abc import Sequence from dataclasses import dataclass +from pathlib import Path +from typing import Any from hermes_handoff_exec import Outcome, Runner, Vantage from hermes_handoff_model import EPHEMERAL, FAIL, NOT_RUN, PASS, CheckResult, CheckSpec -from hermes_handoff_policy import GITEA_CLIENT +from hermes_handoff_policy import ( + EXPECTED_BASE, + EXPECTED_REMOTE, + EXPECTED_REPO, + GITEA_CLIENT, + arm_ephemeral_policy, + register_ephemeral_pull, +) +from hermes_handoff_rules import strict_json CONFIRMATION = "ARM EPHEMERAL HERMES HANDOFF PUSH" REF_PREFIX = "ephemeral/hermes-handoff-acceptance" REF_RE = re.compile(rf"{REF_PREFIX}/[a-z0-9][a-z0-9-]{{7,63}}\Z") TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9-]{7,63}\Z") +SHA_RE = re.compile(r"[0-9a-f]{40}\Z") DRAFT_TITLE_PREFIX = "WIP: " +EXPECTED_HEAD_REF = "feature/hermes-full-handoff-acceptance" +EXPECTED_REMOTE_URL = "https://scm.bstein.dev/atlas/titan-iac.git" PROTECTED_NAMES = frozenset( { "default", @@ -51,48 +52,113 @@ class ArmingError(ValueError): @dataclass(frozen=True) class ArmRequest: - """Everything the armed mode needs, and nothing it can infer for itself.""" + """Every fixed and caller-confirmed input to one armed run.""" repo: str remote: str token: str confirmation: str - base: str = "main" + base: str = EXPECTED_BASE + expected_head: str = "" + expected_base_sha: str = "" @property def ref(self) -> str: - """Return the exact, unique branch this run is allowed to create.""" return f"{REF_PREFIX}/{self.token}" def normalise_branch(name: str) -> str: """Return a branch name stripped of ref decoration, for comparison.""" - trimmed = name.strip().removeprefix("refs/heads/").strip("/") - return trimmed.lower() + return name.strip().removeprefix("refs/heads/").strip("/").lower() def assert_push_target_allowed(ref: str) -> None: - """Raise unless ``ref`` is an ephemeral branch this harness may create.""" + """Raise unless ``ref`` is the one ephemeral branch grammar.""" if normalise_branch(ref) in PROTECTED_NAMES: raise ArmingError(f"refusing to push to protected branch {ref!r}") if not REF_RE.fullmatch(ref): raise ArmingError(f"push target {ref!r} is not an ephemeral acceptance ref") -def preflight(request: ArmRequest, expected_repo: str) -> None: - """Validate an arming request before any network call is made.""" +def _bounded_read(path: Path, limit: int = 1024 * 1024) -> str: + data = path.read_bytes() + if not data or len(data) > limit: + raise ArmingError(f"local Git metadata is empty or exceeds {limit} bytes") + return data.decode("utf-8", "strict") + + +def _attest_worktree(root: Path, expected_head: str, expected_base_sha: str) -> None: + """Bind cwd, common Git metadata, branch, remote, and exact local HEAD.""" + resolved = root.resolve(strict=True) + marker = resolved / ".git" + if not marker.is_file(): + raise ArmingError( + "armed mode must run at the root of the existing linked worktree" + ) + line = _bounded_read(marker, 4096).strip() + if not line.startswith("gitdir: "): + raise ArmingError("linked-worktree Git metadata is malformed") + gitdir = Path(line.removeprefix("gitdir: ")).resolve(strict=True) + common = (gitdir / _bounded_read(gitdir / "commondir", 4096).strip()).resolve( + strict=True + ) + head = _bounded_read(gitdir / "HEAD", 4096).strip() + expected_ref = f"refs/heads/{EXPECTED_HEAD_REF}" + if head != f"ref: {expected_ref}": + raise ArmingError("armed mode must run from the existing PR #19 feature branch") + ref_path = common / expected_ref + actual_head = _bounded_read(ref_path, 4096).strip() if ref_path.is_file() else "" + if actual_head != expected_head: + raise ArmingError( + "local worktree HEAD does not match the exact reviewed PR head" + ) + remote_ref = common / "refs" / "remotes" / EXPECTED_REMOTE / EXPECTED_BASE + remote_main = ( + _bounded_read(remote_ref, 4096).strip() if remote_ref.is_file() else "" + ) + if remote_main != expected_base_sha: + raise ArmingError("local origin/main does not match the exact release base SHA") + parser = configparser.RawConfigParser(interpolation=None) + try: + parser.read_string(_bounded_read(common / "config")) + remote_url = parser.get(f'remote "{EXPECTED_REMOTE}"', "url") + except (configparser.Error, KeyError) as exc: + raise ArmingError("local Git remote metadata is malformed") from exc + if remote_url != EXPECTED_REMOTE_URL: + raise ArmingError("origin is not the fixed Atlas titan-iac HTTPS repository") + + +def preflight( + request: ArmRequest, expected_repo: str, worktree: Path | None = None +) -> None: + """Validate arming and local provenance before any subprocess/network call.""" if request.confirmation != CONFIRMATION: raise ArmingError("confirmation phrase does not match") - if request.repo != expected_repo: - raise ArmingError(f"repository {request.repo!r} is not the expected {expected_repo!r}") + if expected_repo != EXPECTED_REPO or request.repo != EXPECTED_REPO: + raise ArmingError( + "armed repository is not the fixed Atlas titan-iac repository" + ) + if ( + request.remote != EXPECTED_REMOTE + or normalise_branch(request.base) != EXPECTED_BASE + ): + raise ArmingError("armed remote and pull-request base are fixed to origin/main") if not TOKEN_RE.fullmatch(request.token): - raise ArmingError("ephemeral token must be 8-64 lowercase alphanumeric or dash characters") + raise ArmingError( + "ephemeral token must be 8-64 lowercase alphanumeric or dash characters" + ) + if not SHA_RE.fullmatch(request.expected_head) or not SHA_RE.fullmatch( + request.expected_base_sha + ): + raise ArmingError("exact reviewed head and release base SHAs are required") assert_push_target_allowed(request.ref) - if not normalise_branch(request.base): - raise ArmingError("a pull-request base branch is required") + _attest_worktree( + worktree or Path.cwd(), request.expected_head, request.expected_base_sha + ) + arm_ephemeral_policy(request.ref) -def _spec(identifier: str, title: str, rationale: str = "") -> CheckSpec: +def _spec(identifier: str, title: str) -> CheckSpec: return CheckSpec( id=f"ephemeral.{identifier}", title=title, @@ -101,7 +167,6 @@ def _spec(identifier: str, title: str, rationale: str = "") -> CheckSpec: steps=(), mandatory=True, scope=EPHEMERAL, - rationale=rationale, ) @@ -111,11 +176,10 @@ def _result( status: str, reason: str, outcomes: Sequence[Outcome] = (), - evidence: dict | None = None, - rationale: str = "", + evidence: dict[str, Any] | None = None, ) -> CheckResult: return CheckResult( - spec=_spec(identifier, title, rationale), + spec=_spec(identifier, title), status=status, reason=reason, evidence=evidence or {}, @@ -124,97 +188,271 @@ def _result( def _protected_refusal_result() -> CheckResult: - """Exercise the protected-branch guard on this run, not on a past one.""" - refused: list[str] = [] accepted: list[str] = [] for candidate in ("main", "master", "refs/heads/main", "HEAD", f"{REF_PREFIX}/x"): try: assert_push_target_allowed(candidate) except ArmingError: - refused.append(candidate) - else: - accepted.append(candidate) - evidence = {"refused": refused, "accepted": accepted} - if accepted: - return _result( - "protected-branch-refusal", - "Arming against a protected branch is refused before any network call", - FAIL, - f"the guard accepted {accepted}", - evidence=evidence, - ) + continue + accepted.append(candidate) return _result( "protected-branch-refusal", "Arming against a protected branch is refused before any network call", - PASS, - "every protected and malformed push target was refused", - evidence=evidence, + FAIL if accepted else PASS, + f"guard accepted {accepted}" + if accepted + else "every protected and malformed target was refused", + evidence={"accepted": accepted}, ) def _detail(outcome: Outcome) -> str: - """Return the most informative bounded explanation an outcome carries.""" - return outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}" + return ( + outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}" + ) -def _pull_request_number(outcome: Outcome) -> int | None: - match = re.search(r'"number"\s*:\s*(\d+)', outcome.stdout) - return int(match.group(1)) if match else None +def _pr_problems(payload: Any, request: ArmRequest) -> list[str]: + if not isinstance(payload, dict): + return ["pull request is not an object"] + wanted = { + "state": "open", + "draft": True, + "merged": False, + "base": {"ref": EXPECTED_BASE, "sha": request.expected_base_sha}, + "head": {"ref": request.ref, "sha": request.expected_head}, + } + problems: list[str] = [] + if ( + not isinstance(payload.get("number"), int) + or isinstance(payload.get("number"), bool) + or payload["number"] <= 0 + ): + problems.append("number is malformed") + for field in ("state", "draft", "merged"): + if ( + type(payload.get(field)) is not type(wanted[field]) + or payload.get(field) != wanted[field] + ): + problems.append(f"{field} does not match") + for group, values in (("base", wanted["base"]), ("head", wanted["head"])): + observed = payload.get(group) + if not isinstance(observed, dict): + problems.append(f"{group} is malformed") + continue + for field, value in values.items(): + if observed.get(field) != value: + problems.append(f"{group}.{field} does not match") + return problems -def run_armed(runner: Runner, vantage: Vantage, request: ArmRequest) -> list[CheckResult]: - """Push a unique ephemeral ref, open a draft pull request, and remove both. +def _discover(outcome: Outcome, request: ArmRequest) -> tuple[list[int], str]: + if not outcome.ok: + return ([], f"pull-request discovery failed: {_detail(outcome)}") + try: + payload = strict_json(outcome.stdout) + except ValueError as exc: + return ([], f"pull-request discovery returned malformed JSON: {exc}") + if not isinstance(payload, list): + return ([], "pull-request discovery did not return an array") + matches: list[int] = [] + for index, item in enumerate(payload): + if ( + not isinstance(item, dict) + or not isinstance(item.get("head"), dict) + or not isinstance(item.get("base"), dict) + ): + return ([], f"pull-request discovery row {index + 1} is malformed") + if item["head"].get("ref") != request.ref: + continue + problems = _pr_problems(item, request) + if problems: + return ([], f"discovered pull request is invalid: {', '.join(problems)}") + matches.append(item["number"]) + if len(matches) != 1: + return ( + matches, + f"expected one exact draft pull request, discovered {len(matches)}", + ) + return (matches, "") - Every step is checked and every artifact is removed. Cleanup runs even when - an earlier step failed, and the cleanup verification is reported as its own - mandatory check so a half-removed branch cannot pass as success. - """ - api = f"/api/v1/repos/{request.repo}" - results = [_protected_refusal_result()] + +def _closed_pr_problems(payload: Any, request: ArmRequest) -> list[str]: + """Validate the same exact PR identity with state changed to closed.""" + problems = _pr_problems(payload, request) + if isinstance(payload, dict) and payload.get("state") == "closed": + problems = [ + problem for problem in problems if problem != "state does not match" + ] + elif "state does not match" not in problems: + problems.append("state is not closed") + return problems + + +def _cleanup( + runner: Runner, + vantage: Vantage, + request: ArmRequest, + api: str, + numbers: Sequence[int], + discovery_error: str = "", +) -> CheckResult: outcomes: list[Outcome] = [] + problems = [discovery_error] if discovery_error else [] + for number in sorted(set(numbers)): + closed = runner.run( + (GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"), + vantage, + ) + outcomes.append(closed) + if not closed.ok: + problems.append(f"pull request #{number} close failed") + continue + try: + payload = strict_json(closed.stdout) + except ValueError: + problems.append(f"pull request #{number} close response is malformed") + continue + expected = _closed_pr_problems(payload, request) + if expected: + problems.append(f"pull request #{number} close response does not match") + deleted = runner.run( + (GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage + ) + outcomes.append(deleted) + if not deleted.ok: + problems.append("ephemeral branch deletion failed") + remaining = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage + ) + outcomes.append(remaining) + if not remaining.ok or remaining.stdout.strip(): + problems.append("ephemeral branch absence could not be verified") + for number in sorted(set(numbers)): + observed = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage) + outcomes.append(observed) + if not observed.ok: + problems.append(f"pull request #{number} closure could not be verified") + continue + try: + payload = strict_json(observed.stdout) + except ValueError: + problems.append(f"pull request #{number} verification is malformed") + continue + expected = _closed_pr_problems(payload, request) + if expected: + problems.append(f"pull request #{number} is not exactly closed") + return _result( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + FAIL if problems else PASS, + "; ".join(problems) + if problems + else "the exact branch and draft pull request are gone", + outcomes, + {"ref": request.ref, "pull_requests": sorted(set(numbers))}, + ) - existing = runner.run(("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage) - outcomes.append(existing) - if not existing.ok: - results.append( + +def run_armed( + runner: Runner, vantage: Vantage, request: ArmRequest +) -> list[CheckResult]: + """Create, discover, validate, and clean one exact ephemeral branch/PR.""" + guard = _protected_refusal_result() + if guard.status != PASS: + return [ + guard, _result( "feature-branch-push", "A unique ephemeral feature branch can be pushed and removed", NOT_RUN, - f"could not confirm the ref is unused: {_detail(existing)}", - outcomes, - ) + "protected-ref guard failed", + ), + _result( + "draft-pull-request", + "A draft pull request can be opened and closed", + NOT_RUN, + "protected-ref guard failed", + ), + _result( + "cleanup-verified", + "Every ephemeral artifact is removed", + NOT_RUN, + "nothing was created", + ), + ] + api = f"/api/v1/repos/{request.repo}" + reserve = min(6000.0, max(300.0, runner.command_timeout * 10)) + existing = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), + vantage, + reserve_seconds=reserve, + ) + if not existing.ok or existing.stdout.strip(): + reason = ( + _detail(existing) if not existing.ok else "the ephemeral ref already exists" ) - return results - if existing.stdout.strip(): - results.append( + return [ + guard, _result( "feature-branch-push", "A unique ephemeral feature branch can be pushed and removed", FAIL, - f"{request.ref} already exists on {request.remote}; pick a fresh token", - outcomes, - ) - ) - return results - - pushed = runner.run( - ("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"), vantage + reason, + [existing], + ), + _result( + "draft-pull-request", + "A draft pull request can be opened and closed", + NOT_RUN, + "branch preflight failed", + ), + _result( + "cleanup-verified", + "Every ephemeral artifact is removed", + NOT_RUN, + "nothing was created", + ), + ] + local_head = runner.run( + ("git", "rev-parse", "HEAD"), vantage, reserve_seconds=reserve ) - outcomes.append(pushed) - results.append( - _result( - "feature-branch-push", - "A unique ephemeral feature branch can be pushed and removed", - PASS if pushed.ok else FAIL, - f"pushed {request.ref}" if pushed.ok else f"push failed: {_detail(pushed)}", - [pushed], - {"ref": request.ref}, + pushed = ( + runner.run( + ("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"), + vantage, + reserve_seconds=reserve, + ) + if local_head.ok and local_head.stdout.strip() == request.expected_head + else Outcome( + argv=("git", "push"), + vantage=vantage.name, + error="local HEAD changed after static preflight", ) ) - - number: int | None = None - if pushed.ok: + verified = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), + vantage, + reserve_seconds=reserve, + ) + branch_ok = verified.ok and verified.stdout.split() == [ + request.expected_head, + f"refs/heads/{request.ref}", + ] + branch_result = _result( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + PASS if branch_ok else FAIL, + "the exact reviewed HEAD was pushed" + if branch_ok + else "push could not be verified exactly", + [local_head, pushed, verified], + {"ref": request.ref, "head": request.expected_head}, + ) + numbers: list[int] = [] + discovery_error = "" + create_outcomes: list[Outcome] = [] + if branch_ok: created = runner.run( ( GITEA_CLIENT, @@ -225,89 +463,33 @@ def run_armed(runner: Runner, vantage: Vantage, request: ArmRequest) -> list[Che "--field", f"head={request.ref}", "--field", - f"base={normalise_branch(request.base)}", + f"base={EXPECTED_BASE}", "--field", "body=Ephemeral acceptance probe. Closed and deleted by the harness.", ), vantage, + reserve_seconds=reserve, ) - outcomes.append(created) - number = _pull_request_number(created) if created.ok else None - results.append( - _result( - "draft-pull-request", - "A draft pull request can be opened and closed against the ephemeral branch", - PASS if created.ok and number else FAIL, - f"opened draft pull request #{number}" - if created.ok and number - else f"draft creation failed: {_detail(created)}", - [created], - {"pull_request": number}, - ) + discovered = runner.run( + (GITEA_CLIENT, "GET", f"{api}/pulls?state=all&limit=50"), + vantage, + reserve_seconds=reserve, ) + create_outcomes = [created, discovered] + numbers, discovery_error = _discover(discovered, request) + for number in numbers: + register_ephemeral_pull(number) else: - results.append( - _result( - "draft-pull-request", - "A draft pull request can be opened and closed against the ephemeral branch", - NOT_RUN, - "the ephemeral branch was never pushed", - ) - ) - - results.append(_cleanup(runner, vantage, request, api, number)) - return results - - -def _cleanup( - runner: Runner, vantage: Vantage, request: ArmRequest, api: str, number: int | None -) -> CheckResult: - """Close the pull request, delete the ref, and prove both are gone.""" - outcomes: list[Outcome] = [] - problems: list[str] = [] - - if number is not None: - closed = runner.run( - (GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"), vantage - ) - outcomes.append(closed) - if not closed.ok: - problems.append(f"pull request #{number} could not be closed") - - deleted = runner.run((GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage) - outcomes.append(deleted) - - remaining = runner.run( - ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage - ) - outcomes.append(remaining) - if not remaining.ran: - problems.append(f"could not verify removal: {_detail(remaining)}") - elif remaining.stdout.strip(): - problems.append(f"{request.ref} still exists on {request.remote}") - - if number is not None: - state = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage) - outcomes.append(state) - if not state.ok: - problems.append(f"could not confirm pull request #{number} is closed") - elif '"state":"open"' in state.stdout.replace(" ", ""): - problems.append(f"pull request #{number} is still open") - - if problems: - return _result( - "cleanup-verified", - "Every ephemeral ref and pull request is removed and the removal is verified", - FAIL, - "; ".join(problems), - outcomes, - {"ref": request.ref, "pull_request": number}, - ) - return _result( - "cleanup-verified", - "Every ephemeral ref and pull request is removed and the removal is verified", - PASS, - "the ephemeral ref and pull request are gone and their absence was re-checked", - outcomes, - {"ref": request.ref, "pull_request": number}, + discovery_error = "branch creation was not verified" + draft_result = _result( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + PASS if len(numbers) == 1 and not discovery_error else FAIL, + f"discovered exact draft pull request #{numbers[0]}" + if len(numbers) == 1 and not discovery_error + else discovery_error, + create_outcomes, + {"pull_requests": numbers}, ) + cleanup = _cleanup(runner, vantage, request, api, numbers, discovery_error) + return [guard, branch_result, draft_result, cleanup] diff --git a/scripts/ops/hermes_handoff_evaluators.py b/scripts/ops/hermes_handoff_evaluators.py index 72e3558c..61e9e06b 100644 --- a/scripts/ops/hermes_handoff_evaluators.py +++ b/scripts/ops/hermes_handoff_evaluators.py @@ -22,12 +22,15 @@ from hermes_handoff_model import ATTEMPT, FAIL, NOT_RUN, PASS, REVIEW, CheckSpec from hermes_handoff_rules import ( EVALUATORS, Evaluation, + dotted, evaluate, evaluator, + exact_equal, lines, looks_denied, missing_steps, step_keys, + strict_json, ) __all__ = ["EVALUATORS", "Evaluation", "evaluate", "evaluator"] @@ -41,19 +44,25 @@ def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation return blocked attempts = step_keys(spec, ATTEMPT) if not attempts: - return Evaluation(NOT_RUN, "deny checks require a real attempt, not only a review") + return Evaluation( + NOT_RUN, "deny checks require a real attempt, not only a review" + ) evidence: dict[str, Any] = {} for key in step_keys(spec, REVIEW): verdict = outcomes[key].stdout.strip().lower() evidence[key] = verdict if verdict != "no": return Evaluation( - FAIL, f"authorization review {key} reported {verdict or 'nothing'}", evidence + FAIL, + f"authorization review {key} reported {verdict or 'nothing'}", + evidence, ) for key in attempts: outcome = outcomes[key] if outcome.ok: - return Evaluation(FAIL, f"attempt {key} succeeded but must be refused", evidence) + return Evaluation( + FAIL, f"attempt {key} succeeded but must be refused", evidence + ) if not looks_denied(outcome): return Evaluation( NOT_RUN, @@ -64,6 +73,28 @@ def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation return Evaluation(PASS, "review and live attempt were both refused", evidence) +@evaluator("review_denied") +def evaluate_review_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass only when every side-effect-free authorization review says no.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + reviews = step_keys(spec, REVIEW) + if not reviews: + return Evaluation(NOT_RUN, "a denial check requires an authorization review") + evidence: dict[str, str] = {} + for key in reviews: + verdict = outcomes[key].stdout.strip().lower() + evidence[key] = verdict + if verdict != "no": + return Evaluation( + FAIL, + f"authorization review {key} reported {verdict or 'nothing'}", + evidence, + ) + return Evaluation(PASS, "every authorization review was denied", evidence) + + @evaluator("allowed") def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: """Pass when every required step succeeded and every review said yes.""" @@ -84,7 +115,9 @@ def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluatio ) continue if not outcome.ok: - return Evaluation(FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence) + return Evaluation( + FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence + ) evidence[step.key] = "succeeded" return Evaluation(PASS, "required access is present", evidence) @@ -93,7 +126,9 @@ def _matched(observed: set[str], patterns: tuple[str, ...]) -> set[str]: return {name for pattern in patterns for name in observed if pattern in name} -def _line_step(spec: CheckSpec, outcomes: dict[str, Outcome]) -> tuple[set[str], str] | Evaluation: +def _line_step( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> tuple[set[str], str] | Evaluation: key = spec.expect["step"] outcome = outcomes[key] if not outcome.ok: @@ -117,7 +152,9 @@ def evaluate_names_absent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Eval ) evidence = {"observed_count": len(observed), "offenders": offenders} if offenders: - return Evaluation(FAIL, f"forbidden names present: {', '.join(offenders)}", evidence) + return Evaluation( + FAIL, f"forbidden names present: {', '.join(offenders)}", evidence + ) return Evaluation(PASS, "no forbidden names are present", evidence) @@ -139,14 +176,18 @@ def evaluate_names_present(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Eva ) evidence = {"observed_count": len(observed), "missing": missing} if missing: - return Evaluation(FAIL, f"required names absent: {', '.join(missing)}", evidence) + return Evaluation( + FAIL, f"required names absent: {', '.join(missing)}", evidence + ) if not observed: return Evaluation(NOT_RUN, "step produced no lines to examine", evidence) return Evaluation(PASS, "every required name is present", evidence) @evaluator("stdout_matches") -def evaluate_stdout_matches(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: +def evaluate_stdout_matches( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: """Compare a step's trimmed stdout against an expected value or set.""" blocked = missing_steps(spec, outcomes) if blocked: @@ -160,12 +201,16 @@ def evaluate_stdout_matches(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Ev allowed = expected if isinstance(expected, list) else [expected] evidence = {"observed": actual} if actual not in allowed: - return Evaluation(FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence) + return Evaluation( + FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence + ) return Evaluation(PASS, "output matches the expected value", evidence) @evaluator("distinct_count") -def evaluate_distinct_count(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: +def evaluate_distinct_count( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: """Pass when a step yields enough distinct rows — and, optionally, exactly N. Distinctness is the point for placement checks: three worker pods pinned to @@ -182,16 +227,24 @@ def evaluate_distinct_count(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Ev values = lines(outcome) distinct = sorted(set(values)) minimum = spec.expect["minimum"] - evidence = {"total": len(values), "distinct": len(distinct), "values": distinct[:16]} + evidence = { + "total": len(values), + "distinct": len(distinct), + "values": distinct[:16], + } if not values: return Evaluation(NOT_RUN, f"step {key} produced no rows to count", evidence) if len(distinct) < minimum: return Evaluation( - FAIL, f"{len(distinct)} distinct values, expected at least {minimum}", evidence + FAIL, + f"{len(distinct)} distinct values, expected at least {minimum}", + evidence, ) total_equals = spec.expect.get("total_equals") if total_equals is not None and len(values) != total_equals: - return Evaluation(FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence) + return Evaluation( + FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence + ) return Evaluation(PASS, f"{len(distinct)} distinct values observed", evidence) @@ -217,16 +270,22 @@ def evaluate_lines_match(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evalu evidence = {"rows": len(rows), "offenders": offenders[:8]} if len(rows) < spec.expect.get("minimum", 1): return Evaluation( - NOT_RUN, f"step {key} produced {len(rows)} data rows, expected at least " - f"{spec.expect.get('minimum', 1)}", evidence + NOT_RUN, + f"step {key} produced {len(rows)} data rows, expected at least " + f"{spec.expect.get('minimum', 1)}", + evidence, ) if offenders: - return Evaluation(FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence) + return Evaluation( + FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence + ) return Evaluation(PASS, f"all {len(rows)} rows match the expected shape", evidence) @evaluator("vantages_agree") -def evaluate_vantages_agree(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: +def evaluate_vantages_agree( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: """Pass only when two independent observations report the same fact. A disagreement is a finding in its own right: it usually means the manifest @@ -245,9 +304,13 @@ def evaluate_vantages_agree(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Ev observations[key] = sorted(lines(outcome)) evidence = {key: value[:16] for key, value in observations.items()} if observations[left_key] != observations[right_key]: - return Evaluation(FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence) + return Evaluation( + FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence + ) if not observations[left_key]: - return Evaluation(NOT_RUN, "both vantages returned nothing to compare", evidence) + return Evaluation( + NOT_RUN, "both vantages returned nothing to compare", evidence + ) return Evaluation(PASS, "both vantages report the same state", evidence) @@ -261,6 +324,172 @@ def evaluate_not_armed(spec: CheckSpec, _outcomes: dict[str, Outcome]) -> Evalua ) +# This validator is one compact, linear state machine so every exit observes +# only fully validated state. +# fmt: off +@evaluator("pool_topology") +def evaluate_pool_topology(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Require exactly three Ready ordinal workers on distinct nodes and claims.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + state = outcomes[spec.expect["state_step"]].stdout.strip().split("\t") + if len(state) != 4: + return Evaluation(NOT_RUN, "StatefulSet projection is malformed") + expected = spec.expect.get("replicas", 3) + try: + replicas, ready, current = (int(value) for value in state[:3]) + except ValueError: + return Evaluation(NOT_RUN, "StatefulSet replica fields are malformed") + evidence: dict[str, Any] = {"replicas": replicas, "ready": ready, "current": current, + "automountServiceAccountToken": state[3]} + if (replicas, ready, current, state[3].lower()) != (expected, expected, expected, "false"): + return Evaluation(FAIL, "worker replica or tokenless state does not match the contract", evidence) + workers: dict[int, tuple[str, str]] = {} + names: list[str] = [] + for number, row in enumerate(lines(outcomes[spec.expect["worker_step"]]), 1): + columns = row.split("\t") + if len(columns) != 5: + return Evaluation(NOT_RUN, f"worker row {number} is malformed", evidence) + name, node, phase, ready_state, claims = columns + match = re.fullmatch(rf"{re.escape(spec.expect['name'])}-(\d+)", name) + claim_list = [claim for claim in claims.split(",") if claim] + if (not match or not node or phase != "Running" or ready_state != "True" + or len(claim_list) != 1): + return Evaluation(FAIL, f"worker row {number} is not a Ready isolated worker", evidence) + ordinal = int(match.group(1)) + if ordinal in workers: + return Evaluation(FAIL, f"worker ordinal {ordinal} is duplicated", evidence) + workers[ordinal] = (node, claim_list[0]) + names.append(name) + evidence.update({"workers": sorted(names), "nodes": sorted({v[0] for v in workers.values()}), + "claims": sorted({v[1] for v in workers.values()})}) + if set(workers) != set(range(expected)): + return Evaluation(FAIL, f"worker ordinals are not exactly 0..{expected - 1}", evidence) + if (len({v[0] for v in workers.values()}) != expected + or len({v[1] for v in workers.values()}) != expected): + return Evaluation(FAIL, "workers do not occupy distinct nodes and claims", evidence) + return Evaluation(PASS, "three Ready workers have distinct nodes, ordinals, and claims", evidence) +# fmt: on + + +_SHA_RE = re.compile(r"[0-9a-f]{40}\Z") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}\Z") +_IMAGE_RE = re.compile( + r"[^@\s]+:git-([0-9a-f]{40})-build-([1-9][0-9]*)@(sha256:[0-9a-f]{64})\Z" +) + + +def _strict_object(outcome: Outcome, label: str) -> dict[str, Any] | Evaluation: + try: + payload = strict_json(outcome.stdout) + except ValueError as exc: + return Evaluation(NOT_RUN, f"{label} JSON is malformed: {exc}") + if not isinstance(payload, dict): + return Evaluation(NOT_RUN, f"{label} JSON is not an object") + return payload + + +# fmt: off +@evaluator("release_lineage") +def evaluate_release_lineage(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Atomically bind Git, PR, image, build, Flux, and running revision evidence.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + expected = spec.expect + main_sha = expected["main_sha"] + head_sha = expected["head_sha"] + image = expected["image"] + build_sha = expected["build_sha"] + revision = str(expected["deployment_revision"]) + digest = image.rsplit("@", 1)[-1] + image_match = _IMAGE_RE.fullmatch(image) + if not (_SHA_RE.fullmatch(main_sha) and _SHA_RE.fullmatch(head_sha) + and _SHA_RE.fullmatch(build_sha) and _DIGEST_RE.fullmatch(digest) + and image_match and image_match.group(1) == build_sha and build_sha == main_sha): + return Evaluation(NOT_RUN, "release lineage expectations are malformed") + remote = outcomes["remote-main"].stdout.strip().split() + if remote != [main_sha, "refs/heads/main"]: + return Evaluation(FAIL, "remote main does not match the fixed expected SHA", + {"remote_main": remote[:2]}) + pr = _strict_object(outcomes["reviewed-pr"], "reviewed PR") + if isinstance(pr, Evaluation): + return pr + fields = { + "number": expected["pr_number"], + "state": "open", + "draft": True, + "merged": False, + "base.ref": "main", + "base.sha": main_sha, + "head.ref": expected["head_ref"], + "head.sha": head_sha, + } + for path, wanted in fields.items(): + try: + actual = dotted(pr, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"reviewed PR is missing {path}") + if not exact_equal(actual, wanted): + return Evaluation(FAIL, f"reviewed PR {path} does not match", {path: actual}) + if outcomes["build-source"].stdout.strip() != build_sha: + return Evaluation(FAIL, "build source SHA does not match the fixed release build SHA") + deployment = outcomes["deployment"].stdout.strip().split("\t") + if len(deployment) != 6: + return Evaluation(NOT_RUN, "Deployment projection is malformed") + generation, observed, replicas, ready, deployed_revision, deployed_image = deployment + try: + current = (int(generation) > 0 and generation == observed + and int(replicas) > 0 and replicas == ready) + except ValueError: + return Evaluation(NOT_RUN, "Deployment numeric fields are malformed") + if not current or (deployed_revision, deployed_image) != (revision, image): + return Evaluation(FAIL, "Deployment lineage does not match the release contract") + replica_hash = "" + for number, row in enumerate(lines(outcomes["replicasets"]), 1): + columns = row.split("\t") + if len(columns) != 6: + return Evaluation(NOT_RUN, f"ReplicaSet row {number} is malformed") + _, rs_revision, rs_ready, rs_available, rs_image, rs_hash = columns + if rs_revision != revision: + continue + if (rs_ready != replicas or rs_available != replicas or rs_image != image or not rs_hash): + return Evaluation(FAIL, "active ReplicaSet does not match the Deployment lineage") + if replica_hash: + return Evaluation(FAIL, "more than one ReplicaSet claims the running revision") + replica_hash = rs_hash + if not replica_hash: + return Evaluation(FAIL, "running Deployment revision has no matching ReplicaSet") + pod_rows = lines(outcomes["pods"]) + if len(pod_rows) != int(replicas): + return Evaluation(FAIL, "running pod count does not match the Deployment") + pods: list[str] = [] + for number, row in enumerate(pod_rows, 1): + columns = row.split("\t") + if len(columns) != 6: + return Evaluation(NOT_RUN, f"pod row {number} is malformed") + name, phase, ready_state, pod_image, image_id, pod_hash = columns + if phase != "Running" or ready_state != "True": + return Evaluation(FAIL, f"pod {name or number} is not Ready") + if (pod_image, pod_hash) != (image, replica_hash) or not image_id.endswith(f"@{digest}"): + return Evaluation(FAIL, f"pod {name or number} lineage does not match") + pods.append(name) + flux = outcomes["flux"].stdout.strip().split("\t") + if len(flux) != 6: + return Evaluation(NOT_RUN, "Flux lineage projection is malformed") + name, generation, observed, suspended, ready_state, applied = flux + if (not name or generation != observed or suspended.lower() == "true" or ready_state != "True"): + return Evaluation(FAIL, "Flux source is not current and Ready") + match = re.search(r"sha1:([0-9a-f]{40})\Z", applied) + if not match or match.group(1) != main_sha: + return Evaluation(FAIL, "Flux applied revision is not the expected remote main") + return Evaluation(PASS, "Git, PR, image, build, Flux, and running revision are bound", + {"main_sha": main_sha, "head_sha": head_sha, "image_digest": digest, + "build_sha": build_sha, "deployment_revision": revision, "pods": pods}) +# fmt: on + + # Two observations of the same fact agree whether or not they came from # different vantages; the dual-vantage case is just the important one. EVALUATORS["steps_agree"] = evaluate_vantages_agree diff --git a/scripts/ops/hermes_handoff_exec.py b/scripts/ops/hermes_handoff_exec.py index 200d82e0..08888b13 100644 --- a/scripts/ops/hermes_handoff_exec.py +++ b/scripts/ops/hermes_handoff_exec.py @@ -1,57 +1,100 @@ #!/usr/bin/env python3 -"""Bounded, screened command execution for the handoff acceptance harness. +"""Bounded, screened, provenance-attested command execution. -Every probe runs through :class:`Runner`, which owns the three budgets a -long-running acceptance sweep needs: a per-command timeout, a whole-run -deadline, and a per-capture byte budget. Exhausting the deadline does not -silently shorten the sweep — the remaining probes report ``NOT_RUN``, which is a -NO_GO, so a slow cluster can never be mistaken for a clean one. - -Subprocesses never inherit the caller's environment. A probe that echoed -`$ANTHROPIC_API_KEY` would otherwise be one typo away from a leak, so the child -environment is rebuilt from an allowlist of names whose values the harness -already needs to reach the cluster and the forge. +It bounds pipes, times out and reaps the process group, and discards truncated output. """ from __future__ import annotations +import hashlib +import math import os +import selectors import shlex +import shutil +import signal +import stat import subprocess import time from collections.abc import Callable, Mapping, Sequence +from contextlib import suppress from dataclasses import dataclass, field +from pathlib import Path +from typing import Any -from hermes_handoff_policy import READ_ONLY, PolicyError, check_argv -from hermes_handoff_redaction import DEFAULT_MAX_BYTES, redact, safe_text +from hermes_handoff_policy import GITEA_CLIENT, READ_ONLY, PolicyError, check_argv +from hermes_handoff_redaction import ( + DEFAULT_MAX_BYTES, + TRUNCATION_NOTE, + redact, + safe_text, + scrub, +) DEFAULT_COMMAND_TIMEOUT = 25.0 DEFAULT_RUN_DEADLINE = 900.0 +DEFAULT_CLEANUP_RESERVE = 90.0 +MAX_COMMAND_TIMEOUT = 600.0 +MAX_RUN_DEADLINE = 7200.0 +MAX_OUTPUT_BYTES = 8 * 1024 * 1024 +MAX_CONCURRENCY = 16 +READ_CHUNK = 64 * 1024 +PIPE_DRAIN_SECONDS = 0.25 -# Names whose values the harness needs to reach the cluster and the forge. The -# values are handed to the child process and are never recorded. -ENV_PASSTHROUGH = ( - "GIT_ASKPASS", - "GITEA_BASE_URL", - "HERMES_HOME", - "HOME", - "JENKINS_BASE_URL", - "KUBECONFIG", - "LANG", - "PATH", - "REQUESTS_CA_BUNDLE", - "SSL_CERT_FILE", - "VAULT_ADDR", -) +SAFE_PATH = "/opt/data/tools/bin:/opt/hermes/.venv/bin:/opt/coordinator:/usr/local/bin:/usr/bin:/bin" +# These exact allowlists are intentionally dense: reviewers compare them as a +# single provenance boundary rather than as extensible application config. +# fmt: off +ENV_PASSTHROUGH = ("KUBECONFIG", "LANG", "LC_ALL", "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE") +FIXED_ENVIRONMENT = {"GIT_ASKPASS": "/opt/coordinator/gitea_askpass.sh", + "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", "GITEA_BASE_URL": "https://scm.bstein.dev", + "LC_ALL": "C", "PATH": SAFE_PATH} +EXPECTED_PATHS = { + "kubectl": {"/opt/data/tools/bin/kubectl", "/usr/local/bin/kubectl", "/usr/bin/kubectl"}, + "flux": {"/opt/data/tools/bin/flux", "/usr/local/bin/flux", "/usr/bin/flux"}, + "helm": {"/opt/data/tools/bin/helm", "/usr/local/bin/helm", "/usr/bin/helm"}, + "git": {"/usr/bin/git", "/bin/git"}, + "hermes": {"/opt/hermes/.venv/bin/hermes"}, + "sh": {"/usr/bin/sh", "/bin/sh"}, + GITEA_CLIENT: {GITEA_CLIENT}, +} +EXPECTED_SHA256 = { + "kubectl": {"3d514dbae5dc8c09f773df0ef0f5d449dfad05b3aca5c96b13565f886df345fd"}, + "git": {"a0e562e4bd3c4c79379e91d8c07a10104b2cefe8fac966dc6bd4874a57a807f3"}, + "sh": {"367967c823a0c391e5049b15a67c6a0a629c88b9b6dcdca75ef13ac9d65334b1"}, + "hermes": {"c8419290ef7f1a95f59eadcea9f58f446b1e6535a7f4ffee9f1c0cb1172335e4"}, + GITEA_CLIENT: {"f0943db4e81ce96908e94b0f0e78aa4b5ef95d6c4f7803f9f74a43e761576d85"}, +} DEADLINE_ERROR = "deadline-exceeded" TIMEOUT_ERROR = "timeout" - OPERATOR = "operator" SELF = "self" - -# Tools that accept a `--context` flag directly after the binary name. CONTEXT_AWARE_BINARIES = ("kubectl", "flux", "helm") +POD_COMMAND_PATHS = {"kubectl": "/usr/local/bin/kubectl", "hermes": "/opt/hermes/.venv/bin/hermes", + "git": "/usr/bin/git", "sh": "/bin/sh", GITEA_CLIENT: GITEA_CLIENT} +# fmt: on + + +@dataclass(frozen=True) +class Attestation: + """The exact outer executable used for a probe.""" + + path: str + sha256: str + + +@dataclass(frozen=True) +class Capture: + """Raw bounded process result returned by an executor implementation.""" + + returncode: int + stdout: bytes = b"" + stderr: bytes = b"" + stdout_truncated: bool = False + stderr_truncated: bool = False + timed_out: bool = False @dataclass(frozen=True) @@ -66,116 +109,298 @@ class Outcome: truncated: bool = False duration_ms: int = 0 error: str | None = None + executable_path: str = "" + executable_sha256: str = "" @property def ok(self) -> bool: - """Report whether the command ran to completion and succeeded.""" - return self.error is None and self.returncode == 0 + return self.error is None and self.returncode == 0 and not self.truncated @property def ran(self) -> bool: - """Report whether the command reached the target at all.""" return self.error is None @property def combined(self) -> str: - """Return stdout and stderr joined, for message matching.""" return "\n".join(part for part in (self.stdout, self.stderr) if part) def as_dict(self) -> dict[str, object]: - """Return a JSON-serialisable view of the outcome.""" - return { - "command": redact(shlex.join(self.argv)), - "vantage": self.vantage, - "returncode": self.returncode, - "stdout": self.stdout, - "stderr": self.stderr, - "truncated": self.truncated, - "duration_ms": self.duration_ms, - "error": self.error, - } + return scrub( + { + "command": redact(shlex.join(self.argv)), + "vantage": self.vantage, + "returncode": self.returncode, + "stdout": self.stdout, + "stderr": self.stderr, + "truncated": self.truncated, + "duration_ms": self.duration_ms, + "error": self.error, + "executable_path": self.executable_path, + "executable_sha256": self.executable_sha256, + } + ) @dataclass(frozen=True) class Vantage: - """A named identity and transport a probe can be observed from. - - ``prefix`` is how the in-pod vantage is reached: a ``kubectl exec`` into the - target container, so the inner command authenticates as the pod's own - service account instead of as the operator. ``inserts`` carries per-binary - flags — the operator's `--context`, which must not follow the command into - the pod. - """ + """A named identity and transport used to observe a probe.""" name: str prefix: tuple[str, ...] = () inserts: Mapping[str, tuple[str, ...]] = field(default_factory=dict) env: Mapping[str, str] = field(default_factory=dict) + command_paths: Mapping[str, str] = field(default_factory=dict) description: str = "" def wrap(self, argv: Sequence[str]) -> tuple[str, ...]: - """Return ``argv`` addressed to this vantage.""" command = tuple(argv) if command: + replacement = self.command_paths.get(command[0]) + if replacement: + command = (replacement, *command[1:]) extra = self.inserts.get(command[0].rsplit("/", 1)[-1]) if extra: command = (command[0], *extra, *command[1:]) return (*self.prefix, *command) -def build_environment(vantage: Vantage, source: Mapping[str, str] | None = None) -> dict[str, str]: - """Return the child environment: allow-listed names plus vantage overrides.""" +def _finite_number(name: str, value: Any, minimum: float, maximum: float) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + converted = float(value) + if not math.isfinite(converted) or not minimum <= converted <= maximum: + raise ValueError( + f"{name} must be finite and between {minimum:g} and {maximum:g}" + ) + return converted + + +def _byte_budget(value: Any) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 1 <= value <= MAX_OUTPUT_BYTES + ): + raise ValueError( + f"max_bytes must be an integer between 1 and {MAX_OUTPUT_BYTES}" + ) + return value + + +def _concurrency(value: Any) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 1 <= value <= MAX_CONCURRENCY + ): + raise ValueError( + f"concurrency must be an integer between 1 and {MAX_CONCURRENCY}" + ) + return value + + +def build_environment( + vantage: Vantage, source: Mapping[str, str] | None = None +) -> dict[str, str]: + """Build a deterministic child environment without caller-controlled PATH.""" origin = os.environ if source is None else source environment = {name: origin[name] for name in ENV_PASSTHROUGH if name in origin} - environment.setdefault("LC_ALL", "C") + environment.update(FIXED_ENVIRONMENT) environment.update(vantage.env) + environment["PATH"] = SAFE_PATH + environment["GITEA_BASE_URL"] = FIXED_ENVIRONMENT["GITEA_BASE_URL"] + environment["GIT_ASKPASS"] = FIXED_ENVIRONMENT["GIT_ASKPASS"] return environment +# fmt: off def operator_vantage(kubeconfig: str | None = None, context: str | None = None) -> Vantage: - """Return the external read-only operator vantage.""" - inserts = ( - {binary: ("--context", context) for binary in CONTEXT_AWARE_BINARIES} if context else {} - ) - detail = " ".join( - part for part in (f"context={context}" if context else "", "kubeconfig-pinned" if kubeconfig else "") if part - ) - return Vantage( - name=OPERATOR, - inserts=inserts, - env={"KUBECONFIG": kubeconfig} if kubeconfig else {}, - description=f"operator {detail}".strip(), - ) + inserts = dict.fromkeys(CONTEXT_AWARE_BINARIES, ("--context", context)) if context else {} + detail = " ".join(part for part in (f"context={context}" if context else "", + "kubeconfig-pinned" if kubeconfig else "") if part) + return Vantage(name=OPERATOR, inserts=inserts, + env={"KUBECONFIG": kubeconfig} if kubeconfig else {}, + description=f"operator {detail}".strip()) -def pod_vantage( - namespace: str, - pod: str, - container: str, - operator: Vantage | None = None, -) -> Vantage: - """Return the in-pod vantage: the inner command authenticates as the pod.""" +def pod_vantage(namespace: str, pod: str, container: str, operator: Vantage | None = None) -> Vantage: reach = operator.inserts.get("kubectl", ()) if operator else () - return Vantage( - name=SELF, - prefix=( - "kubectl", - *reach, - "exec", - "--namespace", - namespace, - pod, - "--container", - container, - "--", - ), - env=dict(operator.env) if operator else {}, - description=f"in-pod {namespace}/{pod}[{container}]", + return Vantage(name=SELF, prefix=("kubectl", *reach, "exec", "--namespace", namespace, + pod, "--container", container, "--"), + env=dict(operator.env) if operator else {}, command_paths=POD_COMMAND_PATHS, + description=f"in-pod {namespace}/{pod}[{container}]") +# fmt: on + + +def attest_executable(command: str, environment: Mapping[str, str]) -> Attestation: + """Resolve and hash an executable from a fixed, trusted path set.""" + name = command if command == GITEA_CLIENT else command.rsplit("/", 1)[-1] + candidate = ( + command if "/" in command else shutil.which(command, path=environment["PATH"]) ) + if not candidate: + raise OSError(f"trusted executable is unavailable: {name}") + resolved = str(Path(candidate).resolve(strict=True)) + allowed = { + str(Path(item).resolve(strict=False)) + for item in EXPECTED_PATHS.get(name, set()) + } + if resolved not in allowed: + raise OSError(f"executable resolved outside its trusted paths: {name}") + details = os.stat(resolved) + if ( + not stat.S_ISREG(details.st_mode) + or details.st_mode & 0o022 + or not details.st_mode & 0o111 + ): + raise OSError(f"trusted executable has unsafe ownership mode: {name}") + digest = hashlib.sha256() + with open(resolved, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + observed = digest.hexdigest() + expected = EXPECTED_SHA256.get(name) + if not expected or observed not in expected: + raise OSError(f"executable digest is not release-attested: {name}") + return Attestation(resolved, observed) + + +def _append_bounded(buffer: bytearray, chunk: bytes, limit: int) -> bool: + remaining = limit + 1 - len(buffer) + if remaining > 0: + buffer.extend(chunk[:remaining]) + return len(buffer) > limit or len(chunk) > remaining + + +def _signal_group(process: subprocess.Popen[bytes], signum: int) -> None: + with suppress(ProcessLookupError): + os.killpg(process.pid, signum) + + +def execute_bounded( + argv: Sequence[str], + environment: Mapping[str, str], + timeout: float, + max_bytes: int, + clock: Callable[[], float] = time.monotonic, + popen: Callable[..., subprocess.Popen[bytes]] = subprocess.Popen, +) -> Capture: + """Execute one process with bounded live pipe capture and group cleanup.""" + process = popen( + list(argv), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=dict(environment), + text=False, + close_fds=True, + start_new_session=True, + ) + streams = {"stdout": process.stdout, "stderr": process.stderr} + buffers = {"stdout": bytearray(), "stderr": bytearray()} + truncated = {"stdout": False, "stderr": False} + selector = selectors.DefaultSelector() + for name, stream in streams.items(): + if ( + stream is not None + ): # pragma: no branch - PIPE was requested for both streams + os.set_blocking(stream.fileno(), False) + selector.register(stream, selectors.EVENT_READ, name) + expires = clock() + timeout + timed_out = False + try: + while process.poll() is None: + remaining = expires - clock() + if remaining <= 0: + timed_out = True + break + for key, _ in selector.select(min(remaining, 0.1)): + try: + chunk = os.read(key.fileobj.fileno(), READ_CHUNK) + except BlockingIOError: # pragma: no cover - readiness race, retry only + continue + if not chunk: + selector.unregister(key.fileobj) + continue + name = key.data + truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes) + _signal_group(process, signal.SIGKILL if timed_out else signal.SIGTERM) + drain_until = clock() + PIPE_DRAIN_SECONDS + while selector.get_map() and clock() < drain_until: + for key, _ in selector.select(max(0.0, drain_until - clock())): + try: + chunk = os.read(key.fileobj.fileno(), READ_CHUNK) + except BlockingIOError: # pragma: no cover - readiness race, retry only + continue + if not chunk: + selector.unregister(key.fileobj) + continue + name = key.data + truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes) + _signal_group(process, signal.SIGKILL) + returncode = process.wait(timeout=PIPE_DRAIN_SECONDS) + finally: + selector.close() + for stream in streams.values(): + if ( + stream is not None + ): # pragma: no branch - PIPE was requested for both streams + stream.close() + if ( + process.poll() is None + ): # pragma: no branch - fail-safe for wait/selector exceptions + _signal_group(process, signal.SIGKILL) + process.wait() + return Capture( + returncode=returncode, + stdout=bytes(buffers["stdout"][:max_bytes]), + stderr=bytes(buffers["stderr"][:max_bytes]), + stdout_truncated=truncated["stdout"], + stderr_truncated=truncated["stderr"], + timed_out=timed_out, + ) + + +def _injected_executor( + spawn: Callable[..., subprocess.CompletedProcess[Any]], +) -> Callable[..., Capture]: + """Adapt deterministic test doubles without weakening the production path.""" + + def execute( + argv: Sequence[str], + environment: Mapping[str, str], + timeout: float, + max_bytes: int, + ) -> Capture: + completed = spawn( + list(argv), + capture_output=True, + text=True, + timeout=timeout, + check=False, + env=dict(environment), + ) + stdout = str(completed.stdout or "").encode() + stderr = str(completed.stderr or "").encode() + return Capture( + completed.returncode, + stdout[:max_bytes], + stderr[:max_bytes], + len(stdout) > max_bytes, + len(stderr) > max_bytes, + ) + + return execute + + +def _screen_capture(data: bytes, truncated: bool, budget: int) -> str: + if truncated: + return TRUNCATION_NOTE.encode()[:budget].decode() + return safe_text(data.decode("utf-8", "replace"), budget)[0] class Runner: - """Run probe commands under policy, timeout, deadline, and byte budgets.""" + """Run probes under policy, provenance, timeout, deadline, and byte budgets.""" def __init__( self, @@ -183,70 +408,92 @@ class Runner: command_timeout: float = DEFAULT_COMMAND_TIMEOUT, deadline_seconds: float = DEFAULT_RUN_DEADLINE, max_bytes: int = DEFAULT_MAX_BYTES, + concurrency: int = 1, clock: Callable[[], float] = time.monotonic, - spawn: Callable[..., subprocess.CompletedProcess] = subprocess.run, + executor: Callable[..., Capture] | None = None, + spawn: Callable[..., subprocess.CompletedProcess[Any]] | None = None, environ: Mapping[str, str] | None = None, + attestor: Callable[[str, Mapping[str, str]], Attestation] = attest_executable, ) -> None: self.mode = mode - self.command_timeout = command_timeout - self.max_bytes = max_bytes + self.command_timeout = _finite_number( + "command_timeout", command_timeout, 0.05, MAX_COMMAND_TIMEOUT + ) + deadline = _finite_number( + "deadline_seconds", deadline_seconds, 0.05, MAX_RUN_DEADLINE + ) + self.max_bytes = _byte_budget(max_bytes) + self.concurrency = _concurrency(concurrency) self._clock = clock - self._spawn = spawn + self._executor = executor or ( + _injected_executor(spawn) if spawn else execute_bounded + ) self._environ = environ - self._deadline_at = clock() + deadline_seconds + self._attestor = attestor + self._deadline_at = clock() + deadline self.commands_run = 0 @property def remaining_seconds(self) -> float: - """Return the seconds left before the whole-run deadline.""" return self._deadline_at - self._clock() def run( - self, argv: Sequence[str], vantage: Vantage, max_bytes: int | None = None + self, + argv: Sequence[str], + vantage: Vantage, + max_bytes: int | None = None, + reserve_seconds: float = 0.0, ) -> Outcome: - """Run one command, returning a screened outcome and never raising.""" - budget_bytes = self.max_bytes if max_bytes is None else max_bytes + budget_bytes = self.max_bytes if max_bytes is None else _byte_budget(max_bytes) + reserve = _finite_number( + "reserve_seconds", reserve_seconds, 0.0, MAX_RUN_DEADLINE + ) addressed = vantage.wrap(argv) try: check_argv(addressed, self.mode) except PolicyError as exc: return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}") - - budget = min(self.command_timeout, self.remaining_seconds) + budget = min(self.command_timeout, self.remaining_seconds - reserve) if budget <= 0: return Outcome(argv=addressed, vantage=vantage.name, error=DEADLINE_ERROR) - + environment = build_environment(vantage, self._environ) + try: + attestation = self._attestor(addressed[0], environment) + except (OSError, ValueError) as exc: + return Outcome( + argv=addressed, vantage=vantage.name, error=f"attestation: {exc}" + ) + executed = (attestation.path, *addressed[1:]) + try: + check_argv(executed, self.mode) + except PolicyError as exc: + return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}") started = self._clock() self.commands_run += 1 try: - completed = self._spawn( - list(addressed), - capture_output=True, - text=True, - timeout=budget, - check=False, - env=build_environment(vantage, self._environ), - ) + capture = self._executor(executed, environment, budget, budget_bytes) except subprocess.TimeoutExpired: + capture = Capture(-signal.SIGKILL, timed_out=True) + except OSError as exc: return Outcome( argv=addressed, vantage=vantage.name, - error=TIMEOUT_ERROR, - duration_ms=int((self._clock() - started) * 1000), + error=f"spawn: {exc.strerror or type(exc).__name__}", + executable_path=attestation.path, + executable_sha256=attestation.sha256, ) - except OSError as exc: - return Outcome( - argv=addressed, vantage=vantage.name, error=f"spawn: {exc.strerror or exc}" - ) - - stdout, cut_out = safe_text(completed.stdout or "", budget_bytes) - stderr, cut_err = safe_text(completed.stderr or "", budget_bytes) + was_truncated = capture.stdout_truncated or capture.stderr_truncated + stdout = _screen_capture(capture.stdout, capture.stdout_truncated, budget_bytes) + stderr = _screen_capture(capture.stderr, capture.stderr_truncated, budget_bytes) return Outcome( argv=addressed, vantage=vantage.name, - returncode=completed.returncode, + returncode=capture.returncode, stdout=stdout, stderr=stderr, - truncated=cut_out or cut_err, + truncated=was_truncated, duration_ms=int((self._clock() - started) * 1000), + error=TIMEOUT_ERROR if capture.timed_out else None, + executable_path=attestation.path, + executable_sha256=attestation.sha256, ) diff --git a/scripts/ops/hermes_handoff_json_rules.py b/scripts/ops/hermes_handoff_json_rules.py index 6c10e52d..0f44c6a2 100644 --- a/scripts/ops/hermes_handoff_json_rules.py +++ b/scripts/ops/hermes_handoff_json_rules.py @@ -11,7 +11,8 @@ other rule: an absent path, an unparsable payload, or an empty result set is from __future__ import annotations import datetime as dt -import json +import math +import re from typing import Any from hermes_handoff_exec import Outcome @@ -20,11 +21,33 @@ from hermes_handoff_rules import ( Evaluation, dotted, evaluator, + exact_equal, is_list, missing_steps, parsed_step, + strict_json, ) +_ROUTE_PART_RE = re.compile(r"[a-z0-9][a-z0-9_.-]{0,63}\Z") +_FLUX_NAME_RE = re.compile( + r"[a-z0-9]([-a-z0-9.]*[a-z0-9])?/[a-z0-9]([-a-z0-9.]*[a-z0-9])?\Z" +) + + +def _timestamp(value: Any) -> dt.datetime: + if not isinstance(value, str) or not value.strip(): + raise ValueError("timestamp is not a non-empty string") + observed = dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + if observed.tzinfo is None: + raise ValueError("timestamp has no timezone") + return observed.astimezone(dt.timezone.utc) + + +def _now(value: Any) -> dt.datetime: + if not isinstance(value, dt.datetime) or value.tzinfo is None: + raise ValueError("run clock is not timezone-aware") + return value.astimezone(dt.timezone.utc) + @evaluator("json_field") def evaluate_json_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: @@ -42,10 +65,12 @@ def evaluate_json_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evalua try: actual = dotted(payload, path) except (KeyError, ValueError, IndexError): - return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) evidence[path] = actual allowed = expected if isinstance(expected, list) else [expected] - if actual not in allowed: + if not any(exact_equal(actual, item) for item in allowed): mismatches.append(f"{path}={actual!r} expected one of {allowed!r}") if mismatches: return Evaluation(FAIL, "; ".join(mismatches), evidence) @@ -68,9 +93,17 @@ def evaluate_json_numeric(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Eval try: actual = dotted(payload, path) except (KeyError, ValueError, IndexError): - return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) - if not isinstance(actual, (int, float)) or isinstance(actual, bool): - return Evaluation(NOT_RUN, f"path {path} is not numeric in step {key}", evidence) + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) + if ( + not isinstance(actual, (int, float)) + or isinstance(actual, bool) + or not math.isfinite(float(actual)) + ): + return Evaluation( + NOT_RUN, f"path {path} is not numeric in step {key}", evidence + ) evidence[path] = actual if "min" in bounds and actual < bounds["min"]: problems.append(f"{path}={actual} below minimum {bounds['min']}") @@ -96,23 +129,30 @@ def evaluate_json_recent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evalu if isinstance(parsed, Evaluation): return parsed payload, key = parsed - now = spec.expect["now"] + try: + now = _now(spec.expect["now"]) + except (TypeError, ValueError) as exc: + return Evaluation(NOT_RUN, f"invalid run clock: {exc}") evidence: dict[str, Any] = {} stale: list[str] = [] for path, max_age in spec.expect["fields"].items(): try: raw = dotted(payload, path) except (KeyError, ValueError, IndexError): - return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) try: - observed = dt.datetime.fromisoformat(str(raw).replace("Z", "+00:00")) - except ValueError: - return Evaluation(NOT_RUN, f"path {path} is not an ISO timestamp", evidence) - if observed.tzinfo is None: - observed = observed.replace(tzinfo=dt.timezone.utc) + observed = _timestamp(raw) + except (TypeError, ValueError): + return Evaluation( + NOT_RUN, f"path {path} is not a timezone-aware ISO timestamp", evidence + ) age = (now - observed).total_seconds() evidence[path] = {"age_seconds": int(age)} - if age > max_age: + if age < 0: + stale.append(f"{path} is {abs(int(age))}s in the future") + elif age > max_age: stale.append(f"{path} is {int(age)}s old, limit {max_age}s") if stale: return Evaluation(FAIL, "; ".join(stale), evidence) @@ -161,18 +201,24 @@ def evaluate_all_of_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Eval except (KeyError, ValueError, IndexError): actual = None allowed = expected if isinstance(expected, list) else [expected] - if actual not in allowed: - problems.append(f"{identity}: {path}={actual!r} expected one of {allowed!r}") + if not any(exact_equal(actual, item) for item in allowed): + problems.append( + f"{identity}: {path}={actual!r} expected one of {allowed!r}" + ) evidence["examined"] = seen[:32] if wanted is not None: absent = [item for item in wanted if item not in seen] if absent: - return Evaluation(NOT_RUN, f"expected keys absent from step {key}: {absent}", evidence) + return Evaluation( + NOT_RUN, f"expected keys absent from step {key}: {absent}", evidence + ) elif not seen: return Evaluation(NOT_RUN, f"step {key} returned nothing to examine", evidence) if problems: return Evaluation(FAIL, "; ".join(problems[:8]), evidence) - return Evaluation(PASS, f"{len(seen)} element(s) match the release contract", evidence) + return Evaluation( + PASS, f"{len(seen)} element(s) match the release contract", evidence + ) @evaluator("json_record") @@ -195,21 +241,23 @@ def evaluate_json_record(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evalu try: evidence[label] = dotted(payload, path) except (KeyError, ValueError, IndexError): - return Evaluation(NOT_RUN, f"path {path} is absent from step {key}", evidence) + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) return Evaluation(PASS, "state recorded", evidence) - @evaluator("routing_evidence") -def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: +def evaluate_routing_evidence( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: """Summarise a Switchyard routing log tail into provider/effort coverage. Route identifiers are ``///``, so provider, model family, effort tier, and lane all come from the same field. Fallback and failure evidence is a separate ``fallback_reason`` on the record that - was retried. The parser tolerates a clipped first line: a tail of a growing - file starts mid-record, and discarding one row is better than a NOT_RUN that - reads as an outage. + was retried. Every row must be complete strict JSON; a clipped or malformed + line makes the evidence unusable rather than being silently discarded. """ blocked = missing_steps(spec, outcomes) if blocked: @@ -218,22 +266,64 @@ def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> outcome = outcomes[key] if not outcome.ok: return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") - records = [] - for line in outcome.stdout.splitlines(): + records: list[dict[str, Any]] = [] + routes: list[list[str]] = [] + stamps_by_dimension: dict[tuple[str, str], dt.datetime] = {} + now_value = spec.expect.get("now") + try: + now = _now(now_value) if now_value is not None else None + except (TypeError, ValueError) as exc: + return Evaluation(NOT_RUN, f"invalid routing run clock: {exc}") + for row_number, line in enumerate(outcome.stdout.splitlines(), 1): line = line.strip() if not line: continue try: - records.append(json.loads(line)) - except ValueError: - continue - routes = [ - str(record.get("model") or "").split("/") - for record in records - if str(record.get("model") or "").count("/") == 3 - ] + record = strict_json(line) + except ValueError as exc: + return Evaluation( + NOT_RUN, f"routing row {row_number} is malformed JSON: {exc}" + ) + if not isinstance(record, dict): + return Evaluation(NOT_RUN, f"routing row {row_number} is not an object") + try: + observed = _timestamp(record.get("ts")) + except (TypeError, ValueError) as exc: + return Evaluation( + NOT_RUN, f"routing row {row_number} has invalid timestamp: {exc}" + ) + if now is not None and observed > now: + return Evaluation(FAIL, f"routing row {row_number} has a future timestamp") + model = record.get("model") + if not isinstance(model, str) or not model: + return Evaluation(NOT_RUN, f"routing row {row_number} has no model") + route = model.split("/") + if len(route) == 4 and all(_ROUTE_PART_RE.fullmatch(part) for part in route): + routes.append(route) + for label, value in ( + ("provider", route[1]), + ("effort", route[3]), + ("lane", route[0]), + ): + key_dimension = (label, value) + stamps_by_dimension[key_dimension] = max( + observed, stamps_by_dimension.get(key_dimension, observed) + ) + elif record.get("tier") != "classifier": + return Evaluation( + NOT_RUN, f"routing row {row_number} has malformed route {model!r}" + ) + fallback = record.get("fallback_reason") + if fallback is not None and ( + not isinstance(fallback, str) or not fallback.strip() + ): + return Evaluation( + NOT_RUN, f"routing row {row_number} has malformed fallback_reason" + ) + record["_observed"] = observed + records.append(record) fallbacks = [record for record in records if record.get("fallback_reason")] - stamps = sorted(str(record.get("ts")) for record in records if record.get("ts")) + stamps = sorted(record["_observed"] for record in records) evidence: dict[str, Any] = { "records": len(records), "routed": len(routes), @@ -242,11 +332,15 @@ def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> "families": sorted({route[2] for route in routes}), "efforts": sorted({route[3] for route in routes}), "fallback_events": len(fallbacks), - "fallback_reasons": sorted({str(record["fallback_reason"]) for record in fallbacks}), - "latest_ts": stamps[-1] if stamps else "", + "fallback_reasons": sorted( + {str(record["fallback_reason"]) for record in fallbacks} + ), + "latest_ts": stamps[-1].isoformat() if stamps else "", } if not records: - return Evaluation(NOT_RUN, "routing log tail contained no parsable records", evidence) + return Evaluation( + NOT_RUN, "routing log tail contained no parsable records", evidence + ) missing = [ f"{label}:{item}" for label, wanted in ( @@ -260,18 +354,28 @@ def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> if spec.expect.get("require_fallback_evidence") and not fallbacks: missing.append("fallback:none-observed") max_age = spec.expect.get("max_age_seconds") - now = spec.expect.get("now") if max_age and now: - try: - newest = dt.datetime.fromisoformat(evidence["latest_ts"].replace("Z", "+00:00")) - except ValueError: - return Evaluation(NOT_RUN, "routing log has no parsable newest timestamp", evidence) - age = (now - newest).total_seconds() - evidence["age_seconds"] = int(age) - if age > max_age: - missing.append(f"freshness:{int(age)}s-exceeds-{max_age}s") + ages: dict[str, int] = {} + for label, wanted in ( + ("provider", spec.expect.get("providers", ())), + ("effort", spec.expect.get("efforts", ())), + ("lane", spec.expect.get("lanes", ())), + ): + for item in wanted: + stamp = stamps_by_dimension.get((label, item)) + if stamp is None: + continue + age = int((now - stamp).total_seconds()) + ages[f"{label}:{item}"] = age + if age > max_age: + missing.append( + f"freshness:{label}:{item}:{age}s-exceeds-{max_age}s" + ) + evidence["age_seconds"] = ages if missing: - return Evaluation(FAIL, f"routing evidence missing {', '.join(missing)}", evidence) + return Evaluation( + FAIL, f"routing evidence missing {', '.join(missing)}", evidence + ) return Evaluation(PASS, "routing evidence covers the required lanes", evidence) @@ -279,14 +383,10 @@ def evaluate_routing_evidence(spec: CheckSpec, outcomes: dict[str, Outcome]) -> def evaluate_flux_health(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: """Classify a compact Flux projection into suspensions and unhealthy objects. - Each row is ``namespace/namesuspendType=Status,...``. An object is - unhealthy when it reports ``Healthy=False``, or ``Ready=False`` with no - Healthy condition and no in-flight ``Reconciling=True``. That last clause - matters: Flux flips Ready to False for the duration of a reconcile, so - without it the verdict depends on when the snapshot was taken. Suspended - objects are excluded from the health tally and judged against the - expected-suspension list instead, because a deliberate park is a different - fact from a broken reconcile. + Each row carries name, generation, observed generation, suspension, and all + condition states. A non-suspended object must be current, explicitly + ``Ready=True``, not ``Healthy=False/Unknown``, and not reconciling. Suspended + objects are judged against the exact expected-suspension list. """ blocked = missing_steps(spec, outcomes) if blocked: @@ -299,23 +399,62 @@ def evaluate_flux_health(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evalu suspended: list[str] = [] unhealthy: list[str] = [] total = 0 - for line in outcome.stdout.splitlines(): + for row_number, line in enumerate(outcome.stdout.splitlines(), 1): if not line.strip(): continue + columns = line.split("\t") + if len(columns) != 5: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has {len(columns)} columns" + ) + name, generation, observed_generation, suspend, conditions = columns + name = name.strip() + if not _FLUX_NAME_RE.fullmatch(name): + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed object name" + ) + try: + current_generation = int(generation) + reconciled_generation = int(observed_generation) + except ValueError: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed generations" + ) + if current_generation <= 0 or current_generation != reconciled_generation: + return Evaluation( + FAIL, f"Flux row {row_number} has stale observedGeneration" + ) + if suspend.strip().lower() not in {"", "false", "true"}: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed suspension state" + ) total += 1 - name, _, rest = line.partition("\t") - suspend, _, conditions = rest.partition("\t") if suspend.strip().lower() == "true": - suspended.append(name.strip()) + suspended.append(name) continue - states = dict( - pair.split("=", 1) for pair in conditions.strip().strip(",").split(",") if "=" in pair - ) + states: dict[str, str] = {} + for pair in filter(None, conditions.strip().strip(",").split(",")): + if pair.count("=") != 1: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed condition" + ) + condition, state = pair.split("=", 1) + if ( + not condition + or state not in {"True", "False", "Unknown"} + or condition in states + ): + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed condition state" + ) + states[condition] = state healthy = states.get("Healthy") - reconciling = states.get("Reconciling") == "True" - ready_failed = states.get("Ready") == "False" - if healthy == "False" or (healthy is None and ready_failed and not reconciling): - unhealthy.append(name.strip()) + if ( + states.get("Ready") != "True" + or healthy in {"False", "Unknown"} + or states.get("Reconciling") == "True" + ): + unhealthy.append(name) unexpected = sorted(set(suspended) - expected) evidence = { "total": total, diff --git a/scripts/ops/hermes_handoff_model.py b/scripts/ops/hermes_handoff_model.py index d34b9ebc..d35d39f5 100644 --- a/scripts/ops/hermes_handoff_model.py +++ b/scripts/ops/hermes_handoff_model.py @@ -17,7 +17,7 @@ from dataclasses import dataclass, field from typing import Any from hermes_handoff_exec import Outcome -from hermes_handoff_redaction import contains_credential_shape, scrub +from hermes_handoff_redaction import contains_credential_shape, safe_text, scrub PASS = "PASS" FAIL = "FAIL" @@ -108,6 +108,8 @@ class CheckResult: @property def blocking(self) -> bool: """Report whether this result alone forces NO_GO.""" + if self.status not in STATUSES: + return True return self.spec.mandatory and self.status in {FAIL, NOT_RUN} def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: @@ -172,9 +174,17 @@ class Report: """Return the number of results in each status.""" tally = dict.fromkeys(STATUSES, 0) for result in self.results: - tally[result.status] = tally.get(result.status, 0) + 1 + if result.status in tally: + tally[result.status] += 1 return tally + @property + def invalid_statuses(self) -> list[str]: + """Return check ids whose status is outside the report schema.""" + return [ + result.spec.id for result in self.results if result.status not in STATUSES + ] + @property def blocking(self) -> list[CheckResult]: """Return every result that forces NO_GO, most severe first.""" @@ -187,7 +197,12 @@ class Report: @property def decision(self) -> str: """Return GO only when nothing mandatory failed, skipped, or errored.""" - if self.harness_errors or self.blocking: + if ( + not self.results + or self.invalid_statuses + or self.harness_errors + or self.blocking + ): return NO_GO return GO @@ -201,20 +216,23 @@ class Report: def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: """Return the machine-readable report.""" - return { - "harness": HARNESS, - "schema_version": SCHEMA_VERSION, - "mode": self.mode, - "started_at": self.started_at, - "finished_at": self.finished_at, - "decision": self.decision, - "counts": self.counts, - "baseline": scrub(self.baseline), - "vantages": [vantage.as_dict() for vantage in self.vantages], - "harness_errors": [scrub(error) for error in self.harness_errors], - "blocking": [result.spec.id for result in self.blocking], - "checks": [result.as_dict(include_outcomes) for result in self.results], - } + return scrub( + { + "harness": HARNESS, + "schema_version": SCHEMA_VERSION, + "mode": self.mode, + "started_at": self.started_at, + "finished_at": self.finished_at, + "decision": self.decision, + "counts": self.counts, + "baseline": scrub(self.baseline), + "vantages": [vantage.as_dict() for vantage in self.vantages], + "harness_errors": [scrub(error) for error in self.harness_errors], + "invalid_statuses": self.invalid_statuses, + "blocking": [result.spec.id for result in self.blocking], + "checks": [result.as_dict(include_outcomes) for result in self.results], + } + ) def render_summary(self, width: int = 88) -> str: """Return the concise human summary printed alongside the JSON.""" @@ -239,9 +257,11 @@ class Report: lines.append(" groups:") for name in sorted(groups): bucket = groups[name] - detail = " ".join(f"{status}={bucket[status]}" for status in STATUSES if bucket[status]) + detail = " ".join( + f"{status}={bucket[status]}" for status in STATUSES if bucket[status] + ) lines.append(f" {name:<18}{detail}") - return "\n".join(lines) + return safe_text("\n".join(lines), 32 * 1024)[0] def utc_now(clock: Any = None) -> str: diff --git a/scripts/ops/hermes_handoff_policy.py b/scripts/ops/hermes_handoff_policy.py index 473dd782..fc1a7097 100644 --- a/scripts/ops/hermes_handoff_policy.py +++ b/scripts/ops/hermes_handoff_policy.py @@ -1,24 +1,8 @@ #!/usr/bin/env python3 """Fail-closed command policy for the Hermes handoff acceptance harness. -Two properties have to hold structurally, not by convention: - -* the default run mutates nothing, and -* no probe can pull a credential value into the report. - -Both are enforced here, where an argv becomes a subprocess, so a mistake in the -check catalog surfaces as a policy error when the catalog is built rather than -as a mutation or a leak when it runs. `--as` is rejected on the operator side: -impersonation is not evidence about what Hermes can do, and a harness able to -impersonate would be tempted to substitute it for a real self-probe. It stays -available to the inner command of a `kubectl exec`, where an identity is -probing its own refusal to impersonate rather than borrowing authority. - -Shell is reachable only through frozen templates. Free-form `sh -c` cannot be -screened reliably, but a few probes genuinely need field extraction (report that -a password field is locked without ever reading its hash) or an explicit helper -path, so those scripts are named, reviewed here, and parameter-validated instead -of assembled by callers. +Before spawn it admits only credential-safe, structurally read-only grammar; +shell is limited to frozen templates with absolute tool paths. """ from __future__ import annotations @@ -30,28 +14,42 @@ READ_ONLY = "read-only" ARMED = "ephemeral-armed" MODES = (READ_ONLY, ARMED) -# Rejected everywhere. Impersonation and inline credentials have no honest use -# in an acceptance probe. -FORBIDDEN_ARGS = ( - "--as", - "--as-group", - "--as-uid", - "--token", - "--password", - "--username", - "--client-key", - "--client-certificate", -) -# Reachable only from inside a pod, where an identity is probing its own -# refusal to impersonate rather than borrowing someone else's authority. -IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid") -FORBIDDEN_BINARIES = frozenset({"vault", "sops", "age", "gpg", "openssl", "curl", "wget"}) GITEA_CLIENT = "/opt/coordinator/gitea_api.py" -ALLOWED_BINARIES = frozenset({"kubectl", "flux", "helm", "git", "hermes", "sh", GITEA_CLIENT}) +EXPECTED_REPO = "atlas/titan-iac" +EXPECTED_REMOTE = "origin" +EXPECTED_BASE = "main" + +# fmt: off +FORBIDDEN_ARGS = ("--as", "--as-group", "--as-uid", "--client-certificate", + "--client-key", "--password", "--token", "--username") +# fmt: on +IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid") +FORBIDDEN_BINARIES = frozenset( + {"age", "curl", "gpg", "openssl", "sops", "vault", "wget"} +) +ALLOWED_BINARIES = frozenset( + {"kubectl", "flux", "helm", "git", "hermes", "sh", GITEA_CLIENT} +) +TRUSTED_EXECUTABLE_ROOTS = ( + "/bin/", + "/usr/bin/", + "/usr/local/bin/", + "/opt/data/tools/bin/", + "/opt/hermes/.venv/bin/", + "/opt/coordinator/", +) +TRUSTED_INNER_PATHS = { + "/bin/sh": "sh", + "/usr/bin/git": "git", + "/usr/bin/sh": "sh", + "/usr/local/bin/kubectl": "kubectl", + "/opt/hermes/.venv/bin/hermes": "hermes", + GITEA_CLIENT: GITEA_CLIENT, +} -# Flags that consume the following token, so it is never a subcommand. VALUE_FLAGS = frozenset( { + "-C", "-c", "-l", "-n", @@ -76,50 +74,44 @@ KUBECTL_READ_SUBCOMMANDS = frozenset( { "api-resources", "api-versions", - "auth", - "cluster-info", - "config", - "describe", - "events", "explain", "get", - "logs", "top", "version", } ) -# Reachable only through a dry run: the API server authorises the request and -# discards it. A rejection is the evidence, and nothing is persisted either way. -KUBECTL_DRY_RUN_SUBCOMMANDS = frozenset( - {"annotate", "apply", "create", "delete", "label", "patch", "replace", "scale", "set"} -) KUBECTL_EXEC_SUBCOMMANDS = frozenset({"exec"}) -KUBECTL_SUBCOMMANDS = ( - KUBECTL_READ_SUBCOMMANDS | KUBECTL_DRY_RUN_SUBCOMMANDS | KUBECTL_EXEC_SUBCOMMANDS +KUBECTL_SUBCOMMANDS = KUBECTL_READ_SUBCOMMANDS | KUBECTL_EXEC_SUBCOMMANDS | {"auth"} +SECRET_RESOURCES = frozenset({"secret", "secrets"}) +UNSAFE_PROJECTION_PARTS = ( + ".args", + ".command", + ".data", + ".envfrom", + ".secretkeyref", + ".status.token", + ".stringdata", + ".valuefrom", + "clientconfig", + "password", + "privatekey", ) -DRY_RUN_FLAGS = ("--dry-run=server", "--dry-run=client") - -# Resources whose objects carry credential material. Reading them is permitted -# only in name-only output form, so a regression that grants access still cannot -# deliver a secret value into a report. -NAME_ONLY_RESOURCES = frozenset({"secret", "secrets"}) +SAFE_ENV_VALUE = 'env[?(@.name=="hermes_auto_router_profile")].value' +SAFE_JSON_FIELDS = frozenset(["authenticated", "transport", "state", "model", "effort", "latency_ms", "checked_at", "api_provider", "auth_method", "subscription_type"]) # fmt: skip FLUX_READ_SUBCOMMANDS = frozenset( - {"check", "diff", "events", "export", "get", "stats", "trace", "tree", "version"} + {"check", "events", "get", "stats", "trace", "tree", "version"} ) -HELM_READ_SUBCOMMANDS = frozenset({"get", "history", "list", "status", "version"}) +HELM_READ_SUBCOMMANDS = frozenset({"history", "list", "status", "version"}) GIT_READ_SUBCOMMANDS = frozenset( { "cat-file", - "config", "diff", - "fetch", "for-each-ref", "log", "ls-remote", "ls-tree", "merge-base", - "remote", "rev-list", "rev-parse", "show", @@ -127,60 +119,47 @@ GIT_READ_SUBCOMMANDS = frozenset( } ) GIT_ARMED_SUBCOMMANDS = frozenset({"push"}) -GIT_FORCE_FLAGS = frozenset({"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"}) +GIT_FORCE_FLAGS = frozenset( + {"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"} +) +GIT_UNSAFE_FLAGS = ( + "-c", + "--config-env", + "--exec-path", + "--upload-pack", + "--receive-pack", +) GITEA_READ_METHODS = frozenset({"GET"}) -GITEA_ARMED_METHODS = frozenset({"POST", "PATCH", "DELETE"}) - -# The Hermes CLI is the only honest source for durable session and Kanban -# activity, so a narrow read-only slice of it is on the allowlist. -HERMES_READ_COMMANDS = frozenset({("kanban", "list"), ("kanban", "show"), ("sessions", "list"), ("status",)}) - -# Paths that hold credential material. A probe may prove one exists or is -# unreadable; it may never read it outside a frozen template. -SECRET_PATH_RE = re.compile( - r"(?:^|/)(?:vault/secrets|var/run/secrets|runtime-access|\.credentials\.json" - r"|auth\.json|authorized_keys|id_[a-z0-9]+|\.netrc|shadow|gshadow)(?:$|/)" +HERMES_READ_COMMANDS = frozenset( + {("kanban", "list"), ("kanban", "show"), ("sessions", "list"), ("status",)} ) +# These locations contain credential bytes even when a caller claims it will +# only inspect metadata. They are rejected for every argv, including a frozen +# shell rendering and the inner command of kubectl exec. +SECRET_PATH_RE = re.compile( + r"(?:^|[\s'\"=])(?:/[^\s'\"]*)?(?:" + r"var/run/secrets|run/secrets|vault/secrets|runtime-access|scm-access|pool-access|" + r"\.credentials\.json|auth\.json|\.git-credentials|authorized_keys|" + r"id_[a-z0-9]+|\.netrc|shadow|gshadow|k3s\.yaml" + r")(?:$|[/\s'\"])", + re.IGNORECASE, +) _PARAMETER_RE = re.compile(r"[A-Za-z0-9/][A-Za-z0-9._/@:,-]{0,255}\Z") _PLACEHOLDER_RE = re.compile(r"\{([a-z_]+)\}") +_SAFE_API_PATH_RE = re.compile(r"/[A-Za-z0-9][A-Za-z0-9._~:/?&=%+-]{0,511}\Z") -# Frozen shell templates. Each exists because a plain argv cannot express the -# safety property its probe needs. SHELL_TEMPLATES: dict[str, str] = { - # Report an account's lock state without reading its password hash. - "account_lock_state": ( - 'awk -F: \'$1=="{account}"{{print ($2=="!"||$2=="*"||$2=="!!")?"locked":"unlocked"}}\' {path}' + "env_names": ( + "/usr/bin/env | /usr/bin/sed -n " + "'s/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/\\1/p' | /usr/bin/sort" ), - # Report the account record's non-secret identity fields. - "account_identity": 'awk -F: \'$1=="{account}"{{print $3":"$4":"$6":"$7}}\' {path}', - # Report supplementary group names the account joins, never their members. - "account_groups": "awk -F: '$4~/(^|,){account}(,|$)/{{print $1}}' {path}", - # Prove a path's presence and mode without reading its bytes. - "path_mode": 'if [ -e {path} ]; then stat -c "%a %U:%G" {path}; else echo absent; fi', - # Prove whether a path is readable by this identity, without printing it. - "path_readable": 'if [ -r {path} ]; then echo readable; else echo unreadable; fi', - # List environment variable names only. Values never leave the container. - "env_names": "env | sed -n 's/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/\\1/p' | sort", - # Report a JSON document's top-level key names without any of its values. - "json_keys": ( - 'python3 -c "import json,sys;print(chr(10).join(sorted(json.load(open(sys.argv[1])))))" {path}' - ), - # Report allow-listed non-secret status fields from a provider health file. "json_fields": ( - 'python3 -c "import json,sys;d=json.load(open(sys.argv[1]));' - 'print(json.dumps({{k:d.get(k) for k in sys.argv[2].split(chr(44))}}))" {path} {fields}' + '/opt/hermes/.venv/bin/python -c "import json,sys;d=json.load(open(sys.argv[1]));' + 'print(json.dumps({{k:d.get(k) for k in sys.argv[2].split(chr(44))}}))" ' + "{path} {fields}" ), - # Count non-empty lines, for authorized-key inventories. - "line_count": "awk 'NF{{n++}} END{{print n+0}}' {path}", - # Tail an append-only evidence log. The harness parses it; the container is - # not assumed to carry an interpreter. - "tail_lines": "tail -n {limit} {path}", - # Prove the worker can still read the repository. `kubectl exec` does not - # inherit the coordinator process environment, so the askpass helper is - # named explicitly rather than assumed; the helper reads runtime state and - # the credential never reaches this command line. - "git_ls_remote": "GIT_ASKPASS={askpass} git ls-remote {url} refs/heads/main", + "tail_lines": "/usr/bin/tail -n {limit} {path}", } @@ -199,16 +178,42 @@ def template_parameters(template_name: str) -> set[str]: def render_shell(template_name: str, **parameters: str) -> str: """Return a frozen shell template with validated parameters substituted.""" required = template_parameters(template_name) - template = SHELL_TEMPLATES[template_name] if set(parameters) != required: - raise PolicyError(f"shell template {template_name} expects parameters {sorted(required)}") + raise PolicyError( + f"shell template {template_name} expects parameters {sorted(required)}" + ) for key, value in parameters.items(): if not _PARAMETER_RE.fullmatch(value): raise PolicyError(f"unsafe shell parameter for {template_name}.{key}") - return template.format(**parameters) + if SECRET_PATH_RE.search(value): + raise PolicyError("shell templates may not name credential paths") + if ( + template_name == "json_fields" + and not set(parameters["fields"].split(",")) <= SAFE_JSON_FIELDS + ): + raise PolicyError("json_fields requested a credential-bearing or unknown field") + return SHELL_TEMPLATES[template_name].format(**parameters) _RENDERED: set[str] = set() +_APPROVED_PROJECTIONS: set[str] = set() +_ARMED_REF = "" +_ARMED_PULLS: set[int] = set() + + +# fmt: off +def arm_ephemeral_policy(ref: str) -> None: + global _ARMED_REF + if not re.fullmatch(r"ephemeral/hermes-handoff-acceptance/[a-z0-9][a-z0-9-]{7,63}", ref): + raise PolicyError("armed policy requires one exact ephemeral ref") + _ARMED_REF = ref + _ARMED_PULLS.clear() + + +def register_ephemeral_pull(number: int) -> None: + if not _ARMED_REF or isinstance(number, bool) or not isinstance(number, int) or number <= 0: + raise PolicyError("ephemeral pull registration is malformed") + _ARMED_PULLS.add(number) def shell(template_name: str, **parameters: str) -> tuple[str, ...]: @@ -218,24 +223,56 @@ def shell(template_name: str, **parameters: str) -> tuple[str, ...]: return ("sh", "-c", script) +def projection(value: str) -> str: + normalized = value.lower().replace(" ", "") + if not normalized.startswith("jsonpath=") or any(part in normalized for part in UNSAFE_PROJECTION_PARTS): + raise PolicyError("Kubernetes projection is not credential-safe") + if ".metadata.annotations" in normalized and not any(key in normalized for key in ( + "deployment\\.kubernetes\\.io/revision", "ai\\.bstein\\.dev/config-rev")): + raise PolicyError("Kubernetes annotations are not on the reviewed allowlist") + if ".metadata.labels" in normalized and "pod-template-hash" not in normalized: + raise PolicyError("Kubernetes labels are not on the reviewed allowlist") + if ".env" in normalized and ".value" in normalized and SAFE_ENV_VALUE not in normalized: + raise PolicyError("Kubernetes projection may expose environment values") + _APPROVED_PROJECTIONS.add(normalized) + return value +# fmt: on + + def positionals(argv: tuple[str, ...], start: int = 1) -> Iterator[str]: - """Yield positional arguments, skipping flags and the values they consume.""" + """Yield positional arguments, skipping flags and their values.""" index = start while index < len(argv): item = argv[index] if item == "--": return if item.startswith("-"): - if item in VALUE_FLAGS: - index += 2 - continue - index += 1 + index += 2 if item in VALUE_FLAGS else 1 continue yield item index += 1 -def _reject_forbidden_arguments(argv: tuple[str, ...], allow_impersonation: bool = False) -> None: +def _binary_name(value: str) -> str: + if value == GITEA_CLIENT or ( + value.startswith("/opt/coordinator/") and value.endswith("/gitea_api.py") + ): + return GITEA_CLIENT + if "/" not in value: + return value + mapped = TRUSTED_INNER_PATHS.get(value) + if mapped: + return mapped + if "/../" in value or value.endswith("/..") or "/./" in value: + raise PolicyError(f"executable path is not normalized: {value}") + if not value.startswith(TRUSTED_EXECUTABLE_ROOTS): + raise PolicyError(f"executable path is not trusted: {value}") + return value.rsplit("/", 1)[-1] + + +def _reject_forbidden_arguments( + argv: tuple[str, ...], allow_impersonation: bool +) -> None: permitted = IMPERSONATION_ARGS if allow_impersonation else () boundary = argv.index("--") if "--" in argv else len(argv) for argument in argv[:boundary]: @@ -244,47 +281,85 @@ def _reject_forbidden_arguments(argv: tuple[str, ...], allow_impersonation: bool continue if argument == forbidden or argument.startswith(f"{forbidden}="): raise PolicyError(f"forbidden argument: {forbidden}") + if argument == "--raw" or argument.startswith("--raw="): + raise PolicyError("raw API reads are forbidden") + for argument in argv: + if argument == "--dry-run" or argument.startswith("--dry-run="): + raise PolicyError("dry-run mutation commands are forbidden") def _reject_secret_paths(argv: tuple[str, ...]) -> None: - boundary = argv.index("--") if "--" in argv else len(argv) - for argument in argv[:boundary]: - if SECRET_PATH_RE.search(argument): - raise PolicyError("probes may not name a credential path outside a frozen template") + if any(SECRET_PATH_RE.search(argument) for argument in argv): + raise PolicyError("probes may not name a credential path") -def _assert_name_only_output(argv: tuple[str, ...]) -> None: - if "--output=name" in argv or "-oname" in argv: - return - for index, item in enumerate(argv[:-1]): - if item in {"-o", "--output"} and argv[index + 1] == "name": - return - raise PolicyError("credential-bearing resources may only be read with -o name") +def _resource_name(value: str) -> str: + return value.lower().split("/", 1)[0].split(".", 1)[0] + + +def _output_value(argv: tuple[str, ...]) -> str: + """Return kubectl's output argument in a single canonical form.""" + for index, item in enumerate(argv[1:], 1): + if item in {"-o", "--output"} and index + 1 < len(argv): + return argv[index + 1] + if item.startswith(("-o=", "--output=")): + return item.split("=", 1)[1] + return "" + + +def _check_projection(output: str) -> None: + """Reject full or credential-bearing Kubernetes output projections.""" + lowered = output.lower().replace(" ", "") + if lowered in {"json", "yaml", "wide"} or lowered.startswith("custom-columns"): + raise PolicyError( + "full Kubernetes object output is forbidden; use a safe projection" + ) + if lowered and not (lowered == "name" or lowered.startswith("jsonpath=")): + raise PolicyError( + "Kubernetes output must be name or an approved JSONPath projection" + ) + if lowered.startswith("jsonpath="): + if any(part in lowered for part in UNSAFE_PROJECTION_PARTS): + raise PolicyError("Kubernetes projection may expose runtime credentials") + if ".env" in lowered and SAFE_ENV_VALUE not in lowered: + # Environment names are safe; arbitrary `.value` reads are not. + without_names = lowered.replace("{.name}", "") + if ".value" in without_names: + raise PolicyError("Kubernetes projection may expose environment values") + if lowered not in _APPROVED_PROJECTIONS: + raise PolicyError( + "Kubernetes JSONPath projection is not on the reviewed allowlist" + ) def _check_kubectl(argv: tuple[str, ...], mode: str, allow_impersonation: bool) -> None: words = list(positionals(argv)) - subcommand = next((word for word in words if word in KUBECTL_SUBCOMMANDS), None) - if subcommand is None: + if not words or words[0] not in KUBECTL_SUBCOMMANDS: raise PolicyError("kubectl subcommand is outside the read-only boundary") - if subcommand != words[0]: - raise PolicyError("kubectl subcommand must precede its resource arguments") - resource = words[1].split(".")[0] if len(words) > 1 else "" - if subcommand in {"get", "describe"} and resource in NAME_ONLY_RESOURCES: - _assert_name_only_output(argv) - if subcommand in KUBECTL_READ_SUBCOMMANDS: + subcommand = words[0] + if subcommand == "auth": + if len(words) < 2 or words[1] not in {"can-i", "whoami"}: + raise PolicyError("kubectl auth is limited to can-i and whoami") return - if subcommand in KUBECTL_EXEC_SUBCOMMANDS: + if subcommand == "exec": if "--" not in argv: raise PolicyError("kubectl exec requires an explicit -- command boundary") inner = argv[argv.index("--") + 1 :] if not inner: raise PolicyError("kubectl exec requires an inner command") + if inner[0] not in TRUSTED_INNER_PATHS: + raise PolicyError( + "kubectl exec inner command must use a fixed trusted path" + ) check_argv(inner, mode, allow_impersonation=True) return - if any(flag in argv for flag in DRY_RUN_FLAGS): - return - raise PolicyError(f"kubectl {subcommand} requires a dry run") + resources = words[1].split(",") if len(words) > 1 else [] + if any(_resource_name(resource) in SECRET_RESOURCES for resource in resources): + raise PolicyError("Secret objects may not be read by the harness") + output = _output_value(argv) + if subcommand == "get" and not output: + raise PolicyError("kubectl get requires a reviewed output projection") + _check_projection(output) def _first_positional(binary: str, argv: tuple[str, ...]) -> str: @@ -294,21 +369,39 @@ def _first_positional(binary: str, argv: tuple[str, ...]) -> str: return word -def _check_read_only_tool(binary: str, argv: tuple[str, ...], allowed: frozenset[str]) -> None: +def _check_read_only_tool( + binary: str, argv: tuple[str, ...], allowed: frozenset[str] +) -> None: subcommand = _first_positional(binary, argv) if subcommand not in allowed: raise PolicyError(f"{binary} {subcommand} is outside the read-only boundary") def _check_git(argv: tuple[str, ...], mode: str) -> None: + unsafe = False + for item in argv[1:]: + if item == "-C" or item.startswith(("--git-dir", "--work-tree", "--namespace")): + unsafe = True + if item == "-c" or (item.startswith("-c") and len(item) > 2): + unsafe = True + if any( + item == flag or item.startswith(f"{flag}=") for flag in GIT_UNSAFE_FLAGS[1:] + ): + unsafe = True + if unsafe: + raise PolicyError("git configuration and helper overrides are forbidden") subcommand = _first_positional("git", argv) - if subcommand == "config" and not {"--get", "--get-all", "--list"} & set(argv): - raise PolicyError("git config may only read") if subcommand in GIT_READ_SUBCOMMANDS: return - if subcommand in GIT_ARMED_SUBCOMMANDS and mode == ARMED: + if subcommand == "push" and mode == ARMED: if GIT_FORCE_FLAGS & set(argv): raise PolicyError("git push may not force or fan out across refs") + words = list(positionals(argv)) + if len(words) != 3 or words[1] != EXPECTED_REMOTE: + raise PolicyError("armed git push is pinned to origin and one refspec") + destination = words[2].partition(":")[2] + if not _ARMED_REF or destination != f"refs/heads/{_ARMED_REF}": + raise PolicyError("armed git push is not the preflighted ephemeral ref") return raise PolicyError(f"git {subcommand} is not permitted in {mode} mode") @@ -318,52 +411,76 @@ def _check_hermes(argv: tuple[str, ...]) -> None: if not words: raise PolicyError("hermes requires a subcommand") if words[:2] not in HERMES_READ_COMMANDS and words[:1] not in HERMES_READ_COMMANDS: - raise PolicyError(f"hermes {' '.join(words[:2])} is outside the read-only boundary") + raise PolicyError( + f"hermes {' '.join(words[:2])} is outside the read-only boundary" + ) def _check_gitea(argv: tuple[str, ...], mode: str) -> None: if len(argv) < 3: raise PolicyError("gitea_api.py requires a method and a path") - method = argv[1].upper() + method, path = argv[1].upper(), argv[2] + if not path.startswith("/api/v1/") or not _SAFE_API_PATH_RE.fullmatch(path): + raise PolicyError("gitea_api.py requires a bounded API path") + if path != "/api/v1/user" and not path.startswith(f"/api/v1/repos/{EXPECTED_REPO}"): + raise PolicyError("gitea_api.py is pinned to the Atlas titan-iac repository") if method in GITEA_READ_METHODS: return - if method in GITEA_ARMED_METHODS and mode == ARMED: - return + repo_root = f"/api/v1/repos/{EXPECTED_REPO}" + if mode == ARMED: + fields = [ + argv[index + 1] for index, item in enumerate(argv[:-1]) if item == "--field" + ] + if method == "POST" and path == f"{repo_root}/pulls" and _ARMED_REF: + required = { + "title=WIP: Hermes handoff acceptance ephemeral probe", + f"head={_ARMED_REF}", + f"base={EXPECTED_BASE}", + "body=Ephemeral acceptance probe. Closed and deleted by the harness.", + } + if len(fields) == len(required) and set(fields) == required: + return + match = re.fullmatch(rf"{re.escape(repo_root)}/pulls/([1-9][0-9]*)", path) + if ( + method == "PATCH" + and match + and int(match.group(1)) in _ARMED_PULLS + and fields == ["state=closed"] + ): + return + if ( + method == "DELETE" + and _ARMED_REF + and path == f"{repo_root}/branches/{_ARMED_REF}" + and len(argv) == 3 + ): + return raise PolicyError(f"gitea_api.py {method} is not permitted in {mode} mode") def _check_shell(argv: tuple[str, ...]) -> None: - if len(argv) != 3 or argv[1] != "-c": + if len(argv) != 3 or argv[1] != "-c" or argv[2] not in _RENDERED: raise PolicyError("sh is only reachable as sh -c ") - if argv[2] not in _RENDERED: - raise PolicyError("sh -c script is not a frozen template rendering") def check_argv( argv: tuple[str, ...], mode: str = READ_ONLY, allow_impersonation: bool = False ) -> None: - """Raise :class:`PolicyError` unless ``argv`` is safe to run in ``mode``. - - ``allow_impersonation`` is set only for the inner command of a - ``kubectl exec``. Hermes probing its own refusal to impersonate is evidence; - the operator impersonating Hermes is not, so `--as` stays unreachable from - the outer, operator-side argv. - """ + """Raise :class:`PolicyError` unless ``argv`` is safe in ``mode``.""" if mode not in MODES: raise PolicyError(f"unknown mode: {mode}") if not argv: raise PolicyError("empty command") - binary = argv[0] if argv[0] == GITEA_CLIENT else argv[0].rsplit("/", 1)[-1] + binary = _binary_name(argv[0]) if binary in FORBIDDEN_BINARIES: raise PolicyError(f"forbidden binary: {binary}") if binary not in ALLOWED_BINARIES: raise PolicyError(f"binary is not on the harness allowlist: {binary}") _reject_forbidden_arguments(argv, allow_impersonation) + _reject_secret_paths(argv) if binary == "sh": _check_shell(argv) - return - _reject_secret_paths(argv) - if binary == "kubectl": + elif binary == "kubectl": _check_kubectl(argv, mode, allow_impersonation) elif binary == "flux": _check_read_only_tool("flux", argv, FLUX_READ_SUBCOMMANDS) diff --git a/scripts/ops/hermes_handoff_redaction.py b/scripts/ops/hermes_handoff_redaction.py index f48344a5..f7b97f4c 100644 --- a/scripts/ops/hermes_handoff_redaction.py +++ b/scripts/ops/hermes_handoff_redaction.py @@ -9,9 +9,10 @@ harness therefore also never asks for a secret in the first place. Probes read names, shapes, and status fields; they never read environment values, token files, private keys, cookies, or Vault responses. -Bounding is part of the same barrier. An unbounded capture is how a debug dump -or a stack trace carrying an authorization header ends up in an artifact, so -captures are truncated at a byte budget before they are screened. +Bounding is part of the same barrier. Screening always precedes truncation so a +credential that crosses the byte boundary cannot leave a useful prefix behind. +The process runner additionally discards all captured bytes when a live stream +exceeds its cap; the fixed truncation marker is the only recorded output. """ from __future__ import annotations @@ -62,12 +63,13 @@ SENSITIVE_KEY_EXACT = frozenset( # often than it is a credential, so it is screened by value shape and by the # token patterns rather than by name. The compound forms above still match. -# Values that cannot hold a credential: booleans, numbers, and anything too -# short to be one. Screening them corrupts structured evidence — a redacted -# `automountServiceAccountToken` is exactly the fact a reader needs — without -# protecting anything. +# Ordinary prose values shorter than this threshold are not independently +# treated as secret-shaped. Values under sensitive keys are screened at every +# length, including numeric values. MIN_SECRET_LENGTH = 8 -NON_SECRET_LITERALS = frozenset({"true", "false", "null", "none", "yes", "no", "0", "1", ""}) +NON_SECRET_LITERALS = frozenset( + {"true", "false", "null", "none", "yes", "no", "0", "1", ""} +) _KEY_NOISE_RE = re.compile(r"[^a-z0-9]+") # Ordered: structural matches first so a nested token is not partially rewritten @@ -77,20 +79,23 @@ _PATTERNS: tuple[re.Pattern[str], ...] = ( r"-----BEGIN[ A-Z0-9-]{0,64}-----.*?-----END[ A-Z0-9-]{0,64}-----", re.DOTALL, ), + re.compile(r"-----BEGIN[ A-Z0-9-]{0,64}-----.*", re.DOTALL), re.compile(r"(?i)\b(?:authorization|proxy-authorization)\s*:\s*\S.*"), re.compile(r"(?i)\b(?:set-)?cookie\s*:\s*\S.*"), re.compile(r"(?i)\b(?:bearer|basic|token)\s+[A-Za-z0-9+/_=.:~-]{8,}"), re.compile(r"(?i)\b[a-z][a-z0-9+.-]{1,20}://[^\s/@:]{1,128}:[^\s/@]{1,256}@"), - re.compile(r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}"), + re.compile( + r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}" + ), re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}"), - re.compile(r"(? str: @@ -136,7 +143,7 @@ def _is_high_entropy(candidate: str) -> bool: any(character in "+/_=-" for character in candidate), ) ) - return classes >= 3 + return classes >= 4 def could_hold_secret(value: str) -> bool: @@ -153,13 +160,16 @@ def could_hold_secret(value: str) -> bool: def _redact_assignment(match: re.Match[str]) -> str: value = match.group("value") - if not is_sensitive_key(match.group("key")) or not could_hold_secret(value): + stripped = value.strip().strip("\"'").lower() + if not is_sensitive_key(match.group("key")) or stripped in NON_SECRET_LITERALS: return match.group(0) # Preserve the original quoting so screened JSON stays parsable JSON. quoted = value.strip()[:1] in {'"', "'"} replacement = f'"{REDACTED}"' if quoted else REDACTED key_quote = match.group("quote") - return f"{key_quote}{match.group('key')}{key_quote}{match.group('gap')}{replacement}" + return ( + f"{key_quote}{match.group('key')}{key_quote}{match.group('gap')}{replacement}" + ) def _redact_entropy(match: re.Match[str]) -> str: @@ -183,32 +193,42 @@ def bound(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: encoded = text.encode("utf-8", "replace") if len(encoded) <= max_bytes: return (text, False) - kept = encoded[:max_bytes].decode("utf-8", "ignore") - return (f"{kept}\n{TRUNCATION_NOTE}", True) + marker = TRUNCATION_NOTE.encode()[:max_bytes] + content_budget = max( + 0, max_bytes - len(marker) - (1 if len(marker) < max_bytes else 0) + ) + kept = encoded[:content_budget].decode("utf-8", "ignore") + separator = "\n" if kept and len(marker) < max_bytes else "" + return (f"{kept}{separator}{marker.decode()}", True) def safe_text(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: - """Bound then redact, in that order, so a huge dump cannot outrun screening.""" - bounded, truncated = bound(text, max_bytes) - return (redact(bounded), truncated) + """Redact then bound, so truncation never exposes a credential prefix.""" + return bound(redact(text), max_bytes) def scrub(value: Any, max_bytes: int = DEFAULT_MAX_BYTES) -> Any: """Recursively bound and redact a JSON-shaped value. - Mapping keys are preserved (names are evidence); values under a sensitive - key are dropped outright rather than pattern-screened, because a short - secret would survive screening. + Mapping keys are themselves screened; values under a sensitive key are + dropped outright rather than pattern-screened, because a short secret + would survive screening. """ if isinstance(value, str): return safe_text(value, max_bytes)[0] if isinstance(value, dict): - return { - str(key): REDACTED - if is_sensitive_key(str(key)) and could_hold_secret(str(item)) - else scrub(item, max_bytes) - for key, item in value.items() - } + screened: dict[str, Any] = {} + for key, item in value.items(): + raw_key = str(key) + safe_key = safe_text(raw_key, max_bytes)[0] + sensitive_value = ( + is_sensitive_key(raw_key) + and item is not None + and not isinstance(item, bool) + and item != "" + ) + screened[safe_key] = REDACTED if sensitive_value else scrub(item, max_bytes) + return screened if isinstance(value, (list, tuple)): return [scrub(item, max_bytes) for item in value] return value diff --git a/scripts/ops/hermes_handoff_rules.py b/scripts/ops/hermes_handoff_rules.py index cf9b5b97..5cca2c5a 100644 --- a/scripts/ops/hermes_handoff_rules.py +++ b/scripts/ops/hermes_handoff_rules.py @@ -15,7 +15,7 @@ from dataclasses import dataclass, field from typing import Any from hermes_handoff_exec import Outcome -from hermes_handoff_model import NOT_RUN, CheckSpec +from hermes_handoff_model import NOT_RUN, STATUSES, CheckSpec DENIAL_MARKERS = ( "forbidden", @@ -28,9 +28,7 @@ DENIAL_MARKERS = ( "cannot delete", "cannot patch", "not permitted", - "no such route", - "404 not found", - "403", + "gitea api returned http 403", ) @@ -57,7 +55,9 @@ def evaluator(name: str) -> Callable[[Evaluator], Evaluator]: return register -def missing_steps(spec: CheckSpec, outcomes: Mapping[str, Outcome]) -> Evaluation | None: +def missing_steps( + spec: CheckSpec, outcomes: Mapping[str, Outcome] +) -> Evaluation | None: """Return NOT_RUN when a required step never produced usable evidence.""" for step in spec.steps: if step.optional: @@ -65,6 +65,8 @@ def missing_steps(spec: CheckSpec, outcomes: Mapping[str, Outcome]) -> Evaluatio outcome = outcomes.get(step.key) if outcome is None: return Evaluation(NOT_RUN, f"step {step.key} was never executed") + if outcome.truncated: + return Evaluation(NOT_RUN, f"step {step.key} output was truncated") if not outcome.ran: return Evaluation(NOT_RUN, f"step {step.key} did not run: {outcome.error}") return None @@ -88,6 +90,11 @@ def is_list(value: Any) -> bool: return isinstance(value, Sequence) and not isinstance(value, (str, bytes)) +def exact_equal(actual: Any, expected: Any) -> bool: + """Compare JSON values without Python's bool/int type confusion.""" + return type(actual) is type(expected) and actual == expected + + def dotted(payload: Any, path: str) -> Any: """Return a value addressed by a dotted path. @@ -110,7 +117,9 @@ def dotted(payload: Any, path: str) -> Any: raise KeyError(path) if chunk == "": remainder = ".".join(segments[position + 1 :]) - return [dotted(item, remainder) if remainder else item for item in current] + return [ + dotted(item, remainder) if remainder else item for item in current + ] offset = int(chunk) if offset >= len(current): raise KeyError(path) @@ -118,17 +127,34 @@ def dotted(payload: Any, path: str) -> Any: return current -def parse_json(outcome: Outcome) -> Any: - """Parse a step's stdout as JSON, raising ``ValueError`` when it is not. - - Control characters are tolerated. Task bodies and session titles routinely - carry raw newlines and tabs, and refusing to read a report because a human - pasted a tab into a Kanban card would be a false NO_GO. - """ - text = outcome.stdout.strip() +def strict_json(text: str) -> Any: + """Decode strict JSON, rejecting constants and duplicate object keys.""" + text = text.strip() if not text: raise ValueError("empty output") - return json.loads(text, strict=False) + + def reject_constant(value: str) -> None: + raise ValueError(f"non-finite JSON constant {value}") + + def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key {key!r}") + result[key] = value + return result + + return json.loads( + text, + strict=True, + parse_constant=reject_constant, + object_pairs_hook=unique_object, + ) + + +def parse_json(outcome: Outcome) -> Any: + """Parse one outcome as strict JSON.""" + return strict_json(outcome.stdout) def lines(outcome: Outcome) -> list[str]: @@ -136,7 +162,9 @@ def lines(outcome: Outcome) -> list[str]: return [line.strip() for line in outcome.stdout.splitlines() if line.strip()] -def parsed_step(spec: CheckSpec, outcomes: dict[str, Outcome]) -> tuple[Any, str] | Evaluation: +def parsed_step( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> tuple[Any, str] | Evaluation: """Return the parsed payload of ``expect['step']`` or the NOT_RUN to report.""" key = spec.expect["step"] outcome = outcomes[key] @@ -153,7 +181,19 @@ def evaluate(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: rule = EVALUATORS.get(spec.rule) if rule is None: return Evaluation(NOT_RUN, f"unknown evaluator: {spec.rule}") + truncated = sorted(key for key, outcome in outcomes.items() if outcome.truncated) + if truncated: + return Evaluation( + NOT_RUN, f"truncated evidence from step(s): {', '.join(truncated)}" + ) try: - return rule(spec, outcomes) + result = rule(spec, outcomes) except Exception as exc: # noqa: BLE001 - a harness bug must never read as PASS - return Evaluation(NOT_RUN, f"evaluator {spec.rule} raised {type(exc).__name__}: {exc}") + return Evaluation( + NOT_RUN, f"evaluator {spec.rule} raised {type(exc).__name__}: {exc}" + ) + if result.status not in STATUSES: + return Evaluation( + NOT_RUN, f"evaluator {spec.rule} returned unknown status {result.status!r}" + ) + return result diff --git a/scripts/ops/hermes_handoff_run.py b/scripts/ops/hermes_handoff_run.py index cdfaf3c2..a3b21cd7 100644 --- a/scripts/ops/hermes_handoff_run.py +++ b/scripts/ops/hermes_handoff_run.py @@ -4,8 +4,8 @@ Three things happen before any check runs, and each of them can stop the run on its own: -* the catalog is validated structurally — every rule exists, every deny check - carries a real attempt, no operator-side step can impersonate; +* the catalog is validated structurally — every rule exists, every live-denial + evaluator carries a read-only attempt, no operator-side step can impersonate; * each vantage is resolved and asked who it is; and * the operator and in-pod identities are compared. If they are the same principal there is only one vantage, the dual-vantage evidence would be a @@ -19,7 +19,7 @@ which is exactly the fail-closed answer. from __future__ import annotations -import json +import re from collections.abc import Iterable, Mapping, Sequence from hermes_handoff_catalog import ( @@ -48,7 +48,14 @@ from hermes_handoff_model import ( VantageRecord, utc_now, ) -from hermes_handoff_policy import IMPERSONATION_ARGS +from hermes_handoff_policy import ( + EXPECTED_REMOTE, + EXPECTED_REPO, + IMPERSONATION_ARGS, + PolicyError, + check_argv, + projection, +) UNRECORDED = "[output not recorded]" @@ -66,41 +73,129 @@ def build_catalog(targets: Targets) -> list[CheckSpec]: def validate_catalog(specs: Sequence[CheckSpec]) -> list[str]: """Return the structural problems that make a catalog unsafe to run.""" problems: list[str] = [] + if not specs: + return ["catalog is empty"] seen: set[str] = set() + fake_operator = operator_vantage() + fake_pod = pod_vantage("hermes", "acceptance-probe", "probe", fake_operator) for spec in specs: if spec.id in seen: problems.append(f"{spec.id}: duplicate check id") seen.add(spec.id) if spec.rule not in EVALUATORS and spec.scope != EPHEMERAL: problems.append(f"{spec.id}: unknown rule {spec.rule!r}") + step_names: set[str] = set() for step in spec.steps: + if step.key in step_names: + problems.append(f"{spec.id}: duplicate step key {step.key!r}") + step_names.add(step.key) if step.vantage not in VANTAGE_NAMES: - problems.append(f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}") + problems.append( + f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}" + ) if step.kind not in STEP_KINDS: - problems.append(f"{spec.id}: step {step.key} has unknown kind {step.kind!r}") + problems.append( + f"{spec.id}: step {step.key} has unknown kind {step.kind!r}" + ) if step.vantage == OPERATOR and set(step.argv) & set(IMPERSONATION_ARGS): - problems.append(f"{spec.id}: step {step.key} impersonates from the operator vantage") - if spec.rule == "denied" and not any(step.kind == ATTEMPT for step in spec.steps): - problems.append(f"{spec.id}: a deny check needs a real attempt, not only a review") + problems.append( + f"{spec.id}: step {step.key} impersonates from the operator vantage" + ) + try: + addressed = ( + fake_operator if step.vantage == OPERATOR else fake_pod + ).wrap(step.argv) + check_argv(addressed) + except PolicyError as exc: + problems.append(f"{spec.id}: step {step.key} violates policy: {exc}") + if spec.rule == "denied" and not any( + step.kind == ATTEMPT for step in spec.steps + ): + problems.append( + f"{spec.id}: a deny check needs a real attempt, not only a review" + ) + return problems + + +def validate_targets(targets: Targets) -> list[str]: + """Return release-input problems before a runner or network call exists.""" + problems: list[str] = [] + sha = re.compile(r"[0-9a-f]{40}\Z") + image = re.compile( + r"[^@\s]+:git-([0-9a-f]{40})-build-[1-9][0-9]*@sha256:[0-9a-f]{64}\Z" + ) + if targets.repo != EXPECTED_REPO or targets.remote != EXPECTED_REMOTE: + problems.append( + "repository and remote must be the fixed Atlas titan-iac origin" + ) + if ( + targets.reviewed_pr_number != 19 + or targets.reviewed_head_ref != "feature/hermes-full-handoff-acceptance" + ): + problems.append( + "reviewed PR identity must remain fixed to draft PR #19 and its existing branch" + ) + for label, value in ( + ("baseline commit", targets.baseline_commit), + ("remote main SHA", targets.remote_main_sha), + ("reviewed head SHA", targets.reviewed_head_sha), + ("build SHA", targets.build_sha), + ): + if not sha.fullmatch(value): + problems.append(f"{label} must be an exact lowercase 40-character SHA") + image_match = image.fullmatch(targets.agent_image) + if not image_match or image_match.group(1) != targets.build_sha: + problems.append("agent image tag/digest must bind the exact build SHA") + if targets.build_sha != targets.remote_main_sha: + problems.append("build SHA must equal the exact release main SHA") + if not re.fullmatch(r"[1-9][0-9]{0,8}", str(targets.deployment_revision)): + problems.append("deployment revision must be a positive integer string") + if targets.node_count != 3 or targets.pool_replicas != 3: + problems.append("node and pool counts are fixed at exactly three") + if not targets.expect_telegram_sessions or not targets.chat_config_revision.strip(): + problems.append( + "Telegram continuity and its chat config revision are mandatory" + ) + if set(targets.pool_worker_env) != {"HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL"}: + problems.append("pool assignment evidence must include worker node and ordinal") + heads = dict(targets.dependency_heads) + if len(heads) != len(targets.dependency_heads) or set(heads) != set( + targets.dependency_pull_requests + ): + problems.append("every dependency PR must have one exact current head") + elif any(not sha.fullmatch(value) for value in heads.values()): + problems.append("dependency heads must be exact lowercase 40-character SHAs") + if ( + not isinstance(targets.max_evidence_age_seconds, int) + or targets.max_evidence_age_seconds <= 0 + ): + problems.append("maximum evidence age must be a positive integer") return problems def resolve_identity(runner: Runner, vantage: Vantage) -> tuple[str, str]: """Return the authenticated username of a vantage and any failure detail.""" - outcome = runner.run(("kubectl", "auth", "whoami", "-o", "json"), vantage) + outcome = runner.run( + ( + "kubectl", + "auth", + "whoami", + "-o", + projection("jsonpath={.status.userInfo.username}"), + ), + vantage, + ) if not outcome.ok: return ("", outcome.error or outcome.combined[:200]) - try: - payload = json.loads(outcome.stdout) - except ValueError as exc: - return ("", f"unparsable identity: {exc}") - username = ((payload.get("status") or {}).get("userInfo") or {}).get("username") - if not isinstance(username, str) or not username: + username = outcome.stdout.strip() + if not username or any(character.isspace() for character in username): return ("", "identity response carried no username") return (username, "") -def _first_pod(runner: Runner, operator: Vantage, namespace: str, selector: str) -> tuple[str, str]: +def _first_pod( + runner: Runner, operator: Vantage, namespace: str, selector: str +) -> tuple[str, str]: outcome = runner.run( ( "kubectl", @@ -119,14 +214,21 @@ def _first_pod(runner: Runner, operator: Vantage, namespace: str, selector: str) ) if not outcome.ok: return ("", outcome.error or outcome.combined[:200]) - names = [line.strip().removeprefix("pod/") for line in outcome.stdout.splitlines() if line.strip()] + names = [ + line.strip().removeprefix("pod/") + for line in outcome.stdout.splitlines() + if line.strip() + ] if not names: return ("", f"no running pod matches {selector}") return (sorted(names)[0], "") def resolve_vantages( - runner: Runner, targets: Targets, kubeconfig: str | None = None, context: str | None = None + runner: Runner, + targets: Targets, + kubeconfig: str | None = None, + context: str | None = None, ) -> tuple[dict[str, Vantage], list[VantageRecord]]: """Resolve every vantage the catalog names, recording what each one is.""" operator = operator_vantage(kubeconfig, context) @@ -145,23 +247,37 @@ def resolve_vantages( return (vantages, records) for name, selector, container, description in ( - (SELF, f"app={targets.agent_deployment}", targets.agent_container, "the Hermes agent itself"), + ( + SELF, + f"app={targets.agent_deployment}", + targets.agent_container, + "the Hermes agent itself", + ), ( SWITCHYARD, f"app={targets.switchyard_deployment}", targets.switchyard_container, "the Switchyard routing evidence log", ), - (NODE, f"app={targets.node_daemonset}", targets.node_container, "a node hardening probe"), + ( + NODE, + f"app={targets.node_daemonset}", + targets.node_container, + "a node hardening probe", + ), ): pod, failure = _first_pod(runner, operator, targets.namespace, selector) if not pod: records.append( - VantageRecord(name=name, description=description, available=False, detail=failure) + VantageRecord( + name=name, description=description, available=False, detail=failure + ) ) continue vantage = pod_vantage(targets.namespace, pod, container, operator) - identity, identity_detail = resolve_identity(runner, vantage) if name == SELF else ("", "") + identity, identity_detail = ( + resolve_identity(runner, vantage) if name == SELF else ("", "") + ) records.append( VantageRecord( name=name, @@ -175,11 +291,21 @@ def resolve_vantages( chat_pod = f"{targets.chat_statefulset}-{targets.chat_ordinal}" present = runner.run( - ("kubectl", "--namespace", targets.namespace, "get", f"pod/{chat_pod}", "-o", "name"), + ( + "kubectl", + "--namespace", + targets.namespace, + "get", + f"pod/{chat_pod}", + "-o", + "name", + ), operator, ) if present.ok and present.stdout.strip(): - vantages[CHAT] = pod_vantage(targets.namespace, chat_pod, targets.chat_container, operator) + vantages[CHAT] = pod_vantage( + targets.namespace, chat_pod, targets.chat_container, operator + ) records.append( VantageRecord( name=CHAT, @@ -193,7 +319,9 @@ def resolve_vantages( name=CHAT, description=f"the chat surface ({targets.namespace}/{chat_pod})", available=False, - detail=present.error or present.combined[:200] or "chat tenant pod not found", + detail=present.error + or present.combined[:200] + or "chat tenant pod not found", ) ) return (vantages, records) @@ -206,10 +334,14 @@ def vantage_problems(records: Iterable[VantageRecord]) -> list[str]: hermes = by_name.get(SELF) problems: list[str] = [] if operator is None or not operator.available: - problems.append("the operator vantage is unavailable; no evidence can be collected") + problems.append( + "the operator vantage is unavailable; no evidence can be collected" + ) return problems if hermes is None or not hermes.available: - problems.append("the in-pod Hermes vantage is unavailable; self-probe evidence is missing") + problems.append( + "the in-pod Hermes vantage is unavailable; self-probe evidence is missing" + ) return problems if not hermes.identity: problems.append("the in-pod vantage did not report an identity") @@ -218,6 +350,10 @@ def vantage_problems(records: Iterable[VantageRecord]) -> list[str]: "the operator and in-pod vantages authenticate as the same principal " f"({operator.identity}); run the harness with a separate operator kubeconfig" ) + for name, label in ((SWITCHYARD, "Switchyard"), (CHAT, "chat/Telegram")): + record = by_name.get(name) + if record is None or not record.available: + problems.append(f"the required {label} vantage is unavailable") return problems @@ -231,7 +367,9 @@ def run_check( vantage = vantages.get(step.vantage) if vantage is None: outcome = Outcome( - argv=step.argv, vantage=step.vantage, error=f"vantage {step.vantage} is unavailable" + argv=step.argv, + vantage=step.vantage, + error=f"vantage {step.vantage} is unavailable", ) else: outcome = runner.run(step.argv, vantage, step.max_bytes) @@ -300,18 +438,27 @@ def build_report( harness_errors=problems, ) armed_ids = {result.spec.id for result in extra_results} - report.results = [ - *run_catalog(runner, [spec for spec in specs if spec.id not in armed_ids], vantages), - *extra_results, - ] + runnable = [spec for spec in specs if spec.id not in armed_ids] + if problems: + report.results = [ + CheckResult(spec=spec, status=NOT_RUN, reason="harness preflight failed") + for spec in runnable + ] + report.results.extend(extra_results) + else: + report.results = [*run_catalog(runner, runnable, vantages), *extra_results] report.results.sort(key=lambda result: result.spec.id) if runner.remaining_seconds <= 0: - report.harness_errors.append("the run deadline expired before every check completed") + report.harness_errors.append( + "the run deadline expired before every check completed" + ) report.finished_at = utc_now() return report -def unavailable_report(mode: str, started_at: str, reason: str, targets: Targets) -> Report: +def unavailable_report( + mode: str, started_at: str, reason: str, targets: Targets +) -> Report: """Return a NO_GO report for a run that could not start at all.""" return Report( mode=mode, diff --git a/testing/quality_contract.json b/testing/quality_contract.json index e8793d84..9c3de171 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -14,6 +14,21 @@ } ], "managed_modules": [ + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -25,6 +40,7 @@ "testing/quality_docs.py", "testing/quality_hygiene.py", "testing/quality_coverage.py", + "testing/quality_handoff_mutation.py", "testing/quality_gate.py", "ci/tests/glue/test_ariadne_schedules.py", "ci/tests/glue/test_glue_metrics.py", @@ -35,6 +51,21 @@ "testing/tests/test_quality_gate.py" ], "lint_paths": [ + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -57,6 +88,7 @@ "junit": "build/junit-unit.xml", "coverage_sources": [ "ci/scripts", + "scripts/ops", "scripts.render.dashboards_render_atlas", "services/mailu/scripts", "testing" @@ -118,7 +150,26 @@ ], "hygiene": { "max_lines": 500, + "legacy_line_exceptions": { + "scripts/tests/test_dashboards_render_atlas.py": { + "lines": 516, + "sha256": "afe4923391b157d03bcf0899208f77eeab79a2762da3575a365530b3074c5f25" + }, + "testing/tests/test_hermes_chat_quality.py": { + "lines": 2242, + "sha256": "71e9cafd14460b5132d07dd951514756d091009c23db7703db1575b05e34e75f" + }, + "testing/tests/test_hermes_cli_lanes.py": { + "lines": 1971, + "sha256": "992bf3788c33252e37a695f5307c31b1e8c0646e00a715e3e55f6901788abb62" + }, + "testing/tests/test_hermes_coordinator.py": { + "lines": 510, + "sha256": "290240cf391abc755eaa0abb12d247ad1718e7c9295a2092f349a7eaba4bfe55" + } + }, "line_limit_globs": [ + "scripts/ops/hermes_handoff_*.py", "testing/**/*.py", "ci/scripts/*.py", "ci/tests/**/*.py", @@ -167,7 +218,44 @@ }, "coverage": { "minimum_percent": 95.0, + "minimum_branch_percent": 95.0, + "branch_tracked_files": [ + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", + "ci/scripts/semgrep_report.py", + "testing/quality_coverage.py", + "testing/quality_handoff_mutation.py", + "testing/quality_hygiene.py" + ], "tracked_files": [ + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -178,6 +266,7 @@ "testing/quality_docs.py", "testing/quality_hygiene.py", "testing/quality_coverage.py", + "testing/quality_handoff_mutation.py", "testing/quality_gate.py" ] } diff --git a/testing/quality_coverage.py b/testing/quality_coverage.py index 78d6649f..04355e51 100644 --- a/testing/quality_coverage.py +++ b/testing/quality_coverage.py @@ -3,23 +3,31 @@ from __future__ import annotations import xml.etree.ElementTree as ET +from dataclasses import dataclass from pathlib import Path from typing import Any -def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: - """Load per-file line-rate percentages from a Cobertura XML report.""" +@dataclass(frozen=True) +class CoverageRates: + """Line and branch percentages reported for one source file.""" + + line: float + branch: float | None + + +def _load_percentages(xml_path: Path, root: Path) -> dict[str, CoverageRates]: + """Load per-file line and branch percentages from a Cobertura report.""" tree = ET.parse(xml_path) xml_root = tree.getroot() source_roots = [ - Path(node.text) - for node in xml_root.findall("./sources/source") - if node.text + Path(node.text) for node in xml_root.findall("./sources/source") if node.text ] - percentages: dict[str, float] = {} + percentages: dict[str, CoverageRates] = {} for class_node in xml_root.findall(".//class"): filename = class_node.attrib.get("filename") line_rate = class_node.attrib.get("line-rate") + branch_rate = class_node.attrib.get("branch-rate") if not filename or line_rate is None: continue normalized = filename.replace("\\", "/") @@ -32,7 +40,10 @@ def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]: if candidate.exists(): key = candidate.relative_to(root).as_posix() break - percentages[key] = float(line_rate) * 100.0 + percentages[key] = CoverageRates( + line=float(line_rate) * 100.0, + branch=float(branch_rate) * 100.0 if branch_rate is not None else None, + ) return percentages @@ -46,18 +57,33 @@ def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str] return [f"coverage xml missing: {xml_path.relative_to(root)}"] percentages = _load_percentages(xml_path, root) - minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0)) + coverage_contract = contract.get("coverage", {}) + minimum = float(coverage_contract.get("minimum_percent", 95.0)) + branch_minimum = float(coverage_contract.get("minimum_branch_percent", 95.0)) + branch_tracked = { + path.replace("\\", "/") + for path in coverage_contract.get("branch_tracked_files", ()) + } issues: list[str] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is None: + rates = percentages.get(normalized) + if rates is None: issues.append(f"coverage missing for tracked file: {relative_path}") continue - if percent + 1e-9 < minimum: + if rates.line + 1e-9 < minimum: issues.append( - f"coverage below {minimum:.1f}%: {relative_path} ({percent:.1f}%)" + f"coverage below {minimum:.1f}%: {relative_path} ({rates.line:.1f}%)" + ) + if normalized not in branch_tracked: + continue + if rates.branch is None: + issues.append(f"branch coverage missing for tracked file: {relative_path}") + elif rates.branch + 1e-9 < branch_minimum: + issues.append( + f"branch coverage below {branch_minimum:.1f}%: " + f"{relative_path} ({rates.branch:.1f}%)" ) return issues @@ -77,9 +103,9 @@ def compute_workspace_line_coverage( samples: list[float] = [] for relative_path in contract.get("coverage", {}).get("tracked_files", []): normalized = relative_path.replace("\\", "/") - percent = percentages.get(normalized) - if percent is not None: - samples.append(percent) + rates = percentages.get(normalized) + if rates is not None: + samples.append(rates.line) if not samples: return 0.0 return round(sum(samples) / len(samples), 3) diff --git a/testing/quality_handoff_mutation.py b/testing/quality_handoff_mutation.py new file mode 100644 index 00000000..a0c63fa0 --- /dev/null +++ b/testing/quality_handoff_mutation.py @@ -0,0 +1,209 @@ +"""Deterministic scored mutation gate for Hermes handoff safety invariants.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class Mutation: + """One source mutation and the isolated probe that must kill it.""" + + name: str + path: str + old: str + new: str + probe: str + + +@dataclass(frozen=True) +class MutationReport: + """Scored result; any invalid, surviving, or baseline-broken case is an issue.""" + + total: int + killed: int + issues: tuple[str, ...] + + @property + def score(self) -> float: + return 100.0 if not self.total else round(self.killed * 100.0 / self.total, 2) + + +MUTATIONS = ( + Mutation( + "policy-allows-git-fetch", + "hermes_handoff_policy.py", + ' "for-each-ref",\n', + ' "fetch",\n "for-each-ref",\n', + "from hermes_handoff_policy import *\n" + "try: check_argv(('git','fetch','origin'))\n" + "except PolicyError: raise SystemExit(0)\n" + "raise SystemExit(1)\n", + ), + Mutation( + "redaction-truncates-before-screening", + "hermes_handoff_redaction.py", + " return bound(redact(text), max_bytes)\n", + " return bound(text, max_bytes)\n", + "from hermes_handoff_redaction import safe_text\n" + "assert 'A9' not in safe_text('password=A9secret-prefix', 23)[0]\n", + ), + Mutation( + "empty-report-go", + "hermes_handoff_model.py", + " if (\n" + " not self.results\n" + " or self.invalid_statuses\n" + " or self.harness_errors\n" + " or self.blocking\n" + " ):\n", + " if (\n" + " self.invalid_statuses\n" + " or self.harness_errors\n" + " or self.blocking\n" + " ):\n", + "from hermes_handoff_model import *\nassert Report('read-only','now').decision == NO_GO\n", + ), + Mutation( + "generic-404-is-denial", + "hermes_handoff_rules.py", + "DENIAL_MARKERS = (\n", + 'DENIAL_MARKERS = (\n "404 not found",\n', + "from hermes_handoff_rules import looks_denied\n" + "from hermes_handoff_exec import Outcome\n" + "assert not looks_denied(Outcome(('x',),'x',1,'','404 not found'))\n", + ), + Mutation( + "truncated-evidence-reaches-evaluator", + "hermes_handoff_rules.py", + " if truncated:\n" + " return Evaluation(\n" + " NOT_RUN, f\"truncated evidence from step(s): {', '.join(truncated)}\"\n" + " )\n", + " if False and truncated:\n return Evaluation(NOT_RUN, 'mutant')\n", + "import hermes_handoff_evaluators\n" + "import hermes_handoff_rules as r\n" + "from hermes_handoff_exec import Outcome\n" + "from hermes_handoff_model import *\n" + "@r.evaluator('probe')\ndef probe(*_): return r.Evaluation(PASS,'bad')\n" + "s=CheckSpec('x','x','x','probe')\n" + "assert r.evaluate(s,{'x':Outcome(('x',),'x',0,truncated=True)}).status == NOT_RUN\n", + ), + Mutation( + "truncated-outcome-is-ok", + "hermes_handoff_exec.py", + " return self.error is None and self.returncode == 0 and not self.truncated\n", + " return self.error is None and self.returncode == 0\n", + "from hermes_handoff_exec import Outcome\nassert not Outcome(('x',),'x',0,truncated=True).ok\n", + ), + Mutation( + "nan-is-numeric", + "hermes_handoff_json_rules.py", + " or not math.isfinite(float(actual))\n", + " or False\n", + "import hermes_handoff_json_rules as j\n" + "from hermes_handoff_model import *\nfrom hermes_handoff_exec import Outcome\n" + "s=CheckSpec('x','x','x','json_numeric',expect={'step':'s','fields':{'x':{'min':0}}})\n" + "j.parsed_step=lambda *_: ({'x':float('nan')},'s')\n" + "assert j.evaluate_json_numeric(s,{'s':Outcome(('x',),'x',0)}).status == NOT_RUN\n", + ), + Mutation( + "cleanup-ignores-branch-delete-failure", + "hermes_handoff_ephemeral.py", + ' if not deleted.ok:\n problems.append("ephemeral branch deletion failed")\n', + ' if False and not deleted.ok:\n problems.append("mutant")\n', + "import hermes_handoff_ephemeral as e\nfrom hermes_handoff_exec import Outcome\n" + "class R:\n" + " def __init__(self): self.i=0\n" + " def run(self,*_a,**_k):\n" + " self.i+=1\n" + " return Outcome(('x',),'x',1 if self.i==1 else 0)\n" + "q=e.ArmRequest('atlas/titan-iac','origin','acceptance1',e.CONFIRMATION,expected_head='1'*40,expected_base_sha='2'*40)\n" + "assert e._cleanup(R(),object(),q,'/api/v1/repos/atlas/titan-iac',[]).status == 'FAIL'\n", + ), + Mutation( + "cleanup-accepts-open-pr", + "hermes_handoff_ephemeral.py", + ' elif "state does not match" not in problems:\n problems.append("state is not closed")\n', + ' elif False:\n problems.append("mutant")\n', + "import hermes_handoff_ephemeral as e\n" + "q=e.ArmRequest('atlas/titan-iac','origin','acceptance1',e.CONFIRMATION,expected_head='1'*40,expected_base_sha='2'*40)\n" + "p={'number':1,'state':'open','draft':True,'merged':False,'base':{'ref':'main','sha':'2'*40},'head':{'ref':q.ref,'sha':'1'*40}}\n" + "assert e._closed_pr_problems(p,q)\n", + ), +) + + +def _probe(root: Path, probe: str) -> subprocess.CompletedProcess[str]: + environment = { + "LC_ALL": "C", + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "PYTHONPATH": str(root), + } + return subprocess.run( + [sys.executable, "-c", probe], + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=10, + check=False, + env=environment, + ) + + +def run_gate( + repo_root: Path, mutations: Sequence[Mutation] = MUTATIONS +) -> MutationReport: + """Run baseline and mutated probes in isolated copies and return the score.""" + source_root = repo_root / "scripts" / "ops" + issues: list[str] = [] + killed = 0 + with tempfile.TemporaryDirectory(prefix="hermes-mutation-") as temporary: + clean = Path(temporary) / "clean" + shutil.copytree(source_root, clean) + for mutation in mutations: + source = clean / mutation.path + original = source.read_text(encoding="utf-8") + if original.count(mutation.old) != 1: + issues.append(f"{mutation.name}: mutation anchor count is not one") + continue + baseline = _probe(clean, mutation.probe) + if baseline.returncode != 0: + issues.append(f"{mutation.name}: baseline probe failed") + continue + mutated_text = original.replace(mutation.old, mutation.new, 1) + try: + compile(mutated_text, str(source), "exec") + except SyntaxError: + issues.append(f"{mutation.name}: mutant is syntactically invalid") + continue + mutant = Path(temporary) / f"mutant-{mutation.name}" + shutil.copytree(clean, mutant) + (mutant / mutation.path).write_text(mutated_text, encoding="utf-8") + if _probe(mutant, mutation.probe).returncode != 0: + killed += 1 + else: + issues.append(f"{mutation.name}: mutant survived") + return MutationReport(len(mutations), killed, tuple(issues)) + + +def main() -> int: + """Run the repository mutation contract and print a bounded score.""" + report = run_gate(Path(__file__).resolve().parents[1]) + print( + f"Hermes handoff mutation score: {report.killed}/{report.total} ({report.score:.2f}%)" + ) + for issue in report.issues: + print(issue) + return 0 if not report.issues and report.killed == report.total else 1 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) # pragma: no cover diff --git a/testing/quality_hygiene.py b/testing/quality_hygiene.py index 5060ca02..6b68de3a 100644 --- a/testing/quality_hygiene.py +++ b/testing/quality_hygiene.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import re from collections.abc import Iterable from pathlib import Path @@ -16,6 +17,18 @@ def _expand_globs(root: Path, patterns: Iterable[str]) -> list[Path]: return sorted(matched) +def _is_unchanged_legacy( + config: dict[str, Any], root: Path, path: Path, line_count: int +) -> bool: + """Allow an over-cap legacy file only while its exact bytes stay unchanged.""" + relative = path.relative_to(root).as_posix() + record = config.get("legacy_line_exceptions", {}).get(relative) + if not isinstance(record, dict) or record.get("lines") != line_count: + return False + digest = hashlib.sha256(path.read_bytes()).hexdigest() + return record.get("sha256") == digest + + def run_check(contract: dict[str, Any], root: Path) -> list[str]: """Return human-readable issues for naming and file-size rules.""" config = contract.get("hygiene", {}) @@ -24,8 +37,12 @@ def run_check(contract: dict[str, Any], root: Path) -> list[str]: for path in _expand_globs(root, config.get("line_limit_globs", [])): line_count = sum(1 for _ in path.open("r", encoding="utf-8")) - if line_count > max_lines: - issues.append(f"file exceeds {max_lines} LOC: {path.relative_to(root)} ({line_count})") + if line_count > max_lines and not _is_unchanged_legacy( + config, root, path, line_count + ): + issues.append( + f"file exceeds {max_lines} LOC: {path.relative_to(root)} ({line_count})" + ) for rule in config.get("naming_rules", []): pattern = re.compile(rule["pattern"]) @@ -48,7 +65,9 @@ def count_files_over_line_limit(contract: dict[str, Any], root: Path) -> int: count = 0 for path in _expand_globs(root, config.get("line_limit_globs", [])): line_count = sum(1 for _ in path.open("r", encoding="utf-8")) - if line_count > max_lines: + if line_count > max_lines and not _is_unchanged_legacy( + config, root, path, line_count + ): count += 1 return count diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py new file mode 100644 index 00000000..ecd64eb4 --- /dev/null +++ b/testing/tests/conftest.py @@ -0,0 +1,20 @@ +"""Hermetic runtime-path fixtures shared by repository tests.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +@pytest.fixture(autouse=True) +def isolate_coordinator_gitea_token( + request: pytest.FixtureRequest, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Prevent coordinator tests from observing the mounted production token.""" + if request.node.path.name == "test_hermes_coordinator.py": + monkeypatch.setenv( + "HERMES_GITEA_TOKEN_FILE", str(tmp_path / "absent-gitea-token") + ) diff --git a/testing/tests/test_hermes_auto_router.py b/testing/tests/test_hermes_auto_router.py index c0a31103..f8a333b1 100644 --- a/testing/tests/test_hermes_auto_router.py +++ b/testing/tests/test_hermes_auto_router.py @@ -255,6 +255,12 @@ def test_provider_status_separates_observed_activity_from_plan_quota(monkeypatch monkeypatch.setattr(module, "_claude_account", lambda: { "authenticated": True, "plan": "max", "quota_reported": False, }) + monkeypatch.setattr(module, "_fresh_health", lambda _path, maximum_age=86400.0: {}) + monkeypatch.setattr( + module, + "_configured_models", + lambda provider: ["gpt-5.6-sol"] if provider == "codex" else ["claude-sonnet-5"], + ) payload = module.provider_status_payload() diff --git a/testing/tests/test_hermes_handoff_acceptance.py b/testing/tests/test_hermes_handoff_acceptance.py index 87c8adbe..4f89db35 100644 --- a/testing/tests/test_hermes_handoff_acceptance.py +++ b/testing/tests/test_hermes_handoff_acceptance.py @@ -18,7 +18,9 @@ harness_run = load_handoff_module("hermes_handoff_run") def completed(stdout: str = "", stderr: str = "", returncode: int = 0): - return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) class ScriptedCluster: @@ -44,10 +46,37 @@ class ScriptedCluster: def whoami(username: str): - return completed(stdout=json.dumps({"status": {"userInfo": {"username": username}}})) + return completed(stdout=username) + + +REVIEWED = "8f005458282269ba5c07941814e4237f5d4cf3ac" +SHA = "1" * 40 +VALID_ARGS = [ + "--remote-main-sha", + SHA, + "--reviewed-head-sha", + REVIEWED, + "--agent-image", + f"registry.example/hermes:git-{'1' * 40}-build-7@sha256:{'2' * 64}", + "--build-sha", + SHA, + "--deployment-revision", + "1", + "--chat-config-revision", + "chat-rev", + *[ + item + for number in catalog.Targets().dependency_pull_requests + for item in ("--dependency-head", f"{number}={SHA}") + ], +] HEALTHY_CLUSTER = [ + ( + "exec --namespace hermes hermes-agent-1", + whoami("system:serviceaccount:hermes:hermes-agent"), + ), ("auth whoami", whoami("kubernetes-admin")), ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), ("get pods --selector", completed(stdout="pod/other-1\n")), @@ -60,10 +89,18 @@ def run_cli(monkeypatch, argv, rules): clock = FakeClock() def build_runner(**kwargs): - return exec_module.Runner(clock=clock, spawn=cluster, environ={}, **kwargs) + return exec_module.Runner( + clock=clock, + spawn=cluster, + environ={}, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + **kwargs, + ) monkeypatch.setattr(acceptance, "Runner", build_runner) - return (acceptance.main(argv), cluster) + return (acceptance.main([*VALID_ARGS, *argv]), cluster) def read_report(path): @@ -109,7 +146,15 @@ def test_defaults_fall_back_to_the_shipped_targets() -> None: assert targets.repo == catalog.Targets.repo -def test_a_default_run_is_read_only_and_never_pushes_or_posts(monkeypatch, tmp_path) -> None: +@pytest.mark.parametrize("value", ["bad", "x=" + "1" * 40, "0=" + "1" * 40, "1=BAD"]) +def test_dependency_heads_require_a_positive_pr_and_lowercase_sha(value: str) -> None: + with pytest.raises(Exception): + acceptance.dependency_head(value) + + +def test_a_default_run_is_read_only_and_never_pushes_or_posts( + monkeypatch, tmp_path +) -> None: output = tmp_path / "report.json" status, cluster = run_cli( monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER @@ -124,13 +169,15 @@ def test_a_default_run_is_read_only_and_never_pushes_or_posts(monkeypatch, tmp_p writes = [ call for call in cluster.calls - if {"patch", "create", "delete", "apply", "replace"} & set(call) and "can-i" not in call + if {"patch", "create", "delete", "apply", "replace"} & set(call) + and "can-i" not in call ] - assert writes - assert all("--dry-run=server" in call for call in writes) + assert writes == [] -def test_mutating_checks_report_not_run_until_the_mode_is_armed(monkeypatch, tmp_path) -> None: +def test_mutating_checks_report_not_run_until_the_mode_is_armed( + monkeypatch, tmp_path +) -> None: output = tmp_path / "report.json" run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) @@ -163,7 +210,10 @@ def test_a_zero_state_cluster_is_no_go_with_every_check_accounted_for( assert report["decision"] == model.NO_GO assert report["counts"][model.PASS] == 0 assert sum(report["counts"].values()) == len(report["checks"]) - assert any("in-pod Hermes vantage is unavailable" in error for error in report["harness_errors"]) + assert any( + "in-pod Hermes vantage is unavailable" in error + for error in report["harness_errors"] + ) def test_partial_failure_is_reported_per_check_rather_than_aborting( @@ -186,14 +236,25 @@ def test_partial_failure_is_reported_per_check_rather_than_aborting( assert report["blocking"] -def test_a_vantage_collision_is_a_no_go_even_with_clean_checks(monkeypatch, tmp_path) -> None: +def test_a_vantage_collision_is_a_no_go_even_with_clean_checks( + monkeypatch, tmp_path +) -> None: output = tmp_path / "report.json" rules = [ - ("exec --namespace hermes hermes-agent-1", whoami("system:serviceaccount:hermes:hermes-agent")), + ( + "exec --namespace hermes hermes-agent-1", + whoami("system:serviceaccount:hermes:hermes-agent"), + ), ("auth whoami", whoami("system:serviceaccount:hermes:hermes-agent")), - ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), + ( + "get pods --selector app=hermes-agent", + completed(stdout="pod/hermes-agent-1\n"), + ), ("get pods --selector", completed(stdout="pod/other-1\n")), - ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")), + ( + "get pod/hermes-chat-tenant-0", + completed(stdout="pod/hermes-chat-tenant-0\n"), + ), ] status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) report = read_report(output) @@ -202,16 +263,18 @@ def test_a_vantage_collision_is_a_no_go_even_with_clean_checks(monkeypatch, tmp_ assert any("same principal" in error for error in report["harness_errors"]) -def test_an_exhausted_deadline_leaves_the_remaining_checks_not_run(monkeypatch, tmp_path) -> None: +def test_an_invalid_deadline_stops_before_any_subprocess(monkeypatch, tmp_path) -> None: output = tmp_path / "report.json" status, _ = run_cli( - monkeypatch, ["--output", str(output), "--json-only", "--deadline", "0"], HEALTHY_CLUSTER + monkeypatch, + ["--output", str(output), "--json-only", "--deadline", "0"], + HEALTHY_CLUSTER, ) report = read_report(output) - assert status == acceptance.EXIT_NO_GO + assert status == acceptance.EXIT_USAGE assert report["counts"][model.NOT_RUN] > 0 - assert any("deadline expired" in error for error in report["harness_errors"]) + assert any("invalid execution bound" in error for error in report["harness_errors"]) def test_arming_with_a_wrong_confirmation_stops_before_anything_runs( @@ -239,7 +302,9 @@ def test_arming_with_a_wrong_confirmation_stops_before_anything_runs( assert not any("push" in call for call in cluster.calls) -def test_arming_against_a_protected_push_target_is_refused(monkeypatch, tmp_path) -> None: +def test_arming_against_a_protected_push_target_is_refused( + monkeypatch, tmp_path +) -> None: output = tmp_path / "report.json" status, cluster = run_cli( monkeypatch, @@ -261,17 +326,24 @@ def test_arming_against_a_protected_push_target_is_refused(monkeypatch, tmp_path assert not any("push" in call for call in cluster.calls) -def test_an_armed_run_pushes_one_ephemeral_ref_and_verifies_its_removal( +def test_an_armed_run_does_not_mutate_until_read_only_acceptance_is_go( monkeypatch, tmp_path ) -> None: output = tmp_path / "report.json" + monkeypatch.setattr(acceptance, "preflight", lambda *_args: None) rules = [ ("git ls-remote origin refs/heads/ephemeral", completed(stdout="")), ("git push", completed()), - ("POST /api/v1/repos/atlas/titan-iac/pulls", completed(stdout=json.dumps({"number": 9}))), + ( + "POST /api/v1/repos/atlas/titan-iac/pulls", + completed(stdout=json.dumps({"number": 9})), + ), ("PATCH", completed()), ("DELETE", completed()), - ("GET /api/v1/repos/atlas/titan-iac/pulls/9", completed(stdout='{"state": "closed"}')), + ( + "GET /api/v1/repos/atlas/titan-iac/pulls/9", + completed(stdout='{"state": "closed"}'), + ), *HEALTHY_CLUSTER, ] status, cluster = run_cli( @@ -289,20 +361,14 @@ def test_an_armed_run_pushes_one_ephemeral_ref_and_verifies_its_removal( rules, ) report = read_report(output) - ephemeral = {check["id"]: check for check in report["checks"] if check["group"] == "ephemeral"} - assert report["mode"] == policy.ARMED - assert status == acceptance.EXIT_NO_GO # the rest of the cluster is still unproven - assert ephemeral["ephemeral.feature-branch-push"]["status"] == model.PASS - assert ephemeral["ephemeral.draft-pull-request"]["status"] == model.PASS - assert ephemeral["ephemeral.cleanup-verified"]["status"] == model.PASS - assert ephemeral["ephemeral.protected-branch-refusal"]["status"] == model.PASS - + assert status == acceptance.EXIT_NO_GO + assert any( + "read-only acceptance did not GO" in error for error in report["harness_errors"] + ) pushes = [call for call in cluster.calls if "push" in call] - assert len(pushes) == 1 - assert pushes[0][-1].startswith("HEAD:refs/heads/ephemeral/hermes-handoff-acceptance/") - assert pushes[0][0] == "kubectl" and "exec" in pushes[0] # pushed as the worker, not the operator - assert any("DELETE" in call for call in cluster.calls) + assert pushes == [] + assert not any("DELETE" in call for call in cluster.calls) def test_the_report_is_written_to_stdout_when_no_output_path_is_given( @@ -327,10 +393,14 @@ def test_the_human_summary_accompanies_the_json_unless_suppressed( assert captured.out == "" -def test_a_report_that_fails_its_own_screening_is_replaced_not_published(tmp_path) -> None: +def test_a_report_that_fails_its_own_screening_is_replaced_not_published( + tmp_path, +) -> None: """The last barrier: never write an artifact nobody re-read.""" output = tmp_path / "report.json" - arguments = acceptance.build_parser().parse_args(["--output", str(output), "--json-only"]) + arguments = acceptance.build_parser().parse_args( + ["--output", str(output), "--json-only"] + ) report = model.Report(mode="read-only", started_at="2026-08-17T00:00:00Z") report.harness_errors = ["leaked ghp_ABCDEFGHIJKLMNOPQRSTUV1234"] @@ -348,20 +418,28 @@ def test_a_report_that_fails_its_own_screening_is_replaced_not_published(tmp_pat assert "failed its own credential screening" in payload["harness_errors"][0] -def test_a_screening_failure_forces_no_go_from_the_entry_point(monkeypatch, tmp_path) -> None: +def test_a_screening_failure_forces_no_go_from_the_entry_point( + monkeypatch, tmp_path +) -> None: """The exit status must follow the refusal to publish, not the check tally.""" output = tmp_path / "report.json" - monkeypatch.setattr(acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"]) + monkeypatch.setattr( + acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"] + ) monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) - status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) + status, _ = run_cli( + monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER + ) assert status == acceptance.EXIT_NO_GO assert read_report(output)["decision"] == model.NO_GO -def test_no_recorded_command_output_carries_a_credential_shape(monkeypatch, tmp_path) -> None: +def test_no_recorded_command_output_carries_a_credential_shape( + monkeypatch, tmp_path +) -> None: output = tmp_path / "report.json" leaky = completed(stdout="ANTHROPIC_API_KEY=sk-ant-abcdefghijklmnopqrstuvwxyz\n") run_cli( @@ -380,16 +458,3 @@ def test_the_help_text_describes_the_read_only_default(flag: str, capsys) -> Non acceptance.main([flag]) assert "Read-only by default" in capsys.readouterr().out - - -def test_go_is_reachable_when_nothing_mandatory_is_outstanding(monkeypatch, tmp_path) -> None: - """Exercise the success exit path without pretending the cluster is clean.""" - output = tmp_path / "report.json" - monkeypatch.setattr(harness_run, "build_catalog", lambda _targets: []) - monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) - monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) - - status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) - - assert status == acceptance.EXIT_GO - assert read_report(output)["decision"] == model.GO diff --git a/testing/tests/test_hermes_handoff_acceptance_armed.py b/testing/tests/test_hermes_handoff_acceptance_armed.py new file mode 100644 index 00000000..ded33bb8 --- /dev/null +++ b/testing/tests/test_hermes_handoff_acceptance_armed.py @@ -0,0 +1,120 @@ +"""Static and armed-transition regressions for the handoff CLI.""" + +from __future__ import annotations + +from testing.tests.test_hermes_handoff_acceptance import ( + HEALTHY_CLUSTER, + REVIEWED, + VALID_ARGS, + acceptance, + harness_run, + model, + policy, + read_report, + run_cli, +) + + +def test_an_empty_catalog_fails_before_any_subprocess(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + monkeypatch.setattr(harness_run, "build_catalog", lambda _targets: []) + monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) + monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) + + status, _ = run_cli( + monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER + ) + + assert status == acceptance.EXIT_NO_GO + assert read_report(output)["decision"] == model.NO_GO + + +def test_armed_success_path_replaces_placeholders_only_after_a_go(monkeypatch) -> None: + base = model.CheckSpec( + id="release.pass", title="pass", group="release", rule="allowed" + ) + placeholder = model.CheckSpec( + id="ephemeral.cleanup-verified", + title="pending", + group="ephemeral", + rule="not_armed", + mandatory=False, + scope=model.EPHEMERAL, + ) + report = model.Report( + mode=policy.ARMED, + started_at="2026-08-17T00:00:00Z", + results=[ + model.CheckResult(base, model.PASS), + model.CheckResult(placeholder, model.NOT_RUN), + ], + ) + armed_spec = model.CheckSpec( + id="ephemeral.cleanup-verified", + title="clean", + group="ephemeral", + rule="armed", + scope=model.EPHEMERAL, + ) + armed = model.CheckResult(armed_spec, model.PASS) + monkeypatch.setattr( + acceptance, "resolve_vantages", lambda *_args: ({"operator": object()}, []) + ) + monkeypatch.setattr(acceptance, "preflight", lambda *_args: None) + monkeypatch.setattr(acceptance, "build_report", lambda *_args: report) + monkeypatch.setattr(acceptance, "_armed_results", lambda *_args: [armed]) + monkeypatch.setattr(acceptance, "emit", lambda *_args: []) + status = acceptance.main( + [ + *VALID_ARGS, + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817a", + ] + ) + assert status == acceptance.EXIT_GO + assert ( + next( + result + for result in report.results + if result.spec.id.startswith("ephemeral") + ).status + == model.PASS + ) + monkeypatch.setattr(acceptance, "emit", lambda *_args: ["screening-failure"]) + assert ( + acceptance.main( + [ + *VALID_ARGS, + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817b", + ] + ) + == acceptance.EXIT_NO_GO + ) + + +def test_armed_results_are_bound_to_operator_and_exact_reviewed_head( + monkeypatch, +) -> None: + arguments = acceptance.build_parser().parse_args( + [ + *VALID_ARGS, + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817a", + ] + ) + calls = [] + monkeypatch.setattr(acceptance, "preflight", lambda *args: calls.append(args)) + monkeypatch.setattr(acceptance, "run_armed", lambda *args: [args]) + operator = object() + result = acceptance._armed_results(arguments, object(), {"operator": operator}) + assert calls[0][0].expected_head == REVIEWED + assert result[0][1] is operator diff --git a/testing/tests/test_hermes_handoff_catalog.py b/testing/tests/test_hermes_handoff_catalog.py index 8f5e2de6..21e8031a 100644 --- a/testing/tests/test_hermes_handoff_catalog.py +++ b/testing/tests/test_hermes_handoff_catalog.py @@ -23,8 +23,12 @@ BY_ID = {check.id: check for check in SPECS} VANTAGES = { catalog.OPERATOR: exec_module.operator_vantage(context="atlas-operator"), catalog.SELF: exec_module.pod_vantage("hermes", "hermes-agent-1", "hermes"), - catalog.SWITCHYARD: exec_module.pod_vantage("hermes", "hermes-switchyard-1", "switchyard"), - catalog.NODE: exec_module.pod_vantage("hermes", "hermes-node-ssh-access-1", "key-reconciler"), + catalog.SWITCHYARD: exec_module.pod_vantage( + "hermes", "hermes-switchyard-1", "switchyard" + ), + catalog.NODE: exec_module.pod_vantage( + "hermes", "hermes-node-ssh-access-1", "key-reconciler" + ), catalog.CHAT: exec_module.pod_vantage("hermes", "hermes-chat-tenant-0", "hermes"), } @@ -42,32 +46,41 @@ def test_every_step_is_accepted_by_the_read_only_policy_from_its_vantage() -> No policy.check_argv(addressed, policy.READ_ONLY) -def test_no_step_mutates_outside_a_server_dry_run() -> None: - """`auth can-i create ...` is a read; the subcommand is what decides.""" +def test_no_step_constructs_a_mutation_or_dry_run() -> None: + """Authorization review may name a verb; no executed subcommand mutates.""" for check in SPECS: for step in check.steps: subcommand = next(policy.positionals(step.argv), "") - if subcommand in policy.KUBECTL_DRY_RUN_SUBCOMMANDS: - assert any(flag in step.argv for flag in policy.DRY_RUN_FLAGS), check.id + assert subcommand not in {"apply", "create", "delete", "patch", "replace"} + assert not any(argument.startswith("--dry-run") for argument in step.argv) -def test_no_operator_step_impersonates_and_at_least_one_in_pod_step_does() -> None: +def test_no_catalog_step_constructs_impersonation() -> None: operator_steps = [ - step for check in SPECS for step in check.steps if step.vantage == catalog.OPERATOR + step + for check in SPECS + for step in check.steps + if step.vantage == catalog.OPERATOR ] assert operator_steps - assert not any(set(step.argv) & set(policy.IMPERSONATION_ARGS) for step in operator_steps) + assert not any( + set(step.argv) & set(policy.IMPERSONATION_ARGS) for step in operator_steps + ) - impersonation = BY_ID["access.impersonation-is-denied"] - attempts = [step for step in impersonation.steps if step.kind == model.ATTEMPT] - assert any("--as" in step.argv for step in attempts) + assert not any( + set(step.argv) & set(policy.IMPERSONATION_ARGS) + for check in SPECS + for step in check.steps + ) -def test_every_deny_check_pairs_a_review_with_a_live_attempt() -> None: - deny_checks = [check for check in SPECS if check.rule == "denied"] - assert len(deny_checks) >= 6 +def test_credential_or_mutation_denials_are_review_only() -> None: + deny_checks = [check for check in SPECS if check.rule == "review_denied"] + assert len(deny_checks) >= 5 for check in deny_checks: - assert any(step.kind == model.ATTEMPT for step in check.steps), check.id + assert check.steps and all(step.kind == model.REVIEW for step in check.steps), ( + check.id + ) def test_both_vantages_are_represented_across_the_catalog() -> None: @@ -104,7 +117,9 @@ def test_dependency_pull_requests_each_get_their_own_merge_gate() -> None: for number in TARGETS.dependency_pull_requests: check = BY_ID[f"baseline.dependency-pr-{number}-merged"] assert check.mandatory - assert check.expect["fields"] == {"merged": True, "base.ref": "main"} + assert check.expect["fields"]["merged"] is True + assert check.expect["fields"]["base.ref"] == "main" + assert "head.sha" in check.expect["fields"] def test_the_baseline_commit_gate_names_the_configured_commit() -> None: @@ -113,25 +128,12 @@ def test_the_baseline_commit_gate_names_the_configured_commit() -> None: assert "--is-ancestor" in check.steps[0].argv -def test_optional_checks_appear_only_when_a_site_declares_what_to_expect() -> None: - """A names_present over an empty expectation would pass on any workload.""" - assert "pool.assignment-safety-knobs-are-configured" not in BY_ID - assert "surfaces.chat-runs-the-telegram-topic-revision" not in BY_ID - assert "surfaces.chat-telegram-sessions-are-continuous" not in BY_ID - - configured = harness_run.build_catalog( - catalog.Targets( - now=NOW, - pool_worker_env=("HERMES_POOL_LEASE_SECONDS",), - chat_config_revision="20260816-telegram-topics", - expect_telegram_sessions=True, - ) - ) - identifiers = {check.id for check in configured} - assert "pool.assignment-safety-knobs-are-configured" in identifiers - assert "surfaces.chat-runs-the-telegram-topic-revision" in identifiers - assert "surfaces.chat-telegram-sessions-are-continuous" in identifiers - assert harness_run.validate_catalog(configured) == [] +def test_pool_assignment_and_chat_continuity_are_unconditional() -> None: + assert "pool.assignment-safety-knobs-are-configured" in BY_ID + assert "pool.assignment-fieldrefs-are-bound" in BY_ID + assert "surfaces.chat-runs-the-telegram-topic-revision" in BY_ID + assert "surfaces.chat-telegram-sessions-are-continuous" in BY_ID + assert TARGETS.node_count == TARGETS.pool_replicas == 3 def test_bulk_evidence_is_parsed_but_not_pasted_into_the_report() -> None: @@ -144,7 +146,10 @@ def test_bulk_evidence_is_parsed_but_not_pasted_into_the_report() -> None: def test_freshness_gates_carry_the_run_clock() -> None: - for identifier in ("identity.codex-evidence-is-fresh", "identity.claude-evidence-is-fresh"): + for identifier in ( + "identity.codex-evidence-is-fresh", + "identity.claude-evidence-is-fresh", + ): assert BY_ID[identifier].expect["now"] == NOW routing = BY_ID["routing.provider-effort-and-fallback-evidence"] assert routing.expect["now"] == NOW @@ -170,18 +175,14 @@ def test_check_identifiers_are_unique_and_grouped() -> None: "access.workload-mutation-is-denied", "access.pod-exec-is-denied", "access.required-reads-are-allowed", - "access.pod-logs-are-allowed", "access.flux-and-helm-status-are-allowed", "access.namespace-view-agrees-across-vantages", - "nodes.account-password-is-locked", "nodes.hardening-covers-every-node", "build.builder-service-account-is-tokenless", "build.harbor-immutability-rule-is-applied", - "build.deployed-image-is-digest-pinned", + "release.exact-lineage-is-running", "reliability.finalization-and-replay-patches-are-deployed", - "pool.three-workers-are-ready", - "pool.workers-occupy-distinct-nodes", - "pool.workers-own-distinct-volumes", + "pool.three-workers-have-distinct-nodes-and-volumes", "pool.coordinator-retains-sole-state-ownership", "surfaces.kanban-activity-is-visible", "surfaces.chat-telegram-topic-state-is-durable", diff --git a/testing/tests/test_hermes_handoff_ephemeral.py b/testing/tests/test_hermes_handoff_ephemeral.py index 09cc5d63..9f54601a 100644 --- a/testing/tests/test_hermes_handoff_ephemeral.py +++ b/testing/tests/test_hermes_handoff_ephemeral.py @@ -1,9 +1,10 @@ -"""Contracts for the separately armed, self-cleaning mutation mode.""" +"""Contracts for the strictly bounded, self-cleaning armed mode.""" from __future__ import annotations import json import subprocess +from pathlib import Path import pytest @@ -18,254 +19,319 @@ exec_module = load_handoff_module("hermes_handoff_exec") policy = load_handoff_module("hermes_handoff_policy") model = load_handoff_module("hermes_handoff_model") -REPO = "atlas/titan-iac" TOKEN = "acceptance-20260817a" +HEAD = "8f005458282269ba5c07941814e4237f5d4cf3ac" +MAIN = "1" * 40 VANTAGE = exec_module.operator_vantage() -def request(**overrides) -> object: +def request(**overrides): fields = { - "repo": REPO, - "remote": "origin", + "repo": policy.EXPECTED_REPO, + "remote": policy.EXPECTED_REMOTE, "token": TOKEN, "confirmation": ephemeral.CONFIRMATION, - "base": "main", + "base": policy.EXPECTED_BASE, + "expected_head": HEAD, + "expected_base_sha": MAIN, } fields.update(overrides) return ephemeral.ArmRequest(**fields) def completed(stdout: str = "", stderr: str = "", returncode: int = 0): - return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) -def armed_runner(results: list) -> tuple: +def armed_runner(results: list, mode=policy.ARMED): spawn = FakeSpawn(results) runner = exec_module.Runner( - mode=policy.ARMED, clock=FakeClock(), spawn=spawn, environ={}, deadline_seconds=3600 + mode=mode, + clock=FakeClock(), + spawn=spawn, + environ={}, + deadline_seconds=3600, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), ) - return (runner, spawn) + return runner, spawn def by_id(results: list) -> dict: return {result.spec.id: result for result in results} -def test_the_ephemeral_ref_is_derived_from_the_caller_supplied_token() -> None: - assert request().ref == f"{ephemeral.REF_PREFIX}/{TOKEN}" - - -@pytest.mark.parametrize( - "ref", ["main", "master", "refs/heads/main", "HEAD", "Production", "feature/x", ""] -) -def test_protected_and_malformed_push_targets_are_refused(ref: str) -> None: - with pytest.raises(ephemeral.ArmingError): - ephemeral.assert_push_target_allowed(ref) - - -def test_an_ephemeral_ref_is_the_only_accepted_push_target() -> None: - ephemeral.assert_push_target_allowed(f"{ephemeral.REF_PREFIX}/{TOKEN}") - - -@pytest.mark.parametrize( - ("overrides", "fragment"), - [ - ({"confirmation": "yes please"}, "confirmation phrase"), - ({"confirmation": ""}, "confirmation phrase"), - ({"repo": "atlas/other"}, "is not the expected"), - ({"token": "short"}, "ephemeral token"), - ({"token": "Has-Capitals-Here"}, "ephemeral token"), - ({"token": ""}, "ephemeral token"), - ({"base": ""}, "base branch is required"), - ], -) -def test_preflight_refuses_before_any_network_call(overrides: dict, fragment: str) -> None: - with pytest.raises(ephemeral.ArmingError) as caught: - ephemeral.preflight(request(**overrides), REPO) - assert fragment in str(caught.value) - - -def test_preflight_accepts_a_correctly_armed_request() -> None: - ephemeral.preflight(request(), REPO) - - -@pytest.mark.parametrize( - ("raw", "expected"), [("refs/heads/Main", "main"), (" main ", "main"), ("/x/", "x")] -) -def test_branch_names_are_normalised_before_comparison(raw: str, expected: str) -> None: - assert ephemeral.normalise_branch(raw) == expected - - -def test_the_protected_branch_guard_is_exercised_on_every_armed_run() -> None: - """A guard checked once is a guard nobody notices regressing.""" - runner, _ = armed_runner([completed(stdout="", returncode=2)]) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - guard = results["ephemeral.protected-branch-refusal"] - - assert guard.status == model.PASS - assert guard.evidence["accepted"] == [] - assert "main" in guard.evidence["refused"] - - -def test_a_broken_protected_branch_guard_fails_its_own_self_test(monkeypatch) -> None: - """If the guard ever stops refusing, the armed run says so instead of pushing.""" - monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None) - runner, _ = armed_runner([completed(stdout="", returncode=2)]) - - guard = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[ - "ephemeral.protected-branch-refusal" - ] - - assert guard.status == model.FAIL - assert "main" in guard.evidence["accepted"] - assert "the guard accepted" in guard.reason - - -def test_a_full_armed_run_pushes_opens_a_draft_and_removes_both() -> None: - runner, spawn = armed_runner( - [ - completed(stdout=""), # ls-remote: ref unused - completed(stdout=""), # push - completed(stdout=json.dumps({"number": 42, "draft": True})), # create draft - completed(stdout=""), # close - completed(stdout=""), # delete branch - completed(stdout=""), # ls-remote: gone - completed(stdout=json.dumps({"number": 42, "state": "closed"})), - ] - ) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - - assert results["ephemeral.feature-branch-push"].status == model.PASS - assert results["ephemeral.draft-pull-request"].status == model.PASS - assert results["ephemeral.draft-pull-request"].evidence["pull_request"] == 42 - assert results["ephemeral.cleanup-verified"].status == model.PASS - assert all(result.spec.mandatory for result in results.values()) - assert all(result.spec.scope == model.EPHEMERAL for result in results.values()) - - pushed = [call for call in spawn.calls if call["argv"][:2] == ["git", "push"]] - assert pushed[0]["argv"][-1] == f"HEAD:refs/heads/{request().ref}" - assert "--force" not in pushed[0]["argv"] - - titles = [arg for call in spawn.calls for arg in call["argv"] if arg.startswith("title=")] - assert titles[0].startswith(f"title={ephemeral.DRAFT_TITLE_PREFIX}") - - -def test_an_existing_ref_stops_the_run_before_it_writes() -> None: - runner, spawn = armed_runner([completed(stdout="abc123\trefs/heads/x\n")]) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - - assert results["ephemeral.feature-branch-push"].status == model.FAIL - assert "already exists" in results["ephemeral.feature-branch-push"].reason - assert "ephemeral.draft-pull-request" not in results - assert len(spawn.calls) == 1 - - -def test_an_unverifiable_ref_check_is_not_run_rather_than_assumed_free() -> None: - runner, _ = armed_runner([completed(stderr="network down", returncode=128)]) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - - assert results["ephemeral.feature-branch-push"].status == model.NOT_RUN - - -def test_a_failed_push_still_reports_cleanup_and_skips_the_draft() -> None: - runner, _ = armed_runner( - [ - completed(stdout=""), - completed(stderr="denied", returncode=1), - completed(stdout=""), # delete branch - completed(stdout=""), # ls-remote: gone - ] - ) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - - assert results["ephemeral.feature-branch-push"].status == model.FAIL - assert results["ephemeral.draft-pull-request"].status == model.NOT_RUN - assert results["ephemeral.cleanup-verified"].status == model.PASS - - -def test_a_draft_response_without_a_number_is_a_failure() -> None: - runner, _ = armed_runner( - [ - completed(stdout=""), - completed(stdout=""), - completed(stdout="{}"), - completed(stdout=""), - completed(stdout=""), - ] - ) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - - assert results["ephemeral.draft-pull-request"].status == model.FAIL - - -def test_a_branch_that_survives_cleanup_fails_the_cleanup_check() -> None: - runner, _ = armed_runner( - [ - completed(stdout=""), - completed(stdout=""), - completed(stdout=json.dumps({"number": 7})), - completed(stderr="cannot close", returncode=1), - completed(stderr="cannot delete", returncode=1), - completed(stdout="abc123\trefs/heads/x\n"), - completed(stdout=json.dumps({"number": 7, "state": "open"})), - ] - ) - - results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - cleanup = results["ephemeral.cleanup-verified"] - - assert cleanup.status == model.FAIL - assert "could not be closed" in cleanup.reason - assert "still exists" in cleanup.reason - assert "still open" in cleanup.reason - - -def test_cleanup_that_cannot_be_verified_is_a_failure_not_a_pass() -> None: - runner, _ = armed_runner( - [ - completed(stdout=""), - completed(stdout=""), - completed(stdout=json.dumps({"number": 7})), - completed(stdout=""), - completed(stdout=""), - subprocess.TimeoutExpired(cmd="git", timeout=1.0), - completed(stderr="gone", returncode=1), - ] - ) - - cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))["ephemeral.cleanup-verified"] - - assert cleanup.status == model.FAIL - assert "could not verify removal" in cleanup.reason - assert "could not confirm pull request #7 is closed" in cleanup.reason - - -def test_the_armed_results_replace_the_default_not_run_placeholders() -> None: - runner, _ = armed_runner([completed(stdout="abc\trefs/heads/x\n")]) - identifiers = {result.spec.id for result in ephemeral.run_armed(runner, VANTAGE, request())} - - assert identifiers <= { - "ephemeral.protected-branch-refusal", - "ephemeral.feature-branch-push", - "ephemeral.draft-pull-request", - "ephemeral.cleanup-verified", +def pr_payload(state="open", number=42): + return { + "number": number, + "state": state, + "draft": True, + "merged": False, + "base": {"ref": "main", "sha": MAIN}, + "head": {"ref": request().ref, "sha": HEAD}, } -def test_a_read_only_runner_cannot_execute_the_armed_flow() -> None: - """The mode, not the caller, is what opens the mutations.""" - spawn = FakeSpawn([completed(stdout=""), completed(stdout=""), completed(stdout="")]) - runner = exec_module.Runner(clock=FakeClock(), spawn=spawn, environ={}) +def linked_worktree(tmp_path: Path) -> Path: + root = tmp_path / "worktree" + common = tmp_path / "repo.git" + gitdir = common / "worktrees" / "acceptance" + gitdir.mkdir(parents=True) + (root).mkdir() + (root / ".git").write_text(f"gitdir: {gitdir}\n", encoding="utf-8") + (gitdir / "commondir").write_text("../..\n", encoding="utf-8") + (gitdir / "HEAD").write_text( + f"ref: refs/heads/{ephemeral.EXPECTED_HEAD_REF}\n", encoding="utf-8" + ) + ref = common / "refs" / "heads" / ephemeral.EXPECTED_HEAD_REF + ref.parent.mkdir(parents=True) + ref.write_text(f"{HEAD}\n", encoding="utf-8") + remote_ref = common / "refs" / "remotes" / "origin" / "main" + remote_ref.parent.mkdir(parents=True) + remote_ref.write_text(f"{MAIN}\n", encoding="utf-8") + (common / "config").write_text( + f'[remote "origin"]\n\turl = {ephemeral.EXPECTED_REMOTE_URL}\n', + encoding="utf-8", + ) + return root + +@pytest.mark.parametrize( + "ref", ["main", "master", "refs/heads/main", "HEAD", "feature/x", ""] +) +def test_only_the_unique_ephemeral_ref_is_allowed(ref: str) -> None: + with pytest.raises(ephemeral.ArmingError): + ephemeral.assert_push_target_allowed(ref) + ephemeral.assert_push_target_allowed(request().ref) + + +@pytest.mark.parametrize( + "overrides", + [ + {"confirmation": "wrong"}, + {"repo": "atlas/other"}, + {"remote": "upstream"}, + {"base": "develop"}, + {"token": "short"}, + {"expected_head": "bad"}, + {"expected_base_sha": "bad"}, + ], +) +def test_preflight_rejects_bad_inputs_before_any_runner_exists( + tmp_path: Path, overrides: dict +) -> None: + with pytest.raises(ephemeral.ArmingError): + ephemeral.preflight( + request(**overrides), policy.EXPECTED_REPO, linked_worktree(tmp_path) + ) + + +def test_preflight_attests_the_actual_linked_worktree(tmp_path: Path) -> None: + ephemeral.preflight(request(), policy.EXPECTED_REPO, linked_worktree(tmp_path)) + root = linked_worktree(tmp_path / "wrong") + (Path((root / ".git").read_text().split(": ", 1)[1].strip()) / "HEAD").write_text( + "ref: refs/heads/other\n", encoding="utf-8" + ) + with pytest.raises(ephemeral.ArmingError, match="existing PR #19"): + ephemeral.preflight(request(), policy.EXPECTED_REPO, root) + + +def test_worktree_attestation_rejects_each_malformed_local_source( + tmp_path: Path, +) -> None: + roots = [] + empty = tmp_path / "empty" + empty.mkdir() + roots.append(empty) + bad_marker = linked_worktree(tmp_path / "marker") + (bad_marker / ".git").write_text("not-a-gitdir\n", encoding="utf-8") + roots.append(bad_marker) + wrong_sha = linked_worktree(tmp_path / "sha") + gitdir = Path((wrong_sha / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "refs" / "heads" / ephemeral.EXPECTED_HEAD_REF).write_text("0" * 40) + roots.append(wrong_sha) + wrong_main = linked_worktree(tmp_path / "main") + gitdir = Path((wrong_main / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "refs" / "remotes" / "origin" / "main").write_text("0" * 40) + roots.append(wrong_main) + bad_config = linked_worktree(tmp_path / "config") + gitdir = Path((bad_config / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "config").write_text("[broken\n", encoding="utf-8") + roots.append(bad_config) + wrong_remote = linked_worktree(tmp_path / "remote") + gitdir = Path((wrong_remote / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "config").write_text('[remote "origin"]\nurl=https://example.invalid/x\n') + roots.append(wrong_remote) + for root in roots: + with pytest.raises((ephemeral.ArmingError, FileNotFoundError)): + ephemeral.preflight(request(), policy.EXPECTED_REPO, root) + oversized = tmp_path / "oversized" + oversized.write_text("x" * 5, encoding="utf-8") + with pytest.raises(ephemeral.ArmingError): + ephemeral._bounded_read(oversized, 4) + + +def test_guard_failure_stops_before_runner_calls(monkeypatch) -> None: + monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None) + runner, spawn = armed_runner([]) results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) - push = results["ephemeral.feature-branch-push"] + assert results["ephemeral.protected-branch-refusal"].status == model.FAIL + assert all( + result.status in {model.FAIL, model.NOT_RUN} for result in results.values() + ) + assert spawn.calls == [] - assert push.status == model.FAIL - assert "policy" in push.reason - assert not any(call["argv"][:2] == ["git", "push"] for call in spawn.calls) + +def test_full_armed_run_validates_and_removes_exact_artifacts() -> None: + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, spawn = armed_runner( + [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + completed(stdout=json.dumps(pr_payload())), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ] + ) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert all(result.status == model.PASS for result in results.values()) + push = next(call for call in spawn.calls if call["argv"][1] == "push") + assert push["argv"][-1] == f"HEAD:refs/heads/{request().ref}" + assert all(result.spec.mandatory for result in results.values()) + + +def test_existing_or_unverifiable_ref_never_reaches_a_write() -> None: + for first in (completed(stdout="abc\trefs/heads/x\n"), completed(returncode=128)): + runner, spawn = armed_runner([first]) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert not any( + call["argv"][1] in {"push", "POST", "PATCH", "DELETE"} + for call in spawn.calls + ) + + +def test_failed_or_ambiguous_create_is_discovered_and_cleanup_still_runs() -> None: + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, _ = armed_runner( + [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + completed(returncode=1), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ] + ) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.draft-pull-request"].status == model.PASS + assert results["ephemeral.cleanup-verified"].status == model.PASS + + +def test_pull_request_parsers_reject_every_malformed_or_ambiguous_shape() -> None: + assert ephemeral._pr_problems([], request()) == ["pull request is not an object"] + malformed = pr_payload() + malformed.update({"number": True, "draft": "yes", "base": None}) + malformed["head"] = {"ref": "other", "sha": "bad"} + problems = ephemeral._pr_problems(malformed, request()) + assert {"number is malformed", "draft does not match", "base is malformed"} <= set( + problems + ) + assert ephemeral._discover(exec_module.Outcome((), "x", returncode=1), request())[1] + for body in ( + "not-json", + "{}", + "[null]", + json.dumps([{"head": {"ref": "other"}, "base": {}}]), + ): + numbers, error = ephemeral._discover( + exec_module.Outcome((), "x", 0, body), request() + ) + assert numbers == [] and error + invalid = pr_payload() + invalid["draft"] = False + assert ephemeral._discover( + exec_module.Outcome((), "x", 0, json.dumps([invalid])), request() + )[1] + assert ( + "discovered 0" + in ephemeral._discover(exec_module.Outcome((), "x", 0, "[]"), request())[1] + ) + + +@pytest.mark.parametrize( + "delete_result,remaining,closed", + [ + (completed(returncode=1), completed(), pr_payload(state="closed")), + (completed(), completed(returncode=128), pr_payload(state="closed")), + (completed(), completed(), {}), + (completed(), completed(), pr_payload(state="open")), + ], +) +def test_cleanup_fails_closed_on_partial_or_malformed_evidence( + delete_result, remaining, closed +) -> None: + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, _ = armed_runner( + [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + completed(), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + delete_result, + remaining, + completed(stdout=json.dumps(closed)), + ] + ) + cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[ + "ephemeral.cleanup-verified" + ] + assert cleanup.status == model.FAIL + + +def test_cleanup_fails_when_close_or_verification_commands_are_bad_json_or_failures() -> ( + None +): + api = "/api/v1/repos/atlas/titan-iac" + for scripted in ( + [completed(returncode=1), completed(), completed(), completed(returncode=1)], + [completed(stdout="bad"), completed(), completed(), completed(stdout="bad")], + [ + completed(stdout=json.dumps(pr_payload())), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ], + ): + runner, _ = armed_runner(scripted) + result = ephemeral._cleanup(runner, VANTAGE, request(), api, [42]) + assert result.status == model.FAIL + + +def test_read_only_runner_cannot_reach_push() -> None: + runner, spawn = armed_runner( + [completed(), completed(stdout=f"{HEAD}\n"), completed(), completed()], + mode=policy.READ_ONLY, + ) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert not any(call["argv"][1] == "push" for call in spawn.calls) diff --git a/testing/tests/test_hermes_handoff_evaluators.py b/testing/tests/test_hermes_handoff_evaluators.py index 2e340c7b..4de16c7e 100644 --- a/testing/tests/test_hermes_handoff_evaluators.py +++ b/testing/tests/test_hermes_handoff_evaluators.py @@ -2,14 +2,25 @@ from __future__ import annotations -from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step +import json + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) evaluators = load_handoff_module("hermes_handoff_evaluators") model = load_handoff_module("hermes_handoff_model") REVIEW = model.REVIEW ATTEMPT = model.ATTEMPT -FORBIDDEN = outcome(stderr="Error from server (Forbidden): pods is forbidden", returncode=1) +FORBIDDEN = outcome( + stderr="Error from server (Forbidden): pods is forbidden", returncode=1 +) def deny_spec() -> object: @@ -24,21 +35,27 @@ def evaluate(built, outcomes): def test_a_deny_check_passes_only_on_a_review_no_and_a_live_refusal() -> None: - evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN}) + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN} + ) assert evaluation.status == model.PASS assert evaluation.evidence == {"review": "no", "attempt": "refused"} def test_a_deny_check_fails_when_the_review_says_the_authority_exists() -> None: - evaluation = evaluate(deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN}) + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN} + ) assert evaluation.status == model.FAIL assert "reported yes" in evaluation.reason def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> None: - evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()}) + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()} + ) assert evaluation.status == model.FAIL assert "must be refused" in evaluation.reason @@ -46,7 +63,9 @@ def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> No def test_a_request_that_merely_broke_is_not_evidence_of_a_refusal() -> None: broken = outcome(stderr="connection refused", returncode=7) - evaluation = evaluate(deny_spec(), {"review": outcome(stdout="no"), "attempt": broken}) + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="no"), "attempt": broken} + ) assert evaluation.status == model.NOT_RUN assert "recognisable refusal" in evaluation.reason @@ -62,15 +81,22 @@ def test_a_deny_check_without_a_live_attempt_refuses_to_classify() -> None: assert "require a real attempt" in evaluation.reason -def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> None: +def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> ( + None +): built = spec("allowed", steps=(step("review", kind=REVIEW), step("read"))) - assert evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status == model.PASS + assert ( + evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status + == model.PASS + ) denied = evaluate(built, {"review": outcome(stdout="no"), "read": outcome()}) assert denied.status == model.FAIL and "reported no" in denied.reason - failed = evaluate(built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)}) + failed = evaluate( + built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)} + ) assert failed.status == model.FAIL and "step read failed" in failed.reason @@ -84,7 +110,9 @@ def test_names_absent_flags_exact_names_and_substrings() -> None: clean = evaluate(built, {"env": outcome(stdout="PATH\nHOME\n")}) assert clean.status == model.PASS and clean.evidence["observed_count"] == 2 - leaked = evaluate(built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")}) + leaked = evaluate( + built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")} + ) assert leaked.status == model.FAIL assert leaked.evidence["offenders"] == ["GITEA_TOKEN", "OPENAI_API_KEY"] @@ -92,10 +120,16 @@ def test_names_absent_flags_exact_names_and_substrings() -> None: assert unreadable.status == model.NOT_RUN -def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> None: +def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> ( + None +): built = spec( "names_present", - {"step": "init", "names": ("patch-web-session-activity",), "contains": ("@sha256:",)}, + { + "step": "init", + "names": ("patch-web-session-activity",), + "contains": ("@sha256:",), + }, steps=(step("init"),), ) @@ -110,14 +144,18 @@ def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty( def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None: - single = spec("stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),)) + single = spec( + "stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),) + ) assert evaluate(single, {"s": outcome(stdout=" locked \n")}).status == model.PASS mismatch = evaluate(single, {"s": outcome(stdout="unlocked")}) assert mismatch.status == model.FAIL and mismatch.evidence["observed"] == "unlocked" several = spec( - "stdout_matches", {"step": "s", "equals": ["600 root:root", "absent"]}, steps=(step("s"),) + "stdout_matches", + {"step": "s", "equals": ["600 root:root", "absent"]}, + steps=(step("s"),), ) assert evaluate(several, {"s": outcome(stdout="absent")}).status == model.PASS @@ -126,7 +164,9 @@ def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None: def test_distinct_count_separates_replica_count_from_actual_spread() -> None: built = spec( - "distinct_count", {"step": "n", "minimum": 3, "total_equals": 3}, steps=(step("n"),) + "distinct_count", + {"step": "n", "minimum": 3, "total_equals": 3}, + steps=(step("n"),), ) spread = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-06\ntitan-07\n")}) @@ -135,7 +175,11 @@ def test_distinct_count_separates_replica_count_from_actual_spread() -> None: stacked = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-05\ntitan-05\n")}) assert stacked.status == model.FAIL and "distinct values" in stacked.reason - extra = spec("distinct_count", {"step": "n", "minimum": 1, "total_equals": 2}, steps=(step("n"),)) + extra = spec( + "distinct_count", + {"step": "n", "minimum": 1, "total_equals": 2}, + steps=(step("n"),), + ) over = evaluate(extra, {"n": outcome(stdout="a\nb\nc\n")}) assert over.status == model.FAIL and "expected exactly 2" in over.reason @@ -152,16 +196,22 @@ def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None: agree = evaluate( built, - {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/b\nns/a\n")}, + { + "operator": outcome(stdout="ns/a\nns/b\n"), + "self": outcome(stdout="ns/b\nns/a\n"), + }, ) assert agree.status == model.PASS differ = evaluate( - built, {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")} + built, + {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")}, ) assert differ.status == model.FAIL and "disagree" in differ.reason - empty = evaluate(built, {"operator": outcome(stdout=""), "self": outcome(stdout="")}) + empty = evaluate( + built, {"operator": outcome(stdout=""), "self": outcome(stdout="")} + ) assert empty.status == model.NOT_RUN broken = evaluate(built, {"operator": outcome(returncode=1), "self": outcome()}) @@ -169,7 +219,9 @@ def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None: def test_steps_agree_is_the_same_rule_under_a_neutral_name() -> None: - assert evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"] + assert ( + evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"] + ) def test_lines_match_catches_a_blank_leading_column() -> None: @@ -186,7 +238,10 @@ def test_lines_match_catches_a_blank_leading_column() -> None: failed = evaluate(built, {"s": outcome(stdout=blank)}) assert failed.status == model.FAIL and failed.evidence["offenders"] - assert evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status == model.NOT_RUN + assert ( + evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status + == model.NOT_RUN + ) assert evaluate(built, {"s": outcome(returncode=1)}).status == model.NOT_RUN @@ -198,4 +253,205 @@ def test_not_armed_is_the_fixed_skip_a_default_run_must_produce() -> None: assert evaluation.status == model.NOT_RUN assert evaluation.evidence == {"armed": False} assert evaluation.reason == "ephemeral mutation mode is not armed" - assert evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed" + assert ( + evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed" + ) + + +def test_truncation_fails_closed_before_every_registered_evaluator() -> None: + """No evaluator may reinterpret a clipped required capture as PASS.""" + clipped = outcome(stdout="apparently good", truncated=True) + for name in evaluators.EVALUATORS: + built = spec(name, {"step": "s"}, steps=(step("s"),)) + evaluation = evaluate(built, {"s": clipped}) + assert evaluation.status == model.NOT_RUN, name + assert "truncated" in evaluation.reason, name + + +def test_missing_required_steps_fail_closed_inside_every_step_evaluator() -> None: + for name in evaluators.EVALUATORS: + if name == "not_armed": + continue + built = spec(name, {"step": "s"}, steps=(step("s"),)) + assert evaluate(built, {}).status == model.NOT_RUN, name + + +def test_pool_topology_requires_three_ordinals_nodes_claims_and_ready_state() -> None: + built = spec( + "pool_topology", + { + "state_step": "state", + "worker_step": "workers", + "replicas": 3, + "name": "worker", + }, + steps=(step("state"), step("workers")), + ) + state = outcome(stdout="3\t3\t3\tfalse") + workers = outcome( + stdout=( + "worker-0\tnode-a\tRunning\tTrue\tclaim-0,\n" + "worker-1\tnode-b\tRunning\tTrue\tclaim-1,\n" + "worker-2\tnode-c\tRunning\tTrue\tclaim-2,\n" + ) + ) + assert evaluate(built, {"state": state, "workers": workers}).status == model.PASS + stacked = outcome(stdout=workers.stdout.replace("node-b", "node-a")) + assert evaluate(built, {"state": state, "workers": stacked}).status == model.FAIL + assert ( + evaluate(built, {"state": outcome(stdout="3\t3"), "workers": workers}).status + == model.NOT_RUN + ) + assert ( + evaluate( + built, {"state": outcome(stdout="x\t3\t3\tfalse"), "workers": workers} + ).status + == model.NOT_RUN + ) + assert ( + evaluate( + built, {"state": outcome(stdout="3\t2\t3\tfalse"), "workers": workers} + ).status + == model.FAIL + ) + malformed = outcome(stdout="worker-0\tnode-a\n") + assert ( + evaluate(built, {"state": state, "workers": malformed}).status == model.NOT_RUN + ) + bad_worker = outcome(stdout=workers.stdout.replace("Running", "Pending", 1)) + assert evaluate(built, {"state": state, "workers": bad_worker}).status == model.FAIL + duplicate = outcome(stdout=workers.stdout.replace("worker-1", "worker-0")) + assert evaluate(built, {"state": state, "workers": duplicate}).status == model.FAIL + wrong_ordinals = outcome(stdout=workers.stdout.replace("worker-2", "worker-3")) + assert ( + evaluate(built, {"state": state, "workers": wrong_ordinals}).status + == model.FAIL + ) + + +MAIN = "1" * 40 +HEAD = "2" * 40 +BUILD = MAIN +DIGEST = "sha256:" + "4" * 64 +IMAGE = f"registry.example/hermes:git-{BUILD}-build-7@{DIGEST}" + + +def lineage_spec(): + keys = ( + "remote-main", + "reviewed-pr", + "build-source", + "deployment", + "pods", + "replicasets", + "flux", + ) + return spec( + "release_lineage", + { + "main_sha": MAIN, + "head_sha": HEAD, + "head_ref": "feature/hermes-full-handoff-acceptance", + "pr_number": 19, + "image": IMAGE, + "build_sha": BUILD, + "deployment_revision": "7", + }, + steps=tuple(step(key) for key in keys), + ) + + +def lineage_outcomes(): + pull = { + "number": 19, + "state": "open", + "draft": True, + "merged": False, + "base": {"ref": "main", "sha": MAIN}, + "head": {"ref": "feature/hermes-full-handoff-acceptance", "sha": HEAD}, + } + return { + "remote-main": outcome(stdout=f"{MAIN}\trefs/heads/main\n"), + "reviewed-pr": outcome(stdout=json.dumps(pull)), + "build-source": outcome(stdout=BUILD), + "deployment": outcome(stdout=f"5\t5\t2\t2\t7\t{IMAGE}"), + "replicasets": outcome( + stdout=f"rs-old\t6\t0\t0\t{IMAGE}\told\nrs-live\t7\t2\t2\t{IMAGE}\thash7\n" + ), + "pods": outcome( + stdout=( + f"pod-a\tRunning\tTrue\t{IMAGE}\tdocker-pullable://hermes@{DIGEST}\thash7\n" + f"pod-b\tRunning\tTrue\t{IMAGE}\tcontainerd://hermes@{DIGEST}\thash7\n" + ) + ), + "flux": outcome(stdout=f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{MAIN}"), + } + + +def test_release_lineage_atomically_binds_every_release_identity() -> None: + assert evaluate(lineage_spec(), lineage_outcomes()).status == model.PASS + + +@pytest.mark.parametrize( + ("key", "bad"), + [ + ("remote-main", ""), + ("reviewed-pr", "{}"), + ("build-source", HEAD), + ("deployment", "5\t4\t2\t2\t7\tbad"), + ("replicasets", "malformed"), + ("pods", "malformed"), + ("flux", "hermes\t9\t9\tfalse\tUnknown\tmain@sha1:" + MAIN), + ], +) +def test_release_lineage_fails_closed_on_each_malformed_or_mismatched_source( + key, bad +) -> None: + outcomes = lineage_outcomes() + outcomes[key] = outcome(stdout=bad) + assert evaluate(lineage_spec(), outcomes).status in {model.FAIL, model.NOT_RUN} + + +def test_release_lineage_rejects_structural_partial_failure_variants() -> None: + variants = [] + malformed_expect = lineage_spec() + malformed_expect = model.CheckSpec( + **{ + **malformed_expect.__dict__, + "expect": {**malformed_expect.expect, "main_sha": "bad"}, + } + ) + variants.append((malformed_expect, lineage_outcomes())) + malformed_image = lineage_spec() + malformed_image = model.CheckSpec( + **{ + **malformed_image.__dict__, + "expect": { + **malformed_image.expect, + "image": f"registry.example/hermes@{DIGEST}", + }, + } + ) + variants.append((malformed_image, lineage_outcomes())) + for key, value in ( + ("reviewed-pr", "[]"), + ("deployment", f"x\tx\t2\t2\t7\t{IMAGE}"), + ("replicasets", f"rs\t7\t1\t2\t{IMAGE}\thash7"), + ("replicasets", f"a\t7\t2\t2\t{IMAGE}\th1\nb\t7\t2\t2\t{IMAGE}\th2"), + ("replicasets", f"rs\t6\t2\t2\t{IMAGE}\thash7"), + ("pods", f"pod-a\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7"), + ( + "pods", + f"pod-a\tPending\tFalse\t{IMAGE}\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7", + ), + ( + "pods", + f"pod-a\tRunning\tTrue\tbad\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7", + ), + ("flux", f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{HEAD}"), + ): + outcomes = lineage_outcomes() + outcomes[key] = outcome(stdout=value) + variants.append((lineage_spec(), outcomes)) + for built, outcomes in variants: + assert evaluate(built, outcomes).status in {model.FAIL, model.NOT_RUN} diff --git a/testing/tests/test_hermes_handoff_exec.py b/testing/tests/test_hermes_handoff_exec.py index cd8fc7d2..c192448d 100644 --- a/testing/tests/test_hermes_handoff_exec.py +++ b/testing/tests/test_hermes_handoff_exec.py @@ -2,7 +2,11 @@ from __future__ import annotations +import hashlib import subprocess +import time + +import pytest from testing.tests.test_hermes_handoff_support import ( FakeClock, @@ -15,35 +19,48 @@ policy = load_handoff_module("hermes_handoff_policy") def completed(stdout: str = "", stderr: str = "", returncode: int = 0): - return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) def build(results: list, **kwargs) -> tuple: clock = FakeClock() spawn = FakeSpawn(results) - runner = runner_module.Runner(clock=clock, spawn=spawn, environ={}, **kwargs) + runner = runner_module.Runner( + clock=clock, + spawn=spawn, + environ={}, + attestor=lambda command, _environment: runner_module.Attestation( + command, "a" * 64 + ), + **kwargs, + ) return (runner, spawn, clock) OPERATOR = runner_module.operator_vantage() +SAFE_GET = ("kubectl", "get", "pods", "-o", "name") def test_a_successful_command_is_captured_and_screened() -> None: runner, spawn, _ = build([completed(stdout="password=a-long-enough-secret\n")]) - outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + outcome = runner.run(SAFE_GET, OPERATOR) assert outcome.ok and outcome.ran assert "[redacted]" in outcome.stdout assert outcome.returncode == 0 - assert spawn.calls[0]["argv"] == ["kubectl", "get", "pods"] + assert spawn.calls[0]["argv"] == list(SAFE_GET) assert runner.commands_run == 1 def test_a_failing_command_is_not_ok_but_still_ran() -> None: - runner, _, _ = build([completed(stderr="Error from server (Forbidden)", returncode=1)]) + runner, _, _ = build( + [completed(stderr="Error from server (Forbidden)", returncode=1)] + ) - outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + outcome = runner.run(SAFE_GET, OPERATOR) assert outcome.ran and not outcome.ok assert "Forbidden" in outcome.combined @@ -63,7 +80,7 @@ def test_a_policy_violation_never_reaches_a_subprocess() -> None: def test_a_timeout_is_recorded_rather_than_raised() -> None: runner, _, _ = build([subprocess.TimeoutExpired(cmd="kubectl", timeout=1.0)]) - outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + outcome = runner.run(SAFE_GET, OPERATOR) assert outcome.error == runner_module.TIMEOUT_ERROR assert not outcome.ran @@ -72,7 +89,7 @@ def test_a_timeout_is_recorded_rather_than_raised() -> None: def test_a_missing_binary_is_recorded_rather_than_raised() -> None: runner, _, _ = build([FileNotFoundError(2, "No such file or directory")]) - outcome = runner.run(("kubectl", "get", "pods"), OPERATOR) + outcome = runner.run(SAFE_GET, OPERATOR) assert outcome.error and outcome.error.startswith("spawn:") @@ -80,9 +97,9 @@ def test_a_missing_binary_is_recorded_rather_than_raised() -> None: def test_the_run_deadline_stops_further_probes_without_spawning() -> None: runner, spawn, clock = build([completed(stdout="first")], deadline_seconds=10.0) - assert runner.run(("kubectl", "get", "pods"), OPERATOR).ok + assert runner.run(SAFE_GET, OPERATOR).ok clock.advance(11.0) - second = runner.run(("kubectl", "get", "nodes"), OPERATOR) + second = runner.run(("kubectl", "get", "nodes", "-o", "name"), OPERATOR) assert second.error == runner_module.DEADLINE_ERROR assert len(spawn.calls) == 1 @@ -90,29 +107,42 @@ def test_the_run_deadline_stops_further_probes_without_spawning() -> None: def test_the_command_timeout_is_clamped_by_the_remaining_deadline() -> None: - runner, spawn, clock = build([completed()], command_timeout=30.0, deadline_seconds=10.0) + runner, spawn, clock = build( + [completed()], command_timeout=30.0, deadline_seconds=10.0 + ) clock.advance(6.0) - runner.run(("kubectl", "get", "pods"), OPERATOR) + runner.run(SAFE_GET, OPERATOR) assert spawn.calls[0]["timeout"] == 4.0 def test_output_is_bounded_per_command_and_can_be_raised_per_call() -> None: - runner, _, _ = build([completed(stdout="y" * 200), completed(stdout="y" * 200)], max_bytes=50) + runner, _, _ = build( + [completed(stdout="y" * 200), completed(stdout="y" * 200)], max_bytes=50 + ) - small = runner.run(("kubectl", "get", "pods"), OPERATOR) - large = runner.run(("kubectl", "get", "pods"), OPERATOR, max_bytes=4096) + small = runner.run(SAFE_GET, OPERATOR) + large = runner.run(SAFE_GET, OPERATOR, max_bytes=4096) assert small.truncated assert not large.truncated def test_the_child_environment_is_rebuilt_from_an_allowlist() -> None: - source = {"PATH": "/bin", "KUBECONFIG": "/tmp/kc", "ANTHROPIC_API_KEY": "leak", "HOME": "/root"} + source = { + "PATH": "/bin", + "KUBECONFIG": "/tmp/kc", + "ANTHROPIC_API_KEY": "leak", + "HOME": "/root", + } environment = runner_module.build_environment(OPERATOR, source) - assert environment == {"PATH": "/bin", "KUBECONFIG": "/tmp/kc", "HOME": "/root", "LC_ALL": "C"} + assert environment["PATH"] == runner_module.SAFE_PATH + assert environment["KUBECONFIG"] == "/tmp/kc" + assert environment["GITEA_BASE_URL"] == "https://scm.bstein.dev" + assert environment["GIT_CONFIG_GLOBAL"] == "/dev/null" + assert "HOME" not in environment assert "ANTHROPIC_API_KEY" not in environment @@ -127,7 +157,11 @@ def test_a_vantage_can_override_the_environment_it_needs() -> None: def test_the_operator_context_is_inserted_only_for_context_aware_tools() -> None: vantage = runner_module.operator_vantage(context="atlas-operator") - assert vantage.wrap(("kubectl", "get", "pods"))[:3] == ("kubectl", "--context", "atlas-operator") + assert vantage.wrap(("kubectl", "get", "pods"))[:3] == ( + "kubectl", + "--context", + "atlas-operator", + ) assert vantage.wrap(("git", "status")) == ("git", "status") assert vantage.wrap(()) == () @@ -136,11 +170,18 @@ def test_the_pod_vantage_execs_into_the_container_and_drops_the_context() -> Non operator = runner_module.operator_vantage(context="atlas-operator") vantage = runner_module.pod_vantage("hermes", "hermes-agent-1", "hermes", operator) - wrapped = vantage.wrap(("kubectl", "get", "pods")) + wrapped = vantage.wrap(SAFE_GET) assert wrapped[:3] == ("kubectl", "--context", "atlas-operator") assert wrapped[3] == "exec" - assert wrapped[-4:] == ("--", "kubectl", "get", "pods") + assert wrapped[-6:] == ( + "--", + "/usr/local/bin/kubectl", + "get", + "pods", + "-o", + "name", + ) assert vantage.description == "in-pod hermes/hermes-agent-1[hermes]" policy.check_argv(wrapped) @@ -168,6 +209,155 @@ def test_outcome_serialisation_screens_the_command_line() -> None: def test_combined_output_joins_only_the_streams_that_carry_text() -> None: - assert runner_module.Outcome(argv=(), vantage="x", stdout="a", stderr="b").combined == "a\nb" + assert ( + runner_module.Outcome(argv=(), vantage="x", stdout="a", stderr="b").combined + == "a\nb" + ) assert runner_module.Outcome(argv=(), vantage="x", stdout="a").combined == "a" assert runner_module.Outcome(argv=(), vantage="x").combined == "" + + +@pytest.mark.parametrize( + "kwargs", + [ + {"command_timeout": float("nan")}, + {"command_timeout": float("inf")}, + {"command_timeout": 0}, + {"deadline_seconds": -1}, + {"max_bytes": -1}, + {"max_bytes": 10**100}, + {"concurrency": 0}, + {"concurrency": 17}, + ], +) +def test_execution_bounds_reject_nonfinite_nonpositive_or_unbounded_inputs( + kwargs, +) -> None: + with pytest.raises(ValueError): + runner_module.Runner(**kwargs) + + +def test_execution_bounds_reject_boolean_and_nonnumeric_inputs() -> None: + for value in (True, "1", None): + with pytest.raises(ValueError, match="numeric"): + runner_module.Runner(command_timeout=value) + + +def test_every_reviewer_unsafe_argv_is_refused_without_spawning() -> None: + runner, spawn, _ = build([]) + commands = [ + ("kubectl", "auth", "reconcile", "-f", "x"), + ("kubectl", "config", "view", "--raw"), + ("kubectl", "get", "secret/x", "-o", "name"), + ("kubectl", "get", "--raw=/api/v1/secrets"), + ("kubectl", "delete", "pod/x", "--dry-run=client"), + ("kubectl", "delete", "pod/x", "--", "--dry-run=server"), + ("git", "fetch", "origin"), + ("git", "config", "core.sshCommand", "x"), + ("git", "-ccore.sshCommand=x", "ls-remote", "origin"), + ("helm", "get", "values", "x"), + ] + for command in commands: + assert runner.run(command, OPERATOR).error.startswith("policy:") + assert spawn.calls == [] + assert runner.commands_run == 0 + + +def test_live_capture_bounds_concurrent_stdout_and_stderr() -> None: + capture = runner_module.execute_bounded( + ( + "/usr/bin/python3", + "-c", + "import os;os.write(1,b'x'*2000000);os.write(2,b'y'*2000000)", + ), + {"LC_ALL": "C"}, + timeout=5, + max_bytes=1024, + ) + assert capture.returncode == 0 + assert capture.stdout_truncated and capture.stderr_truncated + assert len(capture.stdout) == len(capture.stderr) == 1024 + + +def test_absolute_timeout_kills_the_process_group_and_reaps_quickly() -> None: + started = time.monotonic() + capture = runner_module.execute_bounded( + ( + "/usr/bin/python3", + "-c", + "import os,signal,time;" + "p=os.fork();" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "time.sleep(30)", + ), + {"LC_ALL": "C"}, + timeout=0.15, + max_bytes=1024, + ) + assert capture.timed_out + assert time.monotonic() - started < 1.5 + + +def test_descendant_holding_pipes_is_bounded_after_the_leader_exits() -> None: + started = time.monotonic() + capture = runner_module.execute_bounded( + ( + "/usr/bin/python3", + "-c", + "import os,signal,time;" + "p=os.fork();" + "(os._exit(0) if p else None);" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "time.sleep(30)", + ), + {"LC_ALL": "C"}, + timeout=2, + max_bytes=1024, + ) + assert not capture.timed_out + assert time.monotonic() - started < 1.5 + + +def test_attestation_rejects_a_reviewer_authored_executable(tmp_path) -> None: + fake = tmp_path / "kubectl" + fake.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + fake.chmod(0o755) + with pytest.raises(OSError, match="trusted paths"): + runner_module.attest_executable(str(fake), {"PATH": str(tmp_path)}) + + +def test_attestation_checks_presence_mode_digest_and_success( + tmp_path, monkeypatch +) -> None: + fake = tmp_path / "tool" + fake.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + fake.chmod(0o755) + digest = hashlib.sha256(fake.read_bytes()).hexdigest() + monkeypatch.setitem(runner_module.EXPECTED_PATHS, "tool", {str(fake)}) + monkeypatch.setitem(runner_module.EXPECTED_SHA256, "tool", {digest}) + assert ( + runner_module.attest_executable("tool", {"PATH": str(tmp_path)}).sha256 + == digest + ) + with pytest.raises(OSError, match="unavailable"): + runner_module.attest_executable("missing", {"PATH": str(tmp_path)}) + fake.chmod(0o777) + with pytest.raises(OSError, match="unsafe ownership mode"): + runner_module.attest_executable("tool", {"PATH": str(tmp_path)}) + fake.chmod(0o755) + monkeypatch.setitem(runner_module.EXPECTED_SHA256, "tool", {"0" * 64}) + with pytest.raises(OSError, match="digest"): + runner_module.attest_executable("tool", {"PATH": str(tmp_path)}) + + +def test_runner_attestation_or_resolved_policy_failure_never_spawns() -> None: + runner, spawn, _ = build([]) + runner._attestor = lambda _command, _environment: (_ for _ in ()).throw( + OSError("no") + ) + assert runner.run(SAFE_GET, OPERATOR).error.startswith("attestation:") + runner._attestor = lambda _command, _environment: runner_module.Attestation( + "/tmp/reviewer-authored/kubectl", "a" * 64 + ) + assert runner.run(SAFE_GET, OPERATOR).error.startswith("policy:") + assert spawn.calls == [] diff --git a/testing/tests/test_hermes_handoff_json_rules.py b/testing/tests/test_hermes_handoff_json_rules.py index 9bb5c673..e74684d7 100644 --- a/testing/tests/test_hermes_handoff_json_rules.py +++ b/testing/tests/test_hermes_handoff_json_rules.py @@ -5,7 +5,14 @@ from __future__ import annotations import datetime as dt import json -from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step +import pytest + +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) evaluators = load_handoff_module("hermes_handoff_evaluators") model = load_handoff_module("hermes_handoff_model") @@ -23,9 +30,13 @@ def one(rule: str, expect: dict): def test_json_field_compares_paths_against_expected_values() -> None: - built = one("json_field", {"fields": {"authenticated": True, "transport": ["a", "b"]}}) + built = one( + "json_field", {"fields": {"authenticated": True, "transport": ["a", "b"]}} + ) - assert evaluate(built, {"authenticated": True, "transport": "b"}).status == model.PASS + assert ( + evaluate(built, {"authenticated": True, "transport": "b"}).status == model.PASS + ) wrong = evaluate(built, {"authenticated": False, "transport": "c"}) assert wrong.status == model.FAIL @@ -40,6 +51,15 @@ def test_json_field_skips_rather_than_guesses_on_unusable_output() -> None: assert evaluate(built, "not json").status == model.NOT_RUN failed = evaluators.evaluate(built, {"s": outcome(returncode=1)}) assert failed.status == model.NOT_RUN + assert evaluate(built, '{"a": 1, "a": 1}').status == model.NOT_RUN + assert evaluate(built, '{"a": NaN}').status == model.NOT_RUN + + +def test_json_equality_does_not_confuse_booleans_and_integers() -> None: + built = one("json_field", {"fields": {"enabled": True}}) + assert evaluate(built, {"enabled": 1}).status == model.FAIL + array = one("all_of_field", {"fields": {"enabled": False}}) + assert evaluate(array, [{"enabled": 0}]).status == model.FAIL def test_json_numeric_enforces_inclusive_bounds_and_rejects_non_numbers() -> None: @@ -63,20 +83,32 @@ def test_json_recent_treats_stale_evidence_as_a_failure() -> None: built = one("json_recent", {"now": NOW, "fields": {"checked_at": 3600}}) fresh = evaluate(built, {"checked_at": "2026-08-17T11:30:00Z"}) - assert fresh.status == model.PASS and fresh.evidence["checked_at"]["age_seconds"] == 1800 + assert ( + fresh.status == model.PASS + and fresh.evidence["checked_at"]["age_seconds"] == 1800 + ) stale = evaluate(built, {"checked_at": "2026-08-16T00:00:00Z"}) assert stale.status == model.FAIL and "limit 3600s" in stale.reason naive = evaluate(built, {"checked_at": "2026-08-17T11:30:00"}) - assert naive.status == model.PASS + assert naive.status == model.NOT_RUN + assert evaluate(built, {"checked_at": "2026-08-17T12:00:01Z"}).status == model.FAIL assert evaluate(built, {"checked_at": "yesterday"}).status == model.NOT_RUN assert evaluate(built, {}).status == model.NOT_RUN + invalid_clock = one("json_recent", {"now": "now", "fields": {"checked_at": 1}}) + assert ( + evaluate(invalid_clock, {"checked_at": "2026-08-17T11:30:00Z"}).status + == model.NOT_RUN + ) def test_all_of_field_selects_keys_and_reports_every_mismatch() -> None: - built = one("all_of_field", {"key_field": "number", "keys": [14, 15], "fields": {"merged": True}}) + built = one( + "all_of_field", + {"key_field": "number", "keys": [14, 15], "fields": {"merged": True}}, + ) payload = [ {"number": 14, "merged": True}, {"number": 15, "merged": False}, @@ -92,7 +124,10 @@ def test_all_of_field_selects_keys_and_reports_every_mismatch() -> None: def test_all_of_field_skips_when_an_expected_key_is_absent_or_nothing_matched() -> None: - keyed = one("all_of_field", {"key_field": "number", "keys": [14], "fields": {"merged": True}}) + keyed = one( + "all_of_field", + {"key_field": "number", "keys": [14], "fields": {"merged": True}}, + ) absent = evaluate(keyed, [{"number": 99, "merged": True}]) assert absent.status == model.NOT_RUN and "expected keys absent" in absent.reason @@ -103,7 +138,9 @@ def test_all_of_field_skips_when_an_expected_key_is_absent_or_nothing_matched() def test_all_of_field_skips_when_the_named_array_path_is_absent() -> None: built = spec( - "all_of_field", {"step": "s", "array": "items", "fields": {"a": 1}}, steps=(step("s"),) + "all_of_field", + {"step": "s", "array": "items", "fields": {"a": 1}}, + steps=(step("s"),), ) evaluation = evaluate(built, {"other": []}) @@ -124,9 +161,16 @@ def test_all_of_field_treats_an_absent_field_as_a_mismatch_not_a_crash() -> None def test_all_of_field_honours_exemptions_and_non_empty_paths() -> None: exempted = one( "all_of_field", - {"key_field": "name", "exempt_keys": ["parked"], "fields": {"suspended": False}}, + { + "key_field": "name", + "exempt_keys": ["parked"], + "fields": {"suspended": False}, + }, ) - payload = [{"name": "parked", "suspended": True}, {"name": "live", "suspended": False}] + payload = [ + {"name": "parked", "suspended": True}, + {"name": "live", "suspended": False}, + ] assert evaluate(exempted, payload).status == model.PASS non_empty = one("all_of_field", {"key_field": "id", "non_empty": ["title"]}) @@ -136,8 +180,13 @@ def test_all_of_field_honours_exemptions_and_non_empty_paths() -> None: def test_json_record_captures_named_paths_or_skips() -> None: - built = one("json_record", {"record": {"numbers": "[].number", "refs": "[].head.ref"}}) - payload = [{"number": 14, "head": {"ref": "a"}}, {"number": 15, "head": {"ref": "b"}}] + built = one( + "json_record", {"record": {"numbers": "[].number", "refs": "[].head.ref"}} + ) + payload = [ + {"number": 14, "head": {"ref": "a"}}, + {"number": 15, "head": {"ref": "b"}}, + ] evaluation = evaluate(built, payload) assert evaluation.status == model.PASS @@ -169,7 +218,11 @@ def routing_spec(**extra): FULL_LOG = routing_log( {"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}, {"ts": "2026-08-17T11:30:00Z", "model": "route/claude/opus/xhigh"}, - {"ts": "2026-08-17T11:45:00Z", "model": "worker/codex/sol/xhigh", "fallback_reason": "unavailable"}, + { + "ts": "2026-08-17T11:45:00Z", + "model": "worker/codex/sol/xhigh", + "fallback_reason": "unavailable", + }, {"ts": "2026-08-17T11:50:00Z", "model": "qwen2.5:14b", "tier": "classifier"}, ) @@ -186,7 +239,9 @@ def test_routing_evidence_summarises_lanes_providers_efforts_and_fallbacks() -> def test_routing_evidence_names_exactly_what_the_window_did_not_show() -> None: - partial = routing_log({"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}) + partial = routing_log( + {"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"} + ) evaluation = evaluate(routing_spec(), partial) @@ -197,28 +252,74 @@ def test_routing_evidence_names_exactly_what_the_window_did_not_show() -> None: assert "fallback:none-observed" in evaluation.reason -def test_routing_evidence_tolerates_a_clipped_tail_but_not_an_empty_one() -> None: +def test_routing_evidence_rejects_a_clipped_tail_and_empty_input() -> None: clipped = '{"ts": "2026-08-17T11:0\n' + FULL_LOG - assert evaluate(routing_spec(), clipped).status == model.PASS + assert evaluate(routing_spec(), clipped).status == model.NOT_RUN assert evaluate(routing_spec(), "").status == model.NOT_RUN assert evaluate(routing_spec(), "garbage\n").status == model.NOT_RUN def test_routing_evidence_can_require_freshness() -> None: fresh = evaluate(routing_spec(max_age_seconds=3600, now=NOW), FULL_LOG) - assert fresh.status == model.PASS and fresh.evidence["age_seconds"] == 600 + assert fresh.status == model.PASS + assert max(fresh.evidence["age_seconds"].values()) <= 3600 stale = evaluate( routing_spec(max_age_seconds=60, now=NOW + dt.timedelta(days=2)), FULL_LOG ) assert stale.status == model.FAIL and "freshness" in stale.reason - undated = routing_log({"model": "route/codex/sol/high"}, {"model": "route/claude/opus/xhigh"}) + undated = routing_log( + {"model": "route/codex/sol/high"}, {"model": "route/claude/opus/xhigh"} + ) unknown = evaluate(routing_spec(max_age_seconds=60, now=NOW), undated) assert unknown.status == model.NOT_RUN +def test_routing_freshness_is_required_per_provider_not_only_globally() -> None: + rows = routing_log( + {"ts": "2026-08-15T11:00:00Z", "model": "route/claude/opus/high"}, + {"ts": "2026-08-17T11:59:00Z", "model": "route/codex/sol/xhigh"}, + { + "ts": "2026-08-17T11:59:00Z", + "model": "worker/codex/sol/high", + "fallback_reason": "down", + }, + ) + evaluation = evaluate(routing_spec(max_age_seconds=3600, now=NOW), rows) + assert evaluation.status == model.FAIL + assert "freshness:provider:claude" in evaluation.reason + missing_dimension = evaluate( + routing_spec(max_age_seconds=3600, now=NOW), + routing_log( + {"ts": "2026-08-17T11:59:00Z", "model": "route/codex/sol/high"}, + { + "ts": "2026-08-17T11:59:00Z", + "model": "worker/codex/sol/xhigh", + "fallback_reason": "down", + }, + ), + ) + assert missing_dimension.status == model.FAIL + + +@pytest.mark.parametrize( + "body", + [ + '["not-an-object"]', + '{"ts":"2026-08-17T12:00:01Z","model":"route/codex/sol/high"}', + '{"ts":"2026-08-17T11:00:00Z"}', + '{"ts":"2026-08-17T11:00:00Z","model":"bad"}', + '{"ts":"2026-08-17T11:00:00Z","model":"route/codex/sol/high","fallback_reason":0}', + ], +) +def test_routing_rows_fail_closed_on_structural_defects(body: str) -> None: + assert evaluate(routing_spec(now=NOW), body).status in {model.FAIL, model.NOT_RUN} + invalid_clock = routing_spec(now="now") + assert evaluate(invalid_clock, FULL_LOG).status == model.NOT_RUN + + def flux_spec(*expected: str): return spec( "flux_health", @@ -229,12 +330,14 @@ def flux_spec(*expected: str): def test_flux_health_separates_parked_objects_from_broken_ones() -> None: rows = ( - "flux-system/core\tfalse\tReady=True,Healthy=True,\n" - "flux-system/parked\ttrue\tReady=False,\n" - "flux-system/broken\tfalse\tReady=False,Healthy=False,\n" + "flux-system/core\t1\t1\tfalse\tReady=True,Healthy=True,\n" + "flux-system/parked\t1\t1\ttrue\tReady=False,\n" + "flux-system/broken\t1\t1\tfalse\tReady=False,Healthy=False,\n" ) - evaluation = evaluators.evaluate(flux_spec("flux-system/parked"), {"s": outcome(stdout=rows)}) + evaluation = evaluators.evaluate( + flux_spec("flux-system/parked"), {"s": outcome(stdout=rows)} + ) assert evaluation.status == model.FAIL assert evaluation.evidence["unhealthy"] == ["flux-system/broken"] @@ -243,7 +346,7 @@ def test_flux_health_separates_parked_objects_from_broken_ones() -> None: def test_flux_health_flags_a_suspension_nobody_declared() -> None: - rows = "flux-system/core\tfalse\tReady=True,\nflux-system/parked\ttrue\tReady=False,\n" + rows = "flux-system/core\t1\t1\tfalse\tReady=True,\nflux-system/parked\t1\t1\ttrue\tReady=False,\n" evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) @@ -251,17 +354,16 @@ def test_flux_health_flags_a_suspension_nobody_declared() -> None: assert evaluation.evidence["unexpected_suspensions"] == ["flux-system/parked"] -def test_flux_health_does_not_call_an_in_flight_reconcile_unhealthy() -> None: - """Flux drops Ready to False for the duration of a reconcile.""" +def test_flux_health_fails_closed_on_in_flight_or_unknown_conditions() -> None: rows = ( - "flux-system/reconciling\tfalse\tReconciling=True,Ready=False,\n" - "flux-system/degraded\tfalse\tReady=False,Healthy=True,\n" - "flux-system/waiting\t\tReady=Unknown,\n" + "flux-system/reconciling\t1\t1\tfalse\tReconciling=True,Ready=False,\n" + "flux-system/degraded\t1\t1\tfalse\tReady=False,Healthy=True,\n" + "flux-system/waiting\t1\t1\tfalse\tReady=Unknown,\n" ) evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) - assert evaluation.status == model.PASS + assert evaluation.status == model.FAIL assert evaluation.evidence["total"] == 3 @@ -269,11 +371,43 @@ def test_blank_rows_in_a_projection_are_ignored_by_both_log_parsers() -> None: padded = "\n\n" + FULL_LOG + "\n\n" assert evaluate(routing_spec(), padded).status == model.PASS - rows = "\n\nflux-system/core\tfalse\tReady=True,\n\n" + rows = "\n\nflux-system/core\t1\t1\tfalse\tReady=True,\n\n" evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) assert evaluation.status == model.PASS and evaluation.evidence["total"] == 1 def test_flux_health_skips_on_an_empty_or_failed_projection() -> None: - assert evaluators.evaluate(flux_spec(), {"s": outcome(stdout="")}).status == model.NOT_RUN - assert evaluators.evaluate(flux_spec(), {"s": outcome(returncode=1)}).status == model.NOT_RUN + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(stdout="")}).status + == model.NOT_RUN + ) + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(returncode=1)}).status + == model.NOT_RUN + ) + malformed = "flux-system/core\t1\t0\tfalse\tReady=True,\n" + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(stdout=malformed)}).status + == model.FAIL + ) + + +@pytest.mark.parametrize( + "row", + [ + "too\tfew", + "BAD NAME\t1\t1\tfalse\tReady=True,", + "flux-system/core\tx\t1\tfalse\tReady=True,", + "flux-system/core\t1\t1\tmaybe\tReady=True,", + "flux-system/core\t1\t1\tfalse\tbroken,", + "flux-system/core\t1\t1\tfalse\tReady=Yes,", + "flux-system/core\t1\t1\tfalse\tReady=True,Ready=True,", + ], +) +def test_flux_rows_reject_malformed_columns_names_generations_and_conditions( + row: str, +) -> None: + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(stdout=row)}).status + == model.NOT_RUN + ) diff --git a/testing/tests/test_hermes_handoff_model.py b/testing/tests/test_hermes_handoff_model.py index 37c3f9b9..e3c46489 100644 --- a/testing/tests/test_hermes_handoff_model.py +++ b/testing/tests/test_hermes_handoff_model.py @@ -6,12 +6,22 @@ import datetime as dt import pytest -from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) model = load_handoff_module("hermes_handoff_model") -def result(status: str, identifier: str = "test.check", mandatory: bool = True, group: str = "g"): +def result( + status: str, + identifier: str = "test.check", + mandatory: bool = True, + group: str = "g", +): return model.CheckResult( spec=model.CheckSpec( id=identifier, title="t", group=group, rule="allowed", mandatory=mandatory @@ -53,14 +63,22 @@ def test_a_mandatory_skip_is_a_no_go_exactly_like_a_failure() -> None: def test_a_harness_error_is_a_no_go_even_when_every_check_passed() -> None: - clean = report(result(model.PASS), errors=("the two vantages are the same principal",)) + clean = report( + result(model.PASS), errors=("the two vantages are the same principal",) + ) assert clean.decision == model.NO_GO -def test_a_zero_state_report_is_a_go_only_because_nothing_was_mandatory() -> None: - """An empty catalog has nothing blocking; the harness errors are what stop it.""" - assert report().decision == model.GO +def test_a_zero_state_or_unknown_status_report_is_no_go() -> None: + """Absence or an unrecognised classification can never round up to GO.""" + assert report().decision == model.NO_GO assert report(errors=("no checks were built",)).decision == model.NO_GO + unknown = result("MAYBE") + assert unknown.blocking is True + built = report(unknown) + assert built.decision == model.NO_GO + assert built.counts == dict.fromkeys(model.STATUSES, 0) + assert "groups:" in report(result(model.PASS)).render_summary() def test_blocking_results_are_ordered_failures_before_skips() -> None: @@ -93,7 +111,9 @@ def test_counts_and_group_summary_tally_every_status() -> None: def test_the_serialised_report_carries_the_verdict_and_its_reasons() -> None: built = report(result(model.FAIL, "x.fail"), result(model.PASS, "y.pass")) built.vantages = [ - model.VantageRecord(name="operator", description="d", identity="u", available=True) + model.VantageRecord( + name="operator", description="d", identity="u", available=True + ) ] payload = built.as_dict() @@ -138,7 +158,11 @@ def test_rationale_is_carried_only_when_a_check_has_one() -> None: def test_distinct_vantages_are_reported_in_first_use_order() -> None: built = spec( "allowed", - steps=(step("a", vantage="operator"), step("b", vantage="self"), step("c", vantage="operator")), + steps=( + step("a", vantage="operator"), + step("b", vantage="self"), + step("c", vantage="operator"), + ), ) assert built.vantages == ("operator", "self") @@ -146,7 +170,9 @@ def test_distinct_vantages_are_reported_in_first_use_order() -> None: def test_the_human_summary_names_the_verdict_vantages_and_blockers() -> None: built = report(result(model.FAIL, "x.fail", group="alpha"), errors=("collision",)) built.vantages = [ - model.VantageRecord(name="operator", description="d", identity="u", available=True), + model.VantageRecord( + name="operator", description="d", identity="u", available=True + ), model.VantageRecord(name="self", description="d", available=False), ] diff --git a/testing/tests/test_hermes_handoff_mutation_gate.py b/testing/tests/test_hermes_handoff_mutation_gate.py new file mode 100644 index 00000000..1e84cfb8 --- /dev/null +++ b/testing/tests/test_hermes_handoff_mutation_gate.py @@ -0,0 +1,51 @@ +"""Tests for the deterministic Hermes handoff mutation gate.""" + +from __future__ import annotations + +from pathlib import Path + +from testing.quality_handoff_mutation import Mutation, MutationReport, main, run_gate + + +ROOT = Path(__file__).parents[2] + + +def test_every_security_mutant_is_killed() -> None: + report = run_gate(ROOT) + assert report.issues == () + assert report.score == 100.0 + assert report.killed == report.total >= 9 + + +def test_invalid_anchor_baseline_syntax_and_survivor_are_reported() -> None: + path = "hermes_handoff_model.py" + cases = ( + Mutation("missing", path, "not present", "x", "pass"), + Mutation( + "baseline", + path, + "SCHEMA_VERSION = 1", + "SCHEMA_VERSION = 2", + "raise SystemExit(1)", + ), + Mutation("syntax", path, "SCHEMA_VERSION = 1", "if", "pass"), + Mutation("survivor", path, "SCHEMA_VERSION = 1", "SCHEMA_VERSION = 2", "pass"), + ) + report = run_gate(ROOT, cases) + assert report.killed == 0 + assert len(report.issues) == 4 + assert report.score == 0.0 + + +def test_main_reports_a_perfect_score(capsys) -> None: + assert main() == 0 + assert "9/9 (100.00%)" in capsys.readouterr().out + + +def test_main_reports_mutation_issues(monkeypatch, capsys) -> None: + monkeypatch.setattr( + "testing.quality_handoff_mutation.run_gate", + lambda _root: MutationReport(1, 0, ("survivor",)), + ) + assert main() == 1 + assert "survivor" in capsys.readouterr().out diff --git a/testing/tests/test_hermes_handoff_policy.py b/testing/tests/test_hermes_handoff_policy.py index 13fdc8b4..90cef791 100644 --- a/testing/tests/test_hermes_handoff_policy.py +++ b/testing/tests/test_hermes_handoff_policy.py @@ -16,16 +16,20 @@ def check(*argv: str, mode: str | None = None, inner: bool = False) -> None: @pytest.mark.parametrize( "argv", [ - ("kubectl", "--namespace", "hermes", "get", "pods"), + ("kubectl", "--namespace", "hermes", "get", "pods", "-o", "name"), ("kubectl", "auth", "can-i", "get", "secrets", "--all-namespaces"), - ("kubectl", "get", "secrets", "--all-namespaces", "-o", "name"), - ("kubectl", "get", "secrets", "--output=name"), - ("kubectl", "logs", "deploy/x", "--tail", "1"), - ("kubectl", "--context", "atlas", "get", "nodes", "-o", "json"), + ("kubectl", "get", "nodes", "-o", "name"), + ( + "kubectl", + "get", + "pods", + "-o", + policy.projection("jsonpath={.status.phase}"), + ), ("flux", "get", "kustomizations", "--all-namespaces"), ("helm", "list", "--all-namespaces"), ("git", "merge-base", "--is-ancestor", "a", "b"), - ("git", "config", "--get", "remote.origin.url"), + ("git", "ls-remote", "origin", "refs/heads/main"), ("hermes", "kanban", "list", "--json"), ("hermes", "sessions", "list", "--source", "telegram"), ("hermes", "status"), @@ -37,40 +41,50 @@ def test_read_only_commands_are_permitted(argv: tuple[str, ...]) -> None: @pytest.mark.parametrize( - ("argv", "fragment"), + "argv", [ - ((), "empty command"), - (("vault", "read", "kv/x"), "forbidden binary"), - (("curl", "https://example.dev"), "forbidden binary"), - (("rm", "-rf", "/"), "allowlist"), - (("kubectl", "get", "pods", "--token", "abc"), "forbidden argument"), - (("kubectl", "get", "pods", "--as=system:admin"), "forbidden argument"), - (("kubectl", "get", "secrets", "-A"), "-o name"), - (("kubectl", "describe", "secrets", "x"), "-o name"), - (("kubectl", "delete", "-n", "get", "secrets"), "requires a dry run"), - (("kubectl", "port-forward", "svc/x", "80"), "outside the read-only boundary"), - (("kubectl", "exec", "pod"), "-- command boundary"), - (("kubectl", "exec", "pod", "--"), "inner command"), - (("kubectl",), "outside the read-only boundary"), - (("flux", "reconcile", "kustomization", "x"), "outside the read-only boundary"), - (("flux",), "requires a subcommand"), - (("helm", "upgrade", "x"), "outside the read-only boundary"), - (("git", "commit", "-m", "x"), "not permitted"), - (("git", "push", "origin", "HEAD:refs/heads/x"), "not permitted"), - (("git", "config", "user.name", "x"), "may only read"), - (("hermes", "kanban", "complete", "t_1"), "outside the read-only boundary"), - (("hermes",), "requires a subcommand"), - ((policy.GITEA_CLIENT, "POST", "/api/v1/x"), "not permitted"), - ((policy.GITEA_CLIENT, "GET"), "method and a path"), - (("sh", "-c", "rm -rf /"), "frozen template"), - (("sh", "echo"), "frozen template"), - (("git", "log", "/etc/shadow"), "credential path"), + (), + ("vault", "read", "kv/x"), + ("curl", "https://example.dev"), + ("rm", "-rf", "/"), + ("kubectl", "get", "pods", "--token", "abc"), + ("kubectl", "get", "pods"), + ("kubectl", "get", "pods", "--as=system:admin"), + ("kubectl", "get", "secrets", "-o", "name"), + ("kubectl", "get", "pods,secrets/x", "-o", "name"), + ("kubectl", "describe", "pod/x"), + ("kubectl", "config", "view", "--raw"), + ("kubectl", "auth", "reconcile", "-f", "x"), + ("kubectl", "delete", "pod/x", "--dry-run=client"), + ("kubectl", "delete", "pod/x", "--", "--dry-run=server"), + ("kubectl", "get", "pods", "-o=json"), + ("kubectl", "get", "pods", "-o", "jsonpath={.spec.containers[*].env[*].value}"), + ("kubectl", "port-forward", "svc/x", "80"), + ("kubectl", "exec", "pod"), + ("kubectl", "exec", "pod", "--"), + ("kubectl",), + ("flux", "reconcile", "kustomization", "x"), + ("flux",), + ("helm", "upgrade", "x"), + ("helm", "get", "values", "x"), + ("git", "commit", "-m", "x"), + ("git", "push", "origin", "HEAD:refs/heads/x"), + ("git", "fetch", "origin"), + ("git", "config", "core.sshCommand", "x"), + ("git", "-ccore.sshCommand=x", "ls-remote", "origin"), + ("git", "--git-dir=/tmp/x", "status"), + ("hermes", "kanban", "complete", "t_1"), + ("hermes",), + (policy.GITEA_CLIENT, "POST", "/api/v1/x"), + (policy.GITEA_CLIENT, "GET"), + ("sh", "-c", "rm -rf /"), + ("sh", "echo"), + ("git", "log", "/etc/shadow"), ], ) -def test_unsafe_commands_are_refused(argv: tuple[str, ...], fragment: str) -> None: - with pytest.raises(policy.PolicyError) as caught: +def test_unsafe_commands_are_refused(argv: tuple[str, ...]) -> None: + with pytest.raises(policy.PolicyError): check(*argv) - assert fragment in str(caught.value) def test_unknown_mode_is_refused() -> None: @@ -78,47 +92,128 @@ def test_unknown_mode_is_refused() -> None: policy.check_argv(("kubectl", "get", "pods"), "whatever") -def test_dry_run_reaches_mutating_subcommands_without_writing() -> None: - check("kubectl", "--namespace", "hermes", "patch", "deploy/x", "--dry-run=server") - check("kubectl", "create", "configmap", "probe", "--dry-run=server") +def test_dry_run_mutation_commands_are_never_constructible() -> None: + with pytest.raises(policy.PolicyError, match="dry-run"): + check("kubectl", "patch", "deploy/x", "--dry-run=server") def test_a_resource_named_after_a_subcommand_cannot_smuggle_a_delete() -> None: """`-n get` must not make a delete look like a read.""" - with pytest.raises(policy.PolicyError, match="requires a dry run"): + with pytest.raises(policy.PolicyError): check("kubectl", "delete", "-n", "get", "configmap", "x") def test_impersonation_is_operator_forbidden_and_in_pod_permitted() -> None: with pytest.raises(policy.PolicyError, match="forbidden argument"): check("kubectl", "get", "namespaces", "--as", "system:admin") - check("kubectl", "get", "namespaces", "--as", "system:admin", inner=True) + check( + "kubectl", + "get", + "namespaces", + "-o", + "name", + "--as", + "system:admin", + inner=True, + ) def test_exec_validates_its_inner_command_and_tolerates_impersonation_inside() -> None: - inner = policy.shell("path_readable", path="/tmp") - check("kubectl", "exec", "--namespace", "hermes", "pod", "--", *inner) - check("kubectl", "exec", "pod", "--", "kubectl", "get", "ns", "--as", "system:admin") - with pytest.raises(policy.PolicyError, match="allowlist"): + script = policy.render_shell("env_names") + policy._RENDERED.add(script) + check( + "kubectl", "exec", "--namespace", "hermes", "pod", "--", "/bin/sh", "-c", script + ) + check( + "kubectl", + "exec", + "pod", + "--", + "/usr/local/bin/kubectl", + "get", + "ns", + "-o", + "name", + "--as", + "system:admin", + ) + with pytest.raises(policy.PolicyError): check("kubectl", "exec", "pod", "--", "bash", "-c", "id") -def test_exec_of_a_shell_template_naming_a_secret_path_is_permitted() -> None: - """The path scan stops at `--`; the tail is a reviewed template, not free text.""" - inner = policy.shell("account_lock_state", account="hermes-agent", path="/host-etc/shadow") - check("kubectl", "exec", "--namespace", "hermes", "pod", "--", *inner) +def test_shell_templates_reject_credential_roots() -> None: + with pytest.raises(policy.PolicyError, match="credential paths"): + policy.shell("json_fields", path="/runtime-access/token", fields="state") + with pytest.raises(policy.PolicyError, match="credential-bearing"): + policy.shell("json_fields", path="/tmp/health.json", fields="state,token") def test_armed_mode_opens_exactly_the_mutations_it_should() -> None: - check("git", "push", "origin", "HEAD:refs/heads/x", mode=policy.ARMED) - check(policy.GITEA_CLIENT, "POST", "/api/v1/x", mode=policy.ARMED) - check(policy.GITEA_CLIENT, "DELETE", "/api/v1/x", mode=policy.ARMED) + ref = "ephemeral/hermes-handoff-acceptance/acceptance1" + policy.arm_ephemeral_policy(ref) + policy.register_ephemeral_pull(9) + check("git", "push", "origin", f"HEAD:refs/heads/{ref}", mode=policy.ARMED) + root = "/api/v1/repos/atlas/titan-iac" + check( + policy.GITEA_CLIENT, + "POST", + f"{root}/pulls", + "--field", + "title=WIP: Hermes handoff acceptance ephemeral probe", + "--field", + f"head={ref}", + "--field", + "base=main", + "--field", + "body=Ephemeral acceptance probe. Closed and deleted by the harness.", + mode=policy.ARMED, + ) + check( + policy.GITEA_CLIENT, + "PATCH", + f"{root}/pulls/9", + "--field", + "state=closed", + mode=policy.ARMED, + ) + check(policy.GITEA_CLIENT, "DELETE", f"{root}/branches/{ref}", mode=policy.ARMED) with pytest.raises(policy.PolicyError, match="may not force"): - check("git", "push", "--force", "origin", "x", mode=policy.ARMED) + check( + "git", + "push", + "--force", + "origin", + f"HEAD:refs/heads/{ref}", + mode=policy.ARMED, + ) with pytest.raises(policy.PolicyError, match="may not force"): check("git", "push", "--all", "origin", mode=policy.ARMED) with pytest.raises(policy.PolicyError, match="not permitted"): - check(policy.GITEA_CLIENT, "PUT", "/api/v1/x", mode=policy.ARMED) + check(policy.GITEA_CLIENT, "PUT", f"{root}/pulls/9", mode=policy.ARMED) + + +def test_arming_and_projection_registration_fail_closed() -> None: + with pytest.raises(policy.PolicyError, match="exact ephemeral ref"): + policy.arm_ephemeral_policy("main") + policy._ARMED_REF = "" + with pytest.raises(policy.PolicyError, match="registration is malformed"): + policy.register_ephemeral_pull(9) + policy.arm_ephemeral_policy("ephemeral/hermes-handoff-acceptance/acceptance1") + + for candidate in ( + "wide", + "jsonpath={.status.token}", + "jsonpath={.metadata.annotations}", + "jsonpath={.metadata.labels.other}", + "jsonpath={.spec.containers[*].env[*].value}", + ): + with pytest.raises(policy.PolicyError): + policy.projection(candidate) + + with pytest.raises(policy.PolicyError): + check("/usr/bin/awk", "x") + with pytest.raises(policy.PolicyError): + check("git", "--exec-path=/tmp", "status") def test_every_frozen_template_renders_and_is_accepted() -> None: @@ -141,11 +236,11 @@ def test_template_rendering_rejects_unknown_names_and_unsafe_parameters() -> Non with pytest.raises(policy.PolicyError, match="unknown shell template"): policy.render_shell("nope") with pytest.raises(policy.PolicyError, match="expects parameters"): - policy.render_shell("path_mode") + policy.render_shell("json_fields") with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): - policy.render_shell("path_mode", path="/tmp; rm -rf /") + policy.render_shell("json_fields", path="/tmp;rm", fields="state") with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): - policy.render_shell("path_mode", path="$(id)") + policy.render_shell("json_fields", path="$(id)", fields="state") def test_positionals_skip_flags_and_stop_at_the_boundary() -> None: @@ -155,5 +250,70 @@ def test_positionals_skip_flags_and_stop_at_the_boundary() -> None: def test_a_subcommand_must_precede_its_resource_arguments() -> None: - with pytest.raises(policy.PolicyError, match="must precede"): + with pytest.raises(policy.PolicyError): check("kubectl", "pods", "get") + + +@pytest.mark.parametrize( + "argv", + [ + ("/usr/bin/../../tmp/kubectl", "get", "pods"), + ("/tmp/reviewer/kubectl", "get", "pods"), + ("kubectl", "get", "pods", "-o", "go-template=x"), + ("kubectl", "get", "pods", "-o", "jsonpath={.status.token}"), + ("kubectl", "get", "pods", "-o", "jsonpath={.metadata.annotations}"), + ("kubectl", "get", "pods", "-o", "jsonpath={.spec.containers[*].env[*].value}"), + ("git", "-C", "/tmp", "status"), + ("git", "--work-tree=/tmp", "status"), + (policy.GITEA_CLIENT, "GET", "not-an-api-path"), + (policy.GITEA_CLIENT, "GET", "/api/v1/repos/atlas/other"), + ], +) +def test_provenance_projection_and_repository_bypasses_are_rejected(argv) -> None: + with pytest.raises(policy.PolicyError): + check(*argv) + + +def test_armed_writes_are_pinned_to_exact_remote_ref_and_endpoints() -> None: + ref = "ephemeral/hermes-handoff-acceptance/acceptance1" + policy.arm_ephemeral_policy(ref) + policy.register_ephemeral_pull(9) + with pytest.raises(policy.PolicyError, match="pinned to origin"): + check( + "git", + "push", + "upstream", + "HEAD:refs/heads/ephemeral/hermes-handoff-acceptance/acceptance1", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError, match="preflighted ephemeral ref"): + check("git", "push", "origin", "HEAD:refs/heads/other", mode=policy.ARMED) + with pytest.raises(policy.PolicyError): + check( + policy.GITEA_CLIENT, + "DELETE", + "/api/v1/repos/atlas/titan-iac", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError): + check( + policy.GITEA_CLIENT, + "PATCH", + "/api/v1/repos/atlas/titan-iac/pulls/8", + "--field", + "state=closed", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError): + check( + policy.GITEA_CLIENT, + "POST", + "/api/v1/repos/atlas/titan-iac/pulls", + "--field", + "title=WIP: wrong branch", + "--field", + "head=main", + "--field", + "base=main", + mode=policy.ARMED, + ) diff --git a/testing/tests/test_hermes_handoff_redaction.py b/testing/tests/test_hermes_handoff_redaction.py index a472d82b..78fe19ec 100644 --- a/testing/tests/test_hermes_handoff_redaction.py +++ b/testing/tests/test_hermes_handoff_redaction.py @@ -63,7 +63,10 @@ def test_screened_json_is_still_parsable_json() -> None: def test_high_entropy_runs_are_screened_but_hex_digests_are_not() -> None: - assert redaction.redact("Zm9vYmFyYmF6cXV4-Quux_1234567890ABCDEFghij") != redaction.REDACTED + assert ( + redaction.redact("Zm9vYmFyYmF6cXV4-Quux_1234567890ABCDEFghij") + != redaction.REDACTED + ) long_mixed = "aB3" + "x9Y2z_" * 10 assert redaction.redact(long_mixed) == redaction.REDACTED assert redaction.redact("f" * 64) == "f" * 64 @@ -108,8 +111,8 @@ def test_value_shape_gates_screening(value: str, possible: bool) -> None: def test_bound_truncates_to_a_byte_budget_and_reports_it() -> None: text, truncated = redaction.bound("x" * 100, max_bytes=10) assert truncated - assert text.startswith("x" * 10) - assert redaction.TRUNCATION_NOTE in text + assert len(text.encode()) <= 10 + assert text == redaction.TRUNCATION_NOTE[:10] text, truncated = redaction.bound("short", max_bytes=64) assert (text, truncated) == ("short", False) @@ -123,17 +126,25 @@ def test_bound_with_no_budget_keeps_nothing() -> None: def test_bound_never_splits_a_multibyte_character() -> None: text, truncated = redaction.bound("é" * 10, max_bytes=5) assert truncated - assert text.startswith("éé") + assert len(text.encode()) <= 5 -def test_safe_text_bounds_before_it_screens() -> None: - """Bounding first is what stops a huge dump from outrunning the screen.""" +def test_safe_text_screens_before_it_bounds() -> None: + """A boundary must never preserve a useful credential prefix.""" payload = "password=" + "a" * 200 text, truncated = redaction.safe_text(payload, max_bytes=20) - assert truncated + assert not truncated assert redaction.REDACTED in text +def test_no_byte_budget_exposes_a_credential_prefix() -> None: + payload = "password=A9secret-prefix-that-must-never-survive" + for budget in range(1, 64): + screened, _ = redaction.safe_text(payload, max_bytes=budget) + assert "A9" not in screened + assert len(screened.encode()) <= budget + + def test_scrub_walks_nested_structures_and_keeps_useful_types() -> None: scrubbed = redaction.scrub( { @@ -152,6 +163,25 @@ def test_scrub_walks_nested_structures_and_keeps_useful_types() -> None: assert scrubbed["note"] == ["tuple", "values"] +def test_scrub_screens_short_numeric_nested_values_and_mapping_keys() -> None: + scrubbed = redaction.scrub( + { + "token": 1234, + "credentials": {"x": "y"}, + "ghp_ABCDEF1234567890": "mapping-key", + "private_key": "x", + } + ) + assert scrubbed["token"] == redaction.REDACTED + assert scrubbed["credentials"] == redaction.REDACTED + assert scrubbed["private_key"] == redaction.REDACTED + assert not any("ghp_" in key for key in scrubbed) + + +def test_unterminated_private_key_blocks_are_screened() -> None: + assert "PRIVATE KEY" not in redaction.redact("-----BEGIN PRIVATE KEY-----\nabc") + + def test_screening_is_idempotent() -> None: once = redaction.redact("api_key=AKIAIOSFODNN7EXAMPLE") assert redaction.redact(once) == once diff --git a/testing/tests/test_hermes_handoff_rules.py b/testing/tests/test_hermes_handoff_rules.py index 11f09840..03d5785f 100644 --- a/testing/tests/test_hermes_handoff_rules.py +++ b/testing/tests/test_hermes_handoff_rules.py @@ -4,7 +4,12 @@ from __future__ import annotations import pytest -from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome, spec, step +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) rules = load_handoff_module("hermes_handoff_rules") model = load_handoff_module("hermes_handoff_model") @@ -33,15 +38,18 @@ def test_dotted_paths_index_and_map(path: str, expected) -> None: assert rules.dotted(PAYLOAD, path) == expected -@pytest.mark.parametrize("path", ["missing", "a.missing", "a.b[9].c", "flag[0]", "flag.b"]) +@pytest.mark.parametrize( + "path", ["missing", "a.missing", "a.b[9].c", "flag[0]", "flag.b"] +) def test_an_absent_path_raises_rather_than_returning_none(path: str) -> None: with pytest.raises(KeyError): rules.dotted(PAYLOAD, path) -def test_parse_json_tolerates_control_characters_but_not_empty_output() -> None: - """Kanban bodies routinely carry raw newlines; that is not a broken read.""" - assert rules.parse_json(outcome(stdout='{"title": "a\nb"}'))["title"] == "a\nb" +def test_parse_json_rejects_raw_control_characters_and_empty_output() -> None: + """Only RFC-compliant, nonempty JSON is accepted as release evidence.""" + with pytest.raises(ValueError): + rules.parse_json(outcome(stdout='{"title": "a\nb"}')) with pytest.raises(ValueError, match="empty output"): rules.parse_json(outcome(stdout=" ")) with pytest.raises(ValueError): @@ -59,10 +67,17 @@ def test_missing_steps_reports_the_first_step_that_produced_nothing() -> None: assert never_run.status == model.NOT_RUN assert "never executed" in never_run.reason - errored = rules.missing_steps(built, {"a": outcome(error="timeout"), "b": outcome()}) + errored = rules.missing_steps( + built, {"a": outcome(error="timeout"), "b": outcome()} + ) assert errored.status == model.NOT_RUN assert "did not run" in errored.reason + truncated = rules.missing_steps( + built, {"a": outcome(truncated=True), "b": outcome()} + ) + assert truncated.status == model.NOT_RUN and "truncated" in truncated.reason + assert rules.missing_steps(built, {"a": outcome(), "b": outcome()}) is None @@ -81,7 +96,9 @@ def test_an_optional_step_may_be_absent() -> None: ("", 0, False), ], ) -def test_denial_detection_needs_a_refusal_not_merely_a_failure(stderr, returncode, denied) -> None: +def test_denial_detection_needs_a_refusal_not_merely_a_failure( + stderr, returncode, denied +) -> None: assert rules.looks_denied(outcome(stderr=stderr, returncode=returncode)) is denied @@ -105,7 +122,10 @@ def test_parsed_step_returns_the_payload_or_the_skip_to_report() -> None: assert failed.status == model.NOT_RUN and "failed" in failed.reason unparsable = rules.parsed_step(built, {"s": outcome(stdout="nope")}) - assert unparsable.status == model.NOT_RUN and "did not return JSON" in unparsable.reason + assert ( + unparsable.status == model.NOT_RUN + and "did not return JSON" in unparsable.reason + ) def test_an_unknown_rule_is_not_run_rather_than_a_crash() -> None: @@ -128,6 +148,17 @@ def test_an_evaluator_that_raises_is_not_run_rather_than_a_pass() -> None: del rules.EVALUATORS["explodes-for-test"] +def test_an_evaluator_returning_an_unknown_status_is_not_run() -> None: + @rules.evaluator("unknown-status-for-test") + def _unknown(_spec, _outcomes): + return rules.Evaluation("MAYBE", "bad") + + evaluation = rules.evaluate(spec("unknown-status-for-test"), {}) + assert evaluation.status == model.NOT_RUN + assert "unknown status" in evaluation.reason + del rules.EVALUATORS["unknown-status-for-test"] + + def test_is_list_distinguishes_arrays_from_strings() -> None: assert rules.is_list([1]) and rules.is_list(()) assert not rules.is_list("text") diff --git a/testing/tests/test_hermes_handoff_run.py b/testing/tests/test_hermes_handoff_run.py index 379bb27d..881ffd55 100644 --- a/testing/tests/test_hermes_handoff_run.py +++ b/testing/tests/test_hermes_handoff_run.py @@ -4,6 +4,7 @@ from __future__ import annotations import datetime as dt import subprocess +from dataclasses import replace from testing.tests.test_hermes_handoff_support import ( FakeClock, @@ -22,20 +23,35 @@ TARGETS = catalog.Targets(now=NOW) def completed(stdout: str = "", stderr: str = "", returncode: int = 0): - return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) def whoami(username: str) -> object: - return completed(stdout=f'{{"status": {{"userInfo": {{"username": "{username}"}}}}}}') + return completed(stdout=username) def runner_for(results: list, **kwargs): return exec_module.Runner( - clock=FakeClock(), spawn=FakeSpawn(results), environ={}, deadline_seconds=3600, **kwargs + clock=FakeClock(), + spawn=FakeSpawn(results), + environ={}, + deadline_seconds=3600, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + **kwargs, ) -def check(rule: str, expect: dict | None = None, steps=(), identifier="test.check", mandatory=True): +def check( + rule: str, + expect: dict | None = None, + steps=(), + identifier="test.check", + mandatory=True, +): return model.CheckSpec( id=identifier, title="t", @@ -51,8 +67,23 @@ def test_the_shipped_catalog_is_valid_and_covers_every_group() -> None: specs = harness_run.build_catalog(TARGETS) assert harness_run.validate_catalog(specs) == [] groups = {spec.group for spec in specs} - assert {"baseline", "identity", "routing", "scopes", "access-denied", "gitops"} <= groups - assert {"forge", "nodes", "build", "reliability", "pool", "surfaces", "ephemeral"} <= groups + assert { + "baseline", + "identity", + "routing", + "scopes", + "access-denied", + "gitops", + } <= groups + assert { + "forge", + "nodes", + "build", + "reliability", + "pool", + "surfaces", + "ephemeral", + } <= groups def test_catalog_validation_names_every_structural_problem() -> None: @@ -61,14 +92,20 @@ def test_catalog_validation_names_every_structural_problem() -> None: check("allowed", identifier="a.dup"), check("allowed", steps=(step("s", vantage="nowhere"),), identifier="b.vantage"), check("allowed", steps=(step("s", kind="guess"),), identifier="c.kind"), - check("denied", steps=(step("s", kind=model.REVIEW),), identifier="d.review-only"), + check( + "denied", steps=(step("s", kind=model.REVIEW),), identifier="d.review-only" + ), model.CheckSpec( id="e.impersonation", title="t", group="g", rule="allowed", steps=( - model.Step(key="s", vantage=catalog.OPERATOR, argv=("kubectl", "get", "ns", "--as", "x")), + model.Step( + key="s", + vantage=catalog.OPERATOR, + argv=("kubectl", "get", "ns", "--as", "x"), + ), ), ), ] @@ -83,23 +120,81 @@ def test_catalog_validation_names_every_structural_problem() -> None: assert "impersonates from the operator vantage" in problems +def test_target_validation_rejects_every_vacuous_or_unbound_release_input() -> None: + broken = replace( + TARGETS, + repo="atlas/other", + remote="upstream", + reviewed_pr_number=20, + reviewed_head_ref="other", + baseline_commit="bad", + remote_main_sha="bad", + reviewed_head_sha="bad", + build_sha="bad", + agent_image="latest", + deployment_revision="0", + node_count=1, + pool_replicas=1, + expect_telegram_sessions=False, + chat_config_revision="", + pool_worker_env=(), + dependency_heads=((14, "bad"), (14, "bad")), + max_evidence_age_seconds=0, + ) + problems = harness_run.validate_targets(broken) + assert len(problems) >= 12 + sha = "1" * 40 + valid = replace( + TARGETS, + remote_main_sha=sha, + reviewed_head_sha=sha, + build_sha=sha, + agent_image=f"registry.example/hermes:git-{sha}-build-7@sha256:{'2' * 64}", + deployment_revision="1", + chat_config_revision="rev", + dependency_heads=tuple( + (number, sha) for number in TARGETS.dependency_pull_requests + ), + ) + assert harness_run.validate_targets(valid) == [] + wrong_image_source = replace( + valid, + agent_image=f"registry.example/hermes:git-{'2' * 40}-build-7@sha256:{'2' * 64}", + ) + assert "image tag/digest" in " ".join( + harness_run.validate_targets(wrong_image_source) + ) + wrong_build = replace(valid, build_sha="2" * 40) + assert "build SHA" in " ".join(harness_run.validate_targets(wrong_build)) + malformed_heads = replace( + valid, + dependency_heads=tuple( + (number, "bad") for number in valid.dependency_pull_requests + ), + ) + assert "dependency heads" in " ".join(harness_run.validate_targets(malformed_heads)) + + def test_identity_resolution_extracts_only_the_username() -> None: runner = runner_for([whoami("system:serviceaccount:hermes:hermes-agent")]) - username, detail = harness_run.resolve_identity(runner, exec_module.operator_vantage()) + username, detail = harness_run.resolve_identity( + runner, exec_module.operator_vantage() + ) assert (username, detail) == ("system:serviceaccount:hermes:hermes-agent", "") failed = harness_run.resolve_identity( - runner_for([completed(stderr="no access", returncode=1)]), exec_module.operator_vantage() + runner_for([completed(stderr="no access", returncode=1)]), + exec_module.operator_vantage(), ) assert failed[0] == "" and "no access" in failed[1] unparsable = harness_run.resolve_identity( - runner_for([completed(stdout="nope")]), exec_module.operator_vantage() + runner_for([completed(stdout="two names")]), exec_module.operator_vantage() ) - assert unparsable[0] == "" and "unparsable identity" in unparsable[1] + assert unparsable[0] == "" and "no username" in unparsable[1] nameless = harness_run.resolve_identity( - runner_for([completed(stdout='{"status": {}}')]), exec_module.operator_vantage() + runner_for([completed(stdout="")]), exec_module.operator_vantage() ) assert nameless[0] == "" and "no username" in nameless[1] @@ -119,7 +214,12 @@ def test_vantages_resolve_to_running_pods_and_record_what_is_missing() -> None: vantages, records = harness_run.resolve_vantages(runner, TARGETS) by_name = {record.name: record for record in records} - assert set(vantages) == {catalog.OPERATOR, catalog.SELF, catalog.SWITCHYARD, catalog.CHAT} + assert set(vantages) == { + catalog.OPERATOR, + catalog.SELF, + catalog.SWITCHYARD, + catalog.CHAT, + } assert by_name[catalog.SELF].identity == "system:serviceaccount:hermes:hermes-agent" assert "hermes-agent-1" in by_name[catalog.SELF].description # deterministic pick assert by_name[catalog.NODE].available is False @@ -133,7 +233,9 @@ def test_an_unreachable_operator_stops_vantage_resolution_immediately() -> None: assert set(vantages) == {catalog.OPERATOR} assert records[0].available is False - assert harness_run.vantage_problems(records)[0].startswith("the operator vantage is unavailable") + assert harness_run.vantage_problems(records)[0].startswith( + "the operator vantage is unavailable" + ) def test_a_missing_chat_pod_is_recorded_rather_than_assumed_present() -> None: @@ -176,8 +278,14 @@ def test_a_selector_that_matches_no_running_pod_leaves_the_vantage_absent() -> N def test_two_vantages_that_are_the_same_principal_invalidate_the_evidence() -> None: """Corroboration needs two identities; one identity twice is a restatement.""" records = [ - model.VantageRecord(name=catalog.OPERATOR, description="d", identity="same", available=True), - model.VantageRecord(name=catalog.SELF, description="d", identity="same", available=True), + model.VantageRecord( + name=catalog.OPERATOR, description="d", identity="same", available=True + ), + model.VantageRecord( + name=catalog.SELF, description="d", identity="same", available=True + ), + model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True), + model.VantageRecord(name=catalog.CHAT, description="d", available=True), ] problems = harness_run.vantage_problems(records) @@ -195,7 +303,10 @@ def test_vantage_problems_flag_a_missing_or_anonymous_self_probe() -> None: assert "in-pod Hermes vantage is unavailable" in absent[0] anonymous = harness_run.vantage_problems( - [operator, model.VantageRecord(name=catalog.SELF, description="d", available=True)] + [ + operator, + model.VantageRecord(name=catalog.SELF, description="d", available=True), + ] ) assert "did not report an identity" in anonymous[0] @@ -205,6 +316,10 @@ def test_vantage_problems_flag_a_missing_or_anonymous_self_probe() -> None: model.VantageRecord( name=catalog.SELF, description="d", identity="agent", available=True ), + model.VantageRecord( + name=catalog.SWITCHYARD, description="d", available=True + ), + model.VantageRecord(name=catalog.CHAT, description="d", available=True), ] ) assert clean == [] @@ -224,11 +339,20 @@ def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None spec = check( "stdout_matches", {"step": "s", "equals": "locked"}, - steps=(model.Step(key="s", vantage=catalog.OPERATOR, argv=("kubectl", "version"), record=False),), + steps=( + model.Step( + key="s", + vantage=catalog.OPERATOR, + argv=("kubectl", "version"), + record=False, + ), + ), ) runner = runner_for([completed(stdout="locked", stderr="noise")]) - result = harness_run.run_check(runner, spec, {catalog.OPERATOR: exec_module.operator_vantage()}) + result = harness_run.run_check( + runner, spec, {catalog.OPERATOR: exec_module.operator_vantage()} + ) assert result.status == model.PASS assert result.outcomes[0].stdout == harness_run.UNRECORDED @@ -237,7 +361,15 @@ def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None def test_an_expired_deadline_leaves_the_report_saying_so() -> None: clock = FakeClock() - runner = exec_module.Runner(clock=clock, spawn=FakeSpawn([]), environ={}, deadline_seconds=1.0) + runner = exec_module.Runner( + clock=clock, + spawn=FakeSpawn([]), + environ={}, + deadline_seconds=1.0, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + ) clock.advance(5.0) spec = check("allowed", steps=(step("s", vantage=catalog.OPERATOR),)) @@ -259,12 +391,24 @@ def test_an_expired_deadline_leaves_the_report_saying_so() -> None: def test_the_report_carries_the_configured_baseline_and_sorts_its_checks() -> None: runner = runner_for([completed(stdout="ok"), completed(stdout="ok")]) specs = [ - check("allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="z.last"), - check("allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="a.first"), + check( + "allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="z.last" + ), + check( + "allowed", + steps=(step("s", vantage=catalog.OPERATOR),), + identifier="a.first", + ), ] records = [ - model.VantageRecord(name=catalog.OPERATOR, description="d", identity="admin", available=True), - model.VantageRecord(name=catalog.SELF, description="d", identity="agent", available=True), + model.VantageRecord( + name=catalog.OPERATOR, description="d", identity="admin", available=True + ), + model.VantageRecord( + name=catalog.SELF, description="d", identity="agent", available=True + ), + model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True), + model.VantageRecord(name=catalog.CHAT, description="d", available=True), ] report = harness_run.build_report( @@ -279,16 +423,23 @@ def test_the_report_carries_the_configured_baseline_and_sorts_its_checks() -> No assert [result.spec.id for result in report.results] == ["a.first", "z.last"] assert report.baseline["baseline_commit"] == TARGETS.baseline_commit - assert report.baseline["dependency_pull_requests"] == list(TARGETS.dependency_pull_requests) + assert report.baseline["dependency_pull_requests"] == list( + TARGETS.dependency_pull_requests + ) assert report.decision == model.GO assert report.finished_at def test_armed_results_replace_the_matching_catalog_placeholders() -> None: - placeholder = check("not_armed", identifier="ephemeral.feature-branch-push", mandatory=False) + placeholder = check( + "not_armed", identifier="ephemeral.feature-branch-push", mandatory=False + ) armed = model.CheckResult( spec=model.CheckSpec( - id="ephemeral.feature-branch-push", title="t", group="ephemeral", rule="armed" + id="ephemeral.feature-branch-push", + title="t", + group="ephemeral", + rule="armed", ), status=model.PASS, reason="pushed", diff --git a/testing/tests/test_hermes_handoff_support.py b/testing/tests/test_hermes_handoff_support.py index 64964a27..f62ead15 100644 --- a/testing/tests/test_hermes_handoff_support.py +++ b/testing/tests/test_hermes_handoff_support.py @@ -33,6 +33,7 @@ def outcome( stderr: str = "", returncode: int | None = 0, error: str | None = None, + truncated: bool = False, vantage: str = "self", argv: tuple[str, ...] = ("kubectl", "get", "pods"), ) -> Any: @@ -44,6 +45,7 @@ def outcome( stdout=stdout, stderr=stderr, error=error, + truncated=truncated, ) @@ -68,12 +70,14 @@ def spec( ) -def step(key: str, kind: str | None = None, vantage: str = "self", optional: bool = False) -> Any: +def step( + key: str, kind: str | None = None, vantage: str = "self", optional: bool = False +) -> Any: """Return a synthetic step.""" return model.Step( key=key, vantage=vantage, - argv=("kubectl", "get", "pods"), + argv=("kubectl", "get", "pods", "-o", "name"), kind=kind or model.READ, optional=optional, ) diff --git a/testing/tests/test_quality_contract.py b/testing/tests/test_quality_contract.py index 6e260a1b..fecae836 100644 --- a/testing/tests/test_quality_contract.py +++ b/testing/tests/test_quality_contract.py @@ -2,13 +2,17 @@ from __future__ import annotations +import hashlib from pathlib import Path import textwrap from testing.quality_contract import load_contract from testing.quality_coverage import run_check as run_coverage_check from testing.quality_docs import run_check as run_docs_check -from testing.quality_hygiene import run_check as run_hygiene_check +from testing.quality_hygiene import ( + count_files_over_line_limit, + run_check as run_hygiene_check, +) def test_bundled_contract_exposes_local_and_jenkins_profiles(): @@ -18,6 +22,21 @@ def test_bundled_contract_exposes_local_and_jenkins_profiles(): assert contract["pytest_suites"]["unit"]["paths"] +def test_handoff_modules_are_exactly_managed_linted_and_covered(): + """The canonical contract must not silently drop a handoff module.""" + contract = load_contract() + expected = { + path.as_posix() for path in Path("scripts/ops").glob("hermes_handoff_*.py") + } + assert len(expected) == 15 + assert expected <= set(contract["managed_modules"]) + assert expected <= set(contract["lint_paths"]) + assert expected <= set(contract["coverage"]["tracked_files"]) + assert expected <= set(contract["coverage"]["branch_tracked_files"]) + assert "scripts/ops" in contract["pytest_suites"]["unit"]["coverage_sources"] + assert "scripts/ops/hermes_handoff_*.py" in contract["hygiene"]["line_limit_globs"] + + def test_docs_check_reports_missing_docstring_and_missing_path(tmp_path: Path): module_path = tmp_path / "managed.py" module_path.write_text("value = 1\n", encoding="utf-8") @@ -43,7 +62,10 @@ def test_docs_check_reports_missing_required_doc_metadata(tmp_path: Path): (tmp_path / "README.md").write_text("", encoding="utf-8") contract = { - "required_docs": [{"path": "README.md", "description": ""}, {"path": "missing.md", "description": "Missing"}], + "required_docs": [ + {"path": "README.md", "description": ""}, + {"path": "missing.md", "description": "Missing"}, + ], "managed_modules": [], "lint_paths": [], "pytest_suites": {"unit": {"description": "", "paths": []}}, @@ -62,6 +84,7 @@ def test_docs_check_reports_missing_required_doc_metadata(tmp_path: Path): def test_hygiene_check_enforces_line_limit_and_name_rules(tmp_path: Path): tests_dir = tmp_path / "tests" tests_dir.mkdir() + (tests_dir / "conftest.py").write_text("value = 1\n", encoding="utf-8") bad_name = tests_dir / "bad-name.py" bad_name.write_text("x = 1\n", encoding="utf-8") long_file = tests_dir / "test_too_long.py" @@ -84,7 +107,34 @@ def test_hygiene_check_enforces_line_limit_and_name_rules(tmp_path: Path): issues = run_hygiene_check(contract, tmp_path) assert any("file exceeds 3 LOC" in issue for issue in issues) - assert any("naming rule failed" in issue and "bad-name.py" in issue for issue in issues) + assert any( + "naming rule failed" in issue and "bad-name.py" in issue for issue in issues + ) + + +def test_hygiene_legacy_exception_is_exact_and_invalidated_by_any_edit(tmp_path: Path): + legacy = tmp_path / "legacy.py" + legacy.write_text("line\n" * 4, encoding="utf-8") + contract = { + "hygiene": { + "max_lines": 3, + "line_limit_globs": ["*.py"], + "legacy_line_exceptions": { + "legacy.py": { + "lines": 4, + "sha256": hashlib.sha256(legacy.read_bytes()).hexdigest(), + } + }, + } + } + assert run_hygiene_check(contract, tmp_path) == [] + assert count_files_over_line_limit(contract, tmp_path) == 0 + + legacy.write_text("edit\n" * 4, encoding="utf-8") + assert run_hygiene_check(contract, tmp_path) == [ + "file exceeds 3 LOC: legacy.py (4)" + ] + assert count_files_over_line_limit(contract, tmp_path) == 1 def test_coverage_check_enforces_per_file_floor(tmp_path: Path): @@ -124,9 +174,9 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path): def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Path): missing_xml = tmp_path / "missing.xml" - assert run_coverage_check({"coverage": {"tracked_files": []}}, tmp_path, missing_xml) == [ - "coverage xml missing: missing.xml" - ] + assert run_coverage_check( + {"coverage": {"tracked_files": []}}, tmp_path, missing_xml + ) == ["coverage xml missing: missing.xml"] source_dir = tmp_path / "pkg" source_dir.mkdir() @@ -143,7 +193,7 @@ def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Pa - + diff --git a/testing/tests/test_quality_coverage_helpers.py b/testing/tests/test_quality_coverage_helpers.py index 5eaa624e..a42df4ed 100644 --- a/testing/tests/test_quality_coverage_helpers.py +++ b/testing/tests/test_quality_coverage_helpers.py @@ -12,10 +12,15 @@ def test_compute_workspace_line_coverage_handles_missing_xml(tmp_path: Path) -> """Missing coverage XML should produce a zero workspace coverage score.""" contract = {"coverage": {"tracked_files": ["managed.py"]}} - assert compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") == 0.0 + assert ( + compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") + == 0.0 + ) -def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path: Path) -> None: +def test_compute_workspace_line_coverage_averages_present_tracked_files( + tmp_path: Path, +) -> None: """Workspace coverage should average only tracked files that appear in the report.""" coverage_xml = tmp_path / "coverage.xml" @@ -41,7 +46,9 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path assert compute_workspace_line_coverage(contract, tmp_path, coverage_xml) == 75.0 -def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: Path) -> None: +def test_run_check_keeps_relative_names_when_source_roots_do_not_match( + tmp_path: Path, +) -> None: """Relative filenames should remain relative when no declared source root contains them.""" coverage_xml = tmp_path / "coverage.xml" @@ -74,3 +81,59 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: ) assert issues == ["coverage below 95.0%: relative.py (80.0%)"] + + +def test_run_check_enforces_explicit_branch_floor(tmp_path: Path) -> None: + """Branch-tracked modules must report and meet the configured floor.""" + coverage_xml = tmp_path / "coverage.xml" + coverage_xml.write_text( + """ + + + + + """, + encoding="utf-8", + ) + issues = run_check( + { + "coverage": { + "minimum_percent": 95, + "minimum_branch_percent": 95, + "tracked_files": ["low.py", "missing.py", "legacy.py"], + "branch_tracked_files": ["low.py", "missing.py"], + } + }, + tmp_path, + coverage_xml, + ) + assert issues == [ + "branch coverage below 95.0%: low.py (90.0%)", + "branch coverage missing for tracked file: missing.py", + ] + passing = run_check( + { + "coverage": { + "minimum_percent": 95, + "minimum_branch_percent": 95, + "tracked_files": ["passing.py"], + "branch_tracked_files": ["passing.py"], + } + }, + tmp_path, + coverage_xml, + ) + assert passing == [] + + +def test_workspace_coverage_returns_zero_when_xml_has_no_tracked_samples( + tmp_path: Path, +) -> None: + xml = tmp_path / "coverage.xml" + xml.write_text("", encoding="utf-8") + assert ( + compute_workspace_line_coverage( + {"coverage": {"tracked_files": ["missing.py"]}}, tmp_path, xml + ) + == 0.0 + ) diff --git a/testing/tests/test_semgrep_report.py b/testing/tests/test_semgrep_report.py index e0cf1735..ef59abd0 100644 --- a/testing/tests/test_semgrep_report.py +++ b/testing/tests/test_semgrep_report.py @@ -100,6 +100,14 @@ def test_build_sonar_issues_uses_safe_defaults_for_sparse_findings() -> None: assert issue["primaryLocation"]["textRange"] == {"startLine": 1, "endLine": 1} +def test_sonar_issue_never_ends_before_its_start_line() -> None: + issue = semgrep_report._sonar_issue_from_finding( + {"path": "app.py", "start": {"line": 10}, "end": {"line": 2}} + ) + assert issue is not None + assert issue["primaryLocation"]["textRange"] == {"startLine": 10, "endLine": 10} + + def test_read_json_handles_invalid_json_and_non_object(tmp_path: Path) -> None: """Report loading should fail closed for bad JSON and wrong top-level shapes.""" @@ -108,7 +116,9 @@ def test_read_json_handles_invalid_json_and_non_object(tmp_path: Path) -> None: list_json = tmp_path / "list.json" list_json.write_text("[]", encoding="utf-8") - assert semgrep_report._read_json(bad_json)["errors"][0]["message"].startswith("invalid JSON:") + assert semgrep_report._read_json(bad_json)["errors"][0]["message"].startswith( + "invalid JSON:" + ) assert semgrep_report._read_json(list_json) == { "errors": [{"message": "report payload is not an object"}], "results": [], @@ -137,16 +147,25 @@ def test_main_handles_missing_or_invalid_input(tmp_path: Path) -> None: payload = json.loads(output.read_text(encoding="utf-8")) assert payload["status"] == "failed" assert payload["errors_total"] == 1 - sonar_payload = json.loads((tmp_path / "build" / "sonar-issues.json").read_text(encoding="utf-8")) + sonar_payload = json.loads( + (tmp_path / "build" / "sonar-issues.json").read_text(encoding="utf-8") + ) assert sonar_payload == {"issues": []} -def test_main_honors_custom_blocking_severity_without_sonar_output(tmp_path: Path) -> None: +def test_main_honors_custom_blocking_severity_without_sonar_output( + tmp_path: Path, +) -> None: """CLI callers can choose stricter blocking severities without writing Sonar issues.""" semgrep_json = tmp_path / "semgrep.json" semgrep_json.write_text( - json.dumps({"results": [{"path": "app.py", "extra": {"severity": "WARNING"}}], "errors": []}), + json.dumps( + { + "results": [{"path": "app.py", "extra": {"severity": "WARNING"}}], + "errors": [], + } + ), encoding="utf-8", ) output = tmp_path / "semgrep-report.json"