diff --git a/ci/scripts/semgrep_report.py b/ci/scripts/semgrep_report.py index 2408f3e9..f46e528d 100644 --- a/ci/scripts/semgrep_report.py +++ b/ci/scripts/semgrep_report.py @@ -27,7 +27,10 @@ def _read_json(path: Path) -> dict[str, Any]: except json.JSONDecodeError as exc: return {"errors": [{"message": f"invalid JSON: {exc}"}], "results": []} if not isinstance(payload, dict): - return {"errors": [{"message": "report payload is not an object"}], "results": []} + return { + "errors": [{"message": "report payload is not an object"}], + "results": [], + } return payload @@ -49,8 +52,16 @@ def build_report( blocking = blocking_severities or BLOCKING_SEVERITIES raw_results = semgrep_payload.get("results", []) raw_errors = semgrep_payload.get("errors", []) - results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else [] - errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else [] + results = ( + [item for item in raw_results if isinstance(item, dict)] + if isinstance(raw_results, list) + else [] + ) + errors = ( + [item for item in raw_errors if isinstance(item, dict)] + if isinstance(raw_errors, list) + else [] + ) severity_counts = dict.fromkeys(SEVERITIES, 0) blocking_findings = 0 @@ -93,16 +104,18 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None: start = finding.get("start") if isinstance(finding.get("start"), dict) else {} end = finding.get("end") if isinstance(finding.get("end"), dict) else {} start_line = _line_number(start.get("line") if isinstance(start, dict) else None) - end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line) - if end_line < start_line: - end_line = start_line + end_line = _line_number( + end.get("line") if isinstance(end, dict) else None, start_line + ) return { "engineId": "semgrep", "ruleId": str(finding.get("check_id") or "semgrep.unknown"), "type": "VULNERABILITY" if severity == "ERROR" else "CODE_SMELL", "severity": SONAR_SEVERITY_BY_SEMGREP[severity], "primaryLocation": { - "message": str(extra.get("message") or finding.get("check_id") or "Semgrep finding"), + "message": str( + extra.get("message") or finding.get("check_id") or "Semgrep finding" + ), "filePath": path, "textRange": { "startLine": start_line, @@ -112,11 +125,19 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None: } -def build_sonar_issues(semgrep_payload: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: +def build_sonar_issues( + semgrep_payload: dict[str, Any], +) -> dict[str, list[dict[str, Any]]]: """Build SonarQube generic external issues from Semgrep JSON output.""" raw_results = semgrep_payload.get("results", []) - results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else [] - issues = [issue for finding in results if (issue := _sonar_issue_from_finding(finding))] + results = ( + [item for item in raw_results if isinstance(item, dict)] + if isinstance(raw_results, list) + else [] + ) + issues = [ + issue for finding in results if (issue := _sonar_issue_from_finding(finding)) + ] return {"issues": issues} @@ -144,7 +165,9 @@ def main(argv: list[str] | None = None) -> int: ) output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + output.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) if args.sonar_issues_output: sonar_output = Path(args.sonar_issues_output) sonar_output.parent.mkdir(parents=True, exist_ok=True) diff --git a/docs/hermes_full_handoff_acceptance.md b/docs/hermes_full_handoff_acceptance.md new file mode 100644 index 00000000..be13c1f0 --- /dev/null +++ b/docs/hermes_full_handoff_acceptance.md @@ -0,0 +1,264 @@ +# Hermes full-handoff acceptance and release runbook + +`scripts/ops/hermes_handoff_acceptance.py` is a fail-closed release gate. It +contains 63 checks in 15 groups. An empty catalog, an unknown result status, a +mandatory `FAIL` or `NOT_RUN`, a harness error, or incomplete evidence makes the +decision `NO_GO`. The four ephemeral checks are non-mandatory `NOT_RUN` in the +default mode and are replaced with mandatory results only in armed mode. + +This runbook describes evidence collection. It does not authorize a merge, +publish, deployment, Flux reconcile, credential read, or live cluster mutation. +Human review of PR #19 and of the resulting report is required. + +## Required release inputs + +Every value that can go stale must be supplied explicitly. Resolve `main`, PR +heads, the image digest, build source SHA, deployment revision, and chat config +revision from their authoritative read-only sources immediately before the run. +Do not copy the snapshot in the dependency section into a later run. + +```sh +scripts/ops/hermes_handoff_acceptance.py \ + --context atlas-operator \ + --remote-main-sha "$REMOTE_MAIN_SHA" \ + --reviewed-head-sha "$REVIEWED_PR19_HEAD" \ + --agent-image "$AGENT_GIT_SHA_BUILD_TAG_AT_SHA256" \ + --build-sha "$IMAGE_BUILD_SOURCE_SHA" \ + --deployment-revision "$RUNNING_DEPLOYMENT_REVISION" \ + --chat-config-revision "$RUNNING_CHAT_CONFIG_REVISION" \ + --dependency-head "12=$PR12_HEAD" \ + --dependency-head "14=$PR14_HEAD" \ + --dependency-head "15=$PR15_HEAD" \ + --dependency-head "16=$PR16_HEAD" \ + --dependency-head "17=$PR17_HEAD" \ + --dependency-head "18=$PR18_HEAD" \ + --dependency-head "20=$PR20_HEAD" \ + --output build/hermes-handoff-acceptance.json +``` + +The fixed topology is three pool workers on three distinct nodes. `--node-count` +and `--pool-replicas` cannot lower that requirement. Pool node/ordinal assignment, +Telegram session continuity, and the chat revision are mandatory. Deliberate Flux +suspensions must be named with repeatable `--expected-suspension` arguments. + +Exit status is `0` only for `GO`, `1` for `NO_GO`, and `2` for invalid execution +bounds. JSON is written to `--output` or stdout; the human summary goes to stderr +unless `--json-only` is passed. + +## Structural safety boundary + +The catalog is validated before a `Runner` exists. Invalid inputs, an empty or +unsafe catalog, and an invalid armed preflight therefore create no subprocess. +The command policy is checked both before and after executable resolution. + +Default mode permits a narrow read grammar: + +- Kubernetes reads use `name` or reviewed JSONPath projections. Secret resources, + raw API reads, full JSON/YAML, `describe`, `cluster-info dump`, credential-bearing + projections, `auth reconcile`, config operations, all mutations, and every + dry-run mutation trick are rejected before spawn. +- Git is read-only. `fetch`, configuration/helper overrides, alternate worktrees, + external diff helpers, and push are rejected. Helm values are not readable. +- Shell commands must be rendered from frozen templates, use fixed executable + paths, and cannot name service-account, Vault, runtime-access, SSH, or other + credential roots. +- Outer impersonation is rejected. A reviewed inner command can use impersonation + only after an explicit `kubectl exec --` boundary; it cannot substitute for the + independent self identity. +- `PATH` is replaced with a fixed path. Every outer executable is resolved to an + allowed real path, checked for unsafe write mode, and compared with a pinned + SHA-256 digest. A caller-authored `kubectl`, `git`, or helper cannot fabricate + evidence. + +The executor uses a new process group, drains stdout and stderr concurrently, +and retains at most the configured bytes while the child is running. A monotonic +absolute timeout terminates and reaps the group, including descendants that keep +pipes open. Timeout, deadline, output, and concurrency values reject booleans, +zero/negative values, non-finite numbers, and values outside fixed bounds. +Truncation discards the affected stream and makes every evaluator `NOT_RUN`. + +Credential screening runs before truncation. Command, output, error, identity, +reason, evidence, JSON, and human-summary fields are screened recursively, +including mapping keys and short/numeric sensitive values. The complete rendered +report is screened again; a remaining credential shape is replaced by a bounded +`NO_GO` stub rather than being written. No credential prefix is retained at a +truncation boundary. + +## Vantages and evidence + +The operator must use an external read-only kubeconfig. The self probe runs in +the actual Hermes agent pod. Both identities are obtained with +`kubectl auth whoami`; equal or missing principals make the run `NO_GO`. Required +Switchyard and chat/Telegram pod vantages must also resolve. Node coverage is +derived from all daemonset pod node names, not one selected pod. + +Dangerous Kubernetes permissions are checked only with side-effect-free +`kubectl auth can-i` reviews. The default catalog never constructs a Secret, +TokenRequest, mutation, attach, port-forward, or arbitrary exec request. The one +forge administrative denial check performs a read-only GET and accepts only an +explicit authorization refusal; generic `404`, `not found`, and `no route` text +are not denial evidence. + +The strict evaluators reject duplicate-key JSON, raw control characters, +`NaN`/infinity, boolean/number equality coercion, future timestamps, stale +provider evidence, and every malformed routing or Flux row. Routing freshness is +required separately for each provider, lane, and effort. Flux objects must be +non-suspended, current at `observedGeneration`, and explicitly `Ready=True`; +unknown or reconciling state is not healthy. + +`release.exact-lineage-is-running` atomically binds: + +- fixed remote `main` SHA and local `origin/main`; +- open draft PR #19, base `main`, existing feature ref, and exact reviewed head; +- the exact `:git--build-@sha256:` image identity, + with the source SHA equal to the release `main` SHA; +- current Deployment generation and revision; +- the one active ReplicaSet revision/hash; every Ready pod image and imageID; +- the Flux-applied `main` revision. + +The pool evaluator requires exactly ordinals 0, 1, and 2, all Ready, on three +distinct nodes and claims, with service-account token automount disabled. The +chat checks always require durable Telegram sessions and the exact running chat +configuration revision. + +## Armed ephemeral mode + +Armed mode is the only write path and runs only after the complete default +catalog reports `GO`. + +```sh +scripts/ops/hermes_handoff_acceptance.py \ + \ + --arm-ephemeral-push \ + --confirm "ARM EPHEMERAL HERMES HANDOFF PUSH" \ + --ephemeral-token acceptance-20260817a \ + --output build/hermes-handoff-acceptance-armed.json +``` + +Before any runner, network request, or cluster vantage is created, preflight +binds the working directory to the existing linked PR #19 worktree, the exact +feature branch and reviewed local HEAD, the fixed HTTPS Atlas origin, base +`main`, confirmation phrase, and one unique ephemeral ref. The command policy is +then pinned to that exact ref. It allows only its push, one exact draft-create +payload, cleanup of the exactly discovered PR number, and deletion of that ref. + +The run reserves at least 300 seconds (up to 6000) for cleanup. It verifies the +pushed SHA, always discovers the PR after create (including ambiguous create +responses), and requires exactly one open draft with the expected base, head ref, +base SHA, and head SHA. Cleanup requires successful PR close, successful branch deletion, +a successful empty `ls-remote`, and a strict re-read of the same closed PR. +Malformed or partial cleanup is `FAIL`; it is never inferred from an error. + +Force, force-with-lease, fan-out, protected refs, other repositories, other PR +numbers, arbitrary PATCH payloads, and repository administration remain denied. + +## Dependency, merge, and conflict graph + +Authoritative snapshot observed 2026-08-17 after refreshing Gitea and Git against +remote `main` `d8f2d818b9a552ea6c2d7fe86554be829bd5ffff`: + +| PR | Head SHA | Draft | Base | Role | +| --- | --- | --- | --- | --- | +| #12 | `465d386da5237f011cb408f1a3d4b7d0c6d8ffae` | no | `main` | AI usage exporter repair | +| #14 | `8c6e3acdacb223f68b1716f0a74bd543e18890e6` | yes | `main` | bounded forge client and least-privilege RBAC | +| #15 | `73fefbb5d92ef07371a9a997f22d848acd2d06ed` | yes | `main` | result-decomposition reliability | +| #16 | `e8c26ecf85a0c802d378fffffb98b7df43b78242` | no | `main` | unprivileged image builder | +| #17 | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | yes | `main` | three-lane placement | +| #18 | `20002527512235b29054f3636b02418168373009` | yes | `main` | distributed three-worker pool | +| #19 | `8f005458282269ba5c07941814e4237f5d4cf3ac` | yes | `main` | reviewed blocked head; this repair replaces it on the same branch | +| #20 | `a242dcc786576ae1a18000cb4941836ff610dff2` | yes | `main` | CLI process-group reaping | + +The repaired #19 SHA cannot be embedded in its own commit without changing that +SHA. Resolve it from Gitea after push and require it as `--reviewed-head-sha`; +the PR handoff records the exact value. All eight snapshot heads merge cleanly +with the snapshot `main` individually. Pairwise `git merge-tree --write-tree` +found conflicts only for: #12+#17, #14+#15, #14+#16, #14+#17, #14+#20, +#15+#16, #15+#17, #16+#20, and #17+#20. Gitea's `mergeable=false` on a draft is +not treated as conflict evidence. + +Use this dependency order, refreshing every head and re-running merge-tree at +each step: + +1. #14 first so the release never restores or prolongs cluster-admin authority. +2. #16, then #15, resolving their Dockerfile and quality-contract changes + additively. Never take one PR's quality contract wholesale. +3. #20 after #15; it depends on the CLI lifecycle work and must be rebased across + the current #14/#16 contract and helper changes. +4. #12 before #17, resolving their Deployment overlap once. +5. #17, then #18, after the authority, image, reliability, and reaping work. +6. #19 last as the integration/acceptance change, only after #12/#14-#18/#20 are + merged and their exact supplied heads are ancestors of refreshed `main`. + +#19 overlaps `testing/quality_contract.json` with #14/#15/#16, +`testing/quality_coverage.py` and its helper tests with #14, +`test_hermes_auto_router.py` with #18, and existing Kustomize content with +#12/#17/#18. Preserve branch coverage, every tracked module, fixture isolation, +and all desired-state changes during conflict resolution. + +An exact post-repair merge-tree check against the heads in the table reports +#19 clean with current `main`, #12, #17, and #20, and conflicting with #14, +#15, #16, and #18. Recompute this graph from the Gitea API immediately before +integration; do not infer conflict status from Gitea's draft `mergeable` field. + +The #16/#19 integration must keep both the immutable digest and the source tag +(`newTag: git--build-` plus the digest) in rendered image identity. +Digest-only rendering loses the attested build-source link and therefore +correctly remains `NO_GO` under the exact-lineage evaluator. + +## Safe rollback guidance + +Every rollback is a reviewed forward fix or selective Git revert followed by +the normal Flux workflow. Never restore `hermes-agent-cluster-admin`, even when +reverting unrelated portions of #14. If a rollback would widen RBAC, stop and +write a forward least-privilege fix instead. + +Reverting #18 removes the StatefulSet desired state, but its +`volumeClaimTemplates` PVCs are retained by the Kubernetes default because no +delete retention policy is set. Do not claim Flux pruning deletes those PVCs. +Any PVC deletion is a separate, explicit, human-reviewed data-destruction task +with exact targets and recovery implications. + +Image rollback means a reviewed commit restoring a previously attested digest; +never retag an immutable image. Do not use direct `kubectl` edits as a durable +rollback. + +## Quality and release evidence + +The canonical quality contract explicitly manages, lints, and tracks all 15 +`scripts/ops/hermes_handoff_*.py` modules. It enforces at least 95% line and +branch coverage per tracked file and the 500-physical-line source cap. Contract +tests derive the module set so a future file cannot silently escape the gate. +Four pre-existing over-cap test files are temporarily locked to their exact +line count and SHA-256; any byte edit invalidates the exception and must split +the file below 500 lines. PR #14 supplies the intended additive test splits. + +```sh +python3 -m testing.quality_gate --profile local --build-dir build +python3 -m testing.quality_handoff_mutation +python3 -m ruff check --select F,B,SIM,C4,UP --ignore B017,UP015,UP035 \ + scripts/ops/hermes_handoff_*.py testing/quality_*.py testing/tests/test_hermes_handoff_*.py +python3 -m ruff format --check \ + scripts/ops/hermes_handoff_*.py testing/quality_*.py testing/tests/test_hermes_handoff_*.py +python3 -m compileall -q scripts/ops testing +kustomize build services/hermes | kubectl apply --dry-run=client --validate=false -f - +git diff --check origin/main...HEAD +``` + +The deterministic mutation gate must kill every policy, redaction, verdict, +numeric, truncation, and cleanup mutant. The adversarial suite includes large +concurrent stdout/stderr, malformed JSON and bytes, zero state, partial failure, +absolute timeout, descendant-held pipes, malformed cleanup, ambiguous creates, +and no-subprocess/no-mutation spies. Render/client dry-run, exact conflict checks, +and bounded diff/full-tree secret scans must also pass. + +The provider-health and Gitea-token-sensitive tests use hermetic fixtures. Their +result must be identical whether synthetic runtime files exist or are absent; +tests must never read the mounted production token. + +## Release decision + +The handoff remains `NO_GO` until all dependency heads are merged, the exact +lineage check passes from an external operator, both default and armed reports +are complete, the branch and PR cleanup is verified, the canonical and +adversarial gates are green, and a human has reviewed PR #19 plus the artifacts. +The harness never merges, deploys, reconciles, publishes, or approves anything. diff --git a/scripts/ops/hermes_handoff_acceptance.py b/scripts/ops/hermes_handoff_acceptance.py new file mode 100755 index 00000000..6b35d49f --- /dev/null +++ b/scripts/ops/hermes_handoff_acceptance.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +"""Fail-closed acceptance harness for the Hermes full handoff. + +Read-only by default. The harness collects evidence from two genuinely separate +places — an external operator vantage and the Hermes agent probing itself from +inside its own pod — classifies every check PASS / FAIL / NOT_RUN / +NOT_APPLICABLE, and emits machine-readable JSON alongside a short human summary. + +Any mandatory FAIL or NOT_RUN is a NO_GO, and so is a harness-level problem: an +unreachable vantage, a catalog whose evidence no longer exists, an expired +deadline, or two vantages that turn out to be the same principal. Nothing here +rounds an absence of evidence up to a pass. + + scripts/ops/hermes_handoff_acceptance.py --help + +The release runbook lists every required freshness and lineage input. + +Mutation lives behind a separate arming flag, needs an exact confirmation +phrase and a unique ref, refuses protected push targets before any network +call, and verifies its own cleanup. A default run reports those checks NOT_RUN. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import sys +from collections.abc import Sequence +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from hermes_handoff_catalog import Targets # noqa: E402 +from hermes_handoff_ephemeral import ( # noqa: E402 + CONFIRMATION, + ArmingError, + ArmRequest, + preflight, + run_armed, +) +from hermes_handoff_exec import ( # noqa: E402 + DEFAULT_COMMAND_TIMEOUT, + DEFAULT_RUN_DEADLINE, + Runner, +) +from hermes_handoff_model import GO, Report, unscreened_fields, utc_now # noqa: E402 +from hermes_handoff_policy import ( # noqa: E402 + ARMED, + EXPECTED_BASE, + EXPECTED_REMOTE, + EXPECTED_REPO, + READ_ONLY, +) +from hermes_handoff_redaction import DEFAULT_MAX_BYTES # noqa: E402 +from hermes_handoff_run import ( # noqa: E402 + build_catalog, + build_report, + resolve_vantages, + unavailable_report, + validate_catalog, + validate_targets, +) + +EXIT_GO = 0 +EXIT_NO_GO = 1 +EXIT_USAGE = 2 + + +def dependency_head(value: str) -> tuple[int, str]: + """Parse one `PR=SHA` dependency-head binding.""" + try: + number, sha = value.split("=", 1) + parsed = int(number) + except (TypeError, ValueError) as exc: + raise argparse.ArgumentTypeError( + "dependency head must be PR=40-character-SHA" + ) from exc + if ( + parsed <= 0 + or len(sha) != 40 + or any(character not in "0123456789abcdef" for character in sha) + ): + raise argparse.ArgumentTypeError( + "dependency head must be PR=lowercase-40-character-SHA" + ) + return (parsed, sha) + + +def build_parser() -> argparse.ArgumentParser: + """Return the command-line interface for the harness.""" + parser = argparse.ArgumentParser( + prog="hermes_handoff_acceptance", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--namespace", default="hermes", help="Hermes namespace.") + parser.add_argument( + "--context", help="Kubernetes context for the operator vantage." + ) + parser.add_argument("--kubeconfig", help="Kubeconfig for the operator vantage.") + parser.add_argument( + "--output", type=Path, help="Write the JSON report here instead of stdout." + ) + parser.add_argument( + "--json-only", action="store_true", help="Suppress the human summary." + ) + parser.add_argument( + "--command-timeout", + type=float, + default=DEFAULT_COMMAND_TIMEOUT, + help="Per-command seconds.", + ) + parser.add_argument( + "--deadline", + type=float, + default=DEFAULT_RUN_DEADLINE, + help="Whole-run seconds.", + ) + parser.add_argument( + "--max-output-bytes", + type=int, + default=DEFAULT_MAX_BYTES, + help="Per-capture byte budget.", + ) + parser.add_argument( + "--concurrency", type=int, default=1, help="Validated scheduling bound." + ) + parser.add_argument( + "--baseline-commit", + default=Targets.baseline_commit, + help="Commit main must descend from.", + ) + parser.add_argument( + "--dependency-pr", + type=int, + action="append", + default=None, + help="Pull request that must be merged before this release is certifiable (repeatable).", + ) + parser.add_argument( + "--dependency-head", + type=dependency_head, + action="append", + default=None, + metavar="PR=SHA", + help="Exact current dependency PR head (repeat for every dependency).", + ) + parser.add_argument( + "--remote-main-sha", default="", help="Exact observed origin/main SHA." + ) + parser.add_argument( + "--reviewed-head-sha", default="", help="Exact reviewed PR #19 head SHA." + ) + parser.add_argument( + "--agent-image", + default="", + help="Exact agent :git--build-@sha256 identity.", + ) + parser.add_argument( + "--build-sha", default="", help="Exact source SHA used for the image build." + ) + parser.add_argument( + "--deployment-revision", default="", help="Exact running Deployment revision." + ) + parser.add_argument( + "--expected-suspension", + action="append", + default=None, + metavar="NAMESPACE/NAME", + help="Flux object allowed to be suspended (repeatable).", + ) + parser.add_argument( + "--node-count", + type=int, + default=3, + help="Nodes the fleet must cover (fixed at 3).", + ) + parser.add_argument( + "--pool-replicas", type=int, default=3, help="Expected pool worker count." + ) + parser.add_argument( + "--chat-config-revision", + default="", + help="Expected chat pod-template config revision.", + ) + parser.add_argument( + "--chat-ordinal", + type=int, + default=0, + help="Chat tenant ordinal that carries the Telegram identity.", + ) + parser.add_argument( + "--expect-telegram-sessions", + action="store_true", + default=True, + help="Require the selected chat tenant to hold durable Telegram-sourced sessions.", + ) + parser.add_argument( + "--pool-worker-env", + action="append", + default=None, + help="Environment variable the pool workers must declare (repeatable).", + ) + parser.add_argument( + "--max-evidence-age", + type=int, + default=86_400, + help="Seconds before provider and routing evidence is treated as stale.", + ) + armed = parser.add_argument_group("ephemeral mutation mode") + armed.add_argument( + "--arm-ephemeral-push", + action="store_true", + help="Permit one ephemeral branch push and draft pull request, then remove both.", + ) + armed.add_argument( + "--confirm", default="", help=f"Exact confirmation phrase: {CONFIRMATION!r}." + ) + armed.add_argument( + "--ephemeral-token", default="", help="Unique 8-64 character ref suffix." + ) + return parser + + +def targets_from_args(arguments: argparse.Namespace, now: dt.datetime) -> Targets: + """Return the catalog targets a parsed command line describes.""" + defaults = Targets() + return Targets( + namespace=arguments.namespace, + baseline_commit=arguments.baseline_commit, + dependency_pull_requests=tuple( + arguments.dependency_pr or defaults.dependency_pull_requests + ), + dependency_heads=tuple(arguments.dependency_head or ()), + remote_main_sha=arguments.remote_main_sha, + reviewed_head_sha=arguments.reviewed_head_sha, + agent_image=arguments.agent_image, + build_sha=arguments.build_sha, + deployment_revision=arguments.deployment_revision, + expected_suspensions=tuple(arguments.expected_suspension or ()), + node_count=arguments.node_count, + pool_replicas=arguments.pool_replicas, + chat_config_revision=arguments.chat_config_revision, + chat_ordinal=arguments.chat_ordinal, + expect_telegram_sessions=arguments.expect_telegram_sessions, + pool_worker_env=tuple(arguments.pool_worker_env or defaults.pool_worker_env), + max_evidence_age_seconds=arguments.max_evidence_age, + now=now, + ) + + +def emit(report: Report, arguments: argparse.Namespace, stream=None) -> list[str]: + """Write the report, refusing to publish one that still looks credential-shaped.""" + payload = report.as_dict() + offenders = unscreened_fields(payload) + if offenders: + payload = { + "harness": payload["harness"], + "decision": "NO_GO", + "harness_errors": [ + "the rendered report failed its own credential screening at " + + ", ".join(offenders[:8]) + ], + } + rendered = json.dumps(payload, indent=2, sort_keys=True, default=str) + if arguments.output: + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(rendered + "\n", encoding="utf-8") + else: + print(rendered, file=stream or sys.stdout) + if not arguments.json_only: + print(report.render_summary(), file=sys.stderr) + return offenders + + +def _armed_results(arguments: argparse.Namespace, runner: Runner, vantages) -> list: + request = ArmRequest( + repo=EXPECTED_REPO, + remote=EXPECTED_REMOTE, + token=arguments.ephemeral_token, + confirmation=arguments.confirm, + base=EXPECTED_BASE, + expected_head=arguments.reviewed_head_sha, + expected_base_sha=arguments.remote_main_sha, + ) + preflight(request, EXPECTED_REPO, Path.cwd()) + vantage = vantages["operator"] + return run_armed(runner, vantage, request) + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the acceptance sweep and return the process exit status.""" + arguments = build_parser().parse_args(argv) + now = dt.datetime.now(dt.timezone.utc) + started_at = utc_now(lambda: now) + targets = targets_from_args(arguments, now) + mode = ARMED if arguments.arm_ephemeral_push else READ_ONLY + + specs = build_catalog(targets) + startup_problems = [*validate_targets(targets), *validate_catalog(specs)] + if arguments.arm_ephemeral_push: + try: + preflight( + ArmRequest( + repo=EXPECTED_REPO, + remote=EXPECTED_REMOTE, + token=arguments.ephemeral_token, + confirmation=arguments.confirm, + base=EXPECTED_BASE, + expected_head=arguments.reviewed_head_sha, + expected_base_sha=arguments.remote_main_sha, + ), + EXPECTED_REPO, + Path.cwd(), + ) + except ArmingError as exc: + startup_problems.append(f"arming refused: {exc}") + if startup_problems: + report = unavailable_report( + mode, started_at, "; ".join(startup_problems), targets + ) + emit(report, arguments) + return EXIT_NO_GO + + try: + runner = Runner( + mode=mode, + command_timeout=arguments.command_timeout, + deadline_seconds=arguments.deadline, + max_bytes=arguments.max_output_bytes, + concurrency=arguments.concurrency, + ) + except ValueError as exc: + report = unavailable_report( + mode, started_at, f"invalid execution bound: {exc}", targets + ) + emit(report, arguments) + return EXIT_USAGE + vantages, records = resolve_vantages( + runner, targets, arguments.kubeconfig, arguments.context + ) + report = build_report(runner, targets, specs, vantages, records, mode, started_at) + if arguments.arm_ephemeral_push: + if report.decision != GO: + report.harness_errors.append( + "armed mutation was not started because read-only acceptance did not GO" + ) + else: + armed_results = _armed_results(arguments, runner, vantages) + armed_ids = {result.spec.id for result in armed_results} + report.results = [ + result for result in report.results if result.spec.id not in armed_ids + ] + report.results.extend(armed_results) + report.results.sort(key=lambda result: result.spec.id) + report.finished_at = utc_now() + offenders = emit(report, arguments) + if offenders: + return EXIT_NO_GO + return EXIT_GO if report.decision == GO else EXIT_NO_GO + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/scripts/ops/hermes_handoff_catalog.py b/scripts/ops/hermes_handoff_catalog.py new file mode 100644 index 00000000..9dc5c98a --- /dev/null +++ b/scripts/ops/hermes_handoff_catalog.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Declarative acceptance catalog: baseline, identity, routing, scopes, access. + +The catalog is data on purpose. A reviewer should be able to read what the +release is asserted to satisfy without reading any plumbing, and the harness +should be able to prove structural properties over every entry — both vantages +represented, no impersonation from outside a pod, and no mutation command of +any kind — before it runs a single command. + +Catalog drift is a NO_GO, not a pass. If a manifest, a field name, or a workload +is renamed, the affected checks report ``NOT_RUN`` because the evidence they +name is not there, and the release stops until either the platform or the +catalog is corrected. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field +from typing import Any + +from hermes_handoff_model import ALWAYS, READ, CheckSpec, Step +from hermes_handoff_policy import projection + +OPERATOR = "operator" +SELF = "self" +SWITCHYARD = "switchyard" +NODE = "node" +CHAT = "chat" +VANTAGE_NAMES = (OPERATOR, SELF, SWITCHYARD, NODE, CHAT) + +# `namegenerationobservedGenerationsuspendconditions`. +FLUX_PROJECTION = projection( + ( + "jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{'\\t'}" + "{.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}" + "{range .status.conditions[*]}{.type}={.status},{end}{'\\n'}{end}" + ).replace("'", '"') +) + +ENV_NAME_PROJECTION = projection( + ( + "jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{'\\n'}{end}{end}" + ).replace("'", '"') +) + +INIT_NAME_PROJECTION = projection( + ( + "jsonpath={range .spec.template.spec.initContainers[*]}{.name}{'\\n'}{end}" + ).replace("'", '"') +) + +NODE_NAME_PROJECTION = projection( + "jsonpath={range .items[*]}{.spec.nodeName}{'\\n'}{end}".replace("'", '"') +) + +POOL_PROJECTION = projection( + ( + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.spec.nodeName}{'\\t'}" + "{.status.phase}{'\\t'}{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}" + "{'\\t'}{range .spec.volumes[*]}{.persistentVolumeClaim.claimName}{','}{end}{'\\n'}{end}" + ).replace("'", '"') +) + +PROVIDER_API_KEY_NAMES = ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + "AZURE_OPENAI_API_KEY", + "CLAUDE_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", +) +FORGE_CREDENTIAL_NAMES = ( + "GITEA_PASSWORD", + "GITEA_TOKEN", + "GITEA_USERNAME", + "GIT_PASSWORD", +) + +CODEX_HEALTH = "/opt/data/provider-health/codex.json" +CLAUDE_HEALTH = "/opt/data/provider-health/claude.json" +CODEX_HEALTH_FIELDS = "authenticated,transport,state,model,effort,latency_ms,checked_at" +CLAUDE_HEALTH_FIELDS = ( + "authenticated,api_provider,auth_method,transport,state,subscription_type," + "model,effort,latency_ms,checked_at" +) +ROUTING_LOG = "/var/lib/switchyard/routing.jsonl" + + +@dataclass(frozen=True) +class Targets: + """Runtime names and expected values the catalog is written against. + + Everything a site can legitimately rename lives here so the assertions stay + readable and a rename is a configuration change rather than a code change. + """ + + namespace: str = "hermes" + agent_deployment: str = "hermes-agent" + agent_container: str = "hermes" + chat_statefulset: str = "hermes-chat-tenant" + chat_container: str = "hermes" + chat_ordinal: int = 0 + chat_router_deployment: str = "hermes-chat-router" + triage_deployment: str = "hermes" + triage_container: str = "hermes" + switchyard_deployment: str = "hermes-switchyard" + switchyard_container: str = "switchyard" + node_daemonset: str = "hermes-node-ssh-access" + node_container: str = "key-reconciler" + pool_statefulset: str = "hermes-execution-worker" + pool_replicas: int = 3 + pool_selector: str = "app=hermes-execution-worker" + coordinator_claims: tuple[str, ...] = ( + "hermes-agent-home", + "hermes-agent-workspace", + ) + node_account: str = "hermes-agent" + node_account_identity: str = "1200:1200:/home/hermes-agent:/bin/bash" + node_count: int = 3 + repo: str = "atlas/titan-iac" + remote: str = "origin" + baseline_commit: str = "ab346f55509d584e457fe26cf90be3078f7a375c" + dependency_pull_requests: tuple[int, ...] = (12, 14, 15, 16, 17, 18, 20) + dependency_heads: tuple[tuple[int, str], ...] = () + remote_main_sha: str = "" + reviewed_pr_number: int = 19 + reviewed_head_ref: str = "feature/hermes-full-handoff-acceptance" + reviewed_head_sha: str = "" + agent_image: str = "" + build_sha: str = "" + deployment_revision: str = "" + expected_suspensions: tuple[str, ...] = () + harbor_namespace: str = "harbor" + harbor_immutability_job: str = "hermes-agent-immutability" + builder_namespace: str = "jenkins" + builder_serviceaccount: str = "hermes-image-builder" + builder_pipeline: str = "ci/Jenkinsfile.hermes-agent-image" + builder_capabilities: str = ( + 'add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]' + ) + agent_init_containers: tuple[str, ...] = ("patch-web-session-activity",) + chat_init_containers: tuple[str, ...] = ("patch-api-server-sessions",) + chat_config_revision: str = "" + pool_worker_env: tuple[str, ...] = ("HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL") + max_evidence_age_seconds: int = 86_400 + routing_tail_lines: int = 600 + routing_tail_bytes: int = 512 * 1024 + listing_bytes: int = 512 * 1024 + git_askpass: str = "/opt/coordinator/gitea_askpass.sh" + expect_telegram_sessions: bool = True + now: Any = None + extra: dict[str, Any] = field(default_factory=dict) + + +def step( + key: str, + vantage: str, + *argv: str, + kind: str = READ, + optional: bool = False, + record: bool = True, + max_bytes: int | None = None, +) -> Step: + """Return a probe step addressed to one vantage.""" + return Step( + key=key, + vantage=vantage, + argv=tuple(argv), + kind=kind, + optional=optional, + record=record, + max_bytes=max_bytes, + ) + + +def check( + identifier: str, + title: str, + group: str, + rule: str, + steps: Sequence[Step], + expect: dict[str, Any] | None = None, + mandatory: bool = True, + scope: str = ALWAYS, + rationale: str = "", +) -> CheckSpec: + """Return one catalog entry.""" + return CheckSpec( + id=identifier, + title=title, + group=group, + rule=rule, + steps=tuple(steps), + expect=expect or {}, + mandatory=mandatory, + scope=scope, + rationale=rationale, + ) diff --git a/scripts/ops/hermes_handoff_checks_access.py b/scripts/ops/hermes_handoff_checks_access.py new file mode 100644 index 00000000..df5c8117 --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_access.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +"""Acceptance checks for what Hermes may and may not do to the cluster. + +Denials use Kubernetes authorization reviews only. Live denied attempts were +removed because a policy regression could turn a supposed probe into a real +credential read or mutation; the default harness never constructs such argv. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + FLUX_PROJECTION, + OPERATOR, + SELF, + Targets, + check, + step, +) +from hermes_handoff_model import REVIEW, CheckSpec + + +def _access(targets: Targets) -> list[CheckSpec]: + namespace = targets.namespace + return [ + check( + "access.no-cluster-admin-binding-for-agent", + "No ClusterRoleBinding grants the agent service account cluster-admin", + "access-denied", + "names_absent", + [ + step( + "bindings", + OPERATOR, + "kubectl", + "get", + "clusterrolebindings", + "-o", + "name", + ) + ], + {"step": "bindings", "contains": ("hermes-agent-cluster-admin",)}, + rationale="The binding inventory and in-pod authorization reviews are independent desired/enforced views.", + ), + check( + "access.secrets-are-denied", + "Hermes cannot read Secrets cluster-wide", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "get", + "secrets", + "--all-namespaces", + kind=REVIEW, + ) + ], + rationale="A live Secret request is never constructed, even when an RBAC regression would allow it.", + ), + check( + "access.service-account-tokens-are-denied", + "Hermes cannot mint a service-account token", + "access-denied", + "review_denied", + [ + step( + "review-token", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "serviceaccounts/token", + "--namespace", + namespace, + kind=REVIEW, + ), + step( + "review-serviceaccount", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "serviceaccounts", + "--namespace", + namespace, + kind=REVIEW, + ), + ], + rationale="Token creation has no credential-safe live probe, so both relevant grants must be denied by review.", + ), + check( + "access.impersonation-is-denied", + "Hermes cannot impersonate another identity", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "impersonate", + "users", + "--all-namespaces", + kind=REVIEW, + ) + ], + rationale="The default harness never constructs an impersonated request.", + ), + check( + "access.workload-mutation-is-denied", + "Hermes cannot mutate its own workload", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "patch", + "deployments", + "--namespace", + namespace, + kind=REVIEW, + ) + ], + rationale="Client-side and server-side dry-run mutation tricks are excluded structurally.", + ), + check( + "access.pod-exec-is-denied", + "Hermes cannot exec into an arbitrary pod", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "pods/exec", + "--all-namespaces", + kind=REVIEW, + ) + ], + rationale="No cross-pod exec request is created by a read-only run.", + ), + check( + "access.attach-portforward-and-kube-system-writes-are-denied", + "Hermes holds no attach, port-forward, or control-plane write authority", + "access-denied", + "review_denied", + [ + step( + "review-attach", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "pods/attach", + "--all-namespaces", + kind=REVIEW, + ), + step( + "review-portforward", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "pods/portforward", + "--all-namespaces", + kind=REVIEW, + ), + step( + "review-create", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "configmaps", + "--namespace", + "kube-system", + kind=REVIEW, + ), + ], + rationale="Every authority is reviewed without issuing the request it would authorize.", + ), + check( + "access.required-reads-are-allowed", + "Hermes retains the reads its operator role depends on", + "access-allowed", + "allowed", + [ + step( + "watch-review", + SELF, + "kubectl", + "auth", + "can-i", + "watch", + "pods", + "--namespace", + namespace, + kind=REVIEW, + ), + step( + "pods", + SELF, + "kubectl", + "--namespace", + namespace, + "get", + "pods", + "-o", + "name", + ), + step("namespaces", SELF, "kubectl", "get", "namespaces", "-o", "name"), + step( + "deployments", + SELF, + "kubectl", + "--namespace", + namespace, + "get", + "deployments", + "-o", + "name", + ), + step("nodes", SELF, "kubectl", "get", "nodes", "-o", "name"), + ], + ), + check( + "access.flux-and-helm-status-are-allowed", + "Hermes can read Flux and Helm reconciliation status", + "access-allowed", + "allowed", + [ + step( + "kustomizations", + SELF, + "kubectl", + "get", + "kustomizations.kustomize.toolkit.fluxcd.io", + "--all-namespaces", + "-o", + "name", + record=False, + ), + step( + "helmreleases", + SELF, + "kubectl", + "get", + "helmreleases.helm.toolkit.fluxcd.io", + "--all-namespaces", + "-o", + "name", + record=False, + ), + ], + ), + check( + "access.namespace-view-agrees-across-vantages", + "The operator and in-pod vantages see the same namespace inventory", + "access-allowed", + "vantages_agree", + [ + step( + "operator", + OPERATOR, + "kubectl", + "get", + "namespaces", + "-o", + "name", + record=False, + ), + step( + "self", + SELF, + "kubectl", + "get", + "namespaces", + "-o", + "name", + record=False, + ), + ], + {"steps": ("operator", "self")}, + rationale="A disagreement here means one vantage is not seeing the cluster the other is certifying.", + ), + ] + + +def _gitops(targets: Targets) -> list[CheckSpec]: + return [ + check( + f"gitops.{kind.split('.')[0]}-reconcile-cleanly", + f"No unexpected {label} suspension or unhealthy reconciliation", + "gitops", + "flux_health", + [ + step( + "objects", + OPERATOR, + "kubectl", + "get", + kind, + "--all-namespaces", + "-o", + FLUX_PROJECTION, + record=False, + ) + ], + {"step": "objects", "expected_suspensions": targets.expected_suspensions}, + ) + for kind, label in ( + ("kustomizations.kustomize.toolkit.fluxcd.io", "Kustomization"), + ("helmreleases.helm.toolkit.fluxcd.io", "HelmRelease"), + ) + ] + + +def access_checks(targets: Targets) -> list[CheckSpec]: + """Return the access-denied, access-allowed, and GitOps health checks.""" + return [*_access(targets), *_gitops(targets)] diff --git a/scripts/ops/hermes_handoff_checks_delivery.py b/scripts/ops/hermes_handoff_checks_delivery.py new file mode 100644 index 00000000..102f93e4 --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_delivery.py @@ -0,0 +1,374 @@ +#!/usr/bin/env python3 +"""Acceptance checks for forge authority, node accounts, images, and regressions. + +What the platform is allowed to do outside its own namespace: which forge +authority a worker holds, what a node account is and is not, how an image is +built and pinned, and whether the fixes the release depends on are actually +running rather than merely merged. + +Where a property has no side-effect-free live probe — a merge that must never be +attempted, a branch protection the harness identity cannot read — the entry says +so in its rationale and asserts the nearest fact that *is* observable, rather +than quietly asserting nothing. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + FORGE_CREDENTIAL_NAMES, + INIT_NAME_PROJECTION, + NODE_NAME_PROJECTION, + OPERATOR, + SELF, + Targets, + check, + step, +) +from hermes_handoff_model import ATTEMPT, CheckSpec +from hermes_handoff_policy import GITEA_CLIENT, projection, shell + + +def _forge(targets: Targets) -> list[CheckSpec]: + api = f"/api/v1/repos/{targets.repo}" + return [ + check( + "forge.worker-holds-no-git-credential-variable", + "No forge credential is exported into the worker process", + "forge", + "names_absent", + [step("env", SELF, *shell("env_names"))], + {"step": "env", "names": FORGE_CREDENTIAL_NAMES}, + rationale="Authentication reaches Git through an askpass helper that reads runtime state, never through the environment.", + ), + check( + "forge.identity-is-not-an-administrator", + "The forge identity the worker uses holds no administrative rights", + "forge", + "json_field", + [step("user", SELF, GITEA_CLIENT, "GET", "/api/v1/user")], + {"step": "user", "fields": {"is_admin": False}}, + ), + check( + "forge.repository-authority-is-push-only", + "The worker can push a branch but cannot administer the repository", + "forge", + "json_field", + [step("repo", SELF, GITEA_CLIENT, "GET", api)], + { + "step": "repo", + "fields": { + "permissions.admin": False, + "permissions.push": True, + "permissions.pull": True, + }, + }, + rationale="Merge, approve, close, and protection changes all require authority this identity is proven not to hold.", + ), + check( + "forge.administrative-route-is-refused", + "An administrative forge route is refused for this identity", + "forge", + "denied", + [ + step( + "attempt", + SELF, + GITEA_CLIENT, + "GET", + f"{api}/branch_protections", + kind=ATTEMPT, + ) + ], + rationale="A live refusal, not a configuration reading. The same lack of authority is what blocks a merge route.", + ), + ] + + +def _nodes(targets: Targets) -> list[CheckSpec]: + return [ + check( + "nodes.hardening-covers-every-node", + "The node account audit runs on every node in the cluster", + "nodes", + "distinct_count", + [ + step( + "coverage", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "pods", + "--selector", + f"app={targets.node_daemonset}", + "-o", + NODE_NAME_PROJECTION, + ) + ], + { + "step": "coverage", + "minimum": targets.node_count, + "total_equals": targets.node_count, + }, + rationale="A hardened fleet with one unenrolled node is an unhardened fleet.", + ), + ] + + +def _build(targets: Targets) -> list[CheckSpec]: + pipeline = f"{targets.remote}/main:{targets.builder_pipeline}" + return [ + check( + "build.builder-service-account-is-tokenless", + "The image builder service account mounts no API token", + "build", + "stdout_matches", + [ + step( + "serviceaccount", + OPERATOR, + "kubectl", + "--namespace", + targets.builder_namespace, + "get", + "serviceaccount", + targets.builder_serviceaccount, + "-o", + projection("jsonpath={.automountServiceAccountToken}"), + ) + ], + {"step": "serviceaccount", "equals": "false"}, + ), + check( + "build.builder-runs-unprivileged-with-exact-capabilities", + "The build pod is unprivileged and adds only the expected capabilities", + "build", + "names_present", + [step("pipeline", OPERATOR, "git", "show", pipeline, record=False)], + { + "step": "pipeline", + "names": ( + targets.builder_capabilities, + "privileged: false", + "allowPrivilegeEscalation: false", + "automountServiceAccountToken: false", + 'drop: ["ALL"]', + ), + }, + rationale="Exact lines, not a substring sweep: a widened capability set has to change one of these lines to take effect.", + ), + check( + "build.harbor-immutability-rule-is-applied", + "The Harbor immutable-tag rule for the agent image has been applied", + "build", + "lines_match", + [ + step( + "job", + OPERATOR, + "kubectl", + "--namespace", + targets.harbor_namespace, + "get", + "job", + targets.harbor_immutability_job, + "-o", + projection("jsonpath={.status.succeeded}"), + ) + ], + {"step": "job", "pattern": r"^[1-9][0-9]*$", "minimum": 1}, + rationale="The job verifies the rule it creates, so a completed run is evidence the rule matched the expected contract.", + ), + _release_lineage(targets), + ] + + +def _release_lineage(targets: Targets) -> CheckSpec: + """Bind the exact reviewed release from Git through the Ready pods.""" + container = targets.agent_container + namespace = targets.namespace + deployment = f"deploy/{targets.agent_deployment}" + deployment_projection = projection( + ( + "jsonpath={.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}" + "{.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}" + "{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}" + f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}' + ).replace("'", '"') + ) + pod_projection = projection( + ( + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.status.phase}{'\\t'}" + "{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}" + f'{{.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}' + f'{{.status.containerStatuses[?(@.name=="{container}")].imageID}}{{"\\t"}}' + "{.metadata.labels.pod-template-hash}{'\\n'}{end}" + ).replace("'", '"') + ) + replicaset_projection = projection( + ( + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}" + "{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}" + "{.status.readyReplicas}{'\\t'}{.status.availableReplicas}{'\\t'}" + f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}' + "{.metadata.labels.pod-template-hash}{'\\n'}{end}" + ).replace("'", '"') + ) + flux_projection = projection( + ( + "jsonpath={.metadata.name}{'\\t'}{.metadata.generation}{'\\t'}" + "{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}" + "{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}" + "{.status.lastAppliedRevision}" + ).replace("'", '"') + ) + return check( + "release.exact-lineage-is-running", + "Remote main, reviewed PR, image, build, Flux, and running revision are identical", + "release", + "release_lineage", + [ + step( + "remote-main", + OPERATOR, + "git", + "ls-remote", + targets.remote, + "refs/heads/main", + ), + step( + "reviewed-pr", + OPERATOR, + GITEA_CLIENT, + "GET", + f"/api/v1/repos/{targets.repo}/pulls/{targets.reviewed_pr_number}", + ), + step( + "build-source", + OPERATOR, + "git", + "rev-parse", + f"{targets.remote}/main", + ), + step( + "deployment", + OPERATOR, + "kubectl", + "--namespace", + namespace, + "get", + deployment, + "-o", + deployment_projection, + ), + step( + "pods", + OPERATOR, + "kubectl", + "--namespace", + namespace, + "get", + "pods", + "--selector", + f"app={targets.agent_deployment}", + "-o", + pod_projection, + ), + step( + "replicasets", + OPERATOR, + "kubectl", + "--namespace", + namespace, + "get", + "replicasets", + "--selector", + f"app={targets.agent_deployment}", + "-o", + replicaset_projection, + ), + step( + "flux", + OPERATOR, + "kubectl", + "--namespace", + "flux-system", + "get", + "kustomization/hermes", + "-o", + flux_projection, + ), + ], + { + "main_sha": targets.remote_main_sha, + "head_sha": targets.reviewed_head_sha, + "head_ref": targets.reviewed_head_ref, + "pr_number": targets.reviewed_pr_number, + "image": targets.agent_image, + "build_sha": targets.build_sha, + "deployment_revision": targets.deployment_revision, + }, + rationale="A PASS requires one exact release identity across every source and running object.", + ) + + +def _reliability(targets: Targets) -> list[CheckSpec]: + return [ + check( + "reliability.finalization-and-replay-patches-are-deployed", + "The agent workload runs the finalization, replay, and session patches", + "reliability", + "names_present", + [ + step( + "init", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.agent_deployment}", + "-o", + INIT_NAME_PROJECTION, + ) + ], + {"step": "init", "names": targets.agent_init_containers}, + rationale="The regression suites live in the repository; this asserts the fixes they cover are actually running.", + ), + check( + "reliability.regression-suites-are-present-on-main", + "The decomposition and worker-recovery regression suites are on main", + "reliability", + "names_present", + [ + step( + "suites", + OPERATOR, + "git", + "ls-tree", + "--name-only", + f"{targets.remote}/main", + "testing/tests/", + ) + ], + { + "step": "suites", + "names": ( + "testing/tests/test_hermes_cli_lanes.py", + "testing/tests/test_hermes_coordinator.py", + "testing/tests/test_hermes_worker_recovery.py", + ), + }, + ), + ] + + +def delivery_checks(targets: Targets) -> list[CheckSpec]: + """Return the forge, node, build, and reliability checks.""" + return [ + *_forge(targets), + *_nodes(targets), + *_build(targets), + *_reliability(targets), + ] diff --git a/scripts/ops/hermes_handoff_checks_platform.py b/scripts/ops/hermes_handoff_checks_platform.py new file mode 100644 index 00000000..7126036b --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_platform.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +"""Acceptance checks for the release baseline, provider identity, and routing. + +These are the facts a handoff rests on before any authority question is asked: +that the tree under test is the merged one, that both providers authenticate +through the subscriptions they are supposed to, that Switchyard actually decided +the routes it claims to, and that chat, agent, and triage are still three +distinct scopes rather than one surface wearing three hostnames. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + CLAUDE_HEALTH, + CLAUDE_HEALTH_FIELDS, + CODEX_HEALTH, + CODEX_HEALTH_FIELDS, + ENV_NAME_PROJECTION, + OPERATOR, + PROVIDER_API_KEY_NAMES, + ROUTING_LOG, + SELF, + SWITCHYARD, + Targets, + check, + step, +) +from hermes_handoff_model import CheckSpec +from hermes_handoff_policy import GITEA_CLIENT, projection, shell + + +def _baseline(targets: Targets) -> list[CheckSpec]: + api = f"/api/v1/repos/{targets.repo}" + dependency_heads = dict(targets.dependency_heads) + checks = [ + check( + "baseline.local-origin-main-is-current", + "The local origin/main ref equals the fixed remote-main SHA", + "baseline", + "stdout_matches", + [step("main", OPERATOR, "git", "rev-parse", f"{targets.remote}/main")], + {"step": "main", "equals": targets.remote_main_sha}, + rationale="The ancestry check is meaningful only after the local tracking ref is bound to the remote observation.", + ), + check( + "baseline.origin-main-descends-merged-work", + "origin/main descends from the merged Hermes isolation baseline", + "baseline", + "allowed", + [ + step( + "ancestor", + OPERATOR, + "git", + "merge-base", + "--is-ancestor", + targets.baseline_commit, + f"{targets.remote}/main", + ) + ], + rationale="Everything downstream assumes the merged worker-isolation baseline is present.", + ), + check( + "baseline.pending-pull-requests-recorded", + "Exact open pull-request heads are recorded, not assumed", + "baseline", + "json_record", + [ + step( + "open", + OPERATOR, + GITEA_CLIENT, + "GET", + f"{api}/pulls?state=open&limit=50", + ) + ], + { + "step": "open", + "record": { + "numbers": "[].number", + "head_refs": "[].head.ref", + "head_shas": "[].head.sha", + "base_shas": "[].base.sha", + "drafts": "[].draft", + "mergeable": "[].mergeable", + }, + }, + rationale="The runbook has to name the heads it was verified against; stale heads are how a release verifies the wrong tree.", + ), + check( + "baseline.operator-checkout-clean", + "The operator checkout has no uncommitted drift", + "baseline", + "stdout_matches", + [step("status", OPERATOR, "git", "status", "--porcelain")], + {"step": "status", "equals": ""}, + mandatory=False, + rationale="Advisory: a dirty checkout does not invalidate cluster evidence, but it does muddy what was compared.", + ), + ] + checks += [ + check( + f"baseline.dependency-pr-{number}-merged", + f"Pull request #{number} is merged into main", + "baseline", + "json_field", + [step("pr", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls/{number}")], + { + "step": "pr", + "fields": { + "merged": True, + "base.ref": "main", + "head.sha": dependency_heads.get(number, ""), + }, + }, + rationale="This harness certifies the post-merge platform; an unmerged dependency means it is measuring something else.", + ) + for number in targets.dependency_pull_requests + ] + return checks + + +def _identity(targets: Targets) -> list[CheckSpec]: + codex = shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS) + claude = shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS) + agent = f"deploy/{targets.agent_deployment}" + return [ + check( + "identity.codex-is-chatgpt-subscription", + "Codex authenticates through the ChatGPT subscription, not an API key", + "identity", + "json_field", + [step("health", SELF, *codex)], + { + "step": "health", + "fields": { + "authenticated": True, + "transport": "codex-chatgpt-subscription", + "state": "available", + }, + }, + rationale="Only allow-listed status fields are read; the credential itself is never opened.", + ), + check( + "identity.codex-evidence-is-fresh", + "Codex authentication evidence is current", + "identity", + "json_recent", + [step("health", SELF, *codex)], + { + "step": "health", + "now": targets.now, + "fields": {"checked_at": targets.max_evidence_age_seconds}, + }, + rationale="Stale provider evidence describes a state the release no longer has.", + ), + check( + "identity.claude-is-firstparty-subscription", + "Claude authenticates as a claude.ai first-party subscription", + "identity", + "json_field", + [step("health", SELF, *claude)], + { + "step": "health", + "fields": { + "authenticated": True, + "api_provider": "firstParty", + "auth_method": "claude.ai", + "transport": "claude-code-cli-subscription", + "state": "available", + }, + }, + ), + check( + "identity.claude-evidence-is-fresh", + "Claude authentication evidence is current", + "identity", + "json_recent", + [step("health", SELF, *claude)], + { + "step": "health", + "now": targets.now, + "fields": {"checked_at": targets.max_evidence_age_seconds}, + }, + ), + check( + "identity.no-provider-api-key-in-pod", + "No provider API-key variable is set in the running agent process", + "identity", + "names_absent", + [step("env", SELF, *shell("env_names"))], + { + "step": "env", + "names": PROVIDER_API_KEY_NAMES, + "contains": ("API_KEY",), + }, + rationale="Names only. The probe lists variable names and never their values.", + ), + check( + "identity.no-provider-api-key-in-manifest", + "No provider API-key variable is declared on the agent workload", + "identity", + "names_absent", + [ + step( + "env", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + agent, + "-o", + ENV_NAME_PROJECTION, + ) + ], + {"step": "env", "names": PROVIDER_API_KEY_NAMES, "contains": ("API_KEY",)}, + rationale="The operator counterpart to the in-pod probe: desired state and running state are asserted separately.", + ), + ] + + +def _routing(targets: Targets) -> list[CheckSpec]: + switchyard = f"deploy/{targets.switchyard_deployment}" + tail = shell("tail_lines", path=ROUTING_LOG, limit=str(targets.routing_tail_lines)) + routing_step = step( + "routing", SWITCHYARD, *tail, record=False, max_bytes=targets.routing_tail_bytes + ) + return [ + check( + "routing.switchyard-is-available", + "Switchyard has a ready replica serving every model-call boundary", + "routing", + "lines_match", + [ + step( + "deployment", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + switchyard, + "-o", + projection("jsonpath={.status.readyReplicas}"), + ) + ], + {"step": "deployment", "pattern": r"^[1-9][0-9]*$", "minimum": 1}, + ), + check( + "routing.provider-effort-and-fallback-evidence", + "Routing evidence covers both providers, all graded efforts, and a real fallback", + "routing", + "routing_evidence", + [routing_step], + { + "step": "routing", + "providers": ("codex", "claude"), + "efforts": ("medium", "high", "xhigh"), + "lanes": ("route", "worker"), + "require_fallback_evidence": True, + "max_age_seconds": targets.max_evidence_age_seconds, + "now": targets.now, + }, + rationale="The routing log records the decided route per boundary; the tail is parsed here and never pasted into the report.", + ), + check( + "routing.codex-latency-recorded", + "Codex boundary latency is measured, not assumed", + "routing", + "json_numeric", + [ + step( + "health", + SELF, + *shell( + "json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS + ), + ) + ], + {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, + ), + check( + "routing.claude-latency-recorded", + "Claude boundary latency is measured, not assumed", + "routing", + "json_numeric", + [ + step( + "health", + SELF, + *shell( + "json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS + ), + ) + ], + {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, + ), + ] + + +def _scope_profile( + targets: Targets, name: str, workload: str, container: str, expected: str +) -> CheckSpec: + profile_projection = projection( + f'jsonpath={{.spec.template.spec.containers[?(@.name=="{container}")]' + '.env[?(@.name=="HERMES_AUTO_ROUTER_PROFILE")].value}' + ) + return check( + f"scopes.{name}-profile-is-distinct", + f"The {name} surface runs under its own routing profile", + "scopes", + "stdout_matches", + [ + step( + "profile", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + workload, + "-o", + profile_projection, + ) + ], + {"step": "profile", "equals": expected}, + rationale="Chat, agent, and triage must not collapse into one scope; each states its own profile.", + ) + + +def _scopes(targets: Targets) -> list[CheckSpec]: + return [ + _scope_profile( + targets, + "agent", + f"deploy/{targets.agent_deployment}", + targets.agent_container, + "agent", + ), + _scope_profile( + targets, + "chat", + f"statefulset/{targets.chat_statefulset}", + targets.chat_container, + "chat", + ), + _scope_profile( + targets, + "triage", + f"deploy/{targets.triage_deployment}", + targets.triage_container, + "triage", + ), + check( + "scopes.surfaces-have-distinct-hosts", + "Chat, agent, and triage are published on distinct hostnames", + "scopes", + "distinct_count", + [ + step( + "hosts", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "ingress", + "-o", + projection( + 'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}' + ), + ) + ], + {"step": "hosts", "minimum": 3}, + ), + ] + + +def platform_checks(targets: Targets) -> list[CheckSpec]: + """Return the baseline, identity, routing, and scope checks.""" + return [ + *_baseline(targets), + *_identity(targets), + *_routing(targets), + *_scopes(targets), + ] diff --git a/scripts/ops/hermes_handoff_checks_workers.py b/scripts/ops/hermes_handoff_checks_workers.py new file mode 100644 index 00000000..eba7df6d --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_workers.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python3 +"""Acceptance checks for the distributed worker pool and the visible surfaces. + +A pool is only distributed if its workers occupy distinct nodes and distinct +volumes and leave state ownership with the coordinator; a replica count alone +satisfies none of that. A surface is only working if the work it did is visible +in it, which is why the session and Kanban checks look at the same data the UI +renders rather than at whether a pod is Running. + +The ephemeral entries are placeholders in a default run. They report NOT_RUN +until the mutation mode is separately armed, and the armed mode replaces them +with real, mandatory results. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + CHAT, + INIT_NAME_PROJECTION, + OPERATOR, + POOL_PROJECTION, + SELF, + Targets, + check, +) +from hermes_handoff_catalog import step as step +from hermes_handoff_model import EPHEMERAL, CheckSpec +from hermes_handoff_policy import projection + +KANBAN_STATUSES = [ + "archived", + "blocked", + "done", + "ready", + "review", + "running", + "scheduled", + "todo", + "triage", +] + + +def _pool(targets: Targets) -> list[CheckSpec]: + selector = targets.pool_selector + return [ + check( + "pool.three-workers-have-distinct-nodes-and-volumes", + "Exactly three Ready ordinal workers occupy distinct nodes and claims", + "pool", + "pool_topology", + [ + step( + "statefulset", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + projection( + "jsonpath={.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}{.status.currentReplicas}{'\\t'}{.spec.template.spec.automountServiceAccountToken}".replace( + "'", '"' + ) + ), + ), + step( + "workers", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "pods", + "--selector", + selector, + "-o", + POOL_PROJECTION, + ), + ], + { + "state_step": "statefulset", + "worker_step": "workers", + "replicas": targets.pool_replicas, + "name": targets.pool_statefulset, + }, + rationale="Replica count, Ready state, ordinal coverage, node placement, PVC isolation, and tokenlessness are one fail-closed assertion.", + ), + check( + "pool.coordinator-retains-sole-state-ownership", + "No pool worker mounts the coordinator's durable state", + "pool", + "names_absent", + [ + step( + "claims", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + projection( + 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}' + ), + ) + ], + {"step": "claims", "names": targets.coordinator_claims}, + rationale="Two writers on one SQLite state file is the duplicate-work and lost-result failure this pool exists to avoid.", + ), + *_pool_env(targets), + ] + + +def _pool_env(targets: Targets) -> list[CheckSpec]: + """Return the mandatory assignment-safety configuration check.""" + return [ + check( + "pool.assignment-safety-knobs-are-configured", + "Pool workers carry the lease and de-duplication configuration", + "pool", + "names_present", + [ + step( + "env", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + projection( + 'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}' + ), + ) + ], + {"step": "env", "names": targets.pool_worker_env}, + rationale="Deployed configuration only; the stale-claim and duplicate-dispatch behaviour itself is covered by the pool's own regression suite.", + ), + check( + "pool.assignment-fieldrefs-are-bound", + "Worker node and ordinal identities come from fixed downward-API fields", + "pool", + "names_present", + [ + step( + "source", + OPERATOR, + "git", + "show", + f"{targets.remote}/main:services/hermes/execution-worker-statefulset.yaml", + record=False, + ) + ], + { + "step": "source", + "names": ( + "name: HERMES_WORKER_ORDINAL", + "fieldPath: metadata.labels['apps.kubernetes.io/pod-index']", + "name: HERMES_WORKER_NODE", + "fieldPath: spec.nodeName", + ), + }, + rationale="This prevents caller-authored ordinal/node values from impersonating another worker assignment.", + ), + ] + + +def _surfaces(targets: Targets) -> list[CheckSpec]: + checks = [ + check( + "surfaces.kanban-activity-is-visible", + "Durable Kanban activity is visible to the agent surface", + "surfaces", + "all_of_field", + [ + step( + "tasks", + SELF, + "hermes", + "kanban", + "list", + "--json", + record=False, + max_bytes=targets.listing_bytes, + ) + ], + {"step": "tasks", "key_field": "id", "fields": {"status": KANBAN_STATUSES}}, + rationale="Non-empty and well-formed. An empty board is NOT_RUN, because it cannot distinguish a quiet week from a broken read.", + ), + check( + "surfaces.durable-sessions-render-without-blank-rows", + "Durable sessions are listed with a preview, a source, and an id", + "surfaces", + "lines_match", + [ + step( + "sessions", + SELF, + "hermes", + "sessions", + "list", + "--limit", + "20", + record=False, + ) + ], + {"step": "sessions", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + rationale="A row whose leading column is blank is the session-render regression seen from the data side.", + ), + check( + "surfaces.agent-ui-session-patches-are-deployed", + "The agent surface runs the session-activity rendering patches", + "surfaces", + "names_present", + [ + step( + "init", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.agent_deployment}", + "-o", + INIT_NAME_PROJECTION, + ) + ], + { + "step": "init", + "names": ("patch-web-session-activity", "patch-api-server-sessions"), + }, + rationale="Blank session rows were a rendering regression; this asserts the patches that fixed it are in the running pod.", + ), + check( + "surfaces.chat-session-patches-are-deployed", + "The chat surface runs the API-session continuity patches", + "surfaces", + "names_present", + [ + step( + "init", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.chat_statefulset}", + "-o", + INIT_NAME_PROJECTION, + ) + ], + {"step": "init", "names": targets.chat_init_containers}, + ), + check( + "surfaces.chat-telegram-topic-state-is-durable", + "Telegram topic and tenant state is held on durable storage", + "surfaces", + "names_present", + [ + step( + "claims", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.chat_router_deployment}", + "-o", + projection( + 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}' + ), + ) + ], + {"step": "claims", "contains": ("state",)}, + rationale="Topic selection survives a router restart only if the tenant state file is on a claim rather than in the pod.", + ), + ] + checks.append( + check( + "surfaces.chat-telegram-sessions-are-continuous", + "The selected chat tenant still holds durable Telegram-sourced sessions", + "surfaces", + "lines_match", + [ + step( + "telegram", + CHAT, + "hermes", + "sessions", + "list", + "--source", + "telegram", + "--limit", + "20", + record=False, + ) + ], + {"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.", + ) + ) + checks.append( + check( + "surfaces.chat-runs-the-telegram-topic-revision", + "The chat surface runs the durable Telegram topic configuration", + "surfaces", + "stdout_matches", + [ + step( + "revision", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.chat_statefulset}", + "-o", + projection( + "jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}" + ), + ) + ], + {"step": "revision", "equals": targets.chat_config_revision}, + rationale="Telegram topic continuity is carried by the config revision the pod template pins.", + ) + ) + return checks + + +def _ephemeral() -> list[CheckSpec]: + return [ + check( + f"ephemeral.{name}", + title, + "ephemeral", + "not_armed", + [], + {"reason": reason}, + mandatory=False, + scope=EPHEMERAL, + rationale="Mutating checks are separately armed. A default run reports them NOT_RUN and never touches the forge.", + ) + for name, title, reason in ( + ( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + "ephemeral mutation mode is not armed", + ), + ( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + "ephemeral mutation mode is not armed", + ), + ( + "protected-branch-refusal", + "Arming against a protected branch is refused before any network call", + "ephemeral mutation mode is not armed", + ), + ( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + "ephemeral mutation mode is not armed", + ), + ) + ] + + +def worker_checks(targets: Targets) -> list[CheckSpec]: + """Return the pool, surface, and ephemeral-placeholder checks.""" + return [*_pool(targets), *_surfaces(targets), *_ephemeral()] diff --git a/scripts/ops/hermes_handoff_ephemeral.py b/scripts/ops/hermes_handoff_ephemeral.py new file mode 100644 index 00000000..fdcfdbad --- /dev/null +++ b/scripts/ops/hermes_handoff_ephemeral.py @@ -0,0 +1,495 @@ +#!/usr/bin/env python3 +"""Strictly preflighted, discoverable, and self-cleaning armed mode.""" + +from __future__ import annotations + +import configparser +import re +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from hermes_handoff_exec import Outcome, Runner, Vantage +from hermes_handoff_model import EPHEMERAL, FAIL, NOT_RUN, PASS, CheckResult, CheckSpec +from hermes_handoff_policy import ( + EXPECTED_BASE, + EXPECTED_REMOTE, + EXPECTED_REPO, + GITEA_CLIENT, + arm_ephemeral_policy, + register_ephemeral_pull, +) +from hermes_handoff_rules import strict_json + +CONFIRMATION = "ARM EPHEMERAL HERMES HANDOFF PUSH" +REF_PREFIX = "ephemeral/hermes-handoff-acceptance" +REF_RE = re.compile(rf"{REF_PREFIX}/[a-z0-9][a-z0-9-]{{7,63}}\Z") +TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9-]{7,63}\Z") +SHA_RE = re.compile(r"[0-9a-f]{40}\Z") +DRAFT_TITLE_PREFIX = "WIP: " +EXPECTED_HEAD_REF = "feature/hermes-full-handoff-acceptance" +EXPECTED_REMOTE_URL = "https://scm.bstein.dev/atlas/titan-iac.git" +PROTECTED_NAMES = frozenset( + { + "default", + "develop", + "head", + "main", + "master", + "prod", + "production", + "release", + "stable", + "trunk", + } +) + + +class ArmingError(ValueError): + """Raised when an ephemeral run is not safe to start.""" + + +@dataclass(frozen=True) +class ArmRequest: + """Every fixed and caller-confirmed input to one armed run.""" + + repo: str + remote: str + token: str + confirmation: str + base: str = EXPECTED_BASE + expected_head: str = "" + expected_base_sha: str = "" + + @property + def ref(self) -> str: + return f"{REF_PREFIX}/{self.token}" + + +def normalise_branch(name: str) -> str: + """Return a branch name stripped of ref decoration, for comparison.""" + return name.strip().removeprefix("refs/heads/").strip("/").lower() + + +def assert_push_target_allowed(ref: str) -> None: + """Raise unless ``ref`` is the one ephemeral branch grammar.""" + if normalise_branch(ref) in PROTECTED_NAMES: + raise ArmingError(f"refusing to push to protected branch {ref!r}") + if not REF_RE.fullmatch(ref): + raise ArmingError(f"push target {ref!r} is not an ephemeral acceptance ref") + + +def _bounded_read(path: Path, limit: int = 1024 * 1024) -> str: + data = path.read_bytes() + if not data or len(data) > limit: + raise ArmingError(f"local Git metadata is empty or exceeds {limit} bytes") + return data.decode("utf-8", "strict") + + +def _attest_worktree(root: Path, expected_head: str, expected_base_sha: str) -> None: + """Bind cwd, common Git metadata, branch, remote, and exact local HEAD.""" + resolved = root.resolve(strict=True) + marker = resolved / ".git" + if not marker.is_file(): + raise ArmingError( + "armed mode must run at the root of the existing linked worktree" + ) + line = _bounded_read(marker, 4096).strip() + if not line.startswith("gitdir: "): + raise ArmingError("linked-worktree Git metadata is malformed") + gitdir = Path(line.removeprefix("gitdir: ")).resolve(strict=True) + common = (gitdir / _bounded_read(gitdir / "commondir", 4096).strip()).resolve( + strict=True + ) + head = _bounded_read(gitdir / "HEAD", 4096).strip() + expected_ref = f"refs/heads/{EXPECTED_HEAD_REF}" + if head != f"ref: {expected_ref}": + raise ArmingError("armed mode must run from the existing PR #19 feature branch") + ref_path = common / expected_ref + actual_head = _bounded_read(ref_path, 4096).strip() if ref_path.is_file() else "" + if actual_head != expected_head: + raise ArmingError( + "local worktree HEAD does not match the exact reviewed PR head" + ) + remote_ref = common / "refs" / "remotes" / EXPECTED_REMOTE / EXPECTED_BASE + remote_main = ( + _bounded_read(remote_ref, 4096).strip() if remote_ref.is_file() else "" + ) + if remote_main != expected_base_sha: + raise ArmingError("local origin/main does not match the exact release base SHA") + parser = configparser.RawConfigParser(interpolation=None) + try: + parser.read_string(_bounded_read(common / "config")) + remote_url = parser.get(f'remote "{EXPECTED_REMOTE}"', "url") + except (configparser.Error, KeyError) as exc: + raise ArmingError("local Git remote metadata is malformed") from exc + if remote_url != EXPECTED_REMOTE_URL: + raise ArmingError("origin is not the fixed Atlas titan-iac HTTPS repository") + + +def preflight( + request: ArmRequest, expected_repo: str, worktree: Path | None = None +) -> None: + """Validate arming and local provenance before any subprocess/network call.""" + if request.confirmation != CONFIRMATION: + raise ArmingError("confirmation phrase does not match") + if expected_repo != EXPECTED_REPO or request.repo != EXPECTED_REPO: + raise ArmingError( + "armed repository is not the fixed Atlas titan-iac repository" + ) + if ( + request.remote != EXPECTED_REMOTE + or normalise_branch(request.base) != EXPECTED_BASE + ): + raise ArmingError("armed remote and pull-request base are fixed to origin/main") + if not TOKEN_RE.fullmatch(request.token): + raise ArmingError( + "ephemeral token must be 8-64 lowercase alphanumeric or dash characters" + ) + if not SHA_RE.fullmatch(request.expected_head) or not SHA_RE.fullmatch( + request.expected_base_sha + ): + raise ArmingError("exact reviewed head and release base SHAs are required") + assert_push_target_allowed(request.ref) + _attest_worktree( + worktree or Path.cwd(), request.expected_head, request.expected_base_sha + ) + arm_ephemeral_policy(request.ref) + + +def _spec(identifier: str, title: str) -> CheckSpec: + return CheckSpec( + id=f"ephemeral.{identifier}", + title=title, + group="ephemeral", + rule="armed", + steps=(), + mandatory=True, + scope=EPHEMERAL, + ) + + +def _result( + identifier: str, + title: str, + status: str, + reason: str, + outcomes: Sequence[Outcome] = (), + evidence: dict[str, Any] | None = None, +) -> CheckResult: + return CheckResult( + spec=_spec(identifier, title), + status=status, + reason=reason, + evidence=evidence or {}, + outcomes=list(outcomes), + ) + + +def _protected_refusal_result() -> CheckResult: + accepted: list[str] = [] + for candidate in ("main", "master", "refs/heads/main", "HEAD", f"{REF_PREFIX}/x"): + try: + assert_push_target_allowed(candidate) + except ArmingError: + continue + accepted.append(candidate) + return _result( + "protected-branch-refusal", + "Arming against a protected branch is refused before any network call", + FAIL if accepted else PASS, + f"guard accepted {accepted}" + if accepted + else "every protected and malformed target was refused", + evidence={"accepted": accepted}, + ) + + +def _detail(outcome: Outcome) -> str: + return ( + outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}" + ) + + +def _pr_problems(payload: Any, request: ArmRequest) -> list[str]: + if not isinstance(payload, dict): + return ["pull request is not an object"] + wanted = { + "state": "open", + "draft": True, + "merged": False, + "base": {"ref": EXPECTED_BASE, "sha": request.expected_base_sha}, + "head": {"ref": request.ref, "sha": request.expected_head}, + } + problems: list[str] = [] + if ( + not isinstance(payload.get("number"), int) + or isinstance(payload.get("number"), bool) + or payload["number"] <= 0 + ): + problems.append("number is malformed") + for field in ("state", "draft", "merged"): + if ( + type(payload.get(field)) is not type(wanted[field]) + or payload.get(field) != wanted[field] + ): + problems.append(f"{field} does not match") + for group, values in (("base", wanted["base"]), ("head", wanted["head"])): + observed = payload.get(group) + if not isinstance(observed, dict): + problems.append(f"{group} is malformed") + continue + for field, value in values.items(): + if observed.get(field) != value: + problems.append(f"{group}.{field} does not match") + return problems + + +def _discover(outcome: Outcome, request: ArmRequest) -> tuple[list[int], str]: + if not outcome.ok: + return ([], f"pull-request discovery failed: {_detail(outcome)}") + try: + payload = strict_json(outcome.stdout) + except ValueError as exc: + return ([], f"pull-request discovery returned malformed JSON: {exc}") + if not isinstance(payload, list): + return ([], "pull-request discovery did not return an array") + matches: list[int] = [] + for index, item in enumerate(payload): + if ( + not isinstance(item, dict) + or not isinstance(item.get("head"), dict) + or not isinstance(item.get("base"), dict) + ): + return ([], f"pull-request discovery row {index + 1} is malformed") + if item["head"].get("ref") != request.ref: + continue + problems = _pr_problems(item, request) + if problems: + return ([], f"discovered pull request is invalid: {', '.join(problems)}") + matches.append(item["number"]) + if len(matches) != 1: + return ( + matches, + f"expected one exact draft pull request, discovered {len(matches)}", + ) + return (matches, "") + + +def _closed_pr_problems(payload: Any, request: ArmRequest) -> list[str]: + """Validate the same exact PR identity with state changed to closed.""" + problems = _pr_problems(payload, request) + if isinstance(payload, dict) and payload.get("state") == "closed": + problems = [ + problem for problem in problems if problem != "state does not match" + ] + elif "state does not match" not in problems: + problems.append("state is not closed") + return problems + + +def _cleanup( + runner: Runner, + vantage: Vantage, + request: ArmRequest, + api: str, + numbers: Sequence[int], + discovery_error: str = "", +) -> CheckResult: + outcomes: list[Outcome] = [] + problems = [discovery_error] if discovery_error else [] + for number in sorted(set(numbers)): + closed = runner.run( + (GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"), + vantage, + ) + outcomes.append(closed) + if not closed.ok: + problems.append(f"pull request #{number} close failed") + continue + try: + payload = strict_json(closed.stdout) + except ValueError: + problems.append(f"pull request #{number} close response is malformed") + continue + expected = _closed_pr_problems(payload, request) + if expected: + problems.append(f"pull request #{number} close response does not match") + deleted = runner.run( + (GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage + ) + outcomes.append(deleted) + if not deleted.ok: + problems.append("ephemeral branch deletion failed") + remaining = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage + ) + outcomes.append(remaining) + if not remaining.ok or remaining.stdout.strip(): + problems.append("ephemeral branch absence could not be verified") + for number in sorted(set(numbers)): + observed = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage) + outcomes.append(observed) + if not observed.ok: + problems.append(f"pull request #{number} closure could not be verified") + continue + try: + payload = strict_json(observed.stdout) + except ValueError: + problems.append(f"pull request #{number} verification is malformed") + continue + expected = _closed_pr_problems(payload, request) + if expected: + problems.append(f"pull request #{number} is not exactly closed") + return _result( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + FAIL if problems else PASS, + "; ".join(problems) + if problems + else "the exact branch and draft pull request are gone", + outcomes, + {"ref": request.ref, "pull_requests": sorted(set(numbers))}, + ) + + +def run_armed( + runner: Runner, vantage: Vantage, request: ArmRequest +) -> list[CheckResult]: + """Create, discover, validate, and clean one exact ephemeral branch/PR.""" + guard = _protected_refusal_result() + if guard.status != PASS: + return [ + guard, + _result( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + NOT_RUN, + "protected-ref guard failed", + ), + _result( + "draft-pull-request", + "A draft pull request can be opened and closed", + NOT_RUN, + "protected-ref guard failed", + ), + _result( + "cleanup-verified", + "Every ephemeral artifact is removed", + NOT_RUN, + "nothing was created", + ), + ] + api = f"/api/v1/repos/{request.repo}" + reserve = min(6000.0, max(300.0, runner.command_timeout * 10)) + existing = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), + vantage, + reserve_seconds=reserve, + ) + if not existing.ok or existing.stdout.strip(): + reason = ( + _detail(existing) if not existing.ok else "the ephemeral ref already exists" + ) + return [ + guard, + _result( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + FAIL, + reason, + [existing], + ), + _result( + "draft-pull-request", + "A draft pull request can be opened and closed", + NOT_RUN, + "branch preflight failed", + ), + _result( + "cleanup-verified", + "Every ephemeral artifact is removed", + NOT_RUN, + "nothing was created", + ), + ] + local_head = runner.run( + ("git", "rev-parse", "HEAD"), vantage, reserve_seconds=reserve + ) + pushed = ( + runner.run( + ("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"), + vantage, + reserve_seconds=reserve, + ) + if local_head.ok and local_head.stdout.strip() == request.expected_head + else Outcome( + argv=("git", "push"), + vantage=vantage.name, + error="local HEAD changed after static preflight", + ) + ) + verified = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), + vantage, + reserve_seconds=reserve, + ) + branch_ok = verified.ok and verified.stdout.split() == [ + request.expected_head, + f"refs/heads/{request.ref}", + ] + branch_result = _result( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + PASS if branch_ok else FAIL, + "the exact reviewed HEAD was pushed" + if branch_ok + else "push could not be verified exactly", + [local_head, pushed, verified], + {"ref": request.ref, "head": request.expected_head}, + ) + numbers: list[int] = [] + discovery_error = "" + create_outcomes: list[Outcome] = [] + if branch_ok: + created = runner.run( + ( + GITEA_CLIENT, + "POST", + f"{api}/pulls", + "--field", + f"title={DRAFT_TITLE_PREFIX}Hermes handoff acceptance ephemeral probe", + "--field", + f"head={request.ref}", + "--field", + f"base={EXPECTED_BASE}", + "--field", + "body=Ephemeral acceptance probe. Closed and deleted by the harness.", + ), + vantage, + reserve_seconds=reserve, + ) + discovered = runner.run( + (GITEA_CLIENT, "GET", f"{api}/pulls?state=all&limit=50"), + vantage, + reserve_seconds=reserve, + ) + create_outcomes = [created, discovered] + numbers, discovery_error = _discover(discovered, request) + for number in numbers: + register_ephemeral_pull(number) + else: + discovery_error = "branch creation was not verified" + draft_result = _result( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + PASS if len(numbers) == 1 and not discovery_error else FAIL, + f"discovered exact draft pull request #{numbers[0]}" + if len(numbers) == 1 and not discovery_error + else discovery_error, + create_outcomes, + {"pull_requests": numbers}, + ) + cleanup = _cleanup(runner, vantage, request, api, numbers, discovery_error) + return [guard, branch_result, draft_result, cleanup] diff --git a/scripts/ops/hermes_handoff_evaluators.py b/scripts/ops/hermes_handoff_evaluators.py new file mode 100644 index 00000000..61e9e06b --- /dev/null +++ b/scripts/ops/hermes_handoff_evaluators.py @@ -0,0 +1,495 @@ +#!/usr/bin/env python3 +"""Text-shaped evaluators for the handoff acceptance harness. + +These rules cover evidence that arrives as a refusal, an exit status, or a list +of names. The ``denied`` rule is the load-bearing one: it refuses to classify on +an authorization review alone — ``kubectl auth can-i`` reports what RBAC says, +not what the API server does — so a real request must have been made and +refused before it will pass. + +Importing this module also registers the JSON-shaped rules, so a caller only has +to import one place to have the full catalog vocabulary available. +""" + +from __future__ import annotations + +import re +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_json_rules import evaluate_flux_health # noqa: F401 (registers rules) +from hermes_handoff_model import ATTEMPT, FAIL, NOT_RUN, PASS, REVIEW, CheckSpec +from hermes_handoff_rules import ( + EVALUATORS, + Evaluation, + dotted, + evaluate, + evaluator, + exact_equal, + lines, + looks_denied, + missing_steps, + step_keys, + strict_json, +) + +__all__ = ["EVALUATORS", "Evaluation", "evaluate", "evaluator"] + + +@evaluator("denied") +def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass only when the review says no and a real request was refused.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + attempts = step_keys(spec, ATTEMPT) + if not attempts: + return Evaluation( + NOT_RUN, "deny checks require a real attempt, not only a review" + ) + evidence: dict[str, Any] = {} + for key in step_keys(spec, REVIEW): + verdict = outcomes[key].stdout.strip().lower() + evidence[key] = verdict + if verdict != "no": + return Evaluation( + FAIL, + f"authorization review {key} reported {verdict or 'nothing'}", + evidence, + ) + for key in attempts: + outcome = outcomes[key] + if outcome.ok: + return Evaluation( + FAIL, f"attempt {key} succeeded but must be refused", evidence + ) + if not looks_denied(outcome): + return Evaluation( + NOT_RUN, + f"attempt {key} failed without a recognisable refusal (rc={outcome.returncode})", + evidence, + ) + evidence[key] = "refused" + return Evaluation(PASS, "review and live attempt were both refused", evidence) + + +@evaluator("review_denied") +def evaluate_review_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass only when every side-effect-free authorization review says no.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + reviews = step_keys(spec, REVIEW) + if not reviews: + return Evaluation(NOT_RUN, "a denial check requires an authorization review") + evidence: dict[str, str] = {} + for key in reviews: + verdict = outcomes[key].stdout.strip().lower() + evidence[key] = verdict + if verdict != "no": + return Evaluation( + FAIL, + f"authorization review {key} reported {verdict or 'nothing'}", + evidence, + ) + return Evaluation(PASS, "every authorization review was denied", evidence) + + +@evaluator("allowed") +def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every required step succeeded and every review said yes.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + evidence: dict[str, Any] = {} + for step in spec.steps: + outcome = outcomes[step.key] + if step.kind == REVIEW: + verdict = outcome.stdout.strip().lower() + evidence[step.key] = verdict + if verdict != "yes": + return Evaluation( + FAIL, + f"authorization review {step.key} reported {verdict or 'nothing'}", + evidence, + ) + continue + if not outcome.ok: + return Evaluation( + FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence + ) + evidence[step.key] = "succeeded" + return Evaluation(PASS, "required access is present", evidence) + + +def _matched(observed: set[str], patterns: tuple[str, ...]) -> set[str]: + return {name for pattern in patterns for name in observed if pattern in name} + + +def _line_step( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> tuple[set[str], str] | Evaluation: + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + return (set(lines(outcome)), key) + + +@evaluator("names_absent") +def evaluate_names_absent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when no forbidden name or substring appears in a step's lines.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + resolved = _line_step(spec, outcomes) + if isinstance(resolved, Evaluation): + return resolved + observed, _ = resolved + offenders = sorted( + (observed & set(spec.expect.get("names", ()))) + | _matched(observed, tuple(spec.expect.get("contains", ()))) + ) + evidence = {"observed_count": len(observed), "offenders": offenders} + if offenders: + return Evaluation( + FAIL, f"forbidden names present: {', '.join(offenders)}", evidence + ) + return Evaluation(PASS, "no forbidden names are present", evidence) + + +@evaluator("names_present") +def evaluate_names_present(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every required name, and every required substring, appears.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + resolved = _line_step(spec, outcomes) + if isinstance(resolved, Evaluation): + return resolved + observed, _ = resolved + missing = sorted(set(spec.expect.get("names", ())) - observed) + missing += sorted( + pattern + for pattern in spec.expect.get("contains", ()) + if not any(pattern in name for name in observed) + ) + evidence = {"observed_count": len(observed), "missing": missing} + if missing: + return Evaluation( + FAIL, f"required names absent: {', '.join(missing)}", evidence + ) + if not observed: + return Evaluation(NOT_RUN, "step produced no lines to examine", evidence) + return Evaluation(PASS, "every required name is present", evidence) + + +@evaluator("stdout_matches") +def evaluate_stdout_matches( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: + """Compare a step's trimmed stdout against an expected value or set.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + actual = outcome.stdout.strip() + expected = spec.expect["equals"] + allowed = expected if isinstance(expected, list) else [expected] + evidence = {"observed": actual} + if actual not in allowed: + return Evaluation( + FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence + ) + return Evaluation(PASS, "output matches the expected value", evidence) + + +@evaluator("distinct_count") +def evaluate_distinct_count( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: + """Pass when a step yields enough distinct rows — and, optionally, exactly N. + + Distinctness is the point for placement checks: three worker pods pinned to + one node satisfy a replica count and defeat the isolation the replica count + was standing in for. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + values = lines(outcome) + distinct = sorted(set(values)) + minimum = spec.expect["minimum"] + evidence = { + "total": len(values), + "distinct": len(distinct), + "values": distinct[:16], + } + if not values: + return Evaluation(NOT_RUN, f"step {key} produced no rows to count", evidence) + if len(distinct) < minimum: + return Evaluation( + FAIL, + f"{len(distinct)} distinct values, expected at least {minimum}", + evidence, + ) + total_equals = spec.expect.get("total_equals") + if total_equals is not None and len(values) != total_equals: + return Evaluation( + FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence + ) + return Evaluation(PASS, f"{len(distinct)} distinct values observed", evidence) + + +@evaluator("lines_match") +def evaluate_lines_match(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every data row of a tabular listing matches a shape. + + Some surfaces only speak tables. A blank leading column is exactly what the + session-render regression looked like, so asserting the shape of each row is + a real check rather than a proxy for one. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + pattern = re.compile(spec.expect["pattern"]) + rows = outcome.stdout.splitlines()[spec.expect.get("skip", 0) :] + rows = [row for row in rows if row.strip()] + offenders = [row[:60] for row in rows if not pattern.search(row)] + evidence = {"rows": len(rows), "offenders": offenders[:8]} + if len(rows) < spec.expect.get("minimum", 1): + return Evaluation( + NOT_RUN, + f"step {key} produced {len(rows)} data rows, expected at least " + f"{spec.expect.get('minimum', 1)}", + evidence, + ) + if offenders: + return Evaluation( + FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence + ) + return Evaluation(PASS, f"all {len(rows)} rows match the expected shape", evidence) + + +@evaluator("vantages_agree") +def evaluate_vantages_agree( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: + """Pass only when two independent observations report the same fact. + + A disagreement is a finding in its own right: it usually means the manifest + on record and the workload actually running have drifted apart, and either + one alone would have looked clean. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + left_key, right_key = spec.expect["steps"] + observations: dict[str, list[str]] = {} + for key in (left_key, right_key): + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + observations[key] = sorted(lines(outcome)) + evidence = {key: value[:16] for key, value in observations.items()} + if observations[left_key] != observations[right_key]: + return Evaluation( + FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence + ) + if not observations[left_key]: + return Evaluation( + NOT_RUN, "both vantages returned nothing to compare", evidence + ) + return Evaluation(PASS, "both vantages report the same state", evidence) + + +@evaluator("not_armed") +def evaluate_not_armed(spec: CheckSpec, _outcomes: dict[str, Outcome]) -> Evaluation: + """Report the fixed NOT_RUN a default, read-only run must produce.""" + return Evaluation( + NOT_RUN, + spec.expect.get("reason", "ephemeral mutation mode is not armed"), + {"armed": False}, + ) + + +# This validator is one compact, linear state machine so every exit observes +# only fully validated state. +# fmt: off +@evaluator("pool_topology") +def evaluate_pool_topology(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Require exactly three Ready ordinal workers on distinct nodes and claims.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + state = outcomes[spec.expect["state_step"]].stdout.strip().split("\t") + if len(state) != 4: + return Evaluation(NOT_RUN, "StatefulSet projection is malformed") + expected = spec.expect.get("replicas", 3) + try: + replicas, ready, current = (int(value) for value in state[:3]) + except ValueError: + return Evaluation(NOT_RUN, "StatefulSet replica fields are malformed") + evidence: dict[str, Any] = {"replicas": replicas, "ready": ready, "current": current, + "automountServiceAccountToken": state[3]} + if (replicas, ready, current, state[3].lower()) != (expected, expected, expected, "false"): + return Evaluation(FAIL, "worker replica or tokenless state does not match the contract", evidence) + workers: dict[int, tuple[str, str]] = {} + names: list[str] = [] + for number, row in enumerate(lines(outcomes[spec.expect["worker_step"]]), 1): + columns = row.split("\t") + if len(columns) != 5: + return Evaluation(NOT_RUN, f"worker row {number} is malformed", evidence) + name, node, phase, ready_state, claims = columns + match = re.fullmatch(rf"{re.escape(spec.expect['name'])}-(\d+)", name) + claim_list = [claim for claim in claims.split(",") if claim] + if (not match or not node or phase != "Running" or ready_state != "True" + or len(claim_list) != 1): + return Evaluation(FAIL, f"worker row {number} is not a Ready isolated worker", evidence) + ordinal = int(match.group(1)) + if ordinal in workers: + return Evaluation(FAIL, f"worker ordinal {ordinal} is duplicated", evidence) + workers[ordinal] = (node, claim_list[0]) + names.append(name) + evidence.update({"workers": sorted(names), "nodes": sorted({v[0] for v in workers.values()}), + "claims": sorted({v[1] for v in workers.values()})}) + if set(workers) != set(range(expected)): + return Evaluation(FAIL, f"worker ordinals are not exactly 0..{expected - 1}", evidence) + if (len({v[0] for v in workers.values()}) != expected + or len({v[1] for v in workers.values()}) != expected): + return Evaluation(FAIL, "workers do not occupy distinct nodes and claims", evidence) + return Evaluation(PASS, "three Ready workers have distinct nodes, ordinals, and claims", evidence) +# fmt: on + + +_SHA_RE = re.compile(r"[0-9a-f]{40}\Z") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}\Z") +_IMAGE_RE = re.compile( + r"[^@\s]+:git-([0-9a-f]{40})-build-([1-9][0-9]*)@(sha256:[0-9a-f]{64})\Z" +) + + +def _strict_object(outcome: Outcome, label: str) -> dict[str, Any] | Evaluation: + try: + payload = strict_json(outcome.stdout) + except ValueError as exc: + return Evaluation(NOT_RUN, f"{label} JSON is malformed: {exc}") + if not isinstance(payload, dict): + return Evaluation(NOT_RUN, f"{label} JSON is not an object") + return payload + + +# fmt: off +@evaluator("release_lineage") +def evaluate_release_lineage(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Atomically bind Git, PR, image, build, Flux, and running revision evidence.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + expected = spec.expect + main_sha = expected["main_sha"] + head_sha = expected["head_sha"] + image = expected["image"] + build_sha = expected["build_sha"] + revision = str(expected["deployment_revision"]) + digest = image.rsplit("@", 1)[-1] + image_match = _IMAGE_RE.fullmatch(image) + if not (_SHA_RE.fullmatch(main_sha) and _SHA_RE.fullmatch(head_sha) + and _SHA_RE.fullmatch(build_sha) and _DIGEST_RE.fullmatch(digest) + and image_match and image_match.group(1) == build_sha and build_sha == main_sha): + return Evaluation(NOT_RUN, "release lineage expectations are malformed") + remote = outcomes["remote-main"].stdout.strip().split() + if remote != [main_sha, "refs/heads/main"]: + return Evaluation(FAIL, "remote main does not match the fixed expected SHA", + {"remote_main": remote[:2]}) + pr = _strict_object(outcomes["reviewed-pr"], "reviewed PR") + if isinstance(pr, Evaluation): + return pr + fields = { + "number": expected["pr_number"], + "state": "open", + "draft": True, + "merged": False, + "base.ref": "main", + "base.sha": main_sha, + "head.ref": expected["head_ref"], + "head.sha": head_sha, + } + for path, wanted in fields.items(): + try: + actual = dotted(pr, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"reviewed PR is missing {path}") + if not exact_equal(actual, wanted): + return Evaluation(FAIL, f"reviewed PR {path} does not match", {path: actual}) + if outcomes["build-source"].stdout.strip() != build_sha: + return Evaluation(FAIL, "build source SHA does not match the fixed release build SHA") + deployment = outcomes["deployment"].stdout.strip().split("\t") + if len(deployment) != 6: + return Evaluation(NOT_RUN, "Deployment projection is malformed") + generation, observed, replicas, ready, deployed_revision, deployed_image = deployment + try: + current = (int(generation) > 0 and generation == observed + and int(replicas) > 0 and replicas == ready) + except ValueError: + return Evaluation(NOT_RUN, "Deployment numeric fields are malformed") + if not current or (deployed_revision, deployed_image) != (revision, image): + return Evaluation(FAIL, "Deployment lineage does not match the release contract") + replica_hash = "" + for number, row in enumerate(lines(outcomes["replicasets"]), 1): + columns = row.split("\t") + if len(columns) != 6: + return Evaluation(NOT_RUN, f"ReplicaSet row {number} is malformed") + _, rs_revision, rs_ready, rs_available, rs_image, rs_hash = columns + if rs_revision != revision: + continue + if (rs_ready != replicas or rs_available != replicas or rs_image != image or not rs_hash): + return Evaluation(FAIL, "active ReplicaSet does not match the Deployment lineage") + if replica_hash: + return Evaluation(FAIL, "more than one ReplicaSet claims the running revision") + replica_hash = rs_hash + if not replica_hash: + return Evaluation(FAIL, "running Deployment revision has no matching ReplicaSet") + pod_rows = lines(outcomes["pods"]) + if len(pod_rows) != int(replicas): + return Evaluation(FAIL, "running pod count does not match the Deployment") + pods: list[str] = [] + for number, row in enumerate(pod_rows, 1): + columns = row.split("\t") + if len(columns) != 6: + return Evaluation(NOT_RUN, f"pod row {number} is malformed") + name, phase, ready_state, pod_image, image_id, pod_hash = columns + if phase != "Running" or ready_state != "True": + return Evaluation(FAIL, f"pod {name or number} is not Ready") + if (pod_image, pod_hash) != (image, replica_hash) or not image_id.endswith(f"@{digest}"): + return Evaluation(FAIL, f"pod {name or number} lineage does not match") + pods.append(name) + flux = outcomes["flux"].stdout.strip().split("\t") + if len(flux) != 6: + return Evaluation(NOT_RUN, "Flux lineage projection is malformed") + name, generation, observed, suspended, ready_state, applied = flux + if (not name or generation != observed or suspended.lower() == "true" or ready_state != "True"): + return Evaluation(FAIL, "Flux source is not current and Ready") + match = re.search(r"sha1:([0-9a-f]{40})\Z", applied) + if not match or match.group(1) != main_sha: + return Evaluation(FAIL, "Flux applied revision is not the expected remote main") + return Evaluation(PASS, "Git, PR, image, build, Flux, and running revision are bound", + {"main_sha": main_sha, "head_sha": head_sha, "image_digest": digest, + "build_sha": build_sha, "deployment_revision": revision, "pods": pods}) +# fmt: on + + +# Two observations of the same fact agree whether or not they came from +# different vantages; the dual-vantage case is just the important one. +EVALUATORS["steps_agree"] = evaluate_vantages_agree diff --git a/scripts/ops/hermes_handoff_exec.py b/scripts/ops/hermes_handoff_exec.py new file mode 100644 index 00000000..08888b13 --- /dev/null +++ b/scripts/ops/hermes_handoff_exec.py @@ -0,0 +1,499 @@ +#!/usr/bin/env python3 +"""Bounded, screened, provenance-attested command execution. + +It bounds pipes, times out and reaps the process group, and discards truncated output. +""" + +from __future__ import annotations + +import hashlib +import math +import os +import selectors +import shlex +import shutil +import signal +import stat +import subprocess +import time +from collections.abc import Callable, Mapping, Sequence +from contextlib import suppress +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from hermes_handoff_policy import GITEA_CLIENT, READ_ONLY, PolicyError, check_argv +from hermes_handoff_redaction import ( + DEFAULT_MAX_BYTES, + TRUNCATION_NOTE, + redact, + safe_text, + scrub, +) + +DEFAULT_COMMAND_TIMEOUT = 25.0 +DEFAULT_RUN_DEADLINE = 900.0 +DEFAULT_CLEANUP_RESERVE = 90.0 +MAX_COMMAND_TIMEOUT = 600.0 +MAX_RUN_DEADLINE = 7200.0 +MAX_OUTPUT_BYTES = 8 * 1024 * 1024 +MAX_CONCURRENCY = 16 +READ_CHUNK = 64 * 1024 +PIPE_DRAIN_SECONDS = 0.25 + +SAFE_PATH = "/opt/data/tools/bin:/opt/hermes/.venv/bin:/opt/coordinator:/usr/local/bin:/usr/bin:/bin" +# These exact allowlists are intentionally dense: reviewers compare them as a +# single provenance boundary rather than as extensible application config. +# fmt: off +ENV_PASSTHROUGH = ("KUBECONFIG", "LANG", "LC_ALL", "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE") +FIXED_ENVIRONMENT = {"GIT_ASKPASS": "/opt/coordinator/gitea_askpass.sh", + "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", "GITEA_BASE_URL": "https://scm.bstein.dev", + "LC_ALL": "C", "PATH": SAFE_PATH} +EXPECTED_PATHS = { + "kubectl": {"/opt/data/tools/bin/kubectl", "/usr/local/bin/kubectl", "/usr/bin/kubectl"}, + "flux": {"/opt/data/tools/bin/flux", "/usr/local/bin/flux", "/usr/bin/flux"}, + "helm": {"/opt/data/tools/bin/helm", "/usr/local/bin/helm", "/usr/bin/helm"}, + "git": {"/usr/bin/git", "/bin/git"}, + "hermes": {"/opt/hermes/.venv/bin/hermes"}, + "sh": {"/usr/bin/sh", "/bin/sh"}, + GITEA_CLIENT: {GITEA_CLIENT}, +} +EXPECTED_SHA256 = { + "kubectl": {"3d514dbae5dc8c09f773df0ef0f5d449dfad05b3aca5c96b13565f886df345fd"}, + "git": {"a0e562e4bd3c4c79379e91d8c07a10104b2cefe8fac966dc6bd4874a57a807f3"}, + "sh": {"367967c823a0c391e5049b15a67c6a0a629c88b9b6dcdca75ef13ac9d65334b1"}, + "hermes": {"c8419290ef7f1a95f59eadcea9f58f446b1e6535a7f4ffee9f1c0cb1172335e4"}, + GITEA_CLIENT: {"f0943db4e81ce96908e94b0f0e78aa4b5ef95d6c4f7803f9f74a43e761576d85"}, +} + +DEADLINE_ERROR = "deadline-exceeded" +TIMEOUT_ERROR = "timeout" +OPERATOR = "operator" +SELF = "self" +CONTEXT_AWARE_BINARIES = ("kubectl", "flux", "helm") +POD_COMMAND_PATHS = {"kubectl": "/usr/local/bin/kubectl", "hermes": "/opt/hermes/.venv/bin/hermes", + "git": "/usr/bin/git", "sh": "/bin/sh", GITEA_CLIENT: GITEA_CLIENT} +# fmt: on + + +@dataclass(frozen=True) +class Attestation: + """The exact outer executable used for a probe.""" + + path: str + sha256: str + + +@dataclass(frozen=True) +class Capture: + """Raw bounded process result returned by an executor implementation.""" + + returncode: int + stdout: bytes = b"" + stderr: bytes = b"" + stdout_truncated: bool = False + stderr_truncated: bool = False + timed_out: bool = False + + +@dataclass(frozen=True) +class Outcome: + """The screened, bounded result of one probe command.""" + + argv: tuple[str, ...] + vantage: str + returncode: int | None = None + stdout: str = "" + stderr: str = "" + truncated: bool = False + duration_ms: int = 0 + error: str | None = None + executable_path: str = "" + executable_sha256: str = "" + + @property + def ok(self) -> bool: + return self.error is None and self.returncode == 0 and not self.truncated + + @property + def ran(self) -> bool: + return self.error is None + + @property + def combined(self) -> str: + return "\n".join(part for part in (self.stdout, self.stderr) if part) + + def as_dict(self) -> dict[str, object]: + return scrub( + { + "command": redact(shlex.join(self.argv)), + "vantage": self.vantage, + "returncode": self.returncode, + "stdout": self.stdout, + "stderr": self.stderr, + "truncated": self.truncated, + "duration_ms": self.duration_ms, + "error": self.error, + "executable_path": self.executable_path, + "executable_sha256": self.executable_sha256, + } + ) + + +@dataclass(frozen=True) +class Vantage: + """A named identity and transport used to observe a probe.""" + + name: str + prefix: tuple[str, ...] = () + inserts: Mapping[str, tuple[str, ...]] = field(default_factory=dict) + env: Mapping[str, str] = field(default_factory=dict) + command_paths: Mapping[str, str] = field(default_factory=dict) + description: str = "" + + def wrap(self, argv: Sequence[str]) -> tuple[str, ...]: + command = tuple(argv) + if command: + replacement = self.command_paths.get(command[0]) + if replacement: + command = (replacement, *command[1:]) + extra = self.inserts.get(command[0].rsplit("/", 1)[-1]) + if extra: + command = (command[0], *extra, *command[1:]) + return (*self.prefix, *command) + + +def _finite_number(name: str, value: Any, minimum: float, maximum: float) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + converted = float(value) + if not math.isfinite(converted) or not minimum <= converted <= maximum: + raise ValueError( + f"{name} must be finite and between {minimum:g} and {maximum:g}" + ) + return converted + + +def _byte_budget(value: Any) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 1 <= value <= MAX_OUTPUT_BYTES + ): + raise ValueError( + f"max_bytes must be an integer between 1 and {MAX_OUTPUT_BYTES}" + ) + return value + + +def _concurrency(value: Any) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 1 <= value <= MAX_CONCURRENCY + ): + raise ValueError( + f"concurrency must be an integer between 1 and {MAX_CONCURRENCY}" + ) + return value + + +def build_environment( + vantage: Vantage, source: Mapping[str, str] | None = None +) -> dict[str, str]: + """Build a deterministic child environment without caller-controlled PATH.""" + origin = os.environ if source is None else source + environment = {name: origin[name] for name in ENV_PASSTHROUGH if name in origin} + environment.update(FIXED_ENVIRONMENT) + environment.update(vantage.env) + environment["PATH"] = SAFE_PATH + environment["GITEA_BASE_URL"] = FIXED_ENVIRONMENT["GITEA_BASE_URL"] + environment["GIT_ASKPASS"] = FIXED_ENVIRONMENT["GIT_ASKPASS"] + return environment + + +# fmt: off +def operator_vantage(kubeconfig: str | None = None, context: str | None = None) -> Vantage: + inserts = dict.fromkeys(CONTEXT_AWARE_BINARIES, ("--context", context)) if context else {} + detail = " ".join(part for part in (f"context={context}" if context else "", + "kubeconfig-pinned" if kubeconfig else "") if part) + return Vantage(name=OPERATOR, inserts=inserts, + env={"KUBECONFIG": kubeconfig} if kubeconfig else {}, + description=f"operator {detail}".strip()) + + +def pod_vantage(namespace: str, pod: str, container: str, operator: Vantage | None = None) -> Vantage: + reach = operator.inserts.get("kubectl", ()) if operator else () + return Vantage(name=SELF, prefix=("kubectl", *reach, "exec", "--namespace", namespace, + pod, "--container", container, "--"), + env=dict(operator.env) if operator else {}, command_paths=POD_COMMAND_PATHS, + description=f"in-pod {namespace}/{pod}[{container}]") +# fmt: on + + +def attest_executable(command: str, environment: Mapping[str, str]) -> Attestation: + """Resolve and hash an executable from a fixed, trusted path set.""" + name = command if command == GITEA_CLIENT else command.rsplit("/", 1)[-1] + candidate = ( + command if "/" in command else shutil.which(command, path=environment["PATH"]) + ) + if not candidate: + raise OSError(f"trusted executable is unavailable: {name}") + resolved = str(Path(candidate).resolve(strict=True)) + allowed = { + str(Path(item).resolve(strict=False)) + for item in EXPECTED_PATHS.get(name, set()) + } + if resolved not in allowed: + raise OSError(f"executable resolved outside its trusted paths: {name}") + details = os.stat(resolved) + if ( + not stat.S_ISREG(details.st_mode) + or details.st_mode & 0o022 + or not details.st_mode & 0o111 + ): + raise OSError(f"trusted executable has unsafe ownership mode: {name}") + digest = hashlib.sha256() + with open(resolved, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + observed = digest.hexdigest() + expected = EXPECTED_SHA256.get(name) + if not expected or observed not in expected: + raise OSError(f"executable digest is not release-attested: {name}") + return Attestation(resolved, observed) + + +def _append_bounded(buffer: bytearray, chunk: bytes, limit: int) -> bool: + remaining = limit + 1 - len(buffer) + if remaining > 0: + buffer.extend(chunk[:remaining]) + return len(buffer) > limit or len(chunk) > remaining + + +def _signal_group(process: subprocess.Popen[bytes], signum: int) -> None: + with suppress(ProcessLookupError): + os.killpg(process.pid, signum) + + +def execute_bounded( + argv: Sequence[str], + environment: Mapping[str, str], + timeout: float, + max_bytes: int, + clock: Callable[[], float] = time.monotonic, + popen: Callable[..., subprocess.Popen[bytes]] = subprocess.Popen, +) -> Capture: + """Execute one process with bounded live pipe capture and group cleanup.""" + process = popen( + list(argv), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=dict(environment), + text=False, + close_fds=True, + start_new_session=True, + ) + streams = {"stdout": process.stdout, "stderr": process.stderr} + buffers = {"stdout": bytearray(), "stderr": bytearray()} + truncated = {"stdout": False, "stderr": False} + selector = selectors.DefaultSelector() + for name, stream in streams.items(): + if ( + stream is not None + ): # pragma: no branch - PIPE was requested for both streams + os.set_blocking(stream.fileno(), False) + selector.register(stream, selectors.EVENT_READ, name) + expires = clock() + timeout + timed_out = False + try: + while process.poll() is None: + remaining = expires - clock() + if remaining <= 0: + timed_out = True + break + for key, _ in selector.select(min(remaining, 0.1)): + try: + chunk = os.read(key.fileobj.fileno(), READ_CHUNK) + except BlockingIOError: # pragma: no cover - readiness race, retry only + continue + if not chunk: + selector.unregister(key.fileobj) + continue + name = key.data + truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes) + _signal_group(process, signal.SIGKILL if timed_out else signal.SIGTERM) + drain_until = clock() + PIPE_DRAIN_SECONDS + while selector.get_map() and clock() < drain_until: + for key, _ in selector.select(max(0.0, drain_until - clock())): + try: + chunk = os.read(key.fileobj.fileno(), READ_CHUNK) + except BlockingIOError: # pragma: no cover - readiness race, retry only + continue + if not chunk: + selector.unregister(key.fileobj) + continue + name = key.data + truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes) + _signal_group(process, signal.SIGKILL) + returncode = process.wait(timeout=PIPE_DRAIN_SECONDS) + finally: + selector.close() + for stream in streams.values(): + if ( + stream is not None + ): # pragma: no branch - PIPE was requested for both streams + stream.close() + if ( + process.poll() is None + ): # pragma: no branch - fail-safe for wait/selector exceptions + _signal_group(process, signal.SIGKILL) + process.wait() + return Capture( + returncode=returncode, + stdout=bytes(buffers["stdout"][:max_bytes]), + stderr=bytes(buffers["stderr"][:max_bytes]), + stdout_truncated=truncated["stdout"], + stderr_truncated=truncated["stderr"], + timed_out=timed_out, + ) + + +def _injected_executor( + spawn: Callable[..., subprocess.CompletedProcess[Any]], +) -> Callable[..., Capture]: + """Adapt deterministic test doubles without weakening the production path.""" + + def execute( + argv: Sequence[str], + environment: Mapping[str, str], + timeout: float, + max_bytes: int, + ) -> Capture: + completed = spawn( + list(argv), + capture_output=True, + text=True, + timeout=timeout, + check=False, + env=dict(environment), + ) + stdout = str(completed.stdout or "").encode() + stderr = str(completed.stderr or "").encode() + return Capture( + completed.returncode, + stdout[:max_bytes], + stderr[:max_bytes], + len(stdout) > max_bytes, + len(stderr) > max_bytes, + ) + + return execute + + +def _screen_capture(data: bytes, truncated: bool, budget: int) -> str: + if truncated: + return TRUNCATION_NOTE.encode()[:budget].decode() + return safe_text(data.decode("utf-8", "replace"), budget)[0] + + +class Runner: + """Run probes under policy, provenance, timeout, deadline, and byte budgets.""" + + def __init__( + self, + mode: str = READ_ONLY, + command_timeout: float = DEFAULT_COMMAND_TIMEOUT, + deadline_seconds: float = DEFAULT_RUN_DEADLINE, + max_bytes: int = DEFAULT_MAX_BYTES, + concurrency: int = 1, + clock: Callable[[], float] = time.monotonic, + executor: Callable[..., Capture] | None = None, + spawn: Callable[..., subprocess.CompletedProcess[Any]] | None = None, + environ: Mapping[str, str] | None = None, + attestor: Callable[[str, Mapping[str, str]], Attestation] = attest_executable, + ) -> None: + self.mode = mode + self.command_timeout = _finite_number( + "command_timeout", command_timeout, 0.05, MAX_COMMAND_TIMEOUT + ) + deadline = _finite_number( + "deadline_seconds", deadline_seconds, 0.05, MAX_RUN_DEADLINE + ) + self.max_bytes = _byte_budget(max_bytes) + self.concurrency = _concurrency(concurrency) + self._clock = clock + self._executor = executor or ( + _injected_executor(spawn) if spawn else execute_bounded + ) + self._environ = environ + self._attestor = attestor + self._deadline_at = clock() + deadline + self.commands_run = 0 + + @property + def remaining_seconds(self) -> float: + return self._deadline_at - self._clock() + + def run( + self, + argv: Sequence[str], + vantage: Vantage, + max_bytes: int | None = None, + reserve_seconds: float = 0.0, + ) -> Outcome: + budget_bytes = self.max_bytes if max_bytes is None else _byte_budget(max_bytes) + reserve = _finite_number( + "reserve_seconds", reserve_seconds, 0.0, MAX_RUN_DEADLINE + ) + addressed = vantage.wrap(argv) + try: + check_argv(addressed, self.mode) + except PolicyError as exc: + return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}") + budget = min(self.command_timeout, self.remaining_seconds - reserve) + if budget <= 0: + return Outcome(argv=addressed, vantage=vantage.name, error=DEADLINE_ERROR) + environment = build_environment(vantage, self._environ) + try: + attestation = self._attestor(addressed[0], environment) + except (OSError, ValueError) as exc: + return Outcome( + argv=addressed, vantage=vantage.name, error=f"attestation: {exc}" + ) + executed = (attestation.path, *addressed[1:]) + try: + check_argv(executed, self.mode) + except PolicyError as exc: + return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}") + started = self._clock() + self.commands_run += 1 + try: + capture = self._executor(executed, environment, budget, budget_bytes) + except subprocess.TimeoutExpired: + capture = Capture(-signal.SIGKILL, timed_out=True) + except OSError as exc: + return Outcome( + argv=addressed, + vantage=vantage.name, + error=f"spawn: {exc.strerror or type(exc).__name__}", + executable_path=attestation.path, + executable_sha256=attestation.sha256, + ) + was_truncated = capture.stdout_truncated or capture.stderr_truncated + stdout = _screen_capture(capture.stdout, capture.stdout_truncated, budget_bytes) + stderr = _screen_capture(capture.stderr, capture.stderr_truncated, budget_bytes) + return Outcome( + argv=addressed, + vantage=vantage.name, + returncode=capture.returncode, + stdout=stdout, + stderr=stderr, + truncated=was_truncated, + duration_ms=int((self._clock() - started) * 1000), + error=TIMEOUT_ERROR if capture.timed_out else None, + executable_path=attestation.path, + executable_sha256=attestation.sha256, + ) diff --git a/scripts/ops/hermes_handoff_json_rules.py b/scripts/ops/hermes_handoff_json_rules.py new file mode 100644 index 00000000..0f44c6a2 --- /dev/null +++ b/scripts/ops/hermes_handoff_json_rules.py @@ -0,0 +1,474 @@ +#!/usr/bin/env python3 +"""JSON-shaped evaluators for the handoff acceptance harness. + +These rules cover the evidence that arrives as structured output — Kubernetes +objects, forge pull requests, provider health files, and the two append-only +evidence logs the platform keeps. They share the fail-closed contract of every +other rule: an absent path, an unparsable payload, or an empty result set is +``NOT_RUN``, never a quiet pass. +""" + +from __future__ import annotations + +import datetime as dt +import math +import re +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_model import FAIL, NOT_RUN, PASS, CheckSpec +from hermes_handoff_rules import ( + Evaluation, + dotted, + evaluator, + exact_equal, + is_list, + missing_steps, + parsed_step, + strict_json, +) + +_ROUTE_PART_RE = re.compile(r"[a-z0-9][a-z0-9_.-]{0,63}\Z") +_FLUX_NAME_RE = re.compile( + r"[a-z0-9]([-a-z0-9.]*[a-z0-9])?/[a-z0-9]([-a-z0-9.]*[a-z0-9])?\Z" +) + + +def _timestamp(value: Any) -> dt.datetime: + if not isinstance(value, str) or not value.strip(): + raise ValueError("timestamp is not a non-empty string") + observed = dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + if observed.tzinfo is None: + raise ValueError("timestamp has no timezone") + return observed.astimezone(dt.timezone.utc) + + +def _now(value: Any) -> dt.datetime: + if not isinstance(value, dt.datetime) or value.tzinfo is None: + raise ValueError("run clock is not timezone-aware") + return value.astimezone(dt.timezone.utc) + + +@evaluator("json_field") +def evaluate_json_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Compare dotted JSON paths of one step's output against expected values.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + evidence: dict[str, Any] = {} + mismatches: list[str] = [] + for path, expected in spec.expect["fields"].items(): + try: + actual = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) + evidence[path] = actual + allowed = expected if isinstance(expected, list) else [expected] + if not any(exact_equal(actual, item) for item in allowed): + mismatches.append(f"{path}={actual!r} expected one of {allowed!r}") + if mismatches: + return Evaluation(FAIL, "; ".join(mismatches), evidence) + return Evaluation(PASS, "observed fields match the release contract", evidence) + + +@evaluator("json_numeric") +def evaluate_json_numeric(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Compare numeric JSON paths against inclusive minimum/maximum bounds.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + evidence: dict[str, Any] = {} + problems: list[str] = [] + for path, bounds in spec.expect["fields"].items(): + try: + actual = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) + if ( + not isinstance(actual, (int, float)) + or isinstance(actual, bool) + or not math.isfinite(float(actual)) + ): + return Evaluation( + NOT_RUN, f"path {path} is not numeric in step {key}", evidence + ) + evidence[path] = actual + if "min" in bounds and actual < bounds["min"]: + problems.append(f"{path}={actual} below minimum {bounds['min']}") + if "max" in bounds and actual > bounds["max"]: + problems.append(f"{path}={actual} above maximum {bounds['max']}") + if problems: + return Evaluation(FAIL, "; ".join(problems), evidence) + return Evaluation(PASS, "numeric fields are within the expected bounds", evidence) + + +@evaluator("json_recent") +def evaluate_json_recent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when a timestamp field is fresh enough to be evidence about now. + + Stale provider or routing evidence is worse than none: it describes a state + the release no longer has. A timestamp older than the allowed age is a FAIL, + and one the harness cannot parse is NOT_RUN. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + try: + now = _now(spec.expect["now"]) + except (TypeError, ValueError) as exc: + return Evaluation(NOT_RUN, f"invalid run clock: {exc}") + evidence: dict[str, Any] = {} + stale: list[str] = [] + for path, max_age in spec.expect["fields"].items(): + try: + raw = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) + try: + observed = _timestamp(raw) + except (TypeError, ValueError): + return Evaluation( + NOT_RUN, f"path {path} is not a timezone-aware ISO timestamp", evidence + ) + age = (now - observed).total_seconds() + evidence[path] = {"age_seconds": int(age)} + if age < 0: + stale.append(f"{path} is {abs(int(age))}s in the future") + elif age > max_age: + stale.append(f"{path} is {int(age)}s old, limit {max_age}s") + if stale: + return Evaluation(FAIL, "; ".join(stale), evidence) + return Evaluation(PASS, "observed timestamps are fresh", evidence) + + +@evaluator("all_of_field") +def evaluate_all_of_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every selected element of a JSON array matches expected fields.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + try: + elements = dotted(payload, spec.expect.get("array", "")) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"array path is absent from step {key}") + if not is_list(elements): + return Evaluation(NOT_RUN, f"step {key} did not return an array") + key_field = spec.expect.get("key_field", "") + wanted = spec.expect.get("keys") + exempt = set(spec.expect.get("exempt_keys", ())) + seen: list[Any] = [] + problems: list[str] = [] + evidence: dict[str, Any] = {} + for element in elements: + identity = dotted(element, key_field) if key_field else None + if wanted is not None and identity not in wanted: + continue + if identity in exempt: + continue + seen.append(identity) + for path in spec.expect.get("non_empty", ()): + try: + value = dotted(element, path) + except (KeyError, ValueError, IndexError): + value = None + if value in (None, "", [], {}): + problems.append(f"{identity}: {path} is empty") + for path, expected in spec.expect.get("fields", {}).items(): + try: + actual = dotted(element, path) + except (KeyError, ValueError, IndexError): + actual = None + allowed = expected if isinstance(expected, list) else [expected] + if not any(exact_equal(actual, item) for item in allowed): + problems.append( + f"{identity}: {path}={actual!r} expected one of {allowed!r}" + ) + evidence["examined"] = seen[:32] + if wanted is not None: + absent = [item for item in wanted if item not in seen] + if absent: + return Evaluation( + NOT_RUN, f"expected keys absent from step {key}: {absent}", evidence + ) + elif not seen: + return Evaluation(NOT_RUN, f"step {key} returned nothing to examine", evidence) + if problems: + return Evaluation(FAIL, "; ".join(problems[:8]), evidence) + return Evaluation( + PASS, f"{len(seen)} element(s) match the release contract", evidence + ) + + +@evaluator("json_record") +def evaluate_json_record(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Capture named JSON paths as evidence; pass when all of them resolve. + + Recording is a release requirement in its own right — the runbook has to + name the exact heads it was verified against — so an unreachable source is a + NOT_RUN rather than an empty section. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + evidence: dict[str, Any] = {} + for label, path in spec.expect["record"].items(): + try: + evidence[label] = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) + return Evaluation(PASS, "state recorded", evidence) + + +@evaluator("routing_evidence") +def evaluate_routing_evidence( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: + """Summarise a Switchyard routing log tail into provider/effort coverage. + + Route identifiers are ``///``, so provider, + model family, effort tier, and lane all come from the same field. Fallback + and failure evidence is a separate ``fallback_reason`` on the record that + was retried. Every row must be complete strict JSON; a clipped or malformed + line makes the evidence unusable rather than being silently discarded. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + records: list[dict[str, Any]] = [] + routes: list[list[str]] = [] + stamps_by_dimension: dict[tuple[str, str], dt.datetime] = {} + now_value = spec.expect.get("now") + try: + now = _now(now_value) if now_value is not None else None + except (TypeError, ValueError) as exc: + return Evaluation(NOT_RUN, f"invalid routing run clock: {exc}") + for row_number, line in enumerate(outcome.stdout.splitlines(), 1): + line = line.strip() + if not line: + continue + try: + record = strict_json(line) + except ValueError as exc: + return Evaluation( + NOT_RUN, f"routing row {row_number} is malformed JSON: {exc}" + ) + if not isinstance(record, dict): + return Evaluation(NOT_RUN, f"routing row {row_number} is not an object") + try: + observed = _timestamp(record.get("ts")) + except (TypeError, ValueError) as exc: + return Evaluation( + NOT_RUN, f"routing row {row_number} has invalid timestamp: {exc}" + ) + if now is not None and observed > now: + return Evaluation(FAIL, f"routing row {row_number} has a future timestamp") + model = record.get("model") + if not isinstance(model, str) or not model: + return Evaluation(NOT_RUN, f"routing row {row_number} has no model") + route = model.split("/") + if len(route) == 4 and all(_ROUTE_PART_RE.fullmatch(part) for part in route): + routes.append(route) + for label, value in ( + ("provider", route[1]), + ("effort", route[3]), + ("lane", route[0]), + ): + key_dimension = (label, value) + stamps_by_dimension[key_dimension] = max( + observed, stamps_by_dimension.get(key_dimension, observed) + ) + elif record.get("tier") != "classifier": + return Evaluation( + NOT_RUN, f"routing row {row_number} has malformed route {model!r}" + ) + fallback = record.get("fallback_reason") + if fallback is not None and ( + not isinstance(fallback, str) or not fallback.strip() + ): + return Evaluation( + NOT_RUN, f"routing row {row_number} has malformed fallback_reason" + ) + record["_observed"] = observed + records.append(record) + fallbacks = [record for record in records if record.get("fallback_reason")] + stamps = sorted(record["_observed"] for record in records) + evidence: dict[str, Any] = { + "records": len(records), + "routed": len(routes), + "lanes": sorted({route[0] for route in routes}), + "providers": sorted({route[1] for route in routes}), + "families": sorted({route[2] for route in routes}), + "efforts": sorted({route[3] for route in routes}), + "fallback_events": len(fallbacks), + "fallback_reasons": sorted( + {str(record["fallback_reason"]) for record in fallbacks} + ), + "latest_ts": stamps[-1].isoformat() if stamps else "", + } + if not records: + return Evaluation( + NOT_RUN, "routing log tail contained no parsable records", evidence + ) + missing = [ + f"{label}:{item}" + for label, wanted in ( + ("provider", spec.expect.get("providers", ())), + ("effort", spec.expect.get("efforts", ())), + ("lane", spec.expect.get("lanes", ())), + ) + for item in wanted + if item not in evidence[f"{label}s"] + ] + if spec.expect.get("require_fallback_evidence") and not fallbacks: + missing.append("fallback:none-observed") + max_age = spec.expect.get("max_age_seconds") + if max_age and now: + ages: dict[str, int] = {} + for label, wanted in ( + ("provider", spec.expect.get("providers", ())), + ("effort", spec.expect.get("efforts", ())), + ("lane", spec.expect.get("lanes", ())), + ): + for item in wanted: + stamp = stamps_by_dimension.get((label, item)) + if stamp is None: + continue + age = int((now - stamp).total_seconds()) + ages[f"{label}:{item}"] = age + if age > max_age: + missing.append( + f"freshness:{label}:{item}:{age}s-exceeds-{max_age}s" + ) + evidence["age_seconds"] = ages + if missing: + return Evaluation( + FAIL, f"routing evidence missing {', '.join(missing)}", evidence + ) + return Evaluation(PASS, "routing evidence covers the required lanes", evidence) + + +@evaluator("flux_health") +def evaluate_flux_health(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Classify a compact Flux projection into suspensions and unhealthy objects. + + Each row carries name, generation, observed generation, suspension, and all + condition states. A non-suspended object must be current, explicitly + ``Ready=True``, not ``Healthy=False/Unknown``, and not reconciling. Suspended + objects are judged against the exact expected-suspension list. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + expected = set(spec.expect.get("expected_suspensions", ())) + suspended: list[str] = [] + unhealthy: list[str] = [] + total = 0 + for row_number, line in enumerate(outcome.stdout.splitlines(), 1): + if not line.strip(): + continue + columns = line.split("\t") + if len(columns) != 5: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has {len(columns)} columns" + ) + name, generation, observed_generation, suspend, conditions = columns + name = name.strip() + if not _FLUX_NAME_RE.fullmatch(name): + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed object name" + ) + try: + current_generation = int(generation) + reconciled_generation = int(observed_generation) + except ValueError: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed generations" + ) + if current_generation <= 0 or current_generation != reconciled_generation: + return Evaluation( + FAIL, f"Flux row {row_number} has stale observedGeneration" + ) + if suspend.strip().lower() not in {"", "false", "true"}: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed suspension state" + ) + total += 1 + if suspend.strip().lower() == "true": + suspended.append(name) + continue + states: dict[str, str] = {} + for pair in filter(None, conditions.strip().strip(",").split(",")): + if pair.count("=") != 1: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed condition" + ) + condition, state = pair.split("=", 1) + if ( + not condition + or state not in {"True", "False", "Unknown"} + or condition in states + ): + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed condition state" + ) + states[condition] = state + healthy = states.get("Healthy") + if ( + states.get("Ready") != "True" + or healthy in {"False", "Unknown"} + or states.get("Reconciling") == "True" + ): + unhealthy.append(name) + unexpected = sorted(set(suspended) - expected) + evidence = { + "total": total, + "suspended": sorted(suspended), + "unexpected_suspensions": unexpected, + "unhealthy": sorted(unhealthy), + } + if not total: + return Evaluation(NOT_RUN, f"step {key} returned no Flux objects", evidence) + problems = [] + if unexpected: + problems.append(f"unexpected suspensions: {', '.join(unexpected)}") + if unhealthy: + problems.append(f"unhealthy: {', '.join(sorted(unhealthy)[:8])}") + if problems: + return Evaluation(FAIL, "; ".join(problems), evidence) + return Evaluation(PASS, f"{total} Flux objects reconciled as expected", evidence) diff --git a/scripts/ops/hermes_handoff_model.py b/scripts/ops/hermes_handoff_model.py new file mode 100644 index 00000000..d35d39f5 --- /dev/null +++ b/scripts/ops/hermes_handoff_model.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +"""Result model, aggregation, and rendering for the handoff acceptance harness. + +The classification is deliberately four-valued. ``NOT_APPLICABLE`` is the only +way a check can be excused, and a check can only reach it by matching a scope +rule that a reviewer wrote down; anything the harness merely failed to observe +is ``NOT_RUN``. Because a mandatory ``NOT_RUN`` is a NO_GO exactly like a +``FAIL``, an unreachable cluster, an exhausted deadline, or a probe nobody +maintained can never round up to a release. +""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_redaction import contains_credential_shape, safe_text, scrub + +PASS = "PASS" +FAIL = "FAIL" +NOT_RUN = "NOT_RUN" +NOT_APPLICABLE = "NOT_APPLICABLE" +STATUSES = (PASS, FAIL, NOT_RUN, NOT_APPLICABLE) + +GO = "GO" +NO_GO = "NO_GO" + +ALWAYS = "always" +EPHEMERAL = "ephemeral" + +HARNESS = "hermes-handoff-acceptance" +SCHEMA_VERSION = 1 + + +REVIEW = "review" +ATTEMPT = "attempt" +READ = "read" +STEP_KINDS = (REVIEW, ATTEMPT, READ) + + +@dataclass(frozen=True) +class Step: + """One command a check needs, and the vantage it must be observed from. + + ``kind`` separates a self access review (``review``) from a real request the + identity is expected to be refused (``attempt``). A deny check needs both: + an authorization review states policy, only an attempt proves the API server + enforces it. + + ``record`` is cleared for bulk evidence — a routing log tail, a large object + listing — which the harness parses but must not paste into the artifact. + ``max_bytes`` raises the capture budget for that same bulk evidence, where a + default-sized truncation would silently narrow the window being examined. + """ + + key: str + vantage: str + argv: tuple[str, ...] + kind: str = READ + optional: bool = False + record: bool = True + max_bytes: int | None = None + + +@dataclass(frozen=True) +class CheckSpec: + """A declarative acceptance check. + + ``rule`` names an evaluator; ``expect`` carries its parameters. Keeping the + catalog declarative means a reviewer reads what is asserted, not how it is + plumbed, and the harness can prove structural properties over every check + (no impersonation, no mutation, both vantages present) before running any. + """ + + id: str + title: str + group: str + rule: str + steps: tuple[Step, ...] = () + expect: Mapping[str, Any] = field(default_factory=dict) + mandatory: bool = True + scope: str = ALWAYS + rationale: str = "" + + @property + def vantages(self) -> tuple[str, ...]: + """Return the distinct vantages this check draws evidence from.""" + seen: list[str] = [] + for step in self.steps: + if step.vantage not in seen: + seen.append(step.vantage) + return tuple(seen) + + +@dataclass +class CheckResult: + """The classified outcome of one acceptance check.""" + + spec: CheckSpec + status: str + reason: str = "" + evidence: dict[str, Any] = field(default_factory=dict) + outcomes: list[Outcome] = field(default_factory=list) + + @property + def blocking(self) -> bool: + """Report whether this result alone forces NO_GO.""" + if self.status not in STATUSES: + return True + return self.spec.mandatory and self.status in {FAIL, NOT_RUN} + + def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: + """Return a JSON-serialisable, screened view of the result.""" + payload: dict[str, Any] = { + "id": self.spec.id, + "title": self.spec.title, + "group": self.spec.group, + "rule": self.spec.rule, + "scope": self.spec.scope, + "mandatory": self.spec.mandatory, + "vantages": list(self.spec.vantages), + "status": self.status, + "reason": self.reason, + "blocking": self.blocking, + "evidence": self.evidence, + } + if self.spec.rationale: + payload["rationale"] = self.spec.rationale + if include_outcomes: + payload["commands"] = [outcome.as_dict() for outcome in self.outcomes] + return scrub(payload) + + +@dataclass +class VantageRecord: + """What the harness proved about one vantage before trusting its evidence.""" + + name: str + description: str + identity: str = "" + available: bool = False + detail: str = "" + + def as_dict(self) -> dict[str, Any]: + """Return a JSON-serialisable view of the vantage.""" + return scrub( + { + "name": self.name, + "description": self.description, + "identity": self.identity, + "available": self.available, + "detail": self.detail, + } + ) + + +@dataclass +class Report: + """The whole acceptance run: metadata, per-check results, and a verdict.""" + + mode: str + started_at: str + baseline: dict[str, Any] = field(default_factory=dict) + vantages: list[VantageRecord] = field(default_factory=list) + results: list[CheckResult] = field(default_factory=list) + harness_errors: list[str] = field(default_factory=list) + finished_at: str = "" + + @property + def counts(self) -> dict[str, int]: + """Return the number of results in each status.""" + tally = dict.fromkeys(STATUSES, 0) + for result in self.results: + if result.status in tally: + tally[result.status] += 1 + return tally + + @property + def invalid_statuses(self) -> list[str]: + """Return check ids whose status is outside the report schema.""" + return [ + result.spec.id for result in self.results if result.status not in STATUSES + ] + + @property + def blocking(self) -> list[CheckResult]: + """Return every result that forces NO_GO, most severe first.""" + order = {FAIL: 0, NOT_RUN: 1} + return sorted( + (result for result in self.results if result.blocking), + key=lambda result: (order.get(result.status, 2), result.spec.id), + ) + + @property + def decision(self) -> str: + """Return GO only when nothing mandatory failed, skipped, or errored.""" + if ( + not self.results + or self.invalid_statuses + or self.harness_errors + or self.blocking + ): + return NO_GO + return GO + + def group_summary(self) -> dict[str, dict[str, int]]: + """Return per-group status counts, for the human summary.""" + groups: dict[str, dict[str, int]] = {} + for result in self.results: + bucket = groups.setdefault(result.spec.group, dict.fromkeys(STATUSES, 0)) + bucket[result.status] = bucket.get(result.status, 0) + 1 + return groups + + def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: + """Return the machine-readable report.""" + return scrub( + { + "harness": HARNESS, + "schema_version": SCHEMA_VERSION, + "mode": self.mode, + "started_at": self.started_at, + "finished_at": self.finished_at, + "decision": self.decision, + "counts": self.counts, + "baseline": scrub(self.baseline), + "vantages": [vantage.as_dict() for vantage in self.vantages], + "harness_errors": [scrub(error) for error in self.harness_errors], + "invalid_statuses": self.invalid_statuses, + "blocking": [result.spec.id for result in self.blocking], + "checks": [result.as_dict(include_outcomes) for result in self.results], + } + ) + + def render_summary(self, width: int = 88) -> str: + """Return the concise human summary printed alongside the JSON.""" + counts = self.counts + lines = [ + f"{HARNESS}: {self.decision}", + f" mode={self.mode} started={self.started_at} checks={len(self.results)}", + " " + " ".join(f"{status}={counts[status]}" for status in STATUSES), + ] + for vantage in self.vantages: + state = "available" if vantage.available else "unavailable" + identity = vantage.identity or "unknown" + lines.append(f" vantage {vantage.name}: {state} identity={identity}") + for error in self.harness_errors: + lines.append(f" harness error: {error}") + if self.blocking: + lines.append(" blocking:") + for result in self.blocking: + lines.append(f" {result.status:<15}{result.spec.id}") + lines.append(f" {'':<15}{result.reason[:width]}") + groups = self.group_summary() + lines.append(" groups:") + for name in sorted(groups): + bucket = groups[name] + detail = " ".join( + f"{status}={bucket[status]}" for status in STATUSES if bucket[status] + ) + lines.append(f" {name:<18}{detail}") + return safe_text("\n".join(lines), 32 * 1024)[0] + + +def utc_now(clock: Any = None) -> str: + """Return an ISO-8601 UTC timestamp, injectable for deterministic tests.""" + now = clock() if clock else dt.datetime.now(dt.timezone.utc) + return now.replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def unscreened_fields(payload: Any, path: str = "$") -> list[str]: + """Return JSON pointers whose text still looks credential-shaped. + + The report is already screened field by field; this is the invariant check + that runs before anything is written, so a screening gap fails the run + instead of publishing an artifact nobody re-read. + """ + offenders: list[str] = [] + if isinstance(payload, str): + if contains_credential_shape(payload): + offenders.append(path) + elif isinstance(payload, Mapping): + for key, value in payload.items(): + offenders.extend(unscreened_fields(value, f"{path}.{key}")) + elif isinstance(payload, Sequence) and not isinstance(payload, (str, bytes)): + for index, value in enumerate(payload): + offenders.extend(unscreened_fields(value, f"{path}[{index}]")) + return offenders diff --git a/scripts/ops/hermes_handoff_policy.py b/scripts/ops/hermes_handoff_policy.py new file mode 100644 index 00000000..fc1a7097 --- /dev/null +++ b/scripts/ops/hermes_handoff_policy.py @@ -0,0 +1,494 @@ +#!/usr/bin/env python3 +"""Fail-closed command policy for the Hermes handoff acceptance harness. + +Before spawn it admits only credential-safe, structurally read-only grammar; +shell is limited to frozen templates with absolute tool paths. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator + +READ_ONLY = "read-only" +ARMED = "ephemeral-armed" +MODES = (READ_ONLY, ARMED) + +GITEA_CLIENT = "/opt/coordinator/gitea_api.py" +EXPECTED_REPO = "atlas/titan-iac" +EXPECTED_REMOTE = "origin" +EXPECTED_BASE = "main" + +# fmt: off +FORBIDDEN_ARGS = ("--as", "--as-group", "--as-uid", "--client-certificate", + "--client-key", "--password", "--token", "--username") +# fmt: on +IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid") +FORBIDDEN_BINARIES = frozenset( + {"age", "curl", "gpg", "openssl", "sops", "vault", "wget"} +) +ALLOWED_BINARIES = frozenset( + {"kubectl", "flux", "helm", "git", "hermes", "sh", GITEA_CLIENT} +) +TRUSTED_EXECUTABLE_ROOTS = ( + "/bin/", + "/usr/bin/", + "/usr/local/bin/", + "/opt/data/tools/bin/", + "/opt/hermes/.venv/bin/", + "/opt/coordinator/", +) +TRUSTED_INNER_PATHS = { + "/bin/sh": "sh", + "/usr/bin/git": "git", + "/usr/bin/sh": "sh", + "/usr/local/bin/kubectl": "kubectl", + "/opt/hermes/.venv/bin/hermes": "hermes", + GITEA_CLIENT: GITEA_CLIENT, +} + +VALUE_FLAGS = frozenset( + { + "-C", + "-c", + "-l", + "-n", + "-o", + "--cluster", + "--container", + "--context", + "--field-selector", + "--kubeconfig", + "--namespace", + "--output", + "--request-timeout", + "--selector", + "--server", + "--since", + "--tail", + "--user", + } +) + +KUBECTL_READ_SUBCOMMANDS = frozenset( + { + "api-resources", + "api-versions", + "explain", + "get", + "top", + "version", + } +) +KUBECTL_EXEC_SUBCOMMANDS = frozenset({"exec"}) +KUBECTL_SUBCOMMANDS = KUBECTL_READ_SUBCOMMANDS | KUBECTL_EXEC_SUBCOMMANDS | {"auth"} +SECRET_RESOURCES = frozenset({"secret", "secrets"}) +UNSAFE_PROJECTION_PARTS = ( + ".args", + ".command", + ".data", + ".envfrom", + ".secretkeyref", + ".status.token", + ".stringdata", + ".valuefrom", + "clientconfig", + "password", + "privatekey", +) +SAFE_ENV_VALUE = 'env[?(@.name=="hermes_auto_router_profile")].value' +SAFE_JSON_FIELDS = frozenset(["authenticated", "transport", "state", "model", "effort", "latency_ms", "checked_at", "api_provider", "auth_method", "subscription_type"]) # fmt: skip + +FLUX_READ_SUBCOMMANDS = frozenset( + {"check", "events", "get", "stats", "trace", "tree", "version"} +) +HELM_READ_SUBCOMMANDS = frozenset({"history", "list", "status", "version"}) +GIT_READ_SUBCOMMANDS = frozenset( + { + "cat-file", + "diff", + "for-each-ref", + "log", + "ls-remote", + "ls-tree", + "merge-base", + "rev-list", + "rev-parse", + "show", + "status", + } +) +GIT_ARMED_SUBCOMMANDS = frozenset({"push"}) +GIT_FORCE_FLAGS = frozenset( + {"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"} +) +GIT_UNSAFE_FLAGS = ( + "-c", + "--config-env", + "--exec-path", + "--upload-pack", + "--receive-pack", +) +GITEA_READ_METHODS = frozenset({"GET"}) +HERMES_READ_COMMANDS = frozenset( + {("kanban", "list"), ("kanban", "show"), ("sessions", "list"), ("status",)} +) + +# These locations contain credential bytes even when a caller claims it will +# only inspect metadata. They are rejected for every argv, including a frozen +# shell rendering and the inner command of kubectl exec. +SECRET_PATH_RE = re.compile( + r"(?:^|[\s'\"=])(?:/[^\s'\"]*)?(?:" + r"var/run/secrets|run/secrets|vault/secrets|runtime-access|scm-access|pool-access|" + r"\.credentials\.json|auth\.json|\.git-credentials|authorized_keys|" + r"id_[a-z0-9]+|\.netrc|shadow|gshadow|k3s\.yaml" + r")(?:$|[/\s'\"])", + re.IGNORECASE, +) +_PARAMETER_RE = re.compile(r"[A-Za-z0-9/][A-Za-z0-9._/@:,-]{0,255}\Z") +_PLACEHOLDER_RE = re.compile(r"\{([a-z_]+)\}") +_SAFE_API_PATH_RE = re.compile(r"/[A-Za-z0-9][A-Za-z0-9._~:/?&=%+-]{0,511}\Z") + +SHELL_TEMPLATES: dict[str, str] = { + "env_names": ( + "/usr/bin/env | /usr/bin/sed -n " + "'s/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/\\1/p' | /usr/bin/sort" + ), + "json_fields": ( + '/opt/hermes/.venv/bin/python -c "import json,sys;d=json.load(open(sys.argv[1]));' + 'print(json.dumps({{k:d.get(k) for k in sys.argv[2].split(chr(44))}}))" ' + "{path} {fields}" + ), + "tail_lines": "/usr/bin/tail -n {limit} {path}", +} + + +class PolicyError(ValueError): + """Raised when a requested command falls outside the safe boundary.""" + + +def template_parameters(template_name: str) -> set[str]: + """Return the parameter names a frozen template requires.""" + template = SHELL_TEMPLATES.get(template_name) + if template is None: + raise PolicyError(f"unknown shell template: {template_name}") + return set(_PLACEHOLDER_RE.findall(template)) + + +def render_shell(template_name: str, **parameters: str) -> str: + """Return a frozen shell template with validated parameters substituted.""" + required = template_parameters(template_name) + if set(parameters) != required: + raise PolicyError( + f"shell template {template_name} expects parameters {sorted(required)}" + ) + for key, value in parameters.items(): + if not _PARAMETER_RE.fullmatch(value): + raise PolicyError(f"unsafe shell parameter for {template_name}.{key}") + if SECRET_PATH_RE.search(value): + raise PolicyError("shell templates may not name credential paths") + if ( + template_name == "json_fields" + and not set(parameters["fields"].split(",")) <= SAFE_JSON_FIELDS + ): + raise PolicyError("json_fields requested a credential-bearing or unknown field") + return SHELL_TEMPLATES[template_name].format(**parameters) + + +_RENDERED: set[str] = set() +_APPROVED_PROJECTIONS: set[str] = set() +_ARMED_REF = "" +_ARMED_PULLS: set[int] = set() + + +# fmt: off +def arm_ephemeral_policy(ref: str) -> None: + global _ARMED_REF + if not re.fullmatch(r"ephemeral/hermes-handoff-acceptance/[a-z0-9][a-z0-9-]{7,63}", ref): + raise PolicyError("armed policy requires one exact ephemeral ref") + _ARMED_REF = ref + _ARMED_PULLS.clear() + + +def register_ephemeral_pull(number: int) -> None: + if not _ARMED_REF or isinstance(number, bool) or not isinstance(number, int) or number <= 0: + raise PolicyError("ephemeral pull registration is malformed") + _ARMED_PULLS.add(number) + + +def shell(template_name: str, **parameters: str) -> tuple[str, ...]: + """Return a validated ``sh -c`` argv built from a frozen template.""" + script = render_shell(template_name, **parameters) + _RENDERED.add(script) + return ("sh", "-c", script) + + +def projection(value: str) -> str: + normalized = value.lower().replace(" ", "") + if not normalized.startswith("jsonpath=") or any(part in normalized for part in UNSAFE_PROJECTION_PARTS): + raise PolicyError("Kubernetes projection is not credential-safe") + if ".metadata.annotations" in normalized and not any(key in normalized for key in ( + "deployment\\.kubernetes\\.io/revision", "ai\\.bstein\\.dev/config-rev")): + raise PolicyError("Kubernetes annotations are not on the reviewed allowlist") + if ".metadata.labels" in normalized and "pod-template-hash" not in normalized: + raise PolicyError("Kubernetes labels are not on the reviewed allowlist") + if ".env" in normalized and ".value" in normalized and SAFE_ENV_VALUE not in normalized: + raise PolicyError("Kubernetes projection may expose environment values") + _APPROVED_PROJECTIONS.add(normalized) + return value +# fmt: on + + +def positionals(argv: tuple[str, ...], start: int = 1) -> Iterator[str]: + """Yield positional arguments, skipping flags and their values.""" + index = start + while index < len(argv): + item = argv[index] + if item == "--": + return + if item.startswith("-"): + index += 2 if item in VALUE_FLAGS else 1 + continue + yield item + index += 1 + + +def _binary_name(value: str) -> str: + if value == GITEA_CLIENT or ( + value.startswith("/opt/coordinator/") and value.endswith("/gitea_api.py") + ): + return GITEA_CLIENT + if "/" not in value: + return value + mapped = TRUSTED_INNER_PATHS.get(value) + if mapped: + return mapped + if "/../" in value or value.endswith("/..") or "/./" in value: + raise PolicyError(f"executable path is not normalized: {value}") + if not value.startswith(TRUSTED_EXECUTABLE_ROOTS): + raise PolicyError(f"executable path is not trusted: {value}") + return value.rsplit("/", 1)[-1] + + +def _reject_forbidden_arguments( + argv: tuple[str, ...], allow_impersonation: bool +) -> None: + permitted = IMPERSONATION_ARGS if allow_impersonation else () + boundary = argv.index("--") if "--" in argv else len(argv) + for argument in argv[:boundary]: + for forbidden in FORBIDDEN_ARGS: + if forbidden in permitted: + continue + if argument == forbidden or argument.startswith(f"{forbidden}="): + raise PolicyError(f"forbidden argument: {forbidden}") + if argument == "--raw" or argument.startswith("--raw="): + raise PolicyError("raw API reads are forbidden") + for argument in argv: + if argument == "--dry-run" or argument.startswith("--dry-run="): + raise PolicyError("dry-run mutation commands are forbidden") + + +def _reject_secret_paths(argv: tuple[str, ...]) -> None: + if any(SECRET_PATH_RE.search(argument) for argument in argv): + raise PolicyError("probes may not name a credential path") + + +def _resource_name(value: str) -> str: + return value.lower().split("/", 1)[0].split(".", 1)[0] + + +def _output_value(argv: tuple[str, ...]) -> str: + """Return kubectl's output argument in a single canonical form.""" + for index, item in enumerate(argv[1:], 1): + if item in {"-o", "--output"} and index + 1 < len(argv): + return argv[index + 1] + if item.startswith(("-o=", "--output=")): + return item.split("=", 1)[1] + return "" + + +def _check_projection(output: str) -> None: + """Reject full or credential-bearing Kubernetes output projections.""" + lowered = output.lower().replace(" ", "") + if lowered in {"json", "yaml", "wide"} or lowered.startswith("custom-columns"): + raise PolicyError( + "full Kubernetes object output is forbidden; use a safe projection" + ) + if lowered and not (lowered == "name" or lowered.startswith("jsonpath=")): + raise PolicyError( + "Kubernetes output must be name or an approved JSONPath projection" + ) + if lowered.startswith("jsonpath="): + if any(part in lowered for part in UNSAFE_PROJECTION_PARTS): + raise PolicyError("Kubernetes projection may expose runtime credentials") + if ".env" in lowered and SAFE_ENV_VALUE not in lowered: + # Environment names are safe; arbitrary `.value` reads are not. + without_names = lowered.replace("{.name}", "") + if ".value" in without_names: + raise PolicyError("Kubernetes projection may expose environment values") + if lowered not in _APPROVED_PROJECTIONS: + raise PolicyError( + "Kubernetes JSONPath projection is not on the reviewed allowlist" + ) + + +def _check_kubectl(argv: tuple[str, ...], mode: str, allow_impersonation: bool) -> None: + words = list(positionals(argv)) + if not words or words[0] not in KUBECTL_SUBCOMMANDS: + raise PolicyError("kubectl subcommand is outside the read-only boundary") + subcommand = words[0] + if subcommand == "auth": + if len(words) < 2 or words[1] not in {"can-i", "whoami"}: + raise PolicyError("kubectl auth is limited to can-i and whoami") + return + if subcommand == "exec": + if "--" not in argv: + raise PolicyError("kubectl exec requires an explicit -- command boundary") + inner = argv[argv.index("--") + 1 :] + if not inner: + raise PolicyError("kubectl exec requires an inner command") + if inner[0] not in TRUSTED_INNER_PATHS: + raise PolicyError( + "kubectl exec inner command must use a fixed trusted path" + ) + check_argv(inner, mode, allow_impersonation=True) + return + resources = words[1].split(",") if len(words) > 1 else [] + if any(_resource_name(resource) in SECRET_RESOURCES for resource in resources): + raise PolicyError("Secret objects may not be read by the harness") + output = _output_value(argv) + if subcommand == "get" and not output: + raise PolicyError("kubectl get requires a reviewed output projection") + _check_projection(output) + + +def _first_positional(binary: str, argv: tuple[str, ...]) -> str: + word = next(positionals(argv), None) + if word is None: + raise PolicyError(f"{binary} requires a subcommand") + return word + + +def _check_read_only_tool( + binary: str, argv: tuple[str, ...], allowed: frozenset[str] +) -> None: + subcommand = _first_positional(binary, argv) + if subcommand not in allowed: + raise PolicyError(f"{binary} {subcommand} is outside the read-only boundary") + + +def _check_git(argv: tuple[str, ...], mode: str) -> None: + unsafe = False + for item in argv[1:]: + if item == "-C" or item.startswith(("--git-dir", "--work-tree", "--namespace")): + unsafe = True + if item == "-c" or (item.startswith("-c") and len(item) > 2): + unsafe = True + if any( + item == flag or item.startswith(f"{flag}=") for flag in GIT_UNSAFE_FLAGS[1:] + ): + unsafe = True + if unsafe: + raise PolicyError("git configuration and helper overrides are forbidden") + subcommand = _first_positional("git", argv) + if subcommand in GIT_READ_SUBCOMMANDS: + return + if subcommand == "push" and mode == ARMED: + if GIT_FORCE_FLAGS & set(argv): + raise PolicyError("git push may not force or fan out across refs") + words = list(positionals(argv)) + if len(words) != 3 or words[1] != EXPECTED_REMOTE: + raise PolicyError("armed git push is pinned to origin and one refspec") + destination = words[2].partition(":")[2] + if not _ARMED_REF or destination != f"refs/heads/{_ARMED_REF}": + raise PolicyError("armed git push is not the preflighted ephemeral ref") + return + raise PolicyError(f"git {subcommand} is not permitted in {mode} mode") + + +def _check_hermes(argv: tuple[str, ...]) -> None: + words = tuple(positionals(argv)) + if not words: + raise PolicyError("hermes requires a subcommand") + if words[:2] not in HERMES_READ_COMMANDS and words[:1] not in HERMES_READ_COMMANDS: + raise PolicyError( + f"hermes {' '.join(words[:2])} is outside the read-only boundary" + ) + + +def _check_gitea(argv: tuple[str, ...], mode: str) -> None: + if len(argv) < 3: + raise PolicyError("gitea_api.py requires a method and a path") + method, path = argv[1].upper(), argv[2] + if not path.startswith("/api/v1/") or not _SAFE_API_PATH_RE.fullmatch(path): + raise PolicyError("gitea_api.py requires a bounded API path") + if path != "/api/v1/user" and not path.startswith(f"/api/v1/repos/{EXPECTED_REPO}"): + raise PolicyError("gitea_api.py is pinned to the Atlas titan-iac repository") + if method in GITEA_READ_METHODS: + return + repo_root = f"/api/v1/repos/{EXPECTED_REPO}" + if mode == ARMED: + fields = [ + argv[index + 1] for index, item in enumerate(argv[:-1]) if item == "--field" + ] + if method == "POST" and path == f"{repo_root}/pulls" and _ARMED_REF: + required = { + "title=WIP: Hermes handoff acceptance ephemeral probe", + f"head={_ARMED_REF}", + f"base={EXPECTED_BASE}", + "body=Ephemeral acceptance probe. Closed and deleted by the harness.", + } + if len(fields) == len(required) and set(fields) == required: + return + match = re.fullmatch(rf"{re.escape(repo_root)}/pulls/([1-9][0-9]*)", path) + if ( + method == "PATCH" + and match + and int(match.group(1)) in _ARMED_PULLS + and fields == ["state=closed"] + ): + return + if ( + method == "DELETE" + and _ARMED_REF + and path == f"{repo_root}/branches/{_ARMED_REF}" + and len(argv) == 3 + ): + return + raise PolicyError(f"gitea_api.py {method} is not permitted in {mode} mode") + + +def _check_shell(argv: tuple[str, ...]) -> None: + if len(argv) != 3 or argv[1] != "-c" or argv[2] not in _RENDERED: + raise PolicyError("sh is only reachable as sh -c ") + + +def check_argv( + argv: tuple[str, ...], mode: str = READ_ONLY, allow_impersonation: bool = False +) -> None: + """Raise :class:`PolicyError` unless ``argv`` is safe in ``mode``.""" + if mode not in MODES: + raise PolicyError(f"unknown mode: {mode}") + if not argv: + raise PolicyError("empty command") + binary = _binary_name(argv[0]) + if binary in FORBIDDEN_BINARIES: + raise PolicyError(f"forbidden binary: {binary}") + if binary not in ALLOWED_BINARIES: + raise PolicyError(f"binary is not on the harness allowlist: {binary}") + _reject_forbidden_arguments(argv, allow_impersonation) + _reject_secret_paths(argv) + if binary == "sh": + _check_shell(argv) + elif binary == "kubectl": + _check_kubectl(argv, mode, allow_impersonation) + elif binary == "flux": + _check_read_only_tool("flux", argv, FLUX_READ_SUBCOMMANDS) + elif binary == "helm": + _check_read_only_tool("helm", argv, HELM_READ_SUBCOMMANDS) + elif binary == "git": + _check_git(argv, mode) + elif binary == "hermes": + _check_hermes(argv) + else: + _check_gitea(argv, mode) diff --git a/scripts/ops/hermes_handoff_redaction.py b/scripts/ops/hermes_handoff_redaction.py new file mode 100644 index 00000000..f7b97f4c --- /dev/null +++ b/scripts/ops/hermes_handoff_redaction.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Credential redaction and output bounding for the handoff acceptance harness. + +Every string the harness records — command output, failure reasons, evidence +values — passes through :func:`redact` before it reaches a report. Screening is +a fail-closed accident barrier, not proof that text is secret-free: a short +secret under an innocuous key is not reliably distinguishable from prose. The +harness therefore also never asks for a secret in the first place. Probes read +names, shapes, and status fields; they never read environment values, token +files, private keys, cookies, or Vault responses. + +Bounding is part of the same barrier. Screening always precedes truncation so a +credential that crosses the byte boundary cannot leave a useful prefix behind. +The process runner additionally discards all captured bytes when a live stream +exceeds its cap; the fixed truncation marker is the only recorded output. +""" + +from __future__ import annotations + +import re +from typing import Any + +REDACTED = "[redacted]" +DEFAULT_MAX_BYTES = 64 * 1024 +TRUNCATION_NOTE = "[truncated]" + +# Keys whose value is never safe to record, matched against a normalised +# (lowercased, separator-stripped) form of the key. +SENSITIVE_KEY_SUFFIXES = ( + "accesskey", + "accesskeyid", + "apikey", + "authtoken", + "clientsecret", + "connectionstring", + "cookie", + "credential", + "credentials", + "passphrase", + "passwd", + "password", + "privatekey", + "secret", + "secretkey", + "sessionkey", + "signature", + "signingkey", + "token", + "webhook", +) +SENSITIVE_KEY_EXACT = frozenset( + { + "auth", + "authorization", + "bearer", + "pat", + "pw", + "sas", + "setcookie", + } +) +# A bare `key:` is a structural field name across Kubernetes and TOML far more +# often than it is a credential, so it is screened by value shape and by the +# token patterns rather than by name. The compound forms above still match. + +# Ordinary prose values shorter than this threshold are not independently +# treated as secret-shaped. Values under sensitive keys are screened at every +# length, including numeric values. +MIN_SECRET_LENGTH = 8 +NON_SECRET_LITERALS = frozenset( + {"true", "false", "null", "none", "yes", "no", "0", "1", ""} +) +_KEY_NOISE_RE = re.compile(r"[^a-z0-9]+") + +# Ordered: structural matches first so a nested token is not partially rewritten +# by a looser rule. +_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile( + r"-----BEGIN[ A-Z0-9-]{0,64}-----.*?-----END[ A-Z0-9-]{0,64}-----", + re.DOTALL, + ), + re.compile(r"-----BEGIN[ A-Z0-9-]{0,64}-----.*", re.DOTALL), + re.compile(r"(?i)\b(?:authorization|proxy-authorization)\s*:\s*\S.*"), + re.compile(r"(?i)\b(?:set-)?cookie\s*:\s*\S.*"), + re.compile(r"(?i)\b(?:bearer|basic|token)\s+[A-Za-z0-9+/_=.:~-]{8,}"), + re.compile(r"(?i)\b[a-z][a-z0-9+.-]{1,20}://[^\s/@:]{1,128}:[^\s/@]{1,256}@"), + re.compile( + r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}" + ), + re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}"), + re.compile(r"(?["']?) + (?P[A-Za-z][A-Za-z0-9_.\ -]{0,63}) + (?P=quote) + (?P\s{0,8}[:=]\s{0,8}) + (?P"[^"\n]{0,4096}"|'[^'\n]{0,4096}'|[^\s,;}\n][^,;}\n]{0,4096}) + """ +) + +# Long, mixed-class runs. Deliberately conservative: prose and hex digests of +# fewer than 44 characters are left alone so commit SHAs stay readable. +_ENTROPY_RE = re.compile( + r"(? str: + """Return a key lowercased with separators removed, for suffix matching.""" + return _KEY_NOISE_RE.sub("", key.lower()) + + +def is_sensitive_key(key: str) -> bool: + """Report whether a key name means its value must never be recorded.""" + compact = normalise_key(key) + if not compact: + return False + if compact in SENSITIVE_KEY_EXACT: + return True + return compact.endswith(SENSITIVE_KEY_SUFFIXES) + + +def _is_high_entropy(candidate: str) -> bool: + """Report whether a long run mixes enough character classes to be a token.""" + classes = sum( + ( + any(character.islower() for character in candidate), + any(character.isupper() for character in candidate), + any(character.isdigit() for character in candidate), + any(character in "+/_=-" for character in candidate), + ) + ) + return classes >= 4 + + +def could_hold_secret(value: str) -> bool: + """Report whether a raw value is long and opaque enough to be a credential.""" + stripped = value.strip().strip("\"'") + if stripped.lower() in NON_SECRET_LITERALS or len(stripped) < MIN_SECRET_LENGTH: + return False + try: + float(stripped) + except ValueError: + return True + return False + + +def _redact_assignment(match: re.Match[str]) -> str: + value = match.group("value") + stripped = value.strip().strip("\"'").lower() + if not is_sensitive_key(match.group("key")) or stripped in NON_SECRET_LITERALS: + return match.group(0) + # Preserve the original quoting so screened JSON stays parsable JSON. + quoted = value.strip()[:1] in {'"', "'"} + replacement = f'"{REDACTED}"' if quoted else REDACTED + key_quote = match.group("quote") + return ( + f"{key_quote}{match.group('key')}{key_quote}{match.group('gap')}{replacement}" + ) + + +def _redact_entropy(match: re.Match[str]) -> str: + return REDACTED if _is_high_entropy(match.group(0)) else match.group(0) + + +def redact(text: str) -> str: + """Return ``text`` with credential-shaped substrings replaced.""" + if not text: + return text + for pattern in _PATTERNS: + text = pattern.sub(REDACTED, text) + text = _ASSIGNMENT_RE.sub(_redact_assignment, text) + return _ENTROPY_RE.sub(_redact_entropy, text) + + +def bound(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: + """Truncate ``text`` to a UTF-8 byte budget, reporting whether it was cut.""" + if max_bytes <= 0: + return ("", bool(text)) + encoded = text.encode("utf-8", "replace") + if len(encoded) <= max_bytes: + return (text, False) + marker = TRUNCATION_NOTE.encode()[:max_bytes] + content_budget = max( + 0, max_bytes - len(marker) - (1 if len(marker) < max_bytes else 0) + ) + kept = encoded[:content_budget].decode("utf-8", "ignore") + separator = "\n" if kept and len(marker) < max_bytes else "" + return (f"{kept}{separator}{marker.decode()}", True) + + +def safe_text(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: + """Redact then bound, so truncation never exposes a credential prefix.""" + return bound(redact(text), max_bytes) + + +def scrub(value: Any, max_bytes: int = DEFAULT_MAX_BYTES) -> Any: + """Recursively bound and redact a JSON-shaped value. + + Mapping keys are themselves screened; values under a sensitive key are + dropped outright rather than pattern-screened, because a short secret + would survive screening. + """ + if isinstance(value, str): + return safe_text(value, max_bytes)[0] + if isinstance(value, dict): + screened: dict[str, Any] = {} + for key, item in value.items(): + raw_key = str(key) + safe_key = safe_text(raw_key, max_bytes)[0] + sensitive_value = ( + is_sensitive_key(raw_key) + and item is not None + and not isinstance(item, bool) + and item != "" + ) + screened[safe_key] = REDACTED if sensitive_value else scrub(item, max_bytes) + return screened + if isinstance(value, (list, tuple)): + return [scrub(item, max_bytes) for item in value] + return value + + +def contains_credential_shape(text: str) -> bool: + """Report whether screening would rewrite ``text``. + + Used by the harness self-test to prove a rendered report is clean before it + is written, and by the unit suite as an assertion helper. + """ + return redact(text) != text diff --git a/scripts/ops/hermes_handoff_rules.py b/scripts/ops/hermes_handoff_rules.py new file mode 100644 index 00000000..5cca2c5a --- /dev/null +++ b/scripts/ops/hermes_handoff_rules.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Shared primitives and dispatch for handoff acceptance evaluators. + +Evaluators are registered by name so the check catalog stays declarative: an +entry says *what* it asserts, not how to plumb it. The dispatch here is the one +place that turns an unexpected exception into ``NOT_RUN`` — a harness bug must +never be indistinguishable from a passing release. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_model import NOT_RUN, STATUSES, CheckSpec + +DENIAL_MARKERS = ( + "forbidden", + "is not allowed", + "unauthorized", + "permission denied", + "cannot get", + "cannot list", + "cannot create", + "cannot delete", + "cannot patch", + "not permitted", + "gitea api returned http 403", +) + + +@dataclass +class Evaluation: + """A classification plus the small facts that justify it.""" + + status: str + reason: str + evidence: dict[str, Any] = field(default_factory=dict) + + +Evaluator = Callable[[CheckSpec, dict[str, Outcome]], Evaluation] +EVALUATORS: dict[str, Evaluator] = {} + + +def evaluator(name: str) -> Callable[[Evaluator], Evaluator]: + """Register an evaluator under the rule name catalog entries use.""" + + def register(function: Evaluator) -> Evaluator: + EVALUATORS[name] = function + return function + + return register + + +def missing_steps( + spec: CheckSpec, outcomes: Mapping[str, Outcome] +) -> Evaluation | None: + """Return NOT_RUN when a required step never produced usable evidence.""" + for step in spec.steps: + if step.optional: + continue + outcome = outcomes.get(step.key) + if outcome is None: + return Evaluation(NOT_RUN, f"step {step.key} was never executed") + if outcome.truncated: + return Evaluation(NOT_RUN, f"step {step.key} output was truncated") + if not outcome.ran: + return Evaluation(NOT_RUN, f"step {step.key} did not run: {outcome.error}") + return None + + +def looks_denied(outcome: Outcome) -> bool: + """Report whether a real request was refused by policy rather than failing.""" + if outcome.ok: + return False + haystack = outcome.combined.lower() + return any(marker in haystack for marker in DENIAL_MARKERS) + + +def step_keys(spec: CheckSpec, kind: str) -> tuple[str, ...]: + """Return the keys of every step of one kind.""" + return tuple(step.key for step in spec.steps if step.kind == kind) + + +def is_list(value: Any) -> bool: + """Report whether a value is a JSON array rather than a string.""" + return isinstance(value, Sequence) and not isinstance(value, (str, bytes)) + + +def exact_equal(actual: Any, expected: Any) -> bool: + """Compare JSON values without Python's bool/int type confusion.""" + return type(actual) is type(expected) and actual == expected + + +def dotted(payload: Any, path: str) -> Any: + """Return a value addressed by a dotted path. + + ``a.b[0].c`` indexes; ``a[].b`` maps the remainder of the path over a list, + which is how the catalog records one field from every pull request or every + Flux object without a bespoke parser per check. + """ + current = payload + segments = [segment for segment in path.split(".") if segment] + for position, raw in enumerate(segments): + name, bracket, indexes = raw.partition("[") + if name: + if not isinstance(current, Mapping) or name not in current: + raise KeyError(path) + current = current[name] + if not bracket: + continue + for chunk in indexes.rstrip("]").split("]["): + if not is_list(current): + raise KeyError(path) + if chunk == "": + remainder = ".".join(segments[position + 1 :]) + return [ + dotted(item, remainder) if remainder else item for item in current + ] + offset = int(chunk) + if offset >= len(current): + raise KeyError(path) + current = current[offset] + return current + + +def strict_json(text: str) -> Any: + """Decode strict JSON, rejecting constants and duplicate object keys.""" + text = text.strip() + if not text: + raise ValueError("empty output") + + def reject_constant(value: str) -> None: + raise ValueError(f"non-finite JSON constant {value}") + + def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key {key!r}") + result[key] = value + return result + + return json.loads( + text, + strict=True, + parse_constant=reject_constant, + object_pairs_hook=unique_object, + ) + + +def parse_json(outcome: Outcome) -> Any: + """Parse one outcome as strict JSON.""" + return strict_json(outcome.stdout) + + +def lines(outcome: Outcome) -> list[str]: + """Return non-empty, stripped stdout lines.""" + return [line.strip() for line in outcome.stdout.splitlines() if line.strip()] + + +def parsed_step( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> tuple[Any, str] | Evaluation: + """Return the parsed payload of ``expect['step']`` or the NOT_RUN to report.""" + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + try: + return (parse_json(outcome), key) + except ValueError as exc: + return Evaluation(NOT_RUN, f"step {key} did not return JSON: {exc}") + + +def evaluate(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Dispatch to the named evaluator, converting any surprise into NOT_RUN.""" + rule = EVALUATORS.get(spec.rule) + if rule is None: + return Evaluation(NOT_RUN, f"unknown evaluator: {spec.rule}") + truncated = sorted(key for key, outcome in outcomes.items() if outcome.truncated) + if truncated: + return Evaluation( + NOT_RUN, f"truncated evidence from step(s): {', '.join(truncated)}" + ) + try: + result = rule(spec, outcomes) + except Exception as exc: # noqa: BLE001 - a harness bug must never read as PASS + return Evaluation( + NOT_RUN, f"evaluator {spec.rule} raised {type(exc).__name__}: {exc}" + ) + if result.status not in STATUSES: + return Evaluation( + NOT_RUN, f"evaluator {spec.rule} returned unknown status {result.status!r}" + ) + return result diff --git a/scripts/ops/hermes_handoff_run.py b/scripts/ops/hermes_handoff_run.py new file mode 100644 index 00000000..a3b21cd7 --- /dev/null +++ b/scripts/ops/hermes_handoff_run.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 +"""Orchestration for the Hermes full-handoff acceptance harness. + +Three things happen before any check runs, and each of them can stop the run on +its own: + +* the catalog is validated structurally — every rule exists, every live-denial + evaluator carries a read-only attempt, no operator-side step can impersonate; +* each vantage is resolved and asked who it is; and +* the operator and in-pod identities are compared. If they are the same + principal there is only one vantage, the dual-vantage evidence would be a + restatement rather than a corroboration, and the run is a NO_GO. + +Everything after that is uniform: run a check's steps against its vantage, hand +the outcomes to the named evaluator, and record the classification. A vantage +that could not be resolved yields ``NOT_RUN`` for every check that needs it, +which is exactly the fail-closed answer. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping, Sequence + +from hermes_handoff_catalog import ( + CHAT, + NODE, + OPERATOR, + SELF, + SWITCHYARD, + VANTAGE_NAMES, + Targets, +) +from hermes_handoff_checks_access import access_checks +from hermes_handoff_checks_delivery import delivery_checks +from hermes_handoff_checks_platform import platform_checks +from hermes_handoff_checks_workers import worker_checks +from hermes_handoff_evaluators import EVALUATORS, evaluate +from hermes_handoff_exec import Outcome, Runner, Vantage, operator_vantage, pod_vantage +from hermes_handoff_model import ( + ATTEMPT, + EPHEMERAL, + NOT_RUN, + STEP_KINDS, + CheckResult, + CheckSpec, + Report, + VantageRecord, + utc_now, +) +from hermes_handoff_policy import ( + EXPECTED_REMOTE, + EXPECTED_REPO, + IMPERSONATION_ARGS, + PolicyError, + check_argv, + projection, +) + +UNRECORDED = "[output not recorded]" + + +def build_catalog(targets: Targets) -> list[CheckSpec]: + """Return the whole acceptance catalog for a set of targets.""" + return [ + *platform_checks(targets), + *access_checks(targets), + *delivery_checks(targets), + *worker_checks(targets), + ] + + +def validate_catalog(specs: Sequence[CheckSpec]) -> list[str]: + """Return the structural problems that make a catalog unsafe to run.""" + problems: list[str] = [] + if not specs: + return ["catalog is empty"] + seen: set[str] = set() + fake_operator = operator_vantage() + fake_pod = pod_vantage("hermes", "acceptance-probe", "probe", fake_operator) + for spec in specs: + if spec.id in seen: + problems.append(f"{spec.id}: duplicate check id") + seen.add(spec.id) + if spec.rule not in EVALUATORS and spec.scope != EPHEMERAL: + problems.append(f"{spec.id}: unknown rule {spec.rule!r}") + step_names: set[str] = set() + for step in spec.steps: + if step.key in step_names: + problems.append(f"{spec.id}: duplicate step key {step.key!r}") + step_names.add(step.key) + if step.vantage not in VANTAGE_NAMES: + problems.append( + f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}" + ) + if step.kind not in STEP_KINDS: + problems.append( + f"{spec.id}: step {step.key} has unknown kind {step.kind!r}" + ) + if step.vantage == OPERATOR and set(step.argv) & set(IMPERSONATION_ARGS): + problems.append( + f"{spec.id}: step {step.key} impersonates from the operator vantage" + ) + try: + addressed = ( + fake_operator if step.vantage == OPERATOR else fake_pod + ).wrap(step.argv) + check_argv(addressed) + except PolicyError as exc: + problems.append(f"{spec.id}: step {step.key} violates policy: {exc}") + if spec.rule == "denied" and not any( + step.kind == ATTEMPT for step in spec.steps + ): + problems.append( + f"{spec.id}: a deny check needs a real attempt, not only a review" + ) + return problems + + +def validate_targets(targets: Targets) -> list[str]: + """Return release-input problems before a runner or network call exists.""" + problems: list[str] = [] + sha = re.compile(r"[0-9a-f]{40}\Z") + image = re.compile( + r"[^@\s]+:git-([0-9a-f]{40})-build-[1-9][0-9]*@sha256:[0-9a-f]{64}\Z" + ) + if targets.repo != EXPECTED_REPO or targets.remote != EXPECTED_REMOTE: + problems.append( + "repository and remote must be the fixed Atlas titan-iac origin" + ) + if ( + targets.reviewed_pr_number != 19 + or targets.reviewed_head_ref != "feature/hermes-full-handoff-acceptance" + ): + problems.append( + "reviewed PR identity must remain fixed to draft PR #19 and its existing branch" + ) + for label, value in ( + ("baseline commit", targets.baseline_commit), + ("remote main SHA", targets.remote_main_sha), + ("reviewed head SHA", targets.reviewed_head_sha), + ("build SHA", targets.build_sha), + ): + if not sha.fullmatch(value): + problems.append(f"{label} must be an exact lowercase 40-character SHA") + image_match = image.fullmatch(targets.agent_image) + if not image_match or image_match.group(1) != targets.build_sha: + problems.append("agent image tag/digest must bind the exact build SHA") + if targets.build_sha != targets.remote_main_sha: + problems.append("build SHA must equal the exact release main SHA") + if not re.fullmatch(r"[1-9][0-9]{0,8}", str(targets.deployment_revision)): + problems.append("deployment revision must be a positive integer string") + if targets.node_count != 3 or targets.pool_replicas != 3: + problems.append("node and pool counts are fixed at exactly three") + if not targets.expect_telegram_sessions or not targets.chat_config_revision.strip(): + problems.append( + "Telegram continuity and its chat config revision are mandatory" + ) + if set(targets.pool_worker_env) != {"HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL"}: + problems.append("pool assignment evidence must include worker node and ordinal") + heads = dict(targets.dependency_heads) + if len(heads) != len(targets.dependency_heads) or set(heads) != set( + targets.dependency_pull_requests + ): + problems.append("every dependency PR must have one exact current head") + elif any(not sha.fullmatch(value) for value in heads.values()): + problems.append("dependency heads must be exact lowercase 40-character SHAs") + if ( + not isinstance(targets.max_evidence_age_seconds, int) + or targets.max_evidence_age_seconds <= 0 + ): + problems.append("maximum evidence age must be a positive integer") + return problems + + +def resolve_identity(runner: Runner, vantage: Vantage) -> tuple[str, str]: + """Return the authenticated username of a vantage and any failure detail.""" + outcome = runner.run( + ( + "kubectl", + "auth", + "whoami", + "-o", + projection("jsonpath={.status.userInfo.username}"), + ), + vantage, + ) + if not outcome.ok: + return ("", outcome.error or outcome.combined[:200]) + username = outcome.stdout.strip() + if not username or any(character.isspace() for character in username): + return ("", "identity response carried no username") + return (username, "") + + +def _first_pod( + runner: Runner, operator: Vantage, namespace: str, selector: str +) -> tuple[str, str]: + outcome = runner.run( + ( + "kubectl", + "--namespace", + namespace, + "get", + "pods", + "--selector", + selector, + "--field-selector", + "status.phase=Running", + "-o", + "name", + ), + operator, + ) + if not outcome.ok: + return ("", outcome.error or outcome.combined[:200]) + names = [ + line.strip().removeprefix("pod/") + for line in outcome.stdout.splitlines() + if line.strip() + ] + if not names: + return ("", f"no running pod matches {selector}") + return (sorted(names)[0], "") + + +def resolve_vantages( + runner: Runner, + targets: Targets, + kubeconfig: str | None = None, + context: str | None = None, +) -> tuple[dict[str, Vantage], list[VantageRecord]]: + """Resolve every vantage the catalog names, recording what each one is.""" + operator = operator_vantage(kubeconfig, context) + username, detail = resolve_identity(runner, operator) + records = [ + VantageRecord( + name=OPERATOR, + description=operator.description or "external read-only operator", + identity=username, + available=bool(username), + detail=detail, + ) + ] + vantages: dict[str, Vantage] = {OPERATOR: operator} + if not username: + return (vantages, records) + + for name, selector, container, description in ( + ( + SELF, + f"app={targets.agent_deployment}", + targets.agent_container, + "the Hermes agent itself", + ), + ( + SWITCHYARD, + f"app={targets.switchyard_deployment}", + targets.switchyard_container, + "the Switchyard routing evidence log", + ), + ( + NODE, + f"app={targets.node_daemonset}", + targets.node_container, + "a node hardening probe", + ), + ): + pod, failure = _first_pod(runner, operator, targets.namespace, selector) + if not pod: + records.append( + VantageRecord( + name=name, description=description, available=False, detail=failure + ) + ) + continue + vantage = pod_vantage(targets.namespace, pod, container, operator) + identity, identity_detail = ( + resolve_identity(runner, vantage) if name == SELF else ("", "") + ) + records.append( + VantageRecord( + name=name, + description=f"{description} ({targets.namespace}/{pod})", + identity=identity, + available=True, + detail=identity_detail, + ) + ) + vantages[name] = vantage + + chat_pod = f"{targets.chat_statefulset}-{targets.chat_ordinal}" + present = runner.run( + ( + "kubectl", + "--namespace", + targets.namespace, + "get", + f"pod/{chat_pod}", + "-o", + "name", + ), + operator, + ) + if present.ok and present.stdout.strip(): + vantages[CHAT] = pod_vantage( + targets.namespace, chat_pod, targets.chat_container, operator + ) + records.append( + VantageRecord( + name=CHAT, + description=f"the chat surface ({targets.namespace}/{chat_pod})", + available=True, + ) + ) + else: + records.append( + VantageRecord( + name=CHAT, + description=f"the chat surface ({targets.namespace}/{chat_pod})", + available=False, + detail=present.error + or present.combined[:200] + or "chat tenant pod not found", + ) + ) + return (vantages, records) + + +def vantage_problems(records: Iterable[VantageRecord]) -> list[str]: + """Return the identity problems that invalidate dual-vantage evidence.""" + by_name = {record.name: record for record in records} + operator = by_name.get(OPERATOR) + hermes = by_name.get(SELF) + problems: list[str] = [] + if operator is None or not operator.available: + problems.append( + "the operator vantage is unavailable; no evidence can be collected" + ) + return problems + if hermes is None or not hermes.available: + problems.append( + "the in-pod Hermes vantage is unavailable; self-probe evidence is missing" + ) + return problems + if not hermes.identity: + problems.append("the in-pod vantage did not report an identity") + elif operator.identity == hermes.identity: + problems.append( + "the operator and in-pod vantages authenticate as the same principal " + f"({operator.identity}); run the harness with a separate operator kubeconfig" + ) + for name, label in ((SWITCHYARD, "Switchyard"), (CHAT, "chat/Telegram")): + record = by_name.get(name) + if record is None or not record.available: + problems.append(f"the required {label} vantage is unavailable") + return problems + + +def run_check( + runner: Runner, spec: CheckSpec, vantages: Mapping[str, Vantage] +) -> CheckResult: + """Run one check's steps and classify the result.""" + outcomes: dict[str, Outcome] = {} + recorded: list[Outcome] = [] + for step in spec.steps: + vantage = vantages.get(step.vantage) + if vantage is None: + outcome = Outcome( + argv=step.argv, + vantage=step.vantage, + error=f"vantage {step.vantage} is unavailable", + ) + else: + outcome = runner.run(step.argv, vantage, step.max_bytes) + outcomes[step.key] = outcome + recorded.append( + outcome + if step.record + else Outcome( + argv=outcome.argv, + vantage=outcome.vantage, + returncode=outcome.returncode, + stdout=UNRECORDED if outcome.stdout else "", + stderr=UNRECORDED if outcome.stderr else "", + truncated=outcome.truncated, + duration_ms=outcome.duration_ms, + error=outcome.error, + ) + ) + evaluation = evaluate(spec, outcomes) + return CheckResult( + spec=spec, + status=evaluation.status, + reason=evaluation.reason, + evidence=evaluation.evidence, + outcomes=recorded, + ) + + +def run_catalog( + runner: Runner, specs: Sequence[CheckSpec], vantages: Mapping[str, Vantage] +) -> list[CheckResult]: + """Run every check in the catalog. + + Nothing is skipped once the deadline passes: the runner returns a + deadline outcome for each remaining command, which classifies as + ``NOT_RUN``, so a truncated sweep is visible in the report rather than + silently shorter. + """ + return [run_check(runner, spec, vantages) for spec in specs] + + +def build_report( + runner: Runner, + targets: Targets, + specs: Sequence[CheckSpec], + vantages: Mapping[str, Vantage], + records: Sequence[VantageRecord], + mode: str, + started_at: str, + extra_results: Sequence[CheckResult] = (), +) -> Report: + """Run the catalog and assemble the report, including structural problems.""" + problems = validate_catalog(specs) + vantage_problems(records) + report = Report( + mode=mode, + started_at=started_at, + baseline={ + "namespace": targets.namespace, + "baseline_commit": targets.baseline_commit, + "dependency_pull_requests": list(targets.dependency_pull_requests), + "expected_suspensions": list(targets.expected_suspensions), + "node_count": targets.node_count, + "pool_replicas": targets.pool_replicas, + }, + vantages=list(records), + harness_errors=problems, + ) + armed_ids = {result.spec.id for result in extra_results} + runnable = [spec for spec in specs if spec.id not in armed_ids] + if problems: + report.results = [ + CheckResult(spec=spec, status=NOT_RUN, reason="harness preflight failed") + for spec in runnable + ] + report.results.extend(extra_results) + else: + report.results = [*run_catalog(runner, runnable, vantages), *extra_results] + report.results.sort(key=lambda result: result.spec.id) + if runner.remaining_seconds <= 0: + report.harness_errors.append( + "the run deadline expired before every check completed" + ) + report.finished_at = utc_now() + return report + + +def unavailable_report( + mode: str, started_at: str, reason: str, targets: Targets +) -> Report: + """Return a NO_GO report for a run that could not start at all.""" + return Report( + mode=mode, + started_at=started_at, + finished_at=utc_now(), + baseline={"namespace": targets.namespace}, + harness_errors=[reason], + results=[ + CheckResult( + spec=CheckSpec( + id="harness.startup", + title="The harness could start and resolve its vantages", + group="harness", + rule="allowed", + ), + status=NOT_RUN, + reason=reason, + ) + ], + ) diff --git a/testing/quality_contract.json b/testing/quality_contract.json index 3f377fc6..573da5aa 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -18,6 +18,21 @@ "dockerfiles/hermes-kaniko-heredoc-runner.py", "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py", "services/hermes/scripts/jenkins_image_build_trigger.py", + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -65,6 +80,7 @@ "testing/quality_docs.py", "testing/quality_hygiene.py", "testing/quality_coverage.py", + "testing/quality_handoff_mutation.py", "testing/quality_gate.py", "ci/tests/glue/test_ariadne_schedules.py", "ci/tests/glue/test_glue_metrics.py", @@ -79,6 +95,21 @@ "dockerfiles/hermes-kaniko-heredoc-runner.py", "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py", "services/hermes/scripts/jenkins_image_build_trigger.py", + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -133,6 +164,7 @@ "coverage_sources": [ "ci/scripts", "dockerfiles", + "scripts/ops", "scripts.render.dashboards_render_atlas", "services/gitea/scripts", "services/hermes/scm-common/scripts", @@ -198,7 +230,26 @@ ], "hygiene": { "max_lines": 500, + "legacy_line_exceptions": { + "scripts/tests/test_dashboards_render_atlas.py": { + "lines": 516, + "sha256": "afe4923391b157d03bcf0899208f77eeab79a2762da3575a365530b3074c5f25" + }, + "testing/tests/test_hermes_chat_quality.py": { + "lines": 2242, + "sha256": "71e9cafd14460b5132d07dd951514756d091009c23db7703db1575b05e34e75f" + }, + "testing/tests/test_hermes_cli_lanes.py": { + "lines": 1971, + "sha256": "992bf3788c33252e37a695f5307c31b1e8c0646e00a715e3e55f6901788abb62" + }, + "testing/tests/test_hermes_coordinator.py": { + "lines": 510, + "sha256": "290240cf391abc755eaa0abb12d247ad1718e7c9295a2092f349a7eaba4bfe55" + } + }, "line_limit_globs": [ + "scripts/ops/hermes_handoff_*.py", "testing/**/*.py", "ci/scripts/*.py", "ci/tests/**/*.py", @@ -258,6 +309,26 @@ "minimum_percent": 95.0, "minimum_branch_percent": 95.0, "branch_tracked_files": [ + "ci/scripts/hermes_image_release.py", + "dockerfiles/hermes-kaniko-heredoc-runner.py", + "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py", + "services/hermes/scripts/jenkins_image_build_trigger.py", + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", + "ci/scripts/semgrep_report.py", "services/gitea/scripts/gitea_branch_protection_check.py", "services/hermes/scm-common/scripts/gitea_api.py", "services/hermes/scm-common/scripts/gitea_api_policy.py", @@ -294,13 +365,30 @@ "services/hermes/scripts/cli_lane_retention.py", "services/hermes/scripts/cli_lane_routing.py", "services/hermes/scripts/cli_lane_runner.py", - "testing/quality_coverage.py" + "testing/quality_coverage.py", + "testing/quality_handoff_mutation.py", + "testing/quality_hygiene.py" ], "tracked_files": [ "ci/scripts/hermes_image_release.py", "dockerfiles/hermes-kaniko-heredoc-runner.py", "services/harbor/scripts/harbor_hermes_agent_immutability_ensure.py", "services/hermes/scripts/jenkins_image_build_trigger.py", + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", "ci/scripts/semgrep_report.py", @@ -347,6 +435,7 @@ "testing/quality_docs.py", "testing/quality_hygiene.py", "testing/quality_coverage.py", + "testing/quality_handoff_mutation.py", "testing/quality_gate.py" ] } diff --git a/testing/quality_coverage.py b/testing/quality_coverage.py index 98cc65d5..04355e51 100644 --- a/testing/quality_coverage.py +++ b/testing/quality_coverage.py @@ -57,19 +57,12 @@ def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str] return [f"coverage xml missing: {xml_path.relative_to(root)}"] percentages = _load_percentages(xml_path, root) - minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0)) coverage_contract = contract.get("coverage", {}) - branch_minimum = coverage_contract.get("minimum_branch_percent") - if branch_minimum is not None: - branch_minimum = float(branch_minimum) + minimum = float(coverage_contract.get("minimum_percent", 95.0)) + branch_minimum = float(coverage_contract.get("minimum_branch_percent", 95.0)) branch_tracked = { path.replace("\\", "/") - for path in coverage_contract.get( - "branch_tracked_files", - coverage_contract.get("tracked_files", []) - if branch_minimum is not None - else [], - ) + for path in coverage_contract.get("branch_tracked_files", ()) } issues: list[str] = [] @@ -83,14 +76,14 @@ def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str] issues.append( f"coverage below {minimum:.1f}%: {relative_path} ({rates.line:.1f}%)" ) - if branch_minimum is None or normalized not in branch_tracked: + if normalized not in branch_tracked: continue if rates.branch is None: issues.append(f"branch coverage missing for tracked file: {relative_path}") elif rates.branch + 1e-9 < branch_minimum: issues.append( - "branch coverage below " - f"{branch_minimum:.1f}%: {relative_path} ({rates.branch:.1f}%)" + f"branch coverage below {branch_minimum:.1f}%: " + f"{relative_path} ({rates.branch:.1f}%)" ) return issues diff --git a/testing/quality_handoff_mutation.py b/testing/quality_handoff_mutation.py new file mode 100644 index 00000000..a0c63fa0 --- /dev/null +++ b/testing/quality_handoff_mutation.py @@ -0,0 +1,209 @@ +"""Deterministic scored mutation gate for Hermes handoff safety invariants.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class Mutation: + """One source mutation and the isolated probe that must kill it.""" + + name: str + path: str + old: str + new: str + probe: str + + +@dataclass(frozen=True) +class MutationReport: + """Scored result; any invalid, surviving, or baseline-broken case is an issue.""" + + total: int + killed: int + issues: tuple[str, ...] + + @property + def score(self) -> float: + return 100.0 if not self.total else round(self.killed * 100.0 / self.total, 2) + + +MUTATIONS = ( + Mutation( + "policy-allows-git-fetch", + "hermes_handoff_policy.py", + ' "for-each-ref",\n', + ' "fetch",\n "for-each-ref",\n', + "from hermes_handoff_policy import *\n" + "try: check_argv(('git','fetch','origin'))\n" + "except PolicyError: raise SystemExit(0)\n" + "raise SystemExit(1)\n", + ), + Mutation( + "redaction-truncates-before-screening", + "hermes_handoff_redaction.py", + " return bound(redact(text), max_bytes)\n", + " return bound(text, max_bytes)\n", + "from hermes_handoff_redaction import safe_text\n" + "assert 'A9' not in safe_text('password=A9secret-prefix', 23)[0]\n", + ), + Mutation( + "empty-report-go", + "hermes_handoff_model.py", + " if (\n" + " not self.results\n" + " or self.invalid_statuses\n" + " or self.harness_errors\n" + " or self.blocking\n" + " ):\n", + " if (\n" + " self.invalid_statuses\n" + " or self.harness_errors\n" + " or self.blocking\n" + " ):\n", + "from hermes_handoff_model import *\nassert Report('read-only','now').decision == NO_GO\n", + ), + Mutation( + "generic-404-is-denial", + "hermes_handoff_rules.py", + "DENIAL_MARKERS = (\n", + 'DENIAL_MARKERS = (\n "404 not found",\n', + "from hermes_handoff_rules import looks_denied\n" + "from hermes_handoff_exec import Outcome\n" + "assert not looks_denied(Outcome(('x',),'x',1,'','404 not found'))\n", + ), + Mutation( + "truncated-evidence-reaches-evaluator", + "hermes_handoff_rules.py", + " if truncated:\n" + " return Evaluation(\n" + " NOT_RUN, f\"truncated evidence from step(s): {', '.join(truncated)}\"\n" + " )\n", + " if False and truncated:\n return Evaluation(NOT_RUN, 'mutant')\n", + "import hermes_handoff_evaluators\n" + "import hermes_handoff_rules as r\n" + "from hermes_handoff_exec import Outcome\n" + "from hermes_handoff_model import *\n" + "@r.evaluator('probe')\ndef probe(*_): return r.Evaluation(PASS,'bad')\n" + "s=CheckSpec('x','x','x','probe')\n" + "assert r.evaluate(s,{'x':Outcome(('x',),'x',0,truncated=True)}).status == NOT_RUN\n", + ), + Mutation( + "truncated-outcome-is-ok", + "hermes_handoff_exec.py", + " return self.error is None and self.returncode == 0 and not self.truncated\n", + " return self.error is None and self.returncode == 0\n", + "from hermes_handoff_exec import Outcome\nassert not Outcome(('x',),'x',0,truncated=True).ok\n", + ), + Mutation( + "nan-is-numeric", + "hermes_handoff_json_rules.py", + " or not math.isfinite(float(actual))\n", + " or False\n", + "import hermes_handoff_json_rules as j\n" + "from hermes_handoff_model import *\nfrom hermes_handoff_exec import Outcome\n" + "s=CheckSpec('x','x','x','json_numeric',expect={'step':'s','fields':{'x':{'min':0}}})\n" + "j.parsed_step=lambda *_: ({'x':float('nan')},'s')\n" + "assert j.evaluate_json_numeric(s,{'s':Outcome(('x',),'x',0)}).status == NOT_RUN\n", + ), + Mutation( + "cleanup-ignores-branch-delete-failure", + "hermes_handoff_ephemeral.py", + ' if not deleted.ok:\n problems.append("ephemeral branch deletion failed")\n', + ' if False and not deleted.ok:\n problems.append("mutant")\n', + "import hermes_handoff_ephemeral as e\nfrom hermes_handoff_exec import Outcome\n" + "class R:\n" + " def __init__(self): self.i=0\n" + " def run(self,*_a,**_k):\n" + " self.i+=1\n" + " return Outcome(('x',),'x',1 if self.i==1 else 0)\n" + "q=e.ArmRequest('atlas/titan-iac','origin','acceptance1',e.CONFIRMATION,expected_head='1'*40,expected_base_sha='2'*40)\n" + "assert e._cleanup(R(),object(),q,'/api/v1/repos/atlas/titan-iac',[]).status == 'FAIL'\n", + ), + Mutation( + "cleanup-accepts-open-pr", + "hermes_handoff_ephemeral.py", + ' elif "state does not match" not in problems:\n problems.append("state is not closed")\n', + ' elif False:\n problems.append("mutant")\n', + "import hermes_handoff_ephemeral as e\n" + "q=e.ArmRequest('atlas/titan-iac','origin','acceptance1',e.CONFIRMATION,expected_head='1'*40,expected_base_sha='2'*40)\n" + "p={'number':1,'state':'open','draft':True,'merged':False,'base':{'ref':'main','sha':'2'*40},'head':{'ref':q.ref,'sha':'1'*40}}\n" + "assert e._closed_pr_problems(p,q)\n", + ), +) + + +def _probe(root: Path, probe: str) -> subprocess.CompletedProcess[str]: + environment = { + "LC_ALL": "C", + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "PYTHONPATH": str(root), + } + return subprocess.run( + [sys.executable, "-c", probe], + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=10, + check=False, + env=environment, + ) + + +def run_gate( + repo_root: Path, mutations: Sequence[Mutation] = MUTATIONS +) -> MutationReport: + """Run baseline and mutated probes in isolated copies and return the score.""" + source_root = repo_root / "scripts" / "ops" + issues: list[str] = [] + killed = 0 + with tempfile.TemporaryDirectory(prefix="hermes-mutation-") as temporary: + clean = Path(temporary) / "clean" + shutil.copytree(source_root, clean) + for mutation in mutations: + source = clean / mutation.path + original = source.read_text(encoding="utf-8") + if original.count(mutation.old) != 1: + issues.append(f"{mutation.name}: mutation anchor count is not one") + continue + baseline = _probe(clean, mutation.probe) + if baseline.returncode != 0: + issues.append(f"{mutation.name}: baseline probe failed") + continue + mutated_text = original.replace(mutation.old, mutation.new, 1) + try: + compile(mutated_text, str(source), "exec") + except SyntaxError: + issues.append(f"{mutation.name}: mutant is syntactically invalid") + continue + mutant = Path(temporary) / f"mutant-{mutation.name}" + shutil.copytree(clean, mutant) + (mutant / mutation.path).write_text(mutated_text, encoding="utf-8") + if _probe(mutant, mutation.probe).returncode != 0: + killed += 1 + else: + issues.append(f"{mutation.name}: mutant survived") + return MutationReport(len(mutations), killed, tuple(issues)) + + +def main() -> int: + """Run the repository mutation contract and print a bounded score.""" + report = run_gate(Path(__file__).resolve().parents[1]) + print( + f"Hermes handoff mutation score: {report.killed}/{report.total} ({report.score:.2f}%)" + ) + for issue in report.issues: + print(issue) + return 0 if not report.issues and report.killed == report.total else 1 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) # pragma: no cover diff --git a/testing/quality_hygiene.py b/testing/quality_hygiene.py index 5060ca02..6b68de3a 100644 --- a/testing/quality_hygiene.py +++ b/testing/quality_hygiene.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import re from collections.abc import Iterable from pathlib import Path @@ -16,6 +17,18 @@ def _expand_globs(root: Path, patterns: Iterable[str]) -> list[Path]: return sorted(matched) +def _is_unchanged_legacy( + config: dict[str, Any], root: Path, path: Path, line_count: int +) -> bool: + """Allow an over-cap legacy file only while its exact bytes stay unchanged.""" + relative = path.relative_to(root).as_posix() + record = config.get("legacy_line_exceptions", {}).get(relative) + if not isinstance(record, dict) or record.get("lines") != line_count: + return False + digest = hashlib.sha256(path.read_bytes()).hexdigest() + return record.get("sha256") == digest + + def run_check(contract: dict[str, Any], root: Path) -> list[str]: """Return human-readable issues for naming and file-size rules.""" config = contract.get("hygiene", {}) @@ -24,8 +37,12 @@ def run_check(contract: dict[str, Any], root: Path) -> list[str]: for path in _expand_globs(root, config.get("line_limit_globs", [])): line_count = sum(1 for _ in path.open("r", encoding="utf-8")) - if line_count > max_lines: - issues.append(f"file exceeds {max_lines} LOC: {path.relative_to(root)} ({line_count})") + if line_count > max_lines and not _is_unchanged_legacy( + config, root, path, line_count + ): + issues.append( + f"file exceeds {max_lines} LOC: {path.relative_to(root)} ({line_count})" + ) for rule in config.get("naming_rules", []): pattern = re.compile(rule["pattern"]) @@ -48,7 +65,9 @@ def count_files_over_line_limit(contract: dict[str, Any], root: Path) -> int: count = 0 for path in _expand_globs(root, config.get("line_limit_globs", [])): line_count = sum(1 for _ in path.open("r", encoding="utf-8")) - if line_count > max_lines: + if line_count > max_lines and not _is_unchanged_legacy( + config, root, path, line_count + ): count += 1 return count diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py index 09dcb0a1..ccf1d2b6 100644 --- a/testing/tests/conftest.py +++ b/testing/tests/conftest.py @@ -1,7 +1,8 @@ -"""Shared pytest boundaries for repository automation tests.""" +"""Shared hermetic and capability boundaries for repository tests.""" from __future__ import annotations +from pathlib import Path import sys import pytest @@ -24,3 +25,16 @@ def _patched_hermes_cli_db_contract(request, monkeypatch): name == "cli_lane_runner" or name.startswith("cli_lane_") ) and hasattr(module, "kanban_capabilities"): monkeypatch.setattr(module, "kanban_capabilities", lambda _db: safe) + + +@pytest.fixture(autouse=True) +def isolate_coordinator_gitea_token( + request: pytest.FixtureRequest, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Prevent coordinator tests from observing the mounted production token.""" + if request.node.path.name == "test_hermes_coordinator.py": + monkeypatch.setenv( + "HERMES_GITEA_TOKEN_FILE", str(tmp_path / "absent-gitea-token") + ) diff --git a/testing/tests/test_hermes_auto_router.py b/testing/tests/test_hermes_auto_router.py index 05af3b01..f8a333b1 100644 --- a/testing/tests/test_hermes_auto_router.py +++ b/testing/tests/test_hermes_auto_router.py @@ -255,7 +255,12 @@ def test_provider_status_separates_observed_activity_from_plan_quota(monkeypatch monkeypatch.setattr(module, "_claude_account", lambda: { "authenticated": True, "plan": "max", "quota_reported": False, }) - monkeypatch.setattr(module, "_fresh_health", lambda _path: {}) + monkeypatch.setattr(module, "_fresh_health", lambda _path, maximum_age=86400.0: {}) + monkeypatch.setattr( + module, + "_configured_models", + lambda provider: ["gpt-5.6-sol"] if provider == "codex" else ["claude-sonnet-5"], + ) payload = module.provider_status_payload() diff --git a/testing/tests/test_hermes_handoff_acceptance.py b/testing/tests/test_hermes_handoff_acceptance.py new file mode 100644 index 00000000..4f89db35 --- /dev/null +++ b/testing/tests/test_hermes_handoff_acceptance.py @@ -0,0 +1,460 @@ +"""End-to-end contracts for the Hermes full-handoff acceptance harness CLI.""" + +from __future__ import annotations + +import json +import subprocess + +import pytest + +from testing.tests.test_hermes_handoff_support import FakeClock, load_handoff_module + +acceptance = load_handoff_module("hermes_handoff_acceptance") +catalog = load_handoff_module("hermes_handoff_catalog") +exec_module = load_handoff_module("hermes_handoff_exec") +model = load_handoff_module("hermes_handoff_model") +policy = load_handoff_module("hermes_handoff_policy") +harness_run = load_handoff_module("hermes_handoff_run") + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) + + +class ScriptedCluster: + """A ``subprocess.run`` stand-in that answers by command shape. + + Real runs issue a couple of hundred commands; scripting them positionally + would make every test a brittle ordering puzzle. Matching on the command + line keeps each test about the behaviour it is checking. + """ + + def __init__(self, rules: list[tuple[str, object]], default=None) -> None: + self.rules = rules + self.default = default or completed(stderr="not found", returncode=1) + self.calls: list[list[str]] = [] + + def __call__(self, argv, **_kwargs): + self.calls.append(list(argv)) + line = " ".join(argv) + for fragment, response in self.rules: + if fragment in line: + return response + return self.default + + +def whoami(username: str): + return completed(stdout=username) + + +REVIEWED = "8f005458282269ba5c07941814e4237f5d4cf3ac" +SHA = "1" * 40 +VALID_ARGS = [ + "--remote-main-sha", + SHA, + "--reviewed-head-sha", + REVIEWED, + "--agent-image", + f"registry.example/hermes:git-{'1' * 40}-build-7@sha256:{'2' * 64}", + "--build-sha", + SHA, + "--deployment-revision", + "1", + "--chat-config-revision", + "chat-rev", + *[ + item + for number in catalog.Targets().dependency_pull_requests + for item in ("--dependency-head", f"{number}={SHA}") + ], +] + + +HEALTHY_CLUSTER = [ + ( + "exec --namespace hermes hermes-agent-1", + whoami("system:serviceaccount:hermes:hermes-agent"), + ), + ("auth whoami", whoami("kubernetes-admin")), + ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), + ("get pods --selector", completed(stdout="pod/other-1\n")), + ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")), +] + + +def run_cli(monkeypatch, argv, rules): + cluster = ScriptedCluster(rules) + clock = FakeClock() + + def build_runner(**kwargs): + return exec_module.Runner( + clock=clock, + spawn=cluster, + environ={}, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + **kwargs, + ) + + monkeypatch.setattr(acceptance, "Runner", build_runner) + return (acceptance.main([*VALID_ARGS, *argv]), cluster) + + +def read_report(path): + return json.loads(path.read_text(encoding="utf-8")) + + +def test_the_parser_exposes_the_documented_controls() -> None: + arguments = acceptance.build_parser().parse_args( + [ + "--namespace", + "hermes", + "--context", + "atlas-operator", + "--dependency-pr", + "14", + "--dependency-pr", + "15", + "--expected-suspension", + "flux-system/parked", + "--node-count", + "21", + "--pool-worker-env", + "HERMES_POOL_LEASE_SECONDS", + "--expect-telegram-sessions", + ] + ) + + targets = acceptance.targets_from_args(arguments, None) + + assert targets.dependency_pull_requests == (14, 15) + assert targets.expected_suspensions == ("flux-system/parked",) + assert targets.node_count == 21 + assert targets.pool_worker_env == ("HERMES_POOL_LEASE_SECONDS",) + assert targets.expect_telegram_sessions is True + + +def test_defaults_fall_back_to_the_shipped_targets() -> None: + arguments = acceptance.build_parser().parse_args([]) + targets = acceptance.targets_from_args(arguments, None) + + assert targets.dependency_pull_requests == catalog.Targets.dependency_pull_requests + assert targets.expected_suspensions == () + assert targets.repo == catalog.Targets.repo + + +@pytest.mark.parametrize("value", ["bad", "x=" + "1" * 40, "0=" + "1" * 40, "1=BAD"]) +def test_dependency_heads_require_a_positive_pr_and_lowercase_sha(value: str) -> None: + with pytest.raises(Exception): + acceptance.dependency_head(value) + + +def test_a_default_run_is_read_only_and_never_pushes_or_posts( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + status, cluster = run_cli( + monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert report["mode"] == policy.READ_ONLY + assert not any("push" in call for call in cluster.calls) + assert not any("POST" in call for call in cluster.calls) + assert not any("DELETE" in call for call in cluster.calls) + writes = [ + call + for call in cluster.calls + if {"patch", "create", "delete", "apply", "replace"} & set(call) + and "can-i" not in call + ] + assert writes == [] + + +def test_mutating_checks_report_not_run_until_the_mode_is_armed( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) + + ephemeral = { + check["id"]: check + for check in read_report(output)["checks"] + if check["group"] == "ephemeral" + } + + assert len(ephemeral) == 4 + for check in ephemeral.values(): + assert check["status"] == model.NOT_RUN + assert check["mandatory"] is False + assert check["scope"] == model.EPHEMERAL + + +def test_a_zero_state_cluster_is_no_go_with_every_check_accounted_for( + monkeypatch, tmp_path +) -> None: + """Nothing observed must never read as nothing wrong.""" + output = tmp_path / "report.json" + status, _ = run_cli( + monkeypatch, + ["--output", str(output), "--json-only"], + [("auth whoami", whoami("kubernetes-admin"))], + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert report["decision"] == model.NO_GO + assert report["counts"][model.PASS] == 0 + assert sum(report["counts"].values()) == len(report["checks"]) + assert any( + "in-pod Hermes vantage is unavailable" in error + for error in report["harness_errors"] + ) + + +def test_partial_failure_is_reported_per_check_rather_than_aborting( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + rules = [ + *HEALTHY_CLUSTER, + ("auth can-i", completed(stdout="no\n", returncode=1)), + ("get namespaces", completed(stdout="namespace/hermes\n")), + ("merge-base", completed()), + ] + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) + report = read_report(output) + + statuses = {check["status"] for check in report["checks"]} + + assert status == acceptance.EXIT_NO_GO + assert model.PASS in statuses and model.NOT_RUN in statuses + assert report["blocking"] + + +def test_a_vantage_collision_is_a_no_go_even_with_clean_checks( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + rules = [ + ( + "exec --namespace hermes hermes-agent-1", + whoami("system:serviceaccount:hermes:hermes-agent"), + ), + ("auth whoami", whoami("system:serviceaccount:hermes:hermes-agent")), + ( + "get pods --selector app=hermes-agent", + completed(stdout="pod/hermes-agent-1\n"), + ), + ("get pods --selector", completed(stdout="pod/other-1\n")), + ( + "get pod/hermes-chat-tenant-0", + completed(stdout="pod/hermes-chat-tenant-0\n"), + ), + ] + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert any("same principal" in error for error in report["harness_errors"]) + + +def test_an_invalid_deadline_stops_before_any_subprocess(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + status, _ = run_cli( + monkeypatch, + ["--output", str(output), "--json-only", "--deadline", "0"], + HEALTHY_CLUSTER, + ) + report = read_report(output) + + assert status == acceptance.EXIT_USAGE + assert report["counts"][model.NOT_RUN] > 0 + assert any("invalid execution bound" in error for error in report["harness_errors"]) + + +def test_arming_with_a_wrong_confirmation_stops_before_anything_runs( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + status, cluster = run_cli( + monkeypatch, + [ + "--output", + str(output), + "--json-only", + "--arm-ephemeral-push", + "--confirm", + "please", + "--ephemeral-token", + "acceptance-20260817a", + ], + HEALTHY_CLUSTER, + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert "arming refused" in report["harness_errors"][0] + assert not any("push" in call for call in cluster.calls) + + +def test_arming_against_a_protected_push_target_is_refused( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + status, cluster = run_cli( + monkeypatch, + [ + "--output", + str(output), + "--json-only", + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "main", + ], + HEALTHY_CLUSTER, + ) + + assert status == acceptance.EXIT_NO_GO + assert "arming refused" in read_report(output)["harness_errors"][0] + assert not any("push" in call for call in cluster.calls) + + +def test_an_armed_run_does_not_mutate_until_read_only_acceptance_is_go( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + monkeypatch.setattr(acceptance, "preflight", lambda *_args: None) + rules = [ + ("git ls-remote origin refs/heads/ephemeral", completed(stdout="")), + ("git push", completed()), + ( + "POST /api/v1/repos/atlas/titan-iac/pulls", + completed(stdout=json.dumps({"number": 9})), + ), + ("PATCH", completed()), + ("DELETE", completed()), + ( + "GET /api/v1/repos/atlas/titan-iac/pulls/9", + completed(stdout='{"state": "closed"}'), + ), + *HEALTHY_CLUSTER, + ] + status, cluster = run_cli( + monkeypatch, + [ + "--output", + str(output), + "--json-only", + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817a", + ], + rules, + ) + report = read_report(output) + assert report["mode"] == policy.ARMED + assert status == acceptance.EXIT_NO_GO + assert any( + "read-only acceptance did not GO" in error for error in report["harness_errors"] + ) + pushes = [call for call in cluster.calls if "push" in call] + assert pushes == [] + assert not any("DELETE" in call for call in cluster.calls) + + +def test_the_report_is_written_to_stdout_when_no_output_path_is_given( + monkeypatch, capsys +) -> None: + status, _ = run_cli(monkeypatch, ["--json-only"], HEALTHY_CLUSTER) + payload = json.loads(capsys.readouterr().out) + + assert status == acceptance.EXIT_NO_GO + assert payload["harness"] == model.HARNESS + + +def test_the_human_summary_accompanies_the_json_unless_suppressed( + monkeypatch, capsys, tmp_path +) -> None: + output = tmp_path / "report.json" + run_cli(monkeypatch, ["--output", str(output)], HEALTHY_CLUSTER) + captured = capsys.readouterr() + + assert model.HARNESS in captured.err + assert "vantage operator" in captured.err + assert captured.out == "" + + +def test_a_report_that_fails_its_own_screening_is_replaced_not_published( + tmp_path, +) -> None: + """The last barrier: never write an artifact nobody re-read.""" + output = tmp_path / "report.json" + arguments = acceptance.build_parser().parse_args( + ["--output", str(output), "--json-only"] + ) + report = model.Report(mode="read-only", started_at="2026-08-17T00:00:00Z") + report.harness_errors = ["leaked ghp_ABCDEFGHIJKLMNOPQRSTUV1234"] + + monkey = report.as_dict + report.as_dict = lambda *_args, **_kwargs: { # type: ignore[method-assign] + **monkey(), + "leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234", + } + offenders = acceptance.emit(report, arguments) + payload = read_report(output) + + assert offenders == ["$.leak"] + assert payload["decision"] == model.NO_GO + assert "leak" not in payload + assert "failed its own credential screening" in payload["harness_errors"][0] + + +def test_a_screening_failure_forces_no_go_from_the_entry_point( + monkeypatch, tmp_path +) -> None: + """The exit status must follow the refusal to publish, not the check tally.""" + output = tmp_path / "report.json" + monkeypatch.setattr( + acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"] + ) + monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) + monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) + + status, _ = run_cli( + monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER + ) + + assert status == acceptance.EXIT_NO_GO + assert read_report(output)["decision"] == model.NO_GO + + +def test_no_recorded_command_output_carries_a_credential_shape( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + leaky = completed(stdout="ANTHROPIC_API_KEY=sk-ant-abcdefghijklmnopqrstuvwxyz\n") + run_cli( + monkeypatch, + ["--output", str(output), "--json-only"], + [*HEALTHY_CLUSTER, ("exec", leaky)], + ) + + assert model.unscreened_fields(read_report(output)) == [] + assert "sk-ant-abcdefghijklmnopqrstuvwxyz" not in output.read_text(encoding="utf-8") + + +@pytest.mark.parametrize("flag", ["--help"]) +def test_the_help_text_describes_the_read_only_default(flag: str, capsys) -> None: + with pytest.raises(SystemExit): + acceptance.main([flag]) + + assert "Read-only by default" in capsys.readouterr().out diff --git a/testing/tests/test_hermes_handoff_acceptance_armed.py b/testing/tests/test_hermes_handoff_acceptance_armed.py new file mode 100644 index 00000000..ded33bb8 --- /dev/null +++ b/testing/tests/test_hermes_handoff_acceptance_armed.py @@ -0,0 +1,120 @@ +"""Static and armed-transition regressions for the handoff CLI.""" + +from __future__ import annotations + +from testing.tests.test_hermes_handoff_acceptance import ( + HEALTHY_CLUSTER, + REVIEWED, + VALID_ARGS, + acceptance, + harness_run, + model, + policy, + read_report, + run_cli, +) + + +def test_an_empty_catalog_fails_before_any_subprocess(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + monkeypatch.setattr(harness_run, "build_catalog", lambda _targets: []) + monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) + monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) + + status, _ = run_cli( + monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER + ) + + assert status == acceptance.EXIT_NO_GO + assert read_report(output)["decision"] == model.NO_GO + + +def test_armed_success_path_replaces_placeholders_only_after_a_go(monkeypatch) -> None: + base = model.CheckSpec( + id="release.pass", title="pass", group="release", rule="allowed" + ) + placeholder = model.CheckSpec( + id="ephemeral.cleanup-verified", + title="pending", + group="ephemeral", + rule="not_armed", + mandatory=False, + scope=model.EPHEMERAL, + ) + report = model.Report( + mode=policy.ARMED, + started_at="2026-08-17T00:00:00Z", + results=[ + model.CheckResult(base, model.PASS), + model.CheckResult(placeholder, model.NOT_RUN), + ], + ) + armed_spec = model.CheckSpec( + id="ephemeral.cleanup-verified", + title="clean", + group="ephemeral", + rule="armed", + scope=model.EPHEMERAL, + ) + armed = model.CheckResult(armed_spec, model.PASS) + monkeypatch.setattr( + acceptance, "resolve_vantages", lambda *_args: ({"operator": object()}, []) + ) + monkeypatch.setattr(acceptance, "preflight", lambda *_args: None) + monkeypatch.setattr(acceptance, "build_report", lambda *_args: report) + monkeypatch.setattr(acceptance, "_armed_results", lambda *_args: [armed]) + monkeypatch.setattr(acceptance, "emit", lambda *_args: []) + status = acceptance.main( + [ + *VALID_ARGS, + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817a", + ] + ) + assert status == acceptance.EXIT_GO + assert ( + next( + result + for result in report.results + if result.spec.id.startswith("ephemeral") + ).status + == model.PASS + ) + monkeypatch.setattr(acceptance, "emit", lambda *_args: ["screening-failure"]) + assert ( + acceptance.main( + [ + *VALID_ARGS, + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817b", + ] + ) + == acceptance.EXIT_NO_GO + ) + + +def test_armed_results_are_bound_to_operator_and_exact_reviewed_head( + monkeypatch, +) -> None: + arguments = acceptance.build_parser().parse_args( + [ + *VALID_ARGS, + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817a", + ] + ) + calls = [] + monkeypatch.setattr(acceptance, "preflight", lambda *args: calls.append(args)) + monkeypatch.setattr(acceptance, "run_armed", lambda *args: [args]) + operator = object() + result = acceptance._armed_results(arguments, object(), {"operator": operator}) + assert calls[0][0].expected_head == REVIEWED + assert result[0][1] is operator diff --git a/testing/tests/test_hermes_handoff_catalog.py b/testing/tests/test_hermes_handoff_catalog.py new file mode 100644 index 00000000..21e8031a --- /dev/null +++ b/testing/tests/test_hermes_handoff_catalog.py @@ -0,0 +1,195 @@ +"""Structural contracts for the handoff acceptance catalog.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module + +catalog = load_handoff_module("hermes_handoff_catalog") +harness_run = load_handoff_module("hermes_handoff_run") +evaluators = load_handoff_module("hermes_handoff_evaluators") +model = load_handoff_module("hermes_handoff_model") +policy = load_handoff_module("hermes_handoff_policy") +exec_module = load_handoff_module("hermes_handoff_exec") + +NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc) +TARGETS = catalog.Targets(now=NOW) +SPECS = harness_run.build_catalog(TARGETS) +BY_ID = {check.id: check for check in SPECS} + +VANTAGES = { + catalog.OPERATOR: exec_module.operator_vantage(context="atlas-operator"), + catalog.SELF: exec_module.pod_vantage("hermes", "hermes-agent-1", "hermes"), + catalog.SWITCHYARD: exec_module.pod_vantage( + "hermes", "hermes-switchyard-1", "switchyard" + ), + catalog.NODE: exec_module.pod_vantage( + "hermes", "hermes-node-ssh-access-1", "key-reconciler" + ), + catalog.CHAT: exec_module.pod_vantage("hermes", "hermes-chat-tenant-0", "hermes"), +} + + +def test_the_catalog_is_structurally_valid() -> None: + assert harness_run.validate_catalog(SPECS) == [] + assert len(SPECS) >= 60 + + +def test_every_step_is_accepted_by_the_read_only_policy_from_its_vantage() -> None: + """A catalog entry that cannot run under policy is a bug found before a run.""" + for check in SPECS: + for step in check.steps: + addressed = VANTAGES[step.vantage].wrap(step.argv) + policy.check_argv(addressed, policy.READ_ONLY) + + +def test_no_step_constructs_a_mutation_or_dry_run() -> None: + """Authorization review may name a verb; no executed subcommand mutates.""" + for check in SPECS: + for step in check.steps: + subcommand = next(policy.positionals(step.argv), "") + assert subcommand not in {"apply", "create", "delete", "patch", "replace"} + assert not any(argument.startswith("--dry-run") for argument in step.argv) + + +def test_no_catalog_step_constructs_impersonation() -> None: + operator_steps = [ + step + for check in SPECS + for step in check.steps + if step.vantage == catalog.OPERATOR + ] + assert operator_steps + assert not any( + set(step.argv) & set(policy.IMPERSONATION_ARGS) for step in operator_steps + ) + + assert not any( + set(step.argv) & set(policy.IMPERSONATION_ARGS) + for check in SPECS + for step in check.steps + ) + + +def test_credential_or_mutation_denials_are_review_only() -> None: + deny_checks = [check for check in SPECS if check.rule == "review_denied"] + assert len(deny_checks) >= 5 + for check in deny_checks: + assert check.steps and all(step.kind == model.REVIEW for step in check.steps), ( + check.id + ) + + +def test_both_vantages_are_represented_across_the_catalog() -> None: + used = {step.vantage for check in SPECS for step in check.steps} + assert catalog.OPERATOR in used and catalog.SELF in used + assert used <= set(catalog.VANTAGE_NAMES) + + +def test_every_rule_named_by_a_check_is_registered() -> None: + for check in SPECS: + if check.scope == model.EPHEMERAL: + assert check.rule == "not_armed" + continue + assert check.rule in evaluators.EVALUATORS, check.id + + +def test_every_evidence_step_referenced_by_expect_exists() -> None: + for check in SPECS: + keys = {step.key for step in check.steps} + named = [check.expect.get("step")] + list(check.expect.get("steps", ())) + for key in filter(None, named): + assert key in keys, f"{check.id} names missing step {key}" + + +def test_the_default_catalog_reports_every_mutating_check_as_not_armed() -> None: + ephemeral = [check for check in SPECS if check.scope == model.EPHEMERAL] + assert len(ephemeral) == 4 + for check in ephemeral: + assert check.mandatory is False + assert evaluators.evaluate(check, {}).status == model.NOT_RUN + + +def test_dependency_pull_requests_each_get_their_own_merge_gate() -> None: + for number in TARGETS.dependency_pull_requests: + check = BY_ID[f"baseline.dependency-pr-{number}-merged"] + assert check.mandatory + assert check.expect["fields"]["merged"] is True + assert check.expect["fields"]["base.ref"] == "main" + assert "head.sha" in check.expect["fields"] + + +def test_the_baseline_commit_gate_names_the_configured_commit() -> None: + check = BY_ID["baseline.origin-main-descends-merged-work"] + assert TARGETS.baseline_commit in check.steps[0].argv + assert "--is-ancestor" in check.steps[0].argv + + +def test_pool_assignment_and_chat_continuity_are_unconditional() -> None: + assert "pool.assignment-safety-knobs-are-configured" in BY_ID + assert "pool.assignment-fieldrefs-are-bound" in BY_ID + assert "surfaces.chat-runs-the-telegram-topic-revision" in BY_ID + assert "surfaces.chat-telegram-sessions-are-continuous" in BY_ID + assert TARGETS.node_count == TARGETS.pool_replicas == 3 + + +def test_bulk_evidence_is_parsed_but_not_pasted_into_the_report() -> None: + routing = BY_ID["routing.provider-effort-and-fallback-evidence"] + assert routing.steps[0].record is False + assert routing.steps[0].max_bytes == TARGETS.routing_tail_bytes + + kanban = BY_ID["surfaces.kanban-activity-is-visible"] + assert kanban.steps[0].record is False + + +def test_freshness_gates_carry_the_run_clock() -> None: + for identifier in ( + "identity.codex-evidence-is-fresh", + "identity.claude-evidence-is-fresh", + ): + assert BY_ID[identifier].expect["now"] == NOW + routing = BY_ID["routing.provider-effort-and-fallback-evidence"] + assert routing.expect["now"] == NOW + assert routing.expect["max_age_seconds"] == TARGETS.max_evidence_age_seconds + + +def test_check_identifiers_are_unique_and_grouped() -> None: + identifiers = [check.id for check in SPECS] + assert len(identifiers) == len(set(identifiers)) + for check in SPECS: + assert check.id.split(".")[0] + assert check.group + + +@pytest.mark.parametrize( + "identifier", + [ + "identity.no-provider-api-key-in-pod", + "identity.no-provider-api-key-in-manifest", + "forge.worker-holds-no-git-credential-variable", + "access.secrets-are-denied", + "access.service-account-tokens-are-denied", + "access.workload-mutation-is-denied", + "access.pod-exec-is-denied", + "access.required-reads-are-allowed", + "access.flux-and-helm-status-are-allowed", + "access.namespace-view-agrees-across-vantages", + "nodes.hardening-covers-every-node", + "build.builder-service-account-is-tokenless", + "build.harbor-immutability-rule-is-applied", + "release.exact-lineage-is-running", + "reliability.finalization-and-replay-patches-are-deployed", + "pool.three-workers-have-distinct-nodes-and-volumes", + "pool.coordinator-retains-sole-state-ownership", + "surfaces.kanban-activity-is-visible", + "surfaces.chat-telegram-topic-state-is-durable", + "gitops.kustomizations-reconcile-cleanly", + "routing.provider-effort-and-fallback-evidence", + "scopes.agent-profile-is-distinct", + ], +) +def test_the_handoff_requirements_each_have_a_mandatory_check(identifier: str) -> None: + assert BY_ID[identifier].mandatory is True diff --git a/testing/tests/test_hermes_handoff_ephemeral.py b/testing/tests/test_hermes_handoff_ephemeral.py new file mode 100644 index 00000000..9f54601a --- /dev/null +++ b/testing/tests/test_hermes_handoff_ephemeral.py @@ -0,0 +1,337 @@ +"""Contracts for the strictly bounded, self-cleaning armed mode.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + FakeClock, + FakeSpawn, + load_handoff_module, +) + +ephemeral = load_handoff_module("hermes_handoff_ephemeral") +exec_module = load_handoff_module("hermes_handoff_exec") +policy = load_handoff_module("hermes_handoff_policy") +model = load_handoff_module("hermes_handoff_model") + +TOKEN = "acceptance-20260817a" +HEAD = "8f005458282269ba5c07941814e4237f5d4cf3ac" +MAIN = "1" * 40 +VANTAGE = exec_module.operator_vantage() + + +def request(**overrides): + fields = { + "repo": policy.EXPECTED_REPO, + "remote": policy.EXPECTED_REMOTE, + "token": TOKEN, + "confirmation": ephemeral.CONFIRMATION, + "base": policy.EXPECTED_BASE, + "expected_head": HEAD, + "expected_base_sha": MAIN, + } + fields.update(overrides) + return ephemeral.ArmRequest(**fields) + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) + + +def armed_runner(results: list, mode=policy.ARMED): + spawn = FakeSpawn(results) + runner = exec_module.Runner( + mode=mode, + clock=FakeClock(), + spawn=spawn, + environ={}, + deadline_seconds=3600, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + ) + return runner, spawn + + +def by_id(results: list) -> dict: + return {result.spec.id: result for result in results} + + +def pr_payload(state="open", number=42): + return { + "number": number, + "state": state, + "draft": True, + "merged": False, + "base": {"ref": "main", "sha": MAIN}, + "head": {"ref": request().ref, "sha": HEAD}, + } + + +def linked_worktree(tmp_path: Path) -> Path: + root = tmp_path / "worktree" + common = tmp_path / "repo.git" + gitdir = common / "worktrees" / "acceptance" + gitdir.mkdir(parents=True) + (root).mkdir() + (root / ".git").write_text(f"gitdir: {gitdir}\n", encoding="utf-8") + (gitdir / "commondir").write_text("../..\n", encoding="utf-8") + (gitdir / "HEAD").write_text( + f"ref: refs/heads/{ephemeral.EXPECTED_HEAD_REF}\n", encoding="utf-8" + ) + ref = common / "refs" / "heads" / ephemeral.EXPECTED_HEAD_REF + ref.parent.mkdir(parents=True) + ref.write_text(f"{HEAD}\n", encoding="utf-8") + remote_ref = common / "refs" / "remotes" / "origin" / "main" + remote_ref.parent.mkdir(parents=True) + remote_ref.write_text(f"{MAIN}\n", encoding="utf-8") + (common / "config").write_text( + f'[remote "origin"]\n\turl = {ephemeral.EXPECTED_REMOTE_URL}\n', + encoding="utf-8", + ) + return root + + +@pytest.mark.parametrize( + "ref", ["main", "master", "refs/heads/main", "HEAD", "feature/x", ""] +) +def test_only_the_unique_ephemeral_ref_is_allowed(ref: str) -> None: + with pytest.raises(ephemeral.ArmingError): + ephemeral.assert_push_target_allowed(ref) + ephemeral.assert_push_target_allowed(request().ref) + + +@pytest.mark.parametrize( + "overrides", + [ + {"confirmation": "wrong"}, + {"repo": "atlas/other"}, + {"remote": "upstream"}, + {"base": "develop"}, + {"token": "short"}, + {"expected_head": "bad"}, + {"expected_base_sha": "bad"}, + ], +) +def test_preflight_rejects_bad_inputs_before_any_runner_exists( + tmp_path: Path, overrides: dict +) -> None: + with pytest.raises(ephemeral.ArmingError): + ephemeral.preflight( + request(**overrides), policy.EXPECTED_REPO, linked_worktree(tmp_path) + ) + + +def test_preflight_attests_the_actual_linked_worktree(tmp_path: Path) -> None: + ephemeral.preflight(request(), policy.EXPECTED_REPO, linked_worktree(tmp_path)) + root = linked_worktree(tmp_path / "wrong") + (Path((root / ".git").read_text().split(": ", 1)[1].strip()) / "HEAD").write_text( + "ref: refs/heads/other\n", encoding="utf-8" + ) + with pytest.raises(ephemeral.ArmingError, match="existing PR #19"): + ephemeral.preflight(request(), policy.EXPECTED_REPO, root) + + +def test_worktree_attestation_rejects_each_malformed_local_source( + tmp_path: Path, +) -> None: + roots = [] + empty = tmp_path / "empty" + empty.mkdir() + roots.append(empty) + bad_marker = linked_worktree(tmp_path / "marker") + (bad_marker / ".git").write_text("not-a-gitdir\n", encoding="utf-8") + roots.append(bad_marker) + wrong_sha = linked_worktree(tmp_path / "sha") + gitdir = Path((wrong_sha / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "refs" / "heads" / ephemeral.EXPECTED_HEAD_REF).write_text("0" * 40) + roots.append(wrong_sha) + wrong_main = linked_worktree(tmp_path / "main") + gitdir = Path((wrong_main / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "refs" / "remotes" / "origin" / "main").write_text("0" * 40) + roots.append(wrong_main) + bad_config = linked_worktree(tmp_path / "config") + gitdir = Path((bad_config / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "config").write_text("[broken\n", encoding="utf-8") + roots.append(bad_config) + wrong_remote = linked_worktree(tmp_path / "remote") + gitdir = Path((wrong_remote / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "config").write_text('[remote "origin"]\nurl=https://example.invalid/x\n') + roots.append(wrong_remote) + for root in roots: + with pytest.raises((ephemeral.ArmingError, FileNotFoundError)): + ephemeral.preflight(request(), policy.EXPECTED_REPO, root) + oversized = tmp_path / "oversized" + oversized.write_text("x" * 5, encoding="utf-8") + with pytest.raises(ephemeral.ArmingError): + ephemeral._bounded_read(oversized, 4) + + +def test_guard_failure_stops_before_runner_calls(monkeypatch) -> None: + monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None) + runner, spawn = armed_runner([]) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.protected-branch-refusal"].status == model.FAIL + assert all( + result.status in {model.FAIL, model.NOT_RUN} for result in results.values() + ) + assert spawn.calls == [] + + +def test_full_armed_run_validates_and_removes_exact_artifacts() -> None: + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, spawn = armed_runner( + [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + completed(stdout=json.dumps(pr_payload())), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ] + ) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert all(result.status == model.PASS for result in results.values()) + push = next(call for call in spawn.calls if call["argv"][1] == "push") + assert push["argv"][-1] == f"HEAD:refs/heads/{request().ref}" + assert all(result.spec.mandatory for result in results.values()) + + +def test_existing_or_unverifiable_ref_never_reaches_a_write() -> None: + for first in (completed(stdout="abc\trefs/heads/x\n"), completed(returncode=128)): + runner, spawn = armed_runner([first]) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert not any( + call["argv"][1] in {"push", "POST", "PATCH", "DELETE"} + for call in spawn.calls + ) + + +def test_failed_or_ambiguous_create_is_discovered_and_cleanup_still_runs() -> None: + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, _ = armed_runner( + [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + completed(returncode=1), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ] + ) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.draft-pull-request"].status == model.PASS + assert results["ephemeral.cleanup-verified"].status == model.PASS + + +def test_pull_request_parsers_reject_every_malformed_or_ambiguous_shape() -> None: + assert ephemeral._pr_problems([], request()) == ["pull request is not an object"] + malformed = pr_payload() + malformed.update({"number": True, "draft": "yes", "base": None}) + malformed["head"] = {"ref": "other", "sha": "bad"} + problems = ephemeral._pr_problems(malformed, request()) + assert {"number is malformed", "draft does not match", "base is malformed"} <= set( + problems + ) + assert ephemeral._discover(exec_module.Outcome((), "x", returncode=1), request())[1] + for body in ( + "not-json", + "{}", + "[null]", + json.dumps([{"head": {"ref": "other"}, "base": {}}]), + ): + numbers, error = ephemeral._discover( + exec_module.Outcome((), "x", 0, body), request() + ) + assert numbers == [] and error + invalid = pr_payload() + invalid["draft"] = False + assert ephemeral._discover( + exec_module.Outcome((), "x", 0, json.dumps([invalid])), request() + )[1] + assert ( + "discovered 0" + in ephemeral._discover(exec_module.Outcome((), "x", 0, "[]"), request())[1] + ) + + +@pytest.mark.parametrize( + "delete_result,remaining,closed", + [ + (completed(returncode=1), completed(), pr_payload(state="closed")), + (completed(), completed(returncode=128), pr_payload(state="closed")), + (completed(), completed(), {}), + (completed(), completed(), pr_payload(state="open")), + ], +) +def test_cleanup_fails_closed_on_partial_or_malformed_evidence( + delete_result, remaining, closed +) -> None: + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, _ = armed_runner( + [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + completed(), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + delete_result, + remaining, + completed(stdout=json.dumps(closed)), + ] + ) + cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[ + "ephemeral.cleanup-verified" + ] + assert cleanup.status == model.FAIL + + +def test_cleanup_fails_when_close_or_verification_commands_are_bad_json_or_failures() -> ( + None +): + api = "/api/v1/repos/atlas/titan-iac" + for scripted in ( + [completed(returncode=1), completed(), completed(), completed(returncode=1)], + [completed(stdout="bad"), completed(), completed(), completed(stdout="bad")], + [ + completed(stdout=json.dumps(pr_payload())), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ], + ): + runner, _ = armed_runner(scripted) + result = ephemeral._cleanup(runner, VANTAGE, request(), api, [42]) + assert result.status == model.FAIL + + +def test_read_only_runner_cannot_reach_push() -> None: + runner, spawn = armed_runner( + [completed(), completed(stdout=f"{HEAD}\n"), completed(), completed()], + mode=policy.READ_ONLY, + ) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert not any(call["argv"][1] == "push" for call in spawn.calls) diff --git a/testing/tests/test_hermes_handoff_evaluators.py b/testing/tests/test_hermes_handoff_evaluators.py new file mode 100644 index 00000000..4de16c7e --- /dev/null +++ b/testing/tests/test_hermes_handoff_evaluators.py @@ -0,0 +1,457 @@ +"""Contracts for the text-shaped handoff acceptance evaluators.""" + +from __future__ import annotations + +import json + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) + +evaluators = load_handoff_module("hermes_handoff_evaluators") +model = load_handoff_module("hermes_handoff_model") + +REVIEW = model.REVIEW +ATTEMPT = model.ATTEMPT +FORBIDDEN = outcome( + stderr="Error from server (Forbidden): pods is forbidden", returncode=1 +) + + +def deny_spec() -> object: + return spec( + "denied", + steps=(step("review", kind=REVIEW), step("attempt", kind=ATTEMPT)), + ) + + +def evaluate(built, outcomes): + return evaluators.evaluate(built, outcomes) + + +def test_a_deny_check_passes_only_on_a_review_no_and_a_live_refusal() -> None: + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN} + ) + + assert evaluation.status == model.PASS + assert evaluation.evidence == {"review": "no", "attempt": "refused"} + + +def test_a_deny_check_fails_when_the_review_says_the_authority_exists() -> None: + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN} + ) + + assert evaluation.status == model.FAIL + assert "reported yes" in evaluation.reason + + +def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> None: + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()} + ) + + assert evaluation.status == model.FAIL + assert "must be refused" in evaluation.reason + + +def test_a_request_that_merely_broke_is_not_evidence_of_a_refusal() -> None: + broken = outcome(stderr="connection refused", returncode=7) + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="no"), "attempt": broken} + ) + + assert evaluation.status == model.NOT_RUN + assert "recognisable refusal" in evaluation.reason + + +def test_a_deny_check_without_a_live_attempt_refuses_to_classify() -> None: + """An authorization review states policy; only an attempt proves enforcement.""" + review_only = spec("denied", steps=(step("review", kind=REVIEW),)) + + evaluation = evaluate(review_only, {"review": outcome(stdout="no")}) + + assert evaluation.status == model.NOT_RUN + assert "require a real attempt" in evaluation.reason + + +def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> ( + None +): + built = spec("allowed", steps=(step("review", kind=REVIEW), step("read"))) + + assert ( + evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status + == model.PASS + ) + + denied = evaluate(built, {"review": outcome(stdout="no"), "read": outcome()}) + assert denied.status == model.FAIL and "reported no" in denied.reason + + failed = evaluate( + built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)} + ) + assert failed.status == model.FAIL and "step read failed" in failed.reason + + +def test_names_absent_flags_exact_names_and_substrings() -> None: + built = spec( + "names_absent", + {"step": "env", "names": ("GITEA_TOKEN",), "contains": ("API_KEY",)}, + steps=(step("env"),), + ) + + clean = evaluate(built, {"env": outcome(stdout="PATH\nHOME\n")}) + assert clean.status == model.PASS and clean.evidence["observed_count"] == 2 + + leaked = evaluate( + built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")} + ) + assert leaked.status == model.FAIL + assert leaked.evidence["offenders"] == ["GITEA_TOKEN", "OPENAI_API_KEY"] + + unreadable = evaluate(built, {"env": outcome(returncode=1)}) + assert unreadable.status == model.NOT_RUN + + +def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> ( + None +): + built = spec( + "names_present", + { + "step": "init", + "names": ("patch-web-session-activity",), + "contains": ("@sha256:",), + }, + steps=(step("init"),), + ) + + good = outcome(stdout="patch-web-session-activity\nimage@sha256:abc\n") + assert evaluate(built, {"init": good}).status == model.PASS + + partial = evaluate(built, {"init": outcome(stdout="patch-web-session-activity\n")}) + assert partial.status == model.FAIL and "@sha256:" in partial.reason + + empty = spec("names_present", {"step": "init"}, steps=(step("init"),)) + assert evaluate(empty, {"init": outcome(stdout="")}).status == model.NOT_RUN + + +def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None: + single = spec( + "stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),) + ) + assert evaluate(single, {"s": outcome(stdout=" locked \n")}).status == model.PASS + + mismatch = evaluate(single, {"s": outcome(stdout="unlocked")}) + assert mismatch.status == model.FAIL and mismatch.evidence["observed"] == "unlocked" + + several = spec( + "stdout_matches", + {"step": "s", "equals": ["600 root:root", "absent"]}, + steps=(step("s"),), + ) + assert evaluate(several, {"s": outcome(stdout="absent")}).status == model.PASS + + assert evaluate(single, {"s": outcome(returncode=1)}).status == model.NOT_RUN + + +def test_distinct_count_separates_replica_count_from_actual_spread() -> None: + built = spec( + "distinct_count", + {"step": "n", "minimum": 3, "total_equals": 3}, + steps=(step("n"),), + ) + + spread = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-06\ntitan-07\n")}) + assert spread.status == model.PASS and spread.evidence["distinct"] == 3 + + stacked = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-05\ntitan-05\n")}) + assert stacked.status == model.FAIL and "distinct values" in stacked.reason + + extra = spec( + "distinct_count", + {"step": "n", "minimum": 1, "total_equals": 2}, + steps=(step("n"),), + ) + over = evaluate(extra, {"n": outcome(stdout="a\nb\nc\n")}) + assert over.status == model.FAIL and "expected exactly 2" in over.reason + + assert evaluate(built, {"n": outcome(stdout="")}).status == model.NOT_RUN + assert evaluate(built, {"n": outcome(returncode=1)}).status == model.NOT_RUN + + +def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None: + built = spec( + "vantages_agree", + {"steps": ("operator", "self")}, + steps=(step("operator", vantage="operator"), step("self")), + ) + + agree = evaluate( + built, + { + "operator": outcome(stdout="ns/a\nns/b\n"), + "self": outcome(stdout="ns/b\nns/a\n"), + }, + ) + assert agree.status == model.PASS + + differ = evaluate( + built, + {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")}, + ) + assert differ.status == model.FAIL and "disagree" in differ.reason + + empty = evaluate( + built, {"operator": outcome(stdout=""), "self": outcome(stdout="")} + ) + assert empty.status == model.NOT_RUN + + broken = evaluate(built, {"operator": outcome(returncode=1), "self": outcome()}) + assert broken.status == model.NOT_RUN + + +def test_steps_agree_is_the_same_rule_under_a_neutral_name() -> None: + assert ( + evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"] + ) + + +def test_lines_match_catches_a_blank_leading_column() -> None: + built = spec( + "lines_match", + {"step": "s", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + steps=(step("s"),), + ) + table = "Preview Src ID\n-------\nwork on task cli 2026\n" + + assert evaluate(built, {"s": outcome(stdout=table)}).status == model.PASS + + blank = "Preview Src ID\n-------\n cli 2026\n" + failed = evaluate(built, {"s": outcome(stdout=blank)}) + assert failed.status == model.FAIL and failed.evidence["offenders"] + + assert ( + evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status + == model.NOT_RUN + ) + assert evaluate(built, {"s": outcome(returncode=1)}).status == model.NOT_RUN + + +def test_not_armed_is_the_fixed_skip_a_default_run_must_produce() -> None: + built = spec("not_armed", {"reason": "ephemeral mutation mode is not armed"}) + + evaluation = evaluate(built, {}) + + assert evaluation.status == model.NOT_RUN + assert evaluation.evidence == {"armed": False} + assert evaluation.reason == "ephemeral mutation mode is not armed" + assert ( + evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed" + ) + + +def test_truncation_fails_closed_before_every_registered_evaluator() -> None: + """No evaluator may reinterpret a clipped required capture as PASS.""" + clipped = outcome(stdout="apparently good", truncated=True) + for name in evaluators.EVALUATORS: + built = spec(name, {"step": "s"}, steps=(step("s"),)) + evaluation = evaluate(built, {"s": clipped}) + assert evaluation.status == model.NOT_RUN, name + assert "truncated" in evaluation.reason, name + + +def test_missing_required_steps_fail_closed_inside_every_step_evaluator() -> None: + for name in evaluators.EVALUATORS: + if name == "not_armed": + continue + built = spec(name, {"step": "s"}, steps=(step("s"),)) + assert evaluate(built, {}).status == model.NOT_RUN, name + + +def test_pool_topology_requires_three_ordinals_nodes_claims_and_ready_state() -> None: + built = spec( + "pool_topology", + { + "state_step": "state", + "worker_step": "workers", + "replicas": 3, + "name": "worker", + }, + steps=(step("state"), step("workers")), + ) + state = outcome(stdout="3\t3\t3\tfalse") + workers = outcome( + stdout=( + "worker-0\tnode-a\tRunning\tTrue\tclaim-0,\n" + "worker-1\tnode-b\tRunning\tTrue\tclaim-1,\n" + "worker-2\tnode-c\tRunning\tTrue\tclaim-2,\n" + ) + ) + assert evaluate(built, {"state": state, "workers": workers}).status == model.PASS + stacked = outcome(stdout=workers.stdout.replace("node-b", "node-a")) + assert evaluate(built, {"state": state, "workers": stacked}).status == model.FAIL + assert ( + evaluate(built, {"state": outcome(stdout="3\t3"), "workers": workers}).status + == model.NOT_RUN + ) + assert ( + evaluate( + built, {"state": outcome(stdout="x\t3\t3\tfalse"), "workers": workers} + ).status + == model.NOT_RUN + ) + assert ( + evaluate( + built, {"state": outcome(stdout="3\t2\t3\tfalse"), "workers": workers} + ).status + == model.FAIL + ) + malformed = outcome(stdout="worker-0\tnode-a\n") + assert ( + evaluate(built, {"state": state, "workers": malformed}).status == model.NOT_RUN + ) + bad_worker = outcome(stdout=workers.stdout.replace("Running", "Pending", 1)) + assert evaluate(built, {"state": state, "workers": bad_worker}).status == model.FAIL + duplicate = outcome(stdout=workers.stdout.replace("worker-1", "worker-0")) + assert evaluate(built, {"state": state, "workers": duplicate}).status == model.FAIL + wrong_ordinals = outcome(stdout=workers.stdout.replace("worker-2", "worker-3")) + assert ( + evaluate(built, {"state": state, "workers": wrong_ordinals}).status + == model.FAIL + ) + + +MAIN = "1" * 40 +HEAD = "2" * 40 +BUILD = MAIN +DIGEST = "sha256:" + "4" * 64 +IMAGE = f"registry.example/hermes:git-{BUILD}-build-7@{DIGEST}" + + +def lineage_spec(): + keys = ( + "remote-main", + "reviewed-pr", + "build-source", + "deployment", + "pods", + "replicasets", + "flux", + ) + return spec( + "release_lineage", + { + "main_sha": MAIN, + "head_sha": HEAD, + "head_ref": "feature/hermes-full-handoff-acceptance", + "pr_number": 19, + "image": IMAGE, + "build_sha": BUILD, + "deployment_revision": "7", + }, + steps=tuple(step(key) for key in keys), + ) + + +def lineage_outcomes(): + pull = { + "number": 19, + "state": "open", + "draft": True, + "merged": False, + "base": {"ref": "main", "sha": MAIN}, + "head": {"ref": "feature/hermes-full-handoff-acceptance", "sha": HEAD}, + } + return { + "remote-main": outcome(stdout=f"{MAIN}\trefs/heads/main\n"), + "reviewed-pr": outcome(stdout=json.dumps(pull)), + "build-source": outcome(stdout=BUILD), + "deployment": outcome(stdout=f"5\t5\t2\t2\t7\t{IMAGE}"), + "replicasets": outcome( + stdout=f"rs-old\t6\t0\t0\t{IMAGE}\told\nrs-live\t7\t2\t2\t{IMAGE}\thash7\n" + ), + "pods": outcome( + stdout=( + f"pod-a\tRunning\tTrue\t{IMAGE}\tdocker-pullable://hermes@{DIGEST}\thash7\n" + f"pod-b\tRunning\tTrue\t{IMAGE}\tcontainerd://hermes@{DIGEST}\thash7\n" + ) + ), + "flux": outcome(stdout=f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{MAIN}"), + } + + +def test_release_lineage_atomically_binds_every_release_identity() -> None: + assert evaluate(lineage_spec(), lineage_outcomes()).status == model.PASS + + +@pytest.mark.parametrize( + ("key", "bad"), + [ + ("remote-main", ""), + ("reviewed-pr", "{}"), + ("build-source", HEAD), + ("deployment", "5\t4\t2\t2\t7\tbad"), + ("replicasets", "malformed"), + ("pods", "malformed"), + ("flux", "hermes\t9\t9\tfalse\tUnknown\tmain@sha1:" + MAIN), + ], +) +def test_release_lineage_fails_closed_on_each_malformed_or_mismatched_source( + key, bad +) -> None: + outcomes = lineage_outcomes() + outcomes[key] = outcome(stdout=bad) + assert evaluate(lineage_spec(), outcomes).status in {model.FAIL, model.NOT_RUN} + + +def test_release_lineage_rejects_structural_partial_failure_variants() -> None: + variants = [] + malformed_expect = lineage_spec() + malformed_expect = model.CheckSpec( + **{ + **malformed_expect.__dict__, + "expect": {**malformed_expect.expect, "main_sha": "bad"}, + } + ) + variants.append((malformed_expect, lineage_outcomes())) + malformed_image = lineage_spec() + malformed_image = model.CheckSpec( + **{ + **malformed_image.__dict__, + "expect": { + **malformed_image.expect, + "image": f"registry.example/hermes@{DIGEST}", + }, + } + ) + variants.append((malformed_image, lineage_outcomes())) + for key, value in ( + ("reviewed-pr", "[]"), + ("deployment", f"x\tx\t2\t2\t7\t{IMAGE}"), + ("replicasets", f"rs\t7\t1\t2\t{IMAGE}\thash7"), + ("replicasets", f"a\t7\t2\t2\t{IMAGE}\th1\nb\t7\t2\t2\t{IMAGE}\th2"), + ("replicasets", f"rs\t6\t2\t2\t{IMAGE}\thash7"), + ("pods", f"pod-a\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7"), + ( + "pods", + f"pod-a\tPending\tFalse\t{IMAGE}\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7", + ), + ( + "pods", + f"pod-a\tRunning\tTrue\tbad\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7", + ), + ("flux", f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{HEAD}"), + ): + outcomes = lineage_outcomes() + outcomes[key] = outcome(stdout=value) + variants.append((lineage_spec(), outcomes)) + for built, outcomes in variants: + assert evaluate(built, outcomes).status in {model.FAIL, model.NOT_RUN} diff --git a/testing/tests/test_hermes_handoff_exec.py b/testing/tests/test_hermes_handoff_exec.py new file mode 100644 index 00000000..c192448d --- /dev/null +++ b/testing/tests/test_hermes_handoff_exec.py @@ -0,0 +1,363 @@ +"""Contracts for bounded, screened command execution in the handoff harness.""" + +from __future__ import annotations + +import hashlib +import subprocess +import time + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + FakeClock, + FakeSpawn, + load_handoff_module, +) + +runner_module = load_handoff_module("hermes_handoff_exec") +policy = load_handoff_module("hermes_handoff_policy") + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) + + +def build(results: list, **kwargs) -> tuple: + clock = FakeClock() + spawn = FakeSpawn(results) + runner = runner_module.Runner( + clock=clock, + spawn=spawn, + environ={}, + attestor=lambda command, _environment: runner_module.Attestation( + command, "a" * 64 + ), + **kwargs, + ) + return (runner, spawn, clock) + + +OPERATOR = runner_module.operator_vantage() +SAFE_GET = ("kubectl", "get", "pods", "-o", "name") + + +def test_a_successful_command_is_captured_and_screened() -> None: + runner, spawn, _ = build([completed(stdout="password=a-long-enough-secret\n")]) + + outcome = runner.run(SAFE_GET, OPERATOR) + + assert outcome.ok and outcome.ran + assert "[redacted]" in outcome.stdout + assert outcome.returncode == 0 + assert spawn.calls[0]["argv"] == list(SAFE_GET) + assert runner.commands_run == 1 + + +def test_a_failing_command_is_not_ok_but_still_ran() -> None: + runner, _, _ = build( + [completed(stderr="Error from server (Forbidden)", returncode=1)] + ) + + outcome = runner.run(SAFE_GET, OPERATOR) + + assert outcome.ran and not outcome.ok + assert "Forbidden" in outcome.combined + + +def test_a_policy_violation_never_reaches_a_subprocess() -> None: + runner, spawn, _ = build([]) + + outcome = runner.run(("kubectl", "delete", "pods", "--all"), OPERATOR) + + assert outcome.error and outcome.error.startswith("policy:") + assert not outcome.ran + assert spawn.calls == [] + assert runner.commands_run == 0 + + +def test_a_timeout_is_recorded_rather_than_raised() -> None: + runner, _, _ = build([subprocess.TimeoutExpired(cmd="kubectl", timeout=1.0)]) + + outcome = runner.run(SAFE_GET, OPERATOR) + + assert outcome.error == runner_module.TIMEOUT_ERROR + assert not outcome.ran + + +def test_a_missing_binary_is_recorded_rather_than_raised() -> None: + runner, _, _ = build([FileNotFoundError(2, "No such file or directory")]) + + outcome = runner.run(SAFE_GET, OPERATOR) + + assert outcome.error and outcome.error.startswith("spawn:") + + +def test_the_run_deadline_stops_further_probes_without_spawning() -> None: + runner, spawn, clock = build([completed(stdout="first")], deadline_seconds=10.0) + + assert runner.run(SAFE_GET, OPERATOR).ok + clock.advance(11.0) + second = runner.run(("kubectl", "get", "nodes", "-o", "name"), OPERATOR) + + assert second.error == runner_module.DEADLINE_ERROR + assert len(spawn.calls) == 1 + assert runner.remaining_seconds < 0 + + +def test_the_command_timeout_is_clamped_by_the_remaining_deadline() -> None: + runner, spawn, clock = build( + [completed()], command_timeout=30.0, deadline_seconds=10.0 + ) + clock.advance(6.0) + + runner.run(SAFE_GET, OPERATOR) + + assert spawn.calls[0]["timeout"] == 4.0 + + +def test_output_is_bounded_per_command_and_can_be_raised_per_call() -> None: + runner, _, _ = build( + [completed(stdout="y" * 200), completed(stdout="y" * 200)], max_bytes=50 + ) + + small = runner.run(SAFE_GET, OPERATOR) + large = runner.run(SAFE_GET, OPERATOR, max_bytes=4096) + + assert small.truncated + assert not large.truncated + + +def test_the_child_environment_is_rebuilt_from_an_allowlist() -> None: + source = { + "PATH": "/bin", + "KUBECONFIG": "/tmp/kc", + "ANTHROPIC_API_KEY": "leak", + "HOME": "/root", + } + environment = runner_module.build_environment(OPERATOR, source) + + assert environment["PATH"] == runner_module.SAFE_PATH + assert environment["KUBECONFIG"] == "/tmp/kc" + assert environment["GITEA_BASE_URL"] == "https://scm.bstein.dev" + assert environment["GIT_CONFIG_GLOBAL"] == "/dev/null" + assert "HOME" not in environment + assert "ANTHROPIC_API_KEY" not in environment + + +def test_a_vantage_can_override_the_environment_it_needs() -> None: + vantage = runner_module.operator_vantage(kubeconfig="/tmp/operator.yaml") + environment = runner_module.build_environment(vantage, {"KUBECONFIG": "/tmp/other"}) + + assert environment["KUBECONFIG"] == "/tmp/operator.yaml" + assert "kubeconfig-pinned" in vantage.description + + +def test_the_operator_context_is_inserted_only_for_context_aware_tools() -> None: + vantage = runner_module.operator_vantage(context="atlas-operator") + + assert vantage.wrap(("kubectl", "get", "pods"))[:3] == ( + "kubectl", + "--context", + "atlas-operator", + ) + assert vantage.wrap(("git", "status")) == ("git", "status") + assert vantage.wrap(()) == () + + +def test_the_pod_vantage_execs_into_the_container_and_drops_the_context() -> None: + operator = runner_module.operator_vantage(context="atlas-operator") + vantage = runner_module.pod_vantage("hermes", "hermes-agent-1", "hermes", operator) + + wrapped = vantage.wrap(SAFE_GET) + + assert wrapped[:3] == ("kubectl", "--context", "atlas-operator") + assert wrapped[3] == "exec" + assert wrapped[-6:] == ( + "--", + "/usr/local/bin/kubectl", + "get", + "pods", + "-o", + "name", + ) + assert vantage.description == "in-pod hermes/hermes-agent-1[hermes]" + policy.check_argv(wrapped) + + +def test_a_pod_vantage_without_an_operator_still_addresses_the_pod() -> None: + vantage = runner_module.pod_vantage("hermes", "pod-a", "hermes") + + assert vantage.wrap(("kubectl", "version"))[0] == "kubectl" + assert vantage.env == {} + + +def test_outcome_serialisation_screens_the_command_line() -> None: + outcome = runner_module.Outcome( + argv=("git", "clone", "https://user:a-long-enough-secret@scm.example.dev/x"), + vantage="operator", + returncode=0, + stdout="ok", + ) + + payload = outcome.as_dict() + + assert "a-long-enough-secret" not in payload["command"] + assert payload["vantage"] == "operator" + assert payload["stdout"] == "ok" + + +def test_combined_output_joins_only_the_streams_that_carry_text() -> None: + assert ( + runner_module.Outcome(argv=(), vantage="x", stdout="a", stderr="b").combined + == "a\nb" + ) + assert runner_module.Outcome(argv=(), vantage="x", stdout="a").combined == "a" + assert runner_module.Outcome(argv=(), vantage="x").combined == "" + + +@pytest.mark.parametrize( + "kwargs", + [ + {"command_timeout": float("nan")}, + {"command_timeout": float("inf")}, + {"command_timeout": 0}, + {"deadline_seconds": -1}, + {"max_bytes": -1}, + {"max_bytes": 10**100}, + {"concurrency": 0}, + {"concurrency": 17}, + ], +) +def test_execution_bounds_reject_nonfinite_nonpositive_or_unbounded_inputs( + kwargs, +) -> None: + with pytest.raises(ValueError): + runner_module.Runner(**kwargs) + + +def test_execution_bounds_reject_boolean_and_nonnumeric_inputs() -> None: + for value in (True, "1", None): + with pytest.raises(ValueError, match="numeric"): + runner_module.Runner(command_timeout=value) + + +def test_every_reviewer_unsafe_argv_is_refused_without_spawning() -> None: + runner, spawn, _ = build([]) + commands = [ + ("kubectl", "auth", "reconcile", "-f", "x"), + ("kubectl", "config", "view", "--raw"), + ("kubectl", "get", "secret/x", "-o", "name"), + ("kubectl", "get", "--raw=/api/v1/secrets"), + ("kubectl", "delete", "pod/x", "--dry-run=client"), + ("kubectl", "delete", "pod/x", "--", "--dry-run=server"), + ("git", "fetch", "origin"), + ("git", "config", "core.sshCommand", "x"), + ("git", "-ccore.sshCommand=x", "ls-remote", "origin"), + ("helm", "get", "values", "x"), + ] + for command in commands: + assert runner.run(command, OPERATOR).error.startswith("policy:") + assert spawn.calls == [] + assert runner.commands_run == 0 + + +def test_live_capture_bounds_concurrent_stdout_and_stderr() -> None: + capture = runner_module.execute_bounded( + ( + "/usr/bin/python3", + "-c", + "import os;os.write(1,b'x'*2000000);os.write(2,b'y'*2000000)", + ), + {"LC_ALL": "C"}, + timeout=5, + max_bytes=1024, + ) + assert capture.returncode == 0 + assert capture.stdout_truncated and capture.stderr_truncated + assert len(capture.stdout) == len(capture.stderr) == 1024 + + +def test_absolute_timeout_kills_the_process_group_and_reaps_quickly() -> None: + started = time.monotonic() + capture = runner_module.execute_bounded( + ( + "/usr/bin/python3", + "-c", + "import os,signal,time;" + "p=os.fork();" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "time.sleep(30)", + ), + {"LC_ALL": "C"}, + timeout=0.15, + max_bytes=1024, + ) + assert capture.timed_out + assert time.monotonic() - started < 1.5 + + +def test_descendant_holding_pipes_is_bounded_after_the_leader_exits() -> None: + started = time.monotonic() + capture = runner_module.execute_bounded( + ( + "/usr/bin/python3", + "-c", + "import os,signal,time;" + "p=os.fork();" + "(os._exit(0) if p else None);" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "time.sleep(30)", + ), + {"LC_ALL": "C"}, + timeout=2, + max_bytes=1024, + ) + assert not capture.timed_out + assert time.monotonic() - started < 1.5 + + +def test_attestation_rejects_a_reviewer_authored_executable(tmp_path) -> None: + fake = tmp_path / "kubectl" + fake.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + fake.chmod(0o755) + with pytest.raises(OSError, match="trusted paths"): + runner_module.attest_executable(str(fake), {"PATH": str(tmp_path)}) + + +def test_attestation_checks_presence_mode_digest_and_success( + tmp_path, monkeypatch +) -> None: + fake = tmp_path / "tool" + fake.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + fake.chmod(0o755) + digest = hashlib.sha256(fake.read_bytes()).hexdigest() + monkeypatch.setitem(runner_module.EXPECTED_PATHS, "tool", {str(fake)}) + monkeypatch.setitem(runner_module.EXPECTED_SHA256, "tool", {digest}) + assert ( + runner_module.attest_executable("tool", {"PATH": str(tmp_path)}).sha256 + == digest + ) + with pytest.raises(OSError, match="unavailable"): + runner_module.attest_executable("missing", {"PATH": str(tmp_path)}) + fake.chmod(0o777) + with pytest.raises(OSError, match="unsafe ownership mode"): + runner_module.attest_executable("tool", {"PATH": str(tmp_path)}) + fake.chmod(0o755) + monkeypatch.setitem(runner_module.EXPECTED_SHA256, "tool", {"0" * 64}) + with pytest.raises(OSError, match="digest"): + runner_module.attest_executable("tool", {"PATH": str(tmp_path)}) + + +def test_runner_attestation_or_resolved_policy_failure_never_spawns() -> None: + runner, spawn, _ = build([]) + runner._attestor = lambda _command, _environment: (_ for _ in ()).throw( + OSError("no") + ) + assert runner.run(SAFE_GET, OPERATOR).error.startswith("attestation:") + runner._attestor = lambda _command, _environment: runner_module.Attestation( + "/tmp/reviewer-authored/kubectl", "a" * 64 + ) + assert runner.run(SAFE_GET, OPERATOR).error.startswith("policy:") + assert spawn.calls == [] diff --git a/testing/tests/test_hermes_handoff_json_rules.py b/testing/tests/test_hermes_handoff_json_rules.py new file mode 100644 index 00000000..e74684d7 --- /dev/null +++ b/testing/tests/test_hermes_handoff_json_rules.py @@ -0,0 +1,413 @@ +"""Contracts for the JSON-shaped handoff acceptance evaluators.""" + +from __future__ import annotations + +import datetime as dt +import json + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) + +evaluators = load_handoff_module("hermes_handoff_evaluators") +model = load_handoff_module("hermes_handoff_model") + +NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc) + + +def evaluate(built, payload): + body = payload if isinstance(payload, str) else json.dumps(payload) + return evaluators.evaluate(built, {"s": outcome(stdout=body)}) + + +def one(rule: str, expect: dict): + return spec(rule, {"step": "s", **expect}, steps=(step("s"),)) + + +def test_json_field_compares_paths_against_expected_values() -> None: + built = one( + "json_field", {"fields": {"authenticated": True, "transport": ["a", "b"]}} + ) + + assert ( + evaluate(built, {"authenticated": True, "transport": "b"}).status == model.PASS + ) + + wrong = evaluate(built, {"authenticated": False, "transport": "c"}) + assert wrong.status == model.FAIL + assert "authenticated=False" in wrong.reason and "transport='c'" in wrong.reason + + assert evaluate(built, {"authenticated": True}).status == model.NOT_RUN + + +def test_json_field_skips_rather_than_guesses_on_unusable_output() -> None: + built = one("json_field", {"fields": {"a": 1}}) + + assert evaluate(built, "not json").status == model.NOT_RUN + failed = evaluators.evaluate(built, {"s": outcome(returncode=1)}) + assert failed.status == model.NOT_RUN + assert evaluate(built, '{"a": 1, "a": 1}').status == model.NOT_RUN + assert evaluate(built, '{"a": NaN}').status == model.NOT_RUN + + +def test_json_equality_does_not_confuse_booleans_and_integers() -> None: + built = one("json_field", {"fields": {"enabled": True}}) + assert evaluate(built, {"enabled": 1}).status == model.FAIL + array = one("all_of_field", {"fields": {"enabled": False}}) + assert evaluate(array, [{"enabled": 0}]).status == model.FAIL + + +def test_json_numeric_enforces_inclusive_bounds_and_rejects_non_numbers() -> None: + built = one("json_numeric", {"fields": {"latency_ms": {"min": 1, "max": 1000}}}) + + assert evaluate(built, {"latency_ms": 500}).status == model.PASS + assert evaluate(built, {"latency_ms": 1}).status == model.PASS + + low = evaluate(built, {"latency_ms": 0}) + assert low.status == model.FAIL and "below minimum" in low.reason + + high = evaluate(built, {"latency_ms": 5000}) + assert high.status == model.FAIL and "above maximum" in high.reason + + assert evaluate(built, {"latency_ms": True}).status == model.NOT_RUN + assert evaluate(built, {"latency_ms": "fast"}).status == model.NOT_RUN + assert evaluate(built, {}).status == model.NOT_RUN + + +def test_json_recent_treats_stale_evidence_as_a_failure() -> None: + built = one("json_recent", {"now": NOW, "fields": {"checked_at": 3600}}) + + fresh = evaluate(built, {"checked_at": "2026-08-17T11:30:00Z"}) + assert ( + fresh.status == model.PASS + and fresh.evidence["checked_at"]["age_seconds"] == 1800 + ) + + stale = evaluate(built, {"checked_at": "2026-08-16T00:00:00Z"}) + assert stale.status == model.FAIL and "limit 3600s" in stale.reason + + naive = evaluate(built, {"checked_at": "2026-08-17T11:30:00"}) + assert naive.status == model.NOT_RUN + assert evaluate(built, {"checked_at": "2026-08-17T12:00:01Z"}).status == model.FAIL + + assert evaluate(built, {"checked_at": "yesterday"}).status == model.NOT_RUN + assert evaluate(built, {}).status == model.NOT_RUN + invalid_clock = one("json_recent", {"now": "now", "fields": {"checked_at": 1}}) + assert ( + evaluate(invalid_clock, {"checked_at": "2026-08-17T11:30:00Z"}).status + == model.NOT_RUN + ) + + +def test_all_of_field_selects_keys_and_reports_every_mismatch() -> None: + built = one( + "all_of_field", + {"key_field": "number", "keys": [14, 15], "fields": {"merged": True}}, + ) + payload = [ + {"number": 14, "merged": True}, + {"number": 15, "merged": False}, + {"number": 16, "merged": False}, + ] + + evaluation = evaluate(built, payload) + + assert evaluation.status == model.FAIL + assert "15: merged=False" in evaluation.reason + assert "16" not in evaluation.reason + assert evaluation.evidence["examined"] == [14, 15] + + +def test_all_of_field_skips_when_an_expected_key_is_absent_or_nothing_matched() -> None: + keyed = one( + "all_of_field", + {"key_field": "number", "keys": [14], "fields": {"merged": True}}, + ) + absent = evaluate(keyed, [{"number": 99, "merged": True}]) + assert absent.status == model.NOT_RUN and "expected keys absent" in absent.reason + + unkeyed = one("all_of_field", {"fields": {"status": ["done"]}}) + assert evaluate(unkeyed, []).status == model.NOT_RUN + assert evaluate(unkeyed, {"not": "an array"}).status == model.NOT_RUN + + +def test_all_of_field_skips_when_the_named_array_path_is_absent() -> None: + built = spec( + "all_of_field", + {"step": "s", "array": "items", "fields": {"a": 1}}, + steps=(step("s"),), + ) + + evaluation = evaluate(built, {"other": []}) + + assert evaluation.status == model.NOT_RUN + assert "array path is absent" in evaluation.reason + + +def test_all_of_field_treats_an_absent_field_as_a_mismatch_not_a_crash() -> None: + built = one("all_of_field", {"key_field": "name", "fields": {"suspended": False}}) + + evaluation = evaluate(built, [{"name": "a"}]) + + assert evaluation.status == model.FAIL + assert "suspended=None" in evaluation.reason + + +def test_all_of_field_honours_exemptions_and_non_empty_paths() -> None: + exempted = one( + "all_of_field", + { + "key_field": "name", + "exempt_keys": ["parked"], + "fields": {"suspended": False}, + }, + ) + payload = [ + {"name": "parked", "suspended": True}, + {"name": "live", "suspended": False}, + ] + assert evaluate(exempted, payload).status == model.PASS + + non_empty = one("all_of_field", {"key_field": "id", "non_empty": ["title"]}) + assert evaluate(non_empty, [{"id": "a", "title": "work"}]).status == model.PASS + blank = evaluate(non_empty, [{"id": "a", "title": ""}, {"id": "b"}]) + assert blank.status == model.FAIL and "is empty" in blank.reason + + +def test_json_record_captures_named_paths_or_skips() -> None: + built = one( + "json_record", {"record": {"numbers": "[].number", "refs": "[].head.ref"}} + ) + payload = [ + {"number": 14, "head": {"ref": "a"}}, + {"number": 15, "head": {"ref": "b"}}, + ] + + evaluation = evaluate(built, payload) + assert evaluation.status == model.PASS + assert evaluation.evidence == {"numbers": [14, 15], "refs": ["a", "b"]} + + missing = evaluate(built, [{"number": 14}]) + assert missing.status == model.NOT_RUN and "absent" in missing.reason + + +def routing_log(*records: dict) -> str: + return "\n".join(json.dumps(record) for record in records) + + +def routing_spec(**extra): + return spec( + "routing_evidence", + { + "step": "s", + "providers": ("codex", "claude"), + "efforts": ("high", "xhigh"), + "lanes": ("route", "worker"), + "require_fallback_evidence": True, + **extra, + }, + steps=(step("s"),), + ) + + +FULL_LOG = routing_log( + {"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}, + {"ts": "2026-08-17T11:30:00Z", "model": "route/claude/opus/xhigh"}, + { + "ts": "2026-08-17T11:45:00Z", + "model": "worker/codex/sol/xhigh", + "fallback_reason": "unavailable", + }, + {"ts": "2026-08-17T11:50:00Z", "model": "qwen2.5:14b", "tier": "classifier"}, +) + + +def test_routing_evidence_summarises_lanes_providers_efforts_and_fallbacks() -> None: + evaluation = evaluate(routing_spec(), FULL_LOG) + + assert evaluation.status == model.PASS + assert evaluation.evidence["providers"] == ["claude", "codex"] + assert evaluation.evidence["efforts"] == ["high", "xhigh"] + assert evaluation.evidence["lanes"] == ["route", "worker"] + assert evaluation.evidence["fallback_reasons"] == ["unavailable"] + assert evaluation.evidence["records"] == 4 and evaluation.evidence["routed"] == 3 + + +def test_routing_evidence_names_exactly_what_the_window_did_not_show() -> None: + partial = routing_log( + {"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"} + ) + + evaluation = evaluate(routing_spec(), partial) + + assert evaluation.status == model.FAIL + assert "provider:claude" in evaluation.reason + assert "effort:xhigh" in evaluation.reason + assert "lane:worker" in evaluation.reason + assert "fallback:none-observed" in evaluation.reason + + +def test_routing_evidence_rejects_a_clipped_tail_and_empty_input() -> None: + clipped = '{"ts": "2026-08-17T11:0\n' + FULL_LOG + + assert evaluate(routing_spec(), clipped).status == model.NOT_RUN + assert evaluate(routing_spec(), "").status == model.NOT_RUN + assert evaluate(routing_spec(), "garbage\n").status == model.NOT_RUN + + +def test_routing_evidence_can_require_freshness() -> None: + fresh = evaluate(routing_spec(max_age_seconds=3600, now=NOW), FULL_LOG) + assert fresh.status == model.PASS + assert max(fresh.evidence["age_seconds"].values()) <= 3600 + + stale = evaluate( + routing_spec(max_age_seconds=60, now=NOW + dt.timedelta(days=2)), FULL_LOG + ) + assert stale.status == model.FAIL and "freshness" in stale.reason + + undated = routing_log( + {"model": "route/codex/sol/high"}, {"model": "route/claude/opus/xhigh"} + ) + unknown = evaluate(routing_spec(max_age_seconds=60, now=NOW), undated) + assert unknown.status == model.NOT_RUN + + +def test_routing_freshness_is_required_per_provider_not_only_globally() -> None: + rows = routing_log( + {"ts": "2026-08-15T11:00:00Z", "model": "route/claude/opus/high"}, + {"ts": "2026-08-17T11:59:00Z", "model": "route/codex/sol/xhigh"}, + { + "ts": "2026-08-17T11:59:00Z", + "model": "worker/codex/sol/high", + "fallback_reason": "down", + }, + ) + evaluation = evaluate(routing_spec(max_age_seconds=3600, now=NOW), rows) + assert evaluation.status == model.FAIL + assert "freshness:provider:claude" in evaluation.reason + missing_dimension = evaluate( + routing_spec(max_age_seconds=3600, now=NOW), + routing_log( + {"ts": "2026-08-17T11:59:00Z", "model": "route/codex/sol/high"}, + { + "ts": "2026-08-17T11:59:00Z", + "model": "worker/codex/sol/xhigh", + "fallback_reason": "down", + }, + ), + ) + assert missing_dimension.status == model.FAIL + + +@pytest.mark.parametrize( + "body", + [ + '["not-an-object"]', + '{"ts":"2026-08-17T12:00:01Z","model":"route/codex/sol/high"}', + '{"ts":"2026-08-17T11:00:00Z"}', + '{"ts":"2026-08-17T11:00:00Z","model":"bad"}', + '{"ts":"2026-08-17T11:00:00Z","model":"route/codex/sol/high","fallback_reason":0}', + ], +) +def test_routing_rows_fail_closed_on_structural_defects(body: str) -> None: + assert evaluate(routing_spec(now=NOW), body).status in {model.FAIL, model.NOT_RUN} + invalid_clock = routing_spec(now="now") + assert evaluate(invalid_clock, FULL_LOG).status == model.NOT_RUN + + +def flux_spec(*expected: str): + return spec( + "flux_health", + {"step": "s", "expected_suspensions": expected}, + steps=(step("s"),), + ) + + +def test_flux_health_separates_parked_objects_from_broken_ones() -> None: + rows = ( + "flux-system/core\t1\t1\tfalse\tReady=True,Healthy=True,\n" + "flux-system/parked\t1\t1\ttrue\tReady=False,\n" + "flux-system/broken\t1\t1\tfalse\tReady=False,Healthy=False,\n" + ) + + evaluation = evaluators.evaluate( + flux_spec("flux-system/parked"), {"s": outcome(stdout=rows)} + ) + + assert evaluation.status == model.FAIL + assert evaluation.evidence["unhealthy"] == ["flux-system/broken"] + assert evaluation.evidence["unexpected_suspensions"] == [] + assert "unhealthy: flux-system/broken" in evaluation.reason + + +def test_flux_health_flags_a_suspension_nobody_declared() -> None: + rows = "flux-system/core\t1\t1\tfalse\tReady=True,\nflux-system/parked\t1\t1\ttrue\tReady=False,\n" + + evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) + + assert evaluation.status == model.FAIL + assert evaluation.evidence["unexpected_suspensions"] == ["flux-system/parked"] + + +def test_flux_health_fails_closed_on_in_flight_or_unknown_conditions() -> None: + rows = ( + "flux-system/reconciling\t1\t1\tfalse\tReconciling=True,Ready=False,\n" + "flux-system/degraded\t1\t1\tfalse\tReady=False,Healthy=True,\n" + "flux-system/waiting\t1\t1\tfalse\tReady=Unknown,\n" + ) + + evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) + + assert evaluation.status == model.FAIL + assert evaluation.evidence["total"] == 3 + + +def test_blank_rows_in_a_projection_are_ignored_by_both_log_parsers() -> None: + padded = "\n\n" + FULL_LOG + "\n\n" + assert evaluate(routing_spec(), padded).status == model.PASS + + rows = "\n\nflux-system/core\t1\t1\tfalse\tReady=True,\n\n" + evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) + assert evaluation.status == model.PASS and evaluation.evidence["total"] == 1 + + +def test_flux_health_skips_on_an_empty_or_failed_projection() -> None: + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(stdout="")}).status + == model.NOT_RUN + ) + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(returncode=1)}).status + == model.NOT_RUN + ) + malformed = "flux-system/core\t1\t0\tfalse\tReady=True,\n" + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(stdout=malformed)}).status + == model.FAIL + ) + + +@pytest.mark.parametrize( + "row", + [ + "too\tfew", + "BAD NAME\t1\t1\tfalse\tReady=True,", + "flux-system/core\tx\t1\tfalse\tReady=True,", + "flux-system/core\t1\t1\tmaybe\tReady=True,", + "flux-system/core\t1\t1\tfalse\tbroken,", + "flux-system/core\t1\t1\tfalse\tReady=Yes,", + "flux-system/core\t1\t1\tfalse\tReady=True,Ready=True,", + ], +) +def test_flux_rows_reject_malformed_columns_names_generations_and_conditions( + row: str, +) -> None: + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(stdout=row)}).status + == model.NOT_RUN + ) diff --git a/testing/tests/test_hermes_handoff_model.py b/testing/tests/test_hermes_handoff_model.py new file mode 100644 index 00000000..e3c46489 --- /dev/null +++ b/testing/tests/test_hermes_handoff_model.py @@ -0,0 +1,206 @@ +"""Contracts for the handoff harness result model, verdict, and rendering.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) + +model = load_handoff_module("hermes_handoff_model") + + +def result( + status: str, + identifier: str = "test.check", + mandatory: bool = True, + group: str = "g", +): + return model.CheckResult( + spec=model.CheckSpec( + id=identifier, title="t", group=group, rule="allowed", mandatory=mandatory + ), + status=status, + reason=f"{status} reason", + ) + + +def report(*results, errors: tuple[str, ...] = ()) -> model.Report: + return model.Report( + mode="read-only", + started_at="2026-08-17T00:00:00Z", + results=list(results), + harness_errors=list(errors), + ) + + +@pytest.mark.parametrize( + ("status", "mandatory", "blocking"), + [ + (model.PASS, True, False), + (model.NOT_APPLICABLE, True, False), + (model.FAIL, True, True), + (model.NOT_RUN, True, True), + (model.FAIL, False, False), + (model.NOT_RUN, False, False), + ], +) +def test_only_a_mandatory_failure_or_skip_blocks(status, mandatory, blocking) -> None: + assert result(status, mandatory=mandatory).blocking is blocking + + +def test_a_mandatory_skip_is_a_no_go_exactly_like_a_failure() -> None: + """The whole point of the four-valued classification.""" + assert report(result(model.NOT_RUN)).decision == model.NO_GO + assert report(result(model.FAIL)).decision == model.NO_GO + assert report(result(model.PASS), result(model.NOT_APPLICABLE)).decision == model.GO + + +def test_a_harness_error_is_a_no_go_even_when_every_check_passed() -> None: + clean = report( + result(model.PASS), errors=("the two vantages are the same principal",) + ) + assert clean.decision == model.NO_GO + + +def test_a_zero_state_or_unknown_status_report_is_no_go() -> None: + """Absence or an unrecognised classification can never round up to GO.""" + assert report().decision == model.NO_GO + assert report(errors=("no checks were built",)).decision == model.NO_GO + unknown = result("MAYBE") + assert unknown.blocking is True + built = report(unknown) + assert built.decision == model.NO_GO + assert built.counts == dict.fromkeys(model.STATUSES, 0) + assert "groups:" in report(result(model.PASS)).render_summary() + + +def test_blocking_results_are_ordered_failures_before_skips() -> None: + ordered = report( + result(model.NOT_RUN, "b.skip"), + result(model.FAIL, "z.fail"), + result(model.PASS, "a.pass"), + ).blocking + + assert [item.spec.id for item in ordered] == ["z.fail", "b.skip"] + + +def test_counts_and_group_summary_tally_every_status() -> None: + built = report( + result(model.PASS, "a", group="alpha"), + result(model.FAIL, "b", group="alpha"), + result(model.NOT_RUN, "c", group="beta"), + ) + + assert built.counts == { + model.PASS: 1, + model.FAIL: 1, + model.NOT_RUN: 1, + model.NOT_APPLICABLE: 0, + } + assert built.group_summary()["alpha"][model.PASS] == 1 + assert built.group_summary()["beta"][model.NOT_RUN] == 1 + + +def test_the_serialised_report_carries_the_verdict_and_its_reasons() -> None: + built = report(result(model.FAIL, "x.fail"), result(model.PASS, "y.pass")) + built.vantages = [ + model.VantageRecord( + name="operator", description="d", identity="u", available=True + ) + ] + + payload = built.as_dict() + + assert payload["decision"] == model.NO_GO + assert payload["blocking"] == ["x.fail"] + assert payload["harness"] == model.HARNESS + assert payload["schema_version"] == model.SCHEMA_VERSION + assert payload["vantages"][0]["identity"] == "u" + assert {check["id"] for check in payload["checks"]} == {"x.fail", "y.pass"} + + +def test_check_serialisation_screens_evidence_and_can_omit_commands() -> None: + built = model.CheckResult( + spec=spec("allowed", identifier="x", steps=(step("a"),)), + status=model.PASS, + evidence={"token": "a-long-enough-secret", "count": 3}, + outcomes=[outcome(stdout="fine")], + ) + + with_commands = built.as_dict() + without = built.as_dict(include_outcomes=False) + + assert with_commands["evidence"]["token"] == "[redacted]" + assert with_commands["evidence"]["count"] == 3 + assert len(with_commands["commands"]) == 1 + assert "commands" not in without + assert with_commands["vantages"] == ["self"] + + +def test_rationale_is_carried_only_when_a_check_has_one() -> None: + without = model.CheckResult(spec=spec("allowed"), status=model.PASS).as_dict() + assert "rationale" not in without + + annotated = model.CheckSpec( + id="x", title="t", group="g", rule="allowed", rationale="because" + ) + with_reason = model.CheckResult(spec=annotated, status=model.PASS).as_dict() + assert with_reason["rationale"] == "because" + + +def test_distinct_vantages_are_reported_in_first_use_order() -> None: + built = spec( + "allowed", + steps=( + step("a", vantage="operator"), + step("b", vantage="self"), + step("c", vantage="operator"), + ), + ) + assert built.vantages == ("operator", "self") + + +def test_the_human_summary_names_the_verdict_vantages_and_blockers() -> None: + built = report(result(model.FAIL, "x.fail", group="alpha"), errors=("collision",)) + built.vantages = [ + model.VantageRecord( + name="operator", description="d", identity="u", available=True + ), + model.VantageRecord(name="self", description="d", available=False), + ] + + summary = built.render_summary() + + assert summary.splitlines()[0] == f"{model.HARNESS}: {model.NO_GO}" + assert "vantage operator: available identity=u" in summary + assert "vantage self: unavailable identity=unknown" in summary + assert "harness error: collision" in summary + assert "x.fail" in summary + assert "alpha" in summary + + +def test_utc_now_renders_a_zulu_timestamp_and_accepts_an_injected_clock() -> None: + fixed = dt.datetime(2026, 8, 17, 9, 30, 15, 123456, tzinfo=dt.timezone.utc) + assert model.utc_now(lambda: fixed) == "2026-08-17T09:30:15Z" + assert model.utc_now().endswith("Z") + + +def test_unscreened_fields_walks_the_whole_payload() -> None: + offenders = model.unscreened_fields( + { + "clean": "hermes", + "nested": {"leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234"}, + "items": [1, "Authorization: Bearer abcdefghijklmnop"], + } + ) + + assert offenders == ["$.nested.leak", "$.items[1]"] + assert model.unscreened_fields({"clean": "hermes"}) == [] + assert model.unscreened_fields(None) == [] diff --git a/testing/tests/test_hermes_handoff_mutation_gate.py b/testing/tests/test_hermes_handoff_mutation_gate.py new file mode 100644 index 00000000..1e84cfb8 --- /dev/null +++ b/testing/tests/test_hermes_handoff_mutation_gate.py @@ -0,0 +1,51 @@ +"""Tests for the deterministic Hermes handoff mutation gate.""" + +from __future__ import annotations + +from pathlib import Path + +from testing.quality_handoff_mutation import Mutation, MutationReport, main, run_gate + + +ROOT = Path(__file__).parents[2] + + +def test_every_security_mutant_is_killed() -> None: + report = run_gate(ROOT) + assert report.issues == () + assert report.score == 100.0 + assert report.killed == report.total >= 9 + + +def test_invalid_anchor_baseline_syntax_and_survivor_are_reported() -> None: + path = "hermes_handoff_model.py" + cases = ( + Mutation("missing", path, "not present", "x", "pass"), + Mutation( + "baseline", + path, + "SCHEMA_VERSION = 1", + "SCHEMA_VERSION = 2", + "raise SystemExit(1)", + ), + Mutation("syntax", path, "SCHEMA_VERSION = 1", "if", "pass"), + Mutation("survivor", path, "SCHEMA_VERSION = 1", "SCHEMA_VERSION = 2", "pass"), + ) + report = run_gate(ROOT, cases) + assert report.killed == 0 + assert len(report.issues) == 4 + assert report.score == 0.0 + + +def test_main_reports_a_perfect_score(capsys) -> None: + assert main() == 0 + assert "9/9 (100.00%)" in capsys.readouterr().out + + +def test_main_reports_mutation_issues(monkeypatch, capsys) -> None: + monkeypatch.setattr( + "testing.quality_handoff_mutation.run_gate", + lambda _root: MutationReport(1, 0, ("survivor",)), + ) + assert main() == 1 + assert "survivor" in capsys.readouterr().out diff --git a/testing/tests/test_hermes_handoff_policy.py b/testing/tests/test_hermes_handoff_policy.py new file mode 100644 index 00000000..90cef791 --- /dev/null +++ b/testing/tests/test_hermes_handoff_policy.py @@ -0,0 +1,319 @@ +"""Contracts for the fail-closed command policy of the handoff harness.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module + +policy = load_handoff_module("hermes_handoff_policy") + + +def check(*argv: str, mode: str | None = None, inner: bool = False) -> None: + policy.check_argv(argv, mode or policy.READ_ONLY, allow_impersonation=inner) + + +@pytest.mark.parametrize( + "argv", + [ + ("kubectl", "--namespace", "hermes", "get", "pods", "-o", "name"), + ("kubectl", "auth", "can-i", "get", "secrets", "--all-namespaces"), + ("kubectl", "get", "nodes", "-o", "name"), + ( + "kubectl", + "get", + "pods", + "-o", + policy.projection("jsonpath={.status.phase}"), + ), + ("flux", "get", "kustomizations", "--all-namespaces"), + ("helm", "list", "--all-namespaces"), + ("git", "merge-base", "--is-ancestor", "a", "b"), + ("git", "ls-remote", "origin", "refs/heads/main"), + ("hermes", "kanban", "list", "--json"), + ("hermes", "sessions", "list", "--source", "telegram"), + ("hermes", "status"), + (policy.GITEA_CLIENT, "GET", "/api/v1/user"), + ], +) +def test_read_only_commands_are_permitted(argv: tuple[str, ...]) -> None: + check(*argv) + + +@pytest.mark.parametrize( + "argv", + [ + (), + ("vault", "read", "kv/x"), + ("curl", "https://example.dev"), + ("rm", "-rf", "/"), + ("kubectl", "get", "pods", "--token", "abc"), + ("kubectl", "get", "pods"), + ("kubectl", "get", "pods", "--as=system:admin"), + ("kubectl", "get", "secrets", "-o", "name"), + ("kubectl", "get", "pods,secrets/x", "-o", "name"), + ("kubectl", "describe", "pod/x"), + ("kubectl", "config", "view", "--raw"), + ("kubectl", "auth", "reconcile", "-f", "x"), + ("kubectl", "delete", "pod/x", "--dry-run=client"), + ("kubectl", "delete", "pod/x", "--", "--dry-run=server"), + ("kubectl", "get", "pods", "-o=json"), + ("kubectl", "get", "pods", "-o", "jsonpath={.spec.containers[*].env[*].value}"), + ("kubectl", "port-forward", "svc/x", "80"), + ("kubectl", "exec", "pod"), + ("kubectl", "exec", "pod", "--"), + ("kubectl",), + ("flux", "reconcile", "kustomization", "x"), + ("flux",), + ("helm", "upgrade", "x"), + ("helm", "get", "values", "x"), + ("git", "commit", "-m", "x"), + ("git", "push", "origin", "HEAD:refs/heads/x"), + ("git", "fetch", "origin"), + ("git", "config", "core.sshCommand", "x"), + ("git", "-ccore.sshCommand=x", "ls-remote", "origin"), + ("git", "--git-dir=/tmp/x", "status"), + ("hermes", "kanban", "complete", "t_1"), + ("hermes",), + (policy.GITEA_CLIENT, "POST", "/api/v1/x"), + (policy.GITEA_CLIENT, "GET"), + ("sh", "-c", "rm -rf /"), + ("sh", "echo"), + ("git", "log", "/etc/shadow"), + ], +) +def test_unsafe_commands_are_refused(argv: tuple[str, ...]) -> None: + with pytest.raises(policy.PolicyError): + check(*argv) + + +def test_unknown_mode_is_refused() -> None: + with pytest.raises(policy.PolicyError, match="unknown mode"): + policy.check_argv(("kubectl", "get", "pods"), "whatever") + + +def test_dry_run_mutation_commands_are_never_constructible() -> None: + with pytest.raises(policy.PolicyError, match="dry-run"): + check("kubectl", "patch", "deploy/x", "--dry-run=server") + + +def test_a_resource_named_after_a_subcommand_cannot_smuggle_a_delete() -> None: + """`-n get` must not make a delete look like a read.""" + with pytest.raises(policy.PolicyError): + check("kubectl", "delete", "-n", "get", "configmap", "x") + + +def test_impersonation_is_operator_forbidden_and_in_pod_permitted() -> None: + with pytest.raises(policy.PolicyError, match="forbidden argument"): + check("kubectl", "get", "namespaces", "--as", "system:admin") + check( + "kubectl", + "get", + "namespaces", + "-o", + "name", + "--as", + "system:admin", + inner=True, + ) + + +def test_exec_validates_its_inner_command_and_tolerates_impersonation_inside() -> None: + script = policy.render_shell("env_names") + policy._RENDERED.add(script) + check( + "kubectl", "exec", "--namespace", "hermes", "pod", "--", "/bin/sh", "-c", script + ) + check( + "kubectl", + "exec", + "pod", + "--", + "/usr/local/bin/kubectl", + "get", + "ns", + "-o", + "name", + "--as", + "system:admin", + ) + with pytest.raises(policy.PolicyError): + check("kubectl", "exec", "pod", "--", "bash", "-c", "id") + + +def test_shell_templates_reject_credential_roots() -> None: + with pytest.raises(policy.PolicyError, match="credential paths"): + policy.shell("json_fields", path="/runtime-access/token", fields="state") + with pytest.raises(policy.PolicyError, match="credential-bearing"): + policy.shell("json_fields", path="/tmp/health.json", fields="state,token") + + +def test_armed_mode_opens_exactly_the_mutations_it_should() -> None: + ref = "ephemeral/hermes-handoff-acceptance/acceptance1" + policy.arm_ephemeral_policy(ref) + policy.register_ephemeral_pull(9) + check("git", "push", "origin", f"HEAD:refs/heads/{ref}", mode=policy.ARMED) + root = "/api/v1/repos/atlas/titan-iac" + check( + policy.GITEA_CLIENT, + "POST", + f"{root}/pulls", + "--field", + "title=WIP: Hermes handoff acceptance ephemeral probe", + "--field", + f"head={ref}", + "--field", + "base=main", + "--field", + "body=Ephemeral acceptance probe. Closed and deleted by the harness.", + mode=policy.ARMED, + ) + check( + policy.GITEA_CLIENT, + "PATCH", + f"{root}/pulls/9", + "--field", + "state=closed", + mode=policy.ARMED, + ) + check(policy.GITEA_CLIENT, "DELETE", f"{root}/branches/{ref}", mode=policy.ARMED) + with pytest.raises(policy.PolicyError, match="may not force"): + check( + "git", + "push", + "--force", + "origin", + f"HEAD:refs/heads/{ref}", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError, match="may not force"): + check("git", "push", "--all", "origin", mode=policy.ARMED) + with pytest.raises(policy.PolicyError, match="not permitted"): + check(policy.GITEA_CLIENT, "PUT", f"{root}/pulls/9", mode=policy.ARMED) + + +def test_arming_and_projection_registration_fail_closed() -> None: + with pytest.raises(policy.PolicyError, match="exact ephemeral ref"): + policy.arm_ephemeral_policy("main") + policy._ARMED_REF = "" + with pytest.raises(policy.PolicyError, match="registration is malformed"): + policy.register_ephemeral_pull(9) + policy.arm_ephemeral_policy("ephemeral/hermes-handoff-acceptance/acceptance1") + + for candidate in ( + "wide", + "jsonpath={.status.token}", + "jsonpath={.metadata.annotations}", + "jsonpath={.metadata.labels.other}", + "jsonpath={.spec.containers[*].env[*].value}", + ): + with pytest.raises(policy.PolicyError): + policy.projection(candidate) + + with pytest.raises(policy.PolicyError): + check("/usr/bin/awk", "x") + with pytest.raises(policy.PolicyError): + check("git", "--exec-path=/tmp", "status") + + +def test_every_frozen_template_renders_and_is_accepted() -> None: + parameters = { + "account": "hermes-agent", + "askpass": "/opt/coordinator/gitea_askpass.sh", + "fields": "state,model", + "limit": "200", + "path": "/host-etc/passwd", + "url": "https://scm.example.dev/atlas/repo.git", + } + for name in policy.SHELL_TEMPLATES: + required = policy.template_parameters(name) + argv = policy.shell(name, **{key: parameters[key] for key in required}) + policy.check_argv(argv) + assert argv[0] == "sh" + + +def test_template_rendering_rejects_unknown_names_and_unsafe_parameters() -> None: + with pytest.raises(policy.PolicyError, match="unknown shell template"): + policy.render_shell("nope") + with pytest.raises(policy.PolicyError, match="expects parameters"): + policy.render_shell("json_fields") + with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): + policy.render_shell("json_fields", path="/tmp;rm", fields="state") + with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): + policy.render_shell("json_fields", path="$(id)", fields="state") + + +def test_positionals_skip_flags_and_stop_at_the_boundary() -> None: + argv = ("kubectl", "--namespace", "hermes", "get", "pods", "--", "ignored") + assert list(policy.positionals(argv)) == ["get", "pods"] + assert list(policy.positionals(("kubectl", "--all-namespaces", "get"))) == ["get"] + + +def test_a_subcommand_must_precede_its_resource_arguments() -> None: + with pytest.raises(policy.PolicyError): + check("kubectl", "pods", "get") + + +@pytest.mark.parametrize( + "argv", + [ + ("/usr/bin/../../tmp/kubectl", "get", "pods"), + ("/tmp/reviewer/kubectl", "get", "pods"), + ("kubectl", "get", "pods", "-o", "go-template=x"), + ("kubectl", "get", "pods", "-o", "jsonpath={.status.token}"), + ("kubectl", "get", "pods", "-o", "jsonpath={.metadata.annotations}"), + ("kubectl", "get", "pods", "-o", "jsonpath={.spec.containers[*].env[*].value}"), + ("git", "-C", "/tmp", "status"), + ("git", "--work-tree=/tmp", "status"), + (policy.GITEA_CLIENT, "GET", "not-an-api-path"), + (policy.GITEA_CLIENT, "GET", "/api/v1/repos/atlas/other"), + ], +) +def test_provenance_projection_and_repository_bypasses_are_rejected(argv) -> None: + with pytest.raises(policy.PolicyError): + check(*argv) + + +def test_armed_writes_are_pinned_to_exact_remote_ref_and_endpoints() -> None: + ref = "ephemeral/hermes-handoff-acceptance/acceptance1" + policy.arm_ephemeral_policy(ref) + policy.register_ephemeral_pull(9) + with pytest.raises(policy.PolicyError, match="pinned to origin"): + check( + "git", + "push", + "upstream", + "HEAD:refs/heads/ephemeral/hermes-handoff-acceptance/acceptance1", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError, match="preflighted ephemeral ref"): + check("git", "push", "origin", "HEAD:refs/heads/other", mode=policy.ARMED) + with pytest.raises(policy.PolicyError): + check( + policy.GITEA_CLIENT, + "DELETE", + "/api/v1/repos/atlas/titan-iac", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError): + check( + policy.GITEA_CLIENT, + "PATCH", + "/api/v1/repos/atlas/titan-iac/pulls/8", + "--field", + "state=closed", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError): + check( + policy.GITEA_CLIENT, + "POST", + "/api/v1/repos/atlas/titan-iac/pulls", + "--field", + "title=WIP: wrong branch", + "--field", + "head=main", + "--field", + "base=main", + mode=policy.ARMED, + ) diff --git a/testing/tests/test_hermes_handoff_redaction.py b/testing/tests/test_hermes_handoff_redaction.py new file mode 100644 index 00000000..78fe19ec --- /dev/null +++ b/testing/tests/test_hermes_handoff_redaction.py @@ -0,0 +1,192 @@ +"""Contracts for the handoff harness credential screen and output bounding.""" + +from __future__ import annotations + +import json + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module + +redaction = load_handoff_module("hermes_handoff_redaction") + + +@pytest.mark.parametrize( + "text", + [ + "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234", + "Authorization: Bearer abcdefghijklmnop", + "cookie: session=abcdefghijklmnop", + "password=hunter2000secret", + "-----BEGIN OPENSSH PRIVATE KEY-----\nabc\n-----END OPENSSH PRIVATE KEY-----", + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI", + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0.SflKxwRJ", + "https://user:supersecretvalue@scm.example.dev/x", + "vault token hvs.CAESIJabcdefghijklmnop", + "aws AKIAIOSFODNN7EXAMPLE", + ], +) +def test_credential_shapes_are_screened(text: str) -> None: + assert redaction.contains_credential_shape(text) + assert redaction.REDACTED in redaction.redact(text) + + +@pytest.mark.parametrize( + "text", + [ + "sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7", + "ab346f55509d584e457fe26cf90be3078f7a375c", + '"automountServiceAccountToken": false', + '"key": "kubernetes.io/arch"', + "namespace: hermes", + "replicas: 3", + "checked_at: 2026-08-17T09:17:07.224400+00:00", + ], +) +def test_ordinary_evidence_survives_screening(text: str) -> None: + assert redaction.redact(text) == text + assert not redaction.contains_credential_shape(text) + + +def test_screened_json_is_still_parsable_json() -> None: + """A screen that corrupts structure would break every downstream evaluator.""" + payload = { + "spec": {"automountServiceAccountToken": True, "key": "kubernetes.io/arch"}, + "data": {"password": "a-long-enough-secret-value"}, + } + screened = redaction.redact(json.dumps(payload, indent=2)) + reparsed = json.loads(screened) + + assert reparsed["spec"]["automountServiceAccountToken"] is True + assert reparsed["spec"]["key"] == "kubernetes.io/arch" + assert reparsed["data"]["password"] == redaction.REDACTED + + +def test_high_entropy_runs_are_screened_but_hex_digests_are_not() -> None: + assert ( + redaction.redact("Zm9vYmFyYmF6cXV4-Quux_1234567890ABCDEFghij") + != redaction.REDACTED + ) + long_mixed = "aB3" + "x9Y2z_" * 10 + assert redaction.redact(long_mixed) == redaction.REDACTED + assert redaction.redact("f" * 64) == "f" * 64 + + +@pytest.mark.parametrize( + ("key", "sensitive"), + [ + ("ANTHROPIC_API_KEY", True), + ("gitea_token", True), + ("clientSecret", True), + ("Set-Cookie", True), + ("automountServiceAccountToken", True), + ("key", False), + ("namespace", False), + ("", False), + ], +) +def test_key_sensitivity_classification(key: str, sensitive: bool) -> None: + assert redaction.is_sensitive_key(key) is sensitive + + +@pytest.mark.parametrize( + ("value", "possible"), + [ + ("true", False), + ("False", False), + ("3", False), + ("12.5", False), + ("short", False), + ("", False), + ("a-long-enough-secret", True), + ('"another-long-secret"', True), + ("1234567890123", False), + ("-12.5e9", False), + ], +) +def test_value_shape_gates_screening(value: str, possible: bool) -> None: + assert redaction.could_hold_secret(value) is possible + + +def test_bound_truncates_to_a_byte_budget_and_reports_it() -> None: + text, truncated = redaction.bound("x" * 100, max_bytes=10) + assert truncated + assert len(text.encode()) <= 10 + assert text == redaction.TRUNCATION_NOTE[:10] + + text, truncated = redaction.bound("short", max_bytes=64) + assert (text, truncated) == ("short", False) + + +def test_bound_with_no_budget_keeps_nothing() -> None: + assert redaction.bound("content", max_bytes=0) == ("", True) + assert redaction.bound("", max_bytes=0) == ("", False) + + +def test_bound_never_splits_a_multibyte_character() -> None: + text, truncated = redaction.bound("é" * 10, max_bytes=5) + assert truncated + assert len(text.encode()) <= 5 + + +def test_safe_text_screens_before_it_bounds() -> None: + """A boundary must never preserve a useful credential prefix.""" + payload = "password=" + "a" * 200 + text, truncated = redaction.safe_text(payload, max_bytes=20) + assert not truncated + assert redaction.REDACTED in text + + +def test_no_byte_budget_exposes_a_credential_prefix() -> None: + payload = "password=A9secret-prefix-that-must-never-survive" + for budget in range(1, 64): + screened, _ = redaction.safe_text(payload, max_bytes=budget) + assert "A9" not in screened + assert len(screened.encode()) <= budget + + +def test_scrub_walks_nested_structures_and_keeps_useful_types() -> None: + scrubbed = redaction.scrub( + { + "token": "a-long-enough-secret", + "automountServiceAccountToken": False, + "items": [{"password": "another-long-secret"}, 3, None], + "note": ("tuple", "values"), + } + ) + + assert scrubbed["token"] == redaction.REDACTED + assert scrubbed["automountServiceAccountToken"] is False + assert scrubbed["items"][0]["password"] == redaction.REDACTED + assert scrubbed["items"][1] == 3 + assert scrubbed["items"][2] is None + assert scrubbed["note"] == ["tuple", "values"] + + +def test_scrub_screens_short_numeric_nested_values_and_mapping_keys() -> None: + scrubbed = redaction.scrub( + { + "token": 1234, + "credentials": {"x": "y"}, + "ghp_ABCDEF1234567890": "mapping-key", + "private_key": "x", + } + ) + assert scrubbed["token"] == redaction.REDACTED + assert scrubbed["credentials"] == redaction.REDACTED + assert scrubbed["private_key"] == redaction.REDACTED + assert not any("ghp_" in key for key in scrubbed) + + +def test_unterminated_private_key_blocks_are_screened() -> None: + assert "PRIVATE KEY" not in redaction.redact("-----BEGIN PRIVATE KEY-----\nabc") + + +def test_screening_is_idempotent() -> None: + once = redaction.redact("api_key=AKIAIOSFODNN7EXAMPLE") + assert redaction.redact(once) == once + assert redaction.redact("") == "" + + +def test_normalise_key_strips_separators() -> None: + assert redaction.normalise_key("Client-Secret_v2") == "clientsecretv2" diff --git a/testing/tests/test_hermes_handoff_rules.py b/testing/tests/test_hermes_handoff_rules.py new file mode 100644 index 00000000..03d5785f --- /dev/null +++ b/testing/tests/test_hermes_handoff_rules.py @@ -0,0 +1,166 @@ +"""Contracts for the shared evaluator primitives and dispatch.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) + +rules = load_handoff_module("hermes_handoff_rules") +model = load_handoff_module("hermes_handoff_model") +load_handoff_module("hermes_handoff_evaluators") + + +PAYLOAD = { + "a": {"b": [{"c": 1}, {"c": 2}]}, + "items": [{"number": 14, "merged": True}, {"number": 15, "merged": False}], + "flag": True, +} + + +@pytest.mark.parametrize( + ("path", "expected"), + [ + ("", PAYLOAD), + ("flag", True), + ("a.b[0].c", 1), + ("a.b[].c", [1, 2]), + ("a.b[]", [{"c": 1}, {"c": 2}]), + ("items[].number", [14, 15]), + ], +) +def test_dotted_paths_index_and_map(path: str, expected) -> None: + assert rules.dotted(PAYLOAD, path) == expected + + +@pytest.mark.parametrize( + "path", ["missing", "a.missing", "a.b[9].c", "flag[0]", "flag.b"] +) +def test_an_absent_path_raises_rather_than_returning_none(path: str) -> None: + with pytest.raises(KeyError): + rules.dotted(PAYLOAD, path) + + +def test_parse_json_rejects_raw_control_characters_and_empty_output() -> None: + """Only RFC-compliant, nonempty JSON is accepted as release evidence.""" + with pytest.raises(ValueError): + rules.parse_json(outcome(stdout='{"title": "a\nb"}')) + with pytest.raises(ValueError, match="empty output"): + rules.parse_json(outcome(stdout=" ")) + with pytest.raises(ValueError): + rules.parse_json(outcome(stdout="not json")) + + +def test_lines_drops_blank_rows_and_trims() -> None: + assert rules.lines(outcome(stdout=" a \n\n b\n")) == ["a", "b"] + + +def test_missing_steps_reports_the_first_step_that_produced_nothing() -> None: + built = spec("allowed", steps=(step("a"), step("b"))) + + never_run = rules.missing_steps(built, {}) + assert never_run.status == model.NOT_RUN + assert "never executed" in never_run.reason + + errored = rules.missing_steps( + built, {"a": outcome(error="timeout"), "b": outcome()} + ) + assert errored.status == model.NOT_RUN + assert "did not run" in errored.reason + + truncated = rules.missing_steps( + built, {"a": outcome(truncated=True), "b": outcome()} + ) + assert truncated.status == model.NOT_RUN and "truncated" in truncated.reason + + assert rules.missing_steps(built, {"a": outcome(), "b": outcome()}) is None + + +def test_an_optional_step_may_be_absent() -> None: + built = spec("allowed", steps=(step("a"), step("b", optional=True))) + assert rules.missing_steps(built, {"a": outcome()}) is None + + +@pytest.mark.parametrize( + ("stderr", "returncode", "denied"), + [ + ("Error from server (Forbidden): pods is forbidden", 1, True), + ("User cannot list resource", 1, True), + ("Gitea API returned HTTP 403", 1, True), + ("connection refused", 1, False), + ("", 0, False), + ], +) +def test_denial_detection_needs_a_refusal_not_merely_a_failure( + stderr, returncode, denied +) -> None: + assert rules.looks_denied(outcome(stderr=stderr, returncode=returncode)) is denied + + +def test_step_keys_selects_by_kind() -> None: + built = spec( + "denied", + steps=(step("r", kind=model.REVIEW), step("a", kind=model.ATTEMPT), step("x")), + ) + assert rules.step_keys(built, model.REVIEW) == ("r",) + assert rules.step_keys(built, model.ATTEMPT) == ("a",) + assert rules.step_keys(built, model.READ) == ("x",) + + +def test_parsed_step_returns_the_payload_or_the_skip_to_report() -> None: + built = spec("json_field", {"step": "s"}, steps=(step("s"),)) + + payload, key = rules.parsed_step(built, {"s": outcome(stdout='{"a": 1}')}) + assert (payload, key) == ({"a": 1}, "s") + + failed = rules.parsed_step(built, {"s": outcome(returncode=1)}) + assert failed.status == model.NOT_RUN and "failed" in failed.reason + + unparsable = rules.parsed_step(built, {"s": outcome(stdout="nope")}) + assert ( + unparsable.status == model.NOT_RUN + and "did not return JSON" in unparsable.reason + ) + + +def test_an_unknown_rule_is_not_run_rather_than_a_crash() -> None: + evaluation = rules.evaluate(spec("no-such-rule"), {}) + assert evaluation.status == model.NOT_RUN + assert "unknown evaluator" in evaluation.reason + + +def test_an_evaluator_that_raises_is_not_run_rather_than_a_pass() -> None: + """A harness bug must never be indistinguishable from a clean release.""" + + @rules.evaluator("explodes-for-test") + def _explode(_spec, _outcomes): + raise RuntimeError("boom") + + evaluation = rules.evaluate(spec("explodes-for-test"), {}) + + assert evaluation.status == model.NOT_RUN + assert "RuntimeError" in evaluation.reason and "boom" in evaluation.reason + del rules.EVALUATORS["explodes-for-test"] + + +def test_an_evaluator_returning_an_unknown_status_is_not_run() -> None: + @rules.evaluator("unknown-status-for-test") + def _unknown(_spec, _outcomes): + return rules.Evaluation("MAYBE", "bad") + + evaluation = rules.evaluate(spec("unknown-status-for-test"), {}) + assert evaluation.status == model.NOT_RUN + assert "unknown status" in evaluation.reason + del rules.EVALUATORS["unknown-status-for-test"] + + +def test_is_list_distinguishes_arrays_from_strings() -> None: + assert rules.is_list([1]) and rules.is_list(()) + assert not rules.is_list("text") + assert not rules.is_list(b"bytes") + assert not rules.is_list({"a": 1}) diff --git a/testing/tests/test_hermes_handoff_run.py b/testing/tests/test_hermes_handoff_run.py new file mode 100644 index 00000000..881ffd55 --- /dev/null +++ b/testing/tests/test_hermes_handoff_run.py @@ -0,0 +1,471 @@ +"""Contracts for handoff acceptance orchestration: vantages, catalog, report.""" + +from __future__ import annotations + +import datetime as dt +import subprocess +from dataclasses import replace + +from testing.tests.test_hermes_handoff_support import ( + FakeClock, + FakeSpawn, + load_handoff_module, + step, +) + +harness_run = load_handoff_module("hermes_handoff_run") +catalog = load_handoff_module("hermes_handoff_catalog") +exec_module = load_handoff_module("hermes_handoff_exec") +model = load_handoff_module("hermes_handoff_model") + +NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc) +TARGETS = catalog.Targets(now=NOW) + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) + + +def whoami(username: str) -> object: + return completed(stdout=username) + + +def runner_for(results: list, **kwargs): + return exec_module.Runner( + clock=FakeClock(), + spawn=FakeSpawn(results), + environ={}, + deadline_seconds=3600, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + **kwargs, + ) + + +def check( + rule: str, + expect: dict | None = None, + steps=(), + identifier="test.check", + mandatory=True, +): + return model.CheckSpec( + id=identifier, + title="t", + group="g", + rule=rule, + steps=tuple(steps), + expect=expect or {}, + mandatory=mandatory, + ) + + +def test_the_shipped_catalog_is_valid_and_covers_every_group() -> None: + specs = harness_run.build_catalog(TARGETS) + assert harness_run.validate_catalog(specs) == [] + groups = {spec.group for spec in specs} + assert { + "baseline", + "identity", + "routing", + "scopes", + "access-denied", + "gitops", + } <= groups + assert { + "forge", + "nodes", + "build", + "reliability", + "pool", + "surfaces", + "ephemeral", + } <= groups + + +def test_catalog_validation_names_every_structural_problem() -> None: + broken = [ + check("no-such-rule", identifier="a.dup"), + check("allowed", identifier="a.dup"), + check("allowed", steps=(step("s", vantage="nowhere"),), identifier="b.vantage"), + check("allowed", steps=(step("s", kind="guess"),), identifier="c.kind"), + check( + "denied", steps=(step("s", kind=model.REVIEW),), identifier="d.review-only" + ), + model.CheckSpec( + id="e.impersonation", + title="t", + group="g", + rule="allowed", + steps=( + model.Step( + key="s", + vantage=catalog.OPERATOR, + argv=("kubectl", "get", "ns", "--as", "x"), + ), + ), + ), + ] + + problems = " | ".join(harness_run.validate_catalog(broken)) + + assert "duplicate check id" in problems + assert "unknown rule" in problems + assert "unknown vantage" in problems + assert "unknown kind" in problems + assert "needs a real attempt" in problems + assert "impersonates from the operator vantage" in problems + + +def test_target_validation_rejects_every_vacuous_or_unbound_release_input() -> None: + broken = replace( + TARGETS, + repo="atlas/other", + remote="upstream", + reviewed_pr_number=20, + reviewed_head_ref="other", + baseline_commit="bad", + remote_main_sha="bad", + reviewed_head_sha="bad", + build_sha="bad", + agent_image="latest", + deployment_revision="0", + node_count=1, + pool_replicas=1, + expect_telegram_sessions=False, + chat_config_revision="", + pool_worker_env=(), + dependency_heads=((14, "bad"), (14, "bad")), + max_evidence_age_seconds=0, + ) + problems = harness_run.validate_targets(broken) + assert len(problems) >= 12 + sha = "1" * 40 + valid = replace( + TARGETS, + remote_main_sha=sha, + reviewed_head_sha=sha, + build_sha=sha, + agent_image=f"registry.example/hermes:git-{sha}-build-7@sha256:{'2' * 64}", + deployment_revision="1", + chat_config_revision="rev", + dependency_heads=tuple( + (number, sha) for number in TARGETS.dependency_pull_requests + ), + ) + assert harness_run.validate_targets(valid) == [] + wrong_image_source = replace( + valid, + agent_image=f"registry.example/hermes:git-{'2' * 40}-build-7@sha256:{'2' * 64}", + ) + assert "image tag/digest" in " ".join( + harness_run.validate_targets(wrong_image_source) + ) + wrong_build = replace(valid, build_sha="2" * 40) + assert "build SHA" in " ".join(harness_run.validate_targets(wrong_build)) + malformed_heads = replace( + valid, + dependency_heads=tuple( + (number, "bad") for number in valid.dependency_pull_requests + ), + ) + assert "dependency heads" in " ".join(harness_run.validate_targets(malformed_heads)) + + +def test_identity_resolution_extracts_only_the_username() -> None: + runner = runner_for([whoami("system:serviceaccount:hermes:hermes-agent")]) + username, detail = harness_run.resolve_identity( + runner, exec_module.operator_vantage() + ) + assert (username, detail) == ("system:serviceaccount:hermes:hermes-agent", "") + + failed = harness_run.resolve_identity( + runner_for([completed(stderr="no access", returncode=1)]), + exec_module.operator_vantage(), + ) + assert failed[0] == "" and "no access" in failed[1] + + unparsable = harness_run.resolve_identity( + runner_for([completed(stdout="two names")]), exec_module.operator_vantage() + ) + assert unparsable[0] == "" and "no username" in unparsable[1] + + nameless = harness_run.resolve_identity( + runner_for([completed(stdout="")]), exec_module.operator_vantage() + ) + assert nameless[0] == "" and "no username" in nameless[1] + + +def test_vantages_resolve_to_running_pods_and_record_what_is_missing() -> None: + runner = runner_for( + [ + whoami("kubernetes-admin"), + completed(stdout="pod/hermes-agent-2\npod/hermes-agent-1\n"), + whoami("system:serviceaccount:hermes:hermes-agent"), + completed(stdout="pod/hermes-switchyard-1\n"), + completed(stderr="no pods", returncode=1), + completed(stdout="pod/hermes-chat-tenant-0\n"), + ] + ) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + by_name = {record.name: record for record in records} + + assert set(vantages) == { + catalog.OPERATOR, + catalog.SELF, + catalog.SWITCHYARD, + catalog.CHAT, + } + assert by_name[catalog.SELF].identity == "system:serviceaccount:hermes:hermes-agent" + assert "hermes-agent-1" in by_name[catalog.SELF].description # deterministic pick + assert by_name[catalog.NODE].available is False + assert by_name[catalog.CHAT].available is True + + +def test_an_unreachable_operator_stops_vantage_resolution_immediately() -> None: + runner = runner_for([completed(stderr="connection refused", returncode=1)]) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + + assert set(vantages) == {catalog.OPERATOR} + assert records[0].available is False + assert harness_run.vantage_problems(records)[0].startswith( + "the operator vantage is unavailable" + ) + + +def test_a_missing_chat_pod_is_recorded_rather_than_assumed_present() -> None: + runner = runner_for( + [ + whoami("kubernetes-admin"), + completed(stdout="pod/hermes-agent-1\n"), + whoami("system:serviceaccount:hermes:hermes-agent"), + completed(stdout="pod/hermes-switchyard-1\n"), + completed(stdout="pod/hermes-node-1\n"), + completed(stderr="pods 'hermes-chat-tenant-0' not found", returncode=1), + ] + ) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + chat = next(record for record in records if record.name == catalog.CHAT) + + assert catalog.CHAT not in vantages + assert chat.available is False and "not found" in chat.detail + + +def test_a_selector_that_matches_no_running_pod_leaves_the_vantage_absent() -> None: + runner = runner_for( + [ + whoami("kubernetes-admin"), + completed(stdout="\n"), + completed(stdout="\n"), + completed(stdout="\n"), + completed(stdout="\n"), + ] + ) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + by_name = {record.name: record for record in records} + + assert set(vantages) == {catalog.OPERATOR} + assert "no running pod matches" in by_name[catalog.SELF].detail + + +def test_two_vantages_that_are_the_same_principal_invalidate_the_evidence() -> None: + """Corroboration needs two identities; one identity twice is a restatement.""" + records = [ + model.VantageRecord( + name=catalog.OPERATOR, description="d", identity="same", available=True + ), + model.VantageRecord( + name=catalog.SELF, description="d", identity="same", available=True + ), + model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True), + model.VantageRecord(name=catalog.CHAT, description="d", available=True), + ] + + problems = harness_run.vantage_problems(records) + + assert len(problems) == 1 + assert "same principal" in problems[0] + + +def test_vantage_problems_flag_a_missing_or_anonymous_self_probe() -> None: + operator = model.VantageRecord( + name=catalog.OPERATOR, description="d", identity="admin", available=True + ) + + absent = harness_run.vantage_problems([operator]) + assert "in-pod Hermes vantage is unavailable" in absent[0] + + anonymous = harness_run.vantage_problems( + [ + operator, + model.VantageRecord(name=catalog.SELF, description="d", available=True), + ] + ) + assert "did not report an identity" in anonymous[0] + + clean = harness_run.vantage_problems( + [ + operator, + model.VantageRecord( + name=catalog.SELF, description="d", identity="agent", available=True + ), + model.VantageRecord( + name=catalog.SWITCHYARD, description="d", available=True + ), + model.VantageRecord(name=catalog.CHAT, description="d", available=True), + ] + ) + assert clean == [] + + +def test_a_check_whose_vantage_is_absent_is_not_run() -> None: + spec = check("allowed", steps=(step("s", vantage=catalog.NODE),)) + + result = harness_run.run_check(runner_for([]), spec, {}) + + assert result.status == model.NOT_RUN + assert "vantage node is unavailable" in result.reason + assert result.outcomes[0].error == "vantage node is unavailable" + + +def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None: + spec = check( + "stdout_matches", + {"step": "s", "equals": "locked"}, + steps=( + model.Step( + key="s", + vantage=catalog.OPERATOR, + argv=("kubectl", "version"), + record=False, + ), + ), + ) + runner = runner_for([completed(stdout="locked", stderr="noise")]) + + result = harness_run.run_check( + runner, spec, {catalog.OPERATOR: exec_module.operator_vantage()} + ) + + assert result.status == model.PASS + assert result.outcomes[0].stdout == harness_run.UNRECORDED + assert result.outcomes[0].stderr == harness_run.UNRECORDED + + +def test_an_expired_deadline_leaves_the_report_saying_so() -> None: + clock = FakeClock() + runner = exec_module.Runner( + clock=clock, + spawn=FakeSpawn([]), + environ={}, + deadline_seconds=1.0, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + ) + clock.advance(5.0) + spec = check("allowed", steps=(step("s", vantage=catalog.OPERATOR),)) + + report = harness_run.build_report( + runner, + TARGETS, + [spec], + {catalog.OPERATOR: exec_module.operator_vantage()}, + [], + "read-only", + "2026-08-17T00:00:00Z", + ) + + assert report.results[0].status == model.NOT_RUN + assert any("deadline expired" in error for error in report.harness_errors) + assert report.decision == model.NO_GO + + +def test_the_report_carries_the_configured_baseline_and_sorts_its_checks() -> None: + runner = runner_for([completed(stdout="ok"), completed(stdout="ok")]) + specs = [ + check( + "allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="z.last" + ), + check( + "allowed", + steps=(step("s", vantage=catalog.OPERATOR),), + identifier="a.first", + ), + ] + records = [ + model.VantageRecord( + name=catalog.OPERATOR, description="d", identity="admin", available=True + ), + model.VantageRecord( + name=catalog.SELF, description="d", identity="agent", available=True + ), + model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True), + model.VantageRecord(name=catalog.CHAT, description="d", available=True), + ] + + report = harness_run.build_report( + runner, + TARGETS, + specs, + {catalog.OPERATOR: exec_module.operator_vantage()}, + records, + "read-only", + "2026-08-17T00:00:00Z", + ) + + assert [result.spec.id for result in report.results] == ["a.first", "z.last"] + assert report.baseline["baseline_commit"] == TARGETS.baseline_commit + assert report.baseline["dependency_pull_requests"] == list( + TARGETS.dependency_pull_requests + ) + assert report.decision == model.GO + assert report.finished_at + + +def test_armed_results_replace_the_matching_catalog_placeholders() -> None: + placeholder = check( + "not_armed", identifier="ephemeral.feature-branch-push", mandatory=False + ) + armed = model.CheckResult( + spec=model.CheckSpec( + id="ephemeral.feature-branch-push", + title="t", + group="ephemeral", + rule="armed", + ), + status=model.PASS, + reason="pushed", + ) + + report = harness_run.build_report( + runner_for([]), + TARGETS, + [placeholder], + {}, + [], + "ephemeral-armed", + "2026-08-17T00:00:00Z", + [armed], + ) + + assert len(report.results) == 1 + assert report.results[0].status == model.PASS + + +def test_an_unstartable_run_reports_no_go_with_a_named_reason() -> None: + report = harness_run.unavailable_report( + "read-only", "2026-08-17T00:00:00Z", "arming refused: bad phrase", TARGETS + ) + + assert report.decision == model.NO_GO + assert report.results[0].spec.id == "harness.startup" + assert report.results[0].status == model.NOT_RUN + assert report.harness_errors == ["arming refused: bad phrase"] diff --git a/testing/tests/test_hermes_handoff_support.py b/testing/tests/test_hermes_handoff_support.py new file mode 100644 index 00000000..f62ead15 --- /dev/null +++ b/testing/tests/test_hermes_handoff_support.py @@ -0,0 +1,114 @@ +"""Shared loader and fixtures for the handoff acceptance harness tests. + +The harness lives in ``scripts/ops`` as sibling modules rather than an installed +package, so tests import it the way the entry point does: by putting that +directory on the path once and importing by name. +""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +OPS = REPO_ROOT / "scripts" / "ops" + +if str(OPS) not in sys.path: + sys.path.insert(0, str(OPS)) + + +def load_handoff_module(name: str) -> Any: + """Import one harness module by name.""" + return importlib.import_module(name) + + +exec_module = load_handoff_module("hermes_handoff_exec") +model = load_handoff_module("hermes_handoff_model") + + +def outcome( + stdout: str = "", + stderr: str = "", + returncode: int | None = 0, + error: str | None = None, + truncated: bool = False, + vantage: str = "self", + argv: tuple[str, ...] = ("kubectl", "get", "pods"), +) -> Any: + """Return a synthetic command outcome for evaluator tests.""" + return exec_module.Outcome( + argv=argv, + vantage=vantage, + returncode=returncode, + stdout=stdout, + stderr=stderr, + error=error, + truncated=truncated, + ) + + +def spec( + rule: str, + expect: dict | None = None, + steps: tuple = (), + identifier: str = "test.check", + mandatory: bool = True, + scope: str | None = None, +) -> Any: + """Return a synthetic check specification.""" + return model.CheckSpec( + id=identifier, + title="test check", + group="test", + rule=rule, + steps=steps, + expect=expect or {}, + mandatory=mandatory, + scope=scope or model.ALWAYS, + ) + + +def step( + key: str, kind: str | None = None, vantage: str = "self", optional: bool = False +) -> Any: + """Return a synthetic step.""" + return model.Step( + key=key, + vantage=vantage, + argv=("kubectl", "get", "pods", "-o", "name"), + kind=kind or model.READ, + optional=optional, + ) + + +class FakeClock: + """A monotonic clock the tests advance by hand.""" + + def __init__(self, start: float = 0.0) -> None: + self.now = start + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + """Move the clock forward.""" + self.now += seconds + + +class FakeSpawn: + """A ``subprocess.run`` stand-in driven by a queue of scripted results.""" + + def __init__(self, results: list) -> None: + self.results = list(results) + self.calls: list[dict] = [] + + def __call__(self, argv, **kwargs): + self.calls.append({"argv": list(argv), **kwargs}) + if not self.results: + raise AssertionError(f"unexpected command: {argv}") + outcome_or_error = self.results.pop(0) + if isinstance(outcome_or_error, Exception): + raise outcome_or_error + return outcome_or_error diff --git a/testing/tests/test_quality_contract.py b/testing/tests/test_quality_contract.py index 6e260a1b..fecae836 100644 --- a/testing/tests/test_quality_contract.py +++ b/testing/tests/test_quality_contract.py @@ -2,13 +2,17 @@ from __future__ import annotations +import hashlib from pathlib import Path import textwrap from testing.quality_contract import load_contract from testing.quality_coverage import run_check as run_coverage_check from testing.quality_docs import run_check as run_docs_check -from testing.quality_hygiene import run_check as run_hygiene_check +from testing.quality_hygiene import ( + count_files_over_line_limit, + run_check as run_hygiene_check, +) def test_bundled_contract_exposes_local_and_jenkins_profiles(): @@ -18,6 +22,21 @@ def test_bundled_contract_exposes_local_and_jenkins_profiles(): assert contract["pytest_suites"]["unit"]["paths"] +def test_handoff_modules_are_exactly_managed_linted_and_covered(): + """The canonical contract must not silently drop a handoff module.""" + contract = load_contract() + expected = { + path.as_posix() for path in Path("scripts/ops").glob("hermes_handoff_*.py") + } + assert len(expected) == 15 + assert expected <= set(contract["managed_modules"]) + assert expected <= set(contract["lint_paths"]) + assert expected <= set(contract["coverage"]["tracked_files"]) + assert expected <= set(contract["coverage"]["branch_tracked_files"]) + assert "scripts/ops" in contract["pytest_suites"]["unit"]["coverage_sources"] + assert "scripts/ops/hermes_handoff_*.py" in contract["hygiene"]["line_limit_globs"] + + def test_docs_check_reports_missing_docstring_and_missing_path(tmp_path: Path): module_path = tmp_path / "managed.py" module_path.write_text("value = 1\n", encoding="utf-8") @@ -43,7 +62,10 @@ def test_docs_check_reports_missing_required_doc_metadata(tmp_path: Path): (tmp_path / "README.md").write_text("", encoding="utf-8") contract = { - "required_docs": [{"path": "README.md", "description": ""}, {"path": "missing.md", "description": "Missing"}], + "required_docs": [ + {"path": "README.md", "description": ""}, + {"path": "missing.md", "description": "Missing"}, + ], "managed_modules": [], "lint_paths": [], "pytest_suites": {"unit": {"description": "", "paths": []}}, @@ -62,6 +84,7 @@ def test_docs_check_reports_missing_required_doc_metadata(tmp_path: Path): def test_hygiene_check_enforces_line_limit_and_name_rules(tmp_path: Path): tests_dir = tmp_path / "tests" tests_dir.mkdir() + (tests_dir / "conftest.py").write_text("value = 1\n", encoding="utf-8") bad_name = tests_dir / "bad-name.py" bad_name.write_text("x = 1\n", encoding="utf-8") long_file = tests_dir / "test_too_long.py" @@ -84,7 +107,34 @@ def test_hygiene_check_enforces_line_limit_and_name_rules(tmp_path: Path): issues = run_hygiene_check(contract, tmp_path) assert any("file exceeds 3 LOC" in issue for issue in issues) - assert any("naming rule failed" in issue and "bad-name.py" in issue for issue in issues) + assert any( + "naming rule failed" in issue and "bad-name.py" in issue for issue in issues + ) + + +def test_hygiene_legacy_exception_is_exact_and_invalidated_by_any_edit(tmp_path: Path): + legacy = tmp_path / "legacy.py" + legacy.write_text("line\n" * 4, encoding="utf-8") + contract = { + "hygiene": { + "max_lines": 3, + "line_limit_globs": ["*.py"], + "legacy_line_exceptions": { + "legacy.py": { + "lines": 4, + "sha256": hashlib.sha256(legacy.read_bytes()).hexdigest(), + } + }, + } + } + assert run_hygiene_check(contract, tmp_path) == [] + assert count_files_over_line_limit(contract, tmp_path) == 0 + + legacy.write_text("edit\n" * 4, encoding="utf-8") + assert run_hygiene_check(contract, tmp_path) == [ + "file exceeds 3 LOC: legacy.py (4)" + ] + assert count_files_over_line_limit(contract, tmp_path) == 1 def test_coverage_check_enforces_per_file_floor(tmp_path: Path): @@ -124,9 +174,9 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path): def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Path): missing_xml = tmp_path / "missing.xml" - assert run_coverage_check({"coverage": {"tracked_files": []}}, tmp_path, missing_xml) == [ - "coverage xml missing: missing.xml" - ] + assert run_coverage_check( + {"coverage": {"tracked_files": []}}, tmp_path, missing_xml + ) == ["coverage xml missing: missing.xml"] source_dir = tmp_path / "pkg" source_dir.mkdir() @@ -143,7 +193,7 @@ def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Pa - + diff --git a/testing/tests/test_quality_coverage_helpers.py b/testing/tests/test_quality_coverage_helpers.py index fc22a8ab..bc5691c3 100644 --- a/testing/tests/test_quality_coverage_helpers.py +++ b/testing/tests/test_quality_coverage_helpers.py @@ -83,58 +83,15 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match( assert issues == ["coverage below 95.0%: relative.py (80.0%)"] -def test_run_check_enforces_branch_floor_and_requires_branch_data( - tmp_path: Path, -) -> None: - """Tracked modules must report and meet the configured branch floor.""" - - coverage_xml = tmp_path / "coverage.xml" - coverage_xml.write_text( - textwrap.dedent( - """\ - - - - - - - - - - - """ - ), - encoding="utf-8", - ) - - issues = run_check( - { - "coverage": { - "minimum_percent": 95.0, - "minimum_branch_percent": 95.0, - "tracked_files": ["low.py", "missing.py"], - } - }, - tmp_path, - coverage_xml, - ) - - assert issues == [ - "branch coverage below 95.0%: low.py (90.0%)", - "branch coverage missing for tracked file: missing.py", - ] - - -def test_run_check_limits_branch_floor_to_explicit_branch_tracked_files( - tmp_path: Path, -) -> None: - """A branch rollout can cover selected security modules without hiding lines.""" - +def test_run_check_enforces_explicit_branch_floor(tmp_path: Path) -> None: + """Branch-tracked modules must report and meet the configured floor.""" coverage_xml = tmp_path / "coverage.xml" coverage_xml.write_text( """ - - + + + + """, encoding="utf-8", ) @@ -143,11 +100,40 @@ def test_run_check_limits_branch_floor_to_explicit_branch_tracked_files( "coverage": { "minimum_percent": 95, "minimum_branch_percent": 95, - "tracked_files": ["new.py", "legacy.py"], - "branch_tracked_files": ["new.py"], + "tracked_files": ["low.py", "missing.py", "legacy.py"], + "branch_tracked_files": ["low.py", "missing.py"], } }, tmp_path, coverage_xml, ) - assert issues == [] + assert issues == [ + "branch coverage below 95.0%: low.py (90.0%)", + "branch coverage missing for tracked file: missing.py", + ] + passing = run_check( + { + "coverage": { + "minimum_percent": 95, + "minimum_branch_percent": 95, + "tracked_files": ["passing.py"], + "branch_tracked_files": ["passing.py"], + } + }, + tmp_path, + coverage_xml, + ) + assert passing == [] + + +def test_workspace_coverage_returns_zero_when_xml_has_no_tracked_samples( + tmp_path: Path, +) -> None: + xml = tmp_path / "coverage.xml" + xml.write_text("", encoding="utf-8") + assert ( + compute_workspace_line_coverage( + {"coverage": {"tracked_files": ["missing.py"]}}, tmp_path, xml + ) + == 0.0 + ) diff --git a/testing/tests/test_semgrep_report.py b/testing/tests/test_semgrep_report.py index e0cf1735..ef59abd0 100644 --- a/testing/tests/test_semgrep_report.py +++ b/testing/tests/test_semgrep_report.py @@ -100,6 +100,14 @@ def test_build_sonar_issues_uses_safe_defaults_for_sparse_findings() -> None: assert issue["primaryLocation"]["textRange"] == {"startLine": 1, "endLine": 1} +def test_sonar_issue_never_ends_before_its_start_line() -> None: + issue = semgrep_report._sonar_issue_from_finding( + {"path": "app.py", "start": {"line": 10}, "end": {"line": 2}} + ) + assert issue is not None + assert issue["primaryLocation"]["textRange"] == {"startLine": 10, "endLine": 10} + + def test_read_json_handles_invalid_json_and_non_object(tmp_path: Path) -> None: """Report loading should fail closed for bad JSON and wrong top-level shapes.""" @@ -108,7 +116,9 @@ def test_read_json_handles_invalid_json_and_non_object(tmp_path: Path) -> None: list_json = tmp_path / "list.json" list_json.write_text("[]", encoding="utf-8") - assert semgrep_report._read_json(bad_json)["errors"][0]["message"].startswith("invalid JSON:") + assert semgrep_report._read_json(bad_json)["errors"][0]["message"].startswith( + "invalid JSON:" + ) assert semgrep_report._read_json(list_json) == { "errors": [{"message": "report payload is not an object"}], "results": [], @@ -137,16 +147,25 @@ def test_main_handles_missing_or_invalid_input(tmp_path: Path) -> None: payload = json.loads(output.read_text(encoding="utf-8")) assert payload["status"] == "failed" assert payload["errors_total"] == 1 - sonar_payload = json.loads((tmp_path / "build" / "sonar-issues.json").read_text(encoding="utf-8")) + sonar_payload = json.loads( + (tmp_path / "build" / "sonar-issues.json").read_text(encoding="utf-8") + ) assert sonar_payload == {"issues": []} -def test_main_honors_custom_blocking_severity_without_sonar_output(tmp_path: Path) -> None: +def test_main_honors_custom_blocking_severity_without_sonar_output( + tmp_path: Path, +) -> None: """CLI callers can choose stricter blocking severities without writing Sonar issues.""" semgrep_json = tmp_path / "semgrep.json" semgrep_json.write_text( - json.dumps({"results": [{"path": "app.py", "extra": {"severity": "WARNING"}}], "errors": []}), + json.dumps( + { + "results": [{"path": "app.py", "extra": {"severity": "WARNING"}}], + "errors": [], + } + ), encoding="utf-8", ) output = tmp_path / "semgrep-report.json"