diff --git a/dockerfiles/Dockerfile.hermes-webui b/dockerfiles/Dockerfile.hermes-webui index a9f22721..52803505 100644 --- a/dockerfiles/Dockerfile.hermes-webui +++ b/dockerfiles/Dockerfile.hermes-webui @@ -111,7 +111,7 @@ RUN /opt/hermes/.venv/bin/python -c 'import cryptography, yaml' \ && grep -Fq "capability.provider!=='local_command'" /opt/hermes-webui/static/atlas-voice.js \ && grep -Fq 'data-priority="maximum"' /opt/hermes-webui/static/index.html \ && grep -Fq 'routing_priority:priority' /opt/hermes-webui/static/atlas-router.js \ - && grep -Fq "'atlas/auto/fast':'AUTO · Fast'" /opt/hermes-webui/static/atlas-router.js \ + && grep -Fq "'atlas/auto/fast':'Automatic · Fast'" /opt/hermes-webui/static/atlas-router.js \ && grep -Fq 'explicit_reasoning_effort' /opt/hermes-webui/api/gateway_chat.py \ && /opt/hermes/.venv/bin/python -m py_compile \ /opt/hermes-webui/api/routes.py \ diff --git a/dockerfiles/hermes-webui-router-patch.py b/dockerfiles/hermes-webui-router-patch.py index 779a9c5d..8bae3eb1 100644 --- a/dockerfiles/hermes-webui-router-patch.py +++ b/dockerfiles/hermes-webui-router-patch.py @@ -23,7 +23,7 @@ replace_once( '''
\n''', '''
diff --git a/dockerfiles/hermes-webui-router.js b/dockerfiles/hermes-webui-router.js index 471c049c..28c2d33c 100644 --- a/dockerfiles/hermes-webui-router.js +++ b/dockerfiles/hermes-webui-router.js @@ -2,14 +2,14 @@ 'use strict'; const PRIORITIES=new Set(['auto','fast','balanced','deep','maximum']); - const LABELS={auto:'AUTO',fast:'FAST',balanced:'BALANCED',deep:'DEEP',maximum:'MAXIMUM'}; + const LABELS={auto:'Automatic',fast:'Fast',balanced:'Balanced',deep:'Deep',maximum:'Maximum'}; const STORAGE_KEY='atlas.hermes.routing-priority'; const EXPLICIT_EFFORT_KEY='atlas.hermes.explicit-reasoning'; const ROUTE_LABELS={ - 'atlas/auto/fast':'AUTO · Fast', - 'atlas/auto/balanced':'AUTO · Balanced', - 'atlas/auto/deep':'AUTO · Deep', - 'atlas/auto/maximum':'AUTO · Maximum', + 'atlas/auto/fast':'Automatic · Fast', + 'atlas/auto/balanced':'Automatic · Balanced', + 'atlas/auto/deep':'Automatic · Deep', + 'atlas/auto/maximum':'Automatic · Maximum', 'atlas/manual/codex/luna':'Codex · Luna', 'atlas/manual/codex/terra':'Codex · Terra', 'atlas/manual/codex/sol':'Codex · SOL', diff --git a/docs/hermes_full_handoff_acceptance.md b/docs/hermes_full_handoff_acceptance.md new file mode 100644 index 00000000..23efe79f --- /dev/null +++ b/docs/hermes_full_handoff_acceptance.md @@ -0,0 +1,341 @@ +# Hermes full-handoff acceptance and release runbook + +`scripts/ops/hermes_handoff_acceptance.py` is a fail-closed release gate. It +contains 63 checks in 15 groups. An empty catalog, an unknown result status, a +mandatory `FAIL` or `NOT_RUN`, a harness error, or incomplete evidence makes the +decision `NO_GO`. The four ephemeral checks are non-mandatory `NOT_RUN` in the +default mode and are replaced with mandatory results only in armed mode. + +This runbook describes evidence collection. It does not authorize a merge, +publish, deployment, Flux reconcile, credential read, or live cluster mutation. +Human review of PR #19 and of the resulting report is required. + +## Required release inputs + +Every value that can go stale must be supplied explicitly. Resolve `main`, PR +heads, the image digest, build source SHA, deployment revision, and chat config +revision from their authoritative read-only sources immediately before the run. +Do not copy the snapshot in the dependency section into a later run. + +```sh +scripts/ops/hermes_handoff_acceptance.py \ + --context atlas-operator \ + --remote-main-sha "$REMOTE_MAIN_SHA" \ + --reviewed-head-sha "$REVIEWED_PR19_HEAD" \ + --agent-image "$AGENT_GIT_SHA_BUILD_TAG_AT_SHA256" \ + --build-sha "$IMAGE_BUILD_SOURCE_SHA" \ + --deployment-revision "$RUNNING_DEPLOYMENT_REVISION" \ + --chat-config-revision "$RUNNING_CHAT_CONFIG_REVISION" \ + --dependency-head "12=$PR12_HEAD" \ + --dependency-head "14=$PR14_HEAD" \ + --dependency-head "15=$PR15_HEAD" \ + --dependency-head "16=$PR16_HEAD" \ + --dependency-head "17=$PR17_HEAD" \ + --dependency-head "18=$PR18_HEAD" \ + --dependency-head "20=$PR20_HEAD" \ + --output build/hermes-handoff-acceptance.json +``` + +The fixed topology is three pool workers on three distinct nodes. `--node-count` +and `--pool-replicas` cannot lower that requirement. Pool node/ordinal assignment, +Telegram session continuity, and the chat revision are mandatory. Deliberate Flux +suspensions must be named with repeatable `--expected-suspension` arguments. + +Exit status is `0` only for `GO`, `1` for `NO_GO`, and `2` for invalid execution +bounds. JSON is written to `--output` or stdout; the human summary goes to stderr +unless `--json-only` is passed. + +## Structural safety boundary + +The catalog is validated before a `Runner` exists. Invalid inputs, an empty or +unsafe catalog, and an invalid armed preflight therefore create no subprocess. +The command policy is checked both before and after executable resolution. + +Default mode permits a narrow read grammar: + +- Kubernetes reads use `name` or reviewed JSONPath projections. Secret resources, + raw API reads, full JSON/YAML, `describe`, `cluster-info dump`, credential-bearing + projections, `auth reconcile`, config operations, all mutations, and every + dry-run mutation trick are rejected before spawn. +- Git is read-only. `fetch`, configuration/helper overrides, alternate worktrees, + external diff helpers, and push are rejected. +- Gitea paths are pinned to `atlas/titan-iac` on an exact segment boundary, so a + look-alike repository such as `atlas/titan-iac-evil` is refused, and any + relative segment — including a percent-encoded `%2e%2e` — is rejected before + the request is built. +- Shell commands must be rendered from frozen templates, use fixed executable + paths, and cannot name service-account, Vault, runtime-access, SSH, or other + credential roots. +- Impersonation is structurally impossible. `--as`, `--as-group`, and `--as-uid` + are refused for every argv, in every mode, from every vantage, and the inner + command of a `kubectl exec` is re-checked under the same rule rather than being + granted an exemption. `validate_catalog` refuses an impersonating step from any + vantage, not only the operator. A future audited self-probe that genuinely needs + impersonation is a reviewed code change, not a configuration flag. +- The binary allowlist is exactly the set with pinned release digests: + `kubectl`, `git`, `sh`, `hermes`, and the Gitea client. `flux` and `helm` are + not on it; Flux and Helm status is read through `kubectl` instead, so no + allowlisted binary can be admitted that the executor would refuse to attest. +- `PATH` is replaced with a fixed path. Every outer executable is resolved to an + allowed real path, checked for unsafe write mode, and compared with a pinned + SHA-256 digest. A caller-authored `kubectl`, `git`, or helper cannot fabricate + evidence. + +The executor uses a new process group, drains stdout and stderr concurrently, +and retains at most the configured bytes while the child is running. A monotonic +absolute timeout terminates and reaps the group, including descendants that keep +pipes open. Timeout, deadline, and output-byte values reject booleans, +zero/negative values, non-finite numbers, and values outside fixed bounds. There +is no concurrency knob: `run_catalog` is strictly sequential and says so rather +than accepting a bound it would ignore. +Truncation discards the affected stream and makes every evaluator `NOT_RUN`. +Bulk-evidence steps have their bytes replaced before they are recorded, but they +keep the resolved executable path and its attested SHA-256, so withholding output +never withholds the provenance of the binary that produced it. + +Credential screening runs before truncation. Command, output, error, identity, +reason, evidence, JSON, and human-summary fields are screened recursively, +including mapping keys and short/numeric sensitive values. The complete rendered +report is screened again; a remaining credential shape is replaced by a bounded +`NO_GO` stub rather than being written. No credential prefix is retained at a +truncation boundary. + +## Vantages and evidence + +The operator must use an external read-only kubeconfig. The self probe runs in +the actual Hermes agent pod. Both identities are obtained with +`kubectl auth whoami`; equal or missing principals make the run `NO_GO`. Required +Switchyard and chat/Telegram pod vantages must also resolve. Node coverage is +derived from all daemonset pod node names, not one selected pod. + +Dangerous Kubernetes permissions are checked only with side-effect-free +`kubectl auth can-i` reviews. The default catalog never constructs a Secret, +TokenRequest, mutation, attach, port-forward, or arbitrary exec request. The one +forge administrative denial check performs a read-only GET and accepts only an +explicit authorization refusal; generic `404`, `not found`, and `no route` text +are not denial evidence. + +An absence is only evidence when the probe observed something. Both name rules +resolve their step through one guard, so a step that exits `0` with no lines — or +with only separators — is `NOT_RUN`, never `PASS`. This is reachable in practice +two ways, and both are pinned by regressions: a POSIX pipeline reports the status +of its *last* stage, so a missing upstream binary in the frozen `env_names` +template exits `0` with no bytes; and `kubectl -o jsonpath` exits `0` when the +structure it walks has drifted, whether a parent field, a leaf field, or a +container filter. Because all five absence checks are mandatory, silence costs +the release its `GO`. The pool claim projection emits one `=` line +per template volume so a volume without a PVC still counts as an observation. + +The strict evaluators reject duplicate-key JSON, raw control characters, +`NaN`/infinity, boolean/number equality coercion, future timestamps, stale +provider evidence, and every malformed routing or Flux row. Routing freshness is +required separately for each provider, lane, and effort. Flux objects must be +non-suspended, current at `observedGeneration`, and explicitly `Ready=True`; +unknown or reconciling state is not healthy. + +`release.exact-lineage-is-running` atomically binds: + +- fixed remote `main` SHA and local `origin/main`; +- open draft PR #19, base `main`, existing feature ref, and exact reviewed head; +- the exact `:git--build-@sha256:` image identity, + with the source SHA equal to the release `main` SHA; +- current Deployment generation and revision; +- the one active ReplicaSet revision/hash; every Ready pod image and imageID; +- the Flux-applied `main` revision. + +The pool evaluator requires exactly ordinals 0, 1, and 2, all Ready, on three +distinct nodes and claims, with service-account token automount disabled. The +chat checks always require durable Telegram sessions and the exact running chat +configuration revision. + +## Armed ephemeral mode + +Armed mode is the only write path and runs only after the complete default +catalog reports `GO`. + +```sh +scripts/ops/hermes_handoff_acceptance.py \ + \ + --arm-ephemeral-push \ + --confirm "ARM EPHEMERAL HERMES HANDOFF PUSH" \ + --ephemeral-token acceptance-20260817a \ + --output build/hermes-handoff-acceptance-armed.json +``` + +Before any runner, network request, or cluster vantage is created, preflight +binds the working directory to the existing linked PR #19 worktree, the exact +feature branch and reviewed local HEAD, the fixed HTTPS Atlas origin, base +`main`, confirmation phrase, and one unique ephemeral ref. The command policy is +then pinned to that exact ref. It allows only its push, one exact draft-create +payload, cleanup of the exactly discovered PR number, and deletion of that ref. + +The run reserves at least 300 seconds (up to 6000) for cleanup. It verifies the +pushed SHA, then identifies the pull request from two independent sources: the +create response itself, and the pull index read page by page to its last page +(50 per page, 20 pages maximum) rather than a single truncated first page. Either +source alone names a pull request this run definitely made, because preflight +proved the ephemeral ref did not exist. Every number found from either source is +registered and closed, so a failed or unusable index can no longer leave an +orphan open. A `GO` still requires exactly one open draft with the expected base, +head ref, base SHA, and head SHA, and the two sources must agree. + +Cleanup requires successful PR close, successful branch deletion, a successful +empty `ls-remote`, and a strict re-read of the same closed PR. Malformed or +partial cleanup is `FAIL`; it is never inferred from an error. When anything is +uncertain the `draft-pull-request` evidence carries `residue_ref` and an exact +`manual_cleanup` command list — the `PATCH .../pulls/ --field state=closed` +calls for every number found, the `DELETE .../branches/` call, and, when no +number could be determined at all, the exact ref to sweep the index for by hand. + +Force, force-with-lease, fan-out, protected refs, other repositories, other PR +numbers, arbitrary PATCH payloads, and repository administration remain denied. + +## Dependency, merge, and conflict graph + +Authoritative snapshot observed 2026-08-17 after refreshing Gitea and Git against +remote `main` `d8f2d818b9a552ea6c2d7fe86554be829bd5ffff`: + +| PR | Head SHA | Draft | Base | Role | +| --- | --- | --- | --- | --- | +| #12 | `ed43dbaa7f094e1841aa760862e80015deb9d80c` | no | `main` | AI usage exporter repair | +| #14 | `8c6e3acdacb223f68b1716f0a74bd543e18890e6` | yes | `main` | bounded forge client and least-privilege RBAC | +| #15 | `73fefbb5d92ef07371a9a997f22d848acd2d06ed` | yes | `main` | result-decomposition reliability | +| #16 | `e8c26ecf85a0c802d378fffffb98b7df43b78242` | no | `main` | unprivileged image builder | +| #17 | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | yes | `main` | three-lane placement | +| #18 | `689bcb6e48414854447988ba174143a307d308b9` | yes | `main` | distributed three-worker pool | +| #19 | `c808baff40a12a90f43c2e5e39b94d139f88ea29` | yes | `main` | head this repair replaces on the same branch | +| #20 | `a242dcc786576ae1a18000cb4941836ff610dff2` | yes | `main` | CLI process-group reaping | + +The repaired #19 SHA cannot be embedded in its own commit without changing that +SHA. Resolve it from Gitea after push and require it as `--reviewed-head-sha`; +the PR handoff records the exact value. All eight heads in this table, including +the repaired #19, merge cleanly with `main` individually. Pairwise +`git merge-tree --write-tree` over all 28 combinations found conflicts for: +#12+#17, #12+#18, #12+#19, #14+#15, #14+#16, #14+#17, #14+#19, #14+#20, #15+#16, +#15+#17, #15+#19, #16+#19, #16+#20, #17+#18, and #17+#20. Gitea's +`mergeable=false` on a draft is not treated as conflict evidence. + +This graph differs from the previous snapshot in exactly two ways. #12 and #18 +both advanced, which adds #12+#18, #17+#18, and #12+#19. And #19 no longer +duplicates the repo-wide hygiene legacy-exception mechanism that #18 also +carries, which makes #18+#19 clean. Recompute before every integration step; +nothing here is durable. + +Use this dependency order, refreshing every head and re-running merge-tree at +each step: + +1. #14 first so the release never restores or prolongs cluster-admin authority. +2. #16, then #15, resolving their Dockerfile and quality-contract changes + additively. Never take one PR's quality contract wholesale. +3. #20 after #15; it depends on the CLI lifecycle work and must be rebased across + the current #14/#16 contract and helper changes. +4. #12 before #17, resolving their Deployment overlap once. +5. #17, then #18, after the authority, image, reliability, and reaping work. +6. #19 last as the integration/acceptance change, only after #12/#14-#18/#20 are + merged and their exact supplied heads are ancestors of refreshed `main`. + +An exact post-repair merge-tree check reports #19 clean with current `main`, #17, +#18, and #20, and conflicting with #12, #14, #15, and #16 on these files: + +| Pair | Overlapping files | +| --- | --- | +| #12+#19 | `testing/tests/test_hermes_auto_router.py` | +| #14+#19 | `testing/quality_contract.json`, `testing/quality_coverage.py`, `testing/tests/test_quality_coverage_helpers.py` | +| #15+#19 | `testing/tests/conftest.py` | +| #16+#19 | `testing/quality_contract.json` | + +Every one of those overlaps is additive on the #19 side. Resolve the contract by +union — take every PR's managed modules, lint paths, tracked files, and branch +tracked files, and never take one PR's quality contract wholesale. #19 adds two +fixture-isolation fixtures to `conftest.py` and two lines to the auto-router test; +keep both sides. Preserve branch coverage, every tracked module, fixture +isolation, and all desired-state changes during conflict resolution. + +Recompute this graph from the Gitea API immediately before integration; do not +infer conflict status from Gitea's draft `mergeable` field. + +The #16/#19 integration must keep both the immutable digest and the source tag +(`newTag: git--build-` plus the digest) in rendered image identity. +Digest-only rendering loses the attested build-source link and therefore +correctly remains `NO_GO` under the exact-lineage evaluator. + +## Safe rollback guidance + +Every rollback is a reviewed forward fix or selective Git revert followed by +the normal Flux workflow. Never restore `hermes-agent-cluster-admin`, even when +reverting unrelated portions of #14. If a rollback would widen RBAC, stop and +write a forward least-privilege fix instead. + +Reverting #18 removes the StatefulSet desired state, but its +`volumeClaimTemplates` PVCs are retained by the Kubernetes default because no +delete retention policy is set. Do not claim Flux pruning deletes those PVCs. +Any PVC deletion is a separate, explicit, human-reviewed data-destruction task +with exact targets and recovery implications. + +Image rollback means a reviewed commit restoring a previously attested digest; +never retag an immutable image. Do not use direct `kubectl` edits as a durable +rollback. + +## Quality and release evidence + +The canonical quality contract explicitly manages, lints, and tracks all 16 +`scripts/ops/hermes_handoff_*.py` modules. It enforces at least 95% line and +branch coverage per tracked file and the 500-physical-line source cap. Contract +tests derive the module set so a future file cannot silently escape the gate. +This change is purely additive to the contract: it registers its own modules and +changes no repo-wide hygiene semantics. + +The gate's `hygiene` step is red on `main` and stays red here. Exactly four +pre-existing test files exceed the 500-line cap — +`scripts/tests/test_dashboards_render_atlas.py` (516), +`testing/tests/test_hermes_chat_quality.py` (2242), +`testing/tests/test_hermes_cli_lanes.py` (1971), and +`testing/tests/test_hermes_coordinator.py` (510). All four are byte-identical at +merge-base `30259b52` and at current `origin/main`, none is touched by this +change, and the same four are the only issues a pristine `main` reports. Splitting +them, or grandfathering them, is a repo-wide contract decision that belongs to the +canonical contract change carried by PR #14/#15, not to this acceptance harness. +Every other gate step — docs, smell, unit, coverage — is green, and every file +this change adds or edits is under the cap. + +```sh +python3 -m testing.quality_gate --profile local --build-dir build +python3 -m testing.quality_handoff_mutation +python3 -m ruff check --select F,B,SIM,C4,UP --ignore B017,UP015,UP035 \ + scripts/ops/hermes_handoff_*.py testing/quality_coverage.py \ + testing/quality_handoff_mutation.py testing/tests/test_hermes_handoff_*.py +python3 -m ruff format --check \ + scripts/ops/hermes_handoff_*.py testing/quality_coverage.py \ + testing/quality_handoff_mutation.py testing/tests/test_hermes_handoff_*.py +python3 -m compileall -q scripts/ops testing +kustomize build services/hermes | kubectl apply --dry-run=client --validate=false -f - +git diff --check origin/main...HEAD +``` + +The format scope names the files this change owns. A wider `testing/quality_*.py` +glob picks up `testing/quality_gate.py`, which is unmodified here and not +format-clean, so the documented command would fail for a reason this change did +not cause. `ruff format` is evidence, not a gate step: the canonical gate runs +`ruff check` only. + +The deterministic mutation gate must kill every policy, redaction, verdict, +numeric, truncation, provenance, and cleanup mutant, including one that restores +the zero-evidence pass, one that restores the bare-prefix repository pin, one +that stops re-checking a `kubectl exec` inner command, and one that drops binary +attestation from an unrecorded step. The adversarial suite includes large +concurrent stdout/stderr, malformed JSON and bytes, zero state, partial failure, +absolute timeout, descendant-held pipes, malformed cleanup, ambiguous creates, +and no-subprocess/no-mutation spies. Render/client dry-run, exact conflict checks, +and bounded diff/full-tree secret scans must also pass. + +The provider-health and Gitea-token-sensitive tests use hermetic fixtures. Their +result must be identical whether synthetic runtime files exist or are absent; +tests must never read the mounted production token. + +## Release decision + +The handoff remains `NO_GO` until all dependency heads are merged, the exact +lineage check passes from an external operator, both default and armed reports +are complete, the branch and PR cleanup is verified, the canonical and +adversarial gates are green, and a human has reviewed PR #19 plus the artifacts. +The harness never merges, deploys, reconciles, publishes, or approves anything. diff --git a/scripts/ops/hermes_handoff_acceptance.py b/scripts/ops/hermes_handoff_acceptance.py new file mode 100755 index 00000000..6dbc69d2 --- /dev/null +++ b/scripts/ops/hermes_handoff_acceptance.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +"""Fail-closed acceptance harness for the Hermes full handoff. + +Read-only by default. The harness collects evidence from two genuinely separate +places — an external operator vantage and the Hermes agent probing itself from +inside its own pod — classifies every check PASS / FAIL / NOT_RUN / +NOT_APPLICABLE, and emits machine-readable JSON alongside a short human summary. + +Any mandatory FAIL or NOT_RUN is a NO_GO, and so is a harness-level problem: an +unreachable vantage, a catalog whose evidence no longer exists, an expired +deadline, or two vantages that turn out to be the same principal. Nothing here +rounds an absence of evidence up to a pass. + + scripts/ops/hermes_handoff_acceptance.py --help + +The release runbook lists every required freshness and lineage input. + +Mutation lives behind a separate arming flag, needs an exact confirmation +phrase and a unique ref, refuses protected push targets before any network +call, and verifies its own cleanup. A default run reports those checks NOT_RUN. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import sys +from collections.abc import Sequence +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from hermes_handoff_catalog import Targets # noqa: E402 +from hermes_handoff_ephemeral import ( # noqa: E402 + CONFIRMATION, + ArmingError, + ArmRequest, + preflight, + run_armed, +) +from hermes_handoff_exec import ( # noqa: E402 + DEFAULT_COMMAND_TIMEOUT, + DEFAULT_RUN_DEADLINE, + Runner, +) +from hermes_handoff_model import GO, Report, unscreened_fields, utc_now # noqa: E402 +from hermes_handoff_policy import ( # noqa: E402 + ARMED, + EXPECTED_BASE, + EXPECTED_REMOTE, + EXPECTED_REPO, + READ_ONLY, +) +from hermes_handoff_redaction import DEFAULT_MAX_BYTES # noqa: E402 +from hermes_handoff_run import ( # noqa: E402 + build_catalog, + build_report, + resolve_vantages, + unavailable_report, + validate_catalog, + validate_targets, +) + +EXIT_GO = 0 +EXIT_NO_GO = 1 +EXIT_USAGE = 2 + + +def dependency_head(value: str) -> tuple[int, str]: + """Parse one `PR=SHA` dependency-head binding.""" + try: + number, sha = value.split("=", 1) + parsed = int(number) + except (TypeError, ValueError) as exc: + raise argparse.ArgumentTypeError( + "dependency head must be PR=40-character-SHA" + ) from exc + if ( + parsed <= 0 + or len(sha) != 40 + or any(character not in "0123456789abcdef" for character in sha) + ): + raise argparse.ArgumentTypeError( + "dependency head must be PR=lowercase-40-character-SHA" + ) + return (parsed, sha) + + +def build_parser() -> argparse.ArgumentParser: + """Return the command-line interface for the harness.""" + parser = argparse.ArgumentParser( + prog="hermes_handoff_acceptance", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--namespace", default="hermes", help="Hermes namespace.") + parser.add_argument( + "--context", help="Kubernetes context for the operator vantage." + ) + parser.add_argument("--kubeconfig", help="Kubeconfig for the operator vantage.") + parser.add_argument( + "--output", type=Path, help="Write the JSON report here instead of stdout." + ) + parser.add_argument( + "--json-only", action="store_true", help="Suppress the human summary." + ) + parser.add_argument( + "--command-timeout", + type=float, + default=DEFAULT_COMMAND_TIMEOUT, + help="Per-command seconds.", + ) + parser.add_argument( + "--deadline", + type=float, + default=DEFAULT_RUN_DEADLINE, + help="Whole-run seconds.", + ) + parser.add_argument( + "--max-output-bytes", + type=int, + default=DEFAULT_MAX_BYTES, + help="Per-capture byte budget.", + ) + parser.add_argument( + "--baseline-commit", + default=Targets.baseline_commit, + help="Commit main must descend from.", + ) + parser.add_argument( + "--dependency-pr", + type=int, + action="append", + default=None, + help="Pull request that must be merged before this release is certifiable (repeatable).", + ) + parser.add_argument( + "--dependency-head", + type=dependency_head, + action="append", + default=None, + metavar="PR=SHA", + help="Exact current dependency PR head (repeat for every dependency).", + ) + parser.add_argument( + "--remote-main-sha", default="", help="Exact observed origin/main SHA." + ) + parser.add_argument( + "--reviewed-head-sha", default="", help="Exact reviewed PR #19 head SHA." + ) + parser.add_argument( + "--agent-image", + default="", + help="Exact agent :git--build-@sha256 identity.", + ) + parser.add_argument( + "--build-sha", default="", help="Exact source SHA used for the image build." + ) + parser.add_argument( + "--deployment-revision", default="", help="Exact running Deployment revision." + ) + parser.add_argument( + "--expected-suspension", + action="append", + default=None, + metavar="NAMESPACE/NAME", + help="Flux object allowed to be suspended (repeatable).", + ) + parser.add_argument( + "--node-count", + type=int, + default=3, + help="Nodes the fleet must cover (fixed at 3).", + ) + parser.add_argument( + "--pool-replicas", type=int, default=3, help="Expected pool worker count." + ) + parser.add_argument( + "--chat-config-revision", + default="", + help="Expected chat pod-template config revision.", + ) + parser.add_argument( + "--chat-ordinal", + type=int, + default=0, + help="Chat tenant ordinal that carries the Telegram identity.", + ) + parser.add_argument( + "--pool-worker-env", + action="append", + default=None, + help="Environment variable the pool workers must declare (repeatable).", + ) + parser.add_argument( + "--max-evidence-age", + type=int, + default=86_400, + help="Seconds before provider and routing evidence is treated as stale.", + ) + armed = parser.add_argument_group("ephemeral mutation mode") + armed.add_argument( + "--arm-ephemeral-push", + action="store_true", + help="Permit one ephemeral branch push and draft pull request, then remove both.", + ) + armed.add_argument( + "--confirm", default="", help=f"Exact confirmation phrase: {CONFIRMATION!r}." + ) + armed.add_argument( + "--ephemeral-token", default="", help="Unique 8-64 character ref suffix." + ) + return parser + + +def targets_from_args(arguments: argparse.Namespace, now: dt.datetime) -> Targets: + """Return the catalog targets a parsed command line describes.""" + defaults = Targets() + return Targets( + namespace=arguments.namespace, + baseline_commit=arguments.baseline_commit, + dependency_pull_requests=tuple( + arguments.dependency_pr or defaults.dependency_pull_requests + ), + dependency_heads=tuple(arguments.dependency_head or ()), + remote_main_sha=arguments.remote_main_sha, + reviewed_head_sha=arguments.reviewed_head_sha, + agent_image=arguments.agent_image, + build_sha=arguments.build_sha, + deployment_revision=arguments.deployment_revision, + expected_suspensions=tuple(arguments.expected_suspension or ()), + node_count=arguments.node_count, + pool_replicas=arguments.pool_replicas, + chat_config_revision=arguments.chat_config_revision, + chat_ordinal=arguments.chat_ordinal, + pool_worker_env=tuple(arguments.pool_worker_env or defaults.pool_worker_env), + max_evidence_age_seconds=arguments.max_evidence_age, + now=now, + ) + + +def emit(report: Report, arguments: argparse.Namespace, stream=None) -> list[str]: + """Write the report, refusing to publish one that still looks credential-shaped.""" + payload = report.as_dict() + offenders = unscreened_fields(payload) + if offenders: + payload = { + "harness": payload["harness"], + "decision": "NO_GO", + "harness_errors": [ + "the rendered report failed its own credential screening at " + + ", ".join(offenders[:8]) + ], + } + rendered = json.dumps(payload, indent=2, sort_keys=True, default=str) + if arguments.output: + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(rendered + "\n", encoding="utf-8") + else: + print(rendered, file=stream or sys.stdout) + if not arguments.json_only: + print(report.render_summary(), file=sys.stderr) + return offenders + + +def _armed_results(arguments: argparse.Namespace, runner: Runner, vantages) -> list: + request = ArmRequest( + repo=EXPECTED_REPO, + remote=EXPECTED_REMOTE, + token=arguments.ephemeral_token, + confirmation=arguments.confirm, + base=EXPECTED_BASE, + expected_head=arguments.reviewed_head_sha, + expected_base_sha=arguments.remote_main_sha, + ) + preflight(request, EXPECTED_REPO, Path.cwd()) + vantage = vantages["operator"] + return run_armed(runner, vantage, request) + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the acceptance sweep and return the process exit status.""" + arguments = build_parser().parse_args(argv) + now = dt.datetime.now(dt.timezone.utc) + started_at = utc_now(lambda: now) + targets = targets_from_args(arguments, now) + mode = ARMED if arguments.arm_ephemeral_push else READ_ONLY + + specs = build_catalog(targets) + startup_problems = [*validate_targets(targets), *validate_catalog(specs)] + if arguments.arm_ephemeral_push: + try: + preflight( + ArmRequest( + repo=EXPECTED_REPO, + remote=EXPECTED_REMOTE, + token=arguments.ephemeral_token, + confirmation=arguments.confirm, + base=EXPECTED_BASE, + expected_head=arguments.reviewed_head_sha, + expected_base_sha=arguments.remote_main_sha, + ), + EXPECTED_REPO, + Path.cwd(), + ) + except ArmingError as exc: + startup_problems.append(f"arming refused: {exc}") + if startup_problems: + report = unavailable_report( + mode, started_at, "; ".join(startup_problems), targets + ) + emit(report, arguments) + return EXIT_NO_GO + + try: + runner = Runner( + mode=mode, + command_timeout=arguments.command_timeout, + deadline_seconds=arguments.deadline, + max_bytes=arguments.max_output_bytes, + ) + except ValueError as exc: + report = unavailable_report( + mode, started_at, f"invalid execution bound: {exc}", targets + ) + emit(report, arguments) + return EXIT_USAGE + vantages, records = resolve_vantages( + runner, targets, arguments.kubeconfig, arguments.context + ) + report = build_report(runner, targets, specs, vantages, records, mode, started_at) + if arguments.arm_ephemeral_push: + if report.decision != GO: + report.harness_errors.append( + "armed mutation was not started because read-only acceptance did not GO" + ) + else: + armed_results = _armed_results(arguments, runner, vantages) + armed_ids = {result.spec.id for result in armed_results} + report.results = [ + result for result in report.results if result.spec.id not in armed_ids + ] + report.results.extend(armed_results) + report.results.sort(key=lambda result: result.spec.id) + report.finished_at = utc_now() + offenders = emit(report, arguments) + if offenders: + return EXIT_NO_GO + return EXIT_GO if report.decision == GO else EXIT_NO_GO + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/scripts/ops/hermes_handoff_arming.py b/scripts/ops/hermes_handoff_arming.py new file mode 100644 index 00000000..5aa596c4 --- /dev/null +++ b/scripts/ops/hermes_handoff_arming.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Arming, provenance, and push-target rules for the armed ephemeral mode. + +Nothing here touches the network or spawns a process. It is the half of armed +mode that must be true before a single command exists: the confirmation phrase, +the fixed repository and base, the one ephemeral ref grammar, and a local +worktree whose HEAD and ``origin/main`` are byte-exact with the reviewed release. +""" + +from __future__ import annotations + +import configparser +import re +from dataclasses import dataclass +from pathlib import Path + +from hermes_handoff_policy import ( + EXPECTED_BASE, + EXPECTED_REMOTE, + EXPECTED_REPO, + arm_ephemeral_policy, +) + +CONFIRMATION = "ARM EPHEMERAL HERMES HANDOFF PUSH" +REF_PREFIX = "ephemeral/hermes-handoff-acceptance" +REF_RE = re.compile(rf"{REF_PREFIX}/[a-z0-9][a-z0-9-]{{7,63}}\Z") +TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9-]{7,63}\Z") +SHA_RE = re.compile(r"[0-9a-f]{40}\Z") +EXPECTED_HEAD_REF = "feature/hermes-full-handoff-acceptance" +EXPECTED_REMOTE_URL = "https://scm.bstein.dev/atlas/titan-iac.git" +PROTECTED_NAMES = frozenset( + { + "default", + "develop", + "head", + "main", + "master", + "prod", + "production", + "release", + "stable", + "trunk", + } +) + + +class ArmingError(ValueError): + """Raised when an ephemeral run is not safe to start.""" + + +@dataclass(frozen=True) +class ArmRequest: + """Every fixed and caller-confirmed input to one armed run.""" + + repo: str + remote: str + token: str + confirmation: str + base: str = EXPECTED_BASE + expected_head: str = "" + expected_base_sha: str = "" + + @property + def ref(self) -> str: + return f"{REF_PREFIX}/{self.token}" + + +def normalise_branch(name: str) -> str: + """Return a branch name stripped of ref decoration, for comparison.""" + return name.strip().removeprefix("refs/heads/").strip("/").lower() + + +def assert_push_target_allowed(ref: str) -> None: + """Raise unless ``ref`` is the one ephemeral branch grammar.""" + if normalise_branch(ref) in PROTECTED_NAMES: + raise ArmingError(f"refusing to push to protected branch {ref!r}") + if not REF_RE.fullmatch(ref): + raise ArmingError(f"push target {ref!r} is not an ephemeral acceptance ref") + + +def bounded_read(path: Path, limit: int = 1024 * 1024) -> str: + data = path.read_bytes() + if not data or len(data) > limit: + raise ArmingError(f"local Git metadata is empty or exceeds {limit} bytes") + return data.decode("utf-8", "strict") + + +def _attest_worktree(root: Path, expected_head: str, expected_base_sha: str) -> None: + """Bind cwd, common Git metadata, branch, remote, and exact local HEAD.""" + resolved = root.resolve(strict=True) + marker = resolved / ".git" + if not marker.is_file(): + raise ArmingError( + "armed mode must run at the root of the existing linked worktree" + ) + line = bounded_read(marker, 4096).strip() + if not line.startswith("gitdir: "): + raise ArmingError("linked-worktree Git metadata is malformed") + gitdir = Path(line.removeprefix("gitdir: ")).resolve(strict=True) + common = (gitdir / bounded_read(gitdir / "commondir", 4096).strip()).resolve( + strict=True + ) + head = bounded_read(gitdir / "HEAD", 4096).strip() + expected_ref = f"refs/heads/{EXPECTED_HEAD_REF}" + if head != f"ref: {expected_ref}": + raise ArmingError("armed mode must run from the existing PR #19 feature branch") + ref_path = common / expected_ref + actual_head = bounded_read(ref_path, 4096).strip() if ref_path.is_file() else "" + if actual_head != expected_head: + raise ArmingError( + "local worktree HEAD does not match the exact reviewed PR head" + ) + remote_ref = common / "refs" / "remotes" / EXPECTED_REMOTE / EXPECTED_BASE + remote_main = bounded_read(remote_ref, 4096).strip() if remote_ref.is_file() else "" + if remote_main != expected_base_sha: + raise ArmingError("local origin/main does not match the exact release base SHA") + parser = configparser.RawConfigParser(interpolation=None) + try: + parser.read_string(bounded_read(common / "config")) + remote_url = parser.get(f'remote "{EXPECTED_REMOTE}"', "url") + except (configparser.Error, KeyError) as exc: + raise ArmingError("local Git remote metadata is malformed") from exc + if remote_url != EXPECTED_REMOTE_URL: + raise ArmingError("origin is not the fixed Atlas titan-iac HTTPS repository") + + +def preflight( + request: ArmRequest, expected_repo: str, worktree: Path | None = None +) -> None: + """Validate arming and local provenance before any subprocess/network call.""" + if request.confirmation != CONFIRMATION: + raise ArmingError("confirmation phrase does not match") + if expected_repo != EXPECTED_REPO or request.repo != EXPECTED_REPO: + raise ArmingError( + "armed repository is not the fixed Atlas titan-iac repository" + ) + if ( + request.remote != EXPECTED_REMOTE + or normalise_branch(request.base) != EXPECTED_BASE + ): + raise ArmingError("armed remote and pull-request base are fixed to origin/main") + if not TOKEN_RE.fullmatch(request.token): + raise ArmingError( + "ephemeral token must be 8-64 lowercase alphanumeric or dash characters" + ) + if not SHA_RE.fullmatch(request.expected_head) or not SHA_RE.fullmatch( + request.expected_base_sha + ): + raise ArmingError("exact reviewed head and release base SHAs are required") + assert_push_target_allowed(request.ref) + _attest_worktree( + worktree or Path.cwd(), request.expected_head, request.expected_base_sha + ) + arm_ephemeral_policy(request.ref) diff --git a/scripts/ops/hermes_handoff_catalog.py b/scripts/ops/hermes_handoff_catalog.py new file mode 100644 index 00000000..59334dec --- /dev/null +++ b/scripts/ops/hermes_handoff_catalog.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Declarative acceptance catalog: baseline, identity, routing, scopes, access. + +The catalog is data on purpose. A reviewer should be able to read what the +release is asserted to satisfy without reading any plumbing, and the harness +should be able to prove structural properties over every entry — both vantages +represented, no impersonation from outside a pod, and no mutation command of +any kind — before it runs a single command. + +Catalog drift is a NO_GO, not a pass. If a manifest, a field name, or a workload +is renamed, the affected checks report ``NOT_RUN`` because the evidence they +name is not there, and the release stops until either the platform or the +catalog is corrected. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field +from typing import Any + +from hermes_handoff_model import ALWAYS, READ, CheckSpec, Step +from hermes_handoff_policy import projection + +OPERATOR = "operator" +SELF = "self" +SWITCHYARD = "switchyard" +NODE = "node" +CHAT = "chat" +VANTAGE_NAMES = (OPERATOR, SELF, SWITCHYARD, NODE, CHAT) + +# `namegenerationobservedGenerationsuspendconditions`. +FLUX_PROJECTION = projection( + ( + "jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{'\\t'}" + "{.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}" + "{range .status.conditions[*]}{.type}={.status},{end}{'\\n'}{end}" + ).replace("'", '"') +) + +ENV_NAME_PROJECTION = projection( + ( + "jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{'\\n'}{end}{end}" + ).replace("'", '"') +) + +INIT_NAME_PROJECTION = projection( + ( + "jsonpath={range .spec.template.spec.initContainers[*]}{.name}{'\\n'}{end}" + ).replace("'", '"') +) + +NODE_NAME_PROJECTION = projection( + "jsonpath={range .items[*]}{.spec.nodeName}{'\\n'}{end}".replace("'", '"') +) + +POOL_PROJECTION = projection( + ( + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.spec.nodeName}{'\\t'}" + "{.status.phase}{'\\t'}{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}" + "{'\\t'}{range .spec.volumes[*]}{.persistentVolumeClaim.claimName}{','}{end}{'\\n'}{end}" + ).replace("'", '"') +) + +PROVIDER_API_KEY_NAMES = ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + "AZURE_OPENAI_API_KEY", + "CLAUDE_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", +) +FORGE_CREDENTIAL_NAMES = ( + "GITEA_PASSWORD", + "GITEA_TOKEN", + "GITEA_USERNAME", + "GIT_PASSWORD", +) + +CODEX_HEALTH = "/opt/data/provider-health/codex.json" +CLAUDE_HEALTH = "/opt/data/provider-health/claude.json" +CODEX_HEALTH_FIELDS = "authenticated,transport,state,model,effort,latency_ms,checked_at" +CLAUDE_HEALTH_FIELDS = ( + "authenticated,api_provider,auth_method,transport,state,subscription_type," + "model,effort,latency_ms,checked_at" +) +ROUTING_LOG = "/var/lib/switchyard/routing.jsonl" + + +@dataclass(frozen=True) +class Targets: + """Runtime names and expected values the catalog is written against. + + Everything a site can legitimately rename lives here so the assertions stay + readable and a rename is a configuration change rather than a code change. + """ + + namespace: str = "hermes" + agent_deployment: str = "hermes-agent" + agent_container: str = "hermes" + chat_statefulset: str = "hermes-chat-tenant" + chat_container: str = "hermes" + chat_ordinal: int = 0 + chat_router_deployment: str = "hermes-chat-router" + triage_deployment: str = "hermes" + triage_container: str = "hermes" + switchyard_deployment: str = "hermes-switchyard" + switchyard_container: str = "switchyard" + node_daemonset: str = "hermes-node-ssh-access" + node_container: str = "key-reconciler" + pool_statefulset: str = "hermes-execution-worker" + pool_replicas: int = 3 + pool_selector: str = "app=hermes-execution-worker" + coordinator_claims: tuple[str, ...] = ( + "hermes-agent-home", + "hermes-agent-workspace", + ) + node_account: str = "hermes-agent" + node_account_identity: str = "1200:1200:/home/hermes-agent:/bin/bash" + node_count: int = 3 + repo: str = "atlas/titan-iac" + remote: str = "origin" + baseline_commit: str = "ab346f55509d584e457fe26cf90be3078f7a375c" + dependency_pull_requests: tuple[int, ...] = (12, 14, 15, 16, 17, 18, 20) + dependency_heads: tuple[tuple[int, str], ...] = () + remote_main_sha: str = "" + reviewed_pr_number: int = 19 + reviewed_head_ref: str = "feature/hermes-full-handoff-acceptance" + reviewed_head_sha: str = "" + agent_image: str = "" + build_sha: str = "" + deployment_revision: str = "" + expected_suspensions: tuple[str, ...] = () + harbor_namespace: str = "harbor" + harbor_immutability_job: str = "hermes-agent-immutability" + builder_namespace: str = "jenkins" + builder_serviceaccount: str = "hermes-image-builder" + builder_pipeline: str = "ci/Jenkinsfile.hermes-agent-image" + builder_capabilities: str = ( + 'add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]' + ) + agent_init_containers: tuple[str, ...] = ("patch-web-session-activity",) + chat_init_containers: tuple[str, ...] = ("patch-api-server-sessions",) + chat_config_revision: str = "" + pool_worker_env: tuple[str, ...] = ("HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL") + max_evidence_age_seconds: int = 86_400 + routing_tail_lines: int = 600 + routing_tail_bytes: int = 512 * 1024 + listing_bytes: int = 512 * 1024 + git_askpass: str = "/opt/coordinator/gitea_askpass.sh" + now: Any = None + extra: dict[str, Any] = field(default_factory=dict) + + +def step( + key: str, + vantage: str, + *argv: str, + kind: str = READ, + optional: bool = False, + record: bool = True, + max_bytes: int | None = None, +) -> Step: + """Return a probe step addressed to one vantage.""" + return Step( + key=key, + vantage=vantage, + argv=tuple(argv), + kind=kind, + optional=optional, + record=record, + max_bytes=max_bytes, + ) + + +def check( + identifier: str, + title: str, + group: str, + rule: str, + steps: Sequence[Step], + expect: dict[str, Any] | None = None, + mandatory: bool = True, + scope: str = ALWAYS, + rationale: str = "", +) -> CheckSpec: + """Return one catalog entry.""" + return CheckSpec( + id=identifier, + title=title, + group=group, + rule=rule, + steps=tuple(steps), + expect=expect or {}, + mandatory=mandatory, + scope=scope, + rationale=rationale, + ) diff --git a/scripts/ops/hermes_handoff_checks_access.py b/scripts/ops/hermes_handoff_checks_access.py new file mode 100644 index 00000000..df5c8117 --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_access.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +"""Acceptance checks for what Hermes may and may not do to the cluster. + +Denials use Kubernetes authorization reviews only. Live denied attempts were +removed because a policy regression could turn a supposed probe into a real +credential read or mutation; the default harness never constructs such argv. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + FLUX_PROJECTION, + OPERATOR, + SELF, + Targets, + check, + step, +) +from hermes_handoff_model import REVIEW, CheckSpec + + +def _access(targets: Targets) -> list[CheckSpec]: + namespace = targets.namespace + return [ + check( + "access.no-cluster-admin-binding-for-agent", + "No ClusterRoleBinding grants the agent service account cluster-admin", + "access-denied", + "names_absent", + [ + step( + "bindings", + OPERATOR, + "kubectl", + "get", + "clusterrolebindings", + "-o", + "name", + ) + ], + {"step": "bindings", "contains": ("hermes-agent-cluster-admin",)}, + rationale="The binding inventory and in-pod authorization reviews are independent desired/enforced views.", + ), + check( + "access.secrets-are-denied", + "Hermes cannot read Secrets cluster-wide", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "get", + "secrets", + "--all-namespaces", + kind=REVIEW, + ) + ], + rationale="A live Secret request is never constructed, even when an RBAC regression would allow it.", + ), + check( + "access.service-account-tokens-are-denied", + "Hermes cannot mint a service-account token", + "access-denied", + "review_denied", + [ + step( + "review-token", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "serviceaccounts/token", + "--namespace", + namespace, + kind=REVIEW, + ), + step( + "review-serviceaccount", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "serviceaccounts", + "--namespace", + namespace, + kind=REVIEW, + ), + ], + rationale="Token creation has no credential-safe live probe, so both relevant grants must be denied by review.", + ), + check( + "access.impersonation-is-denied", + "Hermes cannot impersonate another identity", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "impersonate", + "users", + "--all-namespaces", + kind=REVIEW, + ) + ], + rationale="The default harness never constructs an impersonated request.", + ), + check( + "access.workload-mutation-is-denied", + "Hermes cannot mutate its own workload", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "patch", + "deployments", + "--namespace", + namespace, + kind=REVIEW, + ) + ], + rationale="Client-side and server-side dry-run mutation tricks are excluded structurally.", + ), + check( + "access.pod-exec-is-denied", + "Hermes cannot exec into an arbitrary pod", + "access-denied", + "review_denied", + [ + step( + "review", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "pods/exec", + "--all-namespaces", + kind=REVIEW, + ) + ], + rationale="No cross-pod exec request is created by a read-only run.", + ), + check( + "access.attach-portforward-and-kube-system-writes-are-denied", + "Hermes holds no attach, port-forward, or control-plane write authority", + "access-denied", + "review_denied", + [ + step( + "review-attach", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "pods/attach", + "--all-namespaces", + kind=REVIEW, + ), + step( + "review-portforward", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "pods/portforward", + "--all-namespaces", + kind=REVIEW, + ), + step( + "review-create", + SELF, + "kubectl", + "auth", + "can-i", + "create", + "configmaps", + "--namespace", + "kube-system", + kind=REVIEW, + ), + ], + rationale="Every authority is reviewed without issuing the request it would authorize.", + ), + check( + "access.required-reads-are-allowed", + "Hermes retains the reads its operator role depends on", + "access-allowed", + "allowed", + [ + step( + "watch-review", + SELF, + "kubectl", + "auth", + "can-i", + "watch", + "pods", + "--namespace", + namespace, + kind=REVIEW, + ), + step( + "pods", + SELF, + "kubectl", + "--namespace", + namespace, + "get", + "pods", + "-o", + "name", + ), + step("namespaces", SELF, "kubectl", "get", "namespaces", "-o", "name"), + step( + "deployments", + SELF, + "kubectl", + "--namespace", + namespace, + "get", + "deployments", + "-o", + "name", + ), + step("nodes", SELF, "kubectl", "get", "nodes", "-o", "name"), + ], + ), + check( + "access.flux-and-helm-status-are-allowed", + "Hermes can read Flux and Helm reconciliation status", + "access-allowed", + "allowed", + [ + step( + "kustomizations", + SELF, + "kubectl", + "get", + "kustomizations.kustomize.toolkit.fluxcd.io", + "--all-namespaces", + "-o", + "name", + record=False, + ), + step( + "helmreleases", + SELF, + "kubectl", + "get", + "helmreleases.helm.toolkit.fluxcd.io", + "--all-namespaces", + "-o", + "name", + record=False, + ), + ], + ), + check( + "access.namespace-view-agrees-across-vantages", + "The operator and in-pod vantages see the same namespace inventory", + "access-allowed", + "vantages_agree", + [ + step( + "operator", + OPERATOR, + "kubectl", + "get", + "namespaces", + "-o", + "name", + record=False, + ), + step( + "self", + SELF, + "kubectl", + "get", + "namespaces", + "-o", + "name", + record=False, + ), + ], + {"steps": ("operator", "self")}, + rationale="A disagreement here means one vantage is not seeing the cluster the other is certifying.", + ), + ] + + +def _gitops(targets: Targets) -> list[CheckSpec]: + return [ + check( + f"gitops.{kind.split('.')[0]}-reconcile-cleanly", + f"No unexpected {label} suspension or unhealthy reconciliation", + "gitops", + "flux_health", + [ + step( + "objects", + OPERATOR, + "kubectl", + "get", + kind, + "--all-namespaces", + "-o", + FLUX_PROJECTION, + record=False, + ) + ], + {"step": "objects", "expected_suspensions": targets.expected_suspensions}, + ) + for kind, label in ( + ("kustomizations.kustomize.toolkit.fluxcd.io", "Kustomization"), + ("helmreleases.helm.toolkit.fluxcd.io", "HelmRelease"), + ) + ] + + +def access_checks(targets: Targets) -> list[CheckSpec]: + """Return the access-denied, access-allowed, and GitOps health checks.""" + return [*_access(targets), *_gitops(targets)] diff --git a/scripts/ops/hermes_handoff_checks_delivery.py b/scripts/ops/hermes_handoff_checks_delivery.py new file mode 100644 index 00000000..102f93e4 --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_delivery.py @@ -0,0 +1,374 @@ +#!/usr/bin/env python3 +"""Acceptance checks for forge authority, node accounts, images, and regressions. + +What the platform is allowed to do outside its own namespace: which forge +authority a worker holds, what a node account is and is not, how an image is +built and pinned, and whether the fixes the release depends on are actually +running rather than merely merged. + +Where a property has no side-effect-free live probe — a merge that must never be +attempted, a branch protection the harness identity cannot read — the entry says +so in its rationale and asserts the nearest fact that *is* observable, rather +than quietly asserting nothing. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + FORGE_CREDENTIAL_NAMES, + INIT_NAME_PROJECTION, + NODE_NAME_PROJECTION, + OPERATOR, + SELF, + Targets, + check, + step, +) +from hermes_handoff_model import ATTEMPT, CheckSpec +from hermes_handoff_policy import GITEA_CLIENT, projection, shell + + +def _forge(targets: Targets) -> list[CheckSpec]: + api = f"/api/v1/repos/{targets.repo}" + return [ + check( + "forge.worker-holds-no-git-credential-variable", + "No forge credential is exported into the worker process", + "forge", + "names_absent", + [step("env", SELF, *shell("env_names"))], + {"step": "env", "names": FORGE_CREDENTIAL_NAMES}, + rationale="Authentication reaches Git through an askpass helper that reads runtime state, never through the environment.", + ), + check( + "forge.identity-is-not-an-administrator", + "The forge identity the worker uses holds no administrative rights", + "forge", + "json_field", + [step("user", SELF, GITEA_CLIENT, "GET", "/api/v1/user")], + {"step": "user", "fields": {"is_admin": False}}, + ), + check( + "forge.repository-authority-is-push-only", + "The worker can push a branch but cannot administer the repository", + "forge", + "json_field", + [step("repo", SELF, GITEA_CLIENT, "GET", api)], + { + "step": "repo", + "fields": { + "permissions.admin": False, + "permissions.push": True, + "permissions.pull": True, + }, + }, + rationale="Merge, approve, close, and protection changes all require authority this identity is proven not to hold.", + ), + check( + "forge.administrative-route-is-refused", + "An administrative forge route is refused for this identity", + "forge", + "denied", + [ + step( + "attempt", + SELF, + GITEA_CLIENT, + "GET", + f"{api}/branch_protections", + kind=ATTEMPT, + ) + ], + rationale="A live refusal, not a configuration reading. The same lack of authority is what blocks a merge route.", + ), + ] + + +def _nodes(targets: Targets) -> list[CheckSpec]: + return [ + check( + "nodes.hardening-covers-every-node", + "The node account audit runs on every node in the cluster", + "nodes", + "distinct_count", + [ + step( + "coverage", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "pods", + "--selector", + f"app={targets.node_daemonset}", + "-o", + NODE_NAME_PROJECTION, + ) + ], + { + "step": "coverage", + "minimum": targets.node_count, + "total_equals": targets.node_count, + }, + rationale="A hardened fleet with one unenrolled node is an unhardened fleet.", + ), + ] + + +def _build(targets: Targets) -> list[CheckSpec]: + pipeline = f"{targets.remote}/main:{targets.builder_pipeline}" + return [ + check( + "build.builder-service-account-is-tokenless", + "The image builder service account mounts no API token", + "build", + "stdout_matches", + [ + step( + "serviceaccount", + OPERATOR, + "kubectl", + "--namespace", + targets.builder_namespace, + "get", + "serviceaccount", + targets.builder_serviceaccount, + "-o", + projection("jsonpath={.automountServiceAccountToken}"), + ) + ], + {"step": "serviceaccount", "equals": "false"}, + ), + check( + "build.builder-runs-unprivileged-with-exact-capabilities", + "The build pod is unprivileged and adds only the expected capabilities", + "build", + "names_present", + [step("pipeline", OPERATOR, "git", "show", pipeline, record=False)], + { + "step": "pipeline", + "names": ( + targets.builder_capabilities, + "privileged: false", + "allowPrivilegeEscalation: false", + "automountServiceAccountToken: false", + 'drop: ["ALL"]', + ), + }, + rationale="Exact lines, not a substring sweep: a widened capability set has to change one of these lines to take effect.", + ), + check( + "build.harbor-immutability-rule-is-applied", + "The Harbor immutable-tag rule for the agent image has been applied", + "build", + "lines_match", + [ + step( + "job", + OPERATOR, + "kubectl", + "--namespace", + targets.harbor_namespace, + "get", + "job", + targets.harbor_immutability_job, + "-o", + projection("jsonpath={.status.succeeded}"), + ) + ], + {"step": "job", "pattern": r"^[1-9][0-9]*$", "minimum": 1}, + rationale="The job verifies the rule it creates, so a completed run is evidence the rule matched the expected contract.", + ), + _release_lineage(targets), + ] + + +def _release_lineage(targets: Targets) -> CheckSpec: + """Bind the exact reviewed release from Git through the Ready pods.""" + container = targets.agent_container + namespace = targets.namespace + deployment = f"deploy/{targets.agent_deployment}" + deployment_projection = projection( + ( + "jsonpath={.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}" + "{.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}" + "{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}" + f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}' + ).replace("'", '"') + ) + pod_projection = projection( + ( + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.status.phase}{'\\t'}" + "{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}" + f'{{.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}' + f'{{.status.containerStatuses[?(@.name=="{container}")].imageID}}{{"\\t"}}' + "{.metadata.labels.pod-template-hash}{'\\n'}{end}" + ).replace("'", '"') + ) + replicaset_projection = projection( + ( + "jsonpath={range .items[*]}{.metadata.name}{'\\t'}" + "{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}" + "{.status.readyReplicas}{'\\t'}{.status.availableReplicas}{'\\t'}" + f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}' + "{.metadata.labels.pod-template-hash}{'\\n'}{end}" + ).replace("'", '"') + ) + flux_projection = projection( + ( + "jsonpath={.metadata.name}{'\\t'}{.metadata.generation}{'\\t'}" + "{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}" + "{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}" + "{.status.lastAppliedRevision}" + ).replace("'", '"') + ) + return check( + "release.exact-lineage-is-running", + "Remote main, reviewed PR, image, build, Flux, and running revision are identical", + "release", + "release_lineage", + [ + step( + "remote-main", + OPERATOR, + "git", + "ls-remote", + targets.remote, + "refs/heads/main", + ), + step( + "reviewed-pr", + OPERATOR, + GITEA_CLIENT, + "GET", + f"/api/v1/repos/{targets.repo}/pulls/{targets.reviewed_pr_number}", + ), + step( + "build-source", + OPERATOR, + "git", + "rev-parse", + f"{targets.remote}/main", + ), + step( + "deployment", + OPERATOR, + "kubectl", + "--namespace", + namespace, + "get", + deployment, + "-o", + deployment_projection, + ), + step( + "pods", + OPERATOR, + "kubectl", + "--namespace", + namespace, + "get", + "pods", + "--selector", + f"app={targets.agent_deployment}", + "-o", + pod_projection, + ), + step( + "replicasets", + OPERATOR, + "kubectl", + "--namespace", + namespace, + "get", + "replicasets", + "--selector", + f"app={targets.agent_deployment}", + "-o", + replicaset_projection, + ), + step( + "flux", + OPERATOR, + "kubectl", + "--namespace", + "flux-system", + "get", + "kustomization/hermes", + "-o", + flux_projection, + ), + ], + { + "main_sha": targets.remote_main_sha, + "head_sha": targets.reviewed_head_sha, + "head_ref": targets.reviewed_head_ref, + "pr_number": targets.reviewed_pr_number, + "image": targets.agent_image, + "build_sha": targets.build_sha, + "deployment_revision": targets.deployment_revision, + }, + rationale="A PASS requires one exact release identity across every source and running object.", + ) + + +def _reliability(targets: Targets) -> list[CheckSpec]: + return [ + check( + "reliability.finalization-and-replay-patches-are-deployed", + "The agent workload runs the finalization, replay, and session patches", + "reliability", + "names_present", + [ + step( + "init", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.agent_deployment}", + "-o", + INIT_NAME_PROJECTION, + ) + ], + {"step": "init", "names": targets.agent_init_containers}, + rationale="The regression suites live in the repository; this asserts the fixes they cover are actually running.", + ), + check( + "reliability.regression-suites-are-present-on-main", + "The decomposition and worker-recovery regression suites are on main", + "reliability", + "names_present", + [ + step( + "suites", + OPERATOR, + "git", + "ls-tree", + "--name-only", + f"{targets.remote}/main", + "testing/tests/", + ) + ], + { + "step": "suites", + "names": ( + "testing/tests/test_hermes_cli_lanes.py", + "testing/tests/test_hermes_coordinator.py", + "testing/tests/test_hermes_worker_recovery.py", + ), + }, + ), + ] + + +def delivery_checks(targets: Targets) -> list[CheckSpec]: + """Return the forge, node, build, and reliability checks.""" + return [ + *_forge(targets), + *_nodes(targets), + *_build(targets), + *_reliability(targets), + ] diff --git a/scripts/ops/hermes_handoff_checks_platform.py b/scripts/ops/hermes_handoff_checks_platform.py new file mode 100644 index 00000000..7126036b --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_platform.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +"""Acceptance checks for the release baseline, provider identity, and routing. + +These are the facts a handoff rests on before any authority question is asked: +that the tree under test is the merged one, that both providers authenticate +through the subscriptions they are supposed to, that Switchyard actually decided +the routes it claims to, and that chat, agent, and triage are still three +distinct scopes rather than one surface wearing three hostnames. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + CLAUDE_HEALTH, + CLAUDE_HEALTH_FIELDS, + CODEX_HEALTH, + CODEX_HEALTH_FIELDS, + ENV_NAME_PROJECTION, + OPERATOR, + PROVIDER_API_KEY_NAMES, + ROUTING_LOG, + SELF, + SWITCHYARD, + Targets, + check, + step, +) +from hermes_handoff_model import CheckSpec +from hermes_handoff_policy import GITEA_CLIENT, projection, shell + + +def _baseline(targets: Targets) -> list[CheckSpec]: + api = f"/api/v1/repos/{targets.repo}" + dependency_heads = dict(targets.dependency_heads) + checks = [ + check( + "baseline.local-origin-main-is-current", + "The local origin/main ref equals the fixed remote-main SHA", + "baseline", + "stdout_matches", + [step("main", OPERATOR, "git", "rev-parse", f"{targets.remote}/main")], + {"step": "main", "equals": targets.remote_main_sha}, + rationale="The ancestry check is meaningful only after the local tracking ref is bound to the remote observation.", + ), + check( + "baseline.origin-main-descends-merged-work", + "origin/main descends from the merged Hermes isolation baseline", + "baseline", + "allowed", + [ + step( + "ancestor", + OPERATOR, + "git", + "merge-base", + "--is-ancestor", + targets.baseline_commit, + f"{targets.remote}/main", + ) + ], + rationale="Everything downstream assumes the merged worker-isolation baseline is present.", + ), + check( + "baseline.pending-pull-requests-recorded", + "Exact open pull-request heads are recorded, not assumed", + "baseline", + "json_record", + [ + step( + "open", + OPERATOR, + GITEA_CLIENT, + "GET", + f"{api}/pulls?state=open&limit=50", + ) + ], + { + "step": "open", + "record": { + "numbers": "[].number", + "head_refs": "[].head.ref", + "head_shas": "[].head.sha", + "base_shas": "[].base.sha", + "drafts": "[].draft", + "mergeable": "[].mergeable", + }, + }, + rationale="The runbook has to name the heads it was verified against; stale heads are how a release verifies the wrong tree.", + ), + check( + "baseline.operator-checkout-clean", + "The operator checkout has no uncommitted drift", + "baseline", + "stdout_matches", + [step("status", OPERATOR, "git", "status", "--porcelain")], + {"step": "status", "equals": ""}, + mandatory=False, + rationale="Advisory: a dirty checkout does not invalidate cluster evidence, but it does muddy what was compared.", + ), + ] + checks += [ + check( + f"baseline.dependency-pr-{number}-merged", + f"Pull request #{number} is merged into main", + "baseline", + "json_field", + [step("pr", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls/{number}")], + { + "step": "pr", + "fields": { + "merged": True, + "base.ref": "main", + "head.sha": dependency_heads.get(number, ""), + }, + }, + rationale="This harness certifies the post-merge platform; an unmerged dependency means it is measuring something else.", + ) + for number in targets.dependency_pull_requests + ] + return checks + + +def _identity(targets: Targets) -> list[CheckSpec]: + codex = shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS) + claude = shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS) + agent = f"deploy/{targets.agent_deployment}" + return [ + check( + "identity.codex-is-chatgpt-subscription", + "Codex authenticates through the ChatGPT subscription, not an API key", + "identity", + "json_field", + [step("health", SELF, *codex)], + { + "step": "health", + "fields": { + "authenticated": True, + "transport": "codex-chatgpt-subscription", + "state": "available", + }, + }, + rationale="Only allow-listed status fields are read; the credential itself is never opened.", + ), + check( + "identity.codex-evidence-is-fresh", + "Codex authentication evidence is current", + "identity", + "json_recent", + [step("health", SELF, *codex)], + { + "step": "health", + "now": targets.now, + "fields": {"checked_at": targets.max_evidence_age_seconds}, + }, + rationale="Stale provider evidence describes a state the release no longer has.", + ), + check( + "identity.claude-is-firstparty-subscription", + "Claude authenticates as a claude.ai first-party subscription", + "identity", + "json_field", + [step("health", SELF, *claude)], + { + "step": "health", + "fields": { + "authenticated": True, + "api_provider": "firstParty", + "auth_method": "claude.ai", + "transport": "claude-code-cli-subscription", + "state": "available", + }, + }, + ), + check( + "identity.claude-evidence-is-fresh", + "Claude authentication evidence is current", + "identity", + "json_recent", + [step("health", SELF, *claude)], + { + "step": "health", + "now": targets.now, + "fields": {"checked_at": targets.max_evidence_age_seconds}, + }, + ), + check( + "identity.no-provider-api-key-in-pod", + "No provider API-key variable is set in the running agent process", + "identity", + "names_absent", + [step("env", SELF, *shell("env_names"))], + { + "step": "env", + "names": PROVIDER_API_KEY_NAMES, + "contains": ("API_KEY",), + }, + rationale="Names only. The probe lists variable names and never their values.", + ), + check( + "identity.no-provider-api-key-in-manifest", + "No provider API-key variable is declared on the agent workload", + "identity", + "names_absent", + [ + step( + "env", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + agent, + "-o", + ENV_NAME_PROJECTION, + ) + ], + {"step": "env", "names": PROVIDER_API_KEY_NAMES, "contains": ("API_KEY",)}, + rationale="The operator counterpart to the in-pod probe: desired state and running state are asserted separately.", + ), + ] + + +def _routing(targets: Targets) -> list[CheckSpec]: + switchyard = f"deploy/{targets.switchyard_deployment}" + tail = shell("tail_lines", path=ROUTING_LOG, limit=str(targets.routing_tail_lines)) + routing_step = step( + "routing", SWITCHYARD, *tail, record=False, max_bytes=targets.routing_tail_bytes + ) + return [ + check( + "routing.switchyard-is-available", + "Switchyard has a ready replica serving every model-call boundary", + "routing", + "lines_match", + [ + step( + "deployment", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + switchyard, + "-o", + projection("jsonpath={.status.readyReplicas}"), + ) + ], + {"step": "deployment", "pattern": r"^[1-9][0-9]*$", "minimum": 1}, + ), + check( + "routing.provider-effort-and-fallback-evidence", + "Routing evidence covers both providers, all graded efforts, and a real fallback", + "routing", + "routing_evidence", + [routing_step], + { + "step": "routing", + "providers": ("codex", "claude"), + "efforts": ("medium", "high", "xhigh"), + "lanes": ("route", "worker"), + "require_fallback_evidence": True, + "max_age_seconds": targets.max_evidence_age_seconds, + "now": targets.now, + }, + rationale="The routing log records the decided route per boundary; the tail is parsed here and never pasted into the report.", + ), + check( + "routing.codex-latency-recorded", + "Codex boundary latency is measured, not assumed", + "routing", + "json_numeric", + [ + step( + "health", + SELF, + *shell( + "json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS + ), + ) + ], + {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, + ), + check( + "routing.claude-latency-recorded", + "Claude boundary latency is measured, not assumed", + "routing", + "json_numeric", + [ + step( + "health", + SELF, + *shell( + "json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS + ), + ) + ], + {"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}}, + ), + ] + + +def _scope_profile( + targets: Targets, name: str, workload: str, container: str, expected: str +) -> CheckSpec: + profile_projection = projection( + f'jsonpath={{.spec.template.spec.containers[?(@.name=="{container}")]' + '.env[?(@.name=="HERMES_AUTO_ROUTER_PROFILE")].value}' + ) + return check( + f"scopes.{name}-profile-is-distinct", + f"The {name} surface runs under its own routing profile", + "scopes", + "stdout_matches", + [ + step( + "profile", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + workload, + "-o", + profile_projection, + ) + ], + {"step": "profile", "equals": expected}, + rationale="Chat, agent, and triage must not collapse into one scope; each states its own profile.", + ) + + +def _scopes(targets: Targets) -> list[CheckSpec]: + return [ + _scope_profile( + targets, + "agent", + f"deploy/{targets.agent_deployment}", + targets.agent_container, + "agent", + ), + _scope_profile( + targets, + "chat", + f"statefulset/{targets.chat_statefulset}", + targets.chat_container, + "chat", + ), + _scope_profile( + targets, + "triage", + f"deploy/{targets.triage_deployment}", + targets.triage_container, + "triage", + ), + check( + "scopes.surfaces-have-distinct-hosts", + "Chat, agent, and triage are published on distinct hostnames", + "scopes", + "distinct_count", + [ + step( + "hosts", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "ingress", + "-o", + projection( + 'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}' + ), + ) + ], + {"step": "hosts", "minimum": 3}, + ), + ] + + +def platform_checks(targets: Targets) -> list[CheckSpec]: + """Return the baseline, identity, routing, and scope checks.""" + return [ + *_baseline(targets), + *_identity(targets), + *_routing(targets), + *_scopes(targets), + ] diff --git a/scripts/ops/hermes_handoff_checks_workers.py b/scripts/ops/hermes_handoff_checks_workers.py new file mode 100644 index 00000000..d62c719c --- /dev/null +++ b/scripts/ops/hermes_handoff_checks_workers.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 +"""Acceptance checks for the distributed worker pool and the visible surfaces. + +A pool is only distributed if its workers occupy distinct nodes and distinct +volumes and leave state ownership with the coordinator; a replica count alone +satisfies none of that. A surface is only working if the work it did is visible +in it, which is why the session and Kanban checks look at the same data the UI +renders rather than at whether a pod is Running. + +The ephemeral entries are placeholders in a default run. They report NOT_RUN +until the mutation mode is separately armed, and the armed mode replaces them +with real, mandatory results. +""" + +from __future__ import annotations + +from hermes_handoff_catalog import ( + CHAT, + INIT_NAME_PROJECTION, + OPERATOR, + POOL_PROJECTION, + SELF, + Targets, + check, +) +from hermes_handoff_catalog import step as step +from hermes_handoff_model import EPHEMERAL, CheckSpec +from hermes_handoff_policy import projection + +KANBAN_STATUSES = [ + "archived", + "blocked", + "done", + "ready", + "review", + "running", + "scheduled", + "todo", + "triage", +] + + +def _pool(targets: Targets) -> list[CheckSpec]: + selector = targets.pool_selector + return [ + check( + "pool.three-workers-have-distinct-nodes-and-volumes", + "Exactly three Ready ordinal workers occupy distinct nodes and claims", + "pool", + "pool_topology", + [ + step( + "statefulset", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + projection( + "jsonpath={.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}{.status.currentReplicas}{'\\t'}{.spec.template.spec.automountServiceAccountToken}".replace( + "'", '"' + ) + ), + ), + step( + "workers", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + "pods", + "--selector", + selector, + "-o", + POOL_PROJECTION, + ), + ], + { + "state_step": "statefulset", + "worker_step": "workers", + "replicas": targets.pool_replicas, + "name": targets.pool_statefulset, + }, + rationale="Replica count, Ready state, ordinal coverage, node placement, PVC isolation, and tokenlessness are one fail-closed assertion.", + ), + check( + "pool.coordinator-retains-sole-state-ownership", + "No pool worker mounts the coordinator's durable state", + "pool", + "names_absent", + [ + step( + "claims", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + projection( + "jsonpath={range .spec.template.spec.volumes[*]}{.name}{'='}" + "{.persistentVolumeClaim.claimName}{'\\n'}{end}".replace( + "'", '"' + ) + ), + ) + ], + {"step": "claims", "contains": targets.coordinator_claims}, + rationale="Two writers on one SQLite state file is the duplicate-work and lost-result failure this pool exists to avoid. One line per template volume keeps a drifted or empty projection NOT_RUN instead of a silent pass.", + ), + *_pool_env(targets), + ] + + +def _pool_env(targets: Targets) -> list[CheckSpec]: + """Return the mandatory assignment-safety configuration check.""" + return [ + check( + "pool.assignment-safety-knobs-are-configured", + "Pool workers carry the lease and de-duplication configuration", + "pool", + "names_present", + [ + step( + "env", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.pool_statefulset}", + "-o", + projection( + 'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}' + ), + ) + ], + {"step": "env", "names": targets.pool_worker_env}, + rationale="Deployed configuration only; the stale-claim and duplicate-dispatch behaviour itself is covered by the pool's own regression suite.", + ), + check( + "pool.assignment-fieldrefs-are-bound", + "Worker node and ordinal identities come from fixed downward-API fields", + "pool", + "names_present", + [ + step( + "source", + OPERATOR, + "git", + "show", + f"{targets.remote}/main:services/hermes/execution-worker-statefulset.yaml", + record=False, + ) + ], + { + "step": "source", + "names": ( + "name: HERMES_WORKER_ORDINAL", + "fieldPath: metadata.labels['apps.kubernetes.io/pod-index']", + "name: HERMES_WORKER_NODE", + "fieldPath: spec.nodeName", + ), + }, + rationale="This prevents caller-authored ordinal/node values from impersonating another worker assignment.", + ), + ] + + +def _surfaces(targets: Targets) -> list[CheckSpec]: + checks = [ + check( + "surfaces.kanban-activity-is-visible", + "Durable Kanban activity is visible to the agent surface", + "surfaces", + "all_of_field", + [ + step( + "tasks", + SELF, + "hermes", + "kanban", + "list", + "--json", + record=False, + max_bytes=targets.listing_bytes, + ) + ], + {"step": "tasks", "key_field": "id", "fields": {"status": KANBAN_STATUSES}}, + rationale="Non-empty and well-formed. An empty board is NOT_RUN, because it cannot distinguish a quiet week from a broken read.", + ), + check( + "surfaces.durable-sessions-render-without-blank-rows", + "Durable sessions are listed with a preview, a source, and an id", + "surfaces", + "lines_match", + [ + step( + "sessions", + SELF, + "hermes", + "sessions", + "list", + "--limit", + "20", + record=False, + ) + ], + {"step": "sessions", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + rationale="A row whose leading column is blank is the session-render regression seen from the data side.", + ), + check( + "surfaces.agent-ui-session-patches-are-deployed", + "The agent surface runs the session-activity rendering patches", + "surfaces", + "names_present", + [ + step( + "init", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.agent_deployment}", + "-o", + INIT_NAME_PROJECTION, + ) + ], + { + "step": "init", + "names": ("patch-web-session-activity", "patch-api-server-sessions"), + }, + rationale="Blank session rows were a rendering regression; this asserts the patches that fixed it are in the running pod.", + ), + check( + "surfaces.chat-session-patches-are-deployed", + "The chat surface runs the API-session continuity patches", + "surfaces", + "names_present", + [ + step( + "init", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.chat_statefulset}", + "-o", + INIT_NAME_PROJECTION, + ) + ], + {"step": "init", "names": targets.chat_init_containers}, + ), + check( + "surfaces.chat-telegram-topic-state-is-durable", + "Telegram topic and tenant state is held on durable storage", + "surfaces", + "names_present", + [ + step( + "claims", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"deploy/{targets.chat_router_deployment}", + "-o", + projection( + 'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}' + ), + ) + ], + {"step": "claims", "contains": ("state",)}, + rationale="Topic selection survives a router restart only if the tenant state file is on a claim rather than in the pod.", + ), + ] + checks.append( + check( + "surfaces.chat-telegram-sessions-are-continuous", + "The selected chat tenant still holds durable Telegram-sourced sessions", + "surfaces", + "lines_match", + [ + step( + "telegram", + CHAT, + "hermes", + "sessions", + "list", + "--source", + "telegram", + "--limit", + "20", + record=False, + ) + ], + {"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.", + ) + ) + checks.append( + check( + "surfaces.chat-runs-the-telegram-topic-revision", + "The chat surface runs the durable Telegram topic configuration", + "surfaces", + "stdout_matches", + [ + step( + "revision", + OPERATOR, + "kubectl", + "--namespace", + targets.namespace, + "get", + f"statefulset/{targets.chat_statefulset}", + "-o", + projection( + "jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}" + ), + ) + ], + {"step": "revision", "equals": targets.chat_config_revision}, + rationale="Telegram topic continuity is carried by the config revision the pod template pins.", + ) + ) + return checks + + +def _ephemeral() -> list[CheckSpec]: + return [ + check( + f"ephemeral.{name}", + title, + "ephemeral", + "not_armed", + [], + {"reason": reason}, + mandatory=False, + scope=EPHEMERAL, + rationale="Mutating checks are separately armed. A default run reports them NOT_RUN and never touches the forge.", + ) + for name, title, reason in ( + ( + "feature-branch-push", + "A unique ephemeral feature branch can be pushed and removed", + "ephemeral mutation mode is not armed", + ), + ( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + "ephemeral mutation mode is not armed", + ), + ( + "protected-branch-refusal", + "Arming against a protected branch is refused before any network call", + "ephemeral mutation mode is not armed", + ), + ( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + "ephemeral mutation mode is not armed", + ), + ) + ] + + +def worker_checks(targets: Targets) -> list[CheckSpec]: + """Return the pool, surface, and ephemeral-placeholder checks.""" + return [*_pool(targets), *_surfaces(targets), *_ephemeral()] diff --git a/scripts/ops/hermes_handoff_ephemeral.py b/scripts/ops/hermes_handoff_ephemeral.py new file mode 100644 index 00000000..ca1ea594 --- /dev/null +++ b/scripts/ops/hermes_handoff_ephemeral.py @@ -0,0 +1,439 @@ +#!/usr/bin/env python3 +"""Discoverable, residue-surfacing, self-cleaning armed ephemeral run. + +The create response — not the index — is the authoritative name of a pull +request this harness made, and the index is read to its last page. When either +source is uncertain the run fails closed and reports the exact residue plus the +exact commands that remove it, so an operator is never left guessing what was +created. + +Arming, provenance, and push-target rules live in :mod:`hermes_handoff_arming` +and are re-exported here for callers that treat armed mode as one surface. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from hermes_handoff_arming import ( + CONFIRMATION, + REF_PREFIX, + ArmingError, + ArmRequest, + assert_push_target_allowed, + bounded_read, + normalise_branch, + preflight, +) +from hermes_handoff_exec import Outcome, Runner, Vantage +from hermes_handoff_model import EPHEMERAL, FAIL, NOT_RUN, PASS, CheckResult, CheckSpec +from hermes_handoff_policy import ( + EXPECTED_BASE, + GITEA_CLIENT, + register_ephemeral_pull, +) +from hermes_handoff_rules import strict_json + +__all__ = [ + "CONFIRMATION", + "DISCOVERY_PAGE_BUDGET", + "DISCOVERY_PAGE_SIZE", + "REF_PREFIX", + "ArmRequest", + "ArmingError", + "assert_push_target_allowed", + "bounded_read", + "normalise_branch", + "preflight", + "run_armed", +] + +DRAFT_TITLE_PREFIX = "WIP: " +DISCOVERY_PAGE_SIZE = 50 +DISCOVERY_PAGE_BUDGET = 20 + + +def _spec(identifier: str, title: str) -> CheckSpec: + return CheckSpec( + id=f"ephemeral.{identifier}", + title=title, + group="ephemeral", + rule="armed", + steps=(), + mandatory=True, + scope=EPHEMERAL, + ) + + +def _result( + identifier: str, + title: str, + status: str, + reason: str, + outcomes: Sequence[Outcome] = (), + evidence: dict[str, Any] | None = None, +) -> CheckResult: + return CheckResult( + spec=_spec(identifier, title), + status=status, + reason=reason, + evidence=evidence or {}, + outcomes=list(outcomes), + ) + + +def _protected_refusal_result() -> CheckResult: + accepted: list[str] = [] + for candidate in ("main", "master", "refs/heads/main", "HEAD", f"{REF_PREFIX}/x"): + try: + assert_push_target_allowed(candidate) + except ArmingError: + continue + accepted.append(candidate) + return _result( + "protected-branch-refusal", + "Arming against a protected branch is refused before any network call", + FAIL if accepted else PASS, + f"guard accepted {accepted}" + if accepted + else "every protected and malformed target was refused", + evidence={"accepted": accepted}, + ) + + +def _detail(outcome: Outcome) -> str: + return ( + outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}" + ) + + +def _pr_problems(payload: Any, request: ArmRequest) -> list[str]: + if not isinstance(payload, dict): + return ["pull request is not an object"] + wanted = { + "state": "open", + "draft": True, + "merged": False, + "base": {"ref": EXPECTED_BASE, "sha": request.expected_base_sha}, + "head": {"ref": request.ref, "sha": request.expected_head}, + } + problems: list[str] = [] + if ( + not isinstance(payload.get("number"), int) + or isinstance(payload.get("number"), bool) + or payload["number"] <= 0 + ): + problems.append("number is malformed") + for field in ("state", "draft", "merged"): + if ( + type(payload.get(field)) is not type(wanted[field]) + or payload.get(field) != wanted[field] + ): + problems.append(f"{field} does not match") + for group, values in (("base", wanted["base"]), ("head", wanted["head"])): + observed = payload.get(group) + if not isinstance(observed, dict): + problems.append(f"{group} is malformed") + continue + for field, value in values.items(): + if observed.get(field) != value: + problems.append(f"{group}.{field} does not match") + return problems + + +def _created_number(outcome: Outcome, request: ArmRequest) -> tuple[int, str]: + """Return the number the create response itself reports. + + The create response is the only source that names a pull request the harness + definitely made. Relying on the index alone means an index that fails or + answers ambiguously leaves an unnamed, unclosed pull request behind. + """ + if not outcome.ok: + return (0, f"pull-request creation failed: {_detail(outcome)}") + try: + payload = strict_json(outcome.stdout) + except ValueError as exc: + return (0, f"pull-request creation returned malformed JSON: {exc}") + problems = _pr_problems(payload, request) + if problems: + return (0, f"created pull request is invalid: {', '.join(problems)}") + return (payload["number"], "") + + +def _index_pages( + runner: Runner, vantage: Vantage, api: str, reserve: float +) -> tuple[list[Outcome], list[Any], str]: + """Read every page of the pull index; a truncated page can hide a match.""" + outcomes: list[Outcome] = [] + rows: list[Any] = [] + for page in range(1, DISCOVERY_PAGE_BUDGET + 1): + outcome = runner.run( + ( + GITEA_CLIENT, + "GET", + f"{api}/pulls?state=all&limit={DISCOVERY_PAGE_SIZE}&page={page}", + ), + vantage, + reserve_seconds=reserve, + ) + outcomes.append(outcome) + if not outcome.ok: + return (outcomes, [], f"discovery page {page} failed: {_detail(outcome)}") + try: + payload = strict_json(outcome.stdout) + except ValueError as exc: + return (outcomes, [], f"discovery page {page} is malformed JSON: {exc}") + if not isinstance(payload, list): + return (outcomes, [], f"discovery page {page} did not return an array") + rows.extend(payload) + if len(payload) < DISCOVERY_PAGE_SIZE: + return (outcomes, rows, "") + return (outcomes, [], "pull-request discovery exceeded its page budget") + + +def _residue(request: ArmRequest, numbers: Sequence[int], api: str) -> dict[str, Any]: + """Name the exact residue and the exact commands that remove it by hand.""" + manual = [ + f"{GITEA_CLIENT} PATCH {api}/pulls/{number} --field state=closed" + for number in numbers + ] + if not numbers: + manual.append( + f"{GITEA_CLIENT} GET {api}/pulls?state=all&limit={DISCOVERY_PAGE_SIZE}" + f" then close by hand every pull whose head ref is {request.ref}" + ) + manual.append(f"{GITEA_CLIENT} DELETE {api}/branches/{request.ref}") + return {"residue_ref": request.ref, "manual_cleanup": manual} + + +def _match_pulls(rows: Sequence[Any], request: ArmRequest) -> tuple[list[int], str]: + matches: list[int] = [] + for index, item in enumerate(rows): + if ( + not isinstance(item, dict) + or not isinstance(item.get("head"), dict) + or not isinstance(item.get("base"), dict) + ): + return ([], f"pull-request discovery row {index + 1} is malformed") + if item["head"].get("ref") != request.ref: + continue + problems = _pr_problems(item, request) + if problems: + return ([], f"discovered pull request is invalid: {', '.join(problems)}") + matches.append(item["number"]) + if len(matches) != 1: + return ( + matches, + f"expected one exact draft pull request, discovered {len(matches)}", + ) + return (matches, "") + + +def _closed_pr_problems(payload: Any, request: ArmRequest) -> list[str]: + """Validate the same exact PR identity with state changed to closed.""" + problems = _pr_problems(payload, request) + if isinstance(payload, dict) and payload.get("state") == "closed": + problems = [ + problem for problem in problems if problem != "state does not match" + ] + elif "state does not match" not in problems: + problems.append("state is not closed") + return problems + + +def _cleanup( + runner: Runner, + vantage: Vantage, + request: ArmRequest, + api: str, + numbers: Sequence[int], + discovery_error: str = "", +) -> CheckResult: + outcomes: list[Outcome] = [] + problems = [discovery_error] if discovery_error else [] + for number in sorted(set(numbers)): + closed = runner.run( + (GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"), + vantage, + ) + outcomes.append(closed) + if not closed.ok: + problems.append(f"pull request #{number} close failed") + continue + try: + payload = strict_json(closed.stdout) + except ValueError: + problems.append(f"pull request #{number} close response is malformed") + continue + expected = _closed_pr_problems(payload, request) + if expected: + problems.append(f"pull request #{number} close response does not match") + deleted = runner.run( + (GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage + ) + outcomes.append(deleted) + if not deleted.ok: + problems.append("ephemeral branch deletion failed") + remaining = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage + ) + outcomes.append(remaining) + if not remaining.ok or remaining.stdout.strip(): + problems.append("ephemeral branch absence could not be verified") + for number in sorted(set(numbers)): + observed = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage) + outcomes.append(observed) + if not observed.ok: + problems.append(f"pull request #{number} closure could not be verified") + continue + try: + payload = strict_json(observed.stdout) + except ValueError: + problems.append(f"pull request #{number} verification is malformed") + continue + expected = _closed_pr_problems(payload, request) + if expected: + problems.append(f"pull request #{number} is not exactly closed") + return _result( + "cleanup-verified", + "Every ephemeral ref and pull request is removed and the removal is verified", + FAIL if problems else PASS, + "; ".join(problems) + if problems + else "the exact branch and draft pull request are gone", + outcomes, + {"ref": request.ref, "pull_requests": sorted(set(numbers))}, + ) + + +def _nothing_created(branch: CheckResult, draft_reason: str) -> list[CheckResult]: + """Report the two results that follow a run which created no artifact.""" + return [ + branch, + _result( + "draft-pull-request", + "A draft pull request can be opened and closed", + NOT_RUN, + draft_reason, + ), + _result( + "cleanup-verified", + "Every ephemeral artifact is removed", + NOT_RUN, + "nothing was created", + ), + ] + + +def run_armed( + runner: Runner, vantage: Vantage, request: ArmRequest +) -> list[CheckResult]: + """Create, discover, validate, and clean one exact ephemeral branch/PR.""" + guard = _protected_refusal_result() + branch_title = "A unique ephemeral feature branch can be pushed and removed" + if guard.status != PASS: + unrun = _result("feature-branch-push", branch_title, NOT_RUN, "guard failed") + return [guard, *_nothing_created(unrun, "protected-ref guard failed")] + api = f"/api/v1/repos/{request.repo}" + reserve = min(6000.0, max(300.0, runner.command_timeout * 10)) + existing = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), + vantage, + reserve_seconds=reserve, + ) + if not existing.ok or existing.stdout.strip(): + reason = ( + _detail(existing) if not existing.ok else "the ephemeral ref already exists" + ) + refused = _result("feature-branch-push", branch_title, FAIL, reason, [existing]) + return [guard, *_nothing_created(refused, "branch preflight failed")] + local_head = runner.run( + ("git", "rev-parse", "HEAD"), vantage, reserve_seconds=reserve + ) + pushed = ( + runner.run( + ("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"), + vantage, + reserve_seconds=reserve, + ) + if local_head.ok and local_head.stdout.strip() == request.expected_head + else Outcome( + argv=("git", "push"), + vantage=vantage.name, + error="local HEAD changed after static preflight", + ) + ) + verified = runner.run( + ("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), + vantage, + reserve_seconds=reserve, + ) + branch_ok = verified.ok and verified.stdout.split() == [ + request.expected_head, + f"refs/heads/{request.ref}", + ] + branch_result = _result( + "feature-branch-push", + branch_title, + PASS if branch_ok else FAIL, + "the exact reviewed HEAD was pushed" + if branch_ok + else "push could not be verified exactly", + [local_head, pushed, verified], + {"ref": request.ref, "head": request.expected_head}, + ) + numbers: list[int] = [] + problems: list[str] = ["branch creation was not verified"] + create_outcomes: list[Outcome] = [] + if branch_ok: + created = runner.run( + ( + GITEA_CLIENT, + "POST", + f"{api}/pulls", + "--field", + f"title={DRAFT_TITLE_PREFIX}Hermes handoff acceptance ephemeral probe", + "--field", + f"head={request.ref}", + "--field", + f"base={EXPECTED_BASE}", + "--field", + "body=Ephemeral acceptance probe. Closed and deleted by the harness.", + ), + vantage, + reserve_seconds=reserve, + ) + number, create_error = _created_number(created, request) + pages, rows, index_error = _index_pages(runner, vantage, api, reserve) + found, match_error = ( + _match_pulls(rows, request) if not index_error else ([], "") + ) + create_outcomes = [created, *pages] + # Either source alone names a pull request this run definitely made: the + # ephemeral ref was proven absent in preflight, so any pull on it is ours. + numbers = sorted({item for item in (number, *found) if item}) + problems = [text for text in (index_error, match_error) if text] + if number and found and found != [number]: + problems.append(f"created #{number} is not the discovered {found}") + if not numbers: + problems.append( + create_error or "no exact draft pull request was identified" + ) + for item in numbers: + register_ephemeral_pull(item) + exact = len(numbers) == 1 and not problems + draft_result = _result( + "draft-pull-request", + "A draft pull request can be opened and closed against the ephemeral branch", + PASS if exact else FAIL, + f"discovered exact draft pull request #{numbers[0]}" + if exact + else "; ".join(problems), + create_outcomes, + {"pull_requests": numbers} + if exact + else {"pull_requests": numbers, **_residue(request, numbers, api)}, + ) + cleanup = _cleanup(runner, vantage, request, api, numbers, "; ".join(problems)) + return [guard, branch_result, draft_result, cleanup] diff --git a/scripts/ops/hermes_handoff_evaluators.py b/scripts/ops/hermes_handoff_evaluators.py new file mode 100644 index 00000000..ea0a21b8 --- /dev/null +++ b/scripts/ops/hermes_handoff_evaluators.py @@ -0,0 +1,497 @@ +#!/usr/bin/env python3 +"""Text-shaped evaluators for the handoff acceptance harness. + +These rules cover evidence that arrives as a refusal, an exit status, or a list +of names. The ``denied`` rule is the load-bearing one: it refuses to classify on +an authorization review alone — ``kubectl auth can-i`` reports what RBAC says, +not what the API server does — so a real request must have been made and +refused before it will pass. + +An absence is only evidence when the probe observed something, so a name step +that exits 0 with no lines is ``NOT_RUN``: drift produces silence, not a pass. + +Importing this module also registers the JSON-shaped rules, so a caller only has +to import one place to have the full catalog vocabulary available. +""" + +from __future__ import annotations + +import re +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_json_rules import evaluate_flux_health # noqa: F401 (registers rules) +from hermes_handoff_model import ATTEMPT, FAIL, NOT_RUN, PASS, REVIEW, CheckSpec +from hermes_handoff_rules import ( + EVALUATORS, + Evaluation, + dotted, + evaluate, + evaluator, + exact_equal, + lines, + looks_denied, + missing_steps, + step_keys, + strict_json, +) + +__all__ = ["EVALUATORS", "Evaluation", "evaluate", "evaluator"] + + +@evaluator("denied") +def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass only when the review says no and a real request was refused.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + attempts = step_keys(spec, ATTEMPT) + if not attempts: + return Evaluation( + NOT_RUN, "deny checks require a real attempt, not only a review" + ) + evidence: dict[str, Any] = {} + for key in step_keys(spec, REVIEW): + verdict = outcomes[key].stdout.strip().lower() + evidence[key] = verdict + if verdict != "no": + return Evaluation( + FAIL, + f"authorization review {key} reported {verdict or 'nothing'}", + evidence, + ) + for key in attempts: + outcome = outcomes[key] + if outcome.ok: + return Evaluation( + FAIL, f"attempt {key} succeeded but must be refused", evidence + ) + if not looks_denied(outcome): + return Evaluation( + NOT_RUN, + f"attempt {key} failed without a recognisable refusal (rc={outcome.returncode})", + evidence, + ) + evidence[key] = "refused" + return Evaluation(PASS, "review and live attempt were both refused", evidence) + + +@evaluator("review_denied") +def evaluate_review_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass only when every side-effect-free authorization review says no.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + reviews = step_keys(spec, REVIEW) + if not reviews: + return Evaluation(NOT_RUN, "a denial check requires an authorization review") + evidence: dict[str, str] = {} + for key in reviews: + verdict = outcomes[key].stdout.strip().lower() + evidence[key] = verdict + if verdict != "no": + return Evaluation( + FAIL, + f"authorization review {key} reported {verdict or 'nothing'}", + evidence, + ) + return Evaluation(PASS, "every authorization review was denied", evidence) + + +@evaluator("allowed") +def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every required step succeeded and every review said yes.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + evidence: dict[str, Any] = {} + for step in spec.steps: + outcome = outcomes[step.key] + if step.kind == REVIEW: + verdict = outcome.stdout.strip().lower() + evidence[step.key] = verdict + if verdict != "yes": + return Evaluation( + FAIL, + f"authorization review {step.key} reported {verdict or 'nothing'}", + evidence, + ) + continue + if not outcome.ok: + return Evaluation( + FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence + ) + evidence[step.key] = "succeeded" + return Evaluation(PASS, "required access is present", evidence) + + +def _matched(observed: set[str], patterns: tuple[str, ...]) -> set[str]: + return {name for pattern in patterns for name in observed if pattern in name} + + +def _line_step( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> tuple[set[str], str] | Evaluation: + """Return a step's distinct lines, or the NOT_RUN that silence must produce.""" + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + observed = set(lines(outcome)) + if not observed: + return Evaluation(NOT_RUN, f"step {key} exited 0 with no lines to examine") + return (observed, key) + + +@evaluator("names_absent") +def evaluate_names_absent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when a step observed names and none of them is forbidden.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + resolved = _line_step(spec, outcomes) + if isinstance(resolved, Evaluation): + return resolved + observed, _ = resolved + offenders = sorted( + (observed & set(spec.expect.get("names", ()))) + | _matched(observed, tuple(spec.expect.get("contains", ()))) + ) + evidence = {"observed_count": len(observed), "offenders": offenders} + if offenders: + return Evaluation( + FAIL, f"forbidden names present: {', '.join(offenders)}", evidence + ) + return Evaluation(PASS, "no forbidden names are present", evidence) + + +@evaluator("names_present") +def evaluate_names_present(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every required name, and every required substring, appears.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + resolved = _line_step(spec, outcomes) + if isinstance(resolved, Evaluation): + return resolved + observed, _ = resolved + missing = sorted(set(spec.expect.get("names", ())) - observed) + missing += sorted( + pattern + for pattern in spec.expect.get("contains", ()) + if not any(pattern in name for name in observed) + ) + evidence = {"observed_count": len(observed), "missing": missing} + if missing: + return Evaluation( + FAIL, f"required names absent: {', '.join(missing)}", evidence + ) + return Evaluation(PASS, "every required name is present", evidence) + + +@evaluator("stdout_matches") +def evaluate_stdout_matches( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: + """Compare a step's trimmed stdout against an expected value or set.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + actual = outcome.stdout.strip() + expected = spec.expect["equals"] + allowed = expected if isinstance(expected, list) else [expected] + evidence = {"observed": actual} + if actual not in allowed: + return Evaluation( + FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence + ) + return Evaluation(PASS, "output matches the expected value", evidence) + + +@evaluator("distinct_count") +def evaluate_distinct_count( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: + """Pass when a step yields enough distinct rows — and, optionally, exactly N. + + Distinctness is the point for placement checks: three worker pods pinned to + one node satisfy a replica count and defeat the isolation the replica count + was standing in for. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + values = lines(outcome) + distinct = sorted(set(values)) + minimum = spec.expect["minimum"] + evidence = { + "total": len(values), + "distinct": len(distinct), + "values": distinct[:16], + } + if not values: + return Evaluation(NOT_RUN, f"step {key} produced no rows to count", evidence) + if len(distinct) < minimum: + return Evaluation( + FAIL, + f"{len(distinct)} distinct values, expected at least {minimum}", + evidence, + ) + total_equals = spec.expect.get("total_equals") + if total_equals is not None and len(values) != total_equals: + return Evaluation( + FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence + ) + return Evaluation(PASS, f"{len(distinct)} distinct values observed", evidence) + + +@evaluator("lines_match") +def evaluate_lines_match(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every data row of a tabular listing matches a shape. + + Some surfaces only speak tables. A blank leading column is exactly what the + session-render regression looked like, so asserting the shape of each row is + a real check rather than a proxy for one. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + pattern = re.compile(spec.expect["pattern"]) + rows = outcome.stdout.splitlines()[spec.expect.get("skip", 0) :] + rows = [row for row in rows if row.strip()] + offenders = [row[:60] for row in rows if not pattern.search(row)] + evidence = {"rows": len(rows), "offenders": offenders[:8]} + if len(rows) < spec.expect.get("minimum", 1): + return Evaluation( + NOT_RUN, + f"step {key} produced {len(rows)} data rows, expected at least " + f"{spec.expect.get('minimum', 1)}", + evidence, + ) + if offenders: + return Evaluation( + FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence + ) + return Evaluation(PASS, f"all {len(rows)} rows match the expected shape", evidence) + + +@evaluator("vantages_agree") +def evaluate_vantages_agree( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: + """Pass only when two independent observations report the same fact. + + A disagreement is a finding in its own right: it usually means the manifest + on record and the workload actually running have drifted apart, and either + one alone would have looked clean. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + left_key, right_key = spec.expect["steps"] + observations: dict[str, list[str]] = {} + for key in (left_key, right_key): + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + observations[key] = sorted(lines(outcome)) + evidence = {key: value[:16] for key, value in observations.items()} + if observations[left_key] != observations[right_key]: + return Evaluation( + FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence + ) + if not observations[left_key]: + return Evaluation( + NOT_RUN, "both vantages returned nothing to compare", evidence + ) + return Evaluation(PASS, "both vantages report the same state", evidence) + + +@evaluator("not_armed") +def evaluate_not_armed(spec: CheckSpec, _outcomes: dict[str, Outcome]) -> Evaluation: + """Report the fixed NOT_RUN a default, read-only run must produce.""" + reason = spec.expect.get("reason", "ephemeral mutation mode is not armed") + return Evaluation(NOT_RUN, reason, {"armed": False}) + + +# This validator is one compact, linear state machine so every exit observes +# only fully validated state. +# fmt: off +@evaluator("pool_topology") +def evaluate_pool_topology(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Require exactly three Ready ordinal workers on distinct nodes and claims.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + state = outcomes[spec.expect["state_step"]].stdout.strip().split("\t") + if len(state) != 4: + return Evaluation(NOT_RUN, "StatefulSet projection is malformed") + expected = spec.expect.get("replicas", 3) + try: + replicas, ready, current = (int(value) for value in state[:3]) + except ValueError: + return Evaluation(NOT_RUN, "StatefulSet replica fields are malformed") + evidence: dict[str, Any] = {"replicas": replicas, "ready": ready, "current": current, + "automountServiceAccountToken": state[3]} + if (replicas, ready, current, state[3].lower()) != (expected, expected, expected, "false"): + return Evaluation(FAIL, "worker replica or tokenless state does not match the contract", evidence) + workers: dict[int, tuple[str, str]] = {} + names: list[str] = [] + for number, row in enumerate(lines(outcomes[spec.expect["worker_step"]]), 1): + columns = row.split("\t") + if len(columns) != 5: + return Evaluation(NOT_RUN, f"worker row {number} is malformed", evidence) + name, node, phase, ready_state, claims = columns + match = re.fullmatch(rf"{re.escape(spec.expect['name'])}-(\d+)", name) + claim_list = [claim for claim in claims.split(",") if claim] + if (not match or not node or phase != "Running" or ready_state != "True" + or len(claim_list) != 1): + return Evaluation(FAIL, f"worker row {number} is not a Ready isolated worker", evidence) + ordinal = int(match.group(1)) + if ordinal in workers: + return Evaluation(FAIL, f"worker ordinal {ordinal} is duplicated", evidence) + workers[ordinal] = (node, claim_list[0]) + names.append(name) + evidence.update({"workers": sorted(names), "nodes": sorted({v[0] for v in workers.values()}), + "claims": sorted({v[1] for v in workers.values()})}) + if set(workers) != set(range(expected)): + return Evaluation(FAIL, f"worker ordinals are not exactly 0..{expected - 1}", evidence) + if (len({v[0] for v in workers.values()}) != expected + or len({v[1] for v in workers.values()}) != expected): + return Evaluation(FAIL, "workers do not occupy distinct nodes and claims", evidence) + return Evaluation(PASS, "three Ready workers have distinct nodes, ordinals, and claims", evidence) +# fmt: on + + +_SHA_RE = re.compile(r"[0-9a-f]{40}\Z") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}\Z") +_IMAGE_RE = re.compile( + r"[^@\s]+:git-([0-9a-f]{40})-build-([1-9][0-9]*)@(sha256:[0-9a-f]{64})\Z" +) + + +def _strict_object(outcome: Outcome, label: str) -> dict[str, Any] | Evaluation: + try: + payload = strict_json(outcome.stdout) + except ValueError as exc: + return Evaluation(NOT_RUN, f"{label} JSON is malformed: {exc}") + if not isinstance(payload, dict): + return Evaluation(NOT_RUN, f"{label} JSON is not an object") + return payload + + +# fmt: off +@evaluator("release_lineage") +def evaluate_release_lineage(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Atomically bind Git, PR, image, build, Flux, and running revision evidence.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + expected = spec.expect + main_sha = expected["main_sha"] + head_sha = expected["head_sha"] + image = expected["image"] + build_sha = expected["build_sha"] + revision = str(expected["deployment_revision"]) + digest = image.rsplit("@", 1)[-1] + image_match = _IMAGE_RE.fullmatch(image) + if not (_SHA_RE.fullmatch(main_sha) and _SHA_RE.fullmatch(head_sha) + and _SHA_RE.fullmatch(build_sha) and _DIGEST_RE.fullmatch(digest) + and image_match and image_match.group(1) == build_sha and build_sha == main_sha): + return Evaluation(NOT_RUN, "release lineage expectations are malformed") + remote = outcomes["remote-main"].stdout.strip().split() + if remote != [main_sha, "refs/heads/main"]: + return Evaluation(FAIL, "remote main does not match the fixed expected SHA", + {"remote_main": remote[:2]}) + pr = _strict_object(outcomes["reviewed-pr"], "reviewed PR") + if isinstance(pr, Evaluation): + return pr + fields = { + "number": expected["pr_number"], + "state": "open", + "draft": True, + "merged": False, + "base.ref": "main", + "base.sha": main_sha, + "head.ref": expected["head_ref"], + "head.sha": head_sha, + } + for path, wanted in fields.items(): + try: + actual = dotted(pr, path) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"reviewed PR is missing {path}") + if not exact_equal(actual, wanted): + return Evaluation(FAIL, f"reviewed PR {path} does not match", {path: actual}) + if outcomes["build-source"].stdout.strip() != build_sha: + return Evaluation(FAIL, "build source SHA does not match the fixed release build SHA") + deployment = outcomes["deployment"].stdout.strip().split("\t") + if len(deployment) != 6: + return Evaluation(NOT_RUN, "Deployment projection is malformed") + generation, observed, replicas, ready, deployed_revision, deployed_image = deployment + try: + current = (int(generation) > 0 and generation == observed + and int(replicas) > 0 and replicas == ready) + except ValueError: + return Evaluation(NOT_RUN, "Deployment numeric fields are malformed") + if not current or (deployed_revision, deployed_image) != (revision, image): + return Evaluation(FAIL, "Deployment lineage does not match the release contract") + replica_hash = "" + for number, row in enumerate(lines(outcomes["replicasets"]), 1): + columns = row.split("\t") + if len(columns) != 6: + return Evaluation(NOT_RUN, f"ReplicaSet row {number} is malformed") + _, rs_revision, rs_ready, rs_available, rs_image, rs_hash = columns + if rs_revision != revision: + continue + if (rs_ready != replicas or rs_available != replicas or rs_image != image or not rs_hash): + return Evaluation(FAIL, "active ReplicaSet does not match the Deployment lineage") + if replica_hash: + return Evaluation(FAIL, "more than one ReplicaSet claims the running revision") + replica_hash = rs_hash + if not replica_hash: + return Evaluation(FAIL, "running Deployment revision has no matching ReplicaSet") + pod_rows = lines(outcomes["pods"]) + if len(pod_rows) != int(replicas): + return Evaluation(FAIL, "running pod count does not match the Deployment") + pods: list[str] = [] + for number, row in enumerate(pod_rows, 1): + columns = row.split("\t") + if len(columns) != 6: + return Evaluation(NOT_RUN, f"pod row {number} is malformed") + name, phase, ready_state, pod_image, image_id, pod_hash = columns + if phase != "Running" or ready_state != "True": + return Evaluation(FAIL, f"pod {name or number} is not Ready") + if (pod_image, pod_hash) != (image, replica_hash) or not image_id.endswith(f"@{digest}"): + return Evaluation(FAIL, f"pod {name or number} lineage does not match") + pods.append(name) + flux = outcomes["flux"].stdout.strip().split("\t") + if len(flux) != 6: + return Evaluation(NOT_RUN, "Flux lineage projection is malformed") + name, generation, observed, suspended, ready_state, applied = flux + if (not name or generation != observed or suspended.lower() == "true" or ready_state != "True"): + return Evaluation(FAIL, "Flux source is not current and Ready") + match = re.search(r"sha1:([0-9a-f]{40})\Z", applied) + if not match or match.group(1) != main_sha: + return Evaluation(FAIL, "Flux applied revision is not the expected remote main") + return Evaluation(PASS, "Git, PR, image, build, Flux, and running revision are bound", + {"main_sha": main_sha, "head_sha": head_sha, "image_digest": digest, + "build_sha": build_sha, "deployment_revision": revision, "pods": pods}) +# fmt: on + + +# Two observations of the same fact agree whether or not they came from +# different vantages; the dual-vantage case is just the important one. +EVALUATORS["steps_agree"] = evaluate_vantages_agree diff --git a/scripts/ops/hermes_handoff_exec.py b/scripts/ops/hermes_handoff_exec.py new file mode 100644 index 00000000..d3edbe09 --- /dev/null +++ b/scripts/ops/hermes_handoff_exec.py @@ -0,0 +1,482 @@ +#!/usr/bin/env python3 +"""Bounded, screened, provenance-attested command execution. + +It bounds pipes, times out and reaps the process group, and discards truncated output. +""" + +from __future__ import annotations + +import hashlib +import math +import os +import selectors +import shlex +import shutil +import signal +import stat +import subprocess +import time +from collections.abc import Callable, Mapping, Sequence +from contextlib import suppress +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from hermes_handoff_policy import GITEA_CLIENT, READ_ONLY, PolicyError, check_argv +from hermes_handoff_redaction import ( + DEFAULT_MAX_BYTES, + TRUNCATION_NOTE, + redact, + safe_text, + scrub, +) + +DEFAULT_COMMAND_TIMEOUT = 25.0 +DEFAULT_RUN_DEADLINE = 900.0 +DEFAULT_CLEANUP_RESERVE = 90.0 +MAX_COMMAND_TIMEOUT = 600.0 +MAX_RUN_DEADLINE = 7200.0 +MAX_OUTPUT_BYTES = 8 * 1024 * 1024 +READ_CHUNK = 64 * 1024 +PIPE_DRAIN_SECONDS = 0.25 + +SAFE_PATH = "/opt/data/tools/bin:/opt/hermes/.venv/bin:/opt/coordinator:/usr/local/bin:/usr/bin:/bin" +# These exact allowlists are intentionally dense: reviewers compare them as a +# single provenance boundary rather than as extensible application config. +# fmt: off +ENV_PASSTHROUGH = ("KUBECONFIG", "LANG", "LC_ALL", "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE") +FIXED_ENVIRONMENT = {"GIT_ASKPASS": "/opt/coordinator/gitea_askpass.sh", + "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", "GITEA_BASE_URL": "https://scm.bstein.dev", + "LC_ALL": "C", "PATH": SAFE_PATH} +EXPECTED_PATHS = { + "kubectl": {"/opt/data/tools/bin/kubectl", "/usr/local/bin/kubectl", "/usr/bin/kubectl"}, + "git": {"/usr/bin/git", "/bin/git"}, + "hermes": {"/opt/hermes/.venv/bin/hermes"}, + "sh": {"/usr/bin/sh", "/bin/sh"}, + GITEA_CLIENT: {GITEA_CLIENT}, +} +EXPECTED_SHA256 = { + "kubectl": {"3d514dbae5dc8c09f773df0ef0f5d449dfad05b3aca5c96b13565f886df345fd"}, + "git": {"a0e562e4bd3c4c79379e91d8c07a10104b2cefe8fac966dc6bd4874a57a807f3"}, + "sh": {"367967c823a0c391e5049b15a67c6a0a629c88b9b6dcdca75ef13ac9d65334b1"}, + "hermes": {"c8419290ef7f1a95f59eadcea9f58f446b1e6535a7f4ffee9f1c0cb1172335e4"}, + GITEA_CLIENT: {"f0943db4e81ce96908e94b0f0e78aa4b5ef95d6c4f7803f9f74a43e761576d85"}, +} + +DEADLINE_ERROR = "deadline-exceeded" +TIMEOUT_ERROR = "timeout" +OPERATOR = "operator" +SELF = "self" +CONTEXT_AWARE_BINARIES = ("kubectl",) +POD_COMMAND_PATHS = {"kubectl": "/usr/local/bin/kubectl", "hermes": "/opt/hermes/.venv/bin/hermes", + "git": "/usr/bin/git", "sh": "/bin/sh", GITEA_CLIENT: GITEA_CLIENT} +# fmt: on + + +@dataclass(frozen=True) +class Attestation: + """The exact outer executable used for a probe.""" + + path: str + sha256: str + + +@dataclass(frozen=True) +class Capture: + """Raw bounded process result returned by an executor implementation.""" + + returncode: int + stdout: bytes = b"" + stderr: bytes = b"" + stdout_truncated: bool = False + stderr_truncated: bool = False + timed_out: bool = False + + +@dataclass(frozen=True) +class Outcome: + """The screened, bounded result of one probe command.""" + + argv: tuple[str, ...] + vantage: str + returncode: int | None = None + stdout: str = "" + stderr: str = "" + truncated: bool = False + duration_ms: int = 0 + error: str | None = None + executable_path: str = "" + executable_sha256: str = "" + + @property + def ok(self) -> bool: + return self.error is None and self.returncode == 0 and not self.truncated + + @property + def ran(self) -> bool: + return self.error is None + + @property + def combined(self) -> str: + return "\n".join(part for part in (self.stdout, self.stderr) if part) + + def as_dict(self) -> dict[str, object]: + return scrub( + { + "command": redact(shlex.join(self.argv)), + "vantage": self.vantage, + "returncode": self.returncode, + "stdout": self.stdout, + "stderr": self.stderr, + "truncated": self.truncated, + "duration_ms": self.duration_ms, + "error": self.error, + "executable_path": self.executable_path, + "executable_sha256": self.executable_sha256, + } + ) + + +@dataclass(frozen=True) +class Vantage: + """A named identity and transport used to observe a probe.""" + + name: str + prefix: tuple[str, ...] = () + inserts: Mapping[str, tuple[str, ...]] = field(default_factory=dict) + env: Mapping[str, str] = field(default_factory=dict) + command_paths: Mapping[str, str] = field(default_factory=dict) + description: str = "" + + def wrap(self, argv: Sequence[str]) -> tuple[str, ...]: + command = tuple(argv) + if command: + replacement = self.command_paths.get(command[0]) + if replacement: + command = (replacement, *command[1:]) + extra = self.inserts.get(command[0].rsplit("/", 1)[-1]) + if extra: + command = (command[0], *extra, *command[1:]) + return (*self.prefix, *command) + + +def _finite_number(name: str, value: Any, minimum: float, maximum: float) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + converted = float(value) + if not math.isfinite(converted) or not minimum <= converted <= maximum: + raise ValueError( + f"{name} must be finite and between {minimum:g} and {maximum:g}" + ) + return converted + + +def _byte_budget(value: Any) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 1 <= value <= MAX_OUTPUT_BYTES + ): + raise ValueError( + f"max_bytes must be an integer between 1 and {MAX_OUTPUT_BYTES}" + ) + return value + + +def build_environment( + vantage: Vantage, source: Mapping[str, str] | None = None +) -> dict[str, str]: + """Build a deterministic child environment without caller-controlled PATH.""" + origin = os.environ if source is None else source + environment = {name: origin[name] for name in ENV_PASSTHROUGH if name in origin} + environment.update(FIXED_ENVIRONMENT) + environment.update(vantage.env) + environment["PATH"] = SAFE_PATH + environment["GITEA_BASE_URL"] = FIXED_ENVIRONMENT["GITEA_BASE_URL"] + environment["GIT_ASKPASS"] = FIXED_ENVIRONMENT["GIT_ASKPASS"] + return environment + + +# fmt: off +def operator_vantage(kubeconfig: str | None = None, context: str | None = None) -> Vantage: + inserts = dict.fromkeys(CONTEXT_AWARE_BINARIES, ("--context", context)) if context else {} + detail = " ".join(part for part in (f"context={context}" if context else "", + "kubeconfig-pinned" if kubeconfig else "") if part) + return Vantage(name=OPERATOR, inserts=inserts, + env={"KUBECONFIG": kubeconfig} if kubeconfig else {}, + description=f"operator {detail}".strip()) + + +def pod_vantage(namespace: str, pod: str, container: str, operator: Vantage | None = None) -> Vantage: + reach = operator.inserts.get("kubectl", ()) if operator else () + return Vantage(name=SELF, prefix=("kubectl", *reach, "exec", "--namespace", namespace, + pod, "--container", container, "--"), + env=dict(operator.env) if operator else {}, command_paths=POD_COMMAND_PATHS, + description=f"in-pod {namespace}/{pod}[{container}]") +# fmt: on + + +def attest_executable(command: str, environment: Mapping[str, str]) -> Attestation: + """Resolve and hash an executable from a fixed, trusted path set.""" + name = command if command == GITEA_CLIENT else command.rsplit("/", 1)[-1] + candidate = ( + command if "/" in command else shutil.which(command, path=environment["PATH"]) + ) + if not candidate: + raise OSError(f"trusted executable is unavailable: {name}") + resolved = str(Path(candidate).resolve(strict=True)) + allowed = { + str(Path(item).resolve(strict=False)) + for item in EXPECTED_PATHS.get(name, set()) + } + if resolved not in allowed: + raise OSError(f"executable resolved outside its trusted paths: {name}") + details = os.stat(resolved) + if ( + not stat.S_ISREG(details.st_mode) + or details.st_mode & 0o022 + or not details.st_mode & 0o111 + ): + raise OSError(f"trusted executable has unsafe ownership mode: {name}") + digest = hashlib.sha256() + with open(resolved, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + observed = digest.hexdigest() + expected = EXPECTED_SHA256.get(name) + if not expected or observed not in expected: + raise OSError(f"executable digest is not release-attested: {name}") + return Attestation(resolved, observed) + + +def _append_bounded(buffer: bytearray, chunk: bytes, limit: int) -> bool: + remaining = limit + 1 - len(buffer) + if remaining > 0: + buffer.extend(chunk[:remaining]) + return len(buffer) > limit or len(chunk) > remaining + + +def _signal_group(process: subprocess.Popen[bytes], signum: int) -> None: + with suppress(ProcessLookupError): + os.killpg(process.pid, signum) + + +def execute_bounded( + argv: Sequence[str], + environment: Mapping[str, str], + timeout: float, + max_bytes: int, + clock: Callable[[], float] = time.monotonic, + popen: Callable[..., subprocess.Popen[bytes]] = subprocess.Popen, +) -> Capture: + """Execute one process with bounded live pipe capture and group cleanup.""" + process = popen( + list(argv), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=dict(environment), + text=False, + close_fds=True, + start_new_session=True, + ) + streams = {"stdout": process.stdout, "stderr": process.stderr} + buffers = {"stdout": bytearray(), "stderr": bytearray()} + truncated = {"stdout": False, "stderr": False} + selector = selectors.DefaultSelector() + for name, stream in streams.items(): + if ( + stream is not None + ): # pragma: no branch - PIPE was requested for both streams + os.set_blocking(stream.fileno(), False) + selector.register(stream, selectors.EVENT_READ, name) + expires = clock() + timeout + timed_out = False + try: + while process.poll() is None: + remaining = expires - clock() + if remaining <= 0: + timed_out = True + break + for key, _ in selector.select(min(remaining, 0.1)): + try: + chunk = os.read(key.fileobj.fileno(), READ_CHUNK) + except BlockingIOError: # pragma: no cover - readiness race, retry only + continue + if not chunk: + selector.unregister(key.fileobj) + continue + name = key.data + truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes) + _signal_group(process, signal.SIGKILL if timed_out else signal.SIGTERM) + drain_until = clock() + PIPE_DRAIN_SECONDS + while selector.get_map() and clock() < drain_until: + for key, _ in selector.select(max(0.0, drain_until - clock())): + try: + chunk = os.read(key.fileobj.fileno(), READ_CHUNK) + except BlockingIOError: # pragma: no cover - readiness race, retry only + continue + if not chunk: + selector.unregister(key.fileobj) + continue + name = key.data + truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes) + _signal_group(process, signal.SIGKILL) + returncode = process.wait(timeout=PIPE_DRAIN_SECONDS) + finally: + selector.close() + for stream in streams.values(): + if ( + stream is not None + ): # pragma: no branch - PIPE was requested for both streams + stream.close() + if ( + process.poll() is None + ): # pragma: no branch - fail-safe for wait/selector exceptions + _signal_group(process, signal.SIGKILL) + process.wait() + return Capture( + returncode=returncode, + stdout=bytes(buffers["stdout"][:max_bytes]), + stderr=bytes(buffers["stderr"][:max_bytes]), + stdout_truncated=truncated["stdout"], + stderr_truncated=truncated["stderr"], + timed_out=timed_out, + ) + + +def _injected_executor( + spawn: Callable[..., subprocess.CompletedProcess[Any]], +) -> Callable[..., Capture]: + """Adapt deterministic test doubles without weakening the production path.""" + + def execute( + argv: Sequence[str], + environment: Mapping[str, str], + timeout: float, + max_bytes: int, + ) -> Capture: + completed = spawn( + list(argv), + capture_output=True, + text=True, + timeout=timeout, + check=False, + env=dict(environment), + ) + stdout = str(completed.stdout or "").encode() + stderr = str(completed.stderr or "").encode() + return Capture( + completed.returncode, + stdout[:max_bytes], + stderr[:max_bytes], + len(stdout) > max_bytes, + len(stderr) > max_bytes, + ) + + return execute + + +def _screen_capture(data: bytes, truncated: bool, budget: int) -> str: + if truncated: + return TRUNCATION_NOTE.encode()[:budget].decode() + return safe_text(data.decode("utf-8", "replace"), budget)[0] + + +class Runner: + """Run probes under policy, provenance, timeout, deadline, and byte budgets.""" + + def __init__( + self, + mode: str = READ_ONLY, + command_timeout: float = DEFAULT_COMMAND_TIMEOUT, + deadline_seconds: float = DEFAULT_RUN_DEADLINE, + max_bytes: int = DEFAULT_MAX_BYTES, + clock: Callable[[], float] = time.monotonic, + executor: Callable[..., Capture] | None = None, + spawn: Callable[..., subprocess.CompletedProcess[Any]] | None = None, + environ: Mapping[str, str] | None = None, + attestor: Callable[[str, Mapping[str, str]], Attestation] = attest_executable, + ) -> None: + self.mode = mode + self.command_timeout = _finite_number( + "command_timeout", command_timeout, 0.05, MAX_COMMAND_TIMEOUT + ) + deadline = _finite_number( + "deadline_seconds", deadline_seconds, 0.05, MAX_RUN_DEADLINE + ) + self.max_bytes = _byte_budget(max_bytes) + self._clock = clock + self._executor = executor or ( + _injected_executor(spawn) if spawn else execute_bounded + ) + self._environ = environ + self._attestor = attestor + self._deadline_at = clock() + deadline + self.commands_run = 0 + + @property + def remaining_seconds(self) -> float: + return self._deadline_at - self._clock() + + def run( + self, + argv: Sequence[str], + vantage: Vantage, + max_bytes: int | None = None, + reserve_seconds: float = 0.0, + ) -> Outcome: + budget_bytes = self.max_bytes if max_bytes is None else _byte_budget(max_bytes) + reserve = _finite_number( + "reserve_seconds", reserve_seconds, 0.0, MAX_RUN_DEADLINE + ) + addressed = vantage.wrap(argv) + try: + check_argv(addressed, self.mode) + except PolicyError as exc: + return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}") + budget = min(self.command_timeout, self.remaining_seconds - reserve) + if budget <= 0: + return Outcome(argv=addressed, vantage=vantage.name, error=DEADLINE_ERROR) + environment = build_environment(vantage, self._environ) + try: + attestation = self._attestor(addressed[0], environment) + except (OSError, ValueError) as exc: + return Outcome( + argv=addressed, vantage=vantage.name, error=f"attestation: {exc}" + ) + executed = (attestation.path, *addressed[1:]) + try: + check_argv(executed, self.mode) + except PolicyError as exc: + return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}") + started = self._clock() + self.commands_run += 1 + try: + capture = self._executor(executed, environment, budget, budget_bytes) + except subprocess.TimeoutExpired: + capture = Capture(-signal.SIGKILL, timed_out=True) + except OSError as exc: + return Outcome( + argv=addressed, + vantage=vantage.name, + error=f"spawn: {exc.strerror or type(exc).__name__}", + executable_path=attestation.path, + executable_sha256=attestation.sha256, + ) + was_truncated = capture.stdout_truncated or capture.stderr_truncated + stdout = _screen_capture(capture.stdout, capture.stdout_truncated, budget_bytes) + stderr = _screen_capture(capture.stderr, capture.stderr_truncated, budget_bytes) + return Outcome( + argv=addressed, + vantage=vantage.name, + returncode=capture.returncode, + stdout=stdout, + stderr=stderr, + truncated=was_truncated, + duration_ms=int((self._clock() - started) * 1000), + error=TIMEOUT_ERROR if capture.timed_out else None, + executable_path=attestation.path, + executable_sha256=attestation.sha256, + ) diff --git a/scripts/ops/hermes_handoff_json_rules.py b/scripts/ops/hermes_handoff_json_rules.py new file mode 100644 index 00000000..0f44c6a2 --- /dev/null +++ b/scripts/ops/hermes_handoff_json_rules.py @@ -0,0 +1,474 @@ +#!/usr/bin/env python3 +"""JSON-shaped evaluators for the handoff acceptance harness. + +These rules cover the evidence that arrives as structured output — Kubernetes +objects, forge pull requests, provider health files, and the two append-only +evidence logs the platform keeps. They share the fail-closed contract of every +other rule: an absent path, an unparsable payload, or an empty result set is +``NOT_RUN``, never a quiet pass. +""" + +from __future__ import annotations + +import datetime as dt +import math +import re +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_model import FAIL, NOT_RUN, PASS, CheckSpec +from hermes_handoff_rules import ( + Evaluation, + dotted, + evaluator, + exact_equal, + is_list, + missing_steps, + parsed_step, + strict_json, +) + +_ROUTE_PART_RE = re.compile(r"[a-z0-9][a-z0-9_.-]{0,63}\Z") +_FLUX_NAME_RE = re.compile( + r"[a-z0-9]([-a-z0-9.]*[a-z0-9])?/[a-z0-9]([-a-z0-9.]*[a-z0-9])?\Z" +) + + +def _timestamp(value: Any) -> dt.datetime: + if not isinstance(value, str) or not value.strip(): + raise ValueError("timestamp is not a non-empty string") + observed = dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + if observed.tzinfo is None: + raise ValueError("timestamp has no timezone") + return observed.astimezone(dt.timezone.utc) + + +def _now(value: Any) -> dt.datetime: + if not isinstance(value, dt.datetime) or value.tzinfo is None: + raise ValueError("run clock is not timezone-aware") + return value.astimezone(dt.timezone.utc) + + +@evaluator("json_field") +def evaluate_json_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Compare dotted JSON paths of one step's output against expected values.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + evidence: dict[str, Any] = {} + mismatches: list[str] = [] + for path, expected in spec.expect["fields"].items(): + try: + actual = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) + evidence[path] = actual + allowed = expected if isinstance(expected, list) else [expected] + if not any(exact_equal(actual, item) for item in allowed): + mismatches.append(f"{path}={actual!r} expected one of {allowed!r}") + if mismatches: + return Evaluation(FAIL, "; ".join(mismatches), evidence) + return Evaluation(PASS, "observed fields match the release contract", evidence) + + +@evaluator("json_numeric") +def evaluate_json_numeric(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Compare numeric JSON paths against inclusive minimum/maximum bounds.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + evidence: dict[str, Any] = {} + problems: list[str] = [] + for path, bounds in spec.expect["fields"].items(): + try: + actual = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) + if ( + not isinstance(actual, (int, float)) + or isinstance(actual, bool) + or not math.isfinite(float(actual)) + ): + return Evaluation( + NOT_RUN, f"path {path} is not numeric in step {key}", evidence + ) + evidence[path] = actual + if "min" in bounds and actual < bounds["min"]: + problems.append(f"{path}={actual} below minimum {bounds['min']}") + if "max" in bounds and actual > bounds["max"]: + problems.append(f"{path}={actual} above maximum {bounds['max']}") + if problems: + return Evaluation(FAIL, "; ".join(problems), evidence) + return Evaluation(PASS, "numeric fields are within the expected bounds", evidence) + + +@evaluator("json_recent") +def evaluate_json_recent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when a timestamp field is fresh enough to be evidence about now. + + Stale provider or routing evidence is worse than none: it describes a state + the release no longer has. A timestamp older than the allowed age is a FAIL, + and one the harness cannot parse is NOT_RUN. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + try: + now = _now(spec.expect["now"]) + except (TypeError, ValueError) as exc: + return Evaluation(NOT_RUN, f"invalid run clock: {exc}") + evidence: dict[str, Any] = {} + stale: list[str] = [] + for path, max_age in spec.expect["fields"].items(): + try: + raw = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) + try: + observed = _timestamp(raw) + except (TypeError, ValueError): + return Evaluation( + NOT_RUN, f"path {path} is not a timezone-aware ISO timestamp", evidence + ) + age = (now - observed).total_seconds() + evidence[path] = {"age_seconds": int(age)} + if age < 0: + stale.append(f"{path} is {abs(int(age))}s in the future") + elif age > max_age: + stale.append(f"{path} is {int(age)}s old, limit {max_age}s") + if stale: + return Evaluation(FAIL, "; ".join(stale), evidence) + return Evaluation(PASS, "observed timestamps are fresh", evidence) + + +@evaluator("all_of_field") +def evaluate_all_of_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Pass when every selected element of a JSON array matches expected fields.""" + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + try: + elements = dotted(payload, spec.expect.get("array", "")) + except (KeyError, ValueError, IndexError): + return Evaluation(NOT_RUN, f"array path is absent from step {key}") + if not is_list(elements): + return Evaluation(NOT_RUN, f"step {key} did not return an array") + key_field = spec.expect.get("key_field", "") + wanted = spec.expect.get("keys") + exempt = set(spec.expect.get("exempt_keys", ())) + seen: list[Any] = [] + problems: list[str] = [] + evidence: dict[str, Any] = {} + for element in elements: + identity = dotted(element, key_field) if key_field else None + if wanted is not None and identity not in wanted: + continue + if identity in exempt: + continue + seen.append(identity) + for path in spec.expect.get("non_empty", ()): + try: + value = dotted(element, path) + except (KeyError, ValueError, IndexError): + value = None + if value in (None, "", [], {}): + problems.append(f"{identity}: {path} is empty") + for path, expected in spec.expect.get("fields", {}).items(): + try: + actual = dotted(element, path) + except (KeyError, ValueError, IndexError): + actual = None + allowed = expected if isinstance(expected, list) else [expected] + if not any(exact_equal(actual, item) for item in allowed): + problems.append( + f"{identity}: {path}={actual!r} expected one of {allowed!r}" + ) + evidence["examined"] = seen[:32] + if wanted is not None: + absent = [item for item in wanted if item not in seen] + if absent: + return Evaluation( + NOT_RUN, f"expected keys absent from step {key}: {absent}", evidence + ) + elif not seen: + return Evaluation(NOT_RUN, f"step {key} returned nothing to examine", evidence) + if problems: + return Evaluation(FAIL, "; ".join(problems[:8]), evidence) + return Evaluation( + PASS, f"{len(seen)} element(s) match the release contract", evidence + ) + + +@evaluator("json_record") +def evaluate_json_record(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Capture named JSON paths as evidence; pass when all of them resolve. + + Recording is a release requirement in its own right — the runbook has to + name the exact heads it was verified against — so an unreachable source is a + NOT_RUN rather than an empty section. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + parsed = parsed_step(spec, outcomes) + if isinstance(parsed, Evaluation): + return parsed + payload, key = parsed + evidence: dict[str, Any] = {} + for label, path in spec.expect["record"].items(): + try: + evidence[label] = dotted(payload, path) + except (KeyError, ValueError, IndexError): + return Evaluation( + NOT_RUN, f"path {path} is absent from step {key}", evidence + ) + return Evaluation(PASS, "state recorded", evidence) + + +@evaluator("routing_evidence") +def evaluate_routing_evidence( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> Evaluation: + """Summarise a Switchyard routing log tail into provider/effort coverage. + + Route identifiers are ``///``, so provider, + model family, effort tier, and lane all come from the same field. Fallback + and failure evidence is a separate ``fallback_reason`` on the record that + was retried. Every row must be complete strict JSON; a clipped or malformed + line makes the evidence unusable rather than being silently discarded. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + records: list[dict[str, Any]] = [] + routes: list[list[str]] = [] + stamps_by_dimension: dict[tuple[str, str], dt.datetime] = {} + now_value = spec.expect.get("now") + try: + now = _now(now_value) if now_value is not None else None + except (TypeError, ValueError) as exc: + return Evaluation(NOT_RUN, f"invalid routing run clock: {exc}") + for row_number, line in enumerate(outcome.stdout.splitlines(), 1): + line = line.strip() + if not line: + continue + try: + record = strict_json(line) + except ValueError as exc: + return Evaluation( + NOT_RUN, f"routing row {row_number} is malformed JSON: {exc}" + ) + if not isinstance(record, dict): + return Evaluation(NOT_RUN, f"routing row {row_number} is not an object") + try: + observed = _timestamp(record.get("ts")) + except (TypeError, ValueError) as exc: + return Evaluation( + NOT_RUN, f"routing row {row_number} has invalid timestamp: {exc}" + ) + if now is not None and observed > now: + return Evaluation(FAIL, f"routing row {row_number} has a future timestamp") + model = record.get("model") + if not isinstance(model, str) or not model: + return Evaluation(NOT_RUN, f"routing row {row_number} has no model") + route = model.split("/") + if len(route) == 4 and all(_ROUTE_PART_RE.fullmatch(part) for part in route): + routes.append(route) + for label, value in ( + ("provider", route[1]), + ("effort", route[3]), + ("lane", route[0]), + ): + key_dimension = (label, value) + stamps_by_dimension[key_dimension] = max( + observed, stamps_by_dimension.get(key_dimension, observed) + ) + elif record.get("tier") != "classifier": + return Evaluation( + NOT_RUN, f"routing row {row_number} has malformed route {model!r}" + ) + fallback = record.get("fallback_reason") + if fallback is not None and ( + not isinstance(fallback, str) or not fallback.strip() + ): + return Evaluation( + NOT_RUN, f"routing row {row_number} has malformed fallback_reason" + ) + record["_observed"] = observed + records.append(record) + fallbacks = [record for record in records if record.get("fallback_reason")] + stamps = sorted(record["_observed"] for record in records) + evidence: dict[str, Any] = { + "records": len(records), + "routed": len(routes), + "lanes": sorted({route[0] for route in routes}), + "providers": sorted({route[1] for route in routes}), + "families": sorted({route[2] for route in routes}), + "efforts": sorted({route[3] for route in routes}), + "fallback_events": len(fallbacks), + "fallback_reasons": sorted( + {str(record["fallback_reason"]) for record in fallbacks} + ), + "latest_ts": stamps[-1].isoformat() if stamps else "", + } + if not records: + return Evaluation( + NOT_RUN, "routing log tail contained no parsable records", evidence + ) + missing = [ + f"{label}:{item}" + for label, wanted in ( + ("provider", spec.expect.get("providers", ())), + ("effort", spec.expect.get("efforts", ())), + ("lane", spec.expect.get("lanes", ())), + ) + for item in wanted + if item not in evidence[f"{label}s"] + ] + if spec.expect.get("require_fallback_evidence") and not fallbacks: + missing.append("fallback:none-observed") + max_age = spec.expect.get("max_age_seconds") + if max_age and now: + ages: dict[str, int] = {} + for label, wanted in ( + ("provider", spec.expect.get("providers", ())), + ("effort", spec.expect.get("efforts", ())), + ("lane", spec.expect.get("lanes", ())), + ): + for item in wanted: + stamp = stamps_by_dimension.get((label, item)) + if stamp is None: + continue + age = int((now - stamp).total_seconds()) + ages[f"{label}:{item}"] = age + if age > max_age: + missing.append( + f"freshness:{label}:{item}:{age}s-exceeds-{max_age}s" + ) + evidence["age_seconds"] = ages + if missing: + return Evaluation( + FAIL, f"routing evidence missing {', '.join(missing)}", evidence + ) + return Evaluation(PASS, "routing evidence covers the required lanes", evidence) + + +@evaluator("flux_health") +def evaluate_flux_health(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Classify a compact Flux projection into suspensions and unhealthy objects. + + Each row carries name, generation, observed generation, suspension, and all + condition states. A non-suspended object must be current, explicitly + ``Ready=True``, not ``Healthy=False/Unknown``, and not reconciling. Suspended + objects are judged against the exact expected-suspension list. + """ + blocked = missing_steps(spec, outcomes) + if blocked: + return blocked + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + expected = set(spec.expect.get("expected_suspensions", ())) + suspended: list[str] = [] + unhealthy: list[str] = [] + total = 0 + for row_number, line in enumerate(outcome.stdout.splitlines(), 1): + if not line.strip(): + continue + columns = line.split("\t") + if len(columns) != 5: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has {len(columns)} columns" + ) + name, generation, observed_generation, suspend, conditions = columns + name = name.strip() + if not _FLUX_NAME_RE.fullmatch(name): + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed object name" + ) + try: + current_generation = int(generation) + reconciled_generation = int(observed_generation) + except ValueError: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed generations" + ) + if current_generation <= 0 or current_generation != reconciled_generation: + return Evaluation( + FAIL, f"Flux row {row_number} has stale observedGeneration" + ) + if suspend.strip().lower() not in {"", "false", "true"}: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed suspension state" + ) + total += 1 + if suspend.strip().lower() == "true": + suspended.append(name) + continue + states: dict[str, str] = {} + for pair in filter(None, conditions.strip().strip(",").split(",")): + if pair.count("=") != 1: + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed condition" + ) + condition, state = pair.split("=", 1) + if ( + not condition + or state not in {"True", "False", "Unknown"} + or condition in states + ): + return Evaluation( + NOT_RUN, f"Flux row {row_number} has malformed condition state" + ) + states[condition] = state + healthy = states.get("Healthy") + if ( + states.get("Ready") != "True" + or healthy in {"False", "Unknown"} + or states.get("Reconciling") == "True" + ): + unhealthy.append(name) + unexpected = sorted(set(suspended) - expected) + evidence = { + "total": total, + "suspended": sorted(suspended), + "unexpected_suspensions": unexpected, + "unhealthy": sorted(unhealthy), + } + if not total: + return Evaluation(NOT_RUN, f"step {key} returned no Flux objects", evidence) + problems = [] + if unexpected: + problems.append(f"unexpected suspensions: {', '.join(unexpected)}") + if unhealthy: + problems.append(f"unhealthy: {', '.join(sorted(unhealthy)[:8])}") + if problems: + return Evaluation(FAIL, "; ".join(problems), evidence) + return Evaluation(PASS, f"{total} Flux objects reconciled as expected", evidence) diff --git a/scripts/ops/hermes_handoff_model.py b/scripts/ops/hermes_handoff_model.py new file mode 100644 index 00000000..d35d39f5 --- /dev/null +++ b/scripts/ops/hermes_handoff_model.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +"""Result model, aggregation, and rendering for the handoff acceptance harness. + +The classification is deliberately four-valued. ``NOT_APPLICABLE`` is the only +way a check can be excused, and a check can only reach it by matching a scope +rule that a reviewer wrote down; anything the harness merely failed to observe +is ``NOT_RUN``. Because a mandatory ``NOT_RUN`` is a NO_GO exactly like a +``FAIL``, an unreachable cluster, an exhausted deadline, or a probe nobody +maintained can never round up to a release. +""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_redaction import contains_credential_shape, safe_text, scrub + +PASS = "PASS" +FAIL = "FAIL" +NOT_RUN = "NOT_RUN" +NOT_APPLICABLE = "NOT_APPLICABLE" +STATUSES = (PASS, FAIL, NOT_RUN, NOT_APPLICABLE) + +GO = "GO" +NO_GO = "NO_GO" + +ALWAYS = "always" +EPHEMERAL = "ephemeral" + +HARNESS = "hermes-handoff-acceptance" +SCHEMA_VERSION = 1 + + +REVIEW = "review" +ATTEMPT = "attempt" +READ = "read" +STEP_KINDS = (REVIEW, ATTEMPT, READ) + + +@dataclass(frozen=True) +class Step: + """One command a check needs, and the vantage it must be observed from. + + ``kind`` separates a self access review (``review``) from a real request the + identity is expected to be refused (``attempt``). A deny check needs both: + an authorization review states policy, only an attempt proves the API server + enforces it. + + ``record`` is cleared for bulk evidence — a routing log tail, a large object + listing — which the harness parses but must not paste into the artifact. + ``max_bytes`` raises the capture budget for that same bulk evidence, where a + default-sized truncation would silently narrow the window being examined. + """ + + key: str + vantage: str + argv: tuple[str, ...] + kind: str = READ + optional: bool = False + record: bool = True + max_bytes: int | None = None + + +@dataclass(frozen=True) +class CheckSpec: + """A declarative acceptance check. + + ``rule`` names an evaluator; ``expect`` carries its parameters. Keeping the + catalog declarative means a reviewer reads what is asserted, not how it is + plumbed, and the harness can prove structural properties over every check + (no impersonation, no mutation, both vantages present) before running any. + """ + + id: str + title: str + group: str + rule: str + steps: tuple[Step, ...] = () + expect: Mapping[str, Any] = field(default_factory=dict) + mandatory: bool = True + scope: str = ALWAYS + rationale: str = "" + + @property + def vantages(self) -> tuple[str, ...]: + """Return the distinct vantages this check draws evidence from.""" + seen: list[str] = [] + for step in self.steps: + if step.vantage not in seen: + seen.append(step.vantage) + return tuple(seen) + + +@dataclass +class CheckResult: + """The classified outcome of one acceptance check.""" + + spec: CheckSpec + status: str + reason: str = "" + evidence: dict[str, Any] = field(default_factory=dict) + outcomes: list[Outcome] = field(default_factory=list) + + @property + def blocking(self) -> bool: + """Report whether this result alone forces NO_GO.""" + if self.status not in STATUSES: + return True + return self.spec.mandatory and self.status in {FAIL, NOT_RUN} + + def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: + """Return a JSON-serialisable, screened view of the result.""" + payload: dict[str, Any] = { + "id": self.spec.id, + "title": self.spec.title, + "group": self.spec.group, + "rule": self.spec.rule, + "scope": self.spec.scope, + "mandatory": self.spec.mandatory, + "vantages": list(self.spec.vantages), + "status": self.status, + "reason": self.reason, + "blocking": self.blocking, + "evidence": self.evidence, + } + if self.spec.rationale: + payload["rationale"] = self.spec.rationale + if include_outcomes: + payload["commands"] = [outcome.as_dict() for outcome in self.outcomes] + return scrub(payload) + + +@dataclass +class VantageRecord: + """What the harness proved about one vantage before trusting its evidence.""" + + name: str + description: str + identity: str = "" + available: bool = False + detail: str = "" + + def as_dict(self) -> dict[str, Any]: + """Return a JSON-serialisable view of the vantage.""" + return scrub( + { + "name": self.name, + "description": self.description, + "identity": self.identity, + "available": self.available, + "detail": self.detail, + } + ) + + +@dataclass +class Report: + """The whole acceptance run: metadata, per-check results, and a verdict.""" + + mode: str + started_at: str + baseline: dict[str, Any] = field(default_factory=dict) + vantages: list[VantageRecord] = field(default_factory=list) + results: list[CheckResult] = field(default_factory=list) + harness_errors: list[str] = field(default_factory=list) + finished_at: str = "" + + @property + def counts(self) -> dict[str, int]: + """Return the number of results in each status.""" + tally = dict.fromkeys(STATUSES, 0) + for result in self.results: + if result.status in tally: + tally[result.status] += 1 + return tally + + @property + def invalid_statuses(self) -> list[str]: + """Return check ids whose status is outside the report schema.""" + return [ + result.spec.id for result in self.results if result.status not in STATUSES + ] + + @property + def blocking(self) -> list[CheckResult]: + """Return every result that forces NO_GO, most severe first.""" + order = {FAIL: 0, NOT_RUN: 1} + return sorted( + (result for result in self.results if result.blocking), + key=lambda result: (order.get(result.status, 2), result.spec.id), + ) + + @property + def decision(self) -> str: + """Return GO only when nothing mandatory failed, skipped, or errored.""" + if ( + not self.results + or self.invalid_statuses + or self.harness_errors + or self.blocking + ): + return NO_GO + return GO + + def group_summary(self) -> dict[str, dict[str, int]]: + """Return per-group status counts, for the human summary.""" + groups: dict[str, dict[str, int]] = {} + for result in self.results: + bucket = groups.setdefault(result.spec.group, dict.fromkeys(STATUSES, 0)) + bucket[result.status] = bucket.get(result.status, 0) + 1 + return groups + + def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]: + """Return the machine-readable report.""" + return scrub( + { + "harness": HARNESS, + "schema_version": SCHEMA_VERSION, + "mode": self.mode, + "started_at": self.started_at, + "finished_at": self.finished_at, + "decision": self.decision, + "counts": self.counts, + "baseline": scrub(self.baseline), + "vantages": [vantage.as_dict() for vantage in self.vantages], + "harness_errors": [scrub(error) for error in self.harness_errors], + "invalid_statuses": self.invalid_statuses, + "blocking": [result.spec.id for result in self.blocking], + "checks": [result.as_dict(include_outcomes) for result in self.results], + } + ) + + def render_summary(self, width: int = 88) -> str: + """Return the concise human summary printed alongside the JSON.""" + counts = self.counts + lines = [ + f"{HARNESS}: {self.decision}", + f" mode={self.mode} started={self.started_at} checks={len(self.results)}", + " " + " ".join(f"{status}={counts[status]}" for status in STATUSES), + ] + for vantage in self.vantages: + state = "available" if vantage.available else "unavailable" + identity = vantage.identity or "unknown" + lines.append(f" vantage {vantage.name}: {state} identity={identity}") + for error in self.harness_errors: + lines.append(f" harness error: {error}") + if self.blocking: + lines.append(" blocking:") + for result in self.blocking: + lines.append(f" {result.status:<15}{result.spec.id}") + lines.append(f" {'':<15}{result.reason[:width]}") + groups = self.group_summary() + lines.append(" groups:") + for name in sorted(groups): + bucket = groups[name] + detail = " ".join( + f"{status}={bucket[status]}" for status in STATUSES if bucket[status] + ) + lines.append(f" {name:<18}{detail}") + return safe_text("\n".join(lines), 32 * 1024)[0] + + +def utc_now(clock: Any = None) -> str: + """Return an ISO-8601 UTC timestamp, injectable for deterministic tests.""" + now = clock() if clock else dt.datetime.now(dt.timezone.utc) + return now.replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def unscreened_fields(payload: Any, path: str = "$") -> list[str]: + """Return JSON pointers whose text still looks credential-shaped. + + The report is already screened field by field; this is the invariant check + that runs before anything is written, so a screening gap fails the run + instead of publishing an artifact nobody re-read. + """ + offenders: list[str] = [] + if isinstance(payload, str): + if contains_credential_shape(payload): + offenders.append(path) + elif isinstance(payload, Mapping): + for key, value in payload.items(): + offenders.extend(unscreened_fields(value, f"{path}.{key}")) + elif isinstance(payload, Sequence) and not isinstance(payload, (str, bytes)): + for index, value in enumerate(payload): + offenders.extend(unscreened_fields(value, f"{path}[{index}]")) + return offenders diff --git a/scripts/ops/hermes_handoff_policy.py b/scripts/ops/hermes_handoff_policy.py new file mode 100644 index 00000000..11c8bc1c --- /dev/null +++ b/scripts/ops/hermes_handoff_policy.py @@ -0,0 +1,493 @@ +#!/usr/bin/env python3 +"""Fail-closed command policy for the Hermes handoff acceptance harness. + +Before spawn it admits only credential-safe, structurally read-only grammar; +shell is limited to frozen templates with absolute tool paths. + +Impersonation is structurally impossible: ``--as`` and its siblings are refused +for every argv, in every mode, from every vantage, and the inner command of a +``kubectl exec`` is re-checked under the same rule rather than being trusted. +Only binaries with a pinned release digest are on the allowlist, so nothing can +be admitted here that the executor would then refuse to attest. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator +from urllib.parse import unquote + +READ_ONLY = "read-only" +ARMED = "ephemeral-armed" +MODES = (READ_ONLY, ARMED) + +GITEA_CLIENT = "/opt/coordinator/gitea_api.py" +EXPECTED_REPO = "atlas/titan-iac" +EXPECTED_REMOTE = "origin" +EXPECTED_BASE = "main" + +# fmt: off +FORBIDDEN_ARGS = ("--as", "--as-group", "--as-uid", "--client-certificate", + "--client-key", "--password", "--token", "--username") +# fmt: on +IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid") +FORBIDDEN_BINARIES = frozenset( + {"age", "curl", "gpg", "openssl", "sops", "vault", "wget"} +) +ALLOWED_BINARIES = frozenset({"kubectl", "git", "hermes", "sh", GITEA_CLIENT}) +TRUSTED_EXECUTABLE_ROOTS = ( + "/bin/", + "/usr/bin/", + "/usr/local/bin/", + "/opt/data/tools/bin/", + "/opt/hermes/.venv/bin/", + "/opt/coordinator/", +) +TRUSTED_INNER_PATHS = { + "/bin/sh": "sh", + "/usr/bin/git": "git", + "/usr/bin/sh": "sh", + "/usr/local/bin/kubectl": "kubectl", + "/opt/hermes/.venv/bin/hermes": "hermes", + GITEA_CLIENT: GITEA_CLIENT, +} + +VALUE_FLAGS = frozenset( + { + "-C", + "-c", + "-l", + "-n", + "-o", + "--cluster", + "--container", + "--context", + "--field-selector", + "--kubeconfig", + "--namespace", + "--output", + "--request-timeout", + "--selector", + "--server", + "--since", + "--tail", + "--user", + } +) + +KUBECTL_READ_SUBCOMMANDS = frozenset( + { + "api-resources", + "api-versions", + "explain", + "get", + "top", + "version", + } +) +KUBECTL_EXEC_SUBCOMMANDS = frozenset({"exec"}) +KUBECTL_SUBCOMMANDS = KUBECTL_READ_SUBCOMMANDS | KUBECTL_EXEC_SUBCOMMANDS | {"auth"} +SECRET_RESOURCES = frozenset({"secret", "secrets"}) +UNSAFE_PROJECTION_PARTS = ( + ".args", + ".command", + ".data", + ".envfrom", + ".secretkeyref", + ".status.token", + ".stringdata", + ".valuefrom", + "clientconfig", + "password", + "privatekey", +) +SAFE_ENV_VALUE = 'env[?(@.name=="hermes_auto_router_profile")].value' +SAFE_JSON_FIELDS = frozenset(["authenticated", "transport", "state", "model", "effort", "latency_ms", "checked_at", "api_provider", "auth_method", "subscription_type"]) # fmt: skip + +GIT_READ_SUBCOMMANDS = frozenset( + { + "cat-file", + "diff", + "for-each-ref", + "log", + "ls-remote", + "ls-tree", + "merge-base", + "rev-list", + "rev-parse", + "show", + "status", + } +) +GIT_FORCE_FLAGS = frozenset( + {"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"} +) +GIT_UNSAFE_FLAGS = ( + "-c", + "--config-env", + "--exec-path", + "--upload-pack", + "--receive-pack", +) +GITEA_READ_METHODS = frozenset({"GET"}) +HERMES_READ_COMMANDS = frozenset( + {("kanban", "list"), ("kanban", "show"), ("sessions", "list"), ("status",)} +) + +# These locations contain credential bytes even when a caller claims it will +# only inspect metadata. They are rejected for every argv, including a frozen +# shell rendering and the inner command of kubectl exec. +SECRET_PATH_RE = re.compile( + r"(?:^|[\s'\"=])(?:/[^\s'\"]*)?(?:" + r"var/run/secrets|run/secrets|vault/secrets|runtime-access|scm-access|pool-access|" + r"\.credentials\.json|auth\.json|\.git-credentials|authorized_keys|" + r"id_[a-z0-9]+|\.netrc|shadow|gshadow|k3s\.yaml" + r")(?:$|[/\s'\"])", + re.IGNORECASE, +) +_PARAMETER_RE = re.compile(r"[A-Za-z0-9/][A-Za-z0-9._/@:,-]{0,255}\Z") +_PLACEHOLDER_RE = re.compile(r"\{([a-z_]+)\}") +_SAFE_API_PATH_RE = re.compile(r"/[A-Za-z0-9][A-Za-z0-9._~:/?&=%+-]{0,511}\Z") + +SHELL_TEMPLATES: dict[str, str] = { + "env_names": ( + "/usr/bin/env | /usr/bin/sed -n " + "'s/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/\\1/p' | /usr/bin/sort" + ), + "json_fields": ( + '/opt/hermes/.venv/bin/python -c "import json,sys;d=json.load(open(sys.argv[1]));' + 'print(json.dumps({{k:d.get(k) for k in sys.argv[2].split(chr(44))}}))" ' + "{path} {fields}" + ), + "tail_lines": "/usr/bin/tail -n {limit} {path}", +} + + +class PolicyError(ValueError): + """Raised when a requested command falls outside the safe boundary.""" + + +def template_parameters(template_name: str) -> set[str]: + """Return the parameter names a frozen template requires.""" + template = SHELL_TEMPLATES.get(template_name) + if template is None: + raise PolicyError(f"unknown shell template: {template_name}") + return set(_PLACEHOLDER_RE.findall(template)) + + +def render_shell(template_name: str, **parameters: str) -> str: + """Return a frozen shell template with validated parameters substituted.""" + required = template_parameters(template_name) + if set(parameters) != required: + raise PolicyError( + f"shell template {template_name} expects parameters {sorted(required)}" + ) + for key, value in parameters.items(): + if not _PARAMETER_RE.fullmatch(value): + raise PolicyError(f"unsafe shell parameter for {template_name}.{key}") + if SECRET_PATH_RE.search(value): + raise PolicyError("shell templates may not name credential paths") + if ( + template_name == "json_fields" + and not set(parameters["fields"].split(",")) <= SAFE_JSON_FIELDS + ): + raise PolicyError("json_fields requested a credential-bearing or unknown field") + return SHELL_TEMPLATES[template_name].format(**parameters) + + +_RENDERED: set[str] = set() +_APPROVED_PROJECTIONS: set[str] = set() +_ARMED_REF = "" +_ARMED_PULLS: set[int] = set() + + +# fmt: off +def arm_ephemeral_policy(ref: str) -> None: + global _ARMED_REF + if not re.fullmatch(r"ephemeral/hermes-handoff-acceptance/[a-z0-9][a-z0-9-]{7,63}", ref): + raise PolicyError("armed policy requires one exact ephemeral ref") + _ARMED_REF = ref + _ARMED_PULLS.clear() + + +def register_ephemeral_pull(number: int) -> None: + if not _ARMED_REF or isinstance(number, bool) or not isinstance(number, int) or number <= 0: + raise PolicyError("ephemeral pull registration is malformed") + _ARMED_PULLS.add(number) + + +def shell(template_name: str, **parameters: str) -> tuple[str, ...]: + """Return a validated ``sh -c`` argv built from a frozen template.""" + script = render_shell(template_name, **parameters) + _RENDERED.add(script) + return ("sh", "-c", script) + + +def projection(value: str) -> str: + normalized = value.lower().replace(" ", "") + if not normalized.startswith("jsonpath=") or any(part in normalized for part in UNSAFE_PROJECTION_PARTS): + raise PolicyError("Kubernetes projection is not credential-safe") + if ".metadata.annotations" in normalized and not any(key in normalized for key in ( + "deployment\\.kubernetes\\.io/revision", "ai\\.bstein\\.dev/config-rev")): + raise PolicyError("Kubernetes annotations are not on the reviewed allowlist") + if ".metadata.labels" in normalized and "pod-template-hash" not in normalized: + raise PolicyError("Kubernetes labels are not on the reviewed allowlist") + if ".env" in normalized and ".value" in normalized and SAFE_ENV_VALUE not in normalized: + raise PolicyError("Kubernetes projection may expose environment values") + _APPROVED_PROJECTIONS.add(normalized) + return value +# fmt: on + + +def positionals(argv: tuple[str, ...], start: int = 1) -> Iterator[str]: + """Yield positional arguments, skipping flags and their values.""" + index = start + while index < len(argv): + item = argv[index] + if item == "--": + return + if item.startswith("-"): + index += 2 if item in VALUE_FLAGS else 1 + continue + yield item + index += 1 + + +def _binary_name(value: str) -> str: + if value == GITEA_CLIENT or ( + value.startswith("/opt/coordinator/") and value.endswith("/gitea_api.py") + ): + return GITEA_CLIENT + if "/" not in value: + return value + mapped = TRUSTED_INNER_PATHS.get(value) + if mapped: + return mapped + if "/../" in value or value.endswith("/..") or "/./" in value: + raise PolicyError(f"executable path is not normalized: {value}") + if not value.startswith(TRUSTED_EXECUTABLE_ROOTS): + raise PolicyError(f"executable path is not trusted: {value}") + return value.rsplit("/", 1)[-1] + + +def _reject_forbidden_arguments(argv: tuple[str, ...]) -> None: + """Refuse credential and impersonation arguments up to the ``--`` boundary. + + Everything after ``--`` is the inner command of a ``kubectl exec``, which + :func:`_check_kubectl` submits to :func:`check_argv` again, so it is held to + exactly this rule rather than being exempted from it. + """ + boundary = argv.index("--") if "--" in argv else len(argv) + for argument in argv[:boundary]: + for forbidden in FORBIDDEN_ARGS: + if argument == forbidden or argument.startswith(f"{forbidden}="): + raise PolicyError(f"forbidden argument: {forbidden}") + if argument == "--raw" or argument.startswith("--raw="): + raise PolicyError("raw API reads are forbidden") + for argument in argv: + if argument == "--dry-run" or argument.startswith("--dry-run="): + raise PolicyError("dry-run mutation commands are forbidden") + + +def _reject_secret_paths(argv: tuple[str, ...]) -> None: + if any(SECRET_PATH_RE.search(argument) for argument in argv): + raise PolicyError("probes may not name a credential path") + + +def _resource_name(value: str) -> str: + return value.lower().split("/", 1)[0].split(".", 1)[0] + + +def _output_value(argv: tuple[str, ...]) -> str: + """Return kubectl's output argument in a single canonical form.""" + for index, item in enumerate(argv[1:], 1): + if item in {"-o", "--output"} and index + 1 < len(argv): + return argv[index + 1] + if item.startswith(("-o=", "--output=")): + return item.split("=", 1)[1] + return "" + + +def _check_projection(output: str) -> None: + """Reject full or credential-bearing Kubernetes output projections.""" + lowered = output.lower().replace(" ", "") + if lowered in {"json", "yaml", "wide"} or lowered.startswith("custom-columns"): + raise PolicyError( + "full Kubernetes object output is forbidden; use a safe projection" + ) + if lowered and not (lowered == "name" or lowered.startswith("jsonpath=")): + raise PolicyError( + "Kubernetes output must be name or an approved JSONPath projection" + ) + if lowered.startswith("jsonpath="): + if any(part in lowered for part in UNSAFE_PROJECTION_PARTS): + raise PolicyError("Kubernetes projection may expose runtime credentials") + if ".env" in lowered and SAFE_ENV_VALUE not in lowered: + # Environment names are safe; arbitrary `.value` reads are not. + without_names = lowered.replace("{.name}", "") + if ".value" in without_names: + raise PolicyError("Kubernetes projection may expose environment values") + if lowered not in _APPROVED_PROJECTIONS: + raise PolicyError( + "Kubernetes JSONPath projection is not on the reviewed allowlist" + ) + + +def _check_kubectl(argv: tuple[str, ...], mode: str) -> None: + words = list(positionals(argv)) + if not words or words[0] not in KUBECTL_SUBCOMMANDS: + raise PolicyError("kubectl subcommand is outside the read-only boundary") + subcommand = words[0] + if subcommand == "auth": + if len(words) < 2 or words[1] not in {"can-i", "whoami"}: + raise PolicyError("kubectl auth is limited to can-i and whoami") + return + if subcommand == "exec": + if "--" not in argv: + raise PolicyError("kubectl exec requires an explicit -- command boundary") + inner = argv[argv.index("--") + 1 :] + if not inner: + raise PolicyError("kubectl exec requires an inner command") + if inner[0] not in TRUSTED_INNER_PATHS: + raise PolicyError( + "kubectl exec inner command must use a fixed trusted path" + ) + check_argv(inner, mode) + return + resources = words[1].split(",") if len(words) > 1 else [] + if any(_resource_name(resource) in SECRET_RESOURCES for resource in resources): + raise PolicyError("Secret objects may not be read by the harness") + output = _output_value(argv) + if subcommand == "get" and not output: + raise PolicyError("kubectl get requires a reviewed output projection") + _check_projection(output) + + +def _check_git(argv: tuple[str, ...], mode: str) -> None: + unsafe = False + for item in argv[1:]: + if item == "-C" or item.startswith(("--git-dir", "--work-tree", "--namespace")): + unsafe = True + if item == "-c" or (item.startswith("-c") and len(item) > 2): + unsafe = True + if any( + item == flag or item.startswith(f"{flag}=") for flag in GIT_UNSAFE_FLAGS[1:] + ): + unsafe = True + if unsafe: + raise PolicyError("git configuration and helper overrides are forbidden") + subcommand = next(positionals(argv), None) + if subcommand is None: + raise PolicyError("git requires a subcommand") + if subcommand in GIT_READ_SUBCOMMANDS: + return + if subcommand == "push" and mode == ARMED: + if GIT_FORCE_FLAGS & set(argv): + raise PolicyError("git push may not force or fan out across refs") + words = list(positionals(argv)) + if len(words) != 3 or words[1] != EXPECTED_REMOTE: + raise PolicyError("armed git push is pinned to origin and one refspec") + destination = words[2].partition(":")[2] + if not _ARMED_REF or destination != f"refs/heads/{_ARMED_REF}": + raise PolicyError("armed git push is not the preflighted ephemeral ref") + return + raise PolicyError(f"git {subcommand} is not permitted in {mode} mode") + + +def _check_hermes(argv: tuple[str, ...]) -> None: + words = tuple(positionals(argv)) + if not words: + raise PolicyError("hermes requires a subcommand") + if words[:2] not in HERMES_READ_COMMANDS and words[:1] not in HERMES_READ_COMMANDS: + raise PolicyError( + f"hermes {' '.join(words[:2])} is outside the read-only boundary" + ) + + +def _reject_dot_segments(path: str) -> None: + """Refuse any relative segment, including a percent-encoded one. + + The repository pin below is a textual prefix test, so a single ``..`` would + let a pinned path address another repository or the token endpoints once a + router normalised it. Decode first: ``%2e%2e`` is the same segment. + """ + decoded = unquote(unquote(path)).split("?", 1)[0] + if any(segment in {"", ".", ".."} for segment in decoded.split("/")[1:]): + raise PolicyError("gitea_api.py paths may not contain relative segments") + + +def _check_gitea(argv: tuple[str, ...], mode: str) -> None: + if len(argv) < 3: + raise PolicyError("gitea_api.py requires a method and a path") + method, path = argv[1].upper(), argv[2] + if not path.startswith("/api/v1/") or not _SAFE_API_PATH_RE.fullmatch(path): + raise PolicyError("gitea_api.py requires a bounded API path") + _reject_dot_segments(path) + repo_root = f"/api/v1/repos/{EXPECTED_REPO}" + if ( + path != "/api/v1/user" + and path != repo_root + and not path.startswith((f"{repo_root}/", f"{repo_root}?")) + ): + raise PolicyError("gitea_api.py is pinned to the Atlas titan-iac repository") + if method in GITEA_READ_METHODS: + return + if mode == ARMED: + fields = [ + argv[index + 1] for index, item in enumerate(argv[:-1]) if item == "--field" + ] + if method == "POST" and path == f"{repo_root}/pulls" and _ARMED_REF: + required = { + "title=WIP: Hermes handoff acceptance ephemeral probe", + f"head={_ARMED_REF}", + f"base={EXPECTED_BASE}", + "body=Ephemeral acceptance probe. Closed and deleted by the harness.", + } + if len(fields) == len(required) and set(fields) == required: + return + match = re.fullmatch(rf"{re.escape(repo_root)}/pulls/([1-9][0-9]*)", path) + if ( + method == "PATCH" + and match + and int(match.group(1)) in _ARMED_PULLS + and fields == ["state=closed"] + ): + return + if ( + method == "DELETE" + and _ARMED_REF + and path == f"{repo_root}/branches/{_ARMED_REF}" + and len(argv) == 3 + ): + return + raise PolicyError(f"gitea_api.py {method} is not permitted in {mode} mode") + + +def _check_shell(argv: tuple[str, ...]) -> None: + if len(argv) != 3 or argv[1] != "-c" or argv[2] not in _RENDERED: + raise PolicyError("sh is only reachable as sh -c ") + + +def check_argv(argv: tuple[str, ...], mode: str = READ_ONLY) -> None: + """Raise :class:`PolicyError` unless ``argv`` is safe in ``mode``.""" + if mode not in MODES: + raise PolicyError(f"unknown mode: {mode}") + if not argv: + raise PolicyError("empty command") + binary = _binary_name(argv[0]) + if binary in FORBIDDEN_BINARIES: + raise PolicyError(f"forbidden binary: {binary}") + if binary not in ALLOWED_BINARIES: + raise PolicyError(f"binary is not on the harness allowlist: {binary}") + _reject_forbidden_arguments(argv) + _reject_secret_paths(argv) + if binary == "sh": + _check_shell(argv) + elif binary == "kubectl": + _check_kubectl(argv, mode) + elif binary == "git": + _check_git(argv, mode) + elif binary == "hermes": + _check_hermes(argv) + else: + _check_gitea(argv, mode) diff --git a/scripts/ops/hermes_handoff_redaction.py b/scripts/ops/hermes_handoff_redaction.py new file mode 100644 index 00000000..f7b97f4c --- /dev/null +++ b/scripts/ops/hermes_handoff_redaction.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Credential redaction and output bounding for the handoff acceptance harness. + +Every string the harness records — command output, failure reasons, evidence +values — passes through :func:`redact` before it reaches a report. Screening is +a fail-closed accident barrier, not proof that text is secret-free: a short +secret under an innocuous key is not reliably distinguishable from prose. The +harness therefore also never asks for a secret in the first place. Probes read +names, shapes, and status fields; they never read environment values, token +files, private keys, cookies, or Vault responses. + +Bounding is part of the same barrier. Screening always precedes truncation so a +credential that crosses the byte boundary cannot leave a useful prefix behind. +The process runner additionally discards all captured bytes when a live stream +exceeds its cap; the fixed truncation marker is the only recorded output. +""" + +from __future__ import annotations + +import re +from typing import Any + +REDACTED = "[redacted]" +DEFAULT_MAX_BYTES = 64 * 1024 +TRUNCATION_NOTE = "[truncated]" + +# Keys whose value is never safe to record, matched against a normalised +# (lowercased, separator-stripped) form of the key. +SENSITIVE_KEY_SUFFIXES = ( + "accesskey", + "accesskeyid", + "apikey", + "authtoken", + "clientsecret", + "connectionstring", + "cookie", + "credential", + "credentials", + "passphrase", + "passwd", + "password", + "privatekey", + "secret", + "secretkey", + "sessionkey", + "signature", + "signingkey", + "token", + "webhook", +) +SENSITIVE_KEY_EXACT = frozenset( + { + "auth", + "authorization", + "bearer", + "pat", + "pw", + "sas", + "setcookie", + } +) +# A bare `key:` is a structural field name across Kubernetes and TOML far more +# often than it is a credential, so it is screened by value shape and by the +# token patterns rather than by name. The compound forms above still match. + +# Ordinary prose values shorter than this threshold are not independently +# treated as secret-shaped. Values under sensitive keys are screened at every +# length, including numeric values. +MIN_SECRET_LENGTH = 8 +NON_SECRET_LITERALS = frozenset( + {"true", "false", "null", "none", "yes", "no", "0", "1", ""} +) +_KEY_NOISE_RE = re.compile(r"[^a-z0-9]+") + +# Ordered: structural matches first so a nested token is not partially rewritten +# by a looser rule. +_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile( + r"-----BEGIN[ A-Z0-9-]{0,64}-----.*?-----END[ A-Z0-9-]{0,64}-----", + re.DOTALL, + ), + re.compile(r"-----BEGIN[ A-Z0-9-]{0,64}-----.*", re.DOTALL), + re.compile(r"(?i)\b(?:authorization|proxy-authorization)\s*:\s*\S.*"), + re.compile(r"(?i)\b(?:set-)?cookie\s*:\s*\S.*"), + re.compile(r"(?i)\b(?:bearer|basic|token)\s+[A-Za-z0-9+/_=.:~-]{8,}"), + re.compile(r"(?i)\b[a-z][a-z0-9+.-]{1,20}://[^\s/@:]{1,128}:[^\s/@]{1,256}@"), + re.compile( + r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}" + ), + re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}"), + re.compile(r"(?["']?) + (?P[A-Za-z][A-Za-z0-9_.\ -]{0,63}) + (?P=quote) + (?P\s{0,8}[:=]\s{0,8}) + (?P"[^"\n]{0,4096}"|'[^'\n]{0,4096}'|[^\s,;}\n][^,;}\n]{0,4096}) + """ +) + +# Long, mixed-class runs. Deliberately conservative: prose and hex digests of +# fewer than 44 characters are left alone so commit SHAs stay readable. +_ENTROPY_RE = re.compile( + r"(? str: + """Return a key lowercased with separators removed, for suffix matching.""" + return _KEY_NOISE_RE.sub("", key.lower()) + + +def is_sensitive_key(key: str) -> bool: + """Report whether a key name means its value must never be recorded.""" + compact = normalise_key(key) + if not compact: + return False + if compact in SENSITIVE_KEY_EXACT: + return True + return compact.endswith(SENSITIVE_KEY_SUFFIXES) + + +def _is_high_entropy(candidate: str) -> bool: + """Report whether a long run mixes enough character classes to be a token.""" + classes = sum( + ( + any(character.islower() for character in candidate), + any(character.isupper() for character in candidate), + any(character.isdigit() for character in candidate), + any(character in "+/_=-" for character in candidate), + ) + ) + return classes >= 4 + + +def could_hold_secret(value: str) -> bool: + """Report whether a raw value is long and opaque enough to be a credential.""" + stripped = value.strip().strip("\"'") + if stripped.lower() in NON_SECRET_LITERALS or len(stripped) < MIN_SECRET_LENGTH: + return False + try: + float(stripped) + except ValueError: + return True + return False + + +def _redact_assignment(match: re.Match[str]) -> str: + value = match.group("value") + stripped = value.strip().strip("\"'").lower() + if not is_sensitive_key(match.group("key")) or stripped in NON_SECRET_LITERALS: + return match.group(0) + # Preserve the original quoting so screened JSON stays parsable JSON. + quoted = value.strip()[:1] in {'"', "'"} + replacement = f'"{REDACTED}"' if quoted else REDACTED + key_quote = match.group("quote") + return ( + f"{key_quote}{match.group('key')}{key_quote}{match.group('gap')}{replacement}" + ) + + +def _redact_entropy(match: re.Match[str]) -> str: + return REDACTED if _is_high_entropy(match.group(0)) else match.group(0) + + +def redact(text: str) -> str: + """Return ``text`` with credential-shaped substrings replaced.""" + if not text: + return text + for pattern in _PATTERNS: + text = pattern.sub(REDACTED, text) + text = _ASSIGNMENT_RE.sub(_redact_assignment, text) + return _ENTROPY_RE.sub(_redact_entropy, text) + + +def bound(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: + """Truncate ``text`` to a UTF-8 byte budget, reporting whether it was cut.""" + if max_bytes <= 0: + return ("", bool(text)) + encoded = text.encode("utf-8", "replace") + if len(encoded) <= max_bytes: + return (text, False) + marker = TRUNCATION_NOTE.encode()[:max_bytes] + content_budget = max( + 0, max_bytes - len(marker) - (1 if len(marker) < max_bytes else 0) + ) + kept = encoded[:content_budget].decode("utf-8", "ignore") + separator = "\n" if kept and len(marker) < max_bytes else "" + return (f"{kept}{separator}{marker.decode()}", True) + + +def safe_text(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]: + """Redact then bound, so truncation never exposes a credential prefix.""" + return bound(redact(text), max_bytes) + + +def scrub(value: Any, max_bytes: int = DEFAULT_MAX_BYTES) -> Any: + """Recursively bound and redact a JSON-shaped value. + + Mapping keys are themselves screened; values under a sensitive key are + dropped outright rather than pattern-screened, because a short secret + would survive screening. + """ + if isinstance(value, str): + return safe_text(value, max_bytes)[0] + if isinstance(value, dict): + screened: dict[str, Any] = {} + for key, item in value.items(): + raw_key = str(key) + safe_key = safe_text(raw_key, max_bytes)[0] + sensitive_value = ( + is_sensitive_key(raw_key) + and item is not None + and not isinstance(item, bool) + and item != "" + ) + screened[safe_key] = REDACTED if sensitive_value else scrub(item, max_bytes) + return screened + if isinstance(value, (list, tuple)): + return [scrub(item, max_bytes) for item in value] + return value + + +def contains_credential_shape(text: str) -> bool: + """Report whether screening would rewrite ``text``. + + Used by the harness self-test to prove a rendered report is clean before it + is written, and by the unit suite as an assertion helper. + """ + return redact(text) != text diff --git a/scripts/ops/hermes_handoff_rules.py b/scripts/ops/hermes_handoff_rules.py new file mode 100644 index 00000000..5cca2c5a --- /dev/null +++ b/scripts/ops/hermes_handoff_rules.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Shared primitives and dispatch for handoff acceptance evaluators. + +Evaluators are registered by name so the check catalog stays declarative: an +entry says *what* it asserts, not how to plumb it. The dispatch here is the one +place that turns an unexpected exception into ``NOT_RUN`` — a harness bug must +never be indistinguishable from a passing release. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from hermes_handoff_exec import Outcome +from hermes_handoff_model import NOT_RUN, STATUSES, CheckSpec + +DENIAL_MARKERS = ( + "forbidden", + "is not allowed", + "unauthorized", + "permission denied", + "cannot get", + "cannot list", + "cannot create", + "cannot delete", + "cannot patch", + "not permitted", + "gitea api returned http 403", +) + + +@dataclass +class Evaluation: + """A classification plus the small facts that justify it.""" + + status: str + reason: str + evidence: dict[str, Any] = field(default_factory=dict) + + +Evaluator = Callable[[CheckSpec, dict[str, Outcome]], Evaluation] +EVALUATORS: dict[str, Evaluator] = {} + + +def evaluator(name: str) -> Callable[[Evaluator], Evaluator]: + """Register an evaluator under the rule name catalog entries use.""" + + def register(function: Evaluator) -> Evaluator: + EVALUATORS[name] = function + return function + + return register + + +def missing_steps( + spec: CheckSpec, outcomes: Mapping[str, Outcome] +) -> Evaluation | None: + """Return NOT_RUN when a required step never produced usable evidence.""" + for step in spec.steps: + if step.optional: + continue + outcome = outcomes.get(step.key) + if outcome is None: + return Evaluation(NOT_RUN, f"step {step.key} was never executed") + if outcome.truncated: + return Evaluation(NOT_RUN, f"step {step.key} output was truncated") + if not outcome.ran: + return Evaluation(NOT_RUN, f"step {step.key} did not run: {outcome.error}") + return None + + +def looks_denied(outcome: Outcome) -> bool: + """Report whether a real request was refused by policy rather than failing.""" + if outcome.ok: + return False + haystack = outcome.combined.lower() + return any(marker in haystack for marker in DENIAL_MARKERS) + + +def step_keys(spec: CheckSpec, kind: str) -> tuple[str, ...]: + """Return the keys of every step of one kind.""" + return tuple(step.key for step in spec.steps if step.kind == kind) + + +def is_list(value: Any) -> bool: + """Report whether a value is a JSON array rather than a string.""" + return isinstance(value, Sequence) and not isinstance(value, (str, bytes)) + + +def exact_equal(actual: Any, expected: Any) -> bool: + """Compare JSON values without Python's bool/int type confusion.""" + return type(actual) is type(expected) and actual == expected + + +def dotted(payload: Any, path: str) -> Any: + """Return a value addressed by a dotted path. + + ``a.b[0].c`` indexes; ``a[].b`` maps the remainder of the path over a list, + which is how the catalog records one field from every pull request or every + Flux object without a bespoke parser per check. + """ + current = payload + segments = [segment for segment in path.split(".") if segment] + for position, raw in enumerate(segments): + name, bracket, indexes = raw.partition("[") + if name: + if not isinstance(current, Mapping) or name not in current: + raise KeyError(path) + current = current[name] + if not bracket: + continue + for chunk in indexes.rstrip("]").split("]["): + if not is_list(current): + raise KeyError(path) + if chunk == "": + remainder = ".".join(segments[position + 1 :]) + return [ + dotted(item, remainder) if remainder else item for item in current + ] + offset = int(chunk) + if offset >= len(current): + raise KeyError(path) + current = current[offset] + return current + + +def strict_json(text: str) -> Any: + """Decode strict JSON, rejecting constants and duplicate object keys.""" + text = text.strip() + if not text: + raise ValueError("empty output") + + def reject_constant(value: str) -> None: + raise ValueError(f"non-finite JSON constant {value}") + + def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key {key!r}") + result[key] = value + return result + + return json.loads( + text, + strict=True, + parse_constant=reject_constant, + object_pairs_hook=unique_object, + ) + + +def parse_json(outcome: Outcome) -> Any: + """Parse one outcome as strict JSON.""" + return strict_json(outcome.stdout) + + +def lines(outcome: Outcome) -> list[str]: + """Return non-empty, stripped stdout lines.""" + return [line.strip() for line in outcome.stdout.splitlines() if line.strip()] + + +def parsed_step( + spec: CheckSpec, outcomes: dict[str, Outcome] +) -> tuple[Any, str] | Evaluation: + """Return the parsed payload of ``expect['step']`` or the NOT_RUN to report.""" + key = spec.expect["step"] + outcome = outcomes[key] + if not outcome.ok: + return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})") + try: + return (parse_json(outcome), key) + except ValueError as exc: + return Evaluation(NOT_RUN, f"step {key} did not return JSON: {exc}") + + +def evaluate(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation: + """Dispatch to the named evaluator, converting any surprise into NOT_RUN.""" + rule = EVALUATORS.get(spec.rule) + if rule is None: + return Evaluation(NOT_RUN, f"unknown evaluator: {spec.rule}") + truncated = sorted(key for key, outcome in outcomes.items() if outcome.truncated) + if truncated: + return Evaluation( + NOT_RUN, f"truncated evidence from step(s): {', '.join(truncated)}" + ) + try: + result = rule(spec, outcomes) + except Exception as exc: # noqa: BLE001 - a harness bug must never read as PASS + return Evaluation( + NOT_RUN, f"evaluator {spec.rule} raised {type(exc).__name__}: {exc}" + ) + if result.status not in STATUSES: + return Evaluation( + NOT_RUN, f"evaluator {spec.rule} returned unknown status {result.status!r}" + ) + return result diff --git a/scripts/ops/hermes_handoff_run.py b/scripts/ops/hermes_handoff_run.py new file mode 100644 index 00000000..45459dc8 --- /dev/null +++ b/scripts/ops/hermes_handoff_run.py @@ -0,0 +1,484 @@ +#!/usr/bin/env python3 +"""Orchestration for the Hermes full-handoff acceptance harness. + +Three things happen before any check runs, and each of them can stop the run on +its own: + +* the catalog is validated structurally — every rule exists, every live-denial + evaluator carries a read-only attempt, no operator-side step can impersonate; +* each vantage is resolved and asked who it is; and +* the operator and in-pod identities are compared. If they are the same + principal there is only one vantage, the dual-vantage evidence would be a + restatement rather than a corroboration, and the run is a NO_GO. + +Everything after that is uniform: run a check's steps against its vantage, hand +the outcomes to the named evaluator, and record the classification. A vantage +that could not be resolved yields ``NOT_RUN`` for every check that needs it, +which is exactly the fail-closed answer. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping, Sequence + +from hermes_handoff_catalog import ( + CHAT, + NODE, + OPERATOR, + SELF, + SWITCHYARD, + VANTAGE_NAMES, + Targets, +) +from hermes_handoff_checks_access import access_checks +from hermes_handoff_checks_delivery import delivery_checks +from hermes_handoff_checks_platform import platform_checks +from hermes_handoff_checks_workers import worker_checks +from hermes_handoff_evaluators import EVALUATORS, evaluate +from hermes_handoff_exec import Outcome, Runner, Vantage, operator_vantage, pod_vantage +from hermes_handoff_model import ( + ATTEMPT, + EPHEMERAL, + NOT_RUN, + STEP_KINDS, + CheckResult, + CheckSpec, + Report, + VantageRecord, + utc_now, +) +from hermes_handoff_policy import ( + EXPECTED_REMOTE, + EXPECTED_REPO, + IMPERSONATION_ARGS, + PolicyError, + check_argv, + projection, +) + +UNRECORDED = "[output not recorded]" + + +def build_catalog(targets: Targets) -> list[CheckSpec]: + """Return the whole acceptance catalog for a set of targets.""" + return [ + *platform_checks(targets), + *access_checks(targets), + *delivery_checks(targets), + *worker_checks(targets), + ] + + +def validate_catalog(specs: Sequence[CheckSpec]) -> list[str]: + """Return the structural problems that make a catalog unsafe to run.""" + problems: list[str] = [] + if not specs: + return ["catalog is empty"] + seen: set[str] = set() + fake_operator = operator_vantage() + fake_pod = pod_vantage("hermes", "acceptance-probe", "probe", fake_operator) + for spec in specs: + if spec.id in seen: + problems.append(f"{spec.id}: duplicate check id") + seen.add(spec.id) + if spec.rule not in EVALUATORS and spec.scope != EPHEMERAL: + problems.append(f"{spec.id}: unknown rule {spec.rule!r}") + step_names: set[str] = set() + for step in spec.steps: + if step.key in step_names: + problems.append(f"{spec.id}: duplicate step key {step.key!r}") + step_names.add(step.key) + if step.vantage not in VANTAGE_NAMES: + problems.append( + f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}" + ) + if step.kind not in STEP_KINDS: + problems.append( + f"{spec.id}: step {step.key} has unknown kind {step.kind!r}" + ) + if set(step.argv) & set(IMPERSONATION_ARGS): + problems.append( + f"{spec.id}: step {step.key} impersonates from the " + f"{step.vantage} vantage" + ) + try: + addressed = ( + fake_operator if step.vantage == OPERATOR else fake_pod + ).wrap(step.argv) + check_argv(addressed) + except PolicyError as exc: + problems.append(f"{spec.id}: step {step.key} violates policy: {exc}") + if spec.rule == "denied" and not any( + step.kind == ATTEMPT for step in spec.steps + ): + problems.append( + f"{spec.id}: a deny check needs a real attempt, not only a review" + ) + return problems + + +def validate_targets(targets: Targets) -> list[str]: + """Return release-input problems before a runner or network call exists.""" + problems: list[str] = [] + sha = re.compile(r"[0-9a-f]{40}\Z") + image = re.compile( + r"[^@\s]+:git-([0-9a-f]{40})-build-[1-9][0-9]*@sha256:[0-9a-f]{64}\Z" + ) + if targets.repo != EXPECTED_REPO or targets.remote != EXPECTED_REMOTE: + problems.append( + "repository and remote must be the fixed Atlas titan-iac origin" + ) + if ( + targets.reviewed_pr_number != 19 + or targets.reviewed_head_ref != "feature/hermes-full-handoff-acceptance" + ): + problems.append( + "reviewed PR identity must remain fixed to draft PR #19 and its existing branch" + ) + for label, value in ( + ("baseline commit", targets.baseline_commit), + ("remote main SHA", targets.remote_main_sha), + ("reviewed head SHA", targets.reviewed_head_sha), + ("build SHA", targets.build_sha), + ): + if not sha.fullmatch(value): + problems.append(f"{label} must be an exact lowercase 40-character SHA") + image_match = image.fullmatch(targets.agent_image) + if not image_match or image_match.group(1) != targets.build_sha: + problems.append("agent image tag/digest must bind the exact build SHA") + if targets.build_sha != targets.remote_main_sha: + problems.append("build SHA must equal the exact release main SHA") + if not re.fullmatch(r"[1-9][0-9]{0,8}", str(targets.deployment_revision)): + problems.append("deployment revision must be a positive integer string") + if targets.node_count != 3 or targets.pool_replicas != 3: + problems.append("node and pool counts are fixed at exactly three") + if not targets.chat_config_revision.strip(): + problems.append( + "Telegram continuity and its chat config revision are mandatory" + ) + if set(targets.pool_worker_env) != {"HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL"}: + problems.append("pool assignment evidence must include worker node and ordinal") + heads = dict(targets.dependency_heads) + if len(heads) != len(targets.dependency_heads) or set(heads) != set( + targets.dependency_pull_requests + ): + problems.append("every dependency PR must have one exact current head") + elif any(not sha.fullmatch(value) for value in heads.values()): + problems.append("dependency heads must be exact lowercase 40-character SHAs") + if ( + not isinstance(targets.max_evidence_age_seconds, int) + or targets.max_evidence_age_seconds <= 0 + ): + problems.append("maximum evidence age must be a positive integer") + return problems + + +def resolve_identity(runner: Runner, vantage: Vantage) -> tuple[str, str]: + """Return the authenticated username of a vantage and any failure detail.""" + outcome = runner.run( + ( + "kubectl", + "auth", + "whoami", + "-o", + projection("jsonpath={.status.userInfo.username}"), + ), + vantage, + ) + if not outcome.ok: + return ("", outcome.error or outcome.combined[:200]) + username = outcome.stdout.strip() + if not username or any(character.isspace() for character in username): + return ("", "identity response carried no username") + return (username, "") + + +def _first_pod( + runner: Runner, operator: Vantage, namespace: str, selector: str +) -> tuple[str, str]: + outcome = runner.run( + ( + "kubectl", + "--namespace", + namespace, + "get", + "pods", + "--selector", + selector, + "--field-selector", + "status.phase=Running", + "-o", + "name", + ), + operator, + ) + if not outcome.ok: + return ("", outcome.error or outcome.combined[:200]) + names = [ + line.strip().removeprefix("pod/") + for line in outcome.stdout.splitlines() + if line.strip() + ] + if not names: + return ("", f"no running pod matches {selector}") + return (sorted(names)[0], "") + + +def resolve_vantages( + runner: Runner, + targets: Targets, + kubeconfig: str | None = None, + context: str | None = None, +) -> tuple[dict[str, Vantage], list[VantageRecord]]: + """Resolve every vantage the catalog names, recording what each one is.""" + operator = operator_vantage(kubeconfig, context) + username, detail = resolve_identity(runner, operator) + records = [ + VantageRecord( + name=OPERATOR, + description=operator.description or "external read-only operator", + identity=username, + available=bool(username), + detail=detail, + ) + ] + vantages: dict[str, Vantage] = {OPERATOR: operator} + if not username: + return (vantages, records) + + for name, selector, container, description in ( + ( + SELF, + f"app={targets.agent_deployment}", + targets.agent_container, + "the Hermes agent itself", + ), + ( + SWITCHYARD, + f"app={targets.switchyard_deployment}", + targets.switchyard_container, + "the Switchyard routing evidence log", + ), + ( + NODE, + f"app={targets.node_daemonset}", + targets.node_container, + "a node hardening probe", + ), + ): + pod, failure = _first_pod(runner, operator, targets.namespace, selector) + if not pod: + records.append( + VantageRecord( + name=name, description=description, available=False, detail=failure + ) + ) + continue + vantage = pod_vantage(targets.namespace, pod, container, operator) + identity, identity_detail = ( + resolve_identity(runner, vantage) if name == SELF else ("", "") + ) + records.append( + VantageRecord( + name=name, + description=f"{description} ({targets.namespace}/{pod})", + identity=identity, + available=True, + detail=identity_detail, + ) + ) + vantages[name] = vantage + + chat_pod = f"{targets.chat_statefulset}-{targets.chat_ordinal}" + present = runner.run( + ( + "kubectl", + "--namespace", + targets.namespace, + "get", + f"pod/{chat_pod}", + "-o", + "name", + ), + operator, + ) + if present.ok and present.stdout.strip(): + vantages[CHAT] = pod_vantage( + targets.namespace, chat_pod, targets.chat_container, operator + ) + records.append( + VantageRecord( + name=CHAT, + description=f"the chat surface ({targets.namespace}/{chat_pod})", + available=True, + ) + ) + else: + records.append( + VantageRecord( + name=CHAT, + description=f"the chat surface ({targets.namespace}/{chat_pod})", + available=False, + detail=present.error + or present.combined[:200] + or "chat tenant pod not found", + ) + ) + return (vantages, records) + + +def vantage_problems(records: Iterable[VantageRecord]) -> list[str]: + """Return the identity problems that invalidate dual-vantage evidence.""" + by_name = {record.name: record for record in records} + operator = by_name.get(OPERATOR) + hermes = by_name.get(SELF) + problems: list[str] = [] + if operator is None or not operator.available: + problems.append( + "the operator vantage is unavailable; no evidence can be collected" + ) + return problems + if hermes is None or not hermes.available: + problems.append( + "the in-pod Hermes vantage is unavailable; self-probe evidence is missing" + ) + return problems + if not hermes.identity: + problems.append("the in-pod vantage did not report an identity") + elif operator.identity == hermes.identity: + problems.append( + "the operator and in-pod vantages authenticate as the same principal " + f"({operator.identity}); run the harness with a separate operator kubeconfig" + ) + for name, label in ((SWITCHYARD, "Switchyard"), (CHAT, "chat/Telegram")): + record = by_name.get(name) + if record is None or not record.available: + problems.append(f"the required {label} vantage is unavailable") + return problems + + +def run_check( + runner: Runner, spec: CheckSpec, vantages: Mapping[str, Vantage] +) -> CheckResult: + """Run one check's steps and classify the result.""" + outcomes: dict[str, Outcome] = {} + recorded: list[Outcome] = [] + for step in spec.steps: + vantage = vantages.get(step.vantage) + if vantage is None: + outcome = Outcome( + argv=step.argv, + vantage=step.vantage, + error=f"vantage {step.vantage} is unavailable", + ) + else: + outcome = runner.run(step.argv, vantage, step.max_bytes) + outcomes[step.key] = outcome + recorded.append( + outcome + if step.record + else Outcome( + argv=outcome.argv, + vantage=outcome.vantage, + returncode=outcome.returncode, + stdout=UNRECORDED if outcome.stdout else "", + stderr=UNRECORDED if outcome.stderr else "", + truncated=outcome.truncated, + duration_ms=outcome.duration_ms, + error=outcome.error, + executable_path=outcome.executable_path, + executable_sha256=outcome.executable_sha256, + ) + ) + evaluation = evaluate(spec, outcomes) + return CheckResult( + spec=spec, + status=evaluation.status, + reason=evaluation.reason, + evidence=evaluation.evidence, + outcomes=recorded, + ) + + +def run_catalog( + runner: Runner, specs: Sequence[CheckSpec], vantages: Mapping[str, Vantage] +) -> list[CheckResult]: + """Run every check in the catalog. + + Nothing is skipped once the deadline passes: the runner returns a + deadline outcome for each remaining command, which classifies as + ``NOT_RUN``, so a truncated sweep is visible in the report rather than + silently shorter. + """ + return [run_check(runner, spec, vantages) for spec in specs] + + +def build_report( + runner: Runner, + targets: Targets, + specs: Sequence[CheckSpec], + vantages: Mapping[str, Vantage], + records: Sequence[VantageRecord], + mode: str, + started_at: str, + extra_results: Sequence[CheckResult] = (), +) -> Report: + """Run the catalog and assemble the report, including structural problems.""" + problems = validate_catalog(specs) + vantage_problems(records) + report = Report( + mode=mode, + started_at=started_at, + baseline={ + "namespace": targets.namespace, + "baseline_commit": targets.baseline_commit, + "dependency_pull_requests": list(targets.dependency_pull_requests), + "expected_suspensions": list(targets.expected_suspensions), + "node_count": targets.node_count, + "pool_replicas": targets.pool_replicas, + }, + vantages=list(records), + harness_errors=problems, + ) + armed_ids = {result.spec.id for result in extra_results} + runnable = [spec for spec in specs if spec.id not in armed_ids] + if problems: + report.results = [ + CheckResult(spec=spec, status=NOT_RUN, reason="harness preflight failed") + for spec in runnable + ] + report.results.extend(extra_results) + else: + report.results = [*run_catalog(runner, runnable, vantages), *extra_results] + report.results.sort(key=lambda result: result.spec.id) + if runner.remaining_seconds <= 0: + report.harness_errors.append( + "the run deadline expired before every check completed" + ) + report.finished_at = utc_now() + return report + + +def unavailable_report( + mode: str, started_at: str, reason: str, targets: Targets +) -> Report: + """Return a NO_GO report for a run that could not start at all.""" + return Report( + mode=mode, + started_at=started_at, + finished_at=utc_now(), + baseline={"namespace": targets.namespace}, + harness_errors=[reason], + results=[ + CheckResult( + spec=CheckSpec( + id="harness.startup", + title="The harness could start and resolve its vantages", + group="harness", + rule="allowed", + ), + status=NOT_RUN, + reason=reason, + ) + ], + ) diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 7ae26434..c36a792e 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260817-cli-auto-quota-routing-v1" + ai.bstein.dev/config-rev: "20260818-ai-usage-exporter-split-v3" prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "9010" diff --git a/services/hermes/chat-statefulset.yaml b/services/hermes/chat-statefulset.yaml index dda35cd7..3d509116 100644 --- a/services/hermes/chat-statefulset.yaml +++ b/services/hermes/chat-statefulset.yaml @@ -204,6 +204,9 @@ spec: - | /opt/hermes/.venv/bin/python /opt/coordinator/patch_api_server_sessions.py \ /opt/hermes/gateway/platforms/api_server.py /patched/api_server.py + grep -Fq 'conversation_history = compact_telegram_history(conversation_history)' /patched/api_server.py + grep -Fq 'full_history = compact_telegram_history(full_history)' /patched/api_server.py + grep -Fq 'conversation_history_snapshot = compact_telegram_history(' /patched/api_server.py /opt/hermes/.venv/bin/python /opt/coordinator/migrate_telegram_api_sessions.py \ /opt/data/state.db /opt/data/response_store.db securityContext: @@ -284,6 +287,7 @@ spec: - {name: runtime-access, mountPath: /runtime-access} - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} - {name: api-server-patch, mountPath: /opt/hermes/gateway/platforms/api_server.py, subPath: api_server.py} + - {name: coordinator, mountPath: /opt/hermes/gateway/platforms/telegram_continuity.py, subPath: migrate_telegram_api_sessions.py, readOnly: true} - {name: stream-recovery-patch, mountPath: /opt/hermes/agent/conversation_loop.py, subPath: conversation_loop.py} - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/environments/local.py, subPath: local.py} - {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py} diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 7abeb694..4636e301 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -97,7 +97,9 @@ configMapGenerator: - name: hermes-coordinator namespace: hermes files: + - ai_usage_codex.py=scripts/ai_usage_codex.py - ai_usage_exporter.py=scripts/ai_usage_exporter.py + - ai_usage_http.py=scripts/ai_usage_http.py - claude=scripts/claude - claude_command_policy.py=scripts/claude_command_policy.py - cli_lane_board.py=scripts/cli_lane_board.py diff --git a/services/hermes/router/main.go b/services/hermes/router/main.go index 16d7656c..db115242 100644 --- a/services/hermes/router/main.go +++ b/services/hermes/router/main.go @@ -27,8 +27,9 @@ type linkRecord struct { } type telegramTopic struct { - Label string `json:"label"` - LastUsed int64 `json:"last_used"` + Label string `json:"label"` + LastUsed int64 `json:"last_used"` + Generation int64 `json:"generation,omitempty"` } type telegramTopicState struct { @@ -372,7 +373,12 @@ func (router *tenantRouter) ServeHTTP(writer http.ResponseWriter, request *http. return } proxy := httputil.NewSingleHostReverseProxy(target) - proxy.ModifyResponse = injectChatBridge + proxy.ModifyResponse = func(response *http.Response) error { + if err := boundSessionSnapshot(response); err != nil { + return err + } + return injectChatBridge(response) + } proxy.ErrorHandler = func(writer http.ResponseWriter, _ *http.Request, proxyErr error) { log.Printf("tenant slot %d unavailable", slot) http.Error(writer, "private chat runtime is starting", http.StatusBadGateway) diff --git a/services/hermes/router/main_test.go b/services/hermes/router/main_test.go index a17b020a..152ebeca 100644 --- a/services/hermes/router/main_test.go +++ b/services/hermes/router/main_test.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "io" "net/http" "net/http/httptest" @@ -126,6 +127,9 @@ func TestRouterProxiesWebUIAndAddsTelegramShortcut(t *testing.T) { if !strings.Contains(response.Body.String(), "hermes-chat-bridge.js") || !strings.Contains(response.Body.String(), "hermes-chat-bridge.css") { t.Fatal("Telegram shortcut assets were not injected") } + if !strings.Contains(response.Body.String(), "hermes-session-continuity.js") { + t.Fatal("session continuity fallback was not injected") + } assetRequest := httptest.NewRequest(http.MethodGet, "/hermes-chat-bridge.js", nil) assetRequest.Header.Set("X-Forwarded-User", "subject") assetResponse := httptest.NewRecorder() @@ -145,6 +149,75 @@ func TestRouterProxiesWebUIAndAddsTelegramShortcut(t *testing.T) { } } +func TestSessionContinuityAssetHasAccessiblePollFallback(t *testing.T) { + router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(int) string { return "" }) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodGet, "/hermes-session-continuity.js", nil) + request.Header.Set("X-Forwarded-User", "subject") + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("got status %d", response.Code) + } + asset := response.Body.String() + for _, expected := range []string{ + "aria-live", "aria-busy", "/api/sessions/", "/messages?limit=24&hermes_fallback=1", + "Latest stored activity", "no renderable messages", + "Session updates disconnected", "oauth2/start?rd=", "payload.session_id", + "session.ended_at == null", "latest.observed", "addEventListener('online'", + "addEventListener('pageshow'", "schedule(document.hidden", "fetch(", + } { + if !strings.Contains(asset, expected) { + t.Fatalf("session fallback omitted %q", expected) + } + } + for _, forbidden := range []string{ + "location.replace", "location.reload", "history.replaceState", "/api/session?", "WebSocket", + } { + if strings.Contains(asset, forbidden) { + t.Fatalf("session fallback interferes with native continuity via %q", forbidden) + } + } +} + +func TestSessionContinuityPollIsBoundedByRouter(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + messages := make([]map[string]int, 30) + for index := range messages { + messages[index] = map[string]int{"index": index} + } + writer.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(writer).Encode(map[string]any{ + "session_id": "resolved", "messages": messages, + }) + })) + defer backend.Close() + router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(int) string { return backend.URL }) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest( + http.MethodGet, + "/api/sessions/root/messages?limit=24&hermes_fallback=1", + nil, + ) + request.Header.Set("X-Forwarded-User", "subject") + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("got status %d", response.Code) + } + var payload sessionSnapshot + if err := json.NewDecoder(response.Body).Decode(&payload); err != nil { + t.Fatal(err) + } + if len(payload.Messages) != sessionSnapshotItems || payload.TotalMessages != 30 { + t.Fatalf("router returned unbounded fallback: %#v", payload) + } +} + func TestTelegramPageExplainsOneTimeOperatorSetup(t *testing.T) { router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(slot int) string { return "" }) if err != nil { @@ -294,11 +367,52 @@ func TestPrivateFileBrowserIsTenantAuthenticatedAndHidesKanban(t *testing.T) { assetRequest.Header.Set("X-Forwarded-User", "subject") assetResponse := httptest.NewRecorder() router.ServeHTTP(assetResponse, assetRequest) - if !strings.Contains(assetResponse.Body.String(), "endpoint('list'") || !strings.Contains(assetResponse.Body.String(), `'[data-panel="kanban"]'`) { + if !strings.Contains(assetResponse.Body.String(), "endpoint('list'") || !strings.Contains(assetResponse.Body.String(), `'[data-panel="' + panel + '"]'`) { t.Fatal("file browsing or chat-only navigation policy is missing") } } +func TestChatBridgeHidesOnlyNonChatCockpitChrome(t *testing.T) { + router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(slot int) string { return "" }) + if err != nil { + t.Fatal(err) + } + asset := func(path string) string { + request := httptest.NewRequest(http.MethodGet, path, nil) + request.Header.Set("X-Forwarded-User", "subject") + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("%s got status %d", path, response.Code) + } + return response.Body.String() + } + js := asset("/hermes-chat-bridge.js") + css := asset("/hermes-chat-bridge.css") + + // The whole cockpit (Kanban plus every non-chat rail panel) is declutter-hidden + // both pre-JS (CSS, no flash) and by the runtime .hidden reinforcement. + for _, panel := range []string{"kanban", "logs", "insights", "memory", "skills", "workspaces", "todos", "tasks", "profiles"} { + if !strings.Contains(css, `[data-panel="`+panel+`"]`) { + t.Fatalf("bridge CSS does not hide cockpit panel %q", panel) + } + if !strings.Contains(js, `'`+panel+`'`) { + t.Fatalf("bridge JS does not mark cockpit panel %q hidden", panel) + } + } + if !strings.Contains(css, "display:none") || !strings.Contains(css, "#settingsModel") { + t.Fatal("bridge CSS must hide the cockpit panels and the redundant settings model selector") + } + + // The clean chat experience (chat surface, history, composer, model chip) + // must never be part of the hide set. + for _, keep := range []string{`[data-panel="chat"]`, `[data-panel="history"]`, `[data-panel="sessions"]`, `[data-panel="composer"]`, "composerRoutingChip", "composerRoutingLabel"} { + if strings.Contains(css, keep) || strings.Contains(js, keep) { + t.Fatalf("declutter wrongly targets a core chat control: %q", keep) + } + } +} + func TestWebUIModelAndReasoningOverridesAreProxied(t *testing.T) { backend := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { if request.Method != http.MethodPost { diff --git a/services/hermes/router/session_continuity.go b/services/hermes/router/session_continuity.go new file mode 100644 index 00000000..af42b35d --- /dev/null +++ b/services/hermes/router/session_continuity.go @@ -0,0 +1,97 @@ +package main + +const sessionContinuityJS = `(() => { + const match = location.pathname.match(/^\/session\/([^/]+)\/?$/); + if (!match) return; + let sessionId; + try { sessionId = decodeURIComponent(match[1]); } catch (_) { sessionId = match[1]; } + const started = Date.now(); + let timer = 0; + let request = null; + const card = document.createElement('aside'); + card.id = 'hermes-session-continuity'; + card.setAttribute('role', 'status'); + card.setAttribute('aria-live', 'polite'); + card.setAttribute('aria-atomic', 'true'); + card.hidden = true; + card.style.cssText = 'position:fixed;right:16px;bottom:16px;z-index:1000;max-width:min(420px,calc(100vw - 32px));box-sizing:border-box;padding:10px 12px;border:1px solid #475569;border-radius:10px;background:#111827;color:#e5e7eb;font:13px/1.4 system-ui,sans-serif;box-shadow:0 8px 28px #0008'; + const message = document.createElement('span'); + const newChat = document.createElement('a'); + newChat.href = '/'; + newChat.textContent = ' Start a new chat.'; + newChat.style.color = '#7dd3fc'; + card.append(message, newChat); + document.body.appendChild(card); + + const show = (text, busy, link) => { + message.textContent = text; + card.hidden = false; + card.setAttribute('aria-busy', busy ? 'true' : 'false'); + newChat.hidden = !link; + }; + const hide = () => { card.hidden = true; card.setAttribute('aria-busy', 'false'); }; + const activityLabel = (messages) => { + const latest = messages[messages.length - 1] || {}; + if (latest.tool_name) return 'tool ' + String(latest.tool_name).slice(0, 80); + if (Array.isArray(latest.tool_calls) && latest.tool_calls.length) { + const call = latest.tool_calls[latest.tool_calls.length - 1] || {}; + return 'tool ' + String((call.function || {}).name || 'activity').slice(0, 80); + } + if (latest.activity_event) return String(latest.activity_event).replace(/[._-]+/g, ' ').slice(0, 80); + if (latest.observed && typeof latest.content === 'string') return latest.content.slice(0, 120); + return latest.role === 'assistant' ? 'assistant update' : latest.role === 'user' ? 'request stored' : 'working'; + }; + const authOrMissing = (response) => { + if (response.status === 401 || response.status === 403) { + const rd = location.pathname + location.search + location.hash; + location.assign('/oauth2/start?rd=' + encodeURIComponent(rd)); + return 'auth'; + } + if (response.status === 404) { + show('This session is unavailable to this account.', false, true); + schedule(10000); + return 'missing'; + } + return ''; + }; + const schedule = (delay) => { + clearTimeout(timer); + timer = window.setTimeout(poll, delay); + }; + async function poll() { + if (request) return; + request = new AbortController(); + const timeout = window.setTimeout(() => request && request.abort(), 8000); + try { + const root = '/api/sessions/' + encodeURIComponent(sessionId); + const messageResponse = await fetch(root + '/messages?limit=24&hermes_fallback=1', {cache:'no-store', credentials:'same-origin', signal:request.signal}); + if (authOrMissing(messageResponse)) return; + if (!messageResponse.ok) throw new Error('session messages poll failed'); + const payload = await messageResponse.json(); + const messages = Array.isArray(payload.messages) ? payload.messages : []; + const resolvedId = typeof payload.session_id === 'string' && payload.session_id ? payload.session_id : sessionId; + const detailResponse = await fetch('/api/sessions/' + encodeURIComponent(resolvedId), {cache:'no-store', credentials:'same-origin', signal:request.signal}); + if (authOrMissing(detailResponse)) return; + if (!detailResponse.ok) throw new Error('session detail poll failed'); + const session = await detailResponse.json(); + const lastActive = Number(session.last_active || session.started_at || 0); + const durableWorker = session.source === 'api_server' && Boolean(session.parent_session_id); + const recentlyInteractive = Number.isFinite(lastActive) && Date.now() / 1000 - lastActive < 300; + const active = session.ended_at == null && messages.length > 0 && (durableWorker || recentlyInteractive); + if (active) show('Hermes is working. Latest stored activity: ' + activityLabel(messages) + '.', true, false); + else if (!messages.length && Date.now() - started >= 4000) show('This session has no renderable messages yet. It may be new or no longer available.', false, true); + else hide(); + schedule(document.hidden ? 10000 : 2500); + } catch (_) { + show('Session updates disconnected. Retrying without changing this session…', true, false); + schedule(document.hidden ? 10000 : 3000); + } finally { + clearTimeout(timeout); + request = null; + } + } + addEventListener('online', () => schedule(0)); + addEventListener('pageshow', () => schedule(0)); + document.addEventListener('visibilitychange', () => schedule(0)); + schedule(1200); +})();` diff --git a/services/hermes/router/session_snapshot.go b/services/hermes/router/session_snapshot.go new file mode 100644 index 00000000..237dd19b --- /dev/null +++ b/services/hermes/router/session_snapshot.go @@ -0,0 +1,65 @@ +package main + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "strconv" + "strings" +) + +const ( + sessionSnapshotItems = 24 + sessionSnapshotBytes = 8 << 20 +) + +type sessionSnapshot struct { + SessionID string `json:"session_id"` + Messages []json.RawMessage `json:"messages"` + TotalMessages int `json:"total_messages"` +} + +// boundSessionSnapshot caps only the continuity fallback response. Native +// WebUI requests remain untouched, including when an older backend ignores +// its optional `limit` query parameter. +func boundSessionSnapshot(response *http.Response) error { + request := response.Request + if request == nil || response.StatusCode != http.StatusOK || + request.URL.Query().Get("hermes_fallback") != "1" || + !strings.HasPrefix(request.URL.Path, "/api/sessions/") || + !strings.HasSuffix(request.URL.Path, "/messages") { + return nil + } + body, err := io.ReadAll(io.LimitReader(response.Body, sessionSnapshotBytes+1)) + if err != nil { + return err + } + _ = response.Body.Close() + if len(body) > sessionSnapshotBytes { + return errors.New("session snapshot exceeds safe response limit") + } + var payload sessionSnapshot + if err := json.Unmarshal(body, &payload); err != nil || payload.Messages == nil { + return errors.New("session snapshot is malformed") + } + total := len(payload.Messages) + if payload.TotalMessages > total { + total = payload.TotalMessages + } + if len(payload.Messages) > sessionSnapshotItems { + payload.Messages = payload.Messages[len(payload.Messages)-sessionSnapshotItems:] + } + payload.TotalMessages = total + body, err = json.Marshal(payload) + if err != nil { + return err + } + response.Body = io.NopCloser(strings.NewReader(string(body))) + response.ContentLength = int64(len(body)) + response.Header.Set("Content-Length", strconv.Itoa(len(body))) + response.Header.Set("Content-Type", "application/json; charset=utf-8") + response.Header.Set("Cache-Control", "no-store") + response.Header.Del("ETag") + return nil +} diff --git a/services/hermes/router/session_snapshot_test.go b/services/hermes/router/session_snapshot_test.go new file mode 100644 index 00000000..64aee010 --- /dev/null +++ b/services/hermes/router/session_snapshot_test.go @@ -0,0 +1,113 @@ +package main + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "strings" + "testing" + "testing/iotest" +) + +func snapshotResponse(target, body string) *http.Response { + request, _ := http.NewRequest(http.MethodGet, target, nil) + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(body)), + Request: request, + } +} + +func TestBoundSessionSnapshotKeepsOnlyRecentMessages(t *testing.T) { + messages := make([]map[string]int, 30) + for index := range messages { + messages[index] = map[string]int{"index": index} + } + body, _ := json.Marshal(map[string]any{ + "session_id": "resolved", "messages": messages, + }) + response := snapshotResponse( + "http://tenant/api/sessions/root/messages?limit=24&hermes_fallback=1", + string(body), + ) + response.Header.Set("ETag", "stale") + if err := boundSessionSnapshot(response); err != nil { + t.Fatal(err) + } + var bounded sessionSnapshot + if err := json.NewDecoder(response.Body).Decode(&bounded); err != nil { + t.Fatal(err) + } + if len(bounded.Messages) != 24 || bounded.TotalMessages != 30 { + t.Fatalf("snapshot was not bounded: %#v", bounded) + } + var first map[string]int + if err := json.Unmarshal(bounded.Messages[0], &first); err != nil || first["index"] != 6 { + t.Fatalf("snapshot did not retain the recent tail: %#v, %v", first, err) + } + if response.Header.Get("Cache-Control") != "no-store" || response.Header.Get("ETag") != "" { + t.Fatal("bounded snapshot retained cache metadata") + } +} + +func TestBoundSessionSnapshotPreservesLargerReportedTotal(t *testing.T) { + response := snapshotResponse( + "http://tenant/api/sessions/root/messages?hermes_fallback=1", + `{"session_id":"leaf","messages":[],"total_messages":100}`, + ) + if err := boundSessionSnapshot(response); err != nil { + t.Fatal(err) + } + var bounded sessionSnapshot + if err := json.NewDecoder(response.Body).Decode(&bounded); err != nil { + t.Fatal(err) + } + if bounded.SessionID != "leaf" || bounded.TotalMessages != 100 { + t.Fatalf("reported total was lost: %#v", bounded) + } +} + +func TestBoundSessionSnapshotFailsClosedOnMalformedOrOversizedBody(t *testing.T) { + for name, body := range map[string]string{ + "malformed": `{`, + "oversized": strings.Repeat("x", sessionSnapshotBytes+1), + } { + t.Run(name, func(t *testing.T) { + response := snapshotResponse( + "http://tenant/api/sessions/root/messages?hermes_fallback=1", + body, + ) + if err := boundSessionSnapshot(response); err == nil { + t.Fatal("unsafe snapshot was accepted") + } + }) + } + response := snapshotResponse( + "http://tenant/api/sessions/root/messages?hermes_fallback=1", `{}`, + ) + response.Body = io.NopCloser(iotest.ErrReader(errors.New("read failed"))) + if err := boundSessionSnapshot(response); err == nil { + t.Fatal("snapshot body read failure was ignored") + } +} + +func TestBoundSessionSnapshotLeavesNativeAndErrorResponsesUntouched(t *testing.T) { + for _, response := range []*http.Response{ + {StatusCode: http.StatusOK}, + snapshotResponse("http://tenant/api/sessions/root/messages", `{}`), + snapshotResponse("http://tenant/api/sessions/root", `{}`), + } { + if err := boundSessionSnapshot(response); err != nil { + t.Fatal(err) + } + } + errorResponse := snapshotResponse( + "http://tenant/api/sessions/root/messages?hermes_fallback=1", `{}`, + ) + errorResponse.StatusCode = http.StatusNotFound + if err := boundSessionSnapshot(errorResponse); err != nil { + t.Fatal(err) + } +} diff --git a/services/hermes/router/telegram.go b/services/hermes/router/telegram.go index c4cc3b66..a2d591df 100644 --- a/services/hermes/router/telegram.go +++ b/services/hermes/router/telegram.go @@ -24,20 +24,21 @@ type telegramConfig struct { } type telegramBot struct { - config telegramConfig - router *tenantRouter - apiBase string - fileBase string - client *http.Client - mediaClient *http.Client - agentClient *http.Client - mu sync.RWMutex - botUsername string - ready bool - lastError string - workLimit chan struct{} - slotLocks map[int]*sync.Mutex - slotLocksMux sync.Mutex + config telegramConfig + router *tenantRouter + apiBase string + fileBase string + client *http.Client + mediaClient *http.Client + agentClient *http.Client + mu sync.RWMutex + botUsername string + ready bool + lastError string + workLimit chan struct{} + activityEvery time.Duration + slotLocks map[int]*sync.Mutex + slotLocksMux sync.Mutex } type telegramUpdate struct { @@ -96,15 +97,16 @@ func readTelegramConfig(path string) (telegramConfig, error) { func newTelegramBot(config telegramConfig, router *tenantRouter) *telegramBot { return &telegramBot{ - config: config, - router: router, - apiBase: "https://api.telegram.org/bot" + config.BotToken, - fileBase: "https://api.telegram.org/file/bot" + config.BotToken, - client: &http.Client{Timeout: 70 * time.Second}, - mediaClient: newTenantMediaClient(), - agentClient: &http.Client{Timeout: 15 * time.Minute}, - workLimit: make(chan struct{}, 4), - slotLocks: map[int]*sync.Mutex{}, + config: config, + router: router, + apiBase: "https://api.telegram.org/bot" + config.BotToken, + fileBase: "https://api.telegram.org/file/bot" + config.BotToken, + client: &http.Client{Timeout: 70 * time.Second}, + mediaClient: newTenantMediaClient(), + agentClient: &http.Client{Timeout: 15 * time.Minute}, + workLimit: make(chan struct{}, 4), + activityEvery: 4 * time.Second, + slotLocks: map[int]*sync.Mutex{}, } } @@ -238,16 +240,6 @@ func (bot *telegramBot) run() { } } -func commandParts(text string) (string, []string) { - fields := strings.Fields(strings.TrimSpace(text)) - if len(fields) == 0 || !strings.HasPrefix(fields[0], "/") { - return "", nil - } - command := strings.TrimPrefix(strings.ToLower(fields[0]), "/") - command, _, _ = strings.Cut(command, "@") - return command, fields[1:] -} - func (bot *telegramBot) handleUpdate(update telegramUpdate) { message := update.Message if message == nil || message.From == nil || message.Chat.Type != "private" || message.Chat.ID != message.From.ID { @@ -276,7 +268,7 @@ func (bot *telegramBot) handleUpdate(update telegramUpdate) { return } if command == "help" { - _ = bot.sendText(message.Chat.ID, "Send text or a photo to chat with Hermes. Use /topic to create or switch a durable topic, /topics to list topics, or /unlink to disconnect this Telegram account. Model and intensity controls are available in Hermes WebUI.") + _ = bot.sendText(message.Chat.ID, "Send text or a photo to chat with Hermes. Photos are available for analysis on the message that carries them. Image edits can reuse the latest image Hermes generated; inbound Telegram photos are not stored as editable artifacts. Use /topic to create or switch a durable topic, /topics to list topics, or /unlink to disconnect this Telegram account. Model and intensity controls are available in Hermes WebUI.") return } slot, linked := bot.router.telegramSlot(userID) @@ -332,7 +324,9 @@ func (bot *telegramBot) handleUpdate(update telegramUpdate) { _ = bot.sendText(message.Chat.ID, "Hermes could not save the active topic right now. Please try again shortly.") return } - _ = bot.sendAction(message.Chat.ID, "typing") + text = telegramInputPrompt(text, hasPhoto) + stopActivity := bot.startTelegramActivity(message.Chat.ID) + defer stopActivity() var reply string if hasPhoto { media, fetchErr := bot.fetchTelegramPhoto(message.Photo) @@ -429,6 +423,10 @@ func (bot *telegramBot) askTenantRequest(slot int, bodyReader io.Reader, updateI func (bot *telegramBot) sendAction(chatID int64, action string) error { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() + return bot.sendActionContext(ctx, chatID, action) +} + +func (bot *telegramBot) sendActionContext(ctx context.Context, chatID int64, action string) error { return bot.call(ctx, "sendChatAction", url.Values{ "chat_id": {strconv.FormatInt(chatID, 10)}, "action": {action}, diff --git a/services/hermes/router/telegram_input.go b/services/hermes/router/telegram_input.go index 40be7f96..5e2c5956 100644 --- a/services/hermes/router/telegram_input.go +++ b/services/hermes/router/telegram_input.go @@ -12,6 +12,7 @@ import ( "os" "path" "strings" + "sync" "time" ) @@ -22,6 +23,93 @@ type telegramInputMedia struct { Path string } +func commandParts(text string) (string, []string) { + fields := strings.Fields(strings.TrimSpace(text)) + if len(fields) == 0 || !strings.HasPrefix(fields[0], "/") { + return "", nil + } + command := strings.TrimPrefix(strings.ToLower(fields[0]), "/") + command, _, _ = strings.Cut(command, "@") + return command, fields[1:] +} + +const inboundTelegramEditNotice = "Transport note: this Telegram photo is available for visual analysis on this turn, but it is not stored as a generated-image artifact. The image_edit_latest tools can reuse only an image Hermes generated. Do not claim an exact edit of this inbound photo; explain the limitation and offer either a new image inspired by it or an edit of the latest Hermes-generated image." + +func looksLikeImageEdit(text string) bool { + lower := strings.ToLower(strings.Join(strings.Fields(text), " ")) + for _, phrase := range []string{ + "edit this", "edit the image", "edit the photo", "change this image", + "change this photo", "modify this", "retouch", "turn this into", + "turn this cat", "turn this dog", "remove from the image", + "replace in the image", "edit it", "change it", "modify it", "crop it", + "resize it", "upscale it", + } { + if strings.Contains(lower, phrase) { + return true + } + } + hasTarget := false + for _, target := range []string{"image", "photo", "picture", "portrait", "selfie", "screenshot"} { + if strings.Contains(lower, target) { + hasTarget = true + break + } + } + if hasTarget { + for _, action := range []string{ + "add ", "blur", "brighten", "change", "color", "convert", "crop", + "edit", "make ", "modify", "remove", "replace", "resize", "sharpen", + "transform", "turn ", "upscale", + } { + if strings.Contains(lower, action) { + return true + } + } + } + return false +} + +func telegramInputPrompt(text string, hasPhoto bool) string { + if hasPhoto { + return inboundTelegramEditNotice + "\n\nUser request: " + text + } + if !looksLikeImageEdit(text) { + return text + } + return "Transport note: no image is attached to this Telegram message. Use image_edit_latest only if Hermes previously generated an image in this private tenant. Otherwise do not claim access to an earlier inbound Telegram photo; offer a new image or ask the user to use Hermes WebUI with the source attachment.\n\nUser request: " + text +} + +func (bot *telegramBot) startTelegramActivity(chatID int64) func() { + done := make(chan struct{}) + stopped := make(chan struct{}) + ctx, cancel := context.WithCancel(context.Background()) + every := bot.activityEvery + if every <= 0 { + every = 4 * time.Second + } + go func() { + defer close(stopped) + ticker := time.NewTicker(every) + defer ticker.Stop() + for { + _ = bot.sendActionContext(ctx, chatID, "typing") + select { + case <-done: + return + case <-ticker.C: + } + } + }() + var once sync.Once + return func() { + once.Do(func() { + close(done) + cancel() + }) + <-stopped + } +} + func largestTelegramPhoto(photos []telegramPhotoSize) (telegramPhotoSize, error) { var selected telegramPhotoSize for _, photo := range photos { diff --git a/services/hermes/router/telegram_media.go b/services/hermes/router/telegram_media.go index 90be976f..d4dacca9 100644 --- a/services/hermes/router/telegram_media.go +++ b/services/hermes/router/telegram_media.go @@ -24,7 +24,11 @@ const ( tenantMediaLimit = 50 << 20 ) -var telegramMediaMarker = regexp.MustCompile(`MEDIA:([^\s\)\]]+)`) +var ( + telegramMediaMarker = regexp.MustCompile(`(?i)MEDIA:\s*([^\s\)\]\}"']+)`) + telegramMediaLabel = regexp.MustCompile(`(?i)MEDIA:`) + telegramPrivatePath = regexp.MustCompile(`(?i)(?:/opt/data/(?:cache/images|workspace)|/workspace)/[^\s\)\]\}"']+`) +) type telegramReply struct { Text string @@ -50,6 +54,8 @@ func parseTelegramReply(text string) telegramReply { } } cleaned := telegramMediaMarker.ReplaceAllString(text, "") + cleaned = telegramPrivatePath.ReplaceAllString(cleaned, "[private attachment]") + cleaned = telegramMediaLabel.ReplaceAllString(cleaned, "") lines := strings.Split(cleaned, "\n") compact := make([]string, 0, len(lines)) blank := false diff --git a/services/hermes/router/telegram_media_test.go b/services/hermes/router/telegram_media_test.go index ee36f3d0..ff59bb2c 100644 --- a/services/hermes/router/telegram_media_test.go +++ b/services/hermes/router/telegram_media_test.go @@ -124,6 +124,20 @@ func TestParseTelegramReplyRemovesOneOrMoreInternalMarkers(t *testing.T) { } } +func TestParseTelegramReplyScrubsMalformedMarkersAndBarePrivatePaths(t *testing.T) { + reply := parseTelegramReply( + "Delivered. MEDIA: /opt/data/cache/images/one.png and /workspace/private/two.png; MEDIA:", + ) + if len(reply.MediaPaths) != 1 || reply.MediaPaths[0] != "/opt/data/cache/images/one.png" { + t.Fatalf("valid spaced marker was not retained for delivery: %#v", reply) + } + for _, unsafe := range []string{"MEDIA:", "/opt/data/", "/workspace/"} { + if strings.Contains(reply.Text, unsafe) { + t.Fatalf("unsafe transport detail %q leaked in %q", unsafe, reply.Text) + } + } +} + func TestNormalizeTenantMediaPath(t *testing.T) { allowed := map[string]string{ "/workspace/./renders//portrait.png": "/opt/data/workspace/renders/portrait.png", diff --git a/services/hermes/router/telegram_test.go b/services/hermes/router/telegram_test.go index 317dcadc..b192ad14 100644 --- a/services/hermes/router/telegram_test.go +++ b/services/hermes/router/telegram_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "sync/atomic" "testing" "time" ) @@ -273,6 +274,144 @@ func TestTelegramTopicValidationAndLegacyGeneralConversation(t *testing.T) { } } +func TestTelegramTopicContinuityExpiresOnlyAfterIdleWeek(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "state.json") + base := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC) + now := base + router, err := newTenantRouter(statePath, 1, func(int) string { return "" }) + if err != nil { + t.Fatal(err) + } + router.now = func() time.Time { return now } + first, err := router.selectTelegramTopic("123", "Week plan") + if err != nil { + t.Fatal(err) + } + for _, elapsed := range []time.Duration{time.Hour, 24 * time.Hour} { + now = base.Add(elapsed) + resumed, touchErr := router.touchTelegramTopic("123") + if touchErr != nil || resumed.Conversation != first.Conversation { + t.Fatalf("topic changed after %s: %#v, %v", elapsed, resumed, touchErr) + } + } + // Exactly seven idle days remains the same durable conversation. + now = base.Add(8 * 24 * time.Hour) + exactWeek, err := router.touchTelegramTopic("123") + if err != nil || exactWeek.Conversation != first.Conversation { + t.Fatalf("topic changed at exact idle week: %#v, %v", exactWeek, err) + } + reloaded, err := newTenantRouter(statePath, 1, func(int) string { return "" }) + if err != nil { + t.Fatal(err) + } + reloaded.now = func() time.Time { return now } + if active := reloaded.activeTelegramTopic("123"); active != exactWeek { + t.Fatalf("router restart changed topic: %#v != %#v", active, exactWeek) + } + now = now.Add(7*24*time.Hour + time.Second) + rotated, err := reloaded.touchTelegramTopic("123") + if err != nil { + t.Fatal(err) + } + if rotated.Conversation == first.Conversation || !strings.HasSuffix(rotated.Conversation, "-g1") { + t.Fatalf("expired topic did not rotate deterministically: %#v", rotated) + } + again, err := newTenantRouter(statePath, 1, func(int) string { return "" }) + if err != nil { + t.Fatal(err) + } + if active := again.activeTelegramTopic("123"); active != rotated { + t.Fatalf("rotated identity changed after restart: %#v != %#v", active, rotated) + } +} + +func TestTelegramActivityRepeatsAndStops(t *testing.T) { + var actions atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/sendChatAction" { + t.Fatalf("unexpected Telegram method %s", request.URL.Path) + } + actions.Add(1) + writer.Header().Set("Content-Type", "application/json") + _, _ = writer.Write([]byte(`{"ok":true,"result":true}`)) + })) + defer server.Close() + bot := &telegramBot{ + apiBase: server.URL, + client: &http.Client{Timeout: time.Second}, + activityEvery: 5 * time.Millisecond, + } + stop := bot.startTelegramActivity(42) + deadline := time.Now().Add(time.Second) + for actions.Load() < 2 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + stop() + stoppedAt := actions.Load() + time.Sleep(20 * time.Millisecond) + if stoppedAt < 2 || actions.Load() != stoppedAt { + t.Fatalf("activity did not progress and stop: before=%d after=%d", stoppedAt, actions.Load()) + } +} + +func TestTelegramActivityStopCancelsInflightRequest(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) { + close(started) + select { + case <-request.Context().Done(): + case <-release: + } + })) + defer server.Close() + bot := &telegramBot{ + apiBase: server.URL, + client: &http.Client{Timeout: time.Minute}, + activityEvery: time.Hour, + } + stop := bot.startTelegramActivity(42) + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("activity request did not start") + } + stopped := make(chan struct{}) + go func() { + stop() + close(stopped) + }() + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("activity stop did not cancel the in-flight request") + } + close(release) +} + +func TestTelegramImageEditPromptStatesTransportBoundary(t *testing.T) { + withPhoto := telegramInputPrompt("Edit this photo into a watercolor", true) + if !strings.Contains(withPhoto, "available for visual analysis") || + !strings.Contains(withPhoto, "not stored as a generated-image artifact") { + t.Fatalf("inbound attachment boundary missing: %q", withPhoto) + } + withoutPhoto := telegramInputPrompt("Edit this photo into a watercolor", false) + if !strings.Contains(withoutPhoto, "no image is attached") || + !strings.Contains(withoutPhoto, "image_edit_latest only if Hermes previously generated") { + t.Fatalf("prior generated image boundary missing: %q", withoutPhoto) + } + for _, request := range []string{"Make the photo brighter", "Crop it closer"} { + if got := telegramInputPrompt(request, false); !strings.Contains(got, "no image is attached") { + t.Fatalf("edit variant omitted attachment boundary: %q", got) + } + } + plain := telegramInputPrompt("Explain this photograph", true) + if !strings.Contains(plain, "available for visual analysis") || + !strings.Contains(plain, "User request: Explain this photograph") { + t.Fatalf("photo analysis boundary missing: %q", plain) + } +} + func TestTelegramInputRejectsUnsafePathsAndNonImages(t *testing.T) { for _, value := range []string{"", "../secret", "/absolute/image.jpg", `photos\image.jpg`, "photos/../../secret"} { if _, err := safeTelegramFilePath(value); err == nil { diff --git a/services/hermes/router/telegram_topics.go b/services/hermes/router/telegram_topics.go index d5e9bed6..59c1447a 100644 --- a/services/hermes/router/telegram_topics.go +++ b/services/hermes/router/telegram_topics.go @@ -5,12 +5,16 @@ import ( "encoding/hex" "errors" "sort" + "strconv" "strings" "unicode" "unicode/utf8" ) -const telegramDefaultTopic = "General" +const ( + telegramDefaultTopic = "General" + telegramTopicIdleTTL = 7 * 24 * 60 * 60 +) type activeTelegramTopic struct { Conversation string @@ -42,12 +46,35 @@ func telegramTopicID(label string) string { } func telegramTopicConversation(topicID string) string { + return telegramTopicConversationGeneration(topicID, 0) +} + +func telegramTopicConversationGeneration(topicID string, generation int64) string { if topicID == "" || topicID == "general" { // Preserve the original named conversation so existing Telegram context // survives rollout into topic-aware routing. - return "telegram" + if generation == 0 { + return "telegram" + } + topicID = "general" } - return "telegram-topic-" + topicID + conversation := "telegram-topic-" + topicID + if generation > 0 { + conversation += "-g" + strconv.FormatInt(generation, 10) + } + return conversation +} + +func topicExpired(topic telegramTopic, now int64) bool { + return topic.LastUsed > 0 && now > topic.LastUsed && now-topic.LastUsed > telegramTopicIdleTTL +} + +func touchTopic(topic telegramTopic, now int64) telegramTopic { + if topicExpired(topic, now) { + topic.Generation++ + } + topic.LastUsed = now + return topic } func (router *tenantRouter) activeTelegramTopic(userID string) activeTelegramTopic { @@ -63,7 +90,10 @@ func (router *tenantRouter) activeTelegramTopic(userID string) activeTelegramTop if topic, found := state.Topics[topicID]; found && strings.TrimSpace(topic.Label) != "" { label = topic.Label } - return activeTelegramTopic{Conversation: telegramTopicConversation(topicID), Label: label} + return activeTelegramTopic{ + Conversation: telegramTopicConversationGeneration(topicID, state.Topics[topicID].Generation), + Label: label, + } } func (router *tenantRouter) selectTelegramTopic(userID, rawLabel string) (activeTelegramTopic, error) { @@ -83,12 +113,18 @@ func (router *tenantRouter) selectTelegramTopic(userID, rawLabel string) (active state.Topics = map[string]telegramTopic{} } state.Active = topicID - state.Topics[topicID] = telegramTopic{Label: label, LastUsed: router.now().Unix()} + topic := state.Topics[topicID] + topic.Label = label + topic = touchTopic(topic, router.now().Unix()) + state.Topics[topicID] = topic router.state.TelegramTopics[identity] = state if err := router.saveLocked(); err != nil { return activeTelegramTopic{}, err } - return activeTelegramTopic{Conversation: telegramTopicConversation(topicID), Label: label}, nil + return activeTelegramTopic{ + Conversation: telegramTopicConversationGeneration(topicID, topic.Generation), + Label: label, + }, nil } func (router *tenantRouter) touchTelegramTopic(userID string) (activeTelegramTopic, error) { @@ -108,13 +144,16 @@ func (router *tenantRouter) touchTelegramTopic(userID string) (activeTelegramTop if strings.TrimSpace(topic.Label) == "" { topic.Label = telegramDefaultTopic } - topic.LastUsed = router.now().Unix() + topic = touchTopic(topic, router.now().Unix()) state.Topics[topicID] = topic router.state.TelegramTopics[identity] = state if err := router.saveLocked(); err != nil { return activeTelegramTopic{}, err } - return activeTelegramTopic{Conversation: telegramTopicConversation(topicID), Label: topic.Label}, nil + return activeTelegramTopic{ + Conversation: telegramTopicConversationGeneration(topicID, topic.Generation), + Label: topic.Label, + }, nil } func (router *tenantRouter) telegramTopics(userID string) []telegramTopic { diff --git a/services/hermes/router/web.go b/services/hermes/router/web.go index 0ea02a20..adb596a6 100644 --- a/services/hermes/router/web.go +++ b/services/hermes/router/web.go @@ -93,6 +93,8 @@ const privateFilesPage = ` ` const bridgeCSS = ` +[data-panel="kanban"],[data-panel="logs"],[data-panel="insights"],[data-panel="memory"],[data-panel="skills"],[data-panel="workspaces"],[data-panel="todos"],[data-panel="tasks"],[data-panel="profiles"]{display:none!important} +#settingsModel,label:has(#settingsModel),.settings-row:has(#settingsModel){display:none!important} #hermes-files-sidebar,#hermes-telegram-sidebar{display:flex;align-items:center} .hermes-link-page{margin:0;min-height:100vh;display:grid;place-items:center;background:#0f172a;color:#e2e8f0;font:16px/1.5 system-ui,sans-serif} .hermes-link-card{width:min(620px,calc(100% - 40px));box-sizing:border-box;padding:32px;border:1px solid #334155;border-radius:18px;background:#111827;box-shadow:0 20px 60px #0006} @@ -105,8 +107,11 @@ const bridgeCSS = ` const bridgeJS = `(() => { const page = document.querySelector('[data-telegram-page]'); const filesPage = document.querySelector('[data-files-page]'); + const chatAdministrationPanels = ['kanban','logs','insights','memory','skills','workspaces','todos','tasks','profiles']; const hideChatAdministration = () => { - document.querySelectorAll('[data-panel="kanban"]').forEach((node) => { node.hidden = true; }); + chatAdministrationPanels.forEach((panel) => { + document.querySelectorAll('[data-panel="' + panel + '"]').forEach((node) => { node.hidden = true; }); + }); }; hideChatAdministration(); if (!page && !filesPage) { @@ -159,7 +164,7 @@ const bridgeJS = `(() => { const observer = new MutationObserver(() => { if (scheduled) return; scheduled = true; - requestAnimationFrame(() => { scheduled = false; installSidebarLinks(); }); + requestAnimationFrame(() => { scheduled = false; hideChatAdministration(); installSidebarLinks(); }); }); installSidebarLinks(); observer.observe(document.body, {childList:true, subtree:true}); @@ -354,6 +359,15 @@ func (router *tenantRouter) serveTelegramWeb(writer http.ResponseWriter, request writer.Header().Set("Cache-Control", "public, max-age=3600") _, _ = io.WriteString(writer, bridgeJS) return true + case "/hermes-session-continuity.js": + if request.Method != http.MethodGet { + http.Error(writer, "method not allowed", http.StatusMethodNotAllowed) + return true + } + writer.Header().Set("Content-Type", "application/javascript; charset=utf-8") + writer.Header().Set("Cache-Control", "public, max-age=3600") + _, _ = io.WriteString(writer, sessionContinuityJS) + return true case "/telegram": if request.Method != http.MethodGet { http.Error(writer, "method not allowed", http.StatusMethodNotAllowed) @@ -455,6 +469,9 @@ func injectChatBridge(response *http.Response) error { content = strings.Replace(content, "", ``, 1) content = strings.Replace(content, "", ``, 1) } + if !strings.Contains(content, "hermes-session-continuity.js") { + content = strings.Replace(content, "", ``, 1) + } response.Body = io.NopCloser(strings.NewReader(content)) response.ContentLength = int64(len(content)) response.Header.Set("Content-Length", strconv.Itoa(len(content))) diff --git a/services/hermes/scripts/ai_usage_codex.py b/services/hermes/scripts/ai_usage_codex.py new file mode 100644 index 00000000..6214cbf5 --- /dev/null +++ b/services/hermes/scripts/ai_usage_codex.py @@ -0,0 +1,138 @@ +"""Bounded Codex app-server query and child-process cleanup.""" + +from __future__ import annotations + +import json +import os +import selectors +import subprocess +import time +from typing import Any + + +CODEX_BIN = os.environ.get("ATLAS_AI_CODEX_BIN", "/opt/data/tools/bin/codex") +CODEX_HOME = os.environ.get("CODEX_HOME", "/runtime-access/codex") +CODEX_CLEANUP_TIMEOUT_SECONDS = 5 + + +def _close_stream(stream: Any) -> bool: + """Close a subprocess pipe without allowing cleanup errors to escape.""" + if stream is None: + return True + try: + stream.close() + except Exception: + return False + return True + + +def _cleanup_codex_process( + process: subprocess.Popen[str], selector: selectors.BaseSelector | None +) -> bool: + """Stop and reap a Codex child within two fixed waits, then close its pipes.""" + cleanup_ok = True + if selector is not None: + try: + selector.close() + except Exception: + cleanup_ok = False + + # Closing the request pipe first also gives a responsive app-server an EOF. + cleanup_ok = _close_stream(getattr(process, "stdin", None)) and cleanup_ok + try: + process.terminate() + except ProcessLookupError: + pass + except Exception: + cleanup_ok = False + + needs_kill = False + try: + process.wait(timeout=CODEX_CLEANUP_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + needs_kill = True + except Exception: + cleanup_ok = False + needs_kill = True + + if needs_kill: + try: + process.kill() + except ProcessLookupError: + pass + except Exception: + cleanup_ok = False + try: + process.wait(timeout=CODEX_CLEANUP_TIMEOUT_SECONDS) + except Exception: + cleanup_ok = False + + cleanup_ok = _close_stream(getattr(process, "stdout", None)) and cleanup_ok + cleanup_ok = _close_stream(getattr(process, "stderr", None)) and cleanup_ok + return cleanup_ok + + +def query_codex(timeout: float = 20) -> tuple[dict[str, Any], dict[str, Any]]: + """Read Codex account quota and usage through its structured app-server protocol.""" + process = subprocess.Popen( + [CODEX_BIN, "app-server", "--stdio"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + bufsize=1, + env={**os.environ, "CODEX_HOME": CODEX_HOME}, + ) + requests = ( + { + "id": 1, + "method": "initialize", + "params": { + "clientInfo": { + "name": "atlas-ai-usage-exporter", + "title": "Atlas AI Usage Exporter", + "version": "1.0.0", + }, + "capabilities": {"experimentalApi": True}, + }, + }, + {"method": "initialized", "params": {}}, + {"id": 2, "method": "account/rateLimits/read", "params": None}, + {"id": 3, "method": "account/usage/read", "params": None}, + ) + selector: selectors.BaseSelector | None = None + rate_result: dict[str, Any] + usage_result: dict[str, Any] + try: + if process.stdin is None or process.stdout is None: + raise RuntimeError("Codex app-server pipes are unavailable") + for message in requests: + process.stdin.write(json.dumps(message, separators=(",", ":")) + "\n") + process.stdin.flush() + selector = selectors.DefaultSelector() + selector.register(process.stdout, selectors.EVENT_READ) + responses: dict[int, dict[str, Any]] = {} + deadline = time.monotonic() + timeout + while len(responses) < 3 and time.monotonic() < deadline: + for key, _ in selector.select(min(1, max(0, deadline - time.monotonic()))): + line = key.fileobj.readline() + if not line: + continue + message = json.loads(line) + if message.get("id") in (1, 2, 3): + responses[int(message["id"])] = message + for response_id in (1, 2, 3): + response = responses.get(response_id) + if ( + not response + or "error" in response + or not isinstance(response.get("result"), dict) + ): + raise RuntimeError(f"Codex app-server response {response_id} failed") + rate_result = responses[2]["result"] + usage_result = responses[3]["result"] + finally: + cleanup_ok = _cleanup_codex_process(process, selector) + if not cleanup_ok: + raise RuntimeError("Codex app-server cleanup failed") + return rate_result, usage_result diff --git a/services/hermes/scripts/ai_usage_exporter.py b/services/hermes/scripts/ai_usage_exporter.py index 178c2c54..b0db51fb 100644 --- a/services/hermes/scripts/ai_usage_exporter.py +++ b/services/hermes/scripts/ai_usage_exporter.py @@ -5,20 +5,18 @@ from __future__ import annotations import json import os -import selectors -import subprocess import threading import time from dataclasses import dataclass, field from datetime import UTC, date, datetime, timedelta -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Any from urllib.request import Request, urlopen +import ai_usage_codex as codex_query +import ai_usage_http as http_engine + -CODEX_BIN = os.environ.get("ATLAS_AI_CODEX_BIN", "/opt/data/tools/bin/codex") -CODEX_HOME = os.environ.get("CODEX_HOME", "/runtime-access/codex") CLAUDE_CREDENTIALS = Path( os.environ.get( "ATLAS_AI_CLAUDE_CREDENTIALS", @@ -48,6 +46,8 @@ METRIC_HELP = { "atlas_ai_quota_used_percent": "Used percentage in a first-party coding CLI quota window.", "atlas_ai_quota_window_duration_seconds": "Nominal duration of a coding CLI quota window.", } +POLL_STARTUP_GRACE_SECONDS = 45 +POLL_PROGRESS_BUDGET_SECONDS = 60 @dataclass(frozen=True) @@ -75,7 +75,9 @@ def _number(value: Any) -> float | None: if isinstance(value, bool) or not isinstance(value, (int, float)): return None converted = float(value) - return converted if converted == converted and abs(converted) != float("inf") else None + return ( + converted if converted == converted and abs(converted) != float("inf") else None + ) def _timestamp(value: Any) -> float | None: @@ -112,7 +114,10 @@ def _codex_limit_name(limit_id: str, snapshot: dict[str, Any]) -> str: def parse_codex_payloads( - rate_response: dict[str, Any], usage_response: dict[str, Any], *, today: date | None = None + rate_response: dict[str, Any], + usage_response: dict[str, Any], + *, + today: date | None = None, ) -> list[Sample]: """Convert structured Codex app-server responses into bounded metrics.""" samples: list[Sample] = [] @@ -140,15 +145,23 @@ def parse_codex_payloads( samples.extend( ( Sample("atlas_ai_quota_used_percent", labels, used), - Sample("atlas_ai_quota_remaining_percent", labels, max(0, 100 - used)), + Sample( + "atlas_ai_quota_remaining_percent", labels, max(0, 100 - used) + ), ) ) reset = _timestamp(raw_window.get("resetsAt")) duration = _number(raw_window.get("windowDurationMins")) if reset is not None: - samples.append(Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset)) + samples.append( + Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset) + ) if duration is not None: - samples.append(Sample("atlas_ai_quota_window_duration_seconds", labels, duration * 60)) + samples.append( + Sample( + "atlas_ai_quota_window_duration_seconds", labels, duration * 60 + ) + ) summary = usage_response.get("summary") if isinstance(summary, dict): @@ -189,8 +202,16 @@ def parse_codex_payloads( anchor_day = max((start for start, _ in parsed_daily), default=current_day) for period, days in (("latest_day", 1), ("seven_day", 7), ("thirty_day", 30)): earliest = anchor_day - timedelta(days=days - 1) - value = sum(tokens for start, tokens in parsed_daily if earliest <= start <= anchor_day) - samples.append(Sample("atlas_ai_account_tokens", {"provider": "openai", "period": period}, value)) + value = sum( + tokens for start, tokens in parsed_daily if earliest <= start <= anchor_day + ) + samples.append( + Sample( + "atlas_ai_account_tokens", + {"provider": "openai", "period": period}, + value, + ) + ) return samples @@ -213,7 +234,9 @@ def parse_claude_payload(payload: dict[str, Any]) -> list[Sample]: ) reset = _timestamp(window.get("resets_at")) if reset is not None: - samples.append(Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset)) + samples.append( + Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset) + ) extra = payload.get("extra_usage") if isinstance(extra, dict): enabled = extra.get("is_enabled") @@ -228,64 +251,7 @@ def parse_claude_payload(payload: dict[str, Any]) -> list[Sample]: return samples -def query_codex(timeout: float = 20) -> tuple[dict[str, Any], dict[str, Any]]: - """Read Codex account quota and usage through its structured app-server protocol.""" - process = subprocess.Popen( - [CODEX_BIN, "app-server", "--stdio"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - bufsize=1, - env={**os.environ, "CODEX_HOME": CODEX_HOME}, - ) - requests = ( - { - "id": 1, - "method": "initialize", - "params": { - "clientInfo": { - "name": "atlas-ai-usage-exporter", - "title": "Atlas AI Usage Exporter", - "version": "1.0.0", - }, - "capabilities": {"experimentalApi": True}, - }, - }, - {"method": "initialized", "params": {}}, - {"id": 2, "method": "account/rateLimits/read", "params": None}, - {"id": 3, "method": "account/usage/read", "params": None}, - ) - try: - if process.stdin is None or process.stdout is None: - raise RuntimeError("Codex app-server pipes are unavailable") - for message in requests: - process.stdin.write(json.dumps(message, separators=(",", ":")) + "\n") - process.stdin.flush() - selector = selectors.DefaultSelector() - selector.register(process.stdout, selectors.EVENT_READ) - responses: dict[int, dict[str, Any]] = {} - deadline = time.monotonic() + timeout - while len(responses) < 3 and time.monotonic() < deadline: - for key, _ in selector.select(min(1, max(0, deadline - time.monotonic()))): - line = key.fileobj.readline() - if not line: - continue - message = json.loads(line) - if message.get("id") in (1, 2, 3): - responses[int(message["id"])] = message - for response_id in (1, 2, 3): - response = responses.get(response_id) - if not response or "error" in response or not isinstance(response.get("result"), dict): - raise RuntimeError(f"Codex app-server response {response_id} failed") - return responses[2]["result"], responses[3]["result"] - finally: - process.terminate() - try: - process.wait(timeout=5) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=5) +query_codex = codex_query.query_codex def query_claude() -> dict[str, Any]: @@ -322,6 +288,24 @@ class Collector: self._lock = threading.Lock() self._providers = {name: ProviderState() for name in ("openai", "anthropic")} + def record_failure( + self, + provider: str, + *, + started: float | None = None, + monotonic_started: float | None = None, + ) -> None: + """Record a failed attempt while retaining the provider's last good samples.""" + attempted_at = time.time() if started is None else started + duration_started = ( + time.monotonic() if monotonic_started is None else monotonic_started + ) + with self._lock: + state = self._providers[provider] + state.last_attempt = attempted_at + state.duration = max(0, time.monotonic() - duration_started) + state.fetch_success = False + def refresh_provider(self, provider: str) -> None: """Refresh one provider while retaining the last good values on failure.""" started = time.time() @@ -333,19 +317,24 @@ class Collector: samples = parse_claude_payload(query_claude()) else: raise ValueError("unknown provider") - success = True - except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as error: - print(f"{provider} quota collection deferred: {type(error).__name__}", flush=True) - samples = [] - success = False + except Exception as error: + print( + f"{provider} quota collection deferred: {type(error).__name__}", + flush=True, + ) + self.record_failure( + provider, + started=started, + monotonic_started=monotonic_started, + ) + return with self._lock: state = self._providers[provider] state.last_attempt = started state.duration = time.monotonic() - monotonic_started - state.fetch_success = success - if success: - state.samples = samples - state.last_success = time.time() + state.fetch_success = True + state.samples = samples + state.last_success = time.time() def render(self) -> bytes: """Render the current provider states in Prometheus text format.""" @@ -360,10 +349,24 @@ class Collector: samples.extend(state.samples) samples.extend( ( - Sample("atlas_ai_quota_fetch_success", labels, float(state.fetch_success)), - Sample("atlas_ai_quota_last_attempt_timestamp_seconds", labels, state.last_attempt), - Sample("atlas_ai_quota_last_success_timestamp_seconds", labels, state.last_success), - Sample("atlas_ai_quota_fetch_duration_seconds", labels, state.duration), + Sample( + "atlas_ai_quota_fetch_success", + labels, + float(state.fetch_success), + ), + Sample( + "atlas_ai_quota_last_attempt_timestamp_seconds", + labels, + state.last_attempt, + ), + Sample( + "atlas_ai_quota_last_success_timestamp_seconds", + labels, + state.last_success, + ), + Sample( + "atlas_ai_quota_fetch_duration_seconds", labels, state.duration + ), ) ) lines: list[str] = [] @@ -381,38 +384,112 @@ class Collector: return ("\n".join(lines) + "\n").encode("utf-8") +class PollingEngine: + """Run isolated provider polls and track bounded forward progress.""" + + def __init__( + self, + collector: Collector, + *, + interval: float, + startup_grace: float = POLL_STARTUP_GRACE_SECONDS, + progress_timeout: float | None = None, + clock: Any = time.monotonic, + ) -> None: + self.collector = collector + self.interval = interval + self.startup_grace = startup_grace + self.progress_timeout = ( + interval + POLL_PROGRESS_BUDGET_SECONDS + if progress_timeout is None + else progress_timeout + ) + self._clock = clock + self._started_at = clock() + self._last_progress: float | None = None + self._progress_lock = threading.Lock() + self._thread: threading.Thread | None = None + + def _mark_progress(self) -> None: + """Record completion of one bounded provider attempt.""" + with self._progress_lock: + self._last_progress = self._clock() + + def poll_once(self) -> None: + """Refresh every provider even if another provider fails unexpectedly.""" + for provider in ("openai", "anthropic"): + started = time.time() + monotonic_started = time.monotonic() + try: + self.collector.refresh_provider(provider) + except Exception as error: + print( + f"{provider} quota collection isolated: {type(error).__name__}", + flush=True, + ) + try: + self.collector.record_failure( + provider, + started=started, + monotonic_started=monotonic_started, + ) + except Exception as record_error: + print( + f"{provider} quota failure accounting deferred: " + f"{type(record_error).__name__}", + flush=True, + ) + finally: + self._mark_progress() + + def run(self) -> None: + """Poll forever without allowing a cycle-level exception to stop the thread.""" + while True: + try: + self.poll_once() + except Exception as error: + print( + f"quota polling cycle deferred: {type(error).__name__}", flush=True + ) + time.sleep(self.interval) + + def start(self) -> None: + """Start the daemon poller exactly once.""" + if self._thread is not None: + return + self._thread = threading.Thread( + target=self.run, + name="ai-usage-poller", + daemon=True, + ) + self._thread.start() + + def is_healthy(self) -> bool: + """Report thread liveness and progress, independent of provider success.""" + thread = self._thread + if thread is None or not thread.is_alive(): + return False + now = self._clock() + with self._progress_lock: + last_progress = self._last_progress + if last_progress is None: + return now - self._started_at <= self.startup_grace + return now - last_progress <= self.progress_timeout + + +make_handler = http_engine.make_handler +Server = http_engine.Server + + def main() -> int: """Poll quota APIs and serve only sanitized metrics and health endpoints.""" collector = Collector() interval = max(60, int(os.environ.get("ATLAS_AI_USAGE_INTERVAL_SECONDS", "300"))) - def polling_loop() -> None: - while True: - for provider in ("openai", "anthropic"): - collector.refresh_provider(provider) - time.sleep(interval) - - class Handler(BaseHTTPRequestHandler): - def do_GET(self) -> None: # noqa: N802 - if self.path == "/metrics": - payload = collector.render() - self.send_response(200) - self.send_header("Content-Type", "text/plain; version=0.0.4") - self.send_header("Content-Length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - elif self.path == "/healthz": - self.send_response(200) - self.end_headers() - else: - self.send_error(404) - - def log_message(self, _format: str, *_args: object) -> None: - return - - threading.Thread(target=polling_loop, daemon=True).start() + poller = PollingEngine(collector, interval=interval) + poller.start() port = int(os.environ.get("ATLAS_AI_USAGE_PORT", "9010")) - server = ThreadingHTTPServer(("0.0.0.0", port), Handler) + server = Server(("0.0.0.0", port), make_handler(collector, poller)) server.serve_forever() return 0 diff --git a/services/hermes/scripts/ai_usage_http.py b/services/hermes/scripts/ai_usage_http.py new file mode 100644 index 00000000..8558740f --- /dev/null +++ b/services/hermes/scripts/ai_usage_http.py @@ -0,0 +1,49 @@ +"""Disconnect-safe HTTP serving for the AI usage exporter.""" + +from __future__ import annotations + +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + + +def make_handler(collector: Any, poller: Any) -> type[BaseHTTPRequestHandler]: + """Build an HTTP handler bound to one collector and polling engine.""" + + class Handler(BaseHTTPRequestHandler): + def _respond(self, status: int, payload: bytes, content_type: str) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def do_GET(self) -> None: # noqa: N802 + try: + if self.path == "/metrics": + self._respond( + 200, + collector.render(), + "text/plain; version=0.0.4", + ) + elif self.path == "/healthz": + healthy = poller.is_healthy() + self._respond( + 200 if healthy else 503, + b"ok\n" if healthy else b"poller unhealthy\n", + "text/plain; charset=utf-8", + ) + else: + self.send_error(404) + except (BrokenPipeError, ConnectionResetError): + return + + def log_message(self, _format: str, *_args: object) -> None: + return + + return Handler + + +class Server(ThreadingHTTPServer): + """Threaded metrics server that does not retain disconnected clients.""" + + daemon_threads = True diff --git a/services/hermes/scripts/migrate_telegram_api_sessions.py b/services/hermes/scripts/migrate_telegram_api_sessions.py index 82f58456..9d4597f9 100644 --- a/services/hermes/scripts/migrate_telegram_api_sessions.py +++ b/services/hermes/scripts/migrate_telegram_api_sessions.py @@ -1,16 +1,236 @@ #!/usr/bin/env python3 -"""Backfill Telegram origin metadata for durable API conversations.""" +"""Bound Telegram context and backfill durable API-session metadata.""" from __future__ import annotations import argparse import json +import re import sqlite3 from pathlib import Path +from typing import Any + + +MAX_RESPONSE_BYTES = 32 << 20 +MAX_HISTORY_ITEMS = 80 +MAX_HISTORY_BYTES = 192_000 +MAX_RECENT_TURNS = 12 +MAX_ITEM_CHARS = 8_000 +MAX_SUMMARY_CHARS = 12_000 +SUMMARY_PREFIX = "Telegram continuity summary (older bounded context):" +ATTACHMENT_NOTE = ( + "[Earlier Telegram image omitted from bounded context; attach it again " + "in Hermes WebUI if the original pixels are required.]" +) +_CONVERSATION = re.compile(r"^telegram(?:-topic-[a-z0-9-]{1,160})?$") +_MEDIA_PATH = re.compile( + r'(?:MEDIA:\s*)?(?:/opt/data/(?:cache/images|workspace)|/workspace)/[^\s\]\[)}`"]+', + re.IGNORECASE, +) +_DATA_IMAGE = re.compile(r"data:image/[^;,\s]+;base64,[A-Za-z0-9+/=]+", re.IGNORECASE) +_MESSAGE_IDENTITY_COLUMNS = ( + "session_id", + "role", + "content", + "tool_call_id", + "tool_calls", + "tool_name", + "timestamp", + "token_count", + "finish_reason", + "reasoning", + "reasoning_content", + "reasoning_details", + "codex_reasoning_items", + "codex_message_items", + "platform_message_id", + "observed", + "active", + "compacted", +) + + +def _text(value: Any) -> str: + """Return bounded context text without reusable transport-only images.""" + if isinstance(value, str): + text = value + elif isinstance(value, list): + parts: list[str] = [] + for part in value: + if not isinstance(part, dict): + continue + kind = str(part.get("type") or "").lower() + if "image" in kind or "image_url" in part: + parts.append(ATTACHMENT_NOTE) + continue + candidate = part.get("text", part.get("content", "")) + if isinstance(candidate, str): + parts.append(candidate) + text = "\n".join(parts) + else: + try: + text = json.dumps(value, ensure_ascii=False, sort_keys=True) + except (TypeError, ValueError): + text = str(value) + text = _DATA_IMAGE.sub(ATTACHMENT_NOTE, text) + text = _MEDIA_PATH.sub("[private attachment]", text) + return text[:MAX_ITEM_CHARS] + + +def _normalized_item(raw: Any) -> dict[str, Any] | None: + if not isinstance(raw, dict): + return None + role = str(raw.get("role") or "").strip().lower() + if role not in {"assistant", "system", "tool", "user"}: + return None + content = _text(raw.get("content", "")) + if not content and role != "assistant": + return None + item: dict[str, Any] = {"role": role, "content": content} + if raw.get("_db_persisted") is True: + item["_db_persisted"] = True + for key in ("name", "tool_call_id", "tool_name"): + value = raw.get(key) + if isinstance(value, str) and value: + item[key] = value[:256] + calls = raw.get("tool_calls") + if isinstance(calls, list): + bounded_calls = [] + for call in calls[:16]: + if not isinstance(call, dict): + continue + function = ( + call.get("function") if isinstance(call.get("function"), dict) else {} + ) + bounded = { + "id": str(call.get("id") or "")[:256], + "type": str(call.get("type") or "function")[:32], + "function": { + "name": str(function.get("name") or "")[:256], + "arguments": _text(function.get("arguments", ""))[:2_000], + }, + } + bounded_calls.append(bounded) + if bounded_calls: + item["tool_calls"] = bounded_calls + return item + + +def _fingerprint(item: dict[str, Any]) -> str: + try: + visible = {key: value for key, value in item.items() if not key.startswith("_")} + return json.dumps( + visible, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ) + except (TypeError, ValueError): + return repr(item) + + +def _deduplicate(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Drop replay duplicates while preserving intentional repeated prose.""" + result: list[dict[str, Any]] = [] + stable_ids: set[tuple[str, str]] = set() + for item in items: + stable = "" + if isinstance(item.get("tool_call_id"), str): + stable = item["tool_call_id"] + elif isinstance(item.get("id"), str): + stable = item["id"] + if stable: + key = (item["role"], stable) + if key in stable_ids: + continue + stable_ids.add(key) + if result and _fingerprint(result[-1]) == _fingerprint(item): + continue + result.append(item) + # Response replays can append one exact transcript to itself. Remove only + # complete adjacent blocks, not ordinary repeated user phrases. + changed = True + while changed and len(result) > 1: + changed = False + for width in range(len(result) // 2, 0, -1): + if [_fingerprint(x) for x in result[-2 * width : -width]] == [ + _fingerprint(x) for x in result[-width:] + ]: + del result[-width:] + changed = True + break + return result + + +def _summary(older: list[dict[str, Any]]) -> dict[str, str] | None: + lines: list[str] = [] + for item in older: + role = item["role"] + content = str(item.get("content") or "").strip() + if role == "system" and content.startswith(SUMMARY_PREFIX): + candidates = content[len(SUMMARY_PREFIX) :].strip().splitlines() + elif role in {"user", "assistant"} and content: + candidates = [f"- {role.title()}: {content[:600]}"] + else: + continue + for line in candidates: + clean = " ".join(line.split()) + if clean and clean not in lines: + lines.append(clean) + content = SUMMARY_PREFIX + for line in lines[-40:]: + candidate = content + "\n" + line + if len(candidate) > MAX_SUMMARY_CHARS: + break + content = candidate + return ( + { + "role": "system", + "content": content, + "_db_persisted": True, + "_compressed_summary": True, + } + if lines + else None + ) + + +def compact_telegram_history(history: Any) -> list[dict[str, Any]]: + """Return a deterministic summary plus bounded recent Telegram turns.""" + if not isinstance(history, list): + return [] + normalized = _deduplicate( + [item for raw in history if (item := _normalized_item(raw)) is not None] + ) + recent: list[dict[str, Any]] = [] + recent_bytes = 2 + user_turns = 0 + split = len(normalized) + for index in range(len(normalized) - 1, -1, -1): + item = normalized[index] + encoded = len(_fingerprint(item).encode("utf-8")) + 1 + next_turns = user_turns + (item["role"] == "user") + if recent and ( + len(recent) >= MAX_HISTORY_ITEMS + or recent_bytes + encoded > MAX_HISTORY_BYTES + or next_turns > MAX_RECENT_TURNS + ): + break + recent.insert(0, item) + recent_bytes += encoded + user_turns = next_turns + split = index + summary = _summary(normalized[:split]) + result = ([summary] if summary else []) + recent + while ( + len(json.dumps(result, ensure_ascii=False).encode("utf-8")) + > (MAX_HISTORY_BYTES + MAX_SUMMARY_CHARS) + and len(recent) > 1 + ): + recent.pop(0) + result = ([summary] if summary else []) + recent + return result def telegram_sessions(response_store: Path) -> dict[str, str]: - """Return session ID to stable Telegram conversation-key mappings.""" + """Return safe session ID to stable Telegram conversation mappings.""" if not response_store.exists(): return {} connection = sqlite3.connect(f"file:{response_store}?mode=ro", uri=True) @@ -19,21 +239,64 @@ def telegram_sessions(response_store: Path) -> dict[str, str]: """SELECT c.name, r.data FROM conversations AS c JOIN responses AS r ON r.response_id = c.response_id - WHERE c.name = 'telegram' OR c.name LIKE 'telegram-topic-%'""" + WHERE (c.name = 'telegram' OR c.name LIKE 'telegram-topic-%') + AND LENGTH(r.data) <= ? + ORDER BY c.name, r.response_id""", + (MAX_RESPONSE_BYTES,), ).fetchall() + except sqlite3.DatabaseError: + return {} finally: connection.close() result: dict[str, str] = {} for conversation, raw in rows: - try: - session_id = str(json.loads(raw).get("session_id") or "").strip() - except (TypeError, ValueError, json.JSONDecodeError): + name = str(conversation or "") + if not _CONVERSATION.fullmatch(name): continue - if session_id: - result[session_id] = str(conversation) + try: + payload = json.loads(raw) + session_id = str(payload.get("session_id") or "").strip() + except (AttributeError, TypeError, ValueError, json.JSONDecodeError): + continue + if ( + session_id + and len(session_id) <= 256 + and not re.search(r"[\r\n\x00/\\]", session_id) + ): + result.setdefault(session_id, name) return result +def deduplicate_stored_messages( + connection: sqlite3.Connection, session_ids: list[str] +) -> int: + """Remove only byte-for-byte replay rows for known Telegram API sessions.""" + columns = {str(row[1]) for row in connection.execute("PRAGMA table_info(messages)")} + required = {"id", "session_id", "role", "content", "timestamp"} + if not required.issubset(columns): + return 0 + identity = [column for column in _MESSAGE_IDENTITY_COLUMNS if column in columns] + partition = ", ".join(identity) + changed = 0 + for session_id in session_ids: + query = f"""DELETE FROM messages WHERE id IN ( + SELECT id FROM ( + SELECT id, ROW_NUMBER() OVER ( + PARTITION BY {partition} ORDER BY id + ) AS replay_number + FROM messages + WHERE session_id = ? AND EXISTS ( + SELECT 1 FROM sessions + WHERE sessions.id = messages.session_id + AND sessions.source = 'api_server' + ) + ) WHERE replay_number > 1 + )""" # noqa: S608 -- identifiers come only from the fixed allowlist + cursor = connection.execute(query, (session_id,)) + changed += max(cursor.rowcount, 0) + return changed + + def migrate(state_database: Path, response_store: Path) -> int: """Annotate known sessions without creating or rewriting conversations.""" mappings = telegram_sessions(response_store) @@ -43,12 +306,16 @@ def migrate(state_database: Path, response_store: Path) -> int: changed = 0 try: columns = { - str(row[1]) - for row in connection.execute("PRAGMA table_info(sessions)").fetchall() + str(row[1]) for row in connection.execute("PRAGMA table_info(sessions)") } required = { - "id", "source", "session_key", "chat_type", "display_name", - "origin_json", "title", + "id", + "source", + "session_key", + "chat_type", + "display_name", + "origin_json", + "title", } if not required.issubset(columns): return 0 @@ -57,23 +324,38 @@ def migrate(state_database: Path, response_store: Path) -> int: {"platform": "telegram", "session_key": conversation}, separators=(",", ":"), ) - title = "Telegram · General" if conversation == "telegram" else "Telegram" + default_title = ( + "Telegram · General" if conversation == "telegram" else "Telegram" + ) cursor = connection.execute( """UPDATE sessions - SET session_key = COALESCE(session_key, ?), - chat_type = COALESCE(chat_type, 'private'), - display_name = COALESCE(display_name, 'Telegram'), - origin_json = COALESCE(origin_json, ?), - title = COALESCE(title, ?) + SET session_key = ?, chat_type = 'private', + display_name = 'Telegram', origin_json = ?, + title = CASE + WHEN title IS NULL OR TRIM(title) = '' OR + LOWER(TRIM(title)) = 'unassigned' + THEN ? ELSE title END WHERE id = ? AND source = 'api_server' - AND ( - session_key IS NULL OR chat_type IS NULL OR - display_name IS NULL OR origin_json IS NULL OR title IS NULL - )""", - (conversation, origin, title, session_id), + AND (COALESCE(session_key, '') != ? OR + COALESCE(chat_type, '') != 'private' OR + COALESCE(display_name, '') != 'Telegram' OR + COALESCE(origin_json, '') != ? OR title IS NULL OR + TRIM(title) = '' OR LOWER(TRIM(title)) = 'unassigned')""", + ( + conversation, + origin, + default_title, + session_id, + conversation, + origin, + ), ) changed += max(cursor.rowcount, 0) + deduplicate_stored_messages(connection, list(mappings)) connection.commit() + except sqlite3.DatabaseError: + connection.rollback() + return 0 finally: connection.close() return changed diff --git a/services/hermes/scripts/patch_api_server_sessions.py b/services/hermes/scripts/patch_api_server_sessions.py index bc7cafcc..55a57b81 100644 --- a/services/hermes/scripts/patch_api_server_sessions.py +++ b/services/hermes/scripts/patch_api_server_sessions.py @@ -6,15 +6,13 @@ from __future__ import annotations import argparse from pathlib import Path - -BEFORE = ''' model = body.get("model") or self._model_name +BEFORE = """ model = body.get("model") or self._model_name system_prompt = body.get("system_prompt") if system_prompt is not None and not isinstance(system_prompt, str): return web.json_response(_openai_error("system_prompt must be a string", code="invalid_system_prompt"), status=400) db.create_session(session_id, "api_server", model=str(model) if model else None, system_prompt=system_prompt) -''' - -AFTER = ''' model = body.get("model") or self._model_name +""" +AFTER = """ model = body.get("model") or self._model_name system_prompt = body.get("system_prompt") if system_prompt is not None and not isinstance(system_prompt, str): return web.json_response(_openai_error("system_prompt must be a string", code="invalid_system_prompt"), status=400) @@ -46,21 +44,18 @@ AFTER = ''' model = body.get("model") or self._model_name system_prompt=system_prompt, parent_session_id=parent_session_id, ) -''' - -RUNS_BEFORE = ''' run_id = f"run_{uuid.uuid4().hex}" +""" +RUNS_BEFORE = """ run_id = f"run_{uuid.uuid4().hex}" session_id = body.get("session_id") or stored_session_id or run_id # Approval queues gate host-side tool execution and must be isolated -''' - -RESPONSES_SESSION_BEFORE = ''' # Reuse session from previous_response_id chain so the dashboard +""" +RESPONSES_SESSION_BEFORE = """ # Reuse session from previous_response_id chain so the dashboard # groups the entire conversation under one session entry. session_id = stored_session_id or str(uuid.uuid4()) # Per-client model routing for /v1/responses (see model_routes). -''' - -RESPONSES_SESSION_AFTER = ''' # Reuse session from previous_response_id chain so the dashboard +""" +RESPONSES_SESSION_AFTER = """ # Reuse session from previous_response_id chain so the dashboard # groups the entire conversation under one session entry. session_id = stored_session_id or str(uuid.uuid4()) @@ -68,9 +63,7 @@ RESPONSES_SESSION_AFTER = ''' # Reuse session from previous_response_id c # WebUI do not collapse it into an anonymous Api_Server session. Keep # the accepted vocabulary narrow: these headers are presentation and # routing metadata, never an authorization boundary. - conversation_platform = request.headers.get( - "X-Hermes-Conversation-Platform", "" - ).strip().lower() + # X-Hermes-Conversation-Platform was parsed before history compaction. conversation_title = request.headers.get( "X-Hermes-Conversation-Title", "" ).strip() @@ -117,9 +110,41 @@ RESPONSES_SESSION_AFTER = ''' # Reuse session from previous_response_id c db.set_session_title(session_id, conversation_title) # Per-client model routing for /v1/responses (see model_routes). -''' +""" +TRUNCATION_BEFORE = """ # Truncation support + if body.get("truncation") == "auto" and len(conversation_history) > 100: + conversation_history = conversation_history[-100:] +""" +TRUNCATION_AFTER = """ # Telegram keeps a durable compact summary plus bounded recent turns. + conversation_platform = request.headers.get( + "X-Hermes-Conversation-Platform", "" + ).strip().lower() + if conversation_platform == "telegram": + from gateway.platforms.telegram_continuity import compact_telegram_history + conversation_history = compact_telegram_history(conversation_history) + elif body.get("truncation") == "auto" and len(conversation_history) > 100: + conversation_history = conversation_history[-100:] +""" +HISTORY_STORE_BEFORE = """ # Build output items from the current turn only. AIAgent returns a +""" +HISTORY_STORE_AFTER = """ if conversation_platform == "telegram": + full_history = compact_telegram_history(full_history) -RUNS_AFTER = ''' run_id = f"run_{uuid.uuid4().hex}" + # Build output items from the current turn only. AIAgent returns a +""" + +SSE_STORE_BEFORE = """ self._response_store.put(response_id, { +""" + +SSE_STORE_AFTER = """ if gateway_session_key and gateway_session_key.startswith("telegram"): + from gateway.platforms.telegram_continuity import compact_telegram_history + conversation_history_snapshot = compact_telegram_history( + conversation_history_snapshot + ) + self._response_store.put(response_id, { +""" + +RUNS_AFTER = """ run_id = f"run_{uuid.uuid4().hex}" session_id = body.get("session_id") or stored_session_id or run_id # Persist API-run lineage before the agent starts. Automated callers @@ -183,13 +208,13 @@ RUNS_AFTER = ''' run_id = f"run_{uuid.uuid4().hex}" db.reopen_session(session_id) # Approval queues gate host-side tool execution and must be isolated -''' +""" -RUN_CLOSE_BEFORE = ''' finally: +RUN_CLOSE_BEFORE = """ finally: # If the asyncio wrapper is cancelled (for example via -''' +""" -RUN_CLOSE_AFTER = ''' finally: +RUN_CLOSE_AFTER = """ finally: # Parent-linked API workers are durable dashboard sessions. # Close them on every terminal run path so the dashboard can # distinguish a long model wait from completed or failed work. @@ -218,10 +243,10 @@ RUN_CLOSE_AFTER = ''' finally: ) # If the asyncio wrapper is cancelled (for example via -''' +""" -EVENT_CALLBACK_SIGNATURE_BEFORE = ''' def _make_run_event_callback(self, run_id: str, loop: "asyncio.AbstractEventLoop"): -''' +EVENT_CALLBACK_SIGNATURE_BEFORE = """ def _make_run_event_callback(self, run_id: str, loop: "asyncio.AbstractEventLoop"): +""" EVENT_CALLBACK_SIGNATURE_AFTER = ''' _RUN_ACTIVITY_BYTES = 512_000 _RUN_ACTIVITY_FILES = 256 @@ -346,12 +371,12 @@ EVENT_CALLBACK_SIGNATURE_AFTER = ''' _RUN_ACTIVITY_BYTES = 512_000 ): ''' -EVENT_CALLBACK_BODY_BEFORE = ''' def _callback(event_type: str, tool_name: str = None, preview: str = None, args=None, **kwargs): +EVENT_CALLBACK_BODY_BEFORE = """ def _callback(event_type: str, tool_name: str = None, preview: str = None, args=None, **kwargs): ts = time.time() if event_type == "tool.started": -''' +""" -EVENT_CALLBACK_BODY_AFTER = ''' def _callback(event_type: str, tool_name: str = None, preview: str = None, args=None, **kwargs): +EVENT_CALLBACK_BODY_AFTER = """ def _callback(event_type: str, tool_name: str = None, preview: str = None, args=None, **kwargs): ts = time.time() self._record_run_activity( session_id, @@ -361,12 +386,12 @@ EVENT_CALLBACK_BODY_AFTER = ''' def _callback(event_type: str, tool_name: is_error=bool(kwargs.get("is_error", False)), ) if event_type == "tool.started": -''' +""" -EVENT_CALLBACK_END_BEFORE = ''' # _thinking and subagent_progress are intentionally not forwarded -''' +EVENT_CALLBACK_END_BEFORE = """ # _thinking and subagent_progress are intentionally not forwarded +""" -EVENT_CALLBACK_END_AFTER = ''' elif event_type in { +EVENT_CALLBACK_END_AFTER = """ elif event_type in { "subagent.tool", "subagent.progress", "subagent_progress", @@ -386,27 +411,27 @@ EVENT_CALLBACK_END_AFTER = ''' elif event_type in { "tool": tool_name if event_type == "subagent.tool" else None, "preview": safe_preview, }) -''' +""" -EVENT_CALLBACK_CALL_BEFORE = ''' event_cb = self._make_run_event_callback(run_id, loop) -''' +EVENT_CALLBACK_CALL_BEFORE = """ event_cb = self._make_run_event_callback(run_id, loop) +""" -EVENT_CALLBACK_CALL_AFTER = ''' event_cb = self._make_run_event_callback( +EVENT_CALLBACK_CALL_AFTER = """ event_cb = self._make_run_event_callback( run_id, loop, session_id=session_id, ) self._record_run_activity(session_id, "run.started") -''' +""" -RUN_SWEEP_BEFORE = ''' self._run_streams.pop(run_id, None) +RUN_SWEEP_BEFORE = """ self._run_streams.pop(run_id, None) self._run_streams_created.pop(run_id, None) self._active_run_agents.pop(run_id, None) self._active_run_tasks.pop(run_id, None) self._run_approval_sessions.pop(run_id, None) -''' +""" -RUN_SWEEP_AFTER = ''' self._run_streams.pop(run_id, None) +RUN_SWEEP_AFTER = """ self._run_streams.pop(run_id, None) self._run_streams_created.pop(run_id, None) # Stream retention and run lifetime are separate. A long run # can legitimately outlive its unconsumed SSE queue; keep the @@ -416,7 +441,7 @@ RUN_SWEEP_AFTER = ''' self._run_streams.pop(run_id, None) self._active_run_agents.pop(run_id, None) self._active_run_tasks.pop(run_id, None) self._run_approval_sessions.pop(run_id, None) -''' +""" def patch(source: Path, destination: Path) -> None: @@ -444,6 +469,9 @@ def patch(source: Path, destination: Path) -> None: content = content.replace(RUNS_BEFORE, RUNS_AFTER, 1) content = content.replace(RUN_CLOSE_BEFORE, RUN_CLOSE_AFTER, 1) content = content.replace(RESPONSES_SESSION_BEFORE, RESPONSES_SESSION_AFTER, 1) + content = content.replace(TRUNCATION_BEFORE, TRUNCATION_AFTER, 1) + content = content.replace(HISTORY_STORE_BEFORE, HISTORY_STORE_AFTER, 1) + content = content.replace(SSE_STORE_BEFORE, SSE_STORE_AFTER, 1) content = content.replace( EVENT_CALLBACK_SIGNATURE_BEFORE, EVENT_CALLBACK_SIGNATURE_AFTER, diff --git a/testing/quality_contract.json b/testing/quality_contract.json index b5148396..d94de9be 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -84,7 +84,24 @@ "services/hermes/scm-common/scripts/deadline_http.py", "services/hermes/scm-common/scripts/git_pack_objects.py", "services/hermes/scm-common/scripts/receive_pack_scan.py", - "services/hermes/scripts/node_polkit_audit.py" + "services/hermes/scripts/node_polkit_audit.py", + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_arming.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", + "testing/quality_handoff_mutation.py" ], "lint_paths": [ "ci/scripts/hermes_image_release.py", @@ -140,7 +157,23 @@ "services/hermes/scripts/node_account_hardening.py", "services/hermes/scripts/node_account_io.py", "services/hermes/scripts/stage_runtime_access.py", - "services/hermes/scripts/node_polkit_audit.py" + "services/hermes/scripts/node_polkit_audit.py", + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_arming.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py" ], "pytest_suites": { "unit": { @@ -160,7 +193,8 @@ "services/mailu/scripts", "testing", "services/gitea/scripts", - "services/hermes/scm-common/scripts" + "services/hermes/scm-common/scripts", + "scripts/ops" ], "coverage_xml": "build/coverage-unit.xml" }, @@ -238,7 +272,8 @@ "services/hermes/scripts/node_account_hardening.py", "services/hermes/scripts/node_account_io.py", "services/hermes/scripts/stage_runtime_access.py", - "services/hermes/scripts/node_polkit_audit.py" + "services/hermes/scripts/node_polkit_audit.py", + "scripts/ops/hermes_handoff_*.py" ], "naming_rules": [ { @@ -280,6 +315,28 @@ }, "coverage": { "minimum_percent": 95.0, + "minimum_branch_percent": 95.0, + "branch_tracked_files": [ + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_arming.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", + "ci/scripts/semgrep_report.py", + "testing/quality_coverage.py", + "testing/quality_handoff_mutation.py" + ], "tracked_files": [ "ci/scripts/hermes_image_release.py", "dockerfiles/hermes-kaniko-heredoc-runner.py", @@ -343,7 +400,24 @@ "services/hermes/scm-common/scripts/deadline_http.py", "services/hermes/scm-common/scripts/git_pack_objects.py", "services/hermes/scm-common/scripts/receive_pack_scan.py", - "services/hermes/scripts/node_polkit_audit.py" + "services/hermes/scripts/node_polkit_audit.py", + "scripts/ops/hermes_handoff_acceptance.py", + "scripts/ops/hermes_handoff_arming.py", + "scripts/ops/hermes_handoff_catalog.py", + "scripts/ops/hermes_handoff_checks_access.py", + "scripts/ops/hermes_handoff_checks_delivery.py", + "scripts/ops/hermes_handoff_checks_platform.py", + "scripts/ops/hermes_handoff_checks_workers.py", + "scripts/ops/hermes_handoff_ephemeral.py", + "scripts/ops/hermes_handoff_evaluators.py", + "scripts/ops/hermes_handoff_exec.py", + "scripts/ops/hermes_handoff_json_rules.py", + "scripts/ops/hermes_handoff_model.py", + "scripts/ops/hermes_handoff_policy.py", + "scripts/ops/hermes_handoff_redaction.py", + "scripts/ops/hermes_handoff_rules.py", + "scripts/ops/hermes_handoff_run.py", + "testing/quality_handoff_mutation.py" ] } } diff --git a/testing/quality_handoff_mutation.py b/testing/quality_handoff_mutation.py new file mode 100644 index 00000000..95d2df42 --- /dev/null +++ b/testing/quality_handoff_mutation.py @@ -0,0 +1,261 @@ +"""Deterministic scored mutation gate for Hermes handoff safety invariants.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class Mutation: + """One source mutation and the isolated probe that must kill it.""" + + name: str + path: str + old: str + new: str + probe: str + + +@dataclass(frozen=True) +class MutationReport: + """Scored result; any invalid, surviving, or baseline-broken case is an issue.""" + + total: int + killed: int + issues: tuple[str, ...] + + @property + def score(self) -> float: + return 100.0 if not self.total else round(self.killed * 100.0 / self.total, 2) + + +MUTATIONS = ( + Mutation( + "policy-allows-git-fetch", + "hermes_handoff_policy.py", + ' "for-each-ref",\n', + ' "fetch",\n "for-each-ref",\n', + "from hermes_handoff_policy import *\n" + "try: check_argv(('git','fetch','origin'))\n" + "except PolicyError: raise SystemExit(0)\n" + "raise SystemExit(1)\n", + ), + Mutation( + "redaction-truncates-before-screening", + "hermes_handoff_redaction.py", + " return bound(redact(text), max_bytes)\n", + " return bound(text, max_bytes)\n", + "from hermes_handoff_redaction import safe_text\n" + "assert 'A9' not in safe_text('password=A9secret-prefix', 23)[0]\n", + ), + Mutation( + "empty-report-go", + "hermes_handoff_model.py", + " if (\n" + " not self.results\n" + " or self.invalid_statuses\n" + " or self.harness_errors\n" + " or self.blocking\n" + " ):\n", + " if (\n" + " self.invalid_statuses\n" + " or self.harness_errors\n" + " or self.blocking\n" + " ):\n", + "from hermes_handoff_model import *\nassert Report('read-only','now').decision == NO_GO\n", + ), + Mutation( + "generic-404-is-denial", + "hermes_handoff_rules.py", + "DENIAL_MARKERS = (\n", + 'DENIAL_MARKERS = (\n "404 not found",\n', + "from hermes_handoff_rules import looks_denied\n" + "from hermes_handoff_exec import Outcome\n" + "assert not looks_denied(Outcome(('x',),'x',1,'','404 not found'))\n", + ), + Mutation( + "truncated-evidence-reaches-evaluator", + "hermes_handoff_rules.py", + " if truncated:\n" + " return Evaluation(\n" + " NOT_RUN, f\"truncated evidence from step(s): {', '.join(truncated)}\"\n" + " )\n", + " if False and truncated:\n return Evaluation(NOT_RUN, 'mutant')\n", + "import hermes_handoff_evaluators\n" + "import hermes_handoff_rules as r\n" + "from hermes_handoff_exec import Outcome\n" + "from hermes_handoff_model import *\n" + "@r.evaluator('probe')\ndef probe(*_): return r.Evaluation(PASS,'bad')\n" + "s=CheckSpec('x','x','x','probe')\n" + "assert r.evaluate(s,{'x':Outcome(('x',),'x',0,truncated=True)}).status == NOT_RUN\n", + ), + Mutation( + "truncated-outcome-is-ok", + "hermes_handoff_exec.py", + " return self.error is None and self.returncode == 0 and not self.truncated\n", + " return self.error is None and self.returncode == 0\n", + "from hermes_handoff_exec import Outcome\nassert not Outcome(('x',),'x',0,truncated=True).ok\n", + ), + Mutation( + "nan-is-numeric", + "hermes_handoff_json_rules.py", + " or not math.isfinite(float(actual))\n", + " or False\n", + "import hermes_handoff_json_rules as j\n" + "from hermes_handoff_model import *\nfrom hermes_handoff_exec import Outcome\n" + "s=CheckSpec('x','x','x','json_numeric',expect={'step':'s','fields':{'x':{'min':0}}})\n" + "j.parsed_step=lambda *_: ({'x':float('nan')},'s')\n" + "assert j.evaluate_json_numeric(s,{'s':Outcome(('x',),'x',0)}).status == NOT_RUN\n", + ), + Mutation( + "cleanup-ignores-branch-delete-failure", + "hermes_handoff_ephemeral.py", + ' if not deleted.ok:\n problems.append("ephemeral branch deletion failed")\n', + ' if False and not deleted.ok:\n problems.append("mutant")\n', + "import hermes_handoff_ephemeral as e\nfrom hermes_handoff_exec import Outcome\n" + "class R:\n" + " def __init__(self): self.i=0\n" + " def run(self,*_a,**_k):\n" + " self.i+=1\n" + " return Outcome(('x',),'x',1 if self.i==1 else 0)\n" + "q=e.ArmRequest('atlas/titan-iac','origin','acceptance1',e.CONFIRMATION,expected_head='1'*40,expected_base_sha='2'*40)\n" + "assert e._cleanup(R(),object(),q,'/api/v1/repos/atlas/titan-iac',[]).status == 'FAIL'\n", + ), + Mutation( + "cleanup-accepts-open-pr", + "hermes_handoff_ephemeral.py", + ' elif "state does not match" not in problems:\n problems.append("state is not closed")\n', + ' elif False:\n problems.append("mutant")\n', + "import hermes_handoff_ephemeral as e\n" + "q=e.ArmRequest('atlas/titan-iac','origin','acceptance1',e.CONFIRMATION,expected_head='1'*40,expected_base_sha='2'*40)\n" + "p={'number':1,'state':'open','draft':True,'merged':False,'base':{'ref':'main','sha':'2'*40},'head':{'ref':q.ref,'sha':'1'*40}}\n" + "assert e._closed_pr_problems(p,q)\n", + ), + Mutation( + "absence-check-passes-on-zero-evidence", + "hermes_handoff_evaluators.py", + " if not observed:\n" + ' return Evaluation(NOT_RUN, f"step {key} exited 0 with no lines to examine")\n', + " if False and not observed:\n return Evaluation(NOT_RUN, 'mutant')\n", + "import hermes_handoff_evaluators as v\n" + "from hermes_handoff_exec import Outcome\nfrom hermes_handoff_model import *\n" + "s=CheckSpec('x','x','x','names_absent',steps=(Step('e','self',('kubectl',)),)," + "expect={'step':'e','names':('GITEA_TOKEN',)})\n" + "assert v.evaluate(s,{'e':Outcome(('x',),'x',0,'')}).status == NOT_RUN\n", + ), + Mutation( + "repository-pin-is-a-bare-prefix", + "hermes_handoff_policy.py", + ' and not path.startswith((f"{repo_root}/", f"{repo_root}?"))\n', + " and not path.startswith(repo_root)\n", + "from hermes_handoff_policy import *\n" + "try: check_argv((GITEA_CLIENT,'GET','/api/v1/repos/atlas/titan-iac-evil/pulls'))\n" + "except PolicyError: raise SystemExit(0)\n" + "raise SystemExit(1)\n", + ), + Mutation( + "exec-inner-command-is-not-rechecked", + "hermes_handoff_policy.py", + " check_argv(inner, mode)\n return\n", + " return\n", + "from hermes_handoff_policy import *\n" + "argv=('kubectl','exec','pod','--','/usr/local/bin/kubectl','get','ns','-o','name'," + "'--as','system:admin')\n" + "try: check_argv(argv)\n" + "except PolicyError: raise SystemExit(0)\n" + "raise SystemExit(1)\n", + ), + Mutation( + "unrecorded-step-loses-its-binary-attestation", + "hermes_handoff_run.py", + " executable_path=outcome.executable_path,\n" + " executable_sha256=outcome.executable_sha256,\n", + "", + "import hermes_handoff_run as r\nfrom hermes_handoff_exec import Outcome\n" + "from hermes_handoff_model import *\n" + "class V:\n name='self'\n" + "class N:\n" + " def run(self,argv,_v,_b=None): return Outcome(argv,'self',0,'locked'," + "executable_path='/usr/bin/git',executable_sha256='a'*64)\n" + "s=CheckSpec('x','x','x','stdout_matches'," + "steps=(Step('s','self',('git','status'),record=False),)," + "expect={'step':'s','equals':'locked'})\n" + "o=r.run_check(N(),s,{'self':V()}).outcomes[0]\n" + "assert o.executable_sha256 == 'a'*64 and o.executable_path == '/usr/bin/git'\n", + ), +) + + +def _probe(root: Path, probe: str) -> subprocess.CompletedProcess[str]: + environment = { + "LC_ALL": "C", + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "PYTHONPATH": str(root), + } + return subprocess.run( + [sys.executable, "-c", probe], + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=10, + check=False, + env=environment, + ) + + +def run_gate( + repo_root: Path, mutations: Sequence[Mutation] = MUTATIONS +) -> MutationReport: + """Run baseline and mutated probes in isolated copies and return the score.""" + source_root = repo_root / "scripts" / "ops" + issues: list[str] = [] + killed = 0 + with tempfile.TemporaryDirectory(prefix="hermes-mutation-") as temporary: + clean = Path(temporary) / "clean" + shutil.copytree(source_root, clean) + for mutation in mutations: + source = clean / mutation.path + original = source.read_text(encoding="utf-8") + if original.count(mutation.old) != 1: + issues.append(f"{mutation.name}: mutation anchor count is not one") + continue + baseline = _probe(clean, mutation.probe) + if baseline.returncode != 0: + issues.append(f"{mutation.name}: baseline probe failed") + continue + mutated_text = original.replace(mutation.old, mutation.new, 1) + try: + compile(mutated_text, str(source), "exec") + except SyntaxError: + issues.append(f"{mutation.name}: mutant is syntactically invalid") + continue + mutant = Path(temporary) / f"mutant-{mutation.name}" + shutil.copytree(clean, mutant) + (mutant / mutation.path).write_text(mutated_text, encoding="utf-8") + if _probe(mutant, mutation.probe).returncode != 0: + killed += 1 + else: + issues.append(f"{mutation.name}: mutant survived") + return MutationReport(len(mutations), killed, tuple(issues)) + + +def main() -> int: + """Run the repository mutation contract and print a bounded score.""" + report = run_gate(Path(__file__).resolve().parents[1]) + print( + f"Hermes handoff mutation score: {report.killed}/{report.total} ({report.score:.2f}%)" + ) + for issue in report.issues: + print(issue) + return 0 if not report.issues and report.killed == report.total else 1 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) # pragma: no cover diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py index 09dcb0a1..35d86f8f 100644 --- a/testing/tests/conftest.py +++ b/testing/tests/conftest.py @@ -1,12 +1,26 @@ -"""Shared pytest boundaries for repository automation tests.""" +"""Shared pytest boundaries and runtime-path fixtures for repository tests.""" from __future__ import annotations import sys +from pathlib import Path import pytest +@pytest.fixture(autouse=True) +def isolate_coordinator_gitea_token( + request: pytest.FixtureRequest, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Prevent coordinator tests from observing the mounted production token.""" + if request.node.path.name == "test_hermes_coordinator.py": + monkeypatch.setenv( + "HERMES_GITEA_TOKEN_FILE", str(tmp_path / "absent-gitea-token") + ) + + @pytest.fixture(autouse=True) def _patched_hermes_cli_db_contract(request, monkeypatch): """Keep legacy lane test doubles focused on their original safety boundary.""" diff --git a/testing/tests/test_hermes_ai_usage_codex.py b/testing/tests/test_hermes_ai_usage_codex.py new file mode 100644 index 00000000..a77525bf --- /dev/null +++ b/testing/tests/test_hermes_ai_usage_codex.py @@ -0,0 +1,255 @@ +"""Adversarial tests for bounded Codex quota-query process handling.""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from io import StringIO +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "services/hermes/scripts/ai_usage_codex.py" + + +def load_module(): + """Load the standalone Codex query helper as a fresh test module.""" + name = "ai_usage_codex_tested" + spec = importlib.util.spec_from_file_location(name, SCRIPT) + module = importlib.util.module_from_spec(spec) + assert spec and spec.loader + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def test_codex_query_uses_structured_app_server_protocol(tmp_path, monkeypatch): + mod = load_module() + request_log = tmp_path / "requests.jsonl" + mock = tmp_path / "codex-mock" + mock.write_text( + """#!/usr/bin/env python3 +import json, os, sys +messages = [json.loads(sys.stdin.readline()) for _ in range(4)] +with open(os.environ['REQUEST_LOG'], 'w') as handle: + for message in messages: + handle.write(json.dumps(message) + '\\n') +responses = { + 1: {}, + 2: {'rateLimits': {'primary': {'usedPercent': 10}}}, + 3: {'summary': {}, 'dailyUsageBuckets': []}, +} +for response_id, result in responses.items(): + print(json.dumps({'id': response_id, 'result': result}), flush=True) +""" + ) + mock.chmod(0o755) + monkeypatch.setattr(mod, "CODEX_BIN", str(mock)) + monkeypatch.setenv("REQUEST_LOG", str(request_log)) + + rate, usage = mod.query_codex(timeout=5) + requests = [json.loads(line) for line in request_log.read_text().splitlines()] + + assert rate["rateLimits"]["primary"]["usedPercent"] == 10 + assert usage["summary"] == {} + assert [request["method"] for request in requests] == [ + "initialize", + "initialized", + "account/rateLimits/read", + "account/usage/read", + ] + assert all("/status" not in json.dumps(request) for request in requests) + + +def test_codex_cleanup_is_bounded_when_process_cannot_be_reaped(monkeypatch): + mod = load_module() + processes = [] + selectors = [] + + class StubbornProcess: + def __init__(self, *_args, **_kwargs): + self.stdin = StringIO() + self.stdout = StringIO( + "\n".join( + json.dumps({"id": response_id, "result": result}) + for response_id, result in ( + (1, {}), + (2, {"rateLimits": {}}), + (3, {"summary": {}, "dailyUsageBuckets": []}), + ) + ) + + "\n" + ) + self.stderr = None + self.terminate_calls = 0 + self.kill_calls = 0 + self.wait_timeouts = [] + processes.append(self) + + def terminate(self): + self.terminate_calls += 1 + + def kill(self): + self.kill_calls += 1 + + def wait(self, timeout): + self.wait_timeouts.append(timeout) + raise subprocess.TimeoutExpired("codex-mock", timeout) + + class TrackingSelector: + def __init__(self): + self.fileobj = None + self.closed = False + selectors.append(self) + + def register(self, fileobj, _events): + self.fileobj = fileobj + + def select(self, _timeout): + return [(SimpleNamespace(fileobj=self.fileobj), None)] + + def close(self): + self.closed = True + + monkeypatch.setattr(mod.subprocess, "Popen", StubbornProcess) + monkeypatch.setattr(mod.selectors, "DefaultSelector", TrackingSelector) + + with pytest.raises(RuntimeError, match="^Codex app-server cleanup failed$"): + mod.query_codex(timeout=1) + + process = processes[0] + assert process.terminate_calls == 1 + assert process.kill_calls == 1 + assert process.wait_timeouts == [5, 5] + assert process.stdin.closed + assert process.stdout.closed + assert selectors[0].closed + + +def test_cleanup_contains_all_pipe_and_process_api_failures(): + mod = load_module() + + class BrokenStream: + def close(self): + raise OSError("private pipe detail") + + class BrokenSelector: + def close(self): + raise OSError("private selector detail") + + class BrokenProcess: + stdin = BrokenStream() + stdout = BrokenStream() + stderr = BrokenStream() + + def __init__(self): + self.wait_calls = 0 + + def terminate(self): + raise OSError("private terminate detail") + + def wait(self, timeout): + assert timeout == mod.CODEX_CLEANUP_TIMEOUT_SECONDS + self.wait_calls += 1 + raise OSError("private wait detail") + + def kill(self): + raise OSError("private kill detail") + + assert mod._close_stream(None) + assert not mod._close_stream(BrokenStream()) + assert not mod._cleanup_codex_process(BrokenProcess(), BrokenSelector()) + + +def test_cleanup_accepts_already_exited_process_during_terminate_and_kill(): + mod = load_module() + + class AlreadyExited: + stdin = None + stdout = None + stderr = None + + def __init__(self, *, timeout_first): + self.timeout_first = timeout_first + self.wait_calls = 0 + + def terminate(self): + raise ProcessLookupError + + def wait(self, timeout): + self.wait_calls += 1 + if self.timeout_first and self.wait_calls == 1: + raise subprocess.TimeoutExpired("codex", timeout) + + def kill(self): + raise ProcessLookupError + + assert mod._cleanup_codex_process(AlreadyExited(timeout_first=False), None) + assert mod._cleanup_codex_process(AlreadyExited(timeout_first=True), None) + + +def test_query_rejects_missing_pipes_and_structured_errors(monkeypatch): + mod = load_module() + + class Process: + stderr = None + + def __init__(self, *_args, **_kwargs): + self.stdin = None + self.stdout = None + + def terminate(self): + return None + + def wait(self, timeout): + return 0 + + monkeypatch.setattr(mod.subprocess, "Popen", Process) + with pytest.raises(RuntimeError, match="pipes are unavailable"): + mod.query_codex(timeout=0) + + class InvalidProcess(Process): + def __init__(self, *_args, **_kwargs): + self.stdin = StringIO() + self.stdout = SequencedOutput() + + class SequencedOutput: + def __init__(self): + self.lines = iter( + [ + "", + json.dumps({"id": 99, "result": {}}) + "\n", + json.dumps({"id": 1, "result": {}}) + "\n", + json.dumps({"id": 2, "error": {"message": "secret"}}) + "\n", + json.dumps({"id": 3, "result": {}}) + "\n", + ] + ) + + def readline(self): + return next(self.lines) + + def close(self): + return None + + class Selector: + def register(self, fileobj, _events): + self.fileobj = fileobj + + def select(self, _timeout): + return [(SimpleNamespace(fileobj=self.fileobj), None)] + + def close(self): + return None + + monkeypatch.setattr(mod.subprocess, "Popen", InvalidProcess) + monkeypatch.setattr(mod.selectors, "DefaultSelector", Selector) + with pytest.raises( + RuntimeError, match="^Codex app-server response 2 failed$" + ) as caught: + mod.query_codex(timeout=1) + assert "secret" not in str(caught.value) diff --git a/testing/tests/test_hermes_ai_usage_exporter.py b/testing/tests/test_hermes_ai_usage_exporter.py index 9b78b7f3..c33654dc 100644 --- a/testing/tests/test_hermes_ai_usage_exporter.py +++ b/testing/tests/test_hermes_ai_usage_exporter.py @@ -3,9 +3,9 @@ from __future__ import annotations import importlib.util -import json import sys from datetime import date +from io import BytesIO from pathlib import Path @@ -15,6 +15,9 @@ SCRIPT = ROOT / "services/hermes/scripts/ai_usage_exporter.py" def load_module(): """Load the standalone exporter script as a test module.""" + script_directory = str(SCRIPT.parent) + if script_directory not in sys.path: + sys.path.insert(0, script_directory) spec = importlib.util.spec_from_file_location("ai_usage_exporter", SCRIPT) module = importlib.util.module_from_spec(spec) assert spec and spec.loader @@ -54,7 +57,9 @@ def test_codex_payloads_include_real_windows_and_bounded_token_periods(): } samples = mod.parse_codex_payloads(rate, usage, today=date(2026, 8, 16)) - values = {(item.name, tuple(sorted(item.labels.items()))): item.value for item in samples} + values = { + (item.name, tuple(sorted(item.labels.items()))): item.value for item in samples + } codex_labels = ( ("limit", "codex"), @@ -69,18 +74,24 @@ def test_codex_payloads_include_real_windows_and_bounded_token_periods(): assert values[("atlas_ai_quota_used_percent", codex_labels)] == 41 assert values[("atlas_ai_quota_remaining_percent", codex_labels)] == 59 assert values[("atlas_ai_quota_used_percent", spark_labels)] == 0 - assert values[ - ( - "atlas_ai_account_tokens", - (("period", "latest_day"), ("provider", "openai")), - ) - ] == 100 - assert values[ - ( - "atlas_ai_account_tokens", - (("period", "seven_day"), ("provider", "openai")), - ) - ] == 150 + assert ( + values[ + ( + "atlas_ai_account_tokens", + (("period", "latest_day"), ("provider", "openai")), + ) + ] + == 100 + ) + assert ( + values[ + ( + "atlas_ai_account_tokens", + (("period", "seven_day"), ("provider", "openai")), + ) + ] + == 150 + ) def test_claude_payload_uses_only_supported_quota_fields(): @@ -131,45 +142,161 @@ def test_render_reports_failure_and_freshness_without_logging_credentials(): rendered = collector.render().decode() assert 'atlas_ai_quota_fetch_success{provider="openai"} 0' in rendered - assert 'atlas_ai_quota_last_success_timestamp_seconds{provider="openai"} 100' in rendered - assert 'atlas_ai_quota_used_percent{limit="codex",provider="openai",window="seven_day"} 42' in rendered + assert ( + 'atlas_ai_quota_last_success_timestamp_seconds{provider="openai"} 100' + in rendered + ) + assert ( + 'atlas_ai_quota_used_percent{limit="codex",provider="openai",window="seven_day"} 42' + in rendered + ) assert "accessToken" not in rendered assert "refreshToken" not in rendered -def test_codex_query_uses_structured_app_server_protocol(tmp_path, monkeypatch): +def test_unexpected_provider_failure_is_sanitized_and_preserves_last_good( + monkeypatch, capsys +): mod = load_module() - request_log = tmp_path / "requests.jsonl" - mock = tmp_path / "codex-mock" - mock.write_text( - """#!/usr/bin/env python3 -import json, os, sys -messages = [json.loads(sys.stdin.readline()) for _ in range(4)] -with open(os.environ['REQUEST_LOG'], 'w') as handle: - for message in messages: - handle.write(json.dumps(message) + '\\n') -responses = { - 1: {}, - 2: {'rateLimits': {'primary': {'usedPercent': 10}}}, - 3: {'summary': {}, 'dailyUsageBuckets': []}, -} -for response_id, result in responses.items(): - print(json.dumps({'id': response_id, 'result': result}), flush=True) -""" + collector = mod.Collector() + previous = mod.Sample( + "atlas_ai_quota_used_percent", + {"provider": "openai", "limit": "codex", "window": "seven_day"}, + 42, ) - mock.chmod(0o755) - monkeypatch.setattr(mod, "CODEX_BIN", str(mock)) - monkeypatch.setenv("REQUEST_LOG", str(request_log)) + collector._providers["openai"].samples = [previous] + secret = "provider-body-must-not-appear" - rate, usage = mod.query_codex(timeout=5) - requests = [json.loads(line) for line in request_log.read_text().splitlines()] + class UnexpectedProviderFailure(Exception): + pass - assert rate["rateLimits"]["primary"]["usedPercent"] == 10 - assert usage["summary"] == {} - assert [request["method"] for request in requests] == [ - "initialize", - "initialized", - "account/rateLimits/read", - "account/usage/read", - ] - assert all("/status" not in json.dumps(request) for request in requests) + def fail(): + raise UnexpectedProviderFailure(secret) + + monkeypatch.setattr(mod, "query_codex", fail) + collector.refresh_provider("openai") + + output = capsys.readouterr().out + state = collector._providers["openai"] + assert state.samples == [previous] + assert state.fetch_success is False + assert state.last_attempt > 0 + assert "UnexpectedProviderFailure" in output + assert secret not in output + + +def test_polling_engine_isolates_providers_and_future_cycles(monkeypatch, capsys): + mod = load_module() + collector = mod.Collector() + calls = [] + secret = "cleanup-detail-must-not-appear" + + class UnexpectedCleanupFailure(Exception): + pass + + def refresh(provider): + calls.append(provider) + if provider == "openai" and calls.count("openai") == 1: + raise UnexpectedCleanupFailure(secret) + + monkeypatch.setattr(collector, "refresh_provider", refresh) + engine = mod.PollingEngine(collector, interval=60) + + engine.poll_once() + engine.poll_once() + + output = capsys.readouterr().out + assert calls == ["openai", "anthropic", "openai", "anthropic"] + assert collector._providers["openai"].fetch_success is False + assert collector._providers["openai"].last_attempt > 0 + assert "UnexpectedCleanupFailure" in output + assert secret not in output + + +def test_health_tracks_poller_liveness_and_progress_not_provider_success(): + mod = load_module() + now = [100.0] + collector = mod.Collector() + collector._providers["openai"].fetch_success = False + engine = mod.PollingEngine( + collector, + interval=60, + startup_grace=10, + progress_timeout=75, + clock=lambda: now[0], + ) + + class ThreadState: + alive = True + + def is_alive(self): + return self.alive + + thread = ThreadState() + engine._thread = thread + assert engine.is_healthy() + + now[0] = 111.0 + assert not engine.is_healthy() + + engine._mark_progress() + assert engine.is_healthy() + now[0] = 187.0 + assert not engine.is_healthy() + + thread.alive = False + now[0] = 100.0 + assert not engine.is_healthy() + + +def test_health_endpoint_uses_poller_state_and_ignores_provider_failure(): + mod = load_module() + collector = mod.Collector() + + class Poller: + healthy = False + + def is_healthy(self): + return self.healthy + + poller = Poller() + handler_class = mod.make_handler(collector, poller) + handler = object.__new__(handler_class) + handler.path = "/healthz" + statuses = [] + handler.send_response = statuses.append + handler.send_header = lambda _name, _value: None + handler.end_headers = lambda: None + handler.wfile = BytesIO() + + handler.do_GET() + assert statuses == [503] + + poller.healthy = True + statuses.clear() + collector._providers["openai"].fetch_success = False + handler.do_GET() + assert statuses == [200] + + +def test_metrics_response_tolerates_disconnected_client(): + mod = load_module() + collector = mod.Collector() + + class HealthyPoller: + def is_healthy(self): + return True + + class Disconnected(BytesIO): + def write(self, _body): + raise BrokenPipeError + + handler_class = mod.make_handler(collector, HealthyPoller()) + handler = object.__new__(handler_class) + handler.path = "/metrics" + handler.send_response = lambda _status: None + handler.send_header = lambda _name, _value: None + handler.end_headers = lambda: None + handler.wfile = Disconnected() + + handler.do_GET() diff --git a/testing/tests/test_hermes_ai_usage_exporter_coverage.py b/testing/tests/test_hermes_ai_usage_exporter_coverage.py new file mode 100644 index 00000000..affaf336 --- /dev/null +++ b/testing/tests/test_hermes_ai_usage_exporter_coverage.py @@ -0,0 +1,275 @@ +"""Focused edge coverage for AI usage exporter provider and polling boundaries.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from io import BytesIO +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "services/hermes/scripts/ai_usage_exporter.py" + + +def load_module(): + """Load a fresh exporter while making its sibling modules importable.""" + script_directory = str(SCRIPT.parent) + if script_directory not in sys.path: + sys.path.insert(0, script_directory) + name = "ai_usage_exporter_coverage" + spec = importlib.util.spec_from_file_location(name, SCRIPT) + module = importlib.util.module_from_spec(spec) + assert spec and spec.loader + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def test_parser_rejects_nonfinite_and_malformed_optional_fields(): + mod = load_module() + + assert mod._number(True) is None + assert mod._number("1") is None + assert mod._number(float("nan")) is None + assert mod._number(float("inf")) is None + assert mod._timestamp(None) is None + assert mod._timestamp("not-a-time") is None + assert mod._window_name({}, "fallback") == "fallback" + assert mod._codex_limit_name("other", {}) == "additional" + + samples = mod.parse_codex_payloads( + { + "rateLimitsByLimitId": { + 1: {}, + "not-a-snapshot": "invalid", + "unnamed": { + "primary": {"usedPercent": None}, + "secondary": { + "usedPercent": 120, + "resetsAt": "invalid", + }, + }, + } + }, + { + "summary": { + "lifetimeTokens": None, + "currentStreakDays": 2, + "longestStreakDays": 3, + "longestRunningTurnSec": 4, + }, + "dailyUsageBuckets": [ + None, + {"startDate": "invalid", "tokens": 2}, + {"startDate": "2026-08-17", "tokens": None}, + ], + }, + ) + + assert any(sample.value == 120 for sample in samples) + assert not any( + sample.name == "atlas_ai_quota_reset_timestamp_seconds" for sample in samples + ) + assert mod.parse_codex_payloads({"rateLimits": None}, {"summary": None}) + + +def test_claude_parser_ignores_unbounded_fields_and_invalid_extra_usage(): + mod = load_module() + + assert mod.parse_claude_payload( + { + "five_hour": {"utilization": None}, + "seven_day": {"utilization": 10, "resets_at": None}, + "extra_usage": {"is_enabled": "yes"}, + } + ) + assert mod.parse_claude_payload({"extra_usage": "invalid"}) == [] + + +def test_provider_queries_validate_credentials_and_response_shape( + tmp_path, monkeypatch +): + mod = load_module() + credentials = tmp_path / "credentials.json" + credentials.write_text("{}", encoding="utf-8") + monkeypatch.setattr(mod, "CLAUDE_CREDENTIALS", credentials) + + with pytest.raises(RuntimeError, match="credentials are incomplete"): + mod.query_claude() + + credentials.write_text( + json.dumps({"claudeAiOauth": {"accessToken": "runtime-only-token"}}), + encoding="utf-8", + ) + bodies = iter([{"five_hour": {}}, []]) + requests = [] + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self, maximum): + assert maximum == 1 << 20 + return json.dumps(next(bodies)).encode() + + def open_request(request, timeout): + requests.append(request) + assert timeout == 15 + return Response() + + monkeypatch.setattr(mod, "urlopen", open_request) + assert mod.query_claude() == {"five_hour": {}} + assert requests[0].get_header("Authorization") == "Bearer runtime-only-token" + with pytest.raises(RuntimeError, match="response is not an object"): + mod.query_claude() + + assert mod.query_codex is mod.codex_query.query_codex + + +def test_collector_refreshes_both_providers_and_contains_unknown_provider( + monkeypatch, capsys +): + mod = load_module() + collector = mod.Collector() + monkeypatch.setattr( + mod, + "query_codex", + lambda: ({"rateLimits": {"primary": {"usedPercent": 1}}}, {}), + ) + monkeypatch.setattr(mod, "query_claude", lambda: {"five_hour": {"utilization": 2}}) + + collector.refresh_provider("openai") + collector.refresh_provider("anthropic") + assert collector._providers["openai"].fetch_success + assert collector._providers["anthropic"].fetch_success + assert collector._providers["openai"].last_success > 0 + + collector.record_failure("openai") + assert not collector._providers["openai"].fetch_success + with pytest.raises(KeyError, match="unknown"): + collector.refresh_provider("unknown") + assert "unknown quota collection deferred: ValueError" in capsys.readouterr().out + + +def test_polling_failure_accounting_and_cycle_errors_are_sanitized(monkeypatch, capsys): + mod = load_module() + secret = "must-not-leak" + + class Failure(Exception): + pass + + class Collector: + def refresh_provider(self, _provider): + raise Failure(secret) + + def record_failure(self, *_args, **_kwargs): + raise Failure(secret) + + engine = mod.PollingEngine(Collector(), interval=60) + engine.poll_once() + output = capsys.readouterr().out + assert "quota failure accounting deferred: Failure" in output + assert secret not in output + + monkeypatch.setattr( + engine, "poll_once", lambda: (_ for _ in ()).throw(Failure(secret)) + ) + monkeypatch.setattr( + mod.time, "sleep", lambda _interval: (_ for _ in ()).throw(StopIteration) + ) + with pytest.raises(StopIteration): + engine.run() + output = capsys.readouterr().out + assert "quota polling cycle deferred: Failure" in output + assert secret not in output + + +def test_poller_start_is_idempotent_and_main_wires_server(monkeypatch): + mod = load_module() + threads = [] + + class Thread: + def __init__(self, **kwargs): + self.kwargs = kwargs + self.started = False + threads.append(self) + + def start(self): + self.started = True + + monkeypatch.setattr(mod.threading, "Thread", Thread) + engine = mod.PollingEngine(mod.Collector(), interval=60) + engine.start() + engine.start() + assert len(threads) == 1 + assert threads[0].started + assert threads[0].kwargs["daemon"] is True + + events = [] + + class Poller: + def __init__(self, collector, *, interval): + events.append(("poller", collector, interval)) + + def start(self): + events.append("started") + + class Server: + def __init__(self, address, handler): + events.append(("server", address, handler)) + + def serve_forever(self): + events.append("served") + + monkeypatch.setattr(mod, "PollingEngine", Poller) + monkeypatch.setattr(mod, "Server", Server) + monkeypatch.setattr( + mod, "make_handler", lambda collector, poller: (collector, poller) + ) + monkeypatch.setenv("ATLAS_AI_USAGE_INTERVAL_SECONDS", "1") + monkeypatch.setenv("ATLAS_AI_USAGE_PORT", "19010") + + assert mod.main() == 0 + assert events[0][0] == "poller" + assert events[0][2] == 60 + assert events[2][1][1] == 19010 + assert events[-1] == "served" + + +def test_http_handler_covers_not_found_logging_and_connection_reset(): + mod = load_module() + + class Collector: + def render(self): + return b"metrics\n" + + class Poller: + def is_healthy(self): + return True + + handler_class = mod.make_handler(Collector(), Poller()) + handler = object.__new__(handler_class) + handler.path = "/missing" + statuses = [] + handler.send_error = statuses.append + handler.do_GET() + assert statuses == [404] + assert handler.log_message("ignored") is None + + class Reset(BytesIO): + def write(self, _body): + raise ConnectionResetError + + handler.path = "/metrics" + handler.send_response = lambda _status: None + handler.send_header = lambda _name, _value: None + handler.end_headers = lambda: None + handler.wfile = Reset() + handler.do_GET() diff --git a/testing/tests/test_hermes_auto_router.py b/testing/tests/test_hermes_auto_router.py index 05af3b01..f8a333b1 100644 --- a/testing/tests/test_hermes_auto_router.py +++ b/testing/tests/test_hermes_auto_router.py @@ -255,7 +255,12 @@ def test_provider_status_separates_observed_activity_from_plan_quota(monkeypatch monkeypatch.setattr(module, "_claude_account", lambda: { "authenticated": True, "plan": "max", "quota_reported": False, }) - monkeypatch.setattr(module, "_fresh_health", lambda _path: {}) + monkeypatch.setattr(module, "_fresh_health", lambda _path, maximum_age=86400.0: {}) + monkeypatch.setattr( + module, + "_configured_models", + lambda provider: ["gpt-5.6-sol"] if provider == "codex" else ["claude-sonnet-5"], + ) payload = module.provider_status_payload() diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index b513b452..4cf45ffb 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -305,7 +305,7 @@ def test_webui_recovers_auth_and_labels_session_scoped_controls(): assert "session_key.startswith('telegram-topic-')" in telegram_project_patch router = (ROOT / "dockerfiles" / "hermes-webui-router.js").read_text() - assert "'atlas/auto/fast':'AUTO · Fast'" in router + assert "'atlas/auto/fast':'Automatic · Fast'" in router assert "'atlas/manual/codex/sol':'Codex · SOL'" in router assert "'atlas/manual/claude/opus':'Claude · Opus'" in router assert "watchModelOptions('modelSelect')" in router diff --git a/testing/tests/test_hermes_chat_smoothness.py b/testing/tests/test_hermes_chat_smoothness.py new file mode 100644 index 00000000..ee7a32eb --- /dev/null +++ b/testing/tests/test_hermes_chat_smoothness.py @@ -0,0 +1,493 @@ +"""Adversarial contracts for chat-only continuity and truthful rendering.""" + +from __future__ import annotations + +import importlib.util +import json +import runpy +import sqlite3 +import sys +from pathlib import Path + +import pytest +import yaml + + +ROOT = Path(__file__).parents[2] +HERMES = ROOT / "services" / "hermes" + + +def _load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def continuity(): + return _load( + "telegram_continuity", + HERMES / "scripts" / "migrate_telegram_api_sessions.py", + ) + + +def test_context_is_deterministic_bounded_deduplicated_and_truthful(continuity): + turns = [] + for index in range(30): + user = {"role": "user", "content": f"question {index}", "_db_persisted": True} + assistant = { + "role": "assistant", + "content": f"answer {index}", + "_db_persisted": True, + } + turns.extend((user, assistant)) + if index == 4: + # Simulate one complete transcript replay. + turns.extend((user.copy(), assistant.copy())) + turns.extend( + [ + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": { + "name": "browser", + "arguments": "x" * 40_000, + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call-1", + "tool_name": "browser", + "content": "tool output " * 4_000, + }, + { + "role": "user", + "content": [ + {"type": "input_text", "text": "edit this"}, + { + "type": "input_image", + "image_url": "data:image/png;base64," + "a" * 80_000, + }, + ], + }, + { + "role": "assistant", + "content": "Created it MEDIA:/opt/data/cache/images/private.png", + }, + ] + ) + + compacted = continuity.compact_telegram_history(turns) + + assert compacted == continuity.compact_telegram_history(turns) + assert compacted[0]["role"] == "system" + assert compacted[0]["_db_persisted"] is True + assert compacted[0]["content"].startswith(continuity.SUMMARY_PREFIX) + assert sum(item["role"] == "user" for item in compacted[1:]) <= 12 + assert len(compacted[1:]) <= continuity.MAX_HISTORY_ITEMS + assert len(json.dumps(compacted).encode()) <= ( + continuity.MAX_HISTORY_BYTES + continuity.MAX_SUMMARY_CHARS + ) + encoded = json.dumps(compacted) + assert "data:image" not in encoded + assert "MEDIA:" not in encoded + assert "/opt/data/" not in encoded + assert continuity.ATTACHMENT_NOTE in encoded + assert "question 29" in encoded + assert ( + next(item for item in compacted if item.get("content") == "question 29")[ + "_db_persisted" + ] + is True + ) + assert "x" * 3_000 not in encoded + fingerprints = [continuity._fingerprint(item) for item in compacted] + assert all( + left != right + for left, right in zip(fingerprints, fingerprints[1:], strict=False) + ) + + +def test_context_rejects_malformed_history_and_bounds_single_large_item(continuity): + assert continuity.compact_telegram_history(None) == [] + assert continuity.compact_telegram_history({"role": "user"}) == [] + malformed = [ + None, + "text", + {"role": "unknown", "content": "skip"}, + {"role": "tool", "content": ""}, + {"role": "user", "content": {"nested": "value"}}, + {"role": "assistant", "content": "z" * 2_000_000}, + ] + compacted = continuity.compact_telegram_history(malformed) + assert len(compacted) == 2 + assert compacted[0]["content"] == '{"nested": "value"}' + assert len(compacted[1]["content"]) == continuity.MAX_ITEM_CHARS + + +def test_context_defensive_normalization_and_replay_edges(continuity, monkeypatch): + assert continuity._text(["skip", {"type": "text", "text": "keep"}]) == "keep" + assert continuity._text({"not-json-serializable"}) == "{'not-json-serializable'}" + normalized = continuity._normalized_item( + { + "role": "assistant", + "tool_calls": [ + None, + {"id": "call", "function": {"name": "browser", "arguments": {1}}}, + ], + } + ) + assert normalized and normalized["tool_calls"][0]["id"] == "call" + assert continuity._fingerprint({"role": "user", "content": {1}}).startswith( + "{'role':" + ) + + user = {"role": "user", "content": "same"} + assistant = {"role": "assistant", "content": "reply"} + assert continuity._deduplicate( + [user, assistant, user.copy(), assistant.copy()] + ) == [ + user, + assistant, + ] + assert continuity._deduplicate( + [ + {"role": "tool", "content": "first", "id": "stable"}, + {"role": "tool", "content": "replay", "id": "stable"}, + ] + ) == [{"role": "tool", "content": "first", "id": "stable"}] + + summary = continuity._summary( + [ + { + "role": "system", + "content": continuity.SUMMARY_PREFIX + "\n- existing fact", + }, + {"role": "tool", "content": "not durable prose"}, + ] + ) + assert summary and "existing fact" in summary["content"] + monkeypatch.setattr( + continuity, "MAX_SUMMARY_CHARS", len(continuity.SUMMARY_PREFIX) + 1 + ) + assert ( + continuity._summary([{"role": "user", "content": "too long"}])["content"] + == continuity.SUMMARY_PREFIX + ) + + +def test_context_final_serialized_size_guard(continuity, monkeypatch): + monkeypatch.setattr(continuity, "MAX_HISTORY_ITEMS", 2) + monkeypatch.setattr(continuity, "MAX_HISTORY_BYTES", 1_000) + monkeypatch.setattr( + continuity, + "_summary", + lambda _older: { + "role": "system", + "content": "s" * 20_000, + "_db_persisted": True, + }, + ) + compacted = continuity.compact_telegram_history( + [ + {"role": "user", "content": "older"}, + {"role": "assistant", "content": "recent one"}, + {"role": "assistant", "content": "recent two"}, + ] + ) + assert [item["content"] for item in compacted[1:]] == ["recent two"] + + +def _create_session_databases(tmp_path: Path): + state = tmp_path / "state.db" + responses = tmp_path / "response_store.db" + with sqlite3.connect(state) as connection: + connection.execute( + """CREATE TABLE sessions ( + id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT, + display_name TEXT, origin_json TEXT, title TEXT + )""" + ) + connection.executemany( + "INSERT INTO sessions VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + ( + "legacy", + "api_server", + "wrong", + "group", + "Unassigned", + "{}", + "Unassigned", + ), + ("named", "api_server", None, None, None, None, "Keep this title"), + ("other", "cli", None, None, "Unassigned", None, None), + ), + ) + connection.execute( + "CREATE TABLE messages (id INTEGER PRIMARY KEY, session_id TEXT, role TEXT, content TEXT, timestamp REAL)" + ) + connection.executemany( + "INSERT INTO messages VALUES (?, 'legacy', 'user', 'same', 1)", ((1,), (2,)) + ) + with sqlite3.connect(responses) as connection: + connection.execute( + "CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)" + ) + connection.execute( + "CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)" + ) + connection.executemany( + "INSERT INTO responses VALUES (?, ?, 0)", + ( + ("general", json.dumps({"session_id": "legacy"})), + ("topic", json.dumps({"session_id": "named"})), + ("bad-json", "{"), + ("bad-id", json.dumps({"session_id": "../unsafe"})), + ( + "oversized", + json.dumps({"session_id": "other", "padding": "x" * 500}), + ), + ), + ) + connection.executemany( + "INSERT INTO conversations VALUES (?, ?)", + ( + ("telegram", "general"), + ("telegram-topic-plans-g1", "topic"), + ("telegram-topic-malformed", "bad-json"), + ("telegram-topic-unsafe", "bad-id"), + ("telegram-topic-oversized", "oversized"), + ), + ) + return state, responses + + +def test_existing_session_migration_repairs_labels_and_is_idempotent( + tmp_path: Path, continuity, monkeypatch +): + state, responses = _create_session_databases(tmp_path) + monkeypatch.setattr(continuity, "MAX_RESPONSE_BYTES", 300) + + assert continuity.migrate(state, responses) == 2 + assert continuity.migrate(state, responses) == 0 + + with sqlite3.connect(state) as connection: + rows = { + row[0]: row[1:] + for row in connection.execute( + "SELECT id, session_key, chat_type, display_name, origin_json, title " + "FROM sessions ORDER BY id" + ) + } + assert rows["legacy"][:3] == ("telegram", "private", "Telegram") + assert json.loads(rows["legacy"][3]) == { + "platform": "telegram", + "session_key": "telegram", + } + assert rows["legacy"][4] == "Telegram · General" + assert rows["named"][:3] == ( + "telegram-topic-plans-g1", + "private", + "Telegram", + ) + assert rows["named"][4] == "Keep this title" + assert rows["other"] == (None, None, "Unassigned", None, None) + with sqlite3.connect(state) as connection: + assert connection.execute("SELECT COUNT(*) FROM messages").fetchone() == (1,) + + +def test_migration_fails_closed_on_missing_or_malformed_databases( + tmp_path: Path, continuity +): + assert continuity.telegram_sessions(tmp_path / "missing.db") == {} + assert ( + continuity.migrate(tmp_path / "missing-state.db", tmp_path / "missing.db") == 0 + ) + broken = tmp_path / "broken.db" + broken.write_text("not sqlite", encoding="utf-8") + assert continuity.telegram_sessions(broken) == {} + + state = tmp_path / "state.db" + responses = tmp_path / "responses.db" + with sqlite3.connect(state) as connection: + connection.execute("CREATE TABLE sessions (id TEXT PRIMARY KEY)") + with sqlite3.connect(responses) as connection: + connection.execute("CREATE TABLE conversations (name TEXT, response_id TEXT)") + connection.execute("CREATE TABLE responses (response_id TEXT, data TEXT)") + connection.execute( + "INSERT INTO responses VALUES ('valid', '{\"session_id\": \"known\"}')" + ) + connection.execute("INSERT INTO conversations VALUES ('telegram', 'valid')") + assert continuity.migrate(state, responses) == 0 + + +def test_migration_skips_invalid_conversation_and_rolls_back_database_errors( + tmp_path: Path, continuity +): + state = tmp_path / "state.db" + responses = tmp_path / "responses.db" + with sqlite3.connect(state) as connection: + connection.execute( + """CREATE TABLE sessions ( + id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT, + display_name TEXT, origin_json TEXT, title TEXT + )""" + ) + connection.execute( + "INSERT INTO sessions VALUES ('known', 'api_server', NULL, NULL, NULL, NULL, NULL)" + ) + connection.execute( + """CREATE TRIGGER reject_updates BEFORE UPDATE ON sessions + BEGIN SELECT RAISE(FAIL, 'read only for test'); END""" + ) + with sqlite3.connect(responses) as connection: + connection.execute( + "CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)" + ) + connection.execute( + "CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL)" + ) + connection.executemany( + "INSERT INTO responses VALUES (?, ?)", + ( + ("valid", json.dumps({"session_id": "known"})), + ("invalid", json.dumps({"session_id": "ignored"})), + ), + ) + connection.executemany( + "INSERT INTO conversations VALUES (?, ?)", + (("telegram", "valid"), ("telegram-topic-BAD", "invalid")), + ) + + assert continuity.telegram_sessions(responses) == {"known": "telegram"} + assert continuity.migrate(state, responses) == 0 + with sqlite3.connect(state) as connection: + assert connection.execute( + "SELECT session_key FROM sessions WHERE id = 'known'" + ).fetchone() == (None,) + + +def test_api_patch_compacts_input_batch_and_disconnect_snapshots(tmp_path: Path): + patcher = _load( + "chat_api_patch", + HERMES / "scripts" / "patch_api_server_sessions.py", + ) + source = tmp_path / "api_server.py" + destination = tmp_path / "patched.py" + source.write_text( + patcher.BEFORE + + patcher.RUNS_BEFORE + + patcher.RUN_CLOSE_BEFORE + + patcher.TRUNCATION_BEFORE + + patcher.RESPONSES_SESSION_BEFORE + + patcher.EVENT_CALLBACK_SIGNATURE_BEFORE + + patcher.EVENT_CALLBACK_BODY_BEFORE + + patcher.EVENT_CALLBACK_END_BEFORE + + patcher.EVENT_CALLBACK_CALL_BEFORE + + patcher.RUN_SWEEP_BEFORE + + patcher.SSE_STORE_BEFORE + + patcher.HISTORY_STORE_BEFORE, + encoding="utf-8", + ) + + patcher.patch(source, destination) + patched = destination.read_text(encoding="utf-8") + + assert ( + "conversation_history = compact_telegram_history(conversation_history)" + in patched + ) + assert "conversation_history_snapshot = compact_telegram_history(" in patched + assert "full_history = compact_telegram_history(full_history)" in patched + assert patched.count("from gateway.platforms.telegram_continuity import") == 2 + + +def _patch_source(patcher) -> str: + return ( + patcher.BEFORE + + patcher.RUNS_BEFORE + + patcher.RUN_CLOSE_BEFORE + + patcher.RESPONSES_SESSION_BEFORE + + patcher.EVENT_CALLBACK_SIGNATURE_BEFORE + + patcher.EVENT_CALLBACK_BODY_BEFORE + + patcher.EVENT_CALLBACK_END_BEFORE + + patcher.EVENT_CALLBACK_CALL_BEFORE + + patcher.RUN_SWEEP_BEFORE + ) + + +@pytest.mark.parametrize( + "marker_name", + [ + "BEFORE", + "RUNS_BEFORE", + "RUN_CLOSE_BEFORE", + "RESPONSES_SESSION_BEFORE", + "EVENT_CALLBACK_SIGNATURE_BEFORE", + ], +) +def test_api_patch_fails_closed_on_upstream_drift(tmp_path: Path, marker_name: str): + patcher = _load( + "chat_api_patch_drift", + HERMES / "scripts" / "patch_api_server_sessions.py", + ) + source = tmp_path / "api_server.py" + marker = getattr(patcher, marker_name) + source.write_text(_patch_source(patcher).replace(marker, "", 1), encoding="utf-8") + with pytest.raises(RuntimeError, match="patch context changed"): + patcher.patch(source, tmp_path / "patched.py") + + +def test_script_entrypoints(tmp_path: Path, monkeypatch): + continuity_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py" + monkeypatch.setattr( + sys, + "argv", + [ + str(continuity_path), + str(tmp_path / "state.db"), + str(tmp_path / "responses.db"), + ], + ) + with pytest.raises(SystemExit, match="0"): + runpy.run_path(str(continuity_path), run_name="__main__") + + patch_path = HERMES / "scripts" / "patch_api_server_sessions.py" + patcher = _load("chat_api_patch_cli", patch_path) + source = tmp_path / "api_server.py" + destination = tmp_path / "patched.py" + source.write_text(_patch_source(patcher), encoding="utf-8") + monkeypatch.setattr(sys, "argv", [str(patch_path), str(source), str(destination)]) + with pytest.raises(SystemExit, match="0"): + runpy.run_path(str(patch_path), run_name="__main__") + assert destination.exists() + + +def test_chat_mounts_continuity_only_into_isolated_tenants(): + statefulset = yaml.safe_load((HERMES / "chat-statefulset.yaml").read_text()) + pod = statefulset["spec"]["template"]["spec"] + hermes = next(item for item in pod["containers"] if item["name"] == "hermes") + assert { + "name": "coordinator", + "mountPath": "/opt/hermes/gateway/platforms/telegram_continuity.py", + "subPath": "migrate_telegram_api_sessions.py", + "readOnly": True, + } in hermes["volumeMounts"] + patch_init = next( + item + for item in pod["initContainers"] + if item["name"] == "patch-api-server-sessions" + ) + command = patch_init["args"][0] + assert command.count("grep -Fq") == 3 + assert "compact_telegram_history" in command + assert "agent-deployment" not in (HERMES / "chat-statefulset.yaml").read_text() diff --git a/testing/tests/test_hermes_handoff_acceptance.py b/testing/tests/test_hermes_handoff_acceptance.py new file mode 100644 index 00000000..de4a411c --- /dev/null +++ b/testing/tests/test_hermes_handoff_acceptance.py @@ -0,0 +1,460 @@ +"""End-to-end contracts for the Hermes full-handoff acceptance harness CLI.""" + +from __future__ import annotations + +import json +import subprocess + +import pytest + +from testing.tests.test_hermes_handoff_support import FakeClock, load_handoff_module + +acceptance = load_handoff_module("hermes_handoff_acceptance") +catalog = load_handoff_module("hermes_handoff_catalog") +exec_module = load_handoff_module("hermes_handoff_exec") +model = load_handoff_module("hermes_handoff_model") +policy = load_handoff_module("hermes_handoff_policy") +harness_run = load_handoff_module("hermes_handoff_run") + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) + + +class ScriptedCluster: + """A ``subprocess.run`` stand-in that answers by command shape. + + Real runs issue a couple of hundred commands; scripting them positionally + would make every test a brittle ordering puzzle. Matching on the command + line keeps each test about the behaviour it is checking. + """ + + def __init__(self, rules: list[tuple[str, object]], default=None) -> None: + self.rules = rules + self.default = default or completed(stderr="not found", returncode=1) + self.calls: list[list[str]] = [] + + def __call__(self, argv, **_kwargs): + self.calls.append(list(argv)) + line = " ".join(argv) + for fragment, response in self.rules: + if fragment in line: + return response + return self.default + + +def whoami(username: str): + return completed(stdout=username) + + +REVIEWED = "8f005458282269ba5c07941814e4237f5d4cf3ac" +SHA = "1" * 40 +VALID_ARGS = [ + "--remote-main-sha", + SHA, + "--reviewed-head-sha", + REVIEWED, + "--agent-image", + f"registry.example/hermes:git-{'1' * 40}-build-7@sha256:{'2' * 64}", + "--build-sha", + SHA, + "--deployment-revision", + "1", + "--chat-config-revision", + "chat-rev", + *[ + item + for number in catalog.Targets().dependency_pull_requests + for item in ("--dependency-head", f"{number}={SHA}") + ], +] + + +HEALTHY_CLUSTER = [ + ( + "exec --namespace hermes hermes-agent-1", + whoami("system:serviceaccount:hermes:hermes-agent"), + ), + ("auth whoami", whoami("kubernetes-admin")), + ("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")), + ("get pods --selector", completed(stdout="pod/other-1\n")), + ("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")), +] + + +def run_cli(monkeypatch, argv, rules): + cluster = ScriptedCluster(rules) + clock = FakeClock() + + def build_runner(**kwargs): + return exec_module.Runner( + clock=clock, + spawn=cluster, + environ={}, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + **kwargs, + ) + + monkeypatch.setattr(acceptance, "Runner", build_runner) + return (acceptance.main([*VALID_ARGS, *argv]), cluster) + + +def read_report(path): + return json.loads(path.read_text(encoding="utf-8")) + + +def test_the_parser_exposes_the_documented_controls() -> None: + arguments = acceptance.build_parser().parse_args( + [ + "--namespace", + "hermes", + "--context", + "atlas-operator", + "--dependency-pr", + "14", + "--dependency-pr", + "15", + "--expected-suspension", + "flux-system/parked", + "--node-count", + "21", + "--pool-worker-env", + "HERMES_POOL_LEASE_SECONDS", + ] + ) + + targets = acceptance.targets_from_args(arguments, None) + + assert targets.dependency_pull_requests == (14, 15) + assert targets.expected_suspensions == ("flux-system/parked",) + assert targets.node_count == 21 + assert targets.pool_worker_env == ("HERMES_POOL_LEASE_SECONDS",) + assert not hasattr(arguments, "concurrency") + assert not hasattr(arguments, "expect_telegram_sessions") + + +def test_defaults_fall_back_to_the_shipped_targets() -> None: + arguments = acceptance.build_parser().parse_args([]) + targets = acceptance.targets_from_args(arguments, None) + + assert targets.dependency_pull_requests == catalog.Targets.dependency_pull_requests + assert targets.expected_suspensions == () + assert targets.repo == catalog.Targets.repo + + +@pytest.mark.parametrize("value", ["bad", "x=" + "1" * 40, "0=" + "1" * 40, "1=BAD"]) +def test_dependency_heads_require_a_positive_pr_and_lowercase_sha(value: str) -> None: + with pytest.raises(Exception): + acceptance.dependency_head(value) + + +def test_a_default_run_is_read_only_and_never_pushes_or_posts( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + status, cluster = run_cli( + monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert report["mode"] == policy.READ_ONLY + assert not any("push" in call for call in cluster.calls) + assert not any("POST" in call for call in cluster.calls) + assert not any("DELETE" in call for call in cluster.calls) + writes = [ + call + for call in cluster.calls + if {"patch", "create", "delete", "apply", "replace"} & set(call) + and "can-i" not in call + ] + assert writes == [] + + +def test_mutating_checks_report_not_run_until_the_mode_is_armed( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER) + + ephemeral = { + check["id"]: check + for check in read_report(output)["checks"] + if check["group"] == "ephemeral" + } + + assert len(ephemeral) == 4 + for check in ephemeral.values(): + assert check["status"] == model.NOT_RUN + assert check["mandatory"] is False + assert check["scope"] == model.EPHEMERAL + + +def test_a_zero_state_cluster_is_no_go_with_every_check_accounted_for( + monkeypatch, tmp_path +) -> None: + """Nothing observed must never read as nothing wrong.""" + output = tmp_path / "report.json" + status, _ = run_cli( + monkeypatch, + ["--output", str(output), "--json-only"], + [("auth whoami", whoami("kubernetes-admin"))], + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert report["decision"] == model.NO_GO + assert report["counts"][model.PASS] == 0 + assert sum(report["counts"].values()) == len(report["checks"]) + assert any( + "in-pod Hermes vantage is unavailable" in error + for error in report["harness_errors"] + ) + + +def test_partial_failure_is_reported_per_check_rather_than_aborting( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + rules = [ + *HEALTHY_CLUSTER, + ("auth can-i", completed(stdout="no\n", returncode=1)), + ("get namespaces", completed(stdout="namespace/hermes\n")), + ("merge-base", completed()), + ] + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) + report = read_report(output) + + statuses = {check["status"] for check in report["checks"]} + + assert status == acceptance.EXIT_NO_GO + assert model.PASS in statuses and model.NOT_RUN in statuses + assert report["blocking"] + + +def test_a_vantage_collision_is_a_no_go_even_with_clean_checks( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + rules = [ + ( + "exec --namespace hermes hermes-agent-1", + whoami("system:serviceaccount:hermes:hermes-agent"), + ), + ("auth whoami", whoami("system:serviceaccount:hermes:hermes-agent")), + ( + "get pods --selector app=hermes-agent", + completed(stdout="pod/hermes-agent-1\n"), + ), + ("get pods --selector", completed(stdout="pod/other-1\n")), + ( + "get pod/hermes-chat-tenant-0", + completed(stdout="pod/hermes-chat-tenant-0\n"), + ), + ] + status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert any("same principal" in error for error in report["harness_errors"]) + + +def test_an_invalid_deadline_stops_before_any_subprocess(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + status, _ = run_cli( + monkeypatch, + ["--output", str(output), "--json-only", "--deadline", "0"], + HEALTHY_CLUSTER, + ) + report = read_report(output) + + assert status == acceptance.EXIT_USAGE + assert report["counts"][model.NOT_RUN] > 0 + assert any("invalid execution bound" in error for error in report["harness_errors"]) + + +def test_arming_with_a_wrong_confirmation_stops_before_anything_runs( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + status, cluster = run_cli( + monkeypatch, + [ + "--output", + str(output), + "--json-only", + "--arm-ephemeral-push", + "--confirm", + "please", + "--ephemeral-token", + "acceptance-20260817a", + ], + HEALTHY_CLUSTER, + ) + report = read_report(output) + + assert status == acceptance.EXIT_NO_GO + assert "arming refused" in report["harness_errors"][0] + assert not any("push" in call for call in cluster.calls) + + +def test_arming_against_a_protected_push_target_is_refused( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + status, cluster = run_cli( + monkeypatch, + [ + "--output", + str(output), + "--json-only", + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "main", + ], + HEALTHY_CLUSTER, + ) + + assert status == acceptance.EXIT_NO_GO + assert "arming refused" in read_report(output)["harness_errors"][0] + assert not any("push" in call for call in cluster.calls) + + +def test_an_armed_run_does_not_mutate_until_read_only_acceptance_is_go( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + monkeypatch.setattr(acceptance, "preflight", lambda *_args: None) + rules = [ + ("git ls-remote origin refs/heads/ephemeral", completed(stdout="")), + ("git push", completed()), + ( + "POST /api/v1/repos/atlas/titan-iac/pulls", + completed(stdout=json.dumps({"number": 9})), + ), + ("PATCH", completed()), + ("DELETE", completed()), + ( + "GET /api/v1/repos/atlas/titan-iac/pulls/9", + completed(stdout='{"state": "closed"}'), + ), + *HEALTHY_CLUSTER, + ] + status, cluster = run_cli( + monkeypatch, + [ + "--output", + str(output), + "--json-only", + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817a", + ], + rules, + ) + report = read_report(output) + assert report["mode"] == policy.ARMED + assert status == acceptance.EXIT_NO_GO + assert any( + "read-only acceptance did not GO" in error for error in report["harness_errors"] + ) + pushes = [call for call in cluster.calls if "push" in call] + assert pushes == [] + assert not any("DELETE" in call for call in cluster.calls) + + +def test_the_report_is_written_to_stdout_when_no_output_path_is_given( + monkeypatch, capsys +) -> None: + status, _ = run_cli(monkeypatch, ["--json-only"], HEALTHY_CLUSTER) + payload = json.loads(capsys.readouterr().out) + + assert status == acceptance.EXIT_NO_GO + assert payload["harness"] == model.HARNESS + + +def test_the_human_summary_accompanies_the_json_unless_suppressed( + monkeypatch, capsys, tmp_path +) -> None: + output = tmp_path / "report.json" + run_cli(monkeypatch, ["--output", str(output)], HEALTHY_CLUSTER) + captured = capsys.readouterr() + + assert model.HARNESS in captured.err + assert "vantage operator" in captured.err + assert captured.out == "" + + +def test_a_report_that_fails_its_own_screening_is_replaced_not_published( + tmp_path, +) -> None: + """The last barrier: never write an artifact nobody re-read.""" + output = tmp_path / "report.json" + arguments = acceptance.build_parser().parse_args( + ["--output", str(output), "--json-only"] + ) + report = model.Report(mode="read-only", started_at="2026-08-17T00:00:00Z") + report.harness_errors = ["leaked ghp_ABCDEFGHIJKLMNOPQRSTUV1234"] + + monkey = report.as_dict + report.as_dict = lambda *_args, **_kwargs: { # type: ignore[method-assign] + **monkey(), + "leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234", + } + offenders = acceptance.emit(report, arguments) + payload = read_report(output) + + assert offenders == ["$.leak"] + assert payload["decision"] == model.NO_GO + assert "leak" not in payload + assert "failed its own credential screening" in payload["harness_errors"][0] + + +def test_a_screening_failure_forces_no_go_from_the_entry_point( + monkeypatch, tmp_path +) -> None: + """The exit status must follow the refusal to publish, not the check tally.""" + output = tmp_path / "report.json" + monkeypatch.setattr( + acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"] + ) + monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) + monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) + + status, _ = run_cli( + monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER + ) + + assert status == acceptance.EXIT_NO_GO + assert read_report(output)["decision"] == model.NO_GO + + +def test_no_recorded_command_output_carries_a_credential_shape( + monkeypatch, tmp_path +) -> None: + output = tmp_path / "report.json" + leaky = completed(stdout="ANTHROPIC_API_KEY=sk-ant-abcdefghijklmnopqrstuvwxyz\n") + run_cli( + monkeypatch, + ["--output", str(output), "--json-only"], + [*HEALTHY_CLUSTER, ("exec", leaky)], + ) + + assert model.unscreened_fields(read_report(output)) == [] + assert "sk-ant-abcdefghijklmnopqrstuvwxyz" not in output.read_text(encoding="utf-8") + + +@pytest.mark.parametrize("flag", ["--help"]) +def test_the_help_text_describes_the_read_only_default(flag: str, capsys) -> None: + with pytest.raises(SystemExit): + acceptance.main([flag]) + + assert "Read-only by default" in capsys.readouterr().out diff --git a/testing/tests/test_hermes_handoff_acceptance_armed.py b/testing/tests/test_hermes_handoff_acceptance_armed.py new file mode 100644 index 00000000..ded33bb8 --- /dev/null +++ b/testing/tests/test_hermes_handoff_acceptance_armed.py @@ -0,0 +1,120 @@ +"""Static and armed-transition regressions for the handoff CLI.""" + +from __future__ import annotations + +from testing.tests.test_hermes_handoff_acceptance import ( + HEALTHY_CLUSTER, + REVIEWED, + VALID_ARGS, + acceptance, + harness_run, + model, + policy, + read_report, + run_cli, +) + + +def test_an_empty_catalog_fails_before_any_subprocess(monkeypatch, tmp_path) -> None: + output = tmp_path / "report.json" + monkeypatch.setattr(harness_run, "build_catalog", lambda _targets: []) + monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: []) + monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: []) + + status, _ = run_cli( + monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER + ) + + assert status == acceptance.EXIT_NO_GO + assert read_report(output)["decision"] == model.NO_GO + + +def test_armed_success_path_replaces_placeholders_only_after_a_go(monkeypatch) -> None: + base = model.CheckSpec( + id="release.pass", title="pass", group="release", rule="allowed" + ) + placeholder = model.CheckSpec( + id="ephemeral.cleanup-verified", + title="pending", + group="ephemeral", + rule="not_armed", + mandatory=False, + scope=model.EPHEMERAL, + ) + report = model.Report( + mode=policy.ARMED, + started_at="2026-08-17T00:00:00Z", + results=[ + model.CheckResult(base, model.PASS), + model.CheckResult(placeholder, model.NOT_RUN), + ], + ) + armed_spec = model.CheckSpec( + id="ephemeral.cleanup-verified", + title="clean", + group="ephemeral", + rule="armed", + scope=model.EPHEMERAL, + ) + armed = model.CheckResult(armed_spec, model.PASS) + monkeypatch.setattr( + acceptance, "resolve_vantages", lambda *_args: ({"operator": object()}, []) + ) + monkeypatch.setattr(acceptance, "preflight", lambda *_args: None) + monkeypatch.setattr(acceptance, "build_report", lambda *_args: report) + monkeypatch.setattr(acceptance, "_armed_results", lambda *_args: [armed]) + monkeypatch.setattr(acceptance, "emit", lambda *_args: []) + status = acceptance.main( + [ + *VALID_ARGS, + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817a", + ] + ) + assert status == acceptance.EXIT_GO + assert ( + next( + result + for result in report.results + if result.spec.id.startswith("ephemeral") + ).status + == model.PASS + ) + monkeypatch.setattr(acceptance, "emit", lambda *_args: ["screening-failure"]) + assert ( + acceptance.main( + [ + *VALID_ARGS, + "--arm-ephemeral-push", + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817b", + ] + ) + == acceptance.EXIT_NO_GO + ) + + +def test_armed_results_are_bound_to_operator_and_exact_reviewed_head( + monkeypatch, +) -> None: + arguments = acceptance.build_parser().parse_args( + [ + *VALID_ARGS, + "--confirm", + acceptance.CONFIRMATION, + "--ephemeral-token", + "acceptance-20260817a", + ] + ) + calls = [] + monkeypatch.setattr(acceptance, "preflight", lambda *args: calls.append(args)) + monkeypatch.setattr(acceptance, "run_armed", lambda *args: [args]) + operator = object() + result = acceptance._armed_results(arguments, object(), {"operator": operator}) + assert calls[0][0].expected_head == REVIEWED + assert result[0][1] is operator diff --git a/testing/tests/test_hermes_handoff_catalog.py b/testing/tests/test_hermes_handoff_catalog.py new file mode 100644 index 00000000..21e8031a --- /dev/null +++ b/testing/tests/test_hermes_handoff_catalog.py @@ -0,0 +1,195 @@ +"""Structural contracts for the handoff acceptance catalog.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module + +catalog = load_handoff_module("hermes_handoff_catalog") +harness_run = load_handoff_module("hermes_handoff_run") +evaluators = load_handoff_module("hermes_handoff_evaluators") +model = load_handoff_module("hermes_handoff_model") +policy = load_handoff_module("hermes_handoff_policy") +exec_module = load_handoff_module("hermes_handoff_exec") + +NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc) +TARGETS = catalog.Targets(now=NOW) +SPECS = harness_run.build_catalog(TARGETS) +BY_ID = {check.id: check for check in SPECS} + +VANTAGES = { + catalog.OPERATOR: exec_module.operator_vantage(context="atlas-operator"), + catalog.SELF: exec_module.pod_vantage("hermes", "hermes-agent-1", "hermes"), + catalog.SWITCHYARD: exec_module.pod_vantage( + "hermes", "hermes-switchyard-1", "switchyard" + ), + catalog.NODE: exec_module.pod_vantage( + "hermes", "hermes-node-ssh-access-1", "key-reconciler" + ), + catalog.CHAT: exec_module.pod_vantage("hermes", "hermes-chat-tenant-0", "hermes"), +} + + +def test_the_catalog_is_structurally_valid() -> None: + assert harness_run.validate_catalog(SPECS) == [] + assert len(SPECS) >= 60 + + +def test_every_step_is_accepted_by_the_read_only_policy_from_its_vantage() -> None: + """A catalog entry that cannot run under policy is a bug found before a run.""" + for check in SPECS: + for step in check.steps: + addressed = VANTAGES[step.vantage].wrap(step.argv) + policy.check_argv(addressed, policy.READ_ONLY) + + +def test_no_step_constructs_a_mutation_or_dry_run() -> None: + """Authorization review may name a verb; no executed subcommand mutates.""" + for check in SPECS: + for step in check.steps: + subcommand = next(policy.positionals(step.argv), "") + assert subcommand not in {"apply", "create", "delete", "patch", "replace"} + assert not any(argument.startswith("--dry-run") for argument in step.argv) + + +def test_no_catalog_step_constructs_impersonation() -> None: + operator_steps = [ + step + for check in SPECS + for step in check.steps + if step.vantage == catalog.OPERATOR + ] + assert operator_steps + assert not any( + set(step.argv) & set(policy.IMPERSONATION_ARGS) for step in operator_steps + ) + + assert not any( + set(step.argv) & set(policy.IMPERSONATION_ARGS) + for check in SPECS + for step in check.steps + ) + + +def test_credential_or_mutation_denials_are_review_only() -> None: + deny_checks = [check for check in SPECS if check.rule == "review_denied"] + assert len(deny_checks) >= 5 + for check in deny_checks: + assert check.steps and all(step.kind == model.REVIEW for step in check.steps), ( + check.id + ) + + +def test_both_vantages_are_represented_across_the_catalog() -> None: + used = {step.vantage for check in SPECS for step in check.steps} + assert catalog.OPERATOR in used and catalog.SELF in used + assert used <= set(catalog.VANTAGE_NAMES) + + +def test_every_rule_named_by_a_check_is_registered() -> None: + for check in SPECS: + if check.scope == model.EPHEMERAL: + assert check.rule == "not_armed" + continue + assert check.rule in evaluators.EVALUATORS, check.id + + +def test_every_evidence_step_referenced_by_expect_exists() -> None: + for check in SPECS: + keys = {step.key for step in check.steps} + named = [check.expect.get("step")] + list(check.expect.get("steps", ())) + for key in filter(None, named): + assert key in keys, f"{check.id} names missing step {key}" + + +def test_the_default_catalog_reports_every_mutating_check_as_not_armed() -> None: + ephemeral = [check for check in SPECS if check.scope == model.EPHEMERAL] + assert len(ephemeral) == 4 + for check in ephemeral: + assert check.mandatory is False + assert evaluators.evaluate(check, {}).status == model.NOT_RUN + + +def test_dependency_pull_requests_each_get_their_own_merge_gate() -> None: + for number in TARGETS.dependency_pull_requests: + check = BY_ID[f"baseline.dependency-pr-{number}-merged"] + assert check.mandatory + assert check.expect["fields"]["merged"] is True + assert check.expect["fields"]["base.ref"] == "main" + assert "head.sha" in check.expect["fields"] + + +def test_the_baseline_commit_gate_names_the_configured_commit() -> None: + check = BY_ID["baseline.origin-main-descends-merged-work"] + assert TARGETS.baseline_commit in check.steps[0].argv + assert "--is-ancestor" in check.steps[0].argv + + +def test_pool_assignment_and_chat_continuity_are_unconditional() -> None: + assert "pool.assignment-safety-knobs-are-configured" in BY_ID + assert "pool.assignment-fieldrefs-are-bound" in BY_ID + assert "surfaces.chat-runs-the-telegram-topic-revision" in BY_ID + assert "surfaces.chat-telegram-sessions-are-continuous" in BY_ID + assert TARGETS.node_count == TARGETS.pool_replicas == 3 + + +def test_bulk_evidence_is_parsed_but_not_pasted_into_the_report() -> None: + routing = BY_ID["routing.provider-effort-and-fallback-evidence"] + assert routing.steps[0].record is False + assert routing.steps[0].max_bytes == TARGETS.routing_tail_bytes + + kanban = BY_ID["surfaces.kanban-activity-is-visible"] + assert kanban.steps[0].record is False + + +def test_freshness_gates_carry_the_run_clock() -> None: + for identifier in ( + "identity.codex-evidence-is-fresh", + "identity.claude-evidence-is-fresh", + ): + assert BY_ID[identifier].expect["now"] == NOW + routing = BY_ID["routing.provider-effort-and-fallback-evidence"] + assert routing.expect["now"] == NOW + assert routing.expect["max_age_seconds"] == TARGETS.max_evidence_age_seconds + + +def test_check_identifiers_are_unique_and_grouped() -> None: + identifiers = [check.id for check in SPECS] + assert len(identifiers) == len(set(identifiers)) + for check in SPECS: + assert check.id.split(".")[0] + assert check.group + + +@pytest.mark.parametrize( + "identifier", + [ + "identity.no-provider-api-key-in-pod", + "identity.no-provider-api-key-in-manifest", + "forge.worker-holds-no-git-credential-variable", + "access.secrets-are-denied", + "access.service-account-tokens-are-denied", + "access.workload-mutation-is-denied", + "access.pod-exec-is-denied", + "access.required-reads-are-allowed", + "access.flux-and-helm-status-are-allowed", + "access.namespace-view-agrees-across-vantages", + "nodes.hardening-covers-every-node", + "build.builder-service-account-is-tokenless", + "build.harbor-immutability-rule-is-applied", + "release.exact-lineage-is-running", + "reliability.finalization-and-replay-patches-are-deployed", + "pool.three-workers-have-distinct-nodes-and-volumes", + "pool.coordinator-retains-sole-state-ownership", + "surfaces.kanban-activity-is-visible", + "surfaces.chat-telegram-topic-state-is-durable", + "gitops.kustomizations-reconcile-cleanly", + "routing.provider-effort-and-fallback-evidence", + "scopes.agent-profile-is-distinct", + ], +) +def test_the_handoff_requirements_each_have_a_mandatory_check(identifier: str) -> None: + assert BY_ID[identifier].mandatory is True diff --git a/testing/tests/test_hermes_handoff_ephemeral.py b/testing/tests/test_hermes_handoff_ephemeral.py new file mode 100644 index 00000000..c1f993f8 --- /dev/null +++ b/testing/tests/test_hermes_handoff_ephemeral.py @@ -0,0 +1,438 @@ +"""Contracts for the strictly bounded, self-cleaning armed mode.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + FakeClock, + FakeSpawn, + load_handoff_module, +) + +ephemeral = load_handoff_module("hermes_handoff_ephemeral") +arming = load_handoff_module("hermes_handoff_arming") +exec_module = load_handoff_module("hermes_handoff_exec") +policy = load_handoff_module("hermes_handoff_policy") +model = load_handoff_module("hermes_handoff_model") + +TOKEN = "acceptance-20260817a" +HEAD = "8f005458282269ba5c07941814e4237f5d4cf3ac" +MAIN = "1" * 40 +VANTAGE = exec_module.operator_vantage() + + +def request(**overrides): + fields = { + "repo": policy.EXPECTED_REPO, + "remote": policy.EXPECTED_REMOTE, + "token": TOKEN, + "confirmation": ephemeral.CONFIRMATION, + "base": policy.EXPECTED_BASE, + "expected_head": HEAD, + "expected_base_sha": MAIN, + } + fields.update(overrides) + return ephemeral.ArmRequest(**fields) + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) + + +def armed_runner(results: list, mode=policy.ARMED): + spawn = FakeSpawn(results) + runner = exec_module.Runner( + mode=mode, + clock=FakeClock(), + spawn=spawn, + environ={}, + deadline_seconds=3600, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + ) + return runner, spawn + + +def by_id(results: list) -> dict: + return {result.spec.id: result for result in results} + + +def pr_payload(state="open", number=42): + return { + "number": number, + "state": state, + "draft": True, + "merged": False, + "base": {"ref": "main", "sha": MAIN}, + "head": {"ref": request().ref, "sha": HEAD}, + } + + +def linked_worktree(tmp_path: Path) -> Path: + root = tmp_path / "worktree" + common = tmp_path / "repo.git" + gitdir = common / "worktrees" / "acceptance" + gitdir.mkdir(parents=True) + (root).mkdir() + (root / ".git").write_text(f"gitdir: {gitdir}\n", encoding="utf-8") + (gitdir / "commondir").write_text("../..\n", encoding="utf-8") + (gitdir / "HEAD").write_text( + f"ref: refs/heads/{arming.EXPECTED_HEAD_REF}\n", encoding="utf-8" + ) + ref = common / "refs" / "heads" / arming.EXPECTED_HEAD_REF + ref.parent.mkdir(parents=True) + ref.write_text(f"{HEAD}\n", encoding="utf-8") + remote_ref = common / "refs" / "remotes" / "origin" / "main" + remote_ref.parent.mkdir(parents=True) + remote_ref.write_text(f"{MAIN}\n", encoding="utf-8") + (common / "config").write_text( + f'[remote "origin"]\n\turl = {arming.EXPECTED_REMOTE_URL}\n', + encoding="utf-8", + ) + return root + + +@pytest.mark.parametrize( + "ref", ["main", "master", "refs/heads/main", "HEAD", "feature/x", ""] +) +def test_only_the_unique_ephemeral_ref_is_allowed(ref: str) -> None: + with pytest.raises(ephemeral.ArmingError): + ephemeral.assert_push_target_allowed(ref) + ephemeral.assert_push_target_allowed(request().ref) + + +@pytest.mark.parametrize( + "overrides", + [ + {"confirmation": "wrong"}, + {"repo": "atlas/other"}, + {"remote": "upstream"}, + {"base": "develop"}, + {"token": "short"}, + {"expected_head": "bad"}, + {"expected_base_sha": "bad"}, + ], +) +def test_preflight_rejects_bad_inputs_before_any_runner_exists( + tmp_path: Path, overrides: dict +) -> None: + with pytest.raises(ephemeral.ArmingError): + ephemeral.preflight( + request(**overrides), policy.EXPECTED_REPO, linked_worktree(tmp_path) + ) + + +def test_preflight_attests_the_actual_linked_worktree(tmp_path: Path) -> None: + ephemeral.preflight(request(), policy.EXPECTED_REPO, linked_worktree(tmp_path)) + root = linked_worktree(tmp_path / "wrong") + (Path((root / ".git").read_text().split(": ", 1)[1].strip()) / "HEAD").write_text( + "ref: refs/heads/other\n", encoding="utf-8" + ) + with pytest.raises(ephemeral.ArmingError, match="existing PR #19"): + ephemeral.preflight(request(), policy.EXPECTED_REPO, root) + + +def test_worktree_attestation_rejects_each_malformed_local_source( + tmp_path: Path, +) -> None: + roots = [] + empty = tmp_path / "empty" + empty.mkdir() + roots.append(empty) + bad_marker = linked_worktree(tmp_path / "marker") + (bad_marker / ".git").write_text("not-a-gitdir\n", encoding="utf-8") + roots.append(bad_marker) + wrong_sha = linked_worktree(tmp_path / "sha") + gitdir = Path((wrong_sha / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "refs" / "heads" / arming.EXPECTED_HEAD_REF).write_text("0" * 40) + roots.append(wrong_sha) + wrong_main = linked_worktree(tmp_path / "main") + gitdir = Path((wrong_main / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "refs" / "remotes" / "origin" / "main").write_text("0" * 40) + roots.append(wrong_main) + bad_config = linked_worktree(tmp_path / "config") + gitdir = Path((bad_config / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "config").write_text("[broken\n", encoding="utf-8") + roots.append(bad_config) + wrong_remote = linked_worktree(tmp_path / "remote") + gitdir = Path((wrong_remote / ".git").read_text().split(": ", 1)[1].strip()) + common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve() + (common / "config").write_text('[remote "origin"]\nurl=https://example.invalid/x\n') + roots.append(wrong_remote) + for root in roots: + with pytest.raises((ephemeral.ArmingError, FileNotFoundError)): + ephemeral.preflight(request(), policy.EXPECTED_REPO, root) + oversized = tmp_path / "oversized" + oversized.write_text("x" * 5, encoding="utf-8") + with pytest.raises(ephemeral.ArmingError): + ephemeral.bounded_read(oversized, 4) + + +def test_guard_failure_stops_before_runner_calls(monkeypatch) -> None: + monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None) + runner, spawn = armed_runner([]) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.protected-branch-refusal"].status == model.FAIL + assert all( + result.status in {model.FAIL, model.NOT_RUN} for result in results.values() + ) + assert spawn.calls == [] + + +def test_full_armed_run_validates_and_removes_exact_artifacts() -> None: + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, spawn = armed_runner( + [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + completed(stdout=json.dumps(pr_payload())), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ] + ) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert all(result.status == model.PASS for result in results.values()) + push = next(call for call in spawn.calls if call["argv"][1] == "push") + assert push["argv"][-1] == f"HEAD:refs/heads/{request().ref}" + assert all(result.spec.mandatory for result in results.values()) + + +def test_existing_or_unverifiable_ref_never_reaches_a_write() -> None: + for first in (completed(stdout="abc\trefs/heads/x\n"), completed(returncode=128)): + runner, spawn = armed_runner([first]) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert not any( + call["argv"][1] in {"push", "POST", "PATCH", "DELETE"} + for call in spawn.calls + ) + + +def test_failed_or_ambiguous_create_is_discovered_and_cleanup_still_runs() -> None: + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, _ = armed_runner( + [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + completed(returncode=1), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ] + ) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.draft-pull-request"].status == model.PASS + assert results["ephemeral.cleanup-verified"].status == model.PASS + + +def test_pull_request_parsers_reject_every_malformed_or_ambiguous_shape() -> None: + assert ephemeral._pr_problems([], request()) == ["pull request is not an object"] + malformed = pr_payload() + malformed.update({"number": True, "draft": "yes", "base": None}) + malformed["head"] = {"ref": "other", "sha": "bad"} + problems = ephemeral._pr_problems(malformed, request()) + assert {"number is malformed", "draft does not match", "base is malformed"} <= set( + problems + ) + assert ephemeral._created_number( + exec_module.Outcome((), "x", returncode=1), request() + ) == (0, "pull-request creation failed: exit status 1") + for body in ("not-json", "{}"): + number, error = ephemeral._created_number( + exec_module.Outcome((), "x", 0, body), request() + ) + assert number == 0 and error + assert ephemeral._created_number( + exec_module.Outcome((), "x", 0, json.dumps(pr_payload())), request() + ) == (42, "") + for rows in ([None], [{"head": {"ref": "other"}, "base": {}}]): + numbers, error = ephemeral._match_pulls(rows, request()) + assert numbers == [] and error + invalid = pr_payload() + invalid["draft"] = False + assert ephemeral._match_pulls([invalid], request())[1] + assert "discovered 0" in ephemeral._match_pulls([], request())[1] + + +@pytest.mark.parametrize( + "delete_result,remaining,closed", + [ + (completed(returncode=1), completed(), pr_payload(state="closed")), + (completed(), completed(returncode=128), pr_payload(state="closed")), + (completed(), completed(), {}), + (completed(), completed(), pr_payload(state="open")), + ], +) +def test_cleanup_fails_closed_on_partial_or_malformed_evidence( + delete_result, remaining, closed +) -> None: + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, _ = armed_runner( + [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + completed(), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + delete_result, + remaining, + completed(stdout=json.dumps(closed)), + ] + ) + cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[ + "ephemeral.cleanup-verified" + ] + assert cleanup.status == model.FAIL + + +def test_cleanup_fails_when_close_or_verification_commands_are_bad_json_or_failures() -> ( + None +): + api = "/api/v1/repos/atlas/titan-iac" + for scripted in ( + [completed(returncode=1), completed(), completed(), completed(returncode=1)], + [completed(stdout="bad"), completed(), completed(), completed(stdout="bad")], + [ + completed(stdout=json.dumps(pr_payload())), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ], + ): + runner, _ = armed_runner(scripted) + result = ephemeral._cleanup(runner, VANTAGE, request(), api, [42]) + assert result.status == model.FAIL + + +def test_read_only_runner_cannot_reach_push() -> None: + runner, spawn = armed_runner( + [completed(), completed(stdout=f"{HEAD}\n"), completed(), completed()], + mode=policy.READ_ONLY, + ) + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + assert results["ephemeral.feature-branch-push"].status == model.FAIL + assert not any(call["argv"][1] == "push" for call in spawn.calls) + + +def push_prelude(ref_line: str) -> list: + """The four scripted results that get an armed run to a verified push.""" + return [ + completed(), + completed(stdout=f"{HEAD}\n"), + completed(), + completed(stdout=ref_line), + ] + + +def test_pull_request_discovery_reads_every_page() -> None: + """A full first page must not end discovery: the match can be on page two.""" + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + filler = [ + { + "number": index, + "state": "open", + "draft": True, + "merged": False, + "base": {"ref": "main", "sha": MAIN}, + "head": {"ref": "feature/other", "sha": HEAD}, + } + for index in range(1, ephemeral.DISCOVERY_PAGE_SIZE + 1) + ] + runner, spawn = armed_runner( + [ + *push_prelude(ref_line), + completed(stdout=json.dumps(pr_payload())), + completed(stdout=json.dumps(filler)), + completed(stdout=json.dumps([pr_payload()])), + completed(stdout=json.dumps(pr_payload(state="closed"))), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + + assert results["ephemeral.draft-pull-request"].status == model.PASS + pages = [call["argv"][2] for call in spawn.calls if "pulls?state=all" in str(call)] + assert any("page=1" in page for page in pages) + assert any("page=2" in page for page in pages) + + +def test_uncertain_discovery_still_closes_the_created_pull_and_names_residue() -> None: + """The create response names the pull, so an unusable index cannot orphan it.""" + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, spawn = armed_runner( + [ + *push_prelude(ref_line), + completed(stdout=json.dumps(pr_payload())), + completed(returncode=1), + completed(stdout=json.dumps(pr_payload(state="closed"))), + completed(), + completed(), + completed(stdout=json.dumps(pr_payload(state="closed"))), + ] + ) + + results = by_id(ephemeral.run_armed(runner, VANTAGE, request())) + draft = results["ephemeral.draft-pull-request"] + + assert draft.status == model.FAIL + assert draft.evidence["pull_requests"] == [42] + assert draft.evidence["residue_ref"] == request().ref + assert any("pulls/42" in line for line in draft.evidence["manual_cleanup"]) + assert any("branches/" in line for line in draft.evidence["manual_cleanup"]) + # The pull the harness created is closed, not left behind. + assert any( + call["argv"][1] == "PATCH" and call["argv"][2].endswith("/pulls/42") + for call in spawn.calls + ) + assert results["ephemeral.cleanup-verified"].status == model.FAIL + + +def test_a_wholly_unidentifiable_create_names_the_ref_for_manual_cleanup() -> None: + """When neither source names a pull, the operator gets the exact ref to sweep.""" + ref_line = f"{HEAD}\trefs/heads/{request().ref}\n" + runner, _ = armed_runner( + [ + *push_prelude(ref_line), + completed(stdout="not-json"), + completed(returncode=1), + completed(), + completed(), + ] + ) + + draft = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[ + "ephemeral.draft-pull-request" + ] + + assert draft.status == model.FAIL + assert draft.evidence["pull_requests"] == [] + assert draft.evidence["residue_ref"] == request().ref + assert any( + request().ref in line and "close by hand" in line + for line in draft.evidence["manual_cleanup"] + ) diff --git a/testing/tests/test_hermes_handoff_evaluators.py b/testing/tests/test_hermes_handoff_evaluators.py new file mode 100644 index 00000000..4de16c7e --- /dev/null +++ b/testing/tests/test_hermes_handoff_evaluators.py @@ -0,0 +1,457 @@ +"""Contracts for the text-shaped handoff acceptance evaluators.""" + +from __future__ import annotations + +import json + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) + +evaluators = load_handoff_module("hermes_handoff_evaluators") +model = load_handoff_module("hermes_handoff_model") + +REVIEW = model.REVIEW +ATTEMPT = model.ATTEMPT +FORBIDDEN = outcome( + stderr="Error from server (Forbidden): pods is forbidden", returncode=1 +) + + +def deny_spec() -> object: + return spec( + "denied", + steps=(step("review", kind=REVIEW), step("attempt", kind=ATTEMPT)), + ) + + +def evaluate(built, outcomes): + return evaluators.evaluate(built, outcomes) + + +def test_a_deny_check_passes_only_on_a_review_no_and_a_live_refusal() -> None: + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN} + ) + + assert evaluation.status == model.PASS + assert evaluation.evidence == {"review": "no", "attempt": "refused"} + + +def test_a_deny_check_fails_when_the_review_says_the_authority_exists() -> None: + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN} + ) + + assert evaluation.status == model.FAIL + assert "reported yes" in evaluation.reason + + +def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> None: + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()} + ) + + assert evaluation.status == model.FAIL + assert "must be refused" in evaluation.reason + + +def test_a_request_that_merely_broke_is_not_evidence_of_a_refusal() -> None: + broken = outcome(stderr="connection refused", returncode=7) + evaluation = evaluate( + deny_spec(), {"review": outcome(stdout="no"), "attempt": broken} + ) + + assert evaluation.status == model.NOT_RUN + assert "recognisable refusal" in evaluation.reason + + +def test_a_deny_check_without_a_live_attempt_refuses_to_classify() -> None: + """An authorization review states policy; only an attempt proves enforcement.""" + review_only = spec("denied", steps=(step("review", kind=REVIEW),)) + + evaluation = evaluate(review_only, {"review": outcome(stdout="no")}) + + assert evaluation.status == model.NOT_RUN + assert "require a real attempt" in evaluation.reason + + +def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> ( + None +): + built = spec("allowed", steps=(step("review", kind=REVIEW), step("read"))) + + assert ( + evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status + == model.PASS + ) + + denied = evaluate(built, {"review": outcome(stdout="no"), "read": outcome()}) + assert denied.status == model.FAIL and "reported no" in denied.reason + + failed = evaluate( + built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)} + ) + assert failed.status == model.FAIL and "step read failed" in failed.reason + + +def test_names_absent_flags_exact_names_and_substrings() -> None: + built = spec( + "names_absent", + {"step": "env", "names": ("GITEA_TOKEN",), "contains": ("API_KEY",)}, + steps=(step("env"),), + ) + + clean = evaluate(built, {"env": outcome(stdout="PATH\nHOME\n")}) + assert clean.status == model.PASS and clean.evidence["observed_count"] == 2 + + leaked = evaluate( + built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")} + ) + assert leaked.status == model.FAIL + assert leaked.evidence["offenders"] == ["GITEA_TOKEN", "OPENAI_API_KEY"] + + unreadable = evaluate(built, {"env": outcome(returncode=1)}) + assert unreadable.status == model.NOT_RUN + + +def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> ( + None +): + built = spec( + "names_present", + { + "step": "init", + "names": ("patch-web-session-activity",), + "contains": ("@sha256:",), + }, + steps=(step("init"),), + ) + + good = outcome(stdout="patch-web-session-activity\nimage@sha256:abc\n") + assert evaluate(built, {"init": good}).status == model.PASS + + partial = evaluate(built, {"init": outcome(stdout="patch-web-session-activity\n")}) + assert partial.status == model.FAIL and "@sha256:" in partial.reason + + empty = spec("names_present", {"step": "init"}, steps=(step("init"),)) + assert evaluate(empty, {"init": outcome(stdout="")}).status == model.NOT_RUN + + +def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None: + single = spec( + "stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),) + ) + assert evaluate(single, {"s": outcome(stdout=" locked \n")}).status == model.PASS + + mismatch = evaluate(single, {"s": outcome(stdout="unlocked")}) + assert mismatch.status == model.FAIL and mismatch.evidence["observed"] == "unlocked" + + several = spec( + "stdout_matches", + {"step": "s", "equals": ["600 root:root", "absent"]}, + steps=(step("s"),), + ) + assert evaluate(several, {"s": outcome(stdout="absent")}).status == model.PASS + + assert evaluate(single, {"s": outcome(returncode=1)}).status == model.NOT_RUN + + +def test_distinct_count_separates_replica_count_from_actual_spread() -> None: + built = spec( + "distinct_count", + {"step": "n", "minimum": 3, "total_equals": 3}, + steps=(step("n"),), + ) + + spread = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-06\ntitan-07\n")}) + assert spread.status == model.PASS and spread.evidence["distinct"] == 3 + + stacked = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-05\ntitan-05\n")}) + assert stacked.status == model.FAIL and "distinct values" in stacked.reason + + extra = spec( + "distinct_count", + {"step": "n", "minimum": 1, "total_equals": 2}, + steps=(step("n"),), + ) + over = evaluate(extra, {"n": outcome(stdout="a\nb\nc\n")}) + assert over.status == model.FAIL and "expected exactly 2" in over.reason + + assert evaluate(built, {"n": outcome(stdout="")}).status == model.NOT_RUN + assert evaluate(built, {"n": outcome(returncode=1)}).status == model.NOT_RUN + + +def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None: + built = spec( + "vantages_agree", + {"steps": ("operator", "self")}, + steps=(step("operator", vantage="operator"), step("self")), + ) + + agree = evaluate( + built, + { + "operator": outcome(stdout="ns/a\nns/b\n"), + "self": outcome(stdout="ns/b\nns/a\n"), + }, + ) + assert agree.status == model.PASS + + differ = evaluate( + built, + {"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")}, + ) + assert differ.status == model.FAIL and "disagree" in differ.reason + + empty = evaluate( + built, {"operator": outcome(stdout=""), "self": outcome(stdout="")} + ) + assert empty.status == model.NOT_RUN + + broken = evaluate(built, {"operator": outcome(returncode=1), "self": outcome()}) + assert broken.status == model.NOT_RUN + + +def test_steps_agree_is_the_same_rule_under_a_neutral_name() -> None: + assert ( + evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"] + ) + + +def test_lines_match_catches_a_blank_leading_column() -> None: + built = spec( + "lines_match", + {"step": "s", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1}, + steps=(step("s"),), + ) + table = "Preview Src ID\n-------\nwork on task cli 2026\n" + + assert evaluate(built, {"s": outcome(stdout=table)}).status == model.PASS + + blank = "Preview Src ID\n-------\n cli 2026\n" + failed = evaluate(built, {"s": outcome(stdout=blank)}) + assert failed.status == model.FAIL and failed.evidence["offenders"] + + assert ( + evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status + == model.NOT_RUN + ) + assert evaluate(built, {"s": outcome(returncode=1)}).status == model.NOT_RUN + + +def test_not_armed_is_the_fixed_skip_a_default_run_must_produce() -> None: + built = spec("not_armed", {"reason": "ephemeral mutation mode is not armed"}) + + evaluation = evaluate(built, {}) + + assert evaluation.status == model.NOT_RUN + assert evaluation.evidence == {"armed": False} + assert evaluation.reason == "ephemeral mutation mode is not armed" + assert ( + evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed" + ) + + +def test_truncation_fails_closed_before_every_registered_evaluator() -> None: + """No evaluator may reinterpret a clipped required capture as PASS.""" + clipped = outcome(stdout="apparently good", truncated=True) + for name in evaluators.EVALUATORS: + built = spec(name, {"step": "s"}, steps=(step("s"),)) + evaluation = evaluate(built, {"s": clipped}) + assert evaluation.status == model.NOT_RUN, name + assert "truncated" in evaluation.reason, name + + +def test_missing_required_steps_fail_closed_inside_every_step_evaluator() -> None: + for name in evaluators.EVALUATORS: + if name == "not_armed": + continue + built = spec(name, {"step": "s"}, steps=(step("s"),)) + assert evaluate(built, {}).status == model.NOT_RUN, name + + +def test_pool_topology_requires_three_ordinals_nodes_claims_and_ready_state() -> None: + built = spec( + "pool_topology", + { + "state_step": "state", + "worker_step": "workers", + "replicas": 3, + "name": "worker", + }, + steps=(step("state"), step("workers")), + ) + state = outcome(stdout="3\t3\t3\tfalse") + workers = outcome( + stdout=( + "worker-0\tnode-a\tRunning\tTrue\tclaim-0,\n" + "worker-1\tnode-b\tRunning\tTrue\tclaim-1,\n" + "worker-2\tnode-c\tRunning\tTrue\tclaim-2,\n" + ) + ) + assert evaluate(built, {"state": state, "workers": workers}).status == model.PASS + stacked = outcome(stdout=workers.stdout.replace("node-b", "node-a")) + assert evaluate(built, {"state": state, "workers": stacked}).status == model.FAIL + assert ( + evaluate(built, {"state": outcome(stdout="3\t3"), "workers": workers}).status + == model.NOT_RUN + ) + assert ( + evaluate( + built, {"state": outcome(stdout="x\t3\t3\tfalse"), "workers": workers} + ).status + == model.NOT_RUN + ) + assert ( + evaluate( + built, {"state": outcome(stdout="3\t2\t3\tfalse"), "workers": workers} + ).status + == model.FAIL + ) + malformed = outcome(stdout="worker-0\tnode-a\n") + assert ( + evaluate(built, {"state": state, "workers": malformed}).status == model.NOT_RUN + ) + bad_worker = outcome(stdout=workers.stdout.replace("Running", "Pending", 1)) + assert evaluate(built, {"state": state, "workers": bad_worker}).status == model.FAIL + duplicate = outcome(stdout=workers.stdout.replace("worker-1", "worker-0")) + assert evaluate(built, {"state": state, "workers": duplicate}).status == model.FAIL + wrong_ordinals = outcome(stdout=workers.stdout.replace("worker-2", "worker-3")) + assert ( + evaluate(built, {"state": state, "workers": wrong_ordinals}).status + == model.FAIL + ) + + +MAIN = "1" * 40 +HEAD = "2" * 40 +BUILD = MAIN +DIGEST = "sha256:" + "4" * 64 +IMAGE = f"registry.example/hermes:git-{BUILD}-build-7@{DIGEST}" + + +def lineage_spec(): + keys = ( + "remote-main", + "reviewed-pr", + "build-source", + "deployment", + "pods", + "replicasets", + "flux", + ) + return spec( + "release_lineage", + { + "main_sha": MAIN, + "head_sha": HEAD, + "head_ref": "feature/hermes-full-handoff-acceptance", + "pr_number": 19, + "image": IMAGE, + "build_sha": BUILD, + "deployment_revision": "7", + }, + steps=tuple(step(key) for key in keys), + ) + + +def lineage_outcomes(): + pull = { + "number": 19, + "state": "open", + "draft": True, + "merged": False, + "base": {"ref": "main", "sha": MAIN}, + "head": {"ref": "feature/hermes-full-handoff-acceptance", "sha": HEAD}, + } + return { + "remote-main": outcome(stdout=f"{MAIN}\trefs/heads/main\n"), + "reviewed-pr": outcome(stdout=json.dumps(pull)), + "build-source": outcome(stdout=BUILD), + "deployment": outcome(stdout=f"5\t5\t2\t2\t7\t{IMAGE}"), + "replicasets": outcome( + stdout=f"rs-old\t6\t0\t0\t{IMAGE}\told\nrs-live\t7\t2\t2\t{IMAGE}\thash7\n" + ), + "pods": outcome( + stdout=( + f"pod-a\tRunning\tTrue\t{IMAGE}\tdocker-pullable://hermes@{DIGEST}\thash7\n" + f"pod-b\tRunning\tTrue\t{IMAGE}\tcontainerd://hermes@{DIGEST}\thash7\n" + ) + ), + "flux": outcome(stdout=f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{MAIN}"), + } + + +def test_release_lineage_atomically_binds_every_release_identity() -> None: + assert evaluate(lineage_spec(), lineage_outcomes()).status == model.PASS + + +@pytest.mark.parametrize( + ("key", "bad"), + [ + ("remote-main", ""), + ("reviewed-pr", "{}"), + ("build-source", HEAD), + ("deployment", "5\t4\t2\t2\t7\tbad"), + ("replicasets", "malformed"), + ("pods", "malformed"), + ("flux", "hermes\t9\t9\tfalse\tUnknown\tmain@sha1:" + MAIN), + ], +) +def test_release_lineage_fails_closed_on_each_malformed_or_mismatched_source( + key, bad +) -> None: + outcomes = lineage_outcomes() + outcomes[key] = outcome(stdout=bad) + assert evaluate(lineage_spec(), outcomes).status in {model.FAIL, model.NOT_RUN} + + +def test_release_lineage_rejects_structural_partial_failure_variants() -> None: + variants = [] + malformed_expect = lineage_spec() + malformed_expect = model.CheckSpec( + **{ + **malformed_expect.__dict__, + "expect": {**malformed_expect.expect, "main_sha": "bad"}, + } + ) + variants.append((malformed_expect, lineage_outcomes())) + malformed_image = lineage_spec() + malformed_image = model.CheckSpec( + **{ + **malformed_image.__dict__, + "expect": { + **malformed_image.expect, + "image": f"registry.example/hermes@{DIGEST}", + }, + } + ) + variants.append((malformed_image, lineage_outcomes())) + for key, value in ( + ("reviewed-pr", "[]"), + ("deployment", f"x\tx\t2\t2\t7\t{IMAGE}"), + ("replicasets", f"rs\t7\t1\t2\t{IMAGE}\thash7"), + ("replicasets", f"a\t7\t2\t2\t{IMAGE}\th1\nb\t7\t2\t2\t{IMAGE}\th2"), + ("replicasets", f"rs\t6\t2\t2\t{IMAGE}\thash7"), + ("pods", f"pod-a\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7"), + ( + "pods", + f"pod-a\tPending\tFalse\t{IMAGE}\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7", + ), + ( + "pods", + f"pod-a\tRunning\tTrue\tbad\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7", + ), + ("flux", f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{HEAD}"), + ): + outcomes = lineage_outcomes() + outcomes[key] = outcome(stdout=value) + variants.append((lineage_spec(), outcomes)) + for built, outcomes in variants: + assert evaluate(built, outcomes).status in {model.FAIL, model.NOT_RUN} diff --git a/testing/tests/test_hermes_handoff_exec.py b/testing/tests/test_hermes_handoff_exec.py new file mode 100644 index 00000000..fb54d3a5 --- /dev/null +++ b/testing/tests/test_hermes_handoff_exec.py @@ -0,0 +1,360 @@ +"""Contracts for bounded, screened command execution in the handoff harness.""" + +from __future__ import annotations + +import hashlib +import subprocess +import time + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + FakeClock, + FakeSpawn, + load_handoff_module, +) + +runner_module = load_handoff_module("hermes_handoff_exec") +policy = load_handoff_module("hermes_handoff_policy") + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) + + +def build(results: list, **kwargs) -> tuple: + clock = FakeClock() + spawn = FakeSpawn(results) + runner = runner_module.Runner( + clock=clock, + spawn=spawn, + environ={}, + attestor=lambda command, _environment: runner_module.Attestation( + command, "a" * 64 + ), + **kwargs, + ) + return (runner, spawn, clock) + + +OPERATOR = runner_module.operator_vantage() +SAFE_GET = ("kubectl", "get", "pods", "-o", "name") + + +def test_a_successful_command_is_captured_and_screened() -> None: + runner, spawn, _ = build([completed(stdout="password=a-long-enough-secret\n")]) + + outcome = runner.run(SAFE_GET, OPERATOR) + + assert outcome.ok and outcome.ran + assert "[redacted]" in outcome.stdout + assert outcome.returncode == 0 + assert spawn.calls[0]["argv"] == list(SAFE_GET) + assert runner.commands_run == 1 + + +def test_a_failing_command_is_not_ok_but_still_ran() -> None: + runner, _, _ = build( + [completed(stderr="Error from server (Forbidden)", returncode=1)] + ) + + outcome = runner.run(SAFE_GET, OPERATOR) + + assert outcome.ran and not outcome.ok + assert "Forbidden" in outcome.combined + + +def test_a_policy_violation_never_reaches_a_subprocess() -> None: + runner, spawn, _ = build([]) + + outcome = runner.run(("kubectl", "delete", "pods", "--all"), OPERATOR) + + assert outcome.error and outcome.error.startswith("policy:") + assert not outcome.ran + assert spawn.calls == [] + assert runner.commands_run == 0 + + +def test_a_timeout_is_recorded_rather_than_raised() -> None: + runner, _, _ = build([subprocess.TimeoutExpired(cmd="kubectl", timeout=1.0)]) + + outcome = runner.run(SAFE_GET, OPERATOR) + + assert outcome.error == runner_module.TIMEOUT_ERROR + assert not outcome.ran + + +def test_a_missing_binary_is_recorded_rather_than_raised() -> None: + runner, _, _ = build([FileNotFoundError(2, "No such file or directory")]) + + outcome = runner.run(SAFE_GET, OPERATOR) + + assert outcome.error and outcome.error.startswith("spawn:") + + +def test_the_run_deadline_stops_further_probes_without_spawning() -> None: + runner, spawn, clock = build([completed(stdout="first")], deadline_seconds=10.0) + + assert runner.run(SAFE_GET, OPERATOR).ok + clock.advance(11.0) + second = runner.run(("kubectl", "get", "nodes", "-o", "name"), OPERATOR) + + assert second.error == runner_module.DEADLINE_ERROR + assert len(spawn.calls) == 1 + assert runner.remaining_seconds < 0 + + +def test_the_command_timeout_is_clamped_by_the_remaining_deadline() -> None: + runner, spawn, clock = build( + [completed()], command_timeout=30.0, deadline_seconds=10.0 + ) + clock.advance(6.0) + + runner.run(SAFE_GET, OPERATOR) + + assert spawn.calls[0]["timeout"] == 4.0 + + +def test_output_is_bounded_per_command_and_can_be_raised_per_call() -> None: + runner, _, _ = build( + [completed(stdout="y" * 200), completed(stdout="y" * 200)], max_bytes=50 + ) + + small = runner.run(SAFE_GET, OPERATOR) + large = runner.run(SAFE_GET, OPERATOR, max_bytes=4096) + + assert small.truncated + assert not large.truncated + + +def test_the_child_environment_is_rebuilt_from_an_allowlist() -> None: + source = { + "PATH": "/bin", + "KUBECONFIG": "/tmp/kc", + "ANTHROPIC_API_KEY": "leak", + "HOME": "/root", + } + environment = runner_module.build_environment(OPERATOR, source) + + assert environment["PATH"] == runner_module.SAFE_PATH + assert environment["KUBECONFIG"] == "/tmp/kc" + assert environment["GITEA_BASE_URL"] == "https://scm.bstein.dev" + assert environment["GIT_CONFIG_GLOBAL"] == "/dev/null" + assert "HOME" not in environment + assert "ANTHROPIC_API_KEY" not in environment + + +def test_a_vantage_can_override_the_environment_it_needs() -> None: + vantage = runner_module.operator_vantage(kubeconfig="/tmp/operator.yaml") + environment = runner_module.build_environment(vantage, {"KUBECONFIG": "/tmp/other"}) + + assert environment["KUBECONFIG"] == "/tmp/operator.yaml" + assert "kubeconfig-pinned" in vantage.description + + +def test_the_operator_context_is_inserted_only_for_context_aware_tools() -> None: + vantage = runner_module.operator_vantage(context="atlas-operator") + + assert vantage.wrap(("kubectl", "get", "pods"))[:3] == ( + "kubectl", + "--context", + "atlas-operator", + ) + assert vantage.wrap(("git", "status")) == ("git", "status") + assert vantage.wrap(()) == () + + +def test_the_pod_vantage_execs_into_the_container_and_drops_the_context() -> None: + operator = runner_module.operator_vantage(context="atlas-operator") + vantage = runner_module.pod_vantage("hermes", "hermes-agent-1", "hermes", operator) + + wrapped = vantage.wrap(SAFE_GET) + + assert wrapped[:3] == ("kubectl", "--context", "atlas-operator") + assert wrapped[3] == "exec" + assert wrapped[-6:] == ( + "--", + "/usr/local/bin/kubectl", + "get", + "pods", + "-o", + "name", + ) + assert vantage.description == "in-pod hermes/hermes-agent-1[hermes]" + policy.check_argv(wrapped) + + +def test_a_pod_vantage_without_an_operator_still_addresses_the_pod() -> None: + vantage = runner_module.pod_vantage("hermes", "pod-a", "hermes") + + assert vantage.wrap(("kubectl", "version"))[0] == "kubectl" + assert vantage.env == {} + + +def test_outcome_serialisation_screens_the_command_line() -> None: + outcome = runner_module.Outcome( + argv=("git", "clone", "https://user:a-long-enough-secret@scm.example.dev/x"), + vantage="operator", + returncode=0, + stdout="ok", + ) + + payload = outcome.as_dict() + + assert "a-long-enough-secret" not in payload["command"] + assert payload["vantage"] == "operator" + assert payload["stdout"] == "ok" + + +def test_combined_output_joins_only_the_streams_that_carry_text() -> None: + assert ( + runner_module.Outcome(argv=(), vantage="x", stdout="a", stderr="b").combined + == "a\nb" + ) + assert runner_module.Outcome(argv=(), vantage="x", stdout="a").combined == "a" + assert runner_module.Outcome(argv=(), vantage="x").combined == "" + + +@pytest.mark.parametrize( + "kwargs", + [ + {"command_timeout": float("nan")}, + {"command_timeout": float("inf")}, + {"command_timeout": 0}, + {"deadline_seconds": -1}, + {"max_bytes": -1}, + {"max_bytes": 10**100}, + ], +) +def test_execution_bounds_reject_nonfinite_nonpositive_or_unbounded_inputs( + kwargs, +) -> None: + with pytest.raises(ValueError): + runner_module.Runner(**kwargs) + + +def test_execution_bounds_reject_boolean_and_nonnumeric_inputs() -> None: + for value in (True, "1", None): + with pytest.raises(ValueError, match="numeric"): + runner_module.Runner(command_timeout=value) + + +def test_every_reviewer_unsafe_argv_is_refused_without_spawning() -> None: + runner, spawn, _ = build([]) + commands = [ + ("kubectl", "auth", "reconcile", "-f", "x"), + ("kubectl", "config", "view", "--raw"), + ("kubectl", "get", "secret/x", "-o", "name"), + ("kubectl", "get", "--raw=/api/v1/secrets"), + ("kubectl", "delete", "pod/x", "--dry-run=client"), + ("kubectl", "delete", "pod/x", "--", "--dry-run=server"), + ("git", "fetch", "origin"), + ("git", "config", "core.sshCommand", "x"), + ("git", "-ccore.sshCommand=x", "ls-remote", "origin"), + ] + for command in commands: + assert runner.run(command, OPERATOR).error.startswith("policy:") + assert spawn.calls == [] + assert runner.commands_run == 0 + + +def test_live_capture_bounds_concurrent_stdout_and_stderr() -> None: + capture = runner_module.execute_bounded( + ( + "/usr/bin/python3", + "-c", + "import os;os.write(1,b'x'*2000000);os.write(2,b'y'*2000000)", + ), + {"LC_ALL": "C"}, + timeout=5, + max_bytes=1024, + ) + assert capture.returncode == 0 + assert capture.stdout_truncated and capture.stderr_truncated + assert len(capture.stdout) == len(capture.stderr) == 1024 + + +def test_absolute_timeout_kills_the_process_group_and_reaps_quickly() -> None: + started = time.monotonic() + capture = runner_module.execute_bounded( + ( + "/usr/bin/python3", + "-c", + "import os,signal,time;" + "p=os.fork();" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "time.sleep(30)", + ), + {"LC_ALL": "C"}, + timeout=0.15, + max_bytes=1024, + ) + assert capture.timed_out + assert time.monotonic() - started < 1.5 + + +def test_descendant_holding_pipes_is_bounded_after_the_leader_exits() -> None: + started = time.monotonic() + capture = runner_module.execute_bounded( + ( + "/usr/bin/python3", + "-c", + "import os,signal,time;" + "p=os.fork();" + "(os._exit(0) if p else None);" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "time.sleep(30)", + ), + {"LC_ALL": "C"}, + timeout=2, + max_bytes=1024, + ) + assert not capture.timed_out + assert time.monotonic() - started < 1.5 + + +def test_attestation_rejects_a_reviewer_authored_executable(tmp_path) -> None: + fake = tmp_path / "kubectl" + fake.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + fake.chmod(0o755) + with pytest.raises(OSError, match="trusted paths"): + runner_module.attest_executable(str(fake), {"PATH": str(tmp_path)}) + + +def test_attestation_checks_presence_mode_digest_and_success( + tmp_path, monkeypatch +) -> None: + fake = tmp_path / "tool" + fake.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + fake.chmod(0o755) + digest = hashlib.sha256(fake.read_bytes()).hexdigest() + monkeypatch.setitem(runner_module.EXPECTED_PATHS, "tool", {str(fake)}) + monkeypatch.setitem(runner_module.EXPECTED_SHA256, "tool", {digest}) + assert ( + runner_module.attest_executable("tool", {"PATH": str(tmp_path)}).sha256 + == digest + ) + with pytest.raises(OSError, match="unavailable"): + runner_module.attest_executable("missing", {"PATH": str(tmp_path)}) + fake.chmod(0o777) + with pytest.raises(OSError, match="unsafe ownership mode"): + runner_module.attest_executable("tool", {"PATH": str(tmp_path)}) + fake.chmod(0o755) + monkeypatch.setitem(runner_module.EXPECTED_SHA256, "tool", {"0" * 64}) + with pytest.raises(OSError, match="digest"): + runner_module.attest_executable("tool", {"PATH": str(tmp_path)}) + + +def test_runner_attestation_or_resolved_policy_failure_never_spawns() -> None: + runner, spawn, _ = build([]) + runner._attestor = lambda _command, _environment: (_ for _ in ()).throw( + OSError("no") + ) + assert runner.run(SAFE_GET, OPERATOR).error.startswith("attestation:") + runner._attestor = lambda _command, _environment: runner_module.Attestation( + "/tmp/reviewer-authored/kubectl", "a" * 64 + ) + assert runner.run(SAFE_GET, OPERATOR).error.startswith("policy:") + assert spawn.calls == [] diff --git a/testing/tests/test_hermes_handoff_json_rules.py b/testing/tests/test_hermes_handoff_json_rules.py new file mode 100644 index 00000000..e74684d7 --- /dev/null +++ b/testing/tests/test_hermes_handoff_json_rules.py @@ -0,0 +1,413 @@ +"""Contracts for the JSON-shaped handoff acceptance evaluators.""" + +from __future__ import annotations + +import datetime as dt +import json + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) + +evaluators = load_handoff_module("hermes_handoff_evaluators") +model = load_handoff_module("hermes_handoff_model") + +NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc) + + +def evaluate(built, payload): + body = payload if isinstance(payload, str) else json.dumps(payload) + return evaluators.evaluate(built, {"s": outcome(stdout=body)}) + + +def one(rule: str, expect: dict): + return spec(rule, {"step": "s", **expect}, steps=(step("s"),)) + + +def test_json_field_compares_paths_against_expected_values() -> None: + built = one( + "json_field", {"fields": {"authenticated": True, "transport": ["a", "b"]}} + ) + + assert ( + evaluate(built, {"authenticated": True, "transport": "b"}).status == model.PASS + ) + + wrong = evaluate(built, {"authenticated": False, "transport": "c"}) + assert wrong.status == model.FAIL + assert "authenticated=False" in wrong.reason and "transport='c'" in wrong.reason + + assert evaluate(built, {"authenticated": True}).status == model.NOT_RUN + + +def test_json_field_skips_rather_than_guesses_on_unusable_output() -> None: + built = one("json_field", {"fields": {"a": 1}}) + + assert evaluate(built, "not json").status == model.NOT_RUN + failed = evaluators.evaluate(built, {"s": outcome(returncode=1)}) + assert failed.status == model.NOT_RUN + assert evaluate(built, '{"a": 1, "a": 1}').status == model.NOT_RUN + assert evaluate(built, '{"a": NaN}').status == model.NOT_RUN + + +def test_json_equality_does_not_confuse_booleans_and_integers() -> None: + built = one("json_field", {"fields": {"enabled": True}}) + assert evaluate(built, {"enabled": 1}).status == model.FAIL + array = one("all_of_field", {"fields": {"enabled": False}}) + assert evaluate(array, [{"enabled": 0}]).status == model.FAIL + + +def test_json_numeric_enforces_inclusive_bounds_and_rejects_non_numbers() -> None: + built = one("json_numeric", {"fields": {"latency_ms": {"min": 1, "max": 1000}}}) + + assert evaluate(built, {"latency_ms": 500}).status == model.PASS + assert evaluate(built, {"latency_ms": 1}).status == model.PASS + + low = evaluate(built, {"latency_ms": 0}) + assert low.status == model.FAIL and "below minimum" in low.reason + + high = evaluate(built, {"latency_ms": 5000}) + assert high.status == model.FAIL and "above maximum" in high.reason + + assert evaluate(built, {"latency_ms": True}).status == model.NOT_RUN + assert evaluate(built, {"latency_ms": "fast"}).status == model.NOT_RUN + assert evaluate(built, {}).status == model.NOT_RUN + + +def test_json_recent_treats_stale_evidence_as_a_failure() -> None: + built = one("json_recent", {"now": NOW, "fields": {"checked_at": 3600}}) + + fresh = evaluate(built, {"checked_at": "2026-08-17T11:30:00Z"}) + assert ( + fresh.status == model.PASS + and fresh.evidence["checked_at"]["age_seconds"] == 1800 + ) + + stale = evaluate(built, {"checked_at": "2026-08-16T00:00:00Z"}) + assert stale.status == model.FAIL and "limit 3600s" in stale.reason + + naive = evaluate(built, {"checked_at": "2026-08-17T11:30:00"}) + assert naive.status == model.NOT_RUN + assert evaluate(built, {"checked_at": "2026-08-17T12:00:01Z"}).status == model.FAIL + + assert evaluate(built, {"checked_at": "yesterday"}).status == model.NOT_RUN + assert evaluate(built, {}).status == model.NOT_RUN + invalid_clock = one("json_recent", {"now": "now", "fields": {"checked_at": 1}}) + assert ( + evaluate(invalid_clock, {"checked_at": "2026-08-17T11:30:00Z"}).status + == model.NOT_RUN + ) + + +def test_all_of_field_selects_keys_and_reports_every_mismatch() -> None: + built = one( + "all_of_field", + {"key_field": "number", "keys": [14, 15], "fields": {"merged": True}}, + ) + payload = [ + {"number": 14, "merged": True}, + {"number": 15, "merged": False}, + {"number": 16, "merged": False}, + ] + + evaluation = evaluate(built, payload) + + assert evaluation.status == model.FAIL + assert "15: merged=False" in evaluation.reason + assert "16" not in evaluation.reason + assert evaluation.evidence["examined"] == [14, 15] + + +def test_all_of_field_skips_when_an_expected_key_is_absent_or_nothing_matched() -> None: + keyed = one( + "all_of_field", + {"key_field": "number", "keys": [14], "fields": {"merged": True}}, + ) + absent = evaluate(keyed, [{"number": 99, "merged": True}]) + assert absent.status == model.NOT_RUN and "expected keys absent" in absent.reason + + unkeyed = one("all_of_field", {"fields": {"status": ["done"]}}) + assert evaluate(unkeyed, []).status == model.NOT_RUN + assert evaluate(unkeyed, {"not": "an array"}).status == model.NOT_RUN + + +def test_all_of_field_skips_when_the_named_array_path_is_absent() -> None: + built = spec( + "all_of_field", + {"step": "s", "array": "items", "fields": {"a": 1}}, + steps=(step("s"),), + ) + + evaluation = evaluate(built, {"other": []}) + + assert evaluation.status == model.NOT_RUN + assert "array path is absent" in evaluation.reason + + +def test_all_of_field_treats_an_absent_field_as_a_mismatch_not_a_crash() -> None: + built = one("all_of_field", {"key_field": "name", "fields": {"suspended": False}}) + + evaluation = evaluate(built, [{"name": "a"}]) + + assert evaluation.status == model.FAIL + assert "suspended=None" in evaluation.reason + + +def test_all_of_field_honours_exemptions_and_non_empty_paths() -> None: + exempted = one( + "all_of_field", + { + "key_field": "name", + "exempt_keys": ["parked"], + "fields": {"suspended": False}, + }, + ) + payload = [ + {"name": "parked", "suspended": True}, + {"name": "live", "suspended": False}, + ] + assert evaluate(exempted, payload).status == model.PASS + + non_empty = one("all_of_field", {"key_field": "id", "non_empty": ["title"]}) + assert evaluate(non_empty, [{"id": "a", "title": "work"}]).status == model.PASS + blank = evaluate(non_empty, [{"id": "a", "title": ""}, {"id": "b"}]) + assert blank.status == model.FAIL and "is empty" in blank.reason + + +def test_json_record_captures_named_paths_or_skips() -> None: + built = one( + "json_record", {"record": {"numbers": "[].number", "refs": "[].head.ref"}} + ) + payload = [ + {"number": 14, "head": {"ref": "a"}}, + {"number": 15, "head": {"ref": "b"}}, + ] + + evaluation = evaluate(built, payload) + assert evaluation.status == model.PASS + assert evaluation.evidence == {"numbers": [14, 15], "refs": ["a", "b"]} + + missing = evaluate(built, [{"number": 14}]) + assert missing.status == model.NOT_RUN and "absent" in missing.reason + + +def routing_log(*records: dict) -> str: + return "\n".join(json.dumps(record) for record in records) + + +def routing_spec(**extra): + return spec( + "routing_evidence", + { + "step": "s", + "providers": ("codex", "claude"), + "efforts": ("high", "xhigh"), + "lanes": ("route", "worker"), + "require_fallback_evidence": True, + **extra, + }, + steps=(step("s"),), + ) + + +FULL_LOG = routing_log( + {"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}, + {"ts": "2026-08-17T11:30:00Z", "model": "route/claude/opus/xhigh"}, + { + "ts": "2026-08-17T11:45:00Z", + "model": "worker/codex/sol/xhigh", + "fallback_reason": "unavailable", + }, + {"ts": "2026-08-17T11:50:00Z", "model": "qwen2.5:14b", "tier": "classifier"}, +) + + +def test_routing_evidence_summarises_lanes_providers_efforts_and_fallbacks() -> None: + evaluation = evaluate(routing_spec(), FULL_LOG) + + assert evaluation.status == model.PASS + assert evaluation.evidence["providers"] == ["claude", "codex"] + assert evaluation.evidence["efforts"] == ["high", "xhigh"] + assert evaluation.evidence["lanes"] == ["route", "worker"] + assert evaluation.evidence["fallback_reasons"] == ["unavailable"] + assert evaluation.evidence["records"] == 4 and evaluation.evidence["routed"] == 3 + + +def test_routing_evidence_names_exactly_what_the_window_did_not_show() -> None: + partial = routing_log( + {"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"} + ) + + evaluation = evaluate(routing_spec(), partial) + + assert evaluation.status == model.FAIL + assert "provider:claude" in evaluation.reason + assert "effort:xhigh" in evaluation.reason + assert "lane:worker" in evaluation.reason + assert "fallback:none-observed" in evaluation.reason + + +def test_routing_evidence_rejects_a_clipped_tail_and_empty_input() -> None: + clipped = '{"ts": "2026-08-17T11:0\n' + FULL_LOG + + assert evaluate(routing_spec(), clipped).status == model.NOT_RUN + assert evaluate(routing_spec(), "").status == model.NOT_RUN + assert evaluate(routing_spec(), "garbage\n").status == model.NOT_RUN + + +def test_routing_evidence_can_require_freshness() -> None: + fresh = evaluate(routing_spec(max_age_seconds=3600, now=NOW), FULL_LOG) + assert fresh.status == model.PASS + assert max(fresh.evidence["age_seconds"].values()) <= 3600 + + stale = evaluate( + routing_spec(max_age_seconds=60, now=NOW + dt.timedelta(days=2)), FULL_LOG + ) + assert stale.status == model.FAIL and "freshness" in stale.reason + + undated = routing_log( + {"model": "route/codex/sol/high"}, {"model": "route/claude/opus/xhigh"} + ) + unknown = evaluate(routing_spec(max_age_seconds=60, now=NOW), undated) + assert unknown.status == model.NOT_RUN + + +def test_routing_freshness_is_required_per_provider_not_only_globally() -> None: + rows = routing_log( + {"ts": "2026-08-15T11:00:00Z", "model": "route/claude/opus/high"}, + {"ts": "2026-08-17T11:59:00Z", "model": "route/codex/sol/xhigh"}, + { + "ts": "2026-08-17T11:59:00Z", + "model": "worker/codex/sol/high", + "fallback_reason": "down", + }, + ) + evaluation = evaluate(routing_spec(max_age_seconds=3600, now=NOW), rows) + assert evaluation.status == model.FAIL + assert "freshness:provider:claude" in evaluation.reason + missing_dimension = evaluate( + routing_spec(max_age_seconds=3600, now=NOW), + routing_log( + {"ts": "2026-08-17T11:59:00Z", "model": "route/codex/sol/high"}, + { + "ts": "2026-08-17T11:59:00Z", + "model": "worker/codex/sol/xhigh", + "fallback_reason": "down", + }, + ), + ) + assert missing_dimension.status == model.FAIL + + +@pytest.mark.parametrize( + "body", + [ + '["not-an-object"]', + '{"ts":"2026-08-17T12:00:01Z","model":"route/codex/sol/high"}', + '{"ts":"2026-08-17T11:00:00Z"}', + '{"ts":"2026-08-17T11:00:00Z","model":"bad"}', + '{"ts":"2026-08-17T11:00:00Z","model":"route/codex/sol/high","fallback_reason":0}', + ], +) +def test_routing_rows_fail_closed_on_structural_defects(body: str) -> None: + assert evaluate(routing_spec(now=NOW), body).status in {model.FAIL, model.NOT_RUN} + invalid_clock = routing_spec(now="now") + assert evaluate(invalid_clock, FULL_LOG).status == model.NOT_RUN + + +def flux_spec(*expected: str): + return spec( + "flux_health", + {"step": "s", "expected_suspensions": expected}, + steps=(step("s"),), + ) + + +def test_flux_health_separates_parked_objects_from_broken_ones() -> None: + rows = ( + "flux-system/core\t1\t1\tfalse\tReady=True,Healthy=True,\n" + "flux-system/parked\t1\t1\ttrue\tReady=False,\n" + "flux-system/broken\t1\t1\tfalse\tReady=False,Healthy=False,\n" + ) + + evaluation = evaluators.evaluate( + flux_spec("flux-system/parked"), {"s": outcome(stdout=rows)} + ) + + assert evaluation.status == model.FAIL + assert evaluation.evidence["unhealthy"] == ["flux-system/broken"] + assert evaluation.evidence["unexpected_suspensions"] == [] + assert "unhealthy: flux-system/broken" in evaluation.reason + + +def test_flux_health_flags_a_suspension_nobody_declared() -> None: + rows = "flux-system/core\t1\t1\tfalse\tReady=True,\nflux-system/parked\t1\t1\ttrue\tReady=False,\n" + + evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) + + assert evaluation.status == model.FAIL + assert evaluation.evidence["unexpected_suspensions"] == ["flux-system/parked"] + + +def test_flux_health_fails_closed_on_in_flight_or_unknown_conditions() -> None: + rows = ( + "flux-system/reconciling\t1\t1\tfalse\tReconciling=True,Ready=False,\n" + "flux-system/degraded\t1\t1\tfalse\tReady=False,Healthy=True,\n" + "flux-system/waiting\t1\t1\tfalse\tReady=Unknown,\n" + ) + + evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) + + assert evaluation.status == model.FAIL + assert evaluation.evidence["total"] == 3 + + +def test_blank_rows_in_a_projection_are_ignored_by_both_log_parsers() -> None: + padded = "\n\n" + FULL_LOG + "\n\n" + assert evaluate(routing_spec(), padded).status == model.PASS + + rows = "\n\nflux-system/core\t1\t1\tfalse\tReady=True,\n\n" + evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)}) + assert evaluation.status == model.PASS and evaluation.evidence["total"] == 1 + + +def test_flux_health_skips_on_an_empty_or_failed_projection() -> None: + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(stdout="")}).status + == model.NOT_RUN + ) + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(returncode=1)}).status + == model.NOT_RUN + ) + malformed = "flux-system/core\t1\t0\tfalse\tReady=True,\n" + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(stdout=malformed)}).status + == model.FAIL + ) + + +@pytest.mark.parametrize( + "row", + [ + "too\tfew", + "BAD NAME\t1\t1\tfalse\tReady=True,", + "flux-system/core\tx\t1\tfalse\tReady=True,", + "flux-system/core\t1\t1\tmaybe\tReady=True,", + "flux-system/core\t1\t1\tfalse\tbroken,", + "flux-system/core\t1\t1\tfalse\tReady=Yes,", + "flux-system/core\t1\t1\tfalse\tReady=True,Ready=True,", + ], +) +def test_flux_rows_reject_malformed_columns_names_generations_and_conditions( + row: str, +) -> None: + assert ( + evaluators.evaluate(flux_spec(), {"s": outcome(stdout=row)}).status + == model.NOT_RUN + ) diff --git a/testing/tests/test_hermes_handoff_model.py b/testing/tests/test_hermes_handoff_model.py new file mode 100644 index 00000000..e3c46489 --- /dev/null +++ b/testing/tests/test_hermes_handoff_model.py @@ -0,0 +1,206 @@ +"""Contracts for the handoff harness result model, verdict, and rendering.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) + +model = load_handoff_module("hermes_handoff_model") + + +def result( + status: str, + identifier: str = "test.check", + mandatory: bool = True, + group: str = "g", +): + return model.CheckResult( + spec=model.CheckSpec( + id=identifier, title="t", group=group, rule="allowed", mandatory=mandatory + ), + status=status, + reason=f"{status} reason", + ) + + +def report(*results, errors: tuple[str, ...] = ()) -> model.Report: + return model.Report( + mode="read-only", + started_at="2026-08-17T00:00:00Z", + results=list(results), + harness_errors=list(errors), + ) + + +@pytest.mark.parametrize( + ("status", "mandatory", "blocking"), + [ + (model.PASS, True, False), + (model.NOT_APPLICABLE, True, False), + (model.FAIL, True, True), + (model.NOT_RUN, True, True), + (model.FAIL, False, False), + (model.NOT_RUN, False, False), + ], +) +def test_only_a_mandatory_failure_or_skip_blocks(status, mandatory, blocking) -> None: + assert result(status, mandatory=mandatory).blocking is blocking + + +def test_a_mandatory_skip_is_a_no_go_exactly_like_a_failure() -> None: + """The whole point of the four-valued classification.""" + assert report(result(model.NOT_RUN)).decision == model.NO_GO + assert report(result(model.FAIL)).decision == model.NO_GO + assert report(result(model.PASS), result(model.NOT_APPLICABLE)).decision == model.GO + + +def test_a_harness_error_is_a_no_go_even_when_every_check_passed() -> None: + clean = report( + result(model.PASS), errors=("the two vantages are the same principal",) + ) + assert clean.decision == model.NO_GO + + +def test_a_zero_state_or_unknown_status_report_is_no_go() -> None: + """Absence or an unrecognised classification can never round up to GO.""" + assert report().decision == model.NO_GO + assert report(errors=("no checks were built",)).decision == model.NO_GO + unknown = result("MAYBE") + assert unknown.blocking is True + built = report(unknown) + assert built.decision == model.NO_GO + assert built.counts == dict.fromkeys(model.STATUSES, 0) + assert "groups:" in report(result(model.PASS)).render_summary() + + +def test_blocking_results_are_ordered_failures_before_skips() -> None: + ordered = report( + result(model.NOT_RUN, "b.skip"), + result(model.FAIL, "z.fail"), + result(model.PASS, "a.pass"), + ).blocking + + assert [item.spec.id for item in ordered] == ["z.fail", "b.skip"] + + +def test_counts_and_group_summary_tally_every_status() -> None: + built = report( + result(model.PASS, "a", group="alpha"), + result(model.FAIL, "b", group="alpha"), + result(model.NOT_RUN, "c", group="beta"), + ) + + assert built.counts == { + model.PASS: 1, + model.FAIL: 1, + model.NOT_RUN: 1, + model.NOT_APPLICABLE: 0, + } + assert built.group_summary()["alpha"][model.PASS] == 1 + assert built.group_summary()["beta"][model.NOT_RUN] == 1 + + +def test_the_serialised_report_carries_the_verdict_and_its_reasons() -> None: + built = report(result(model.FAIL, "x.fail"), result(model.PASS, "y.pass")) + built.vantages = [ + model.VantageRecord( + name="operator", description="d", identity="u", available=True + ) + ] + + payload = built.as_dict() + + assert payload["decision"] == model.NO_GO + assert payload["blocking"] == ["x.fail"] + assert payload["harness"] == model.HARNESS + assert payload["schema_version"] == model.SCHEMA_VERSION + assert payload["vantages"][0]["identity"] == "u" + assert {check["id"] for check in payload["checks"]} == {"x.fail", "y.pass"} + + +def test_check_serialisation_screens_evidence_and_can_omit_commands() -> None: + built = model.CheckResult( + spec=spec("allowed", identifier="x", steps=(step("a"),)), + status=model.PASS, + evidence={"token": "a-long-enough-secret", "count": 3}, + outcomes=[outcome(stdout="fine")], + ) + + with_commands = built.as_dict() + without = built.as_dict(include_outcomes=False) + + assert with_commands["evidence"]["token"] == "[redacted]" + assert with_commands["evidence"]["count"] == 3 + assert len(with_commands["commands"]) == 1 + assert "commands" not in without + assert with_commands["vantages"] == ["self"] + + +def test_rationale_is_carried_only_when_a_check_has_one() -> None: + without = model.CheckResult(spec=spec("allowed"), status=model.PASS).as_dict() + assert "rationale" not in without + + annotated = model.CheckSpec( + id="x", title="t", group="g", rule="allowed", rationale="because" + ) + with_reason = model.CheckResult(spec=annotated, status=model.PASS).as_dict() + assert with_reason["rationale"] == "because" + + +def test_distinct_vantages_are_reported_in_first_use_order() -> None: + built = spec( + "allowed", + steps=( + step("a", vantage="operator"), + step("b", vantage="self"), + step("c", vantage="operator"), + ), + ) + assert built.vantages == ("operator", "self") + + +def test_the_human_summary_names_the_verdict_vantages_and_blockers() -> None: + built = report(result(model.FAIL, "x.fail", group="alpha"), errors=("collision",)) + built.vantages = [ + model.VantageRecord( + name="operator", description="d", identity="u", available=True + ), + model.VantageRecord(name="self", description="d", available=False), + ] + + summary = built.render_summary() + + assert summary.splitlines()[0] == f"{model.HARNESS}: {model.NO_GO}" + assert "vantage operator: available identity=u" in summary + assert "vantage self: unavailable identity=unknown" in summary + assert "harness error: collision" in summary + assert "x.fail" in summary + assert "alpha" in summary + + +def test_utc_now_renders_a_zulu_timestamp_and_accepts_an_injected_clock() -> None: + fixed = dt.datetime(2026, 8, 17, 9, 30, 15, 123456, tzinfo=dt.timezone.utc) + assert model.utc_now(lambda: fixed) == "2026-08-17T09:30:15Z" + assert model.utc_now().endswith("Z") + + +def test_unscreened_fields_walks_the_whole_payload() -> None: + offenders = model.unscreened_fields( + { + "clean": "hermes", + "nested": {"leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234"}, + "items": [1, "Authorization: Bearer abcdefghijklmnop"], + } + ) + + assert offenders == ["$.nested.leak", "$.items[1]"] + assert model.unscreened_fields({"clean": "hermes"}) == [] + assert model.unscreened_fields(None) == [] diff --git a/testing/tests/test_hermes_handoff_mutation_gate.py b/testing/tests/test_hermes_handoff_mutation_gate.py new file mode 100644 index 00000000..61c60705 --- /dev/null +++ b/testing/tests/test_hermes_handoff_mutation_gate.py @@ -0,0 +1,60 @@ +"""Tests for the deterministic Hermes handoff mutation gate.""" + +from __future__ import annotations + +from pathlib import Path + +from testing.quality_handoff_mutation import ( + MUTATIONS, + Mutation, + MutationReport, + main, + run_gate, +) + + +ROOT = Path(__file__).parents[2] + + +def test_every_security_mutant_is_killed() -> None: + report = run_gate(ROOT) + assert report.issues == () + assert report.score == 100.0 + assert report.killed == report.total >= 9 + + +def test_invalid_anchor_baseline_syntax_and_survivor_are_reported() -> None: + path = "hermes_handoff_model.py" + cases = ( + Mutation("missing", path, "not present", "x", "pass"), + Mutation( + "baseline", + path, + "SCHEMA_VERSION = 1", + "SCHEMA_VERSION = 2", + "raise SystemExit(1)", + ), + Mutation("syntax", path, "SCHEMA_VERSION = 1", "if", "pass"), + Mutation("survivor", path, "SCHEMA_VERSION = 1", "SCHEMA_VERSION = 2", "pass"), + ) + report = run_gate(ROOT, cases) + assert report.killed == 0 + assert len(report.issues) == 4 + assert report.score == 0.0 + + +def test_main_reports_a_perfect_score(capsys) -> None: + """Every mutant must die, and the count must track the declared set.""" + assert main() == 0 + total = len(MUTATIONS) + assert total >= 13 + assert f"{total}/{total} (100.00%)" in capsys.readouterr().out + + +def test_main_reports_mutation_issues(monkeypatch, capsys) -> None: + monkeypatch.setattr( + "testing.quality_handoff_mutation.run_gate", + lambda _root: MutationReport(1, 0, ("survivor",)), + ) + assert main() == 1 + assert "survivor" in capsys.readouterr().out diff --git a/testing/tests/test_hermes_handoff_policy.py b/testing/tests/test_hermes_handoff_policy.py new file mode 100644 index 00000000..41f1fc7e --- /dev/null +++ b/testing/tests/test_hermes_handoff_policy.py @@ -0,0 +1,367 @@ +"""Contracts for the fail-closed command policy of the handoff harness.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module + +policy = load_handoff_module("hermes_handoff_policy") + + +def check(*argv: str, mode: str | None = None) -> None: + policy.check_argv(argv, mode or policy.READ_ONLY) + + +@pytest.mark.parametrize( + "argv", + [ + ("kubectl", "--namespace", "hermes", "get", "pods", "-o", "name"), + ("kubectl", "auth", "can-i", "get", "secrets", "--all-namespaces"), + ("kubectl", "get", "nodes", "-o", "name"), + ( + "kubectl", + "get", + "pods", + "-o", + policy.projection("jsonpath={.status.phase}"), + ), + ("git", "merge-base", "--is-ancestor", "a", "b"), + ("git", "ls-remote", "origin", "refs/heads/main"), + ("hermes", "kanban", "list", "--json"), + ("hermes", "sessions", "list", "--source", "telegram"), + ("hermes", "status"), + (policy.GITEA_CLIENT, "GET", "/api/v1/user"), + ], +) +def test_read_only_commands_are_permitted(argv: tuple[str, ...]) -> None: + check(*argv) + + +@pytest.mark.parametrize( + "argv", + [ + (), + ("vault", "read", "kv/x"), + ("curl", "https://example.dev"), + ("rm", "-rf", "/"), + ("kubectl", "get", "pods", "--token", "abc"), + ("kubectl", "get", "pods"), + ("kubectl", "get", "pods", "--as=system:admin"), + ("kubectl", "get", "secrets", "-o", "name"), + ("kubectl", "get", "pods,secrets/x", "-o", "name"), + ("kubectl", "describe", "pod/x"), + ("kubectl", "config", "view", "--raw"), + ("kubectl", "auth", "reconcile", "-f", "x"), + ("kubectl", "delete", "pod/x", "--dry-run=client"), + ("kubectl", "delete", "pod/x", "--", "--dry-run=server"), + ("kubectl", "get", "pods", "-o=json"), + ("kubectl", "get", "pods", "-o", "jsonpath={.spec.containers[*].env[*].value}"), + ("kubectl", "port-forward", "svc/x", "80"), + ("kubectl", "exec", "pod"), + ("kubectl", "exec", "pod", "--"), + ("kubectl",), + ("flux", "get", "kustomizations", "--all-namespaces"), + ("flux", "reconcile", "kustomization", "x"), + ("flux",), + ("helm", "list", "--all-namespaces"), + ("helm", "upgrade", "x"), + ("helm", "get", "values", "x"), + ("git", "commit", "-m", "x"), + ("git", "push", "origin", "HEAD:refs/heads/x"), + ("git", "fetch", "origin"), + ("git", "config", "core.sshCommand", "x"), + ("git", "-ccore.sshCommand=x", "ls-remote", "origin"), + ("git", "--git-dir=/tmp/x", "status"), + ("hermes", "kanban", "complete", "t_1"), + ("hermes",), + (policy.GITEA_CLIENT, "POST", "/api/v1/x"), + (policy.GITEA_CLIENT, "GET"), + ("sh", "-c", "rm -rf /"), + ("sh", "echo"), + ("git", "log", "/etc/shadow"), + ], +) +def test_unsafe_commands_are_refused(argv: tuple[str, ...]) -> None: + with pytest.raises(policy.PolicyError): + check(*argv) + + +def test_unknown_mode_is_refused() -> None: + with pytest.raises(policy.PolicyError, match="unknown mode"): + policy.check_argv(("kubectl", "get", "pods"), "whatever") + + +def test_dry_run_mutation_commands_are_never_constructible() -> None: + with pytest.raises(policy.PolicyError, match="dry-run"): + check("kubectl", "patch", "deploy/x", "--dry-run=server") + + +def test_a_resource_named_after_a_subcommand_cannot_smuggle_a_delete() -> None: + """`-n get` must not make a delete look like a read.""" + with pytest.raises(policy.PolicyError): + check("kubectl", "delete", "-n", "get", "configmap", "x") + + +@pytest.mark.parametrize("flag", ["--as", "--as-group", "--as-uid"]) +@pytest.mark.parametrize("mode", ["read-only", "ephemeral-armed"]) +def test_impersonation_is_refused_from_every_vantage_and_mode( + flag: str, mode: str +) -> None: + """No vantage may impersonate: not the operator, and not inside a pod. + + The inner command of ``kubectl exec`` used to be granted impersonation + unconditionally, so a self/switchyard/node/chat step carrying ``--as`` was + accepted. The inner argv is now re-checked under the same rule. + """ + for argv in ( + ("kubectl", "get", "namespaces", "-o", "name", flag, "system:admin"), + ("kubectl", "get", "ns", "-o", "name", f"{flag}=system:admin"), + ): + with pytest.raises(policy.PolicyError, match="forbidden argument"): + check(*argv, mode=mode) + inner = ( + "kubectl", + "exec", + "pod", + "--", + "/usr/local/bin/kubectl", + "get", + "ns", + "-o", + "name", + flag, + "system:admin", + ) + with pytest.raises(policy.PolicyError, match="forbidden argument"): + check(*inner, mode=mode) + + +def test_exec_validates_its_inner_command() -> None: + script = policy.render_shell("env_names") + policy._RENDERED.add(script) + check( + "kubectl", "exec", "--namespace", "hermes", "pod", "--", "/bin/sh", "-c", script + ) + check( + "kubectl", + "exec", + "pod", + "--", + "/usr/local/bin/kubectl", + "get", + "ns", + "-o", + "name", + ) + with pytest.raises(policy.PolicyError): + check("kubectl", "exec", "pod", "--", "bash", "-c", "id") + + +def test_the_repository_pin_has_an_exact_boundary_and_rejects_dot_segments() -> None: + """A prefix match without a boundary let a look-alike repository through.""" + for path in ( + "/api/v1/repos/atlas/titan-iac-evil/pulls", + "/api/v1/repos/atlas/titan-iacx", + "/api/v1/repos/atlas/titan-iac../pulls", + ): + with pytest.raises(policy.PolicyError): + check(policy.GITEA_CLIENT, "GET", path) + for path in ( + "/api/v1/repos/atlas/titan-iac/../../user/tokens", + "/api/v1/repos/atlas/titan-iac/%2e%2e/%2e%2e/user/tokens", + "/api/v1/repos/atlas/titan-iac/%252e%252e/user/tokens", + "/api/v1/repos/atlas/titan-iac/./pulls", + "/api/v1/repos/atlas/titan-iac//pulls", + ): + with pytest.raises(policy.PolicyError, match="relative segments"): + check(policy.GITEA_CLIENT, "GET", path) + check(policy.GITEA_CLIENT, "GET", "/api/v1/repos/atlas/titan-iac") + check(policy.GITEA_CLIENT, "GET", "/api/v1/repos/atlas/titan-iac/pulls?state=all") + + +def test_every_allowlisted_binary_is_release_attestable() -> None: + """An allowlisted binary with no pinned digest is dead, misleading config.""" + exec_module = load_handoff_module("hermes_handoff_exec") + assert set(exec_module.EXPECTED_SHA256) == set(policy.ALLOWED_BINARIES) + assert set(exec_module.EXPECTED_PATHS) == set(policy.ALLOWED_BINARIES) + assert set(exec_module.POD_COMMAND_PATHS) == set(policy.ALLOWED_BINARIES) + assert set(exec_module.CONTEXT_AWARE_BINARIES) <= set(policy.ALLOWED_BINARIES) + + +def test_shell_templates_reject_credential_roots() -> None: + with pytest.raises(policy.PolicyError, match="credential paths"): + policy.shell("json_fields", path="/runtime-access/token", fields="state") + with pytest.raises(policy.PolicyError, match="credential-bearing"): + policy.shell("json_fields", path="/tmp/health.json", fields="state,token") + + +def test_armed_mode_opens_exactly_the_mutations_it_should() -> None: + ref = "ephemeral/hermes-handoff-acceptance/acceptance1" + policy.arm_ephemeral_policy(ref) + policy.register_ephemeral_pull(9) + check("git", "push", "origin", f"HEAD:refs/heads/{ref}", mode=policy.ARMED) + root = "/api/v1/repos/atlas/titan-iac" + check( + policy.GITEA_CLIENT, + "POST", + f"{root}/pulls", + "--field", + "title=WIP: Hermes handoff acceptance ephemeral probe", + "--field", + f"head={ref}", + "--field", + "base=main", + "--field", + "body=Ephemeral acceptance probe. Closed and deleted by the harness.", + mode=policy.ARMED, + ) + check( + policy.GITEA_CLIENT, + "PATCH", + f"{root}/pulls/9", + "--field", + "state=closed", + mode=policy.ARMED, + ) + check(policy.GITEA_CLIENT, "DELETE", f"{root}/branches/{ref}", mode=policy.ARMED) + with pytest.raises(policy.PolicyError, match="may not force"): + check( + "git", + "push", + "--force", + "origin", + f"HEAD:refs/heads/{ref}", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError, match="may not force"): + check("git", "push", "--all", "origin", mode=policy.ARMED) + with pytest.raises(policy.PolicyError, match="not permitted"): + check(policy.GITEA_CLIENT, "PUT", f"{root}/pulls/9", mode=policy.ARMED) + + +def test_arming_and_projection_registration_fail_closed() -> None: + with pytest.raises(policy.PolicyError, match="exact ephemeral ref"): + policy.arm_ephemeral_policy("main") + policy._ARMED_REF = "" + with pytest.raises(policy.PolicyError, match="registration is malformed"): + policy.register_ephemeral_pull(9) + policy.arm_ephemeral_policy("ephemeral/hermes-handoff-acceptance/acceptance1") + + for candidate in ( + "wide", + "jsonpath={.status.token}", + "jsonpath={.metadata.annotations}", + "jsonpath={.metadata.labels.other}", + "jsonpath={.spec.containers[*].env[*].value}", + ): + with pytest.raises(policy.PolicyError): + policy.projection(candidate) + + with pytest.raises(policy.PolicyError): + check("/usr/bin/awk", "x") + with pytest.raises(policy.PolicyError): + check("git", "--exec-path=/tmp", "status") + + +def test_every_frozen_template_renders_and_is_accepted() -> None: + parameters = { + "account": "hermes-agent", + "askpass": "/opt/coordinator/gitea_askpass.sh", + "fields": "state,model", + "limit": "200", + "path": "/host-etc/passwd", + "url": "https://scm.example.dev/atlas/repo.git", + } + for name in policy.SHELL_TEMPLATES: + required = policy.template_parameters(name) + argv = policy.shell(name, **{key: parameters[key] for key in required}) + policy.check_argv(argv) + assert argv[0] == "sh" + + +def test_template_rendering_rejects_unknown_names_and_unsafe_parameters() -> None: + with pytest.raises(policy.PolicyError, match="unknown shell template"): + policy.render_shell("nope") + with pytest.raises(policy.PolicyError, match="expects parameters"): + policy.render_shell("json_fields") + with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): + policy.render_shell("json_fields", path="/tmp;rm", fields="state") + with pytest.raises(policy.PolicyError, match="unsafe shell parameter"): + policy.render_shell("json_fields", path="$(id)", fields="state") + + +def test_positionals_skip_flags_and_stop_at_the_boundary() -> None: + argv = ("kubectl", "--namespace", "hermes", "get", "pods", "--", "ignored") + assert list(policy.positionals(argv)) == ["get", "pods"] + assert list(policy.positionals(("kubectl", "--all-namespaces", "get"))) == ["get"] + + +def test_a_subcommand_must_precede_its_resource_arguments() -> None: + with pytest.raises(policy.PolicyError): + check("kubectl", "pods", "get") + + +@pytest.mark.parametrize( + "argv", + [ + ("/usr/bin/../../tmp/kubectl", "get", "pods"), + ("/tmp/reviewer/kubectl", "get", "pods"), + ("kubectl", "get", "pods", "-o", "go-template=x"), + ("kubectl", "get", "pods", "-o", "jsonpath={.status.token}"), + ("kubectl", "get", "pods", "-o", "jsonpath={.metadata.annotations}"), + ("kubectl", "get", "pods", "-o", "jsonpath={.spec.containers[*].env[*].value}"), + ("git", "-C", "/tmp", "status"), + ("git", "--work-tree=/tmp", "status"), + (policy.GITEA_CLIENT, "GET", "not-an-api-path"), + (policy.GITEA_CLIENT, "GET", "/api/v1/repos/atlas/other"), + ], +) +def test_provenance_projection_and_repository_bypasses_are_rejected(argv) -> None: + with pytest.raises(policy.PolicyError): + check(*argv) + + +def test_armed_writes_are_pinned_to_exact_remote_ref_and_endpoints() -> None: + ref = "ephemeral/hermes-handoff-acceptance/acceptance1" + policy.arm_ephemeral_policy(ref) + policy.register_ephemeral_pull(9) + with pytest.raises(policy.PolicyError, match="pinned to origin"): + check( + "git", + "push", + "upstream", + "HEAD:refs/heads/ephemeral/hermes-handoff-acceptance/acceptance1", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError, match="preflighted ephemeral ref"): + check("git", "push", "origin", "HEAD:refs/heads/other", mode=policy.ARMED) + with pytest.raises(policy.PolicyError): + check( + policy.GITEA_CLIENT, + "DELETE", + "/api/v1/repos/atlas/titan-iac", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError): + check( + policy.GITEA_CLIENT, + "PATCH", + "/api/v1/repos/atlas/titan-iac/pulls/8", + "--field", + "state=closed", + mode=policy.ARMED, + ) + with pytest.raises(policy.PolicyError): + check( + policy.GITEA_CLIENT, + "POST", + "/api/v1/repos/atlas/titan-iac/pulls", + "--field", + "title=WIP: wrong branch", + "--field", + "head=main", + "--field", + "base=main", + mode=policy.ARMED, + ) diff --git a/testing/tests/test_hermes_handoff_redaction.py b/testing/tests/test_hermes_handoff_redaction.py new file mode 100644 index 00000000..78fe19ec --- /dev/null +++ b/testing/tests/test_hermes_handoff_redaction.py @@ -0,0 +1,192 @@ +"""Contracts for the handoff harness credential screen and output bounding.""" + +from __future__ import annotations + +import json + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module + +redaction = load_handoff_module("hermes_handoff_redaction") + + +@pytest.mark.parametrize( + "text", + [ + "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234", + "Authorization: Bearer abcdefghijklmnop", + "cookie: session=abcdefghijklmnop", + "password=hunter2000secret", + "-----BEGIN OPENSSH PRIVATE KEY-----\nabc\n-----END OPENSSH PRIVATE KEY-----", + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI", + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0.SflKxwRJ", + "https://user:supersecretvalue@scm.example.dev/x", + "vault token hvs.CAESIJabcdefghijklmnop", + "aws AKIAIOSFODNN7EXAMPLE", + ], +) +def test_credential_shapes_are_screened(text: str) -> None: + assert redaction.contains_credential_shape(text) + assert redaction.REDACTED in redaction.redact(text) + + +@pytest.mark.parametrize( + "text", + [ + "sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7", + "ab346f55509d584e457fe26cf90be3078f7a375c", + '"automountServiceAccountToken": false', + '"key": "kubernetes.io/arch"', + "namespace: hermes", + "replicas: 3", + "checked_at: 2026-08-17T09:17:07.224400+00:00", + ], +) +def test_ordinary_evidence_survives_screening(text: str) -> None: + assert redaction.redact(text) == text + assert not redaction.contains_credential_shape(text) + + +def test_screened_json_is_still_parsable_json() -> None: + """A screen that corrupts structure would break every downstream evaluator.""" + payload = { + "spec": {"automountServiceAccountToken": True, "key": "kubernetes.io/arch"}, + "data": {"password": "a-long-enough-secret-value"}, + } + screened = redaction.redact(json.dumps(payload, indent=2)) + reparsed = json.loads(screened) + + assert reparsed["spec"]["automountServiceAccountToken"] is True + assert reparsed["spec"]["key"] == "kubernetes.io/arch" + assert reparsed["data"]["password"] == redaction.REDACTED + + +def test_high_entropy_runs_are_screened_but_hex_digests_are_not() -> None: + assert ( + redaction.redact("Zm9vYmFyYmF6cXV4-Quux_1234567890ABCDEFghij") + != redaction.REDACTED + ) + long_mixed = "aB3" + "x9Y2z_" * 10 + assert redaction.redact(long_mixed) == redaction.REDACTED + assert redaction.redact("f" * 64) == "f" * 64 + + +@pytest.mark.parametrize( + ("key", "sensitive"), + [ + ("ANTHROPIC_API_KEY", True), + ("gitea_token", True), + ("clientSecret", True), + ("Set-Cookie", True), + ("automountServiceAccountToken", True), + ("key", False), + ("namespace", False), + ("", False), + ], +) +def test_key_sensitivity_classification(key: str, sensitive: bool) -> None: + assert redaction.is_sensitive_key(key) is sensitive + + +@pytest.mark.parametrize( + ("value", "possible"), + [ + ("true", False), + ("False", False), + ("3", False), + ("12.5", False), + ("short", False), + ("", False), + ("a-long-enough-secret", True), + ('"another-long-secret"', True), + ("1234567890123", False), + ("-12.5e9", False), + ], +) +def test_value_shape_gates_screening(value: str, possible: bool) -> None: + assert redaction.could_hold_secret(value) is possible + + +def test_bound_truncates_to_a_byte_budget_and_reports_it() -> None: + text, truncated = redaction.bound("x" * 100, max_bytes=10) + assert truncated + assert len(text.encode()) <= 10 + assert text == redaction.TRUNCATION_NOTE[:10] + + text, truncated = redaction.bound("short", max_bytes=64) + assert (text, truncated) == ("short", False) + + +def test_bound_with_no_budget_keeps_nothing() -> None: + assert redaction.bound("content", max_bytes=0) == ("", True) + assert redaction.bound("", max_bytes=0) == ("", False) + + +def test_bound_never_splits_a_multibyte_character() -> None: + text, truncated = redaction.bound("é" * 10, max_bytes=5) + assert truncated + assert len(text.encode()) <= 5 + + +def test_safe_text_screens_before_it_bounds() -> None: + """A boundary must never preserve a useful credential prefix.""" + payload = "password=" + "a" * 200 + text, truncated = redaction.safe_text(payload, max_bytes=20) + assert not truncated + assert redaction.REDACTED in text + + +def test_no_byte_budget_exposes_a_credential_prefix() -> None: + payload = "password=A9secret-prefix-that-must-never-survive" + for budget in range(1, 64): + screened, _ = redaction.safe_text(payload, max_bytes=budget) + assert "A9" not in screened + assert len(screened.encode()) <= budget + + +def test_scrub_walks_nested_structures_and_keeps_useful_types() -> None: + scrubbed = redaction.scrub( + { + "token": "a-long-enough-secret", + "automountServiceAccountToken": False, + "items": [{"password": "another-long-secret"}, 3, None], + "note": ("tuple", "values"), + } + ) + + assert scrubbed["token"] == redaction.REDACTED + assert scrubbed["automountServiceAccountToken"] is False + assert scrubbed["items"][0]["password"] == redaction.REDACTED + assert scrubbed["items"][1] == 3 + assert scrubbed["items"][2] is None + assert scrubbed["note"] == ["tuple", "values"] + + +def test_scrub_screens_short_numeric_nested_values_and_mapping_keys() -> None: + scrubbed = redaction.scrub( + { + "token": 1234, + "credentials": {"x": "y"}, + "ghp_ABCDEF1234567890": "mapping-key", + "private_key": "x", + } + ) + assert scrubbed["token"] == redaction.REDACTED + assert scrubbed["credentials"] == redaction.REDACTED + assert scrubbed["private_key"] == redaction.REDACTED + assert not any("ghp_" in key for key in scrubbed) + + +def test_unterminated_private_key_blocks_are_screened() -> None: + assert "PRIVATE KEY" not in redaction.redact("-----BEGIN PRIVATE KEY-----\nabc") + + +def test_screening_is_idempotent() -> None: + once = redaction.redact("api_key=AKIAIOSFODNN7EXAMPLE") + assert redaction.redact(once) == once + assert redaction.redact("") == "" + + +def test_normalise_key_strips_separators() -> None: + assert redaction.normalise_key("Client-Secret_v2") == "clientsecretv2" diff --git a/testing/tests/test_hermes_handoff_rules.py b/testing/tests/test_hermes_handoff_rules.py new file mode 100644 index 00000000..03d5785f --- /dev/null +++ b/testing/tests/test_hermes_handoff_rules.py @@ -0,0 +1,166 @@ +"""Contracts for the shared evaluator primitives and dispatch.""" + +from __future__ import annotations + +import pytest + +from testing.tests.test_hermes_handoff_support import ( + load_handoff_module, + outcome, + spec, + step, +) + +rules = load_handoff_module("hermes_handoff_rules") +model = load_handoff_module("hermes_handoff_model") +load_handoff_module("hermes_handoff_evaluators") + + +PAYLOAD = { + "a": {"b": [{"c": 1}, {"c": 2}]}, + "items": [{"number": 14, "merged": True}, {"number": 15, "merged": False}], + "flag": True, +} + + +@pytest.mark.parametrize( + ("path", "expected"), + [ + ("", PAYLOAD), + ("flag", True), + ("a.b[0].c", 1), + ("a.b[].c", [1, 2]), + ("a.b[]", [{"c": 1}, {"c": 2}]), + ("items[].number", [14, 15]), + ], +) +def test_dotted_paths_index_and_map(path: str, expected) -> None: + assert rules.dotted(PAYLOAD, path) == expected + + +@pytest.mark.parametrize( + "path", ["missing", "a.missing", "a.b[9].c", "flag[0]", "flag.b"] +) +def test_an_absent_path_raises_rather_than_returning_none(path: str) -> None: + with pytest.raises(KeyError): + rules.dotted(PAYLOAD, path) + + +def test_parse_json_rejects_raw_control_characters_and_empty_output() -> None: + """Only RFC-compliant, nonempty JSON is accepted as release evidence.""" + with pytest.raises(ValueError): + rules.parse_json(outcome(stdout='{"title": "a\nb"}')) + with pytest.raises(ValueError, match="empty output"): + rules.parse_json(outcome(stdout=" ")) + with pytest.raises(ValueError): + rules.parse_json(outcome(stdout="not json")) + + +def test_lines_drops_blank_rows_and_trims() -> None: + assert rules.lines(outcome(stdout=" a \n\n b\n")) == ["a", "b"] + + +def test_missing_steps_reports_the_first_step_that_produced_nothing() -> None: + built = spec("allowed", steps=(step("a"), step("b"))) + + never_run = rules.missing_steps(built, {}) + assert never_run.status == model.NOT_RUN + assert "never executed" in never_run.reason + + errored = rules.missing_steps( + built, {"a": outcome(error="timeout"), "b": outcome()} + ) + assert errored.status == model.NOT_RUN + assert "did not run" in errored.reason + + truncated = rules.missing_steps( + built, {"a": outcome(truncated=True), "b": outcome()} + ) + assert truncated.status == model.NOT_RUN and "truncated" in truncated.reason + + assert rules.missing_steps(built, {"a": outcome(), "b": outcome()}) is None + + +def test_an_optional_step_may_be_absent() -> None: + built = spec("allowed", steps=(step("a"), step("b", optional=True))) + assert rules.missing_steps(built, {"a": outcome()}) is None + + +@pytest.mark.parametrize( + ("stderr", "returncode", "denied"), + [ + ("Error from server (Forbidden): pods is forbidden", 1, True), + ("User cannot list resource", 1, True), + ("Gitea API returned HTTP 403", 1, True), + ("connection refused", 1, False), + ("", 0, False), + ], +) +def test_denial_detection_needs_a_refusal_not_merely_a_failure( + stderr, returncode, denied +) -> None: + assert rules.looks_denied(outcome(stderr=stderr, returncode=returncode)) is denied + + +def test_step_keys_selects_by_kind() -> None: + built = spec( + "denied", + steps=(step("r", kind=model.REVIEW), step("a", kind=model.ATTEMPT), step("x")), + ) + assert rules.step_keys(built, model.REVIEW) == ("r",) + assert rules.step_keys(built, model.ATTEMPT) == ("a",) + assert rules.step_keys(built, model.READ) == ("x",) + + +def test_parsed_step_returns_the_payload_or_the_skip_to_report() -> None: + built = spec("json_field", {"step": "s"}, steps=(step("s"),)) + + payload, key = rules.parsed_step(built, {"s": outcome(stdout='{"a": 1}')}) + assert (payload, key) == ({"a": 1}, "s") + + failed = rules.parsed_step(built, {"s": outcome(returncode=1)}) + assert failed.status == model.NOT_RUN and "failed" in failed.reason + + unparsable = rules.parsed_step(built, {"s": outcome(stdout="nope")}) + assert ( + unparsable.status == model.NOT_RUN + and "did not return JSON" in unparsable.reason + ) + + +def test_an_unknown_rule_is_not_run_rather_than_a_crash() -> None: + evaluation = rules.evaluate(spec("no-such-rule"), {}) + assert evaluation.status == model.NOT_RUN + assert "unknown evaluator" in evaluation.reason + + +def test_an_evaluator_that_raises_is_not_run_rather_than_a_pass() -> None: + """A harness bug must never be indistinguishable from a clean release.""" + + @rules.evaluator("explodes-for-test") + def _explode(_spec, _outcomes): + raise RuntimeError("boom") + + evaluation = rules.evaluate(spec("explodes-for-test"), {}) + + assert evaluation.status == model.NOT_RUN + assert "RuntimeError" in evaluation.reason and "boom" in evaluation.reason + del rules.EVALUATORS["explodes-for-test"] + + +def test_an_evaluator_returning_an_unknown_status_is_not_run() -> None: + @rules.evaluator("unknown-status-for-test") + def _unknown(_spec, _outcomes): + return rules.Evaluation("MAYBE", "bad") + + evaluation = rules.evaluate(spec("unknown-status-for-test"), {}) + assert evaluation.status == model.NOT_RUN + assert "unknown status" in evaluation.reason + del rules.EVALUATORS["unknown-status-for-test"] + + +def test_is_list_distinguishes_arrays_from_strings() -> None: + assert rules.is_list([1]) and rules.is_list(()) + assert not rules.is_list("text") + assert not rules.is_list(b"bytes") + assert not rules.is_list({"a": 1}) diff --git a/testing/tests/test_hermes_handoff_run.py b/testing/tests/test_hermes_handoff_run.py new file mode 100644 index 00000000..4d87fb3e --- /dev/null +++ b/testing/tests/test_hermes_handoff_run.py @@ -0,0 +1,496 @@ +"""Contracts for handoff acceptance orchestration: vantages, catalog, report.""" + +from __future__ import annotations + +import datetime as dt +import subprocess +from dataclasses import replace + +from testing.tests.test_hermes_handoff_support import ( + FakeClock, + FakeSpawn, + load_handoff_module, + step, +) + +harness_run = load_handoff_module("hermes_handoff_run") +catalog = load_handoff_module("hermes_handoff_catalog") +exec_module = load_handoff_module("hermes_handoff_exec") +model = load_handoff_module("hermes_handoff_model") + +NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc) +TARGETS = catalog.Targets(now=NOW) + + +def completed(stdout: str = "", stderr: str = "", returncode: int = 0): + return subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=stdout, stderr=stderr + ) + + +def whoami(username: str) -> object: + return completed(stdout=username) + + +def runner_for(results: list, **kwargs): + return exec_module.Runner( + clock=FakeClock(), + spawn=FakeSpawn(results), + environ={}, + deadline_seconds=3600, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + **kwargs, + ) + + +def check( + rule: str, + expect: dict | None = None, + steps=(), + identifier="test.check", + mandatory=True, +): + return model.CheckSpec( + id=identifier, + title="t", + group="g", + rule=rule, + steps=tuple(steps), + expect=expect or {}, + mandatory=mandatory, + ) + + +def test_the_shipped_catalog_is_valid_and_covers_every_group() -> None: + specs = harness_run.build_catalog(TARGETS) + assert harness_run.validate_catalog(specs) == [] + groups = {spec.group for spec in specs} + assert { + "baseline", + "identity", + "routing", + "scopes", + "access-denied", + "gitops", + } <= groups + assert { + "forge", + "nodes", + "build", + "reliability", + "pool", + "surfaces", + "ephemeral", + } <= groups + + +def test_catalog_validation_names_every_structural_problem() -> None: + broken = [ + check("no-such-rule", identifier="a.dup"), + check("allowed", identifier="a.dup"), + check("allowed", steps=(step("s", vantage="nowhere"),), identifier="b.vantage"), + check("allowed", steps=(step("s", kind="guess"),), identifier="c.kind"), + check( + "denied", steps=(step("s", kind=model.REVIEW),), identifier="d.review-only" + ), + model.CheckSpec( + id="e.impersonation", + title="t", + group="g", + rule="allowed", + steps=( + model.Step( + key="s", + vantage=catalog.OPERATOR, + argv=("kubectl", "get", "ns", "--as", "x"), + ), + ), + ), + ] + + problems = " | ".join(harness_run.validate_catalog(broken)) + + assert "duplicate check id" in problems + assert "unknown rule" in problems + assert "unknown vantage" in problems + assert "unknown kind" in problems + assert "needs a real attempt" in problems + assert "impersonates from the operator vantage" in problems + + +def test_target_validation_rejects_every_vacuous_or_unbound_release_input() -> None: + broken = replace( + TARGETS, + repo="atlas/other", + remote="upstream", + reviewed_pr_number=20, + reviewed_head_ref="other", + baseline_commit="bad", + remote_main_sha="bad", + reviewed_head_sha="bad", + build_sha="bad", + agent_image="latest", + deployment_revision="0", + node_count=1, + pool_replicas=1, + chat_config_revision="", + pool_worker_env=(), + dependency_heads=((14, "bad"), (14, "bad")), + max_evidence_age_seconds=0, + ) + problems = harness_run.validate_targets(broken) + assert len(problems) >= 12 + sha = "1" * 40 + valid = replace( + TARGETS, + remote_main_sha=sha, + reviewed_head_sha=sha, + build_sha=sha, + agent_image=f"registry.example/hermes:git-{sha}-build-7@sha256:{'2' * 64}", + deployment_revision="1", + chat_config_revision="rev", + dependency_heads=tuple( + (number, sha) for number in TARGETS.dependency_pull_requests + ), + ) + assert harness_run.validate_targets(valid) == [] + wrong_image_source = replace( + valid, + agent_image=f"registry.example/hermes:git-{'2' * 40}-build-7@sha256:{'2' * 64}", + ) + assert "image tag/digest" in " ".join( + harness_run.validate_targets(wrong_image_source) + ) + wrong_build = replace(valid, build_sha="2" * 40) + assert "build SHA" in " ".join(harness_run.validate_targets(wrong_build)) + malformed_heads = replace( + valid, + dependency_heads=tuple( + (number, "bad") for number in valid.dependency_pull_requests + ), + ) + assert "dependency heads" in " ".join(harness_run.validate_targets(malformed_heads)) + + +def test_identity_resolution_extracts_only_the_username() -> None: + runner = runner_for([whoami("system:serviceaccount:hermes:hermes-agent")]) + username, detail = harness_run.resolve_identity( + runner, exec_module.operator_vantage() + ) + assert (username, detail) == ("system:serviceaccount:hermes:hermes-agent", "") + + failed = harness_run.resolve_identity( + runner_for([completed(stderr="no access", returncode=1)]), + exec_module.operator_vantage(), + ) + assert failed[0] == "" and "no access" in failed[1] + + unparsable = harness_run.resolve_identity( + runner_for([completed(stdout="two names")]), exec_module.operator_vantage() + ) + assert unparsable[0] == "" and "no username" in unparsable[1] + + nameless = harness_run.resolve_identity( + runner_for([completed(stdout="")]), exec_module.operator_vantage() + ) + assert nameless[0] == "" and "no username" in nameless[1] + + +def test_vantages_resolve_to_running_pods_and_record_what_is_missing() -> None: + runner = runner_for( + [ + whoami("kubernetes-admin"), + completed(stdout="pod/hermes-agent-2\npod/hermes-agent-1\n"), + whoami("system:serviceaccount:hermes:hermes-agent"), + completed(stdout="pod/hermes-switchyard-1\n"), + completed(stderr="no pods", returncode=1), + completed(stdout="pod/hermes-chat-tenant-0\n"), + ] + ) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + by_name = {record.name: record for record in records} + + assert set(vantages) == { + catalog.OPERATOR, + catalog.SELF, + catalog.SWITCHYARD, + catalog.CHAT, + } + assert by_name[catalog.SELF].identity == "system:serviceaccount:hermes:hermes-agent" + assert "hermes-agent-1" in by_name[catalog.SELF].description # deterministic pick + assert by_name[catalog.NODE].available is False + assert by_name[catalog.CHAT].available is True + + +def test_an_unreachable_operator_stops_vantage_resolution_immediately() -> None: + runner = runner_for([completed(stderr="connection refused", returncode=1)]) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + + assert set(vantages) == {catalog.OPERATOR} + assert records[0].available is False + assert harness_run.vantage_problems(records)[0].startswith( + "the operator vantage is unavailable" + ) + + +def test_a_missing_chat_pod_is_recorded_rather_than_assumed_present() -> None: + runner = runner_for( + [ + whoami("kubernetes-admin"), + completed(stdout="pod/hermes-agent-1\n"), + whoami("system:serviceaccount:hermes:hermes-agent"), + completed(stdout="pod/hermes-switchyard-1\n"), + completed(stdout="pod/hermes-node-1\n"), + completed(stderr="pods 'hermes-chat-tenant-0' not found", returncode=1), + ] + ) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + chat = next(record for record in records if record.name == catalog.CHAT) + + assert catalog.CHAT not in vantages + assert chat.available is False and "not found" in chat.detail + + +def test_a_selector_that_matches_no_running_pod_leaves_the_vantage_absent() -> None: + runner = runner_for( + [ + whoami("kubernetes-admin"), + completed(stdout="\n"), + completed(stdout="\n"), + completed(stdout="\n"), + completed(stdout="\n"), + ] + ) + + vantages, records = harness_run.resolve_vantages(runner, TARGETS) + by_name = {record.name: record for record in records} + + assert set(vantages) == {catalog.OPERATOR} + assert "no running pod matches" in by_name[catalog.SELF].detail + + +def test_two_vantages_that_are_the_same_principal_invalidate_the_evidence() -> None: + """Corroboration needs two identities; one identity twice is a restatement.""" + records = [ + model.VantageRecord( + name=catalog.OPERATOR, description="d", identity="same", available=True + ), + model.VantageRecord( + name=catalog.SELF, description="d", identity="same", available=True + ), + model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True), + model.VantageRecord(name=catalog.CHAT, description="d", available=True), + ] + + problems = harness_run.vantage_problems(records) + + assert len(problems) == 1 + assert "same principal" in problems[0] + + +def test_vantage_problems_flag_a_missing_or_anonymous_self_probe() -> None: + operator = model.VantageRecord( + name=catalog.OPERATOR, description="d", identity="admin", available=True + ) + + absent = harness_run.vantage_problems([operator]) + assert "in-pod Hermes vantage is unavailable" in absent[0] + + anonymous = harness_run.vantage_problems( + [ + operator, + model.VantageRecord(name=catalog.SELF, description="d", available=True), + ] + ) + assert "did not report an identity" in anonymous[0] + + clean = harness_run.vantage_problems( + [ + operator, + model.VantageRecord( + name=catalog.SELF, description="d", identity="agent", available=True + ), + model.VantageRecord( + name=catalog.SWITCHYARD, description="d", available=True + ), + model.VantageRecord(name=catalog.CHAT, description="d", available=True), + ] + ) + assert clean == [] + + +def test_a_check_whose_vantage_is_absent_is_not_run() -> None: + spec = check("allowed", steps=(step("s", vantage=catalog.NODE),)) + + result = harness_run.run_check(runner_for([]), spec, {}) + + assert result.status == model.NOT_RUN + assert "vantage node is unavailable" in result.reason + assert result.outcomes[0].error == "vantage node is unavailable" + + +def test_validate_catalog_rejects_impersonation_from_every_vantage() -> None: + """The operator vantage was the only one guarded; now none is exempt.""" + for vantage in (catalog.OPERATOR, catalog.SELF, catalog.CHAT): + spec = check( + "stdout_matches", + {"step": "s", "equals": "x"}, + steps=( + model.Step( + key="s", + vantage=vantage, + argv=("kubectl", "get", "ns", "-o", "name", "--as", "system:admin"), + ), + ), + ) + problems = harness_run.validate_catalog([spec]) + assert any("impersonates" in problem for problem in problems) + assert any(vantage in problem for problem in problems) + + +def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None: + spec = check( + "stdout_matches", + {"step": "s", "equals": "locked"}, + steps=( + model.Step( + key="s", + vantage=catalog.OPERATOR, + argv=("kubectl", "version"), + record=False, + ), + ), + ) + runner = runner_for([completed(stdout="locked", stderr="noise")]) + + result = harness_run.run_check( + runner, spec, {catalog.OPERATOR: exec_module.operator_vantage()} + ) + + assert result.status == model.PASS + assert result.outcomes[0].stdout == harness_run.UNRECORDED + assert result.outcomes[0].stderr == harness_run.UNRECORDED + # Withholding bulk bytes must not withhold the binary that produced them. + recorded = result.outcomes[0] + assert recorded.executable_path + assert recorded.executable_sha256 + assert recorded.executable_path == "kubectl" + assert recorded.executable_sha256 == "a" * 64 + assert recorded.as_dict()["executable_sha256"] == recorded.executable_sha256 + + +def test_an_expired_deadline_leaves_the_report_saying_so() -> None: + clock = FakeClock() + runner = exec_module.Runner( + clock=clock, + spawn=FakeSpawn([]), + environ={}, + deadline_seconds=1.0, + attestor=lambda command, _environment: exec_module.Attestation( + command, "a" * 64 + ), + ) + clock.advance(5.0) + spec = check("allowed", steps=(step("s", vantage=catalog.OPERATOR),)) + + report = harness_run.build_report( + runner, + TARGETS, + [spec], + {catalog.OPERATOR: exec_module.operator_vantage()}, + [], + "read-only", + "2026-08-17T00:00:00Z", + ) + + assert report.results[0].status == model.NOT_RUN + assert any("deadline expired" in error for error in report.harness_errors) + assert report.decision == model.NO_GO + + +def test_the_report_carries_the_configured_baseline_and_sorts_its_checks() -> None: + runner = runner_for([completed(stdout="ok"), completed(stdout="ok")]) + specs = [ + check( + "allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="z.last" + ), + check( + "allowed", + steps=(step("s", vantage=catalog.OPERATOR),), + identifier="a.first", + ), + ] + records = [ + model.VantageRecord( + name=catalog.OPERATOR, description="d", identity="admin", available=True + ), + model.VantageRecord( + name=catalog.SELF, description="d", identity="agent", available=True + ), + model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True), + model.VantageRecord(name=catalog.CHAT, description="d", available=True), + ] + + report = harness_run.build_report( + runner, + TARGETS, + specs, + {catalog.OPERATOR: exec_module.operator_vantage()}, + records, + "read-only", + "2026-08-17T00:00:00Z", + ) + + assert [result.spec.id for result in report.results] == ["a.first", "z.last"] + assert report.baseline["baseline_commit"] == TARGETS.baseline_commit + assert report.baseline["dependency_pull_requests"] == list( + TARGETS.dependency_pull_requests + ) + assert report.decision == model.GO + assert report.finished_at + + +def test_armed_results_replace_the_matching_catalog_placeholders() -> None: + placeholder = check( + "not_armed", identifier="ephemeral.feature-branch-push", mandatory=False + ) + armed = model.CheckResult( + spec=model.CheckSpec( + id="ephemeral.feature-branch-push", + title="t", + group="ephemeral", + rule="armed", + ), + status=model.PASS, + reason="pushed", + ) + + report = harness_run.build_report( + runner_for([]), + TARGETS, + [placeholder], + {}, + [], + "ephemeral-armed", + "2026-08-17T00:00:00Z", + [armed], + ) + + assert len(report.results) == 1 + assert report.results[0].status == model.PASS + + +def test_an_unstartable_run_reports_no_go_with_a_named_reason() -> None: + report = harness_run.unavailable_report( + "read-only", "2026-08-17T00:00:00Z", "arming refused: bad phrase", TARGETS + ) + + assert report.decision == model.NO_GO + assert report.results[0].spec.id == "harness.startup" + assert report.results[0].status == model.NOT_RUN + assert report.harness_errors == ["arming refused: bad phrase"] diff --git a/testing/tests/test_hermes_handoff_support.py b/testing/tests/test_hermes_handoff_support.py new file mode 100644 index 00000000..f62ead15 --- /dev/null +++ b/testing/tests/test_hermes_handoff_support.py @@ -0,0 +1,114 @@ +"""Shared loader and fixtures for the handoff acceptance harness tests. + +The harness lives in ``scripts/ops`` as sibling modules rather than an installed +package, so tests import it the way the entry point does: by putting that +directory on the path once and importing by name. +""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +OPS = REPO_ROOT / "scripts" / "ops" + +if str(OPS) not in sys.path: + sys.path.insert(0, str(OPS)) + + +def load_handoff_module(name: str) -> Any: + """Import one harness module by name.""" + return importlib.import_module(name) + + +exec_module = load_handoff_module("hermes_handoff_exec") +model = load_handoff_module("hermes_handoff_model") + + +def outcome( + stdout: str = "", + stderr: str = "", + returncode: int | None = 0, + error: str | None = None, + truncated: bool = False, + vantage: str = "self", + argv: tuple[str, ...] = ("kubectl", "get", "pods"), +) -> Any: + """Return a synthetic command outcome for evaluator tests.""" + return exec_module.Outcome( + argv=argv, + vantage=vantage, + returncode=returncode, + stdout=stdout, + stderr=stderr, + error=error, + truncated=truncated, + ) + + +def spec( + rule: str, + expect: dict | None = None, + steps: tuple = (), + identifier: str = "test.check", + mandatory: bool = True, + scope: str | None = None, +) -> Any: + """Return a synthetic check specification.""" + return model.CheckSpec( + id=identifier, + title="test check", + group="test", + rule=rule, + steps=steps, + expect=expect or {}, + mandatory=mandatory, + scope=scope or model.ALWAYS, + ) + + +def step( + key: str, kind: str | None = None, vantage: str = "self", optional: bool = False +) -> Any: + """Return a synthetic step.""" + return model.Step( + key=key, + vantage=vantage, + argv=("kubectl", "get", "pods", "-o", "name"), + kind=kind or model.READ, + optional=optional, + ) + + +class FakeClock: + """A monotonic clock the tests advance by hand.""" + + def __init__(self, start: float = 0.0) -> None: + self.now = start + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + """Move the clock forward.""" + self.now += seconds + + +class FakeSpawn: + """A ``subprocess.run`` stand-in driven by a queue of scripted results.""" + + def __init__(self, results: list) -> None: + self.results = list(results) + self.calls: list[dict] = [] + + def __call__(self, argv, **kwargs): + self.calls.append({"argv": list(argv), **kwargs}) + if not self.results: + raise AssertionError(f"unexpected command: {argv}") + outcome_or_error = self.results.pop(0) + if isinstance(outcome_or_error, Exception): + raise outcome_or_error + return outcome_or_error diff --git a/testing/tests/test_hermes_handoff_zero_evidence.py b/testing/tests/test_hermes_handoff_zero_evidence.py new file mode 100644 index 00000000..8e47768a --- /dev/null +++ b/testing/tests/test_hermes_handoff_zero_evidence.py @@ -0,0 +1,243 @@ +"""Regressions for the zero-evidence fail-open in the absence checks. + +``evaluate_names_absent`` used to return ``PASS`` when its step exited 0 with no +output, so five mandatory checks — the ones asserting that provider API keys, +forge credentials, cluster-admin bindings, and shared coordinator state are +*absent* — could report a pass on no evidence at all and turn a ``NO_GO`` into a +``GO``. These tests pin the real catalog specs, not synthetic look-alikes, and +reproduce both ways silence actually arrives: a POSIX pipeline whose exit status +comes from its last stage, and a ``kubectl -o jsonpath`` over drifted structure. +""" + +from __future__ import annotations + +import datetime +import shutil +import subprocess + +import pytest + +from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome + +catalog = load_handoff_module("hermes_handoff_catalog") +evaluators = load_handoff_module("hermes_handoff_evaluators") +harness_run = load_handoff_module("hermes_handoff_run") +model = load_handoff_module("hermes_handoff_model") +policy = load_handoff_module("hermes_handoff_policy") + +# Every mandatory check whose whole claim is that something is not there. +ABSENCE_CHECKS = ( + "access.no-cluster-admin-binding-for-agent", + "forge.worker-holds-no-git-credential-variable", + "identity.no-provider-api-key-in-manifest", + "identity.no-provider-api-key-in-pod", + "pool.coordinator-retains-sole-state-ownership", +) + + +def real_catalog() -> dict: + targets = catalog.Targets(now=datetime.datetime.now(datetime.UTC)) + return {spec.id: spec for spec in harness_run.build_catalog(targets)} + + +def test_the_five_absence_checks_are_exactly_the_mandatory_names_absent_set() -> None: + """If a sixth absence check appears it must be covered here too.""" + specs = real_catalog() + found = { + identifier + for identifier, spec in specs.items() + if spec.rule == "names_absent" and spec.mandatory + } + assert found == set(ABSENCE_CHECKS) + + +@pytest.mark.parametrize("identifier", ABSENCE_CHECKS) +def test_a_real_absence_check_never_passes_on_zero_observations( + identifier: str, +) -> None: + """rc=0 with no lines is drift, so the verdict must be NOT_RUN, not PASS.""" + spec = real_catalog()[identifier] + step_key = spec.expect["step"] + + silent = evaluators.evaluate(spec, {step_key: outcome(stdout="")}) + assert silent.status == model.NOT_RUN + assert "no lines" in silent.reason + assert silent.status != model.PASS + + whitespace = evaluators.evaluate(spec, {step_key: outcome(stdout="\n \n\t\n")}) + assert whitespace.status == model.NOT_RUN + + +@pytest.mark.parametrize("identifier", ABSENCE_CHECKS) +def test_a_real_absence_check_still_passes_on_real_evidence(identifier: str) -> None: + """The guard must not turn a healthy observation into a false NOT_RUN.""" + spec = real_catalog()[identifier] + step_key = spec.expect["step"] + benign = "clusterrolebinding.rbac.authorization.k8s.io/system:basic-user\nPATH\n" + + passed = evaluators.evaluate(spec, {step_key: outcome(stdout=benign)}) + assert passed.status == model.PASS + assert passed.evidence["observed_count"] == 2 + + +@pytest.mark.parametrize("identifier", ABSENCE_CHECKS) +def test_a_real_absence_check_still_fails_on_a_forbidden_name(identifier: str) -> None: + spec = real_catalog()[identifier] + step_key = spec.expect["step"] + offender = next( + iter(spec.expect.get("names", ()) or spec.expect.get("contains", ())) + ) + + verdict = evaluators.evaluate( + spec, {step_key: outcome(stdout=f"OTHER\n{offender}")} + ) + assert verdict.status == model.FAIL + assert offender in verdict.reason + + +def test_zero_evidence_on_a_mandatory_absence_check_forces_no_go() -> None: + """A NOT_RUN on a mandatory check must not be rounded up to a GO. + + Every other check is held at PASS so the verdict turns on the five absence + checks alone: silence must cost the release its GO. + """ + specs = real_catalog() + silent, green = [], [] + for identifier, spec in specs.items(): + status = model.PASS + if identifier in ABSENCE_CHECKS: + evaluation = evaluators.evaluate( + spec, {spec.expect["step"]: outcome(stdout="")} + ) + assert evaluation.status == model.NOT_RUN + status = evaluation.status + silent.append(model.CheckResult(spec=spec, status=status)) + green.append(model.CheckResult(spec=spec, status=model.PASS)) + + assert all(result.blocking for result in silent if result.status == model.NOT_RUN) + assert model.Report(mode="read-only", started_at="t", results=silent).decision == ( + model.NO_GO + ) + assert ( + model.Report(mode="read-only", started_at="t", results=green).decision + == model.GO + ) + + +def test_a_broken_shell_pipeline_exits_zero_with_no_output() -> None: + """The harness' own frozen template is reachable with rc=0 and 0 bytes. + + A POSIX pipeline reports the status of its *last* stage, so a missing or + renamed upstream binary produces success-with-no-evidence rather than an + error. This is one of the two live paths that made the fail-open reachable. + """ + script = policy.render_shell("env_names").replace("/usr/bin/env", "/usr/bin/absent") + + completed = subprocess.run( # noqa: S603 + ["/bin/sh", "-c", script], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + assert completed.returncode == 0 + assert completed.stdout == "" + + spec = real_catalog()["identity.no-provider-api-key-in-pod"] + verdict = evaluators.evaluate( + spec, {"env": outcome(stdout=completed.stdout, returncode=completed.returncode)} + ) + assert verdict.status == model.NOT_RUN + + +DRIFTED_JSONPATH_OUTPUT = { + # A renamed parent — `containerz` — makes kubectl print nothing at all. + "renamed parent": "", + # A renamed leaf — `.nam` — makes it print only the separators. + "renamed leaf": "\n" * 6, + # A container filter that matches nothing behaves the same way. + "unmatched filter": "", +} + + +@pytest.mark.parametrize("shape", sorted(DRIFTED_JSONPATH_OUTPUT)) +def test_drifted_kubectl_jsonpath_output_is_not_run(shape: str) -> None: + """The exact bytes a drifted ``-o jsonpath`` emits must never read as a pass. + + ``kubectl -o jsonpath`` exits 0 when the structure it walks is missing, so a + renamed field, a renamed leaf, or an unmatched container filter all deliver + success with no usable evidence. Pinned as literal output so the regression + holds with no cluster and no network; the live rc=0 behaviour is asserted + separately by :func:`test_a_live_drifted_projection_still_exits_zero`. + """ + spec = real_catalog()["identity.no-provider-api-key-in-manifest"] + + verdict = evaluators.evaluate( + spec, {"env": outcome(stdout=DRIFTED_JSONPATH_OUTPUT[shape])} + ) + + assert verdict.status == model.NOT_RUN + assert "no lines" in verdict.reason + + +def test_a_live_drifted_projection_still_exits_zero() -> None: + """Confirm against the real tool that drift is success, not an error. + + kubectl has no offline JSONPath renderer, so this reads the live agent + workload — a plain, read-only ``get`` with a name-only projection — and skips + when no cluster is reachable. The hermetic assertion above does not skip. + """ + kubectl = shutil.which("kubectl") + if kubectl is None: # pragma: no cover - kubectl ships with the release image + pytest.skip("kubectl is not installed in this environment") + base = ["--namespace", "hermes", "get", "deploy/hermes-agent", "-o"] + healthy = 'jsonpath={range .spec.template.spec.containers[*]}{.name}{"\\n"}{end}' + probe = subprocess.run( # noqa: S603 + [kubectl, *base, healthy], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + if probe.returncode != 0: + pytest.skip("no read-only cluster access in this environment") + assert probe.stdout.strip() + + for drifted in ( + 'jsonpath={range .spec.template.spec.containerz[*]}{.name}{"\\n"}{end}', + 'jsonpath={range .spec.template.spec.containers[*]}{.nam}{"\\n"}{end}', + ): + result = subprocess.run( # noqa: S603 + [kubectl, *base, drifted], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert result.returncode == 0 + assert result.stdout.strip() == "" + spec = real_catalog()["identity.no-provider-api-key-in-manifest"] + verdict = evaluators.evaluate(spec, {"env": outcome(stdout=result.stdout)}) + assert verdict.status == model.NOT_RUN + + +def test_every_absence_probe_yields_one_line_per_observed_item() -> None: + """A healthy probe must produce evidence, or the guard would misfire. + + The pool claim projection emits ``=`` so a template volume + without a PVC still counts as an observation; an empty result therefore only + happens when the object or the projection has drifted. + """ + spec = real_catalog()["pool.coordinator-retains-sole-state-ownership"] + argv = spec.steps[0].argv + assert '{.name}{"="}' in argv[-1] + + verdict = evaluators.evaluate(spec, {"claims": outcome(stdout="tmp=\nconfig=\n")}) + assert verdict.status == model.PASS + assert verdict.evidence["observed_count"] == 2 + + leaked = evaluators.evaluate( + spec, {"claims": outcome(stdout="state=hermes-agent-home\n")} + ) + assert leaked.status == model.FAIL diff --git a/testing/tests/test_quality_contract.py b/testing/tests/test_quality_contract.py index f7bf38b7..134485bf 100644 --- a/testing/tests/test_quality_contract.py +++ b/testing/tests/test_quality_contract.py @@ -18,6 +18,21 @@ def test_bundled_contract_exposes_local_and_jenkins_profiles(): assert contract["pytest_suites"]["unit"]["paths"] +def test_handoff_modules_are_exactly_managed_linted_and_covered(): + """The canonical contract must not silently drop a handoff module.""" + contract = load_contract() + expected = { + path.as_posix() for path in Path("scripts/ops").glob("hermes_handoff_*.py") + } + assert len(expected) == 16 + assert expected <= set(contract["managed_modules"]) + assert expected <= set(contract["lint_paths"]) + assert expected <= set(contract["coverage"]["tracked_files"]) + assert expected <= set(contract["coverage"]["branch_tracked_files"]) + assert "scripts/ops" in contract["pytest_suites"]["unit"]["coverage_sources"] + assert "scripts/ops/hermes_handoff_*.py" in contract["hygiene"]["line_limit_globs"] + + def test_docs_check_reports_missing_docstring_and_missing_path(tmp_path: Path): module_path = tmp_path / "managed.py" module_path.write_text("value = 1\n", encoding="utf-8") @@ -43,7 +58,10 @@ def test_docs_check_reports_missing_required_doc_metadata(tmp_path: Path): (tmp_path / "README.md").write_text("", encoding="utf-8") contract = { - "required_docs": [{"path": "README.md", "description": ""}, {"path": "missing.md", "description": "Missing"}], + "required_docs": [ + {"path": "README.md", "description": ""}, + {"path": "missing.md", "description": "Missing"}, + ], "managed_modules": [], "lint_paths": [], "pytest_suites": {"unit": {"description": "", "paths": []}}, @@ -62,6 +80,7 @@ def test_docs_check_reports_missing_required_doc_metadata(tmp_path: Path): def test_hygiene_check_enforces_line_limit_and_name_rules(tmp_path: Path): tests_dir = tmp_path / "tests" tests_dir.mkdir() + (tests_dir / "conftest.py").write_text("value = 1\n", encoding="utf-8") bad_name = tests_dir / "bad-name.py" bad_name.write_text("x = 1\n", encoding="utf-8") long_file = tests_dir / "test_too_long.py" @@ -84,7 +103,9 @@ def test_hygiene_check_enforces_line_limit_and_name_rules(tmp_path: Path): issues = run_hygiene_check(contract, tmp_path) assert any("file exceeds 3 LOC" in issue for issue in issues) - assert any("naming rule failed" in issue and "bad-name.py" in issue for issue in issues) + assert any( + "naming rule failed" in issue and "bad-name.py" in issue for issue in issues + ) def test_coverage_check_enforces_per_file_floor(tmp_path: Path): @@ -126,9 +147,9 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path): def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Path): missing_xml = tmp_path / "missing.xml" - assert run_coverage_check({"coverage": {"tracked_files": []}}, tmp_path, missing_xml) == [ - "coverage xml missing: missing.xml" - ] + assert run_coverage_check( + {"coverage": {"tracked_files": []}}, tmp_path, missing_xml + ) == ["coverage xml missing: missing.xml"] source_dir = tmp_path / "pkg" source_dir.mkdir() diff --git a/testing/tests/test_quality_coverage_helpers.py b/testing/tests/test_quality_coverage_helpers.py index 7e6d90f3..0dc038c0 100644 --- a/testing/tests/test_quality_coverage_helpers.py +++ b/testing/tests/test_quality_coverage_helpers.py @@ -12,10 +12,15 @@ def test_compute_workspace_line_coverage_handles_missing_xml(tmp_path: Path) -> """Missing coverage XML should produce a zero workspace coverage score.""" contract = {"coverage": {"tracked_files": ["managed.py"]}} - assert compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") == 0.0 + assert ( + compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") + == 0.0 + ) -def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path: Path) -> None: +def test_compute_workspace_line_coverage_averages_present_tracked_files( + tmp_path: Path, +) -> None: """Workspace coverage should average only tracked files that appear in the report.""" coverage_xml = tmp_path / "coverage.xml" @@ -41,7 +46,9 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path assert compute_workspace_line_coverage(contract, tmp_path, coverage_xml) == 75.0 -def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: Path) -> None: +def test_run_check_keeps_relative_names_when_source_roots_do_not_match( + tmp_path: Path, +) -> None: """Relative filenames should remain relative when no declared source root contains them.""" coverage_xml = tmp_path / "coverage.xml" @@ -76,6 +83,19 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: assert issues == ["line coverage below 95.0%: relative.py (80.0%)"] +def test_workspace_coverage_returns_zero_when_xml_has_no_tracked_samples( + tmp_path: Path, +) -> None: + xml = tmp_path / "coverage.xml" + xml.write_text("", encoding="utf-8") + assert ( + compute_workspace_line_coverage( + {"coverage": {"tracked_files": ["missing.py"]}}, tmp_path, xml + ) + == 0.0 + ) + + def test_run_check_fails_a_file_below_the_branch_floor(tmp_path: Path) -> None: """Full line coverage must never hide a file whose branch coverage is weak.""" diff --git a/testing/tests/test_semgrep_report.py b/testing/tests/test_semgrep_report.py index 7c5e16ab..f8e9e640 100644 --- a/testing/tests/test_semgrep_report.py +++ b/testing/tests/test_semgrep_report.py @@ -100,6 +100,14 @@ def test_build_sonar_issues_uses_safe_defaults_for_sparse_findings() -> None: assert issue["primaryLocation"]["textRange"] == {"startLine": 1, "endLine": 1} +def test_sonar_issue_never_ends_before_its_start_line() -> None: + issue = semgrep_report._sonar_issue_from_finding( + {"path": "app.py", "start": {"line": 10}, "end": {"line": 2}} + ) + assert issue is not None + assert issue["primaryLocation"]["textRange"] == {"startLine": 10, "endLine": 10} + + def test_build_sonar_issues_clamps_inverted_line_ranges() -> None: """An end line before the start line should collapse to a valid range.""" @@ -131,7 +139,9 @@ def test_read_json_handles_invalid_json_and_non_object(tmp_path: Path) -> None: list_json = tmp_path / "list.json" list_json.write_text("[]", encoding="utf-8") - assert semgrep_report._read_json(bad_json)["errors"][0]["message"].startswith("invalid JSON:") + assert semgrep_report._read_json(bad_json)["errors"][0]["message"].startswith( + "invalid JSON:" + ) assert semgrep_report._read_json(list_json) == { "errors": [{"message": "report payload is not an object"}], "results": [], @@ -160,16 +170,25 @@ def test_main_handles_missing_or_invalid_input(tmp_path: Path) -> None: payload = json.loads(output.read_text(encoding="utf-8")) assert payload["status"] == "failed" assert payload["errors_total"] == 1 - sonar_payload = json.loads((tmp_path / "build" / "sonar-issues.json").read_text(encoding="utf-8")) + sonar_payload = json.loads( + (tmp_path / "build" / "sonar-issues.json").read_text(encoding="utf-8") + ) assert sonar_payload == {"issues": []} -def test_main_honors_custom_blocking_severity_without_sonar_output(tmp_path: Path) -> None: +def test_main_honors_custom_blocking_severity_without_sonar_output( + tmp_path: Path, +) -> None: """CLI callers can choose stricter blocking severities without writing Sonar issues.""" semgrep_json = tmp_path / "semgrep.json" semgrep_json.write_text( - json.dumps({"results": [{"path": "app.py", "extra": {"severity": "WARNING"}}], "errors": []}), + json.dumps( + { + "results": [{"path": "app.py", "extra": {"severity": "WARNING"}}], + "errors": [], + } + ), encoding="utf-8", ) output = tmp_path / "semgrep-report.json"