Compare commits
16 Commits
d1e8cda5ed
...
9749e65042
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9749e65042 | ||
|
|
c8bb23ccd3 | ||
|
|
f83c59a1c9 | ||
|
|
2b2c8bda14 | ||
|
|
a9ee1070be | ||
|
|
943ecfd15b | ||
|
|
02e559d7ac | ||
|
|
d208a89d9d | ||
|
|
ed43dbaa7f | ||
|
|
fcc8f650c2 | ||
|
|
c943ad4537 | ||
|
|
c867ae52fb | ||
|
|
c808baff40 | ||
|
|
9d5214f809 | ||
| 8f00545828 | |||
|
|
465d386da5 |
@ -111,7 +111,7 @@ RUN /opt/hermes/.venv/bin/python -c 'import cryptography, yaml' \
|
||||
&& grep -Fq "capability.provider!=='local_command'" /opt/hermes-webui/static/atlas-voice.js \
|
||||
&& grep -Fq 'data-priority="maximum"' /opt/hermes-webui/static/index.html \
|
||||
&& grep -Fq 'routing_priority:priority' /opt/hermes-webui/static/atlas-router.js \
|
||||
&& grep -Fq "'atlas/auto/fast':'AUTO · Fast'" /opt/hermes-webui/static/atlas-router.js \
|
||||
&& grep -Fq "'atlas/auto/fast':'Automatic · Fast'" /opt/hermes-webui/static/atlas-router.js \
|
||||
&& grep -Fq 'explicit_reasoning_effort' /opt/hermes-webui/api/gateway_chat.py \
|
||||
&& /opt/hermes/.venv/bin/python -m py_compile \
|
||||
/opt/hermes-webui/api/routes.py \
|
||||
|
||||
@ -23,7 +23,7 @@ replace_once(
|
||||
''' <div class="composer-ws-wrap">\n''',
|
||||
''' <div class="composer-routing-wrap" id="composerRoutingWrap">
|
||||
<button class="composer-routing-chip" id="composerRoutingChip" type="button" title="Automatic routing priority" aria-haspopup="true" aria-expanded="false" aria-controls="composerRoutingDropdown">
|
||||
<span class="composer-routing-label" id="composerRoutingLabel">AUTO</span>
|
||||
<span class="composer-routing-label" id="composerRoutingLabel">Automatic</span>
|
||||
<span class="composer-routing-chevron" aria-hidden="true">⌄</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
@ -2,14 +2,14 @@
|
||||
'use strict';
|
||||
|
||||
const PRIORITIES=new Set(['auto','fast','balanced','deep','maximum']);
|
||||
const LABELS={auto:'AUTO',fast:'FAST',balanced:'BALANCED',deep:'DEEP',maximum:'MAXIMUM'};
|
||||
const LABELS={auto:'Automatic',fast:'Fast',balanced:'Balanced',deep:'Deep',maximum:'Maximum'};
|
||||
const STORAGE_KEY='atlas.hermes.routing-priority';
|
||||
const EXPLICIT_EFFORT_KEY='atlas.hermes.explicit-reasoning';
|
||||
const ROUTE_LABELS={
|
||||
'atlas/auto/fast':'AUTO · Fast',
|
||||
'atlas/auto/balanced':'AUTO · Balanced',
|
||||
'atlas/auto/deep':'AUTO · Deep',
|
||||
'atlas/auto/maximum':'AUTO · Maximum',
|
||||
'atlas/auto/fast':'Automatic · Fast',
|
||||
'atlas/auto/balanced':'Automatic · Balanced',
|
||||
'atlas/auto/deep':'Automatic · Deep',
|
||||
'atlas/auto/maximum':'Automatic · Maximum',
|
||||
'atlas/manual/codex/luna':'Codex · Luna',
|
||||
'atlas/manual/codex/terra':'Codex · Terra',
|
||||
'atlas/manual/codex/sol':'Codex · SOL',
|
||||
|
||||
341
docs/hermes_full_handoff_acceptance.md
Normal file
341
docs/hermes_full_handoff_acceptance.md
Normal file
@ -0,0 +1,341 @@
|
||||
# Hermes full-handoff acceptance and release runbook
|
||||
|
||||
`scripts/ops/hermes_handoff_acceptance.py` is a fail-closed release gate. It
|
||||
contains 63 checks in 15 groups. An empty catalog, an unknown result status, a
|
||||
mandatory `FAIL` or `NOT_RUN`, a harness error, or incomplete evidence makes the
|
||||
decision `NO_GO`. The four ephemeral checks are non-mandatory `NOT_RUN` in the
|
||||
default mode and are replaced with mandatory results only in armed mode.
|
||||
|
||||
This runbook describes evidence collection. It does not authorize a merge,
|
||||
publish, deployment, Flux reconcile, credential read, or live cluster mutation.
|
||||
Human review of PR #19 and of the resulting report is required.
|
||||
|
||||
## Required release inputs
|
||||
|
||||
Every value that can go stale must be supplied explicitly. Resolve `main`, PR
|
||||
heads, the image digest, build source SHA, deployment revision, and chat config
|
||||
revision from their authoritative read-only sources immediately before the run.
|
||||
Do not copy the snapshot in the dependency section into a later run.
|
||||
|
||||
```sh
|
||||
scripts/ops/hermes_handoff_acceptance.py \
|
||||
--context atlas-operator \
|
||||
--remote-main-sha "$REMOTE_MAIN_SHA" \
|
||||
--reviewed-head-sha "$REVIEWED_PR19_HEAD" \
|
||||
--agent-image "$AGENT_GIT_SHA_BUILD_TAG_AT_SHA256" \
|
||||
--build-sha "$IMAGE_BUILD_SOURCE_SHA" \
|
||||
--deployment-revision "$RUNNING_DEPLOYMENT_REVISION" \
|
||||
--chat-config-revision "$RUNNING_CHAT_CONFIG_REVISION" \
|
||||
--dependency-head "12=$PR12_HEAD" \
|
||||
--dependency-head "14=$PR14_HEAD" \
|
||||
--dependency-head "15=$PR15_HEAD" \
|
||||
--dependency-head "16=$PR16_HEAD" \
|
||||
--dependency-head "17=$PR17_HEAD" \
|
||||
--dependency-head "18=$PR18_HEAD" \
|
||||
--dependency-head "20=$PR20_HEAD" \
|
||||
--output build/hermes-handoff-acceptance.json
|
||||
```
|
||||
|
||||
The fixed topology is three pool workers on three distinct nodes. `--node-count`
|
||||
and `--pool-replicas` cannot lower that requirement. Pool node/ordinal assignment,
|
||||
Telegram session continuity, and the chat revision are mandatory. Deliberate Flux
|
||||
suspensions must be named with repeatable `--expected-suspension` arguments.
|
||||
|
||||
Exit status is `0` only for `GO`, `1` for `NO_GO`, and `2` for invalid execution
|
||||
bounds. JSON is written to `--output` or stdout; the human summary goes to stderr
|
||||
unless `--json-only` is passed.
|
||||
|
||||
## Structural safety boundary
|
||||
|
||||
The catalog is validated before a `Runner` exists. Invalid inputs, an empty or
|
||||
unsafe catalog, and an invalid armed preflight therefore create no subprocess.
|
||||
The command policy is checked both before and after executable resolution.
|
||||
|
||||
Default mode permits a narrow read grammar:
|
||||
|
||||
- Kubernetes reads use `name` or reviewed JSONPath projections. Secret resources,
|
||||
raw API reads, full JSON/YAML, `describe`, `cluster-info dump`, credential-bearing
|
||||
projections, `auth reconcile`, config operations, all mutations, and every
|
||||
dry-run mutation trick are rejected before spawn.
|
||||
- Git is read-only. `fetch`, configuration/helper overrides, alternate worktrees,
|
||||
external diff helpers, and push are rejected.
|
||||
- Gitea paths are pinned to `atlas/titan-iac` on an exact segment boundary, so a
|
||||
look-alike repository such as `atlas/titan-iac-evil` is refused, and any
|
||||
relative segment — including a percent-encoded `%2e%2e` — is rejected before
|
||||
the request is built.
|
||||
- Shell commands must be rendered from frozen templates, use fixed executable
|
||||
paths, and cannot name service-account, Vault, runtime-access, SSH, or other
|
||||
credential roots.
|
||||
- Impersonation is structurally impossible. `--as`, `--as-group`, and `--as-uid`
|
||||
are refused for every argv, in every mode, from every vantage, and the inner
|
||||
command of a `kubectl exec` is re-checked under the same rule rather than being
|
||||
granted an exemption. `validate_catalog` refuses an impersonating step from any
|
||||
vantage, not only the operator. A future audited self-probe that genuinely needs
|
||||
impersonation is a reviewed code change, not a configuration flag.
|
||||
- The binary allowlist is exactly the set with pinned release digests:
|
||||
`kubectl`, `git`, `sh`, `hermes`, and the Gitea client. `flux` and `helm` are
|
||||
not on it; Flux and Helm status is read through `kubectl` instead, so no
|
||||
allowlisted binary can be admitted that the executor would refuse to attest.
|
||||
- `PATH` is replaced with a fixed path. Every outer executable is resolved to an
|
||||
allowed real path, checked for unsafe write mode, and compared with a pinned
|
||||
SHA-256 digest. A caller-authored `kubectl`, `git`, or helper cannot fabricate
|
||||
evidence.
|
||||
|
||||
The executor uses a new process group, drains stdout and stderr concurrently,
|
||||
and retains at most the configured bytes while the child is running. A monotonic
|
||||
absolute timeout terminates and reaps the group, including descendants that keep
|
||||
pipes open. Timeout, deadline, and output-byte values reject booleans,
|
||||
zero/negative values, non-finite numbers, and values outside fixed bounds. There
|
||||
is no concurrency knob: `run_catalog` is strictly sequential and says so rather
|
||||
than accepting a bound it would ignore.
|
||||
Truncation discards the affected stream and makes every evaluator `NOT_RUN`.
|
||||
Bulk-evidence steps have their bytes replaced before they are recorded, but they
|
||||
keep the resolved executable path and its attested SHA-256, so withholding output
|
||||
never withholds the provenance of the binary that produced it.
|
||||
|
||||
Credential screening runs before truncation. Command, output, error, identity,
|
||||
reason, evidence, JSON, and human-summary fields are screened recursively,
|
||||
including mapping keys and short/numeric sensitive values. The complete rendered
|
||||
report is screened again; a remaining credential shape is replaced by a bounded
|
||||
`NO_GO` stub rather than being written. No credential prefix is retained at a
|
||||
truncation boundary.
|
||||
|
||||
## Vantages and evidence
|
||||
|
||||
The operator must use an external read-only kubeconfig. The self probe runs in
|
||||
the actual Hermes agent pod. Both identities are obtained with
|
||||
`kubectl auth whoami`; equal or missing principals make the run `NO_GO`. Required
|
||||
Switchyard and chat/Telegram pod vantages must also resolve. Node coverage is
|
||||
derived from all daemonset pod node names, not one selected pod.
|
||||
|
||||
Dangerous Kubernetes permissions are checked only with side-effect-free
|
||||
`kubectl auth can-i` reviews. The default catalog never constructs a Secret,
|
||||
TokenRequest, mutation, attach, port-forward, or arbitrary exec request. The one
|
||||
forge administrative denial check performs a read-only GET and accepts only an
|
||||
explicit authorization refusal; generic `404`, `not found`, and `no route` text
|
||||
are not denial evidence.
|
||||
|
||||
An absence is only evidence when the probe observed something. Both name rules
|
||||
resolve their step through one guard, so a step that exits `0` with no lines — or
|
||||
with only separators — is `NOT_RUN`, never `PASS`. This is reachable in practice
|
||||
two ways, and both are pinned by regressions: a POSIX pipeline reports the status
|
||||
of its *last* stage, so a missing upstream binary in the frozen `env_names`
|
||||
template exits `0` with no bytes; and `kubectl -o jsonpath` exits `0` when the
|
||||
structure it walks has drifted, whether a parent field, a leaf field, or a
|
||||
container filter. Because all five absence checks are mandatory, silence costs
|
||||
the release its `GO`. The pool claim projection emits one `<volume>=<claim>` line
|
||||
per template volume so a volume without a PVC still counts as an observation.
|
||||
|
||||
The strict evaluators reject duplicate-key JSON, raw control characters,
|
||||
`NaN`/infinity, boolean/number equality coercion, future timestamps, stale
|
||||
provider evidence, and every malformed routing or Flux row. Routing freshness is
|
||||
required separately for each provider, lane, and effort. Flux objects must be
|
||||
non-suspended, current at `observedGeneration`, and explicitly `Ready=True`;
|
||||
unknown or reconciling state is not healthy.
|
||||
|
||||
`release.exact-lineage-is-running` atomically binds:
|
||||
|
||||
- fixed remote `main` SHA and local `origin/main`;
|
||||
- open draft PR #19, base `main`, existing feature ref, and exact reviewed head;
|
||||
- the exact `:git-<source-SHA>-build-<n>@sha256:<digest>` image identity,
|
||||
with the source SHA equal to the release `main` SHA;
|
||||
- current Deployment generation and revision;
|
||||
- the one active ReplicaSet revision/hash; every Ready pod image and imageID;
|
||||
- the Flux-applied `main` revision.
|
||||
|
||||
The pool evaluator requires exactly ordinals 0, 1, and 2, all Ready, on three
|
||||
distinct nodes and claims, with service-account token automount disabled. The
|
||||
chat checks always require durable Telegram sessions and the exact running chat
|
||||
configuration revision.
|
||||
|
||||
## Armed ephemeral mode
|
||||
|
||||
Armed mode is the only write path and runs only after the complete default
|
||||
catalog reports `GO`.
|
||||
|
||||
```sh
|
||||
scripts/ops/hermes_handoff_acceptance.py \
|
||||
<all required release inputs above> \
|
||||
--arm-ephemeral-push \
|
||||
--confirm "ARM EPHEMERAL HERMES HANDOFF PUSH" \
|
||||
--ephemeral-token acceptance-20260817a \
|
||||
--output build/hermes-handoff-acceptance-armed.json
|
||||
```
|
||||
|
||||
Before any runner, network request, or cluster vantage is created, preflight
|
||||
binds the working directory to the existing linked PR #19 worktree, the exact
|
||||
feature branch and reviewed local HEAD, the fixed HTTPS Atlas origin, base
|
||||
`main`, confirmation phrase, and one unique ephemeral ref. The command policy is
|
||||
then pinned to that exact ref. It allows only its push, one exact draft-create
|
||||
payload, cleanup of the exactly discovered PR number, and deletion of that ref.
|
||||
|
||||
The run reserves at least 300 seconds (up to 6000) for cleanup. It verifies the
|
||||
pushed SHA, then identifies the pull request from two independent sources: the
|
||||
create response itself, and the pull index read page by page to its last page
|
||||
(50 per page, 20 pages maximum) rather than a single truncated first page. Either
|
||||
source alone names a pull request this run definitely made, because preflight
|
||||
proved the ephemeral ref did not exist. Every number found from either source is
|
||||
registered and closed, so a failed or unusable index can no longer leave an
|
||||
orphan open. A `GO` still requires exactly one open draft with the expected base,
|
||||
head ref, base SHA, and head SHA, and the two sources must agree.
|
||||
|
||||
Cleanup requires successful PR close, successful branch deletion, a successful
|
||||
empty `ls-remote`, and a strict re-read of the same closed PR. Malformed or
|
||||
partial cleanup is `FAIL`; it is never inferred from an error. When anything is
|
||||
uncertain the `draft-pull-request` evidence carries `residue_ref` and an exact
|
||||
`manual_cleanup` command list — the `PATCH .../pulls/<n> --field state=closed`
|
||||
calls for every number found, the `DELETE .../branches/<ref>` call, and, when no
|
||||
number could be determined at all, the exact ref to sweep the index for by hand.
|
||||
|
||||
Force, force-with-lease, fan-out, protected refs, other repositories, other PR
|
||||
numbers, arbitrary PATCH payloads, and repository administration remain denied.
|
||||
|
||||
## Dependency, merge, and conflict graph
|
||||
|
||||
Authoritative snapshot observed 2026-08-17 after refreshing Gitea and Git against
|
||||
remote `main` `d8f2d818b9a552ea6c2d7fe86554be829bd5ffff`:
|
||||
|
||||
| PR | Head SHA | Draft | Base | Role |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| #12 | `ed43dbaa7f094e1841aa760862e80015deb9d80c` | no | `main` | AI usage exporter repair |
|
||||
| #14 | `8c6e3acdacb223f68b1716f0a74bd543e18890e6` | yes | `main` | bounded forge client and least-privilege RBAC |
|
||||
| #15 | `73fefbb5d92ef07371a9a997f22d848acd2d06ed` | yes | `main` | result-decomposition reliability |
|
||||
| #16 | `e8c26ecf85a0c802d378fffffb98b7df43b78242` | no | `main` | unprivileged image builder |
|
||||
| #17 | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | yes | `main` | three-lane placement |
|
||||
| #18 | `689bcb6e48414854447988ba174143a307d308b9` | yes | `main` | distributed three-worker pool |
|
||||
| #19 | `c808baff40a12a90f43c2e5e39b94d139f88ea29` | yes | `main` | head this repair replaces on the same branch |
|
||||
| #20 | `a242dcc786576ae1a18000cb4941836ff610dff2` | yes | `main` | CLI process-group reaping |
|
||||
|
||||
The repaired #19 SHA cannot be embedded in its own commit without changing that
|
||||
SHA. Resolve it from Gitea after push and require it as `--reviewed-head-sha`;
|
||||
the PR handoff records the exact value. All eight heads in this table, including
|
||||
the repaired #19, merge cleanly with `main` individually. Pairwise
|
||||
`git merge-tree --write-tree` over all 28 combinations found conflicts for:
|
||||
#12+#17, #12+#18, #12+#19, #14+#15, #14+#16, #14+#17, #14+#19, #14+#20, #15+#16,
|
||||
#15+#17, #15+#19, #16+#19, #16+#20, #17+#18, and #17+#20. Gitea's
|
||||
`mergeable=false` on a draft is not treated as conflict evidence.
|
||||
|
||||
This graph differs from the previous snapshot in exactly two ways. #12 and #18
|
||||
both advanced, which adds #12+#18, #17+#18, and #12+#19. And #19 no longer
|
||||
duplicates the repo-wide hygiene legacy-exception mechanism that #18 also
|
||||
carries, which makes #18+#19 clean. Recompute before every integration step;
|
||||
nothing here is durable.
|
||||
|
||||
Use this dependency order, refreshing every head and re-running merge-tree at
|
||||
each step:
|
||||
|
||||
1. #14 first so the release never restores or prolongs cluster-admin authority.
|
||||
2. #16, then #15, resolving their Dockerfile and quality-contract changes
|
||||
additively. Never take one PR's quality contract wholesale.
|
||||
3. #20 after #15; it depends on the CLI lifecycle work and must be rebased across
|
||||
the current #14/#16 contract and helper changes.
|
||||
4. #12 before #17, resolving their Deployment overlap once.
|
||||
5. #17, then #18, after the authority, image, reliability, and reaping work.
|
||||
6. #19 last as the integration/acceptance change, only after #12/#14-#18/#20 are
|
||||
merged and their exact supplied heads are ancestors of refreshed `main`.
|
||||
|
||||
An exact post-repair merge-tree check reports #19 clean with current `main`, #17,
|
||||
#18, and #20, and conflicting with #12, #14, #15, and #16 on these files:
|
||||
|
||||
| Pair | Overlapping files |
|
||||
| --- | --- |
|
||||
| #12+#19 | `testing/tests/test_hermes_auto_router.py` |
|
||||
| #14+#19 | `testing/quality_contract.json`, `testing/quality_coverage.py`, `testing/tests/test_quality_coverage_helpers.py` |
|
||||
| #15+#19 | `testing/tests/conftest.py` |
|
||||
| #16+#19 | `testing/quality_contract.json` |
|
||||
|
||||
Every one of those overlaps is additive on the #19 side. Resolve the contract by
|
||||
union — take every PR's managed modules, lint paths, tracked files, and branch
|
||||
tracked files, and never take one PR's quality contract wholesale. #19 adds two
|
||||
fixture-isolation fixtures to `conftest.py` and two lines to the auto-router test;
|
||||
keep both sides. Preserve branch coverage, every tracked module, fixture
|
||||
isolation, and all desired-state changes during conflict resolution.
|
||||
|
||||
Recompute this graph from the Gitea API immediately before integration; do not
|
||||
infer conflict status from Gitea's draft `mergeable` field.
|
||||
|
||||
The #16/#19 integration must keep both the immutable digest and the source tag
|
||||
(`newTag: git-<main-SHA>-build-<n>` plus the digest) in rendered image identity.
|
||||
Digest-only rendering loses the attested build-source link and therefore
|
||||
correctly remains `NO_GO` under the exact-lineage evaluator.
|
||||
|
||||
## Safe rollback guidance
|
||||
|
||||
Every rollback is a reviewed forward fix or selective Git revert followed by
|
||||
the normal Flux workflow. Never restore `hermes-agent-cluster-admin`, even when
|
||||
reverting unrelated portions of #14. If a rollback would widen RBAC, stop and
|
||||
write a forward least-privilege fix instead.
|
||||
|
||||
Reverting #18 removes the StatefulSet desired state, but its
|
||||
`volumeClaimTemplates` PVCs are retained by the Kubernetes default because no
|
||||
delete retention policy is set. Do not claim Flux pruning deletes those PVCs.
|
||||
Any PVC deletion is a separate, explicit, human-reviewed data-destruction task
|
||||
with exact targets and recovery implications.
|
||||
|
||||
Image rollback means a reviewed commit restoring a previously attested digest;
|
||||
never retag an immutable image. Do not use direct `kubectl` edits as a durable
|
||||
rollback.
|
||||
|
||||
## Quality and release evidence
|
||||
|
||||
The canonical quality contract explicitly manages, lints, and tracks all 16
|
||||
`scripts/ops/hermes_handoff_*.py` modules. It enforces at least 95% line and
|
||||
branch coverage per tracked file and the 500-physical-line source cap. Contract
|
||||
tests derive the module set so a future file cannot silently escape the gate.
|
||||
This change is purely additive to the contract: it registers its own modules and
|
||||
changes no repo-wide hygiene semantics.
|
||||
|
||||
The gate's `hygiene` step is red on `main` and stays red here. Exactly four
|
||||
pre-existing test files exceed the 500-line cap —
|
||||
`scripts/tests/test_dashboards_render_atlas.py` (516),
|
||||
`testing/tests/test_hermes_chat_quality.py` (2242),
|
||||
`testing/tests/test_hermes_cli_lanes.py` (1971), and
|
||||
`testing/tests/test_hermes_coordinator.py` (510). All four are byte-identical at
|
||||
merge-base `30259b52` and at current `origin/main`, none is touched by this
|
||||
change, and the same four are the only issues a pristine `main` reports. Splitting
|
||||
them, or grandfathering them, is a repo-wide contract decision that belongs to the
|
||||
canonical contract change carried by PR #14/#15, not to this acceptance harness.
|
||||
Every other gate step — docs, smell, unit, coverage — is green, and every file
|
||||
this change adds or edits is under the cap.
|
||||
|
||||
```sh
|
||||
python3 -m testing.quality_gate --profile local --build-dir build
|
||||
python3 -m testing.quality_handoff_mutation
|
||||
python3 -m ruff check --select F,B,SIM,C4,UP --ignore B017,UP015,UP035 \
|
||||
scripts/ops/hermes_handoff_*.py testing/quality_coverage.py \
|
||||
testing/quality_handoff_mutation.py testing/tests/test_hermes_handoff_*.py
|
||||
python3 -m ruff format --check \
|
||||
scripts/ops/hermes_handoff_*.py testing/quality_coverage.py \
|
||||
testing/quality_handoff_mutation.py testing/tests/test_hermes_handoff_*.py
|
||||
python3 -m compileall -q scripts/ops testing
|
||||
kustomize build services/hermes | kubectl apply --dry-run=client --validate=false -f -
|
||||
git diff --check origin/main...HEAD
|
||||
```
|
||||
|
||||
The format scope names the files this change owns. A wider `testing/quality_*.py`
|
||||
glob picks up `testing/quality_gate.py`, which is unmodified here and not
|
||||
format-clean, so the documented command would fail for a reason this change did
|
||||
not cause. `ruff format` is evidence, not a gate step: the canonical gate runs
|
||||
`ruff check` only.
|
||||
|
||||
The deterministic mutation gate must kill every policy, redaction, verdict,
|
||||
numeric, truncation, provenance, and cleanup mutant, including one that restores
|
||||
the zero-evidence pass, one that restores the bare-prefix repository pin, one
|
||||
that stops re-checking a `kubectl exec` inner command, and one that drops binary
|
||||
attestation from an unrecorded step. The adversarial suite includes large
|
||||
concurrent stdout/stderr, malformed JSON and bytes, zero state, partial failure,
|
||||
absolute timeout, descendant-held pipes, malformed cleanup, ambiguous creates,
|
||||
and no-subprocess/no-mutation spies. Render/client dry-run, exact conflict checks,
|
||||
and bounded diff/full-tree secret scans must also pass.
|
||||
|
||||
The provider-health and Gitea-token-sensitive tests use hermetic fixtures. Their
|
||||
result must be identical whether synthetic runtime files exist or are absent;
|
||||
tests must never read the mounted production token.
|
||||
|
||||
## Release decision
|
||||
|
||||
The handoff remains `NO_GO` until all dependency heads are merged, the exact
|
||||
lineage check passes from an external operator, both default and armed reports
|
||||
are complete, the branch and PR cleanup is verified, the canonical and
|
||||
adversarial gates are green, and a human has reviewed PR #19 plus the artifacts.
|
||||
The harness never merges, deploys, reconciles, publishes, or approves anything.
|
||||
354
scripts/ops/hermes_handoff_acceptance.py
Executable file
354
scripts/ops/hermes_handoff_acceptance.py
Executable file
@ -0,0 +1,354 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail-closed acceptance harness for the Hermes full handoff.
|
||||
|
||||
Read-only by default. The harness collects evidence from two genuinely separate
|
||||
places — an external operator vantage and the Hermes agent probing itself from
|
||||
inside its own pod — classifies every check PASS / FAIL / NOT_RUN /
|
||||
NOT_APPLICABLE, and emits machine-readable JSON alongside a short human summary.
|
||||
|
||||
Any mandatory FAIL or NOT_RUN is a NO_GO, and so is a harness-level problem: an
|
||||
unreachable vantage, a catalog whose evidence no longer exists, an expired
|
||||
deadline, or two vantages that turn out to be the same principal. Nothing here
|
||||
rounds an absence of evidence up to a pass.
|
||||
|
||||
scripts/ops/hermes_handoff_acceptance.py --help
|
||||
|
||||
The release runbook lists every required freshness and lineage input.
|
||||
|
||||
Mutation lives behind a separate arming flag, needs an exact confirmation
|
||||
phrase and a unique ref, refuses protected push targets before any network
|
||||
call, and verifies its own cleanup. A default run reports those checks NOT_RUN.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from hermes_handoff_catalog import Targets # noqa: E402
|
||||
from hermes_handoff_ephemeral import ( # noqa: E402
|
||||
CONFIRMATION,
|
||||
ArmingError,
|
||||
ArmRequest,
|
||||
preflight,
|
||||
run_armed,
|
||||
)
|
||||
from hermes_handoff_exec import ( # noqa: E402
|
||||
DEFAULT_COMMAND_TIMEOUT,
|
||||
DEFAULT_RUN_DEADLINE,
|
||||
Runner,
|
||||
)
|
||||
from hermes_handoff_model import GO, Report, unscreened_fields, utc_now # noqa: E402
|
||||
from hermes_handoff_policy import ( # noqa: E402
|
||||
ARMED,
|
||||
EXPECTED_BASE,
|
||||
EXPECTED_REMOTE,
|
||||
EXPECTED_REPO,
|
||||
READ_ONLY,
|
||||
)
|
||||
from hermes_handoff_redaction import DEFAULT_MAX_BYTES # noqa: E402
|
||||
from hermes_handoff_run import ( # noqa: E402
|
||||
build_catalog,
|
||||
build_report,
|
||||
resolve_vantages,
|
||||
unavailable_report,
|
||||
validate_catalog,
|
||||
validate_targets,
|
||||
)
|
||||
|
||||
EXIT_GO = 0
|
||||
EXIT_NO_GO = 1
|
||||
EXIT_USAGE = 2
|
||||
|
||||
|
||||
def dependency_head(value: str) -> tuple[int, str]:
|
||||
"""Parse one `PR=SHA` dependency-head binding."""
|
||||
try:
|
||||
number, sha = value.split("=", 1)
|
||||
parsed = int(number)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"dependency head must be PR=40-character-SHA"
|
||||
) from exc
|
||||
if (
|
||||
parsed <= 0
|
||||
or len(sha) != 40
|
||||
or any(character not in "0123456789abcdef" for character in sha)
|
||||
):
|
||||
raise argparse.ArgumentTypeError(
|
||||
"dependency head must be PR=lowercase-40-character-SHA"
|
||||
)
|
||||
return (parsed, sha)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
"""Return the command-line interface for the harness."""
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="hermes_handoff_acceptance",
|
||||
description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument("--namespace", default="hermes", help="Hermes namespace.")
|
||||
parser.add_argument(
|
||||
"--context", help="Kubernetes context for the operator vantage."
|
||||
)
|
||||
parser.add_argument("--kubeconfig", help="Kubeconfig for the operator vantage.")
|
||||
parser.add_argument(
|
||||
"--output", type=Path, help="Write the JSON report here instead of stdout."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json-only", action="store_true", help="Suppress the human summary."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--command-timeout",
|
||||
type=float,
|
||||
default=DEFAULT_COMMAND_TIMEOUT,
|
||||
help="Per-command seconds.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deadline",
|
||||
type=float,
|
||||
default=DEFAULT_RUN_DEADLINE,
|
||||
help="Whole-run seconds.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-output-bytes",
|
||||
type=int,
|
||||
default=DEFAULT_MAX_BYTES,
|
||||
help="Per-capture byte budget.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--baseline-commit",
|
||||
default=Targets.baseline_commit,
|
||||
help="Commit main must descend from.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dependency-pr",
|
||||
type=int,
|
||||
action="append",
|
||||
default=None,
|
||||
help="Pull request that must be merged before this release is certifiable (repeatable).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dependency-head",
|
||||
type=dependency_head,
|
||||
action="append",
|
||||
default=None,
|
||||
metavar="PR=SHA",
|
||||
help="Exact current dependency PR head (repeat for every dependency).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--remote-main-sha", default="", help="Exact observed origin/main SHA."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reviewed-head-sha", default="", help="Exact reviewed PR #19 head SHA."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--agent-image",
|
||||
default="",
|
||||
help="Exact agent :git-<source>-build-<n>@sha256 identity.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--build-sha", default="", help="Exact source SHA used for the image build."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deployment-revision", default="", help="Exact running Deployment revision."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--expected-suspension",
|
||||
action="append",
|
||||
default=None,
|
||||
metavar="NAMESPACE/NAME",
|
||||
help="Flux object allowed to be suspended (repeatable).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--node-count",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Nodes the fleet must cover (fixed at 3).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pool-replicas", type=int, default=3, help="Expected pool worker count."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chat-config-revision",
|
||||
default="",
|
||||
help="Expected chat pod-template config revision.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chat-ordinal",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Chat tenant ordinal that carries the Telegram identity.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pool-worker-env",
|
||||
action="append",
|
||||
default=None,
|
||||
help="Environment variable the pool workers must declare (repeatable).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-evidence-age",
|
||||
type=int,
|
||||
default=86_400,
|
||||
help="Seconds before provider and routing evidence is treated as stale.",
|
||||
)
|
||||
armed = parser.add_argument_group("ephemeral mutation mode")
|
||||
armed.add_argument(
|
||||
"--arm-ephemeral-push",
|
||||
action="store_true",
|
||||
help="Permit one ephemeral branch push and draft pull request, then remove both.",
|
||||
)
|
||||
armed.add_argument(
|
||||
"--confirm", default="", help=f"Exact confirmation phrase: {CONFIRMATION!r}."
|
||||
)
|
||||
armed.add_argument(
|
||||
"--ephemeral-token", default="", help="Unique 8-64 character ref suffix."
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def targets_from_args(arguments: argparse.Namespace, now: dt.datetime) -> Targets:
|
||||
"""Return the catalog targets a parsed command line describes."""
|
||||
defaults = Targets()
|
||||
return Targets(
|
||||
namespace=arguments.namespace,
|
||||
baseline_commit=arguments.baseline_commit,
|
||||
dependency_pull_requests=tuple(
|
||||
arguments.dependency_pr or defaults.dependency_pull_requests
|
||||
),
|
||||
dependency_heads=tuple(arguments.dependency_head or ()),
|
||||
remote_main_sha=arguments.remote_main_sha,
|
||||
reviewed_head_sha=arguments.reviewed_head_sha,
|
||||
agent_image=arguments.agent_image,
|
||||
build_sha=arguments.build_sha,
|
||||
deployment_revision=arguments.deployment_revision,
|
||||
expected_suspensions=tuple(arguments.expected_suspension or ()),
|
||||
node_count=arguments.node_count,
|
||||
pool_replicas=arguments.pool_replicas,
|
||||
chat_config_revision=arguments.chat_config_revision,
|
||||
chat_ordinal=arguments.chat_ordinal,
|
||||
pool_worker_env=tuple(arguments.pool_worker_env or defaults.pool_worker_env),
|
||||
max_evidence_age_seconds=arguments.max_evidence_age,
|
||||
now=now,
|
||||
)
|
||||
|
||||
|
||||
def emit(report: Report, arguments: argparse.Namespace, stream=None) -> list[str]:
|
||||
"""Write the report, refusing to publish one that still looks credential-shaped."""
|
||||
payload = report.as_dict()
|
||||
offenders = unscreened_fields(payload)
|
||||
if offenders:
|
||||
payload = {
|
||||
"harness": payload["harness"],
|
||||
"decision": "NO_GO",
|
||||
"harness_errors": [
|
||||
"the rendered report failed its own credential screening at "
|
||||
+ ", ".join(offenders[:8])
|
||||
],
|
||||
}
|
||||
rendered = json.dumps(payload, indent=2, sort_keys=True, default=str)
|
||||
if arguments.output:
|
||||
arguments.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
arguments.output.write_text(rendered + "\n", encoding="utf-8")
|
||||
else:
|
||||
print(rendered, file=stream or sys.stdout)
|
||||
if not arguments.json_only:
|
||||
print(report.render_summary(), file=sys.stderr)
|
||||
return offenders
|
||||
|
||||
|
||||
def _armed_results(arguments: argparse.Namespace, runner: Runner, vantages) -> list:
|
||||
request = ArmRequest(
|
||||
repo=EXPECTED_REPO,
|
||||
remote=EXPECTED_REMOTE,
|
||||
token=arguments.ephemeral_token,
|
||||
confirmation=arguments.confirm,
|
||||
base=EXPECTED_BASE,
|
||||
expected_head=arguments.reviewed_head_sha,
|
||||
expected_base_sha=arguments.remote_main_sha,
|
||||
)
|
||||
preflight(request, EXPECTED_REPO, Path.cwd())
|
||||
vantage = vantages["operator"]
|
||||
return run_armed(runner, vantage, request)
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
"""Run the acceptance sweep and return the process exit status."""
|
||||
arguments = build_parser().parse_args(argv)
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
started_at = utc_now(lambda: now)
|
||||
targets = targets_from_args(arguments, now)
|
||||
mode = ARMED if arguments.arm_ephemeral_push else READ_ONLY
|
||||
|
||||
specs = build_catalog(targets)
|
||||
startup_problems = [*validate_targets(targets), *validate_catalog(specs)]
|
||||
if arguments.arm_ephemeral_push:
|
||||
try:
|
||||
preflight(
|
||||
ArmRequest(
|
||||
repo=EXPECTED_REPO,
|
||||
remote=EXPECTED_REMOTE,
|
||||
token=arguments.ephemeral_token,
|
||||
confirmation=arguments.confirm,
|
||||
base=EXPECTED_BASE,
|
||||
expected_head=arguments.reviewed_head_sha,
|
||||
expected_base_sha=arguments.remote_main_sha,
|
||||
),
|
||||
EXPECTED_REPO,
|
||||
Path.cwd(),
|
||||
)
|
||||
except ArmingError as exc:
|
||||
startup_problems.append(f"arming refused: {exc}")
|
||||
if startup_problems:
|
||||
report = unavailable_report(
|
||||
mode, started_at, "; ".join(startup_problems), targets
|
||||
)
|
||||
emit(report, arguments)
|
||||
return EXIT_NO_GO
|
||||
|
||||
try:
|
||||
runner = Runner(
|
||||
mode=mode,
|
||||
command_timeout=arguments.command_timeout,
|
||||
deadline_seconds=arguments.deadline,
|
||||
max_bytes=arguments.max_output_bytes,
|
||||
)
|
||||
except ValueError as exc:
|
||||
report = unavailable_report(
|
||||
mode, started_at, f"invalid execution bound: {exc}", targets
|
||||
)
|
||||
emit(report, arguments)
|
||||
return EXIT_USAGE
|
||||
vantages, records = resolve_vantages(
|
||||
runner, targets, arguments.kubeconfig, arguments.context
|
||||
)
|
||||
report = build_report(runner, targets, specs, vantages, records, mode, started_at)
|
||||
if arguments.arm_ephemeral_push:
|
||||
if report.decision != GO:
|
||||
report.harness_errors.append(
|
||||
"armed mutation was not started because read-only acceptance did not GO"
|
||||
)
|
||||
else:
|
||||
armed_results = _armed_results(arguments, runner, vantages)
|
||||
armed_ids = {result.spec.id for result in armed_results}
|
||||
report.results = [
|
||||
result for result in report.results if result.spec.id not in armed_ids
|
||||
]
|
||||
report.results.extend(armed_results)
|
||||
report.results.sort(key=lambda result: result.spec.id)
|
||||
report.finished_at = utc_now()
|
||||
offenders = emit(report, arguments)
|
||||
if offenders:
|
||||
return EXIT_NO_GO
|
||||
return EXIT_GO if report.decision == GO else EXIT_NO_GO
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through main()
|
||||
raise SystemExit(main())
|
||||
154
scripts/ops/hermes_handoff_arming.py
Normal file
154
scripts/ops/hermes_handoff_arming.py
Normal file
@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Arming, provenance, and push-target rules for the armed ephemeral mode.
|
||||
|
||||
Nothing here touches the network or spawns a process. It is the half of armed
|
||||
mode that must be true before a single command exists: the confirmation phrase,
|
||||
the fixed repository and base, the one ephemeral ref grammar, and a local
|
||||
worktree whose HEAD and ``origin/main`` are byte-exact with the reviewed release.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import configparser
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from hermes_handoff_policy import (
|
||||
EXPECTED_BASE,
|
||||
EXPECTED_REMOTE,
|
||||
EXPECTED_REPO,
|
||||
arm_ephemeral_policy,
|
||||
)
|
||||
|
||||
CONFIRMATION = "ARM EPHEMERAL HERMES HANDOFF PUSH"
|
||||
REF_PREFIX = "ephemeral/hermes-handoff-acceptance"
|
||||
REF_RE = re.compile(rf"{REF_PREFIX}/[a-z0-9][a-z0-9-]{{7,63}}\Z")
|
||||
TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9-]{7,63}\Z")
|
||||
SHA_RE = re.compile(r"[0-9a-f]{40}\Z")
|
||||
EXPECTED_HEAD_REF = "feature/hermes-full-handoff-acceptance"
|
||||
EXPECTED_REMOTE_URL = "https://scm.bstein.dev/atlas/titan-iac.git"
|
||||
PROTECTED_NAMES = frozenset(
|
||||
{
|
||||
"default",
|
||||
"develop",
|
||||
"head",
|
||||
"main",
|
||||
"master",
|
||||
"prod",
|
||||
"production",
|
||||
"release",
|
||||
"stable",
|
||||
"trunk",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class ArmingError(ValueError):
|
||||
"""Raised when an ephemeral run is not safe to start."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ArmRequest:
|
||||
"""Every fixed and caller-confirmed input to one armed run."""
|
||||
|
||||
repo: str
|
||||
remote: str
|
||||
token: str
|
||||
confirmation: str
|
||||
base: str = EXPECTED_BASE
|
||||
expected_head: str = ""
|
||||
expected_base_sha: str = ""
|
||||
|
||||
@property
|
||||
def ref(self) -> str:
|
||||
return f"{REF_PREFIX}/{self.token}"
|
||||
|
||||
|
||||
def normalise_branch(name: str) -> str:
|
||||
"""Return a branch name stripped of ref decoration, for comparison."""
|
||||
return name.strip().removeprefix("refs/heads/").strip("/").lower()
|
||||
|
||||
|
||||
def assert_push_target_allowed(ref: str) -> None:
|
||||
"""Raise unless ``ref`` is the one ephemeral branch grammar."""
|
||||
if normalise_branch(ref) in PROTECTED_NAMES:
|
||||
raise ArmingError(f"refusing to push to protected branch {ref!r}")
|
||||
if not REF_RE.fullmatch(ref):
|
||||
raise ArmingError(f"push target {ref!r} is not an ephemeral acceptance ref")
|
||||
|
||||
|
||||
def bounded_read(path: Path, limit: int = 1024 * 1024) -> str:
|
||||
data = path.read_bytes()
|
||||
if not data or len(data) > limit:
|
||||
raise ArmingError(f"local Git metadata is empty or exceeds {limit} bytes")
|
||||
return data.decode("utf-8", "strict")
|
||||
|
||||
|
||||
def _attest_worktree(root: Path, expected_head: str, expected_base_sha: str) -> None:
|
||||
"""Bind cwd, common Git metadata, branch, remote, and exact local HEAD."""
|
||||
resolved = root.resolve(strict=True)
|
||||
marker = resolved / ".git"
|
||||
if not marker.is_file():
|
||||
raise ArmingError(
|
||||
"armed mode must run at the root of the existing linked worktree"
|
||||
)
|
||||
line = bounded_read(marker, 4096).strip()
|
||||
if not line.startswith("gitdir: "):
|
||||
raise ArmingError("linked-worktree Git metadata is malformed")
|
||||
gitdir = Path(line.removeprefix("gitdir: ")).resolve(strict=True)
|
||||
common = (gitdir / bounded_read(gitdir / "commondir", 4096).strip()).resolve(
|
||||
strict=True
|
||||
)
|
||||
head = bounded_read(gitdir / "HEAD", 4096).strip()
|
||||
expected_ref = f"refs/heads/{EXPECTED_HEAD_REF}"
|
||||
if head != f"ref: {expected_ref}":
|
||||
raise ArmingError("armed mode must run from the existing PR #19 feature branch")
|
||||
ref_path = common / expected_ref
|
||||
actual_head = bounded_read(ref_path, 4096).strip() if ref_path.is_file() else ""
|
||||
if actual_head != expected_head:
|
||||
raise ArmingError(
|
||||
"local worktree HEAD does not match the exact reviewed PR head"
|
||||
)
|
||||
remote_ref = common / "refs" / "remotes" / EXPECTED_REMOTE / EXPECTED_BASE
|
||||
remote_main = bounded_read(remote_ref, 4096).strip() if remote_ref.is_file() else ""
|
||||
if remote_main != expected_base_sha:
|
||||
raise ArmingError("local origin/main does not match the exact release base SHA")
|
||||
parser = configparser.RawConfigParser(interpolation=None)
|
||||
try:
|
||||
parser.read_string(bounded_read(common / "config"))
|
||||
remote_url = parser.get(f'remote "{EXPECTED_REMOTE}"', "url")
|
||||
except (configparser.Error, KeyError) as exc:
|
||||
raise ArmingError("local Git remote metadata is malformed") from exc
|
||||
if remote_url != EXPECTED_REMOTE_URL:
|
||||
raise ArmingError("origin is not the fixed Atlas titan-iac HTTPS repository")
|
||||
|
||||
|
||||
def preflight(
|
||||
request: ArmRequest, expected_repo: str, worktree: Path | None = None
|
||||
) -> None:
|
||||
"""Validate arming and local provenance before any subprocess/network call."""
|
||||
if request.confirmation != CONFIRMATION:
|
||||
raise ArmingError("confirmation phrase does not match")
|
||||
if expected_repo != EXPECTED_REPO or request.repo != EXPECTED_REPO:
|
||||
raise ArmingError(
|
||||
"armed repository is not the fixed Atlas titan-iac repository"
|
||||
)
|
||||
if (
|
||||
request.remote != EXPECTED_REMOTE
|
||||
or normalise_branch(request.base) != EXPECTED_BASE
|
||||
):
|
||||
raise ArmingError("armed remote and pull-request base are fixed to origin/main")
|
||||
if not TOKEN_RE.fullmatch(request.token):
|
||||
raise ArmingError(
|
||||
"ephemeral token must be 8-64 lowercase alphanumeric or dash characters"
|
||||
)
|
||||
if not SHA_RE.fullmatch(request.expected_head) or not SHA_RE.fullmatch(
|
||||
request.expected_base_sha
|
||||
):
|
||||
raise ArmingError("exact reviewed head and release base SHAs are required")
|
||||
assert_push_target_allowed(request.ref)
|
||||
_attest_worktree(
|
||||
worktree or Path.cwd(), request.expected_head, request.expected_base_sha
|
||||
)
|
||||
arm_ephemeral_policy(request.ref)
|
||||
202
scripts/ops/hermes_handoff_catalog.py
Normal file
202
scripts/ops/hermes_handoff_catalog.py
Normal file
@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Declarative acceptance catalog: baseline, identity, routing, scopes, access.
|
||||
|
||||
The catalog is data on purpose. A reviewer should be able to read what the
|
||||
release is asserted to satisfy without reading any plumbing, and the harness
|
||||
should be able to prove structural properties over every entry — both vantages
|
||||
represented, no impersonation from outside a pod, and no mutation command of
|
||||
any kind — before it runs a single command.
|
||||
|
||||
Catalog drift is a NO_GO, not a pass. If a manifest, a field name, or a workload
|
||||
is renamed, the affected checks report ``NOT_RUN`` because the evidence they
|
||||
name is not there, and the release stops until either the platform or the
|
||||
catalog is corrected.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from hermes_handoff_model import ALWAYS, READ, CheckSpec, Step
|
||||
from hermes_handoff_policy import projection
|
||||
|
||||
OPERATOR = "operator"
|
||||
SELF = "self"
|
||||
SWITCHYARD = "switchyard"
|
||||
NODE = "node"
|
||||
CHAT = "chat"
|
||||
VANTAGE_NAMES = (OPERATOR, SELF, SWITCHYARD, NODE, CHAT)
|
||||
|
||||
# `name<TAB>generation<TAB>observedGeneration<TAB>suspend<TAB>conditions`.
|
||||
FLUX_PROJECTION = projection(
|
||||
(
|
||||
"jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{'\\t'}"
|
||||
"{.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}"
|
||||
"{range .status.conditions[*]}{.type}={.status},{end}{'\\n'}{end}"
|
||||
).replace("'", '"')
|
||||
)
|
||||
|
||||
ENV_NAME_PROJECTION = projection(
|
||||
(
|
||||
"jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{'\\n'}{end}{end}"
|
||||
).replace("'", '"')
|
||||
)
|
||||
|
||||
INIT_NAME_PROJECTION = projection(
|
||||
(
|
||||
"jsonpath={range .spec.template.spec.initContainers[*]}{.name}{'\\n'}{end}"
|
||||
).replace("'", '"')
|
||||
)
|
||||
|
||||
NODE_NAME_PROJECTION = projection(
|
||||
"jsonpath={range .items[*]}{.spec.nodeName}{'\\n'}{end}".replace("'", '"')
|
||||
)
|
||||
|
||||
POOL_PROJECTION = projection(
|
||||
(
|
||||
"jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.spec.nodeName}{'\\t'}"
|
||||
"{.status.phase}{'\\t'}{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}"
|
||||
"{'\\t'}{range .spec.volumes[*]}{.persistentVolumeClaim.claimName}{','}{end}{'\\n'}{end}"
|
||||
).replace("'", '"')
|
||||
)
|
||||
|
||||
PROVIDER_API_KEY_NAMES = (
|
||||
"ANTHROPIC_API_KEY",
|
||||
"ANTHROPIC_AUTH_TOKEN",
|
||||
"ANTHROPIC_BASE_URL",
|
||||
"AZURE_OPENAI_API_KEY",
|
||||
"CLAUDE_API_KEY",
|
||||
"CLAUDE_CODE_OAUTH_TOKEN",
|
||||
"GEMINI_API_KEY",
|
||||
"GOOGLE_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_BASE_URL",
|
||||
)
|
||||
FORGE_CREDENTIAL_NAMES = (
|
||||
"GITEA_PASSWORD",
|
||||
"GITEA_TOKEN",
|
||||
"GITEA_USERNAME",
|
||||
"GIT_PASSWORD",
|
||||
)
|
||||
|
||||
CODEX_HEALTH = "/opt/data/provider-health/codex.json"
|
||||
CLAUDE_HEALTH = "/opt/data/provider-health/claude.json"
|
||||
CODEX_HEALTH_FIELDS = "authenticated,transport,state,model,effort,latency_ms,checked_at"
|
||||
CLAUDE_HEALTH_FIELDS = (
|
||||
"authenticated,api_provider,auth_method,transport,state,subscription_type,"
|
||||
"model,effort,latency_ms,checked_at"
|
||||
)
|
||||
ROUTING_LOG = "/var/lib/switchyard/routing.jsonl"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Targets:
|
||||
"""Runtime names and expected values the catalog is written against.
|
||||
|
||||
Everything a site can legitimately rename lives here so the assertions stay
|
||||
readable and a rename is a configuration change rather than a code change.
|
||||
"""
|
||||
|
||||
namespace: str = "hermes"
|
||||
agent_deployment: str = "hermes-agent"
|
||||
agent_container: str = "hermes"
|
||||
chat_statefulset: str = "hermes-chat-tenant"
|
||||
chat_container: str = "hermes"
|
||||
chat_ordinal: int = 0
|
||||
chat_router_deployment: str = "hermes-chat-router"
|
||||
triage_deployment: str = "hermes"
|
||||
triage_container: str = "hermes"
|
||||
switchyard_deployment: str = "hermes-switchyard"
|
||||
switchyard_container: str = "switchyard"
|
||||
node_daemonset: str = "hermes-node-ssh-access"
|
||||
node_container: str = "key-reconciler"
|
||||
pool_statefulset: str = "hermes-execution-worker"
|
||||
pool_replicas: int = 3
|
||||
pool_selector: str = "app=hermes-execution-worker"
|
||||
coordinator_claims: tuple[str, ...] = (
|
||||
"hermes-agent-home",
|
||||
"hermes-agent-workspace",
|
||||
)
|
||||
node_account: str = "hermes-agent"
|
||||
node_account_identity: str = "1200:1200:/home/hermes-agent:/bin/bash"
|
||||
node_count: int = 3
|
||||
repo: str = "atlas/titan-iac"
|
||||
remote: str = "origin"
|
||||
baseline_commit: str = "ab346f55509d584e457fe26cf90be3078f7a375c"
|
||||
dependency_pull_requests: tuple[int, ...] = (12, 14, 15, 16, 17, 18, 20)
|
||||
dependency_heads: tuple[tuple[int, str], ...] = ()
|
||||
remote_main_sha: str = ""
|
||||
reviewed_pr_number: int = 19
|
||||
reviewed_head_ref: str = "feature/hermes-full-handoff-acceptance"
|
||||
reviewed_head_sha: str = ""
|
||||
agent_image: str = ""
|
||||
build_sha: str = ""
|
||||
deployment_revision: str = ""
|
||||
expected_suspensions: tuple[str, ...] = ()
|
||||
harbor_namespace: str = "harbor"
|
||||
harbor_immutability_job: str = "hermes-agent-immutability"
|
||||
builder_namespace: str = "jenkins"
|
||||
builder_serviceaccount: str = "hermes-image-builder"
|
||||
builder_pipeline: str = "ci/Jenkinsfile.hermes-agent-image"
|
||||
builder_capabilities: str = (
|
||||
'add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]'
|
||||
)
|
||||
agent_init_containers: tuple[str, ...] = ("patch-web-session-activity",)
|
||||
chat_init_containers: tuple[str, ...] = ("patch-api-server-sessions",)
|
||||
chat_config_revision: str = ""
|
||||
pool_worker_env: tuple[str, ...] = ("HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL")
|
||||
max_evidence_age_seconds: int = 86_400
|
||||
routing_tail_lines: int = 600
|
||||
routing_tail_bytes: int = 512 * 1024
|
||||
listing_bytes: int = 512 * 1024
|
||||
git_askpass: str = "/opt/coordinator/gitea_askpass.sh"
|
||||
now: Any = None
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
def step(
|
||||
key: str,
|
||||
vantage: str,
|
||||
*argv: str,
|
||||
kind: str = READ,
|
||||
optional: bool = False,
|
||||
record: bool = True,
|
||||
max_bytes: int | None = None,
|
||||
) -> Step:
|
||||
"""Return a probe step addressed to one vantage."""
|
||||
return Step(
|
||||
key=key,
|
||||
vantage=vantage,
|
||||
argv=tuple(argv),
|
||||
kind=kind,
|
||||
optional=optional,
|
||||
record=record,
|
||||
max_bytes=max_bytes,
|
||||
)
|
||||
|
||||
|
||||
def check(
|
||||
identifier: str,
|
||||
title: str,
|
||||
group: str,
|
||||
rule: str,
|
||||
steps: Sequence[Step],
|
||||
expect: dict[str, Any] | None = None,
|
||||
mandatory: bool = True,
|
||||
scope: str = ALWAYS,
|
||||
rationale: str = "",
|
||||
) -> CheckSpec:
|
||||
"""Return one catalog entry."""
|
||||
return CheckSpec(
|
||||
id=identifier,
|
||||
title=title,
|
||||
group=group,
|
||||
rule=rule,
|
||||
steps=tuple(steps),
|
||||
expect=expect or {},
|
||||
mandatory=mandatory,
|
||||
scope=scope,
|
||||
rationale=rationale,
|
||||
)
|
||||
339
scripts/ops/hermes_handoff_checks_access.py
Normal file
339
scripts/ops/hermes_handoff_checks_access.py
Normal file
@ -0,0 +1,339 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Acceptance checks for what Hermes may and may not do to the cluster.
|
||||
|
||||
Denials use Kubernetes authorization reviews only. Live denied attempts were
|
||||
removed because a policy regression could turn a supposed probe into a real
|
||||
credential read or mutation; the default harness never constructs such argv.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hermes_handoff_catalog import (
|
||||
FLUX_PROJECTION,
|
||||
OPERATOR,
|
||||
SELF,
|
||||
Targets,
|
||||
check,
|
||||
step,
|
||||
)
|
||||
from hermes_handoff_model import REVIEW, CheckSpec
|
||||
|
||||
|
||||
def _access(targets: Targets) -> list[CheckSpec]:
|
||||
namespace = targets.namespace
|
||||
return [
|
||||
check(
|
||||
"access.no-cluster-admin-binding-for-agent",
|
||||
"No ClusterRoleBinding grants the agent service account cluster-admin",
|
||||
"access-denied",
|
||||
"names_absent",
|
||||
[
|
||||
step(
|
||||
"bindings",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"get",
|
||||
"clusterrolebindings",
|
||||
"-o",
|
||||
"name",
|
||||
)
|
||||
],
|
||||
{"step": "bindings", "contains": ("hermes-agent-cluster-admin",)},
|
||||
rationale="The binding inventory and in-pod authorization reviews are independent desired/enforced views.",
|
||||
),
|
||||
check(
|
||||
"access.secrets-are-denied",
|
||||
"Hermes cannot read Secrets cluster-wide",
|
||||
"access-denied",
|
||||
"review_denied",
|
||||
[
|
||||
step(
|
||||
"review",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"get",
|
||||
"secrets",
|
||||
"--all-namespaces",
|
||||
kind=REVIEW,
|
||||
)
|
||||
],
|
||||
rationale="A live Secret request is never constructed, even when an RBAC regression would allow it.",
|
||||
),
|
||||
check(
|
||||
"access.service-account-tokens-are-denied",
|
||||
"Hermes cannot mint a service-account token",
|
||||
"access-denied",
|
||||
"review_denied",
|
||||
[
|
||||
step(
|
||||
"review-token",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"create",
|
||||
"serviceaccounts/token",
|
||||
"--namespace",
|
||||
namespace,
|
||||
kind=REVIEW,
|
||||
),
|
||||
step(
|
||||
"review-serviceaccount",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"create",
|
||||
"serviceaccounts",
|
||||
"--namespace",
|
||||
namespace,
|
||||
kind=REVIEW,
|
||||
),
|
||||
],
|
||||
rationale="Token creation has no credential-safe live probe, so both relevant grants must be denied by review.",
|
||||
),
|
||||
check(
|
||||
"access.impersonation-is-denied",
|
||||
"Hermes cannot impersonate another identity",
|
||||
"access-denied",
|
||||
"review_denied",
|
||||
[
|
||||
step(
|
||||
"review",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"impersonate",
|
||||
"users",
|
||||
"--all-namespaces",
|
||||
kind=REVIEW,
|
||||
)
|
||||
],
|
||||
rationale="The default harness never constructs an impersonated request.",
|
||||
),
|
||||
check(
|
||||
"access.workload-mutation-is-denied",
|
||||
"Hermes cannot mutate its own workload",
|
||||
"access-denied",
|
||||
"review_denied",
|
||||
[
|
||||
step(
|
||||
"review",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"patch",
|
||||
"deployments",
|
||||
"--namespace",
|
||||
namespace,
|
||||
kind=REVIEW,
|
||||
)
|
||||
],
|
||||
rationale="Client-side and server-side dry-run mutation tricks are excluded structurally.",
|
||||
),
|
||||
check(
|
||||
"access.pod-exec-is-denied",
|
||||
"Hermes cannot exec into an arbitrary pod",
|
||||
"access-denied",
|
||||
"review_denied",
|
||||
[
|
||||
step(
|
||||
"review",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"create",
|
||||
"pods/exec",
|
||||
"--all-namespaces",
|
||||
kind=REVIEW,
|
||||
)
|
||||
],
|
||||
rationale="No cross-pod exec request is created by a read-only run.",
|
||||
),
|
||||
check(
|
||||
"access.attach-portforward-and-kube-system-writes-are-denied",
|
||||
"Hermes holds no attach, port-forward, or control-plane write authority",
|
||||
"access-denied",
|
||||
"review_denied",
|
||||
[
|
||||
step(
|
||||
"review-attach",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"create",
|
||||
"pods/attach",
|
||||
"--all-namespaces",
|
||||
kind=REVIEW,
|
||||
),
|
||||
step(
|
||||
"review-portforward",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"create",
|
||||
"pods/portforward",
|
||||
"--all-namespaces",
|
||||
kind=REVIEW,
|
||||
),
|
||||
step(
|
||||
"review-create",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"create",
|
||||
"configmaps",
|
||||
"--namespace",
|
||||
"kube-system",
|
||||
kind=REVIEW,
|
||||
),
|
||||
],
|
||||
rationale="Every authority is reviewed without issuing the request it would authorize.",
|
||||
),
|
||||
check(
|
||||
"access.required-reads-are-allowed",
|
||||
"Hermes retains the reads its operator role depends on",
|
||||
"access-allowed",
|
||||
"allowed",
|
||||
[
|
||||
step(
|
||||
"watch-review",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"auth",
|
||||
"can-i",
|
||||
"watch",
|
||||
"pods",
|
||||
"--namespace",
|
||||
namespace,
|
||||
kind=REVIEW,
|
||||
),
|
||||
step(
|
||||
"pods",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
namespace,
|
||||
"get",
|
||||
"pods",
|
||||
"-o",
|
||||
"name",
|
||||
),
|
||||
step("namespaces", SELF, "kubectl", "get", "namespaces", "-o", "name"),
|
||||
step(
|
||||
"deployments",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
namespace,
|
||||
"get",
|
||||
"deployments",
|
||||
"-o",
|
||||
"name",
|
||||
),
|
||||
step("nodes", SELF, "kubectl", "get", "nodes", "-o", "name"),
|
||||
],
|
||||
),
|
||||
check(
|
||||
"access.flux-and-helm-status-are-allowed",
|
||||
"Hermes can read Flux and Helm reconciliation status",
|
||||
"access-allowed",
|
||||
"allowed",
|
||||
[
|
||||
step(
|
||||
"kustomizations",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"get",
|
||||
"kustomizations.kustomize.toolkit.fluxcd.io",
|
||||
"--all-namespaces",
|
||||
"-o",
|
||||
"name",
|
||||
record=False,
|
||||
),
|
||||
step(
|
||||
"helmreleases",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"get",
|
||||
"helmreleases.helm.toolkit.fluxcd.io",
|
||||
"--all-namespaces",
|
||||
"-o",
|
||||
"name",
|
||||
record=False,
|
||||
),
|
||||
],
|
||||
),
|
||||
check(
|
||||
"access.namespace-view-agrees-across-vantages",
|
||||
"The operator and in-pod vantages see the same namespace inventory",
|
||||
"access-allowed",
|
||||
"vantages_agree",
|
||||
[
|
||||
step(
|
||||
"operator",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"get",
|
||||
"namespaces",
|
||||
"-o",
|
||||
"name",
|
||||
record=False,
|
||||
),
|
||||
step(
|
||||
"self",
|
||||
SELF,
|
||||
"kubectl",
|
||||
"get",
|
||||
"namespaces",
|
||||
"-o",
|
||||
"name",
|
||||
record=False,
|
||||
),
|
||||
],
|
||||
{"steps": ("operator", "self")},
|
||||
rationale="A disagreement here means one vantage is not seeing the cluster the other is certifying.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _gitops(targets: Targets) -> list[CheckSpec]:
|
||||
return [
|
||||
check(
|
||||
f"gitops.{kind.split('.')[0]}-reconcile-cleanly",
|
||||
f"No unexpected {label} suspension or unhealthy reconciliation",
|
||||
"gitops",
|
||||
"flux_health",
|
||||
[
|
||||
step(
|
||||
"objects",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"get",
|
||||
kind,
|
||||
"--all-namespaces",
|
||||
"-o",
|
||||
FLUX_PROJECTION,
|
||||
record=False,
|
||||
)
|
||||
],
|
||||
{"step": "objects", "expected_suspensions": targets.expected_suspensions},
|
||||
)
|
||||
for kind, label in (
|
||||
("kustomizations.kustomize.toolkit.fluxcd.io", "Kustomization"),
|
||||
("helmreleases.helm.toolkit.fluxcd.io", "HelmRelease"),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def access_checks(targets: Targets) -> list[CheckSpec]:
|
||||
"""Return the access-denied, access-allowed, and GitOps health checks."""
|
||||
return [*_access(targets), *_gitops(targets)]
|
||||
374
scripts/ops/hermes_handoff_checks_delivery.py
Normal file
374
scripts/ops/hermes_handoff_checks_delivery.py
Normal file
@ -0,0 +1,374 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Acceptance checks for forge authority, node accounts, images, and regressions.
|
||||
|
||||
What the platform is allowed to do outside its own namespace: which forge
|
||||
authority a worker holds, what a node account is and is not, how an image is
|
||||
built and pinned, and whether the fixes the release depends on are actually
|
||||
running rather than merely merged.
|
||||
|
||||
Where a property has no side-effect-free live probe — a merge that must never be
|
||||
attempted, a branch protection the harness identity cannot read — the entry says
|
||||
so in its rationale and asserts the nearest fact that *is* observable, rather
|
||||
than quietly asserting nothing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hermes_handoff_catalog import (
|
||||
FORGE_CREDENTIAL_NAMES,
|
||||
INIT_NAME_PROJECTION,
|
||||
NODE_NAME_PROJECTION,
|
||||
OPERATOR,
|
||||
SELF,
|
||||
Targets,
|
||||
check,
|
||||
step,
|
||||
)
|
||||
from hermes_handoff_model import ATTEMPT, CheckSpec
|
||||
from hermes_handoff_policy import GITEA_CLIENT, projection, shell
|
||||
|
||||
|
||||
def _forge(targets: Targets) -> list[CheckSpec]:
|
||||
api = f"/api/v1/repos/{targets.repo}"
|
||||
return [
|
||||
check(
|
||||
"forge.worker-holds-no-git-credential-variable",
|
||||
"No forge credential is exported into the worker process",
|
||||
"forge",
|
||||
"names_absent",
|
||||
[step("env", SELF, *shell("env_names"))],
|
||||
{"step": "env", "names": FORGE_CREDENTIAL_NAMES},
|
||||
rationale="Authentication reaches Git through an askpass helper that reads runtime state, never through the environment.",
|
||||
),
|
||||
check(
|
||||
"forge.identity-is-not-an-administrator",
|
||||
"The forge identity the worker uses holds no administrative rights",
|
||||
"forge",
|
||||
"json_field",
|
||||
[step("user", SELF, GITEA_CLIENT, "GET", "/api/v1/user")],
|
||||
{"step": "user", "fields": {"is_admin": False}},
|
||||
),
|
||||
check(
|
||||
"forge.repository-authority-is-push-only",
|
||||
"The worker can push a branch but cannot administer the repository",
|
||||
"forge",
|
||||
"json_field",
|
||||
[step("repo", SELF, GITEA_CLIENT, "GET", api)],
|
||||
{
|
||||
"step": "repo",
|
||||
"fields": {
|
||||
"permissions.admin": False,
|
||||
"permissions.push": True,
|
||||
"permissions.pull": True,
|
||||
},
|
||||
},
|
||||
rationale="Merge, approve, close, and protection changes all require authority this identity is proven not to hold.",
|
||||
),
|
||||
check(
|
||||
"forge.administrative-route-is-refused",
|
||||
"An administrative forge route is refused for this identity",
|
||||
"forge",
|
||||
"denied",
|
||||
[
|
||||
step(
|
||||
"attempt",
|
||||
SELF,
|
||||
GITEA_CLIENT,
|
||||
"GET",
|
||||
f"{api}/branch_protections",
|
||||
kind=ATTEMPT,
|
||||
)
|
||||
],
|
||||
rationale="A live refusal, not a configuration reading. The same lack of authority is what blocks a merge route.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _nodes(targets: Targets) -> list[CheckSpec]:
|
||||
return [
|
||||
check(
|
||||
"nodes.hardening-covers-every-node",
|
||||
"The node account audit runs on every node in the cluster",
|
||||
"nodes",
|
||||
"distinct_count",
|
||||
[
|
||||
step(
|
||||
"coverage",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
"pods",
|
||||
"--selector",
|
||||
f"app={targets.node_daemonset}",
|
||||
"-o",
|
||||
NODE_NAME_PROJECTION,
|
||||
)
|
||||
],
|
||||
{
|
||||
"step": "coverage",
|
||||
"minimum": targets.node_count,
|
||||
"total_equals": targets.node_count,
|
||||
},
|
||||
rationale="A hardened fleet with one unenrolled node is an unhardened fleet.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _build(targets: Targets) -> list[CheckSpec]:
|
||||
pipeline = f"{targets.remote}/main:{targets.builder_pipeline}"
|
||||
return [
|
||||
check(
|
||||
"build.builder-service-account-is-tokenless",
|
||||
"The image builder service account mounts no API token",
|
||||
"build",
|
||||
"stdout_matches",
|
||||
[
|
||||
step(
|
||||
"serviceaccount",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.builder_namespace,
|
||||
"get",
|
||||
"serviceaccount",
|
||||
targets.builder_serviceaccount,
|
||||
"-o",
|
||||
projection("jsonpath={.automountServiceAccountToken}"),
|
||||
)
|
||||
],
|
||||
{"step": "serviceaccount", "equals": "false"},
|
||||
),
|
||||
check(
|
||||
"build.builder-runs-unprivileged-with-exact-capabilities",
|
||||
"The build pod is unprivileged and adds only the expected capabilities",
|
||||
"build",
|
||||
"names_present",
|
||||
[step("pipeline", OPERATOR, "git", "show", pipeline, record=False)],
|
||||
{
|
||||
"step": "pipeline",
|
||||
"names": (
|
||||
targets.builder_capabilities,
|
||||
"privileged: false",
|
||||
"allowPrivilegeEscalation: false",
|
||||
"automountServiceAccountToken: false",
|
||||
'drop: ["ALL"]',
|
||||
),
|
||||
},
|
||||
rationale="Exact lines, not a substring sweep: a widened capability set has to change one of these lines to take effect.",
|
||||
),
|
||||
check(
|
||||
"build.harbor-immutability-rule-is-applied",
|
||||
"The Harbor immutable-tag rule for the agent image has been applied",
|
||||
"build",
|
||||
"lines_match",
|
||||
[
|
||||
step(
|
||||
"job",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.harbor_namespace,
|
||||
"get",
|
||||
"job",
|
||||
targets.harbor_immutability_job,
|
||||
"-o",
|
||||
projection("jsonpath={.status.succeeded}"),
|
||||
)
|
||||
],
|
||||
{"step": "job", "pattern": r"^[1-9][0-9]*$", "minimum": 1},
|
||||
rationale="The job verifies the rule it creates, so a completed run is evidence the rule matched the expected contract.",
|
||||
),
|
||||
_release_lineage(targets),
|
||||
]
|
||||
|
||||
|
||||
def _release_lineage(targets: Targets) -> CheckSpec:
|
||||
"""Bind the exact reviewed release from Git through the Ready pods."""
|
||||
container = targets.agent_container
|
||||
namespace = targets.namespace
|
||||
deployment = f"deploy/{targets.agent_deployment}"
|
||||
deployment_projection = projection(
|
||||
(
|
||||
"jsonpath={.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}"
|
||||
"{.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}"
|
||||
"{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}"
|
||||
f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}'
|
||||
).replace("'", '"')
|
||||
)
|
||||
pod_projection = projection(
|
||||
(
|
||||
"jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.status.phase}{'\\t'}"
|
||||
"{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}"
|
||||
f'{{.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}'
|
||||
f'{{.status.containerStatuses[?(@.name=="{container}")].imageID}}{{"\\t"}}'
|
||||
"{.metadata.labels.pod-template-hash}{'\\n'}{end}"
|
||||
).replace("'", '"')
|
||||
)
|
||||
replicaset_projection = projection(
|
||||
(
|
||||
"jsonpath={range .items[*]}{.metadata.name}{'\\t'}"
|
||||
"{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}"
|
||||
"{.status.readyReplicas}{'\\t'}{.status.availableReplicas}{'\\t'}"
|
||||
f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}'
|
||||
"{.metadata.labels.pod-template-hash}{'\\n'}{end}"
|
||||
).replace("'", '"')
|
||||
)
|
||||
flux_projection = projection(
|
||||
(
|
||||
"jsonpath={.metadata.name}{'\\t'}{.metadata.generation}{'\\t'}"
|
||||
"{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}"
|
||||
"{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}"
|
||||
"{.status.lastAppliedRevision}"
|
||||
).replace("'", '"')
|
||||
)
|
||||
return check(
|
||||
"release.exact-lineage-is-running",
|
||||
"Remote main, reviewed PR, image, build, Flux, and running revision are identical",
|
||||
"release",
|
||||
"release_lineage",
|
||||
[
|
||||
step(
|
||||
"remote-main",
|
||||
OPERATOR,
|
||||
"git",
|
||||
"ls-remote",
|
||||
targets.remote,
|
||||
"refs/heads/main",
|
||||
),
|
||||
step(
|
||||
"reviewed-pr",
|
||||
OPERATOR,
|
||||
GITEA_CLIENT,
|
||||
"GET",
|
||||
f"/api/v1/repos/{targets.repo}/pulls/{targets.reviewed_pr_number}",
|
||||
),
|
||||
step(
|
||||
"build-source",
|
||||
OPERATOR,
|
||||
"git",
|
||||
"rev-parse",
|
||||
f"{targets.remote}/main",
|
||||
),
|
||||
step(
|
||||
"deployment",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
namespace,
|
||||
"get",
|
||||
deployment,
|
||||
"-o",
|
||||
deployment_projection,
|
||||
),
|
||||
step(
|
||||
"pods",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
namespace,
|
||||
"get",
|
||||
"pods",
|
||||
"--selector",
|
||||
f"app={targets.agent_deployment}",
|
||||
"-o",
|
||||
pod_projection,
|
||||
),
|
||||
step(
|
||||
"replicasets",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
namespace,
|
||||
"get",
|
||||
"replicasets",
|
||||
"--selector",
|
||||
f"app={targets.agent_deployment}",
|
||||
"-o",
|
||||
replicaset_projection,
|
||||
),
|
||||
step(
|
||||
"flux",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
"flux-system",
|
||||
"get",
|
||||
"kustomization/hermes",
|
||||
"-o",
|
||||
flux_projection,
|
||||
),
|
||||
],
|
||||
{
|
||||
"main_sha": targets.remote_main_sha,
|
||||
"head_sha": targets.reviewed_head_sha,
|
||||
"head_ref": targets.reviewed_head_ref,
|
||||
"pr_number": targets.reviewed_pr_number,
|
||||
"image": targets.agent_image,
|
||||
"build_sha": targets.build_sha,
|
||||
"deployment_revision": targets.deployment_revision,
|
||||
},
|
||||
rationale="A PASS requires one exact release identity across every source and running object.",
|
||||
)
|
||||
|
||||
|
||||
def _reliability(targets: Targets) -> list[CheckSpec]:
|
||||
return [
|
||||
check(
|
||||
"reliability.finalization-and-replay-patches-are-deployed",
|
||||
"The agent workload runs the finalization, replay, and session patches",
|
||||
"reliability",
|
||||
"names_present",
|
||||
[
|
||||
step(
|
||||
"init",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"deploy/{targets.agent_deployment}",
|
||||
"-o",
|
||||
INIT_NAME_PROJECTION,
|
||||
)
|
||||
],
|
||||
{"step": "init", "names": targets.agent_init_containers},
|
||||
rationale="The regression suites live in the repository; this asserts the fixes they cover are actually running.",
|
||||
),
|
||||
check(
|
||||
"reliability.regression-suites-are-present-on-main",
|
||||
"The decomposition and worker-recovery regression suites are on main",
|
||||
"reliability",
|
||||
"names_present",
|
||||
[
|
||||
step(
|
||||
"suites",
|
||||
OPERATOR,
|
||||
"git",
|
||||
"ls-tree",
|
||||
"--name-only",
|
||||
f"{targets.remote}/main",
|
||||
"testing/tests/",
|
||||
)
|
||||
],
|
||||
{
|
||||
"step": "suites",
|
||||
"names": (
|
||||
"testing/tests/test_hermes_cli_lanes.py",
|
||||
"testing/tests/test_hermes_coordinator.py",
|
||||
"testing/tests/test_hermes_worker_recovery.py",
|
||||
),
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def delivery_checks(targets: Targets) -> list[CheckSpec]:
|
||||
"""Return the forge, node, build, and reliability checks."""
|
||||
return [
|
||||
*_forge(targets),
|
||||
*_nodes(targets),
|
||||
*_build(targets),
|
||||
*_reliability(targets),
|
||||
]
|
||||
387
scripts/ops/hermes_handoff_checks_platform.py
Normal file
387
scripts/ops/hermes_handoff_checks_platform.py
Normal file
@ -0,0 +1,387 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Acceptance checks for the release baseline, provider identity, and routing.
|
||||
|
||||
These are the facts a handoff rests on before any authority question is asked:
|
||||
that the tree under test is the merged one, that both providers authenticate
|
||||
through the subscriptions they are supposed to, that Switchyard actually decided
|
||||
the routes it claims to, and that chat, agent, and triage are still three
|
||||
distinct scopes rather than one surface wearing three hostnames.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hermes_handoff_catalog import (
|
||||
CLAUDE_HEALTH,
|
||||
CLAUDE_HEALTH_FIELDS,
|
||||
CODEX_HEALTH,
|
||||
CODEX_HEALTH_FIELDS,
|
||||
ENV_NAME_PROJECTION,
|
||||
OPERATOR,
|
||||
PROVIDER_API_KEY_NAMES,
|
||||
ROUTING_LOG,
|
||||
SELF,
|
||||
SWITCHYARD,
|
||||
Targets,
|
||||
check,
|
||||
step,
|
||||
)
|
||||
from hermes_handoff_model import CheckSpec
|
||||
from hermes_handoff_policy import GITEA_CLIENT, projection, shell
|
||||
|
||||
|
||||
def _baseline(targets: Targets) -> list[CheckSpec]:
|
||||
api = f"/api/v1/repos/{targets.repo}"
|
||||
dependency_heads = dict(targets.dependency_heads)
|
||||
checks = [
|
||||
check(
|
||||
"baseline.local-origin-main-is-current",
|
||||
"The local origin/main ref equals the fixed remote-main SHA",
|
||||
"baseline",
|
||||
"stdout_matches",
|
||||
[step("main", OPERATOR, "git", "rev-parse", f"{targets.remote}/main")],
|
||||
{"step": "main", "equals": targets.remote_main_sha},
|
||||
rationale="The ancestry check is meaningful only after the local tracking ref is bound to the remote observation.",
|
||||
),
|
||||
check(
|
||||
"baseline.origin-main-descends-merged-work",
|
||||
"origin/main descends from the merged Hermes isolation baseline",
|
||||
"baseline",
|
||||
"allowed",
|
||||
[
|
||||
step(
|
||||
"ancestor",
|
||||
OPERATOR,
|
||||
"git",
|
||||
"merge-base",
|
||||
"--is-ancestor",
|
||||
targets.baseline_commit,
|
||||
f"{targets.remote}/main",
|
||||
)
|
||||
],
|
||||
rationale="Everything downstream assumes the merged worker-isolation baseline is present.",
|
||||
),
|
||||
check(
|
||||
"baseline.pending-pull-requests-recorded",
|
||||
"Exact open pull-request heads are recorded, not assumed",
|
||||
"baseline",
|
||||
"json_record",
|
||||
[
|
||||
step(
|
||||
"open",
|
||||
OPERATOR,
|
||||
GITEA_CLIENT,
|
||||
"GET",
|
||||
f"{api}/pulls?state=open&limit=50",
|
||||
)
|
||||
],
|
||||
{
|
||||
"step": "open",
|
||||
"record": {
|
||||
"numbers": "[].number",
|
||||
"head_refs": "[].head.ref",
|
||||
"head_shas": "[].head.sha",
|
||||
"base_shas": "[].base.sha",
|
||||
"drafts": "[].draft",
|
||||
"mergeable": "[].mergeable",
|
||||
},
|
||||
},
|
||||
rationale="The runbook has to name the heads it was verified against; stale heads are how a release verifies the wrong tree.",
|
||||
),
|
||||
check(
|
||||
"baseline.operator-checkout-clean",
|
||||
"The operator checkout has no uncommitted drift",
|
||||
"baseline",
|
||||
"stdout_matches",
|
||||
[step("status", OPERATOR, "git", "status", "--porcelain")],
|
||||
{"step": "status", "equals": ""},
|
||||
mandatory=False,
|
||||
rationale="Advisory: a dirty checkout does not invalidate cluster evidence, but it does muddy what was compared.",
|
||||
),
|
||||
]
|
||||
checks += [
|
||||
check(
|
||||
f"baseline.dependency-pr-{number}-merged",
|
||||
f"Pull request #{number} is merged into main",
|
||||
"baseline",
|
||||
"json_field",
|
||||
[step("pr", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls/{number}")],
|
||||
{
|
||||
"step": "pr",
|
||||
"fields": {
|
||||
"merged": True,
|
||||
"base.ref": "main",
|
||||
"head.sha": dependency_heads.get(number, ""),
|
||||
},
|
||||
},
|
||||
rationale="This harness certifies the post-merge platform; an unmerged dependency means it is measuring something else.",
|
||||
)
|
||||
for number in targets.dependency_pull_requests
|
||||
]
|
||||
return checks
|
||||
|
||||
|
||||
def _identity(targets: Targets) -> list[CheckSpec]:
|
||||
codex = shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS)
|
||||
claude = shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS)
|
||||
agent = f"deploy/{targets.agent_deployment}"
|
||||
return [
|
||||
check(
|
||||
"identity.codex-is-chatgpt-subscription",
|
||||
"Codex authenticates through the ChatGPT subscription, not an API key",
|
||||
"identity",
|
||||
"json_field",
|
||||
[step("health", SELF, *codex)],
|
||||
{
|
||||
"step": "health",
|
||||
"fields": {
|
||||
"authenticated": True,
|
||||
"transport": "codex-chatgpt-subscription",
|
||||
"state": "available",
|
||||
},
|
||||
},
|
||||
rationale="Only allow-listed status fields are read; the credential itself is never opened.",
|
||||
),
|
||||
check(
|
||||
"identity.codex-evidence-is-fresh",
|
||||
"Codex authentication evidence is current",
|
||||
"identity",
|
||||
"json_recent",
|
||||
[step("health", SELF, *codex)],
|
||||
{
|
||||
"step": "health",
|
||||
"now": targets.now,
|
||||
"fields": {"checked_at": targets.max_evidence_age_seconds},
|
||||
},
|
||||
rationale="Stale provider evidence describes a state the release no longer has.",
|
||||
),
|
||||
check(
|
||||
"identity.claude-is-firstparty-subscription",
|
||||
"Claude authenticates as a claude.ai first-party subscription",
|
||||
"identity",
|
||||
"json_field",
|
||||
[step("health", SELF, *claude)],
|
||||
{
|
||||
"step": "health",
|
||||
"fields": {
|
||||
"authenticated": True,
|
||||
"api_provider": "firstParty",
|
||||
"auth_method": "claude.ai",
|
||||
"transport": "claude-code-cli-subscription",
|
||||
"state": "available",
|
||||
},
|
||||
},
|
||||
),
|
||||
check(
|
||||
"identity.claude-evidence-is-fresh",
|
||||
"Claude authentication evidence is current",
|
||||
"identity",
|
||||
"json_recent",
|
||||
[step("health", SELF, *claude)],
|
||||
{
|
||||
"step": "health",
|
||||
"now": targets.now,
|
||||
"fields": {"checked_at": targets.max_evidence_age_seconds},
|
||||
},
|
||||
),
|
||||
check(
|
||||
"identity.no-provider-api-key-in-pod",
|
||||
"No provider API-key variable is set in the running agent process",
|
||||
"identity",
|
||||
"names_absent",
|
||||
[step("env", SELF, *shell("env_names"))],
|
||||
{
|
||||
"step": "env",
|
||||
"names": PROVIDER_API_KEY_NAMES,
|
||||
"contains": ("API_KEY",),
|
||||
},
|
||||
rationale="Names only. The probe lists variable names and never their values.",
|
||||
),
|
||||
check(
|
||||
"identity.no-provider-api-key-in-manifest",
|
||||
"No provider API-key variable is declared on the agent workload",
|
||||
"identity",
|
||||
"names_absent",
|
||||
[
|
||||
step(
|
||||
"env",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
agent,
|
||||
"-o",
|
||||
ENV_NAME_PROJECTION,
|
||||
)
|
||||
],
|
||||
{"step": "env", "names": PROVIDER_API_KEY_NAMES, "contains": ("API_KEY",)},
|
||||
rationale="The operator counterpart to the in-pod probe: desired state and running state are asserted separately.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _routing(targets: Targets) -> list[CheckSpec]:
|
||||
switchyard = f"deploy/{targets.switchyard_deployment}"
|
||||
tail = shell("tail_lines", path=ROUTING_LOG, limit=str(targets.routing_tail_lines))
|
||||
routing_step = step(
|
||||
"routing", SWITCHYARD, *tail, record=False, max_bytes=targets.routing_tail_bytes
|
||||
)
|
||||
return [
|
||||
check(
|
||||
"routing.switchyard-is-available",
|
||||
"Switchyard has a ready replica serving every model-call boundary",
|
||||
"routing",
|
||||
"lines_match",
|
||||
[
|
||||
step(
|
||||
"deployment",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
switchyard,
|
||||
"-o",
|
||||
projection("jsonpath={.status.readyReplicas}"),
|
||||
)
|
||||
],
|
||||
{"step": "deployment", "pattern": r"^[1-9][0-9]*$", "minimum": 1},
|
||||
),
|
||||
check(
|
||||
"routing.provider-effort-and-fallback-evidence",
|
||||
"Routing evidence covers both providers, all graded efforts, and a real fallback",
|
||||
"routing",
|
||||
"routing_evidence",
|
||||
[routing_step],
|
||||
{
|
||||
"step": "routing",
|
||||
"providers": ("codex", "claude"),
|
||||
"efforts": ("medium", "high", "xhigh"),
|
||||
"lanes": ("route", "worker"),
|
||||
"require_fallback_evidence": True,
|
||||
"max_age_seconds": targets.max_evidence_age_seconds,
|
||||
"now": targets.now,
|
||||
},
|
||||
rationale="The routing log records the decided route per boundary; the tail is parsed here and never pasted into the report.",
|
||||
),
|
||||
check(
|
||||
"routing.codex-latency-recorded",
|
||||
"Codex boundary latency is measured, not assumed",
|
||||
"routing",
|
||||
"json_numeric",
|
||||
[
|
||||
step(
|
||||
"health",
|
||||
SELF,
|
||||
*shell(
|
||||
"json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS
|
||||
),
|
||||
)
|
||||
],
|
||||
{"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}},
|
||||
),
|
||||
check(
|
||||
"routing.claude-latency-recorded",
|
||||
"Claude boundary latency is measured, not assumed",
|
||||
"routing",
|
||||
"json_numeric",
|
||||
[
|
||||
step(
|
||||
"health",
|
||||
SELF,
|
||||
*shell(
|
||||
"json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS
|
||||
),
|
||||
)
|
||||
],
|
||||
{"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _scope_profile(
|
||||
targets: Targets, name: str, workload: str, container: str, expected: str
|
||||
) -> CheckSpec:
|
||||
profile_projection = projection(
|
||||
f'jsonpath={{.spec.template.spec.containers[?(@.name=="{container}")]'
|
||||
'.env[?(@.name=="HERMES_AUTO_ROUTER_PROFILE")].value}'
|
||||
)
|
||||
return check(
|
||||
f"scopes.{name}-profile-is-distinct",
|
||||
f"The {name} surface runs under its own routing profile",
|
||||
"scopes",
|
||||
"stdout_matches",
|
||||
[
|
||||
step(
|
||||
"profile",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
workload,
|
||||
"-o",
|
||||
profile_projection,
|
||||
)
|
||||
],
|
||||
{"step": "profile", "equals": expected},
|
||||
rationale="Chat, agent, and triage must not collapse into one scope; each states its own profile.",
|
||||
)
|
||||
|
||||
|
||||
def _scopes(targets: Targets) -> list[CheckSpec]:
|
||||
return [
|
||||
_scope_profile(
|
||||
targets,
|
||||
"agent",
|
||||
f"deploy/{targets.agent_deployment}",
|
||||
targets.agent_container,
|
||||
"agent",
|
||||
),
|
||||
_scope_profile(
|
||||
targets,
|
||||
"chat",
|
||||
f"statefulset/{targets.chat_statefulset}",
|
||||
targets.chat_container,
|
||||
"chat",
|
||||
),
|
||||
_scope_profile(
|
||||
targets,
|
||||
"triage",
|
||||
f"deploy/{targets.triage_deployment}",
|
||||
targets.triage_container,
|
||||
"triage",
|
||||
),
|
||||
check(
|
||||
"scopes.surfaces-have-distinct-hosts",
|
||||
"Chat, agent, and triage are published on distinct hostnames",
|
||||
"scopes",
|
||||
"distinct_count",
|
||||
[
|
||||
step(
|
||||
"hosts",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
"ingress",
|
||||
"-o",
|
||||
projection(
|
||||
'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}'
|
||||
),
|
||||
)
|
||||
],
|
||||
{"step": "hosts", "minimum": 3},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def platform_checks(targets: Targets) -> list[CheckSpec]:
|
||||
"""Return the baseline, identity, routing, and scope checks."""
|
||||
return [
|
||||
*_baseline(targets),
|
||||
*_identity(targets),
|
||||
*_routing(targets),
|
||||
*_scopes(targets),
|
||||
]
|
||||
376
scripts/ops/hermes_handoff_checks_workers.py
Normal file
376
scripts/ops/hermes_handoff_checks_workers.py
Normal file
@ -0,0 +1,376 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Acceptance checks for the distributed worker pool and the visible surfaces.
|
||||
|
||||
A pool is only distributed if its workers occupy distinct nodes and distinct
|
||||
volumes and leave state ownership with the coordinator; a replica count alone
|
||||
satisfies none of that. A surface is only working if the work it did is visible
|
||||
in it, which is why the session and Kanban checks look at the same data the UI
|
||||
renders rather than at whether a pod is Running.
|
||||
|
||||
The ephemeral entries are placeholders in a default run. They report NOT_RUN
|
||||
until the mutation mode is separately armed, and the armed mode replaces them
|
||||
with real, mandatory results.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hermes_handoff_catalog import (
|
||||
CHAT,
|
||||
INIT_NAME_PROJECTION,
|
||||
OPERATOR,
|
||||
POOL_PROJECTION,
|
||||
SELF,
|
||||
Targets,
|
||||
check,
|
||||
)
|
||||
from hermes_handoff_catalog import step as step
|
||||
from hermes_handoff_model import EPHEMERAL, CheckSpec
|
||||
from hermes_handoff_policy import projection
|
||||
|
||||
KANBAN_STATUSES = [
|
||||
"archived",
|
||||
"blocked",
|
||||
"done",
|
||||
"ready",
|
||||
"review",
|
||||
"running",
|
||||
"scheduled",
|
||||
"todo",
|
||||
"triage",
|
||||
]
|
||||
|
||||
|
||||
def _pool(targets: Targets) -> list[CheckSpec]:
|
||||
selector = targets.pool_selector
|
||||
return [
|
||||
check(
|
||||
"pool.three-workers-have-distinct-nodes-and-volumes",
|
||||
"Exactly three Ready ordinal workers occupy distinct nodes and claims",
|
||||
"pool",
|
||||
"pool_topology",
|
||||
[
|
||||
step(
|
||||
"statefulset",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"statefulset/{targets.pool_statefulset}",
|
||||
"-o",
|
||||
projection(
|
||||
"jsonpath={.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}{.status.currentReplicas}{'\\t'}{.spec.template.spec.automountServiceAccountToken}".replace(
|
||||
"'", '"'
|
||||
)
|
||||
),
|
||||
),
|
||||
step(
|
||||
"workers",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
"pods",
|
||||
"--selector",
|
||||
selector,
|
||||
"-o",
|
||||
POOL_PROJECTION,
|
||||
),
|
||||
],
|
||||
{
|
||||
"state_step": "statefulset",
|
||||
"worker_step": "workers",
|
||||
"replicas": targets.pool_replicas,
|
||||
"name": targets.pool_statefulset,
|
||||
},
|
||||
rationale="Replica count, Ready state, ordinal coverage, node placement, PVC isolation, and tokenlessness are one fail-closed assertion.",
|
||||
),
|
||||
check(
|
||||
"pool.coordinator-retains-sole-state-ownership",
|
||||
"No pool worker mounts the coordinator's durable state",
|
||||
"pool",
|
||||
"names_absent",
|
||||
[
|
||||
step(
|
||||
"claims",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"statefulset/{targets.pool_statefulset}",
|
||||
"-o",
|
||||
projection(
|
||||
"jsonpath={range .spec.template.spec.volumes[*]}{.name}{'='}"
|
||||
"{.persistentVolumeClaim.claimName}{'\\n'}{end}".replace(
|
||||
"'", '"'
|
||||
)
|
||||
),
|
||||
)
|
||||
],
|
||||
{"step": "claims", "contains": targets.coordinator_claims},
|
||||
rationale="Two writers on one SQLite state file is the duplicate-work and lost-result failure this pool exists to avoid. One line per template volume keeps a drifted or empty projection NOT_RUN instead of a silent pass.",
|
||||
),
|
||||
*_pool_env(targets),
|
||||
]
|
||||
|
||||
|
||||
def _pool_env(targets: Targets) -> list[CheckSpec]:
|
||||
"""Return the mandatory assignment-safety configuration check."""
|
||||
return [
|
||||
check(
|
||||
"pool.assignment-safety-knobs-are-configured",
|
||||
"Pool workers carry the lease and de-duplication configuration",
|
||||
"pool",
|
||||
"names_present",
|
||||
[
|
||||
step(
|
||||
"env",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"statefulset/{targets.pool_statefulset}",
|
||||
"-o",
|
||||
projection(
|
||||
'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}'
|
||||
),
|
||||
)
|
||||
],
|
||||
{"step": "env", "names": targets.pool_worker_env},
|
||||
rationale="Deployed configuration only; the stale-claim and duplicate-dispatch behaviour itself is covered by the pool's own regression suite.",
|
||||
),
|
||||
check(
|
||||
"pool.assignment-fieldrefs-are-bound",
|
||||
"Worker node and ordinal identities come from fixed downward-API fields",
|
||||
"pool",
|
||||
"names_present",
|
||||
[
|
||||
step(
|
||||
"source",
|
||||
OPERATOR,
|
||||
"git",
|
||||
"show",
|
||||
f"{targets.remote}/main:services/hermes/execution-worker-statefulset.yaml",
|
||||
record=False,
|
||||
)
|
||||
],
|
||||
{
|
||||
"step": "source",
|
||||
"names": (
|
||||
"name: HERMES_WORKER_ORDINAL",
|
||||
"fieldPath: metadata.labels['apps.kubernetes.io/pod-index']",
|
||||
"name: HERMES_WORKER_NODE",
|
||||
"fieldPath: spec.nodeName",
|
||||
),
|
||||
},
|
||||
rationale="This prevents caller-authored ordinal/node values from impersonating another worker assignment.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _surfaces(targets: Targets) -> list[CheckSpec]:
|
||||
checks = [
|
||||
check(
|
||||
"surfaces.kanban-activity-is-visible",
|
||||
"Durable Kanban activity is visible to the agent surface",
|
||||
"surfaces",
|
||||
"all_of_field",
|
||||
[
|
||||
step(
|
||||
"tasks",
|
||||
SELF,
|
||||
"hermes",
|
||||
"kanban",
|
||||
"list",
|
||||
"--json",
|
||||
record=False,
|
||||
max_bytes=targets.listing_bytes,
|
||||
)
|
||||
],
|
||||
{"step": "tasks", "key_field": "id", "fields": {"status": KANBAN_STATUSES}},
|
||||
rationale="Non-empty and well-formed. An empty board is NOT_RUN, because it cannot distinguish a quiet week from a broken read.",
|
||||
),
|
||||
check(
|
||||
"surfaces.durable-sessions-render-without-blank-rows",
|
||||
"Durable sessions are listed with a preview, a source, and an id",
|
||||
"surfaces",
|
||||
"lines_match",
|
||||
[
|
||||
step(
|
||||
"sessions",
|
||||
SELF,
|
||||
"hermes",
|
||||
"sessions",
|
||||
"list",
|
||||
"--limit",
|
||||
"20",
|
||||
record=False,
|
||||
)
|
||||
],
|
||||
{"step": "sessions", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
|
||||
rationale="A row whose leading column is blank is the session-render regression seen from the data side.",
|
||||
),
|
||||
check(
|
||||
"surfaces.agent-ui-session-patches-are-deployed",
|
||||
"The agent surface runs the session-activity rendering patches",
|
||||
"surfaces",
|
||||
"names_present",
|
||||
[
|
||||
step(
|
||||
"init",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"deploy/{targets.agent_deployment}",
|
||||
"-o",
|
||||
INIT_NAME_PROJECTION,
|
||||
)
|
||||
],
|
||||
{
|
||||
"step": "init",
|
||||
"names": ("patch-web-session-activity", "patch-api-server-sessions"),
|
||||
},
|
||||
rationale="Blank session rows were a rendering regression; this asserts the patches that fixed it are in the running pod.",
|
||||
),
|
||||
check(
|
||||
"surfaces.chat-session-patches-are-deployed",
|
||||
"The chat surface runs the API-session continuity patches",
|
||||
"surfaces",
|
||||
"names_present",
|
||||
[
|
||||
step(
|
||||
"init",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"statefulset/{targets.chat_statefulset}",
|
||||
"-o",
|
||||
INIT_NAME_PROJECTION,
|
||||
)
|
||||
],
|
||||
{"step": "init", "names": targets.chat_init_containers},
|
||||
),
|
||||
check(
|
||||
"surfaces.chat-telegram-topic-state-is-durable",
|
||||
"Telegram topic and tenant state is held on durable storage",
|
||||
"surfaces",
|
||||
"names_present",
|
||||
[
|
||||
step(
|
||||
"claims",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"deploy/{targets.chat_router_deployment}",
|
||||
"-o",
|
||||
projection(
|
||||
'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}'
|
||||
),
|
||||
)
|
||||
],
|
||||
{"step": "claims", "contains": ("state",)},
|
||||
rationale="Topic selection survives a router restart only if the tenant state file is on a claim rather than in the pod.",
|
||||
),
|
||||
]
|
||||
checks.append(
|
||||
check(
|
||||
"surfaces.chat-telegram-sessions-are-continuous",
|
||||
"The selected chat tenant still holds durable Telegram-sourced sessions",
|
||||
"surfaces",
|
||||
"lines_match",
|
||||
[
|
||||
step(
|
||||
"telegram",
|
||||
CHAT,
|
||||
"hermes",
|
||||
"sessions",
|
||||
"list",
|
||||
"--source",
|
||||
"telegram",
|
||||
"--limit",
|
||||
"20",
|
||||
record=False,
|
||||
)
|
||||
],
|
||||
{"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
|
||||
rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.",
|
||||
)
|
||||
)
|
||||
checks.append(
|
||||
check(
|
||||
"surfaces.chat-runs-the-telegram-topic-revision",
|
||||
"The chat surface runs the durable Telegram topic configuration",
|
||||
"surfaces",
|
||||
"stdout_matches",
|
||||
[
|
||||
step(
|
||||
"revision",
|
||||
OPERATOR,
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"statefulset/{targets.chat_statefulset}",
|
||||
"-o",
|
||||
projection(
|
||||
"jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}"
|
||||
),
|
||||
)
|
||||
],
|
||||
{"step": "revision", "equals": targets.chat_config_revision},
|
||||
rationale="Telegram topic continuity is carried by the config revision the pod template pins.",
|
||||
)
|
||||
)
|
||||
return checks
|
||||
|
||||
|
||||
def _ephemeral() -> list[CheckSpec]:
|
||||
return [
|
||||
check(
|
||||
f"ephemeral.{name}",
|
||||
title,
|
||||
"ephemeral",
|
||||
"not_armed",
|
||||
[],
|
||||
{"reason": reason},
|
||||
mandatory=False,
|
||||
scope=EPHEMERAL,
|
||||
rationale="Mutating checks are separately armed. A default run reports them NOT_RUN and never touches the forge.",
|
||||
)
|
||||
for name, title, reason in (
|
||||
(
|
||||
"feature-branch-push",
|
||||
"A unique ephemeral feature branch can be pushed and removed",
|
||||
"ephemeral mutation mode is not armed",
|
||||
),
|
||||
(
|
||||
"draft-pull-request",
|
||||
"A draft pull request can be opened and closed against the ephemeral branch",
|
||||
"ephemeral mutation mode is not armed",
|
||||
),
|
||||
(
|
||||
"protected-branch-refusal",
|
||||
"Arming against a protected branch is refused before any network call",
|
||||
"ephemeral mutation mode is not armed",
|
||||
),
|
||||
(
|
||||
"cleanup-verified",
|
||||
"Every ephemeral ref and pull request is removed and the removal is verified",
|
||||
"ephemeral mutation mode is not armed",
|
||||
),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def worker_checks(targets: Targets) -> list[CheckSpec]:
|
||||
"""Return the pool, surface, and ephemeral-placeholder checks."""
|
||||
return [*_pool(targets), *_surfaces(targets), *_ephemeral()]
|
||||
439
scripts/ops/hermes_handoff_ephemeral.py
Normal file
439
scripts/ops/hermes_handoff_ephemeral.py
Normal file
@ -0,0 +1,439 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Discoverable, residue-surfacing, self-cleaning armed ephemeral run.
|
||||
|
||||
The create response — not the index — is the authoritative name of a pull
|
||||
request this harness made, and the index is read to its last page. When either
|
||||
source is uncertain the run fails closed and reports the exact residue plus the
|
||||
exact commands that remove it, so an operator is never left guessing what was
|
||||
created.
|
||||
|
||||
Arming, provenance, and push-target rules live in :mod:`hermes_handoff_arming`
|
||||
and are re-exported here for callers that treat armed mode as one surface.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
from hermes_handoff_arming import (
|
||||
CONFIRMATION,
|
||||
REF_PREFIX,
|
||||
ArmingError,
|
||||
ArmRequest,
|
||||
assert_push_target_allowed,
|
||||
bounded_read,
|
||||
normalise_branch,
|
||||
preflight,
|
||||
)
|
||||
from hermes_handoff_exec import Outcome, Runner, Vantage
|
||||
from hermes_handoff_model import EPHEMERAL, FAIL, NOT_RUN, PASS, CheckResult, CheckSpec
|
||||
from hermes_handoff_policy import (
|
||||
EXPECTED_BASE,
|
||||
GITEA_CLIENT,
|
||||
register_ephemeral_pull,
|
||||
)
|
||||
from hermes_handoff_rules import strict_json
|
||||
|
||||
__all__ = [
|
||||
"CONFIRMATION",
|
||||
"DISCOVERY_PAGE_BUDGET",
|
||||
"DISCOVERY_PAGE_SIZE",
|
||||
"REF_PREFIX",
|
||||
"ArmRequest",
|
||||
"ArmingError",
|
||||
"assert_push_target_allowed",
|
||||
"bounded_read",
|
||||
"normalise_branch",
|
||||
"preflight",
|
||||
"run_armed",
|
||||
]
|
||||
|
||||
DRAFT_TITLE_PREFIX = "WIP: "
|
||||
DISCOVERY_PAGE_SIZE = 50
|
||||
DISCOVERY_PAGE_BUDGET = 20
|
||||
|
||||
|
||||
def _spec(identifier: str, title: str) -> CheckSpec:
|
||||
return CheckSpec(
|
||||
id=f"ephemeral.{identifier}",
|
||||
title=title,
|
||||
group="ephemeral",
|
||||
rule="armed",
|
||||
steps=(),
|
||||
mandatory=True,
|
||||
scope=EPHEMERAL,
|
||||
)
|
||||
|
||||
|
||||
def _result(
|
||||
identifier: str,
|
||||
title: str,
|
||||
status: str,
|
||||
reason: str,
|
||||
outcomes: Sequence[Outcome] = (),
|
||||
evidence: dict[str, Any] | None = None,
|
||||
) -> CheckResult:
|
||||
return CheckResult(
|
||||
spec=_spec(identifier, title),
|
||||
status=status,
|
||||
reason=reason,
|
||||
evidence=evidence or {},
|
||||
outcomes=list(outcomes),
|
||||
)
|
||||
|
||||
|
||||
def _protected_refusal_result() -> CheckResult:
|
||||
accepted: list[str] = []
|
||||
for candidate in ("main", "master", "refs/heads/main", "HEAD", f"{REF_PREFIX}/x"):
|
||||
try:
|
||||
assert_push_target_allowed(candidate)
|
||||
except ArmingError:
|
||||
continue
|
||||
accepted.append(candidate)
|
||||
return _result(
|
||||
"protected-branch-refusal",
|
||||
"Arming against a protected branch is refused before any network call",
|
||||
FAIL if accepted else PASS,
|
||||
f"guard accepted {accepted}"
|
||||
if accepted
|
||||
else "every protected and malformed target was refused",
|
||||
evidence={"accepted": accepted},
|
||||
)
|
||||
|
||||
|
||||
def _detail(outcome: Outcome) -> str:
|
||||
return (
|
||||
outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}"
|
||||
)
|
||||
|
||||
|
||||
def _pr_problems(payload: Any, request: ArmRequest) -> list[str]:
|
||||
if not isinstance(payload, dict):
|
||||
return ["pull request is not an object"]
|
||||
wanted = {
|
||||
"state": "open",
|
||||
"draft": True,
|
||||
"merged": False,
|
||||
"base": {"ref": EXPECTED_BASE, "sha": request.expected_base_sha},
|
||||
"head": {"ref": request.ref, "sha": request.expected_head},
|
||||
}
|
||||
problems: list[str] = []
|
||||
if (
|
||||
not isinstance(payload.get("number"), int)
|
||||
or isinstance(payload.get("number"), bool)
|
||||
or payload["number"] <= 0
|
||||
):
|
||||
problems.append("number is malformed")
|
||||
for field in ("state", "draft", "merged"):
|
||||
if (
|
||||
type(payload.get(field)) is not type(wanted[field])
|
||||
or payload.get(field) != wanted[field]
|
||||
):
|
||||
problems.append(f"{field} does not match")
|
||||
for group, values in (("base", wanted["base"]), ("head", wanted["head"])):
|
||||
observed = payload.get(group)
|
||||
if not isinstance(observed, dict):
|
||||
problems.append(f"{group} is malformed")
|
||||
continue
|
||||
for field, value in values.items():
|
||||
if observed.get(field) != value:
|
||||
problems.append(f"{group}.{field} does not match")
|
||||
return problems
|
||||
|
||||
|
||||
def _created_number(outcome: Outcome, request: ArmRequest) -> tuple[int, str]:
|
||||
"""Return the number the create response itself reports.
|
||||
|
||||
The create response is the only source that names a pull request the harness
|
||||
definitely made. Relying on the index alone means an index that fails or
|
||||
answers ambiguously leaves an unnamed, unclosed pull request behind.
|
||||
"""
|
||||
if not outcome.ok:
|
||||
return (0, f"pull-request creation failed: {_detail(outcome)}")
|
||||
try:
|
||||
payload = strict_json(outcome.stdout)
|
||||
except ValueError as exc:
|
||||
return (0, f"pull-request creation returned malformed JSON: {exc}")
|
||||
problems = _pr_problems(payload, request)
|
||||
if problems:
|
||||
return (0, f"created pull request is invalid: {', '.join(problems)}")
|
||||
return (payload["number"], "")
|
||||
|
||||
|
||||
def _index_pages(
|
||||
runner: Runner, vantage: Vantage, api: str, reserve: float
|
||||
) -> tuple[list[Outcome], list[Any], str]:
|
||||
"""Read every page of the pull index; a truncated page can hide a match."""
|
||||
outcomes: list[Outcome] = []
|
||||
rows: list[Any] = []
|
||||
for page in range(1, DISCOVERY_PAGE_BUDGET + 1):
|
||||
outcome = runner.run(
|
||||
(
|
||||
GITEA_CLIENT,
|
||||
"GET",
|
||||
f"{api}/pulls?state=all&limit={DISCOVERY_PAGE_SIZE}&page={page}",
|
||||
),
|
||||
vantage,
|
||||
reserve_seconds=reserve,
|
||||
)
|
||||
outcomes.append(outcome)
|
||||
if not outcome.ok:
|
||||
return (outcomes, [], f"discovery page {page} failed: {_detail(outcome)}")
|
||||
try:
|
||||
payload = strict_json(outcome.stdout)
|
||||
except ValueError as exc:
|
||||
return (outcomes, [], f"discovery page {page} is malformed JSON: {exc}")
|
||||
if not isinstance(payload, list):
|
||||
return (outcomes, [], f"discovery page {page} did not return an array")
|
||||
rows.extend(payload)
|
||||
if len(payload) < DISCOVERY_PAGE_SIZE:
|
||||
return (outcomes, rows, "")
|
||||
return (outcomes, [], "pull-request discovery exceeded its page budget")
|
||||
|
||||
|
||||
def _residue(request: ArmRequest, numbers: Sequence[int], api: str) -> dict[str, Any]:
|
||||
"""Name the exact residue and the exact commands that remove it by hand."""
|
||||
manual = [
|
||||
f"{GITEA_CLIENT} PATCH {api}/pulls/{number} --field state=closed"
|
||||
for number in numbers
|
||||
]
|
||||
if not numbers:
|
||||
manual.append(
|
||||
f"{GITEA_CLIENT} GET {api}/pulls?state=all&limit={DISCOVERY_PAGE_SIZE}"
|
||||
f" then close by hand every pull whose head ref is {request.ref}"
|
||||
)
|
||||
manual.append(f"{GITEA_CLIENT} DELETE {api}/branches/{request.ref}")
|
||||
return {"residue_ref": request.ref, "manual_cleanup": manual}
|
||||
|
||||
|
||||
def _match_pulls(rows: Sequence[Any], request: ArmRequest) -> tuple[list[int], str]:
|
||||
matches: list[int] = []
|
||||
for index, item in enumerate(rows):
|
||||
if (
|
||||
not isinstance(item, dict)
|
||||
or not isinstance(item.get("head"), dict)
|
||||
or not isinstance(item.get("base"), dict)
|
||||
):
|
||||
return ([], f"pull-request discovery row {index + 1} is malformed")
|
||||
if item["head"].get("ref") != request.ref:
|
||||
continue
|
||||
problems = _pr_problems(item, request)
|
||||
if problems:
|
||||
return ([], f"discovered pull request is invalid: {', '.join(problems)}")
|
||||
matches.append(item["number"])
|
||||
if len(matches) != 1:
|
||||
return (
|
||||
matches,
|
||||
f"expected one exact draft pull request, discovered {len(matches)}",
|
||||
)
|
||||
return (matches, "")
|
||||
|
||||
|
||||
def _closed_pr_problems(payload: Any, request: ArmRequest) -> list[str]:
|
||||
"""Validate the same exact PR identity with state changed to closed."""
|
||||
problems = _pr_problems(payload, request)
|
||||
if isinstance(payload, dict) and payload.get("state") == "closed":
|
||||
problems = [
|
||||
problem for problem in problems if problem != "state does not match"
|
||||
]
|
||||
elif "state does not match" not in problems:
|
||||
problems.append("state is not closed")
|
||||
return problems
|
||||
|
||||
|
||||
def _cleanup(
|
||||
runner: Runner,
|
||||
vantage: Vantage,
|
||||
request: ArmRequest,
|
||||
api: str,
|
||||
numbers: Sequence[int],
|
||||
discovery_error: str = "",
|
||||
) -> CheckResult:
|
||||
outcomes: list[Outcome] = []
|
||||
problems = [discovery_error] if discovery_error else []
|
||||
for number in sorted(set(numbers)):
|
||||
closed = runner.run(
|
||||
(GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"),
|
||||
vantage,
|
||||
)
|
||||
outcomes.append(closed)
|
||||
if not closed.ok:
|
||||
problems.append(f"pull request #{number} close failed")
|
||||
continue
|
||||
try:
|
||||
payload = strict_json(closed.stdout)
|
||||
except ValueError:
|
||||
problems.append(f"pull request #{number} close response is malformed")
|
||||
continue
|
||||
expected = _closed_pr_problems(payload, request)
|
||||
if expected:
|
||||
problems.append(f"pull request #{number} close response does not match")
|
||||
deleted = runner.run(
|
||||
(GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage
|
||||
)
|
||||
outcomes.append(deleted)
|
||||
if not deleted.ok:
|
||||
problems.append("ephemeral branch deletion failed")
|
||||
remaining = runner.run(
|
||||
("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage
|
||||
)
|
||||
outcomes.append(remaining)
|
||||
if not remaining.ok or remaining.stdout.strip():
|
||||
problems.append("ephemeral branch absence could not be verified")
|
||||
for number in sorted(set(numbers)):
|
||||
observed = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage)
|
||||
outcomes.append(observed)
|
||||
if not observed.ok:
|
||||
problems.append(f"pull request #{number} closure could not be verified")
|
||||
continue
|
||||
try:
|
||||
payload = strict_json(observed.stdout)
|
||||
except ValueError:
|
||||
problems.append(f"pull request #{number} verification is malformed")
|
||||
continue
|
||||
expected = _closed_pr_problems(payload, request)
|
||||
if expected:
|
||||
problems.append(f"pull request #{number} is not exactly closed")
|
||||
return _result(
|
||||
"cleanup-verified",
|
||||
"Every ephemeral ref and pull request is removed and the removal is verified",
|
||||
FAIL if problems else PASS,
|
||||
"; ".join(problems)
|
||||
if problems
|
||||
else "the exact branch and draft pull request are gone",
|
||||
outcomes,
|
||||
{"ref": request.ref, "pull_requests": sorted(set(numbers))},
|
||||
)
|
||||
|
||||
|
||||
def _nothing_created(branch: CheckResult, draft_reason: str) -> list[CheckResult]:
|
||||
"""Report the two results that follow a run which created no artifact."""
|
||||
return [
|
||||
branch,
|
||||
_result(
|
||||
"draft-pull-request",
|
||||
"A draft pull request can be opened and closed",
|
||||
NOT_RUN,
|
||||
draft_reason,
|
||||
),
|
||||
_result(
|
||||
"cleanup-verified",
|
||||
"Every ephemeral artifact is removed",
|
||||
NOT_RUN,
|
||||
"nothing was created",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def run_armed(
|
||||
runner: Runner, vantage: Vantage, request: ArmRequest
|
||||
) -> list[CheckResult]:
|
||||
"""Create, discover, validate, and clean one exact ephemeral branch/PR."""
|
||||
guard = _protected_refusal_result()
|
||||
branch_title = "A unique ephemeral feature branch can be pushed and removed"
|
||||
if guard.status != PASS:
|
||||
unrun = _result("feature-branch-push", branch_title, NOT_RUN, "guard failed")
|
||||
return [guard, *_nothing_created(unrun, "protected-ref guard failed")]
|
||||
api = f"/api/v1/repos/{request.repo}"
|
||||
reserve = min(6000.0, max(300.0, runner.command_timeout * 10))
|
||||
existing = runner.run(
|
||||
("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"),
|
||||
vantage,
|
||||
reserve_seconds=reserve,
|
||||
)
|
||||
if not existing.ok or existing.stdout.strip():
|
||||
reason = (
|
||||
_detail(existing) if not existing.ok else "the ephemeral ref already exists"
|
||||
)
|
||||
refused = _result("feature-branch-push", branch_title, FAIL, reason, [existing])
|
||||
return [guard, *_nothing_created(refused, "branch preflight failed")]
|
||||
local_head = runner.run(
|
||||
("git", "rev-parse", "HEAD"), vantage, reserve_seconds=reserve
|
||||
)
|
||||
pushed = (
|
||||
runner.run(
|
||||
("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"),
|
||||
vantage,
|
||||
reserve_seconds=reserve,
|
||||
)
|
||||
if local_head.ok and local_head.stdout.strip() == request.expected_head
|
||||
else Outcome(
|
||||
argv=("git", "push"),
|
||||
vantage=vantage.name,
|
||||
error="local HEAD changed after static preflight",
|
||||
)
|
||||
)
|
||||
verified = runner.run(
|
||||
("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"),
|
||||
vantage,
|
||||
reserve_seconds=reserve,
|
||||
)
|
||||
branch_ok = verified.ok and verified.stdout.split() == [
|
||||
request.expected_head,
|
||||
f"refs/heads/{request.ref}",
|
||||
]
|
||||
branch_result = _result(
|
||||
"feature-branch-push",
|
||||
branch_title,
|
||||
PASS if branch_ok else FAIL,
|
||||
"the exact reviewed HEAD was pushed"
|
||||
if branch_ok
|
||||
else "push could not be verified exactly",
|
||||
[local_head, pushed, verified],
|
||||
{"ref": request.ref, "head": request.expected_head},
|
||||
)
|
||||
numbers: list[int] = []
|
||||
problems: list[str] = ["branch creation was not verified"]
|
||||
create_outcomes: list[Outcome] = []
|
||||
if branch_ok:
|
||||
created = runner.run(
|
||||
(
|
||||
GITEA_CLIENT,
|
||||
"POST",
|
||||
f"{api}/pulls",
|
||||
"--field",
|
||||
f"title={DRAFT_TITLE_PREFIX}Hermes handoff acceptance ephemeral probe",
|
||||
"--field",
|
||||
f"head={request.ref}",
|
||||
"--field",
|
||||
f"base={EXPECTED_BASE}",
|
||||
"--field",
|
||||
"body=Ephemeral acceptance probe. Closed and deleted by the harness.",
|
||||
),
|
||||
vantage,
|
||||
reserve_seconds=reserve,
|
||||
)
|
||||
number, create_error = _created_number(created, request)
|
||||
pages, rows, index_error = _index_pages(runner, vantage, api, reserve)
|
||||
found, match_error = (
|
||||
_match_pulls(rows, request) if not index_error else ([], "")
|
||||
)
|
||||
create_outcomes = [created, *pages]
|
||||
# Either source alone names a pull request this run definitely made: the
|
||||
# ephemeral ref was proven absent in preflight, so any pull on it is ours.
|
||||
numbers = sorted({item for item in (number, *found) if item})
|
||||
problems = [text for text in (index_error, match_error) if text]
|
||||
if number and found and found != [number]:
|
||||
problems.append(f"created #{number} is not the discovered {found}")
|
||||
if not numbers:
|
||||
problems.append(
|
||||
create_error or "no exact draft pull request was identified"
|
||||
)
|
||||
for item in numbers:
|
||||
register_ephemeral_pull(item)
|
||||
exact = len(numbers) == 1 and not problems
|
||||
draft_result = _result(
|
||||
"draft-pull-request",
|
||||
"A draft pull request can be opened and closed against the ephemeral branch",
|
||||
PASS if exact else FAIL,
|
||||
f"discovered exact draft pull request #{numbers[0]}"
|
||||
if exact
|
||||
else "; ".join(problems),
|
||||
create_outcomes,
|
||||
{"pull_requests": numbers}
|
||||
if exact
|
||||
else {"pull_requests": numbers, **_residue(request, numbers, api)},
|
||||
)
|
||||
cleanup = _cleanup(runner, vantage, request, api, numbers, "; ".join(problems))
|
||||
return [guard, branch_result, draft_result, cleanup]
|
||||
497
scripts/ops/hermes_handoff_evaluators.py
Normal file
497
scripts/ops/hermes_handoff_evaluators.py
Normal file
@ -0,0 +1,497 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Text-shaped evaluators for the handoff acceptance harness.
|
||||
|
||||
These rules cover evidence that arrives as a refusal, an exit status, or a list
|
||||
of names. The ``denied`` rule is the load-bearing one: it refuses to classify on
|
||||
an authorization review alone — ``kubectl auth can-i`` reports what RBAC says,
|
||||
not what the API server does — so a real request must have been made and
|
||||
refused before it will pass.
|
||||
|
||||
An absence is only evidence when the probe observed something, so a name step
|
||||
that exits 0 with no lines is ``NOT_RUN``: drift produces silence, not a pass.
|
||||
|
||||
Importing this module also registers the JSON-shaped rules, so a caller only has
|
||||
to import one place to have the full catalog vocabulary available.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from hermes_handoff_exec import Outcome
|
||||
from hermes_handoff_json_rules import evaluate_flux_health # noqa: F401 (registers rules)
|
||||
from hermes_handoff_model import ATTEMPT, FAIL, NOT_RUN, PASS, REVIEW, CheckSpec
|
||||
from hermes_handoff_rules import (
|
||||
EVALUATORS,
|
||||
Evaluation,
|
||||
dotted,
|
||||
evaluate,
|
||||
evaluator,
|
||||
exact_equal,
|
||||
lines,
|
||||
looks_denied,
|
||||
missing_steps,
|
||||
step_keys,
|
||||
strict_json,
|
||||
)
|
||||
|
||||
__all__ = ["EVALUATORS", "Evaluation", "evaluate", "evaluator"]
|
||||
|
||||
|
||||
@evaluator("denied")
|
||||
def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass only when the review says no and a real request was refused."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
attempts = step_keys(spec, ATTEMPT)
|
||||
if not attempts:
|
||||
return Evaluation(
|
||||
NOT_RUN, "deny checks require a real attempt, not only a review"
|
||||
)
|
||||
evidence: dict[str, Any] = {}
|
||||
for key in step_keys(spec, REVIEW):
|
||||
verdict = outcomes[key].stdout.strip().lower()
|
||||
evidence[key] = verdict
|
||||
if verdict != "no":
|
||||
return Evaluation(
|
||||
FAIL,
|
||||
f"authorization review {key} reported {verdict or 'nothing'}",
|
||||
evidence,
|
||||
)
|
||||
for key in attempts:
|
||||
outcome = outcomes[key]
|
||||
if outcome.ok:
|
||||
return Evaluation(
|
||||
FAIL, f"attempt {key} succeeded but must be refused", evidence
|
||||
)
|
||||
if not looks_denied(outcome):
|
||||
return Evaluation(
|
||||
NOT_RUN,
|
||||
f"attempt {key} failed without a recognisable refusal (rc={outcome.returncode})",
|
||||
evidence,
|
||||
)
|
||||
evidence[key] = "refused"
|
||||
return Evaluation(PASS, "review and live attempt were both refused", evidence)
|
||||
|
||||
|
||||
@evaluator("review_denied")
|
||||
def evaluate_review_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass only when every side-effect-free authorization review says no."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
reviews = step_keys(spec, REVIEW)
|
||||
if not reviews:
|
||||
return Evaluation(NOT_RUN, "a denial check requires an authorization review")
|
||||
evidence: dict[str, str] = {}
|
||||
for key in reviews:
|
||||
verdict = outcomes[key].stdout.strip().lower()
|
||||
evidence[key] = verdict
|
||||
if verdict != "no":
|
||||
return Evaluation(
|
||||
FAIL,
|
||||
f"authorization review {key} reported {verdict or 'nothing'}",
|
||||
evidence,
|
||||
)
|
||||
return Evaluation(PASS, "every authorization review was denied", evidence)
|
||||
|
||||
|
||||
@evaluator("allowed")
|
||||
def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass when every required step succeeded and every review said yes."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
evidence: dict[str, Any] = {}
|
||||
for step in spec.steps:
|
||||
outcome = outcomes[step.key]
|
||||
if step.kind == REVIEW:
|
||||
verdict = outcome.stdout.strip().lower()
|
||||
evidence[step.key] = verdict
|
||||
if verdict != "yes":
|
||||
return Evaluation(
|
||||
FAIL,
|
||||
f"authorization review {step.key} reported {verdict or 'nothing'}",
|
||||
evidence,
|
||||
)
|
||||
continue
|
||||
if not outcome.ok:
|
||||
return Evaluation(
|
||||
FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence
|
||||
)
|
||||
evidence[step.key] = "succeeded"
|
||||
return Evaluation(PASS, "required access is present", evidence)
|
||||
|
||||
|
||||
def _matched(observed: set[str], patterns: tuple[str, ...]) -> set[str]:
|
||||
return {name for pattern in patterns for name in observed if pattern in name}
|
||||
|
||||
|
||||
def _line_step(
|
||||
spec: CheckSpec, outcomes: dict[str, Outcome]
|
||||
) -> tuple[set[str], str] | Evaluation:
|
||||
"""Return a step's distinct lines, or the NOT_RUN that silence must produce."""
|
||||
key = spec.expect["step"]
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
observed = set(lines(outcome))
|
||||
if not observed:
|
||||
return Evaluation(NOT_RUN, f"step {key} exited 0 with no lines to examine")
|
||||
return (observed, key)
|
||||
|
||||
|
||||
@evaluator("names_absent")
|
||||
def evaluate_names_absent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass when a step observed names and none of them is forbidden."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
resolved = _line_step(spec, outcomes)
|
||||
if isinstance(resolved, Evaluation):
|
||||
return resolved
|
||||
observed, _ = resolved
|
||||
offenders = sorted(
|
||||
(observed & set(spec.expect.get("names", ())))
|
||||
| _matched(observed, tuple(spec.expect.get("contains", ())))
|
||||
)
|
||||
evidence = {"observed_count": len(observed), "offenders": offenders}
|
||||
if offenders:
|
||||
return Evaluation(
|
||||
FAIL, f"forbidden names present: {', '.join(offenders)}", evidence
|
||||
)
|
||||
return Evaluation(PASS, "no forbidden names are present", evidence)
|
||||
|
||||
|
||||
@evaluator("names_present")
|
||||
def evaluate_names_present(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass when every required name, and every required substring, appears."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
resolved = _line_step(spec, outcomes)
|
||||
if isinstance(resolved, Evaluation):
|
||||
return resolved
|
||||
observed, _ = resolved
|
||||
missing = sorted(set(spec.expect.get("names", ())) - observed)
|
||||
missing += sorted(
|
||||
pattern
|
||||
for pattern in spec.expect.get("contains", ())
|
||||
if not any(pattern in name for name in observed)
|
||||
)
|
||||
evidence = {"observed_count": len(observed), "missing": missing}
|
||||
if missing:
|
||||
return Evaluation(
|
||||
FAIL, f"required names absent: {', '.join(missing)}", evidence
|
||||
)
|
||||
return Evaluation(PASS, "every required name is present", evidence)
|
||||
|
||||
|
||||
@evaluator("stdout_matches")
|
||||
def evaluate_stdout_matches(
|
||||
spec: CheckSpec, outcomes: dict[str, Outcome]
|
||||
) -> Evaluation:
|
||||
"""Compare a step's trimmed stdout against an expected value or set."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
key = spec.expect["step"]
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
actual = outcome.stdout.strip()
|
||||
expected = spec.expect["equals"]
|
||||
allowed = expected if isinstance(expected, list) else [expected]
|
||||
evidence = {"observed": actual}
|
||||
if actual not in allowed:
|
||||
return Evaluation(
|
||||
FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence
|
||||
)
|
||||
return Evaluation(PASS, "output matches the expected value", evidence)
|
||||
|
||||
|
||||
@evaluator("distinct_count")
|
||||
def evaluate_distinct_count(
|
||||
spec: CheckSpec, outcomes: dict[str, Outcome]
|
||||
) -> Evaluation:
|
||||
"""Pass when a step yields enough distinct rows — and, optionally, exactly N.
|
||||
|
||||
Distinctness is the point for placement checks: three worker pods pinned to
|
||||
one node satisfy a replica count and defeat the isolation the replica count
|
||||
was standing in for.
|
||||
"""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
key = spec.expect["step"]
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
values = lines(outcome)
|
||||
distinct = sorted(set(values))
|
||||
minimum = spec.expect["minimum"]
|
||||
evidence = {
|
||||
"total": len(values),
|
||||
"distinct": len(distinct),
|
||||
"values": distinct[:16],
|
||||
}
|
||||
if not values:
|
||||
return Evaluation(NOT_RUN, f"step {key} produced no rows to count", evidence)
|
||||
if len(distinct) < minimum:
|
||||
return Evaluation(
|
||||
FAIL,
|
||||
f"{len(distinct)} distinct values, expected at least {minimum}",
|
||||
evidence,
|
||||
)
|
||||
total_equals = spec.expect.get("total_equals")
|
||||
if total_equals is not None and len(values) != total_equals:
|
||||
return Evaluation(
|
||||
FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence
|
||||
)
|
||||
return Evaluation(PASS, f"{len(distinct)} distinct values observed", evidence)
|
||||
|
||||
|
||||
@evaluator("lines_match")
|
||||
def evaluate_lines_match(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass when every data row of a tabular listing matches a shape.
|
||||
|
||||
Some surfaces only speak tables. A blank leading column is exactly what the
|
||||
session-render regression looked like, so asserting the shape of each row is
|
||||
a real check rather than a proxy for one.
|
||||
"""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
key = spec.expect["step"]
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
pattern = re.compile(spec.expect["pattern"])
|
||||
rows = outcome.stdout.splitlines()[spec.expect.get("skip", 0) :]
|
||||
rows = [row for row in rows if row.strip()]
|
||||
offenders = [row[:60] for row in rows if not pattern.search(row)]
|
||||
evidence = {"rows": len(rows), "offenders": offenders[:8]}
|
||||
if len(rows) < spec.expect.get("minimum", 1):
|
||||
return Evaluation(
|
||||
NOT_RUN,
|
||||
f"step {key} produced {len(rows)} data rows, expected at least "
|
||||
f"{spec.expect.get('minimum', 1)}",
|
||||
evidence,
|
||||
)
|
||||
if offenders:
|
||||
return Evaluation(
|
||||
FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence
|
||||
)
|
||||
return Evaluation(PASS, f"all {len(rows)} rows match the expected shape", evidence)
|
||||
|
||||
|
||||
@evaluator("vantages_agree")
|
||||
def evaluate_vantages_agree(
|
||||
spec: CheckSpec, outcomes: dict[str, Outcome]
|
||||
) -> Evaluation:
|
||||
"""Pass only when two independent observations report the same fact.
|
||||
|
||||
A disagreement is a finding in its own right: it usually means the manifest
|
||||
on record and the workload actually running have drifted apart, and either
|
||||
one alone would have looked clean.
|
||||
"""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
left_key, right_key = spec.expect["steps"]
|
||||
observations: dict[str, list[str]] = {}
|
||||
for key in (left_key, right_key):
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
observations[key] = sorted(lines(outcome))
|
||||
evidence = {key: value[:16] for key, value in observations.items()}
|
||||
if observations[left_key] != observations[right_key]:
|
||||
return Evaluation(
|
||||
FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence
|
||||
)
|
||||
if not observations[left_key]:
|
||||
return Evaluation(
|
||||
NOT_RUN, "both vantages returned nothing to compare", evidence
|
||||
)
|
||||
return Evaluation(PASS, "both vantages report the same state", evidence)
|
||||
|
||||
|
||||
@evaluator("not_armed")
|
||||
def evaluate_not_armed(spec: CheckSpec, _outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Report the fixed NOT_RUN a default, read-only run must produce."""
|
||||
reason = spec.expect.get("reason", "ephemeral mutation mode is not armed")
|
||||
return Evaluation(NOT_RUN, reason, {"armed": False})
|
||||
|
||||
|
||||
# This validator is one compact, linear state machine so every exit observes
|
||||
# only fully validated state.
|
||||
# fmt: off
|
||||
@evaluator("pool_topology")
|
||||
def evaluate_pool_topology(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Require exactly three Ready ordinal workers on distinct nodes and claims."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
state = outcomes[spec.expect["state_step"]].stdout.strip().split("\t")
|
||||
if len(state) != 4:
|
||||
return Evaluation(NOT_RUN, "StatefulSet projection is malformed")
|
||||
expected = spec.expect.get("replicas", 3)
|
||||
try:
|
||||
replicas, ready, current = (int(value) for value in state[:3])
|
||||
except ValueError:
|
||||
return Evaluation(NOT_RUN, "StatefulSet replica fields are malformed")
|
||||
evidence: dict[str, Any] = {"replicas": replicas, "ready": ready, "current": current,
|
||||
"automountServiceAccountToken": state[3]}
|
||||
if (replicas, ready, current, state[3].lower()) != (expected, expected, expected, "false"):
|
||||
return Evaluation(FAIL, "worker replica or tokenless state does not match the contract", evidence)
|
||||
workers: dict[int, tuple[str, str]] = {}
|
||||
names: list[str] = []
|
||||
for number, row in enumerate(lines(outcomes[spec.expect["worker_step"]]), 1):
|
||||
columns = row.split("\t")
|
||||
if len(columns) != 5:
|
||||
return Evaluation(NOT_RUN, f"worker row {number} is malformed", evidence)
|
||||
name, node, phase, ready_state, claims = columns
|
||||
match = re.fullmatch(rf"{re.escape(spec.expect['name'])}-(\d+)", name)
|
||||
claim_list = [claim for claim in claims.split(",") if claim]
|
||||
if (not match or not node or phase != "Running" or ready_state != "True"
|
||||
or len(claim_list) != 1):
|
||||
return Evaluation(FAIL, f"worker row {number} is not a Ready isolated worker", evidence)
|
||||
ordinal = int(match.group(1))
|
||||
if ordinal in workers:
|
||||
return Evaluation(FAIL, f"worker ordinal {ordinal} is duplicated", evidence)
|
||||
workers[ordinal] = (node, claim_list[0])
|
||||
names.append(name)
|
||||
evidence.update({"workers": sorted(names), "nodes": sorted({v[0] for v in workers.values()}),
|
||||
"claims": sorted({v[1] for v in workers.values()})})
|
||||
if set(workers) != set(range(expected)):
|
||||
return Evaluation(FAIL, f"worker ordinals are not exactly 0..{expected - 1}", evidence)
|
||||
if (len({v[0] for v in workers.values()}) != expected
|
||||
or len({v[1] for v in workers.values()}) != expected):
|
||||
return Evaluation(FAIL, "workers do not occupy distinct nodes and claims", evidence)
|
||||
return Evaluation(PASS, "three Ready workers have distinct nodes, ordinals, and claims", evidence)
|
||||
# fmt: on
|
||||
|
||||
|
||||
_SHA_RE = re.compile(r"[0-9a-f]{40}\Z")
|
||||
_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}\Z")
|
||||
_IMAGE_RE = re.compile(
|
||||
r"[^@\s]+:git-([0-9a-f]{40})-build-([1-9][0-9]*)@(sha256:[0-9a-f]{64})\Z"
|
||||
)
|
||||
|
||||
|
||||
def _strict_object(outcome: Outcome, label: str) -> dict[str, Any] | Evaluation:
|
||||
try:
|
||||
payload = strict_json(outcome.stdout)
|
||||
except ValueError as exc:
|
||||
return Evaluation(NOT_RUN, f"{label} JSON is malformed: {exc}")
|
||||
if not isinstance(payload, dict):
|
||||
return Evaluation(NOT_RUN, f"{label} JSON is not an object")
|
||||
return payload
|
||||
|
||||
|
||||
# fmt: off
|
||||
@evaluator("release_lineage")
|
||||
def evaluate_release_lineage(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Atomically bind Git, PR, image, build, Flux, and running revision evidence."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
expected = spec.expect
|
||||
main_sha = expected["main_sha"]
|
||||
head_sha = expected["head_sha"]
|
||||
image = expected["image"]
|
||||
build_sha = expected["build_sha"]
|
||||
revision = str(expected["deployment_revision"])
|
||||
digest = image.rsplit("@", 1)[-1]
|
||||
image_match = _IMAGE_RE.fullmatch(image)
|
||||
if not (_SHA_RE.fullmatch(main_sha) and _SHA_RE.fullmatch(head_sha)
|
||||
and _SHA_RE.fullmatch(build_sha) and _DIGEST_RE.fullmatch(digest)
|
||||
and image_match and image_match.group(1) == build_sha and build_sha == main_sha):
|
||||
return Evaluation(NOT_RUN, "release lineage expectations are malformed")
|
||||
remote = outcomes["remote-main"].stdout.strip().split()
|
||||
if remote != [main_sha, "refs/heads/main"]:
|
||||
return Evaluation(FAIL, "remote main does not match the fixed expected SHA",
|
||||
{"remote_main": remote[:2]})
|
||||
pr = _strict_object(outcomes["reviewed-pr"], "reviewed PR")
|
||||
if isinstance(pr, Evaluation):
|
||||
return pr
|
||||
fields = {
|
||||
"number": expected["pr_number"],
|
||||
"state": "open",
|
||||
"draft": True,
|
||||
"merged": False,
|
||||
"base.ref": "main",
|
||||
"base.sha": main_sha,
|
||||
"head.ref": expected["head_ref"],
|
||||
"head.sha": head_sha,
|
||||
}
|
||||
for path, wanted in fields.items():
|
||||
try:
|
||||
actual = dotted(pr, path)
|
||||
except (KeyError, ValueError, IndexError):
|
||||
return Evaluation(NOT_RUN, f"reviewed PR is missing {path}")
|
||||
if not exact_equal(actual, wanted):
|
||||
return Evaluation(FAIL, f"reviewed PR {path} does not match", {path: actual})
|
||||
if outcomes["build-source"].stdout.strip() != build_sha:
|
||||
return Evaluation(FAIL, "build source SHA does not match the fixed release build SHA")
|
||||
deployment = outcomes["deployment"].stdout.strip().split("\t")
|
||||
if len(deployment) != 6:
|
||||
return Evaluation(NOT_RUN, "Deployment projection is malformed")
|
||||
generation, observed, replicas, ready, deployed_revision, deployed_image = deployment
|
||||
try:
|
||||
current = (int(generation) > 0 and generation == observed
|
||||
and int(replicas) > 0 and replicas == ready)
|
||||
except ValueError:
|
||||
return Evaluation(NOT_RUN, "Deployment numeric fields are malformed")
|
||||
if not current or (deployed_revision, deployed_image) != (revision, image):
|
||||
return Evaluation(FAIL, "Deployment lineage does not match the release contract")
|
||||
replica_hash = ""
|
||||
for number, row in enumerate(lines(outcomes["replicasets"]), 1):
|
||||
columns = row.split("\t")
|
||||
if len(columns) != 6:
|
||||
return Evaluation(NOT_RUN, f"ReplicaSet row {number} is malformed")
|
||||
_, rs_revision, rs_ready, rs_available, rs_image, rs_hash = columns
|
||||
if rs_revision != revision:
|
||||
continue
|
||||
if (rs_ready != replicas or rs_available != replicas or rs_image != image or not rs_hash):
|
||||
return Evaluation(FAIL, "active ReplicaSet does not match the Deployment lineage")
|
||||
if replica_hash:
|
||||
return Evaluation(FAIL, "more than one ReplicaSet claims the running revision")
|
||||
replica_hash = rs_hash
|
||||
if not replica_hash:
|
||||
return Evaluation(FAIL, "running Deployment revision has no matching ReplicaSet")
|
||||
pod_rows = lines(outcomes["pods"])
|
||||
if len(pod_rows) != int(replicas):
|
||||
return Evaluation(FAIL, "running pod count does not match the Deployment")
|
||||
pods: list[str] = []
|
||||
for number, row in enumerate(pod_rows, 1):
|
||||
columns = row.split("\t")
|
||||
if len(columns) != 6:
|
||||
return Evaluation(NOT_RUN, f"pod row {number} is malformed")
|
||||
name, phase, ready_state, pod_image, image_id, pod_hash = columns
|
||||
if phase != "Running" or ready_state != "True":
|
||||
return Evaluation(FAIL, f"pod {name or number} is not Ready")
|
||||
if (pod_image, pod_hash) != (image, replica_hash) or not image_id.endswith(f"@{digest}"):
|
||||
return Evaluation(FAIL, f"pod {name or number} lineage does not match")
|
||||
pods.append(name)
|
||||
flux = outcomes["flux"].stdout.strip().split("\t")
|
||||
if len(flux) != 6:
|
||||
return Evaluation(NOT_RUN, "Flux lineage projection is malformed")
|
||||
name, generation, observed, suspended, ready_state, applied = flux
|
||||
if (not name or generation != observed or suspended.lower() == "true" or ready_state != "True"):
|
||||
return Evaluation(FAIL, "Flux source is not current and Ready")
|
||||
match = re.search(r"sha1:([0-9a-f]{40})\Z", applied)
|
||||
if not match or match.group(1) != main_sha:
|
||||
return Evaluation(FAIL, "Flux applied revision is not the expected remote main")
|
||||
return Evaluation(PASS, "Git, PR, image, build, Flux, and running revision are bound",
|
||||
{"main_sha": main_sha, "head_sha": head_sha, "image_digest": digest,
|
||||
"build_sha": build_sha, "deployment_revision": revision, "pods": pods})
|
||||
# fmt: on
|
||||
|
||||
|
||||
# Two observations of the same fact agree whether or not they came from
|
||||
# different vantages; the dual-vantage case is just the important one.
|
||||
EVALUATORS["steps_agree"] = evaluate_vantages_agree
|
||||
482
scripts/ops/hermes_handoff_exec.py
Normal file
482
scripts/ops/hermes_handoff_exec.py
Normal file
@ -0,0 +1,482 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded, screened, provenance-attested command execution.
|
||||
|
||||
It bounds pipes, times out and reaps the process group, and discards truncated output.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
import os
|
||||
import selectors
|
||||
import shlex
|
||||
import shutil
|
||||
import signal
|
||||
import stat
|
||||
import subprocess
|
||||
import time
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from hermes_handoff_policy import GITEA_CLIENT, READ_ONLY, PolicyError, check_argv
|
||||
from hermes_handoff_redaction import (
|
||||
DEFAULT_MAX_BYTES,
|
||||
TRUNCATION_NOTE,
|
||||
redact,
|
||||
safe_text,
|
||||
scrub,
|
||||
)
|
||||
|
||||
DEFAULT_COMMAND_TIMEOUT = 25.0
|
||||
DEFAULT_RUN_DEADLINE = 900.0
|
||||
DEFAULT_CLEANUP_RESERVE = 90.0
|
||||
MAX_COMMAND_TIMEOUT = 600.0
|
||||
MAX_RUN_DEADLINE = 7200.0
|
||||
MAX_OUTPUT_BYTES = 8 * 1024 * 1024
|
||||
READ_CHUNK = 64 * 1024
|
||||
PIPE_DRAIN_SECONDS = 0.25
|
||||
|
||||
SAFE_PATH = "/opt/data/tools/bin:/opt/hermes/.venv/bin:/opt/coordinator:/usr/local/bin:/usr/bin:/bin"
|
||||
# These exact allowlists are intentionally dense: reviewers compare them as a
|
||||
# single provenance boundary rather than as extensible application config.
|
||||
# fmt: off
|
||||
ENV_PASSTHROUGH = ("KUBECONFIG", "LANG", "LC_ALL", "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE")
|
||||
FIXED_ENVIRONMENT = {"GIT_ASKPASS": "/opt/coordinator/gitea_askpass.sh",
|
||||
"GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_NOSYSTEM": "1",
|
||||
"GIT_TERMINAL_PROMPT": "0", "GITEA_BASE_URL": "https://scm.bstein.dev",
|
||||
"LC_ALL": "C", "PATH": SAFE_PATH}
|
||||
EXPECTED_PATHS = {
|
||||
"kubectl": {"/opt/data/tools/bin/kubectl", "/usr/local/bin/kubectl", "/usr/bin/kubectl"},
|
||||
"git": {"/usr/bin/git", "/bin/git"},
|
||||
"hermes": {"/opt/hermes/.venv/bin/hermes"},
|
||||
"sh": {"/usr/bin/sh", "/bin/sh"},
|
||||
GITEA_CLIENT: {GITEA_CLIENT},
|
||||
}
|
||||
EXPECTED_SHA256 = {
|
||||
"kubectl": {"3d514dbae5dc8c09f773df0ef0f5d449dfad05b3aca5c96b13565f886df345fd"},
|
||||
"git": {"a0e562e4bd3c4c79379e91d8c07a10104b2cefe8fac966dc6bd4874a57a807f3"},
|
||||
"sh": {"367967c823a0c391e5049b15a67c6a0a629c88b9b6dcdca75ef13ac9d65334b1"},
|
||||
"hermes": {"c8419290ef7f1a95f59eadcea9f58f446b1e6535a7f4ffee9f1c0cb1172335e4"},
|
||||
GITEA_CLIENT: {"f0943db4e81ce96908e94b0f0e78aa4b5ef95d6c4f7803f9f74a43e761576d85"},
|
||||
}
|
||||
|
||||
DEADLINE_ERROR = "deadline-exceeded"
|
||||
TIMEOUT_ERROR = "timeout"
|
||||
OPERATOR = "operator"
|
||||
SELF = "self"
|
||||
CONTEXT_AWARE_BINARIES = ("kubectl",)
|
||||
POD_COMMAND_PATHS = {"kubectl": "/usr/local/bin/kubectl", "hermes": "/opt/hermes/.venv/bin/hermes",
|
||||
"git": "/usr/bin/git", "sh": "/bin/sh", GITEA_CLIENT: GITEA_CLIENT}
|
||||
# fmt: on
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Attestation:
|
||||
"""The exact outer executable used for a probe."""
|
||||
|
||||
path: str
|
||||
sha256: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Capture:
|
||||
"""Raw bounded process result returned by an executor implementation."""
|
||||
|
||||
returncode: int
|
||||
stdout: bytes = b""
|
||||
stderr: bytes = b""
|
||||
stdout_truncated: bool = False
|
||||
stderr_truncated: bool = False
|
||||
timed_out: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Outcome:
|
||||
"""The screened, bounded result of one probe command."""
|
||||
|
||||
argv: tuple[str, ...]
|
||||
vantage: str
|
||||
returncode: int | None = None
|
||||
stdout: str = ""
|
||||
stderr: str = ""
|
||||
truncated: bool = False
|
||||
duration_ms: int = 0
|
||||
error: str | None = None
|
||||
executable_path: str = ""
|
||||
executable_sha256: str = ""
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.error is None and self.returncode == 0 and not self.truncated
|
||||
|
||||
@property
|
||||
def ran(self) -> bool:
|
||||
return self.error is None
|
||||
|
||||
@property
|
||||
def combined(self) -> str:
|
||||
return "\n".join(part for part in (self.stdout, self.stderr) if part)
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return scrub(
|
||||
{
|
||||
"command": redact(shlex.join(self.argv)),
|
||||
"vantage": self.vantage,
|
||||
"returncode": self.returncode,
|
||||
"stdout": self.stdout,
|
||||
"stderr": self.stderr,
|
||||
"truncated": self.truncated,
|
||||
"duration_ms": self.duration_ms,
|
||||
"error": self.error,
|
||||
"executable_path": self.executable_path,
|
||||
"executable_sha256": self.executable_sha256,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Vantage:
|
||||
"""A named identity and transport used to observe a probe."""
|
||||
|
||||
name: str
|
||||
prefix: tuple[str, ...] = ()
|
||||
inserts: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
env: Mapping[str, str] = field(default_factory=dict)
|
||||
command_paths: Mapping[str, str] = field(default_factory=dict)
|
||||
description: str = ""
|
||||
|
||||
def wrap(self, argv: Sequence[str]) -> tuple[str, ...]:
|
||||
command = tuple(argv)
|
||||
if command:
|
||||
replacement = self.command_paths.get(command[0])
|
||||
if replacement:
|
||||
command = (replacement, *command[1:])
|
||||
extra = self.inserts.get(command[0].rsplit("/", 1)[-1])
|
||||
if extra:
|
||||
command = (command[0], *extra, *command[1:])
|
||||
return (*self.prefix, *command)
|
||||
|
||||
|
||||
def _finite_number(name: str, value: Any, minimum: float, maximum: float) -> float:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise ValueError(f"{name} must be numeric")
|
||||
converted = float(value)
|
||||
if not math.isfinite(converted) or not minimum <= converted <= maximum:
|
||||
raise ValueError(
|
||||
f"{name} must be finite and between {minimum:g} and {maximum:g}"
|
||||
)
|
||||
return converted
|
||||
|
||||
|
||||
def _byte_budget(value: Any) -> int:
|
||||
if (
|
||||
isinstance(value, bool)
|
||||
or not isinstance(value, int)
|
||||
or not 1 <= value <= MAX_OUTPUT_BYTES
|
||||
):
|
||||
raise ValueError(
|
||||
f"max_bytes must be an integer between 1 and {MAX_OUTPUT_BYTES}"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def build_environment(
|
||||
vantage: Vantage, source: Mapping[str, str] | None = None
|
||||
) -> dict[str, str]:
|
||||
"""Build a deterministic child environment without caller-controlled PATH."""
|
||||
origin = os.environ if source is None else source
|
||||
environment = {name: origin[name] for name in ENV_PASSTHROUGH if name in origin}
|
||||
environment.update(FIXED_ENVIRONMENT)
|
||||
environment.update(vantage.env)
|
||||
environment["PATH"] = SAFE_PATH
|
||||
environment["GITEA_BASE_URL"] = FIXED_ENVIRONMENT["GITEA_BASE_URL"]
|
||||
environment["GIT_ASKPASS"] = FIXED_ENVIRONMENT["GIT_ASKPASS"]
|
||||
return environment
|
||||
|
||||
|
||||
# fmt: off
|
||||
def operator_vantage(kubeconfig: str | None = None, context: str | None = None) -> Vantage:
|
||||
inserts = dict.fromkeys(CONTEXT_AWARE_BINARIES, ("--context", context)) if context else {}
|
||||
detail = " ".join(part for part in (f"context={context}" if context else "",
|
||||
"kubeconfig-pinned" if kubeconfig else "") if part)
|
||||
return Vantage(name=OPERATOR, inserts=inserts,
|
||||
env={"KUBECONFIG": kubeconfig} if kubeconfig else {},
|
||||
description=f"operator {detail}".strip())
|
||||
|
||||
|
||||
def pod_vantage(namespace: str, pod: str, container: str, operator: Vantage | None = None) -> Vantage:
|
||||
reach = operator.inserts.get("kubectl", ()) if operator else ()
|
||||
return Vantage(name=SELF, prefix=("kubectl", *reach, "exec", "--namespace", namespace,
|
||||
pod, "--container", container, "--"),
|
||||
env=dict(operator.env) if operator else {}, command_paths=POD_COMMAND_PATHS,
|
||||
description=f"in-pod {namespace}/{pod}[{container}]")
|
||||
# fmt: on
|
||||
|
||||
|
||||
def attest_executable(command: str, environment: Mapping[str, str]) -> Attestation:
|
||||
"""Resolve and hash an executable from a fixed, trusted path set."""
|
||||
name = command if command == GITEA_CLIENT else command.rsplit("/", 1)[-1]
|
||||
candidate = (
|
||||
command if "/" in command else shutil.which(command, path=environment["PATH"])
|
||||
)
|
||||
if not candidate:
|
||||
raise OSError(f"trusted executable is unavailable: {name}")
|
||||
resolved = str(Path(candidate).resolve(strict=True))
|
||||
allowed = {
|
||||
str(Path(item).resolve(strict=False))
|
||||
for item in EXPECTED_PATHS.get(name, set())
|
||||
}
|
||||
if resolved not in allowed:
|
||||
raise OSError(f"executable resolved outside its trusted paths: {name}")
|
||||
details = os.stat(resolved)
|
||||
if (
|
||||
not stat.S_ISREG(details.st_mode)
|
||||
or details.st_mode & 0o022
|
||||
or not details.st_mode & 0o111
|
||||
):
|
||||
raise OSError(f"trusted executable has unsafe ownership mode: {name}")
|
||||
digest = hashlib.sha256()
|
||||
with open(resolved, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
observed = digest.hexdigest()
|
||||
expected = EXPECTED_SHA256.get(name)
|
||||
if not expected or observed not in expected:
|
||||
raise OSError(f"executable digest is not release-attested: {name}")
|
||||
return Attestation(resolved, observed)
|
||||
|
||||
|
||||
def _append_bounded(buffer: bytearray, chunk: bytes, limit: int) -> bool:
|
||||
remaining = limit + 1 - len(buffer)
|
||||
if remaining > 0:
|
||||
buffer.extend(chunk[:remaining])
|
||||
return len(buffer) > limit or len(chunk) > remaining
|
||||
|
||||
|
||||
def _signal_group(process: subprocess.Popen[bytes], signum: int) -> None:
|
||||
with suppress(ProcessLookupError):
|
||||
os.killpg(process.pid, signum)
|
||||
|
||||
|
||||
def execute_bounded(
|
||||
argv: Sequence[str],
|
||||
environment: Mapping[str, str],
|
||||
timeout: float,
|
||||
max_bytes: int,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
popen: Callable[..., subprocess.Popen[bytes]] = subprocess.Popen,
|
||||
) -> Capture:
|
||||
"""Execute one process with bounded live pipe capture and group cleanup."""
|
||||
process = popen(
|
||||
list(argv),
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=dict(environment),
|
||||
text=False,
|
||||
close_fds=True,
|
||||
start_new_session=True,
|
||||
)
|
||||
streams = {"stdout": process.stdout, "stderr": process.stderr}
|
||||
buffers = {"stdout": bytearray(), "stderr": bytearray()}
|
||||
truncated = {"stdout": False, "stderr": False}
|
||||
selector = selectors.DefaultSelector()
|
||||
for name, stream in streams.items():
|
||||
if (
|
||||
stream is not None
|
||||
): # pragma: no branch - PIPE was requested for both streams
|
||||
os.set_blocking(stream.fileno(), False)
|
||||
selector.register(stream, selectors.EVENT_READ, name)
|
||||
expires = clock() + timeout
|
||||
timed_out = False
|
||||
try:
|
||||
while process.poll() is None:
|
||||
remaining = expires - clock()
|
||||
if remaining <= 0:
|
||||
timed_out = True
|
||||
break
|
||||
for key, _ in selector.select(min(remaining, 0.1)):
|
||||
try:
|
||||
chunk = os.read(key.fileobj.fileno(), READ_CHUNK)
|
||||
except BlockingIOError: # pragma: no cover - readiness race, retry only
|
||||
continue
|
||||
if not chunk:
|
||||
selector.unregister(key.fileobj)
|
||||
continue
|
||||
name = key.data
|
||||
truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes)
|
||||
_signal_group(process, signal.SIGKILL if timed_out else signal.SIGTERM)
|
||||
drain_until = clock() + PIPE_DRAIN_SECONDS
|
||||
while selector.get_map() and clock() < drain_until:
|
||||
for key, _ in selector.select(max(0.0, drain_until - clock())):
|
||||
try:
|
||||
chunk = os.read(key.fileobj.fileno(), READ_CHUNK)
|
||||
except BlockingIOError: # pragma: no cover - readiness race, retry only
|
||||
continue
|
||||
if not chunk:
|
||||
selector.unregister(key.fileobj)
|
||||
continue
|
||||
name = key.data
|
||||
truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes)
|
||||
_signal_group(process, signal.SIGKILL)
|
||||
returncode = process.wait(timeout=PIPE_DRAIN_SECONDS)
|
||||
finally:
|
||||
selector.close()
|
||||
for stream in streams.values():
|
||||
if (
|
||||
stream is not None
|
||||
): # pragma: no branch - PIPE was requested for both streams
|
||||
stream.close()
|
||||
if (
|
||||
process.poll() is None
|
||||
): # pragma: no branch - fail-safe for wait/selector exceptions
|
||||
_signal_group(process, signal.SIGKILL)
|
||||
process.wait()
|
||||
return Capture(
|
||||
returncode=returncode,
|
||||
stdout=bytes(buffers["stdout"][:max_bytes]),
|
||||
stderr=bytes(buffers["stderr"][:max_bytes]),
|
||||
stdout_truncated=truncated["stdout"],
|
||||
stderr_truncated=truncated["stderr"],
|
||||
timed_out=timed_out,
|
||||
)
|
||||
|
||||
|
||||
def _injected_executor(
|
||||
spawn: Callable[..., subprocess.CompletedProcess[Any]],
|
||||
) -> Callable[..., Capture]:
|
||||
"""Adapt deterministic test doubles without weakening the production path."""
|
||||
|
||||
def execute(
|
||||
argv: Sequence[str],
|
||||
environment: Mapping[str, str],
|
||||
timeout: float,
|
||||
max_bytes: int,
|
||||
) -> Capture:
|
||||
completed = spawn(
|
||||
list(argv),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
env=dict(environment),
|
||||
)
|
||||
stdout = str(completed.stdout or "").encode()
|
||||
stderr = str(completed.stderr or "").encode()
|
||||
return Capture(
|
||||
completed.returncode,
|
||||
stdout[:max_bytes],
|
||||
stderr[:max_bytes],
|
||||
len(stdout) > max_bytes,
|
||||
len(stderr) > max_bytes,
|
||||
)
|
||||
|
||||
return execute
|
||||
|
||||
|
||||
def _screen_capture(data: bytes, truncated: bool, budget: int) -> str:
|
||||
if truncated:
|
||||
return TRUNCATION_NOTE.encode()[:budget].decode()
|
||||
return safe_text(data.decode("utf-8", "replace"), budget)[0]
|
||||
|
||||
|
||||
class Runner:
|
||||
"""Run probes under policy, provenance, timeout, deadline, and byte budgets."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
mode: str = READ_ONLY,
|
||||
command_timeout: float = DEFAULT_COMMAND_TIMEOUT,
|
||||
deadline_seconds: float = DEFAULT_RUN_DEADLINE,
|
||||
max_bytes: int = DEFAULT_MAX_BYTES,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
executor: Callable[..., Capture] | None = None,
|
||||
spawn: Callable[..., subprocess.CompletedProcess[Any]] | None = None,
|
||||
environ: Mapping[str, str] | None = None,
|
||||
attestor: Callable[[str, Mapping[str, str]], Attestation] = attest_executable,
|
||||
) -> None:
|
||||
self.mode = mode
|
||||
self.command_timeout = _finite_number(
|
||||
"command_timeout", command_timeout, 0.05, MAX_COMMAND_TIMEOUT
|
||||
)
|
||||
deadline = _finite_number(
|
||||
"deadline_seconds", deadline_seconds, 0.05, MAX_RUN_DEADLINE
|
||||
)
|
||||
self.max_bytes = _byte_budget(max_bytes)
|
||||
self._clock = clock
|
||||
self._executor = executor or (
|
||||
_injected_executor(spawn) if spawn else execute_bounded
|
||||
)
|
||||
self._environ = environ
|
||||
self._attestor = attestor
|
||||
self._deadline_at = clock() + deadline
|
||||
self.commands_run = 0
|
||||
|
||||
@property
|
||||
def remaining_seconds(self) -> float:
|
||||
return self._deadline_at - self._clock()
|
||||
|
||||
def run(
|
||||
self,
|
||||
argv: Sequence[str],
|
||||
vantage: Vantage,
|
||||
max_bytes: int | None = None,
|
||||
reserve_seconds: float = 0.0,
|
||||
) -> Outcome:
|
||||
budget_bytes = self.max_bytes if max_bytes is None else _byte_budget(max_bytes)
|
||||
reserve = _finite_number(
|
||||
"reserve_seconds", reserve_seconds, 0.0, MAX_RUN_DEADLINE
|
||||
)
|
||||
addressed = vantage.wrap(argv)
|
||||
try:
|
||||
check_argv(addressed, self.mode)
|
||||
except PolicyError as exc:
|
||||
return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}")
|
||||
budget = min(self.command_timeout, self.remaining_seconds - reserve)
|
||||
if budget <= 0:
|
||||
return Outcome(argv=addressed, vantage=vantage.name, error=DEADLINE_ERROR)
|
||||
environment = build_environment(vantage, self._environ)
|
||||
try:
|
||||
attestation = self._attestor(addressed[0], environment)
|
||||
except (OSError, ValueError) as exc:
|
||||
return Outcome(
|
||||
argv=addressed, vantage=vantage.name, error=f"attestation: {exc}"
|
||||
)
|
||||
executed = (attestation.path, *addressed[1:])
|
||||
try:
|
||||
check_argv(executed, self.mode)
|
||||
except PolicyError as exc:
|
||||
return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}")
|
||||
started = self._clock()
|
||||
self.commands_run += 1
|
||||
try:
|
||||
capture = self._executor(executed, environment, budget, budget_bytes)
|
||||
except subprocess.TimeoutExpired:
|
||||
capture = Capture(-signal.SIGKILL, timed_out=True)
|
||||
except OSError as exc:
|
||||
return Outcome(
|
||||
argv=addressed,
|
||||
vantage=vantage.name,
|
||||
error=f"spawn: {exc.strerror or type(exc).__name__}",
|
||||
executable_path=attestation.path,
|
||||
executable_sha256=attestation.sha256,
|
||||
)
|
||||
was_truncated = capture.stdout_truncated or capture.stderr_truncated
|
||||
stdout = _screen_capture(capture.stdout, capture.stdout_truncated, budget_bytes)
|
||||
stderr = _screen_capture(capture.stderr, capture.stderr_truncated, budget_bytes)
|
||||
return Outcome(
|
||||
argv=addressed,
|
||||
vantage=vantage.name,
|
||||
returncode=capture.returncode,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
truncated=was_truncated,
|
||||
duration_ms=int((self._clock() - started) * 1000),
|
||||
error=TIMEOUT_ERROR if capture.timed_out else None,
|
||||
executable_path=attestation.path,
|
||||
executable_sha256=attestation.sha256,
|
||||
)
|
||||
474
scripts/ops/hermes_handoff_json_rules.py
Normal file
474
scripts/ops/hermes_handoff_json_rules.py
Normal file
@ -0,0 +1,474 @@
|
||||
#!/usr/bin/env python3
|
||||
"""JSON-shaped evaluators for the handoff acceptance harness.
|
||||
|
||||
These rules cover the evidence that arrives as structured output — Kubernetes
|
||||
objects, forge pull requests, provider health files, and the two append-only
|
||||
evidence logs the platform keeps. They share the fail-closed contract of every
|
||||
other rule: an absent path, an unparsable payload, or an empty result set is
|
||||
``NOT_RUN``, never a quiet pass.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import math
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from hermes_handoff_exec import Outcome
|
||||
from hermes_handoff_model import FAIL, NOT_RUN, PASS, CheckSpec
|
||||
from hermes_handoff_rules import (
|
||||
Evaluation,
|
||||
dotted,
|
||||
evaluator,
|
||||
exact_equal,
|
||||
is_list,
|
||||
missing_steps,
|
||||
parsed_step,
|
||||
strict_json,
|
||||
)
|
||||
|
||||
_ROUTE_PART_RE = re.compile(r"[a-z0-9][a-z0-9_.-]{0,63}\Z")
|
||||
_FLUX_NAME_RE = re.compile(
|
||||
r"[a-z0-9]([-a-z0-9.]*[a-z0-9])?/[a-z0-9]([-a-z0-9.]*[a-z0-9])?\Z"
|
||||
)
|
||||
|
||||
|
||||
def _timestamp(value: Any) -> dt.datetime:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise ValueError("timestamp is not a non-empty string")
|
||||
observed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
if observed.tzinfo is None:
|
||||
raise ValueError("timestamp has no timezone")
|
||||
return observed.astimezone(dt.timezone.utc)
|
||||
|
||||
|
||||
def _now(value: Any) -> dt.datetime:
|
||||
if not isinstance(value, dt.datetime) or value.tzinfo is None:
|
||||
raise ValueError("run clock is not timezone-aware")
|
||||
return value.astimezone(dt.timezone.utc)
|
||||
|
||||
|
||||
@evaluator("json_field")
|
||||
def evaluate_json_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Compare dotted JSON paths of one step's output against expected values."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
parsed = parsed_step(spec, outcomes)
|
||||
if isinstance(parsed, Evaluation):
|
||||
return parsed
|
||||
payload, key = parsed
|
||||
evidence: dict[str, Any] = {}
|
||||
mismatches: list[str] = []
|
||||
for path, expected in spec.expect["fields"].items():
|
||||
try:
|
||||
actual = dotted(payload, path)
|
||||
except (KeyError, ValueError, IndexError):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"path {path} is absent from step {key}", evidence
|
||||
)
|
||||
evidence[path] = actual
|
||||
allowed = expected if isinstance(expected, list) else [expected]
|
||||
if not any(exact_equal(actual, item) for item in allowed):
|
||||
mismatches.append(f"{path}={actual!r} expected one of {allowed!r}")
|
||||
if mismatches:
|
||||
return Evaluation(FAIL, "; ".join(mismatches), evidence)
|
||||
return Evaluation(PASS, "observed fields match the release contract", evidence)
|
||||
|
||||
|
||||
@evaluator("json_numeric")
|
||||
def evaluate_json_numeric(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Compare numeric JSON paths against inclusive minimum/maximum bounds."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
parsed = parsed_step(spec, outcomes)
|
||||
if isinstance(parsed, Evaluation):
|
||||
return parsed
|
||||
payload, key = parsed
|
||||
evidence: dict[str, Any] = {}
|
||||
problems: list[str] = []
|
||||
for path, bounds in spec.expect["fields"].items():
|
||||
try:
|
||||
actual = dotted(payload, path)
|
||||
except (KeyError, ValueError, IndexError):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"path {path} is absent from step {key}", evidence
|
||||
)
|
||||
if (
|
||||
not isinstance(actual, (int, float))
|
||||
or isinstance(actual, bool)
|
||||
or not math.isfinite(float(actual))
|
||||
):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"path {path} is not numeric in step {key}", evidence
|
||||
)
|
||||
evidence[path] = actual
|
||||
if "min" in bounds and actual < bounds["min"]:
|
||||
problems.append(f"{path}={actual} below minimum {bounds['min']}")
|
||||
if "max" in bounds and actual > bounds["max"]:
|
||||
problems.append(f"{path}={actual} above maximum {bounds['max']}")
|
||||
if problems:
|
||||
return Evaluation(FAIL, "; ".join(problems), evidence)
|
||||
return Evaluation(PASS, "numeric fields are within the expected bounds", evidence)
|
||||
|
||||
|
||||
@evaluator("json_recent")
|
||||
def evaluate_json_recent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass when a timestamp field is fresh enough to be evidence about now.
|
||||
|
||||
Stale provider or routing evidence is worse than none: it describes a state
|
||||
the release no longer has. A timestamp older than the allowed age is a FAIL,
|
||||
and one the harness cannot parse is NOT_RUN.
|
||||
"""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
parsed = parsed_step(spec, outcomes)
|
||||
if isinstance(parsed, Evaluation):
|
||||
return parsed
|
||||
payload, key = parsed
|
||||
try:
|
||||
now = _now(spec.expect["now"])
|
||||
except (TypeError, ValueError) as exc:
|
||||
return Evaluation(NOT_RUN, f"invalid run clock: {exc}")
|
||||
evidence: dict[str, Any] = {}
|
||||
stale: list[str] = []
|
||||
for path, max_age in spec.expect["fields"].items():
|
||||
try:
|
||||
raw = dotted(payload, path)
|
||||
except (KeyError, ValueError, IndexError):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"path {path} is absent from step {key}", evidence
|
||||
)
|
||||
try:
|
||||
observed = _timestamp(raw)
|
||||
except (TypeError, ValueError):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"path {path} is not a timezone-aware ISO timestamp", evidence
|
||||
)
|
||||
age = (now - observed).total_seconds()
|
||||
evidence[path] = {"age_seconds": int(age)}
|
||||
if age < 0:
|
||||
stale.append(f"{path} is {abs(int(age))}s in the future")
|
||||
elif age > max_age:
|
||||
stale.append(f"{path} is {int(age)}s old, limit {max_age}s")
|
||||
if stale:
|
||||
return Evaluation(FAIL, "; ".join(stale), evidence)
|
||||
return Evaluation(PASS, "observed timestamps are fresh", evidence)
|
||||
|
||||
|
||||
@evaluator("all_of_field")
|
||||
def evaluate_all_of_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Pass when every selected element of a JSON array matches expected fields."""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
parsed = parsed_step(spec, outcomes)
|
||||
if isinstance(parsed, Evaluation):
|
||||
return parsed
|
||||
payload, key = parsed
|
||||
try:
|
||||
elements = dotted(payload, spec.expect.get("array", ""))
|
||||
except (KeyError, ValueError, IndexError):
|
||||
return Evaluation(NOT_RUN, f"array path is absent from step {key}")
|
||||
if not is_list(elements):
|
||||
return Evaluation(NOT_RUN, f"step {key} did not return an array")
|
||||
key_field = spec.expect.get("key_field", "")
|
||||
wanted = spec.expect.get("keys")
|
||||
exempt = set(spec.expect.get("exempt_keys", ()))
|
||||
seen: list[Any] = []
|
||||
problems: list[str] = []
|
||||
evidence: dict[str, Any] = {}
|
||||
for element in elements:
|
||||
identity = dotted(element, key_field) if key_field else None
|
||||
if wanted is not None and identity not in wanted:
|
||||
continue
|
||||
if identity in exempt:
|
||||
continue
|
||||
seen.append(identity)
|
||||
for path in spec.expect.get("non_empty", ()):
|
||||
try:
|
||||
value = dotted(element, path)
|
||||
except (KeyError, ValueError, IndexError):
|
||||
value = None
|
||||
if value in (None, "", [], {}):
|
||||
problems.append(f"{identity}: {path} is empty")
|
||||
for path, expected in spec.expect.get("fields", {}).items():
|
||||
try:
|
||||
actual = dotted(element, path)
|
||||
except (KeyError, ValueError, IndexError):
|
||||
actual = None
|
||||
allowed = expected if isinstance(expected, list) else [expected]
|
||||
if not any(exact_equal(actual, item) for item in allowed):
|
||||
problems.append(
|
||||
f"{identity}: {path}={actual!r} expected one of {allowed!r}"
|
||||
)
|
||||
evidence["examined"] = seen[:32]
|
||||
if wanted is not None:
|
||||
absent = [item for item in wanted if item not in seen]
|
||||
if absent:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"expected keys absent from step {key}: {absent}", evidence
|
||||
)
|
||||
elif not seen:
|
||||
return Evaluation(NOT_RUN, f"step {key} returned nothing to examine", evidence)
|
||||
if problems:
|
||||
return Evaluation(FAIL, "; ".join(problems[:8]), evidence)
|
||||
return Evaluation(
|
||||
PASS, f"{len(seen)} element(s) match the release contract", evidence
|
||||
)
|
||||
|
||||
|
||||
@evaluator("json_record")
|
||||
def evaluate_json_record(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Capture named JSON paths as evidence; pass when all of them resolve.
|
||||
|
||||
Recording is a release requirement in its own right — the runbook has to
|
||||
name the exact heads it was verified against — so an unreachable source is a
|
||||
NOT_RUN rather than an empty section.
|
||||
"""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
parsed = parsed_step(spec, outcomes)
|
||||
if isinstance(parsed, Evaluation):
|
||||
return parsed
|
||||
payload, key = parsed
|
||||
evidence: dict[str, Any] = {}
|
||||
for label, path in spec.expect["record"].items():
|
||||
try:
|
||||
evidence[label] = dotted(payload, path)
|
||||
except (KeyError, ValueError, IndexError):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"path {path} is absent from step {key}", evidence
|
||||
)
|
||||
return Evaluation(PASS, "state recorded", evidence)
|
||||
|
||||
|
||||
@evaluator("routing_evidence")
|
||||
def evaluate_routing_evidence(
|
||||
spec: CheckSpec, outcomes: dict[str, Outcome]
|
||||
) -> Evaluation:
|
||||
"""Summarise a Switchyard routing log tail into provider/effort coverage.
|
||||
|
||||
Route identifiers are ``<lane>/<provider>/<family>/<effort>``, so provider,
|
||||
model family, effort tier, and lane all come from the same field. Fallback
|
||||
and failure evidence is a separate ``fallback_reason`` on the record that
|
||||
was retried. Every row must be complete strict JSON; a clipped or malformed
|
||||
line makes the evidence unusable rather than being silently discarded.
|
||||
"""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
key = spec.expect["step"]
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
records: list[dict[str, Any]] = []
|
||||
routes: list[list[str]] = []
|
||||
stamps_by_dimension: dict[tuple[str, str], dt.datetime] = {}
|
||||
now_value = spec.expect.get("now")
|
||||
try:
|
||||
now = _now(now_value) if now_value is not None else None
|
||||
except (TypeError, ValueError) as exc:
|
||||
return Evaluation(NOT_RUN, f"invalid routing run clock: {exc}")
|
||||
for row_number, line in enumerate(outcome.stdout.splitlines(), 1):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
record = strict_json(line)
|
||||
except ValueError as exc:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"routing row {row_number} is malformed JSON: {exc}"
|
||||
)
|
||||
if not isinstance(record, dict):
|
||||
return Evaluation(NOT_RUN, f"routing row {row_number} is not an object")
|
||||
try:
|
||||
observed = _timestamp(record.get("ts"))
|
||||
except (TypeError, ValueError) as exc:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"routing row {row_number} has invalid timestamp: {exc}"
|
||||
)
|
||||
if now is not None and observed > now:
|
||||
return Evaluation(FAIL, f"routing row {row_number} has a future timestamp")
|
||||
model = record.get("model")
|
||||
if not isinstance(model, str) or not model:
|
||||
return Evaluation(NOT_RUN, f"routing row {row_number} has no model")
|
||||
route = model.split("/")
|
||||
if len(route) == 4 and all(_ROUTE_PART_RE.fullmatch(part) for part in route):
|
||||
routes.append(route)
|
||||
for label, value in (
|
||||
("provider", route[1]),
|
||||
("effort", route[3]),
|
||||
("lane", route[0]),
|
||||
):
|
||||
key_dimension = (label, value)
|
||||
stamps_by_dimension[key_dimension] = max(
|
||||
observed, stamps_by_dimension.get(key_dimension, observed)
|
||||
)
|
||||
elif record.get("tier") != "classifier":
|
||||
return Evaluation(
|
||||
NOT_RUN, f"routing row {row_number} has malformed route {model!r}"
|
||||
)
|
||||
fallback = record.get("fallback_reason")
|
||||
if fallback is not None and (
|
||||
not isinstance(fallback, str) or not fallback.strip()
|
||||
):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"routing row {row_number} has malformed fallback_reason"
|
||||
)
|
||||
record["_observed"] = observed
|
||||
records.append(record)
|
||||
fallbacks = [record for record in records if record.get("fallback_reason")]
|
||||
stamps = sorted(record["_observed"] for record in records)
|
||||
evidence: dict[str, Any] = {
|
||||
"records": len(records),
|
||||
"routed": len(routes),
|
||||
"lanes": sorted({route[0] for route in routes}),
|
||||
"providers": sorted({route[1] for route in routes}),
|
||||
"families": sorted({route[2] for route in routes}),
|
||||
"efforts": sorted({route[3] for route in routes}),
|
||||
"fallback_events": len(fallbacks),
|
||||
"fallback_reasons": sorted(
|
||||
{str(record["fallback_reason"]) for record in fallbacks}
|
||||
),
|
||||
"latest_ts": stamps[-1].isoformat() if stamps else "",
|
||||
}
|
||||
if not records:
|
||||
return Evaluation(
|
||||
NOT_RUN, "routing log tail contained no parsable records", evidence
|
||||
)
|
||||
missing = [
|
||||
f"{label}:{item}"
|
||||
for label, wanted in (
|
||||
("provider", spec.expect.get("providers", ())),
|
||||
("effort", spec.expect.get("efforts", ())),
|
||||
("lane", spec.expect.get("lanes", ())),
|
||||
)
|
||||
for item in wanted
|
||||
if item not in evidence[f"{label}s"]
|
||||
]
|
||||
if spec.expect.get("require_fallback_evidence") and not fallbacks:
|
||||
missing.append("fallback:none-observed")
|
||||
max_age = spec.expect.get("max_age_seconds")
|
||||
if max_age and now:
|
||||
ages: dict[str, int] = {}
|
||||
for label, wanted in (
|
||||
("provider", spec.expect.get("providers", ())),
|
||||
("effort", spec.expect.get("efforts", ())),
|
||||
("lane", spec.expect.get("lanes", ())),
|
||||
):
|
||||
for item in wanted:
|
||||
stamp = stamps_by_dimension.get((label, item))
|
||||
if stamp is None:
|
||||
continue
|
||||
age = int((now - stamp).total_seconds())
|
||||
ages[f"{label}:{item}"] = age
|
||||
if age > max_age:
|
||||
missing.append(
|
||||
f"freshness:{label}:{item}:{age}s-exceeds-{max_age}s"
|
||||
)
|
||||
evidence["age_seconds"] = ages
|
||||
if missing:
|
||||
return Evaluation(
|
||||
FAIL, f"routing evidence missing {', '.join(missing)}", evidence
|
||||
)
|
||||
return Evaluation(PASS, "routing evidence covers the required lanes", evidence)
|
||||
|
||||
|
||||
@evaluator("flux_health")
|
||||
def evaluate_flux_health(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Classify a compact Flux projection into suspensions and unhealthy objects.
|
||||
|
||||
Each row carries name, generation, observed generation, suspension, and all
|
||||
condition states. A non-suspended object must be current, explicitly
|
||||
``Ready=True``, not ``Healthy=False/Unknown``, and not reconciling. Suspended
|
||||
objects are judged against the exact expected-suspension list.
|
||||
"""
|
||||
blocked = missing_steps(spec, outcomes)
|
||||
if blocked:
|
||||
return blocked
|
||||
key = spec.expect["step"]
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
expected = set(spec.expect.get("expected_suspensions", ()))
|
||||
suspended: list[str] = []
|
||||
unhealthy: list[str] = []
|
||||
total = 0
|
||||
for row_number, line in enumerate(outcome.stdout.splitlines(), 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
columns = line.split("\t")
|
||||
if len(columns) != 5:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"Flux row {row_number} has {len(columns)} columns"
|
||||
)
|
||||
name, generation, observed_generation, suspend, conditions = columns
|
||||
name = name.strip()
|
||||
if not _FLUX_NAME_RE.fullmatch(name):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"Flux row {row_number} has malformed object name"
|
||||
)
|
||||
try:
|
||||
current_generation = int(generation)
|
||||
reconciled_generation = int(observed_generation)
|
||||
except ValueError:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"Flux row {row_number} has malformed generations"
|
||||
)
|
||||
if current_generation <= 0 or current_generation != reconciled_generation:
|
||||
return Evaluation(
|
||||
FAIL, f"Flux row {row_number} has stale observedGeneration"
|
||||
)
|
||||
if suspend.strip().lower() not in {"", "false", "true"}:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"Flux row {row_number} has malformed suspension state"
|
||||
)
|
||||
total += 1
|
||||
if suspend.strip().lower() == "true":
|
||||
suspended.append(name)
|
||||
continue
|
||||
states: dict[str, str] = {}
|
||||
for pair in filter(None, conditions.strip().strip(",").split(",")):
|
||||
if pair.count("=") != 1:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"Flux row {row_number} has malformed condition"
|
||||
)
|
||||
condition, state = pair.split("=", 1)
|
||||
if (
|
||||
not condition
|
||||
or state not in {"True", "False", "Unknown"}
|
||||
or condition in states
|
||||
):
|
||||
return Evaluation(
|
||||
NOT_RUN, f"Flux row {row_number} has malformed condition state"
|
||||
)
|
||||
states[condition] = state
|
||||
healthy = states.get("Healthy")
|
||||
if (
|
||||
states.get("Ready") != "True"
|
||||
or healthy in {"False", "Unknown"}
|
||||
or states.get("Reconciling") == "True"
|
||||
):
|
||||
unhealthy.append(name)
|
||||
unexpected = sorted(set(suspended) - expected)
|
||||
evidence = {
|
||||
"total": total,
|
||||
"suspended": sorted(suspended),
|
||||
"unexpected_suspensions": unexpected,
|
||||
"unhealthy": sorted(unhealthy),
|
||||
}
|
||||
if not total:
|
||||
return Evaluation(NOT_RUN, f"step {key} returned no Flux objects", evidence)
|
||||
problems = []
|
||||
if unexpected:
|
||||
problems.append(f"unexpected suspensions: {', '.join(unexpected)}")
|
||||
if unhealthy:
|
||||
problems.append(f"unhealthy: {', '.join(sorted(unhealthy)[:8])}")
|
||||
if problems:
|
||||
return Evaluation(FAIL, "; ".join(problems), evidence)
|
||||
return Evaluation(PASS, f"{total} Flux objects reconciled as expected", evidence)
|
||||
290
scripts/ops/hermes_handoff_model.py
Normal file
290
scripts/ops/hermes_handoff_model.py
Normal file
@ -0,0 +1,290 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Result model, aggregation, and rendering for the handoff acceptance harness.
|
||||
|
||||
The classification is deliberately four-valued. ``NOT_APPLICABLE`` is the only
|
||||
way a check can be excused, and a check can only reach it by matching a scope
|
||||
rule that a reviewer wrote down; anything the harness merely failed to observe
|
||||
is ``NOT_RUN``. Because a mandatory ``NOT_RUN`` is a NO_GO exactly like a
|
||||
``FAIL``, an unreachable cluster, an exhausted deadline, or a probe nobody
|
||||
maintained can never round up to a release.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from hermes_handoff_exec import Outcome
|
||||
from hermes_handoff_redaction import contains_credential_shape, safe_text, scrub
|
||||
|
||||
PASS = "PASS"
|
||||
FAIL = "FAIL"
|
||||
NOT_RUN = "NOT_RUN"
|
||||
NOT_APPLICABLE = "NOT_APPLICABLE"
|
||||
STATUSES = (PASS, FAIL, NOT_RUN, NOT_APPLICABLE)
|
||||
|
||||
GO = "GO"
|
||||
NO_GO = "NO_GO"
|
||||
|
||||
ALWAYS = "always"
|
||||
EPHEMERAL = "ephemeral"
|
||||
|
||||
HARNESS = "hermes-handoff-acceptance"
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
|
||||
REVIEW = "review"
|
||||
ATTEMPT = "attempt"
|
||||
READ = "read"
|
||||
STEP_KINDS = (REVIEW, ATTEMPT, READ)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Step:
|
||||
"""One command a check needs, and the vantage it must be observed from.
|
||||
|
||||
``kind`` separates a self access review (``review``) from a real request the
|
||||
identity is expected to be refused (``attempt``). A deny check needs both:
|
||||
an authorization review states policy, only an attempt proves the API server
|
||||
enforces it.
|
||||
|
||||
``record`` is cleared for bulk evidence — a routing log tail, a large object
|
||||
listing — which the harness parses but must not paste into the artifact.
|
||||
``max_bytes`` raises the capture budget for that same bulk evidence, where a
|
||||
default-sized truncation would silently narrow the window being examined.
|
||||
"""
|
||||
|
||||
key: str
|
||||
vantage: str
|
||||
argv: tuple[str, ...]
|
||||
kind: str = READ
|
||||
optional: bool = False
|
||||
record: bool = True
|
||||
max_bytes: int | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CheckSpec:
|
||||
"""A declarative acceptance check.
|
||||
|
||||
``rule`` names an evaluator; ``expect`` carries its parameters. Keeping the
|
||||
catalog declarative means a reviewer reads what is asserted, not how it is
|
||||
plumbed, and the harness can prove structural properties over every check
|
||||
(no impersonation, no mutation, both vantages present) before running any.
|
||||
"""
|
||||
|
||||
id: str
|
||||
title: str
|
||||
group: str
|
||||
rule: str
|
||||
steps: tuple[Step, ...] = ()
|
||||
expect: Mapping[str, Any] = field(default_factory=dict)
|
||||
mandatory: bool = True
|
||||
scope: str = ALWAYS
|
||||
rationale: str = ""
|
||||
|
||||
@property
|
||||
def vantages(self) -> tuple[str, ...]:
|
||||
"""Return the distinct vantages this check draws evidence from."""
|
||||
seen: list[str] = []
|
||||
for step in self.steps:
|
||||
if step.vantage not in seen:
|
||||
seen.append(step.vantage)
|
||||
return tuple(seen)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CheckResult:
|
||||
"""The classified outcome of one acceptance check."""
|
||||
|
||||
spec: CheckSpec
|
||||
status: str
|
||||
reason: str = ""
|
||||
evidence: dict[str, Any] = field(default_factory=dict)
|
||||
outcomes: list[Outcome] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def blocking(self) -> bool:
|
||||
"""Report whether this result alone forces NO_GO."""
|
||||
if self.status not in STATUSES:
|
||||
return True
|
||||
return self.spec.mandatory and self.status in {FAIL, NOT_RUN}
|
||||
|
||||
def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]:
|
||||
"""Return a JSON-serialisable, screened view of the result."""
|
||||
payload: dict[str, Any] = {
|
||||
"id": self.spec.id,
|
||||
"title": self.spec.title,
|
||||
"group": self.spec.group,
|
||||
"rule": self.spec.rule,
|
||||
"scope": self.spec.scope,
|
||||
"mandatory": self.spec.mandatory,
|
||||
"vantages": list(self.spec.vantages),
|
||||
"status": self.status,
|
||||
"reason": self.reason,
|
||||
"blocking": self.blocking,
|
||||
"evidence": self.evidence,
|
||||
}
|
||||
if self.spec.rationale:
|
||||
payload["rationale"] = self.spec.rationale
|
||||
if include_outcomes:
|
||||
payload["commands"] = [outcome.as_dict() for outcome in self.outcomes]
|
||||
return scrub(payload)
|
||||
|
||||
|
||||
@dataclass
|
||||
class VantageRecord:
|
||||
"""What the harness proved about one vantage before trusting its evidence."""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
identity: str = ""
|
||||
available: bool = False
|
||||
detail: str = ""
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
"""Return a JSON-serialisable view of the vantage."""
|
||||
return scrub(
|
||||
{
|
||||
"name": self.name,
|
||||
"description": self.description,
|
||||
"identity": self.identity,
|
||||
"available": self.available,
|
||||
"detail": self.detail,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Report:
|
||||
"""The whole acceptance run: metadata, per-check results, and a verdict."""
|
||||
|
||||
mode: str
|
||||
started_at: str
|
||||
baseline: dict[str, Any] = field(default_factory=dict)
|
||||
vantages: list[VantageRecord] = field(default_factory=list)
|
||||
results: list[CheckResult] = field(default_factory=list)
|
||||
harness_errors: list[str] = field(default_factory=list)
|
||||
finished_at: str = ""
|
||||
|
||||
@property
|
||||
def counts(self) -> dict[str, int]:
|
||||
"""Return the number of results in each status."""
|
||||
tally = dict.fromkeys(STATUSES, 0)
|
||||
for result in self.results:
|
||||
if result.status in tally:
|
||||
tally[result.status] += 1
|
||||
return tally
|
||||
|
||||
@property
|
||||
def invalid_statuses(self) -> list[str]:
|
||||
"""Return check ids whose status is outside the report schema."""
|
||||
return [
|
||||
result.spec.id for result in self.results if result.status not in STATUSES
|
||||
]
|
||||
|
||||
@property
|
||||
def blocking(self) -> list[CheckResult]:
|
||||
"""Return every result that forces NO_GO, most severe first."""
|
||||
order = {FAIL: 0, NOT_RUN: 1}
|
||||
return sorted(
|
||||
(result for result in self.results if result.blocking),
|
||||
key=lambda result: (order.get(result.status, 2), result.spec.id),
|
||||
)
|
||||
|
||||
@property
|
||||
def decision(self) -> str:
|
||||
"""Return GO only when nothing mandatory failed, skipped, or errored."""
|
||||
if (
|
||||
not self.results
|
||||
or self.invalid_statuses
|
||||
or self.harness_errors
|
||||
or self.blocking
|
||||
):
|
||||
return NO_GO
|
||||
return GO
|
||||
|
||||
def group_summary(self) -> dict[str, dict[str, int]]:
|
||||
"""Return per-group status counts, for the human summary."""
|
||||
groups: dict[str, dict[str, int]] = {}
|
||||
for result in self.results:
|
||||
bucket = groups.setdefault(result.spec.group, dict.fromkeys(STATUSES, 0))
|
||||
bucket[result.status] = bucket.get(result.status, 0) + 1
|
||||
return groups
|
||||
|
||||
def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]:
|
||||
"""Return the machine-readable report."""
|
||||
return scrub(
|
||||
{
|
||||
"harness": HARNESS,
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"mode": self.mode,
|
||||
"started_at": self.started_at,
|
||||
"finished_at": self.finished_at,
|
||||
"decision": self.decision,
|
||||
"counts": self.counts,
|
||||
"baseline": scrub(self.baseline),
|
||||
"vantages": [vantage.as_dict() for vantage in self.vantages],
|
||||
"harness_errors": [scrub(error) for error in self.harness_errors],
|
||||
"invalid_statuses": self.invalid_statuses,
|
||||
"blocking": [result.spec.id for result in self.blocking],
|
||||
"checks": [result.as_dict(include_outcomes) for result in self.results],
|
||||
}
|
||||
)
|
||||
|
||||
def render_summary(self, width: int = 88) -> str:
|
||||
"""Return the concise human summary printed alongside the JSON."""
|
||||
counts = self.counts
|
||||
lines = [
|
||||
f"{HARNESS}: {self.decision}",
|
||||
f" mode={self.mode} started={self.started_at} checks={len(self.results)}",
|
||||
" " + " ".join(f"{status}={counts[status]}" for status in STATUSES),
|
||||
]
|
||||
for vantage in self.vantages:
|
||||
state = "available" if vantage.available else "unavailable"
|
||||
identity = vantage.identity or "unknown"
|
||||
lines.append(f" vantage {vantage.name}: {state} identity={identity}")
|
||||
for error in self.harness_errors:
|
||||
lines.append(f" harness error: {error}")
|
||||
if self.blocking:
|
||||
lines.append(" blocking:")
|
||||
for result in self.blocking:
|
||||
lines.append(f" {result.status:<15}{result.spec.id}")
|
||||
lines.append(f" {'':<15}{result.reason[:width]}")
|
||||
groups = self.group_summary()
|
||||
lines.append(" groups:")
|
||||
for name in sorted(groups):
|
||||
bucket = groups[name]
|
||||
detail = " ".join(
|
||||
f"{status}={bucket[status]}" for status in STATUSES if bucket[status]
|
||||
)
|
||||
lines.append(f" {name:<18}{detail}")
|
||||
return safe_text("\n".join(lines), 32 * 1024)[0]
|
||||
|
||||
|
||||
def utc_now(clock: Any = None) -> str:
|
||||
"""Return an ISO-8601 UTC timestamp, injectable for deterministic tests."""
|
||||
now = clock() if clock else dt.datetime.now(dt.timezone.utc)
|
||||
return now.replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def unscreened_fields(payload: Any, path: str = "$") -> list[str]:
|
||||
"""Return JSON pointers whose text still looks credential-shaped.
|
||||
|
||||
The report is already screened field by field; this is the invariant check
|
||||
that runs before anything is written, so a screening gap fails the run
|
||||
instead of publishing an artifact nobody re-read.
|
||||
"""
|
||||
offenders: list[str] = []
|
||||
if isinstance(payload, str):
|
||||
if contains_credential_shape(payload):
|
||||
offenders.append(path)
|
||||
elif isinstance(payload, Mapping):
|
||||
for key, value in payload.items():
|
||||
offenders.extend(unscreened_fields(value, f"{path}.{key}"))
|
||||
elif isinstance(payload, Sequence) and not isinstance(payload, (str, bytes)):
|
||||
for index, value in enumerate(payload):
|
||||
offenders.extend(unscreened_fields(value, f"{path}[{index}]"))
|
||||
return offenders
|
||||
493
scripts/ops/hermes_handoff_policy.py
Normal file
493
scripts/ops/hermes_handoff_policy.py
Normal file
@ -0,0 +1,493 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail-closed command policy for the Hermes handoff acceptance harness.
|
||||
|
||||
Before spawn it admits only credential-safe, structurally read-only grammar;
|
||||
shell is limited to frozen templates with absolute tool paths.
|
||||
|
||||
Impersonation is structurally impossible: ``--as`` and its siblings are refused
|
||||
for every argv, in every mode, from every vantage, and the inner command of a
|
||||
``kubectl exec`` is re-checked under the same rule rather than being trusted.
|
||||
Only binaries with a pinned release digest are on the allowlist, so nothing can
|
||||
be admitted here that the executor would then refuse to attest.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from urllib.parse import unquote
|
||||
|
||||
READ_ONLY = "read-only"
|
||||
ARMED = "ephemeral-armed"
|
||||
MODES = (READ_ONLY, ARMED)
|
||||
|
||||
GITEA_CLIENT = "/opt/coordinator/gitea_api.py"
|
||||
EXPECTED_REPO = "atlas/titan-iac"
|
||||
EXPECTED_REMOTE = "origin"
|
||||
EXPECTED_BASE = "main"
|
||||
|
||||
# fmt: off
|
||||
FORBIDDEN_ARGS = ("--as", "--as-group", "--as-uid", "--client-certificate",
|
||||
"--client-key", "--password", "--token", "--username")
|
||||
# fmt: on
|
||||
IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid")
|
||||
FORBIDDEN_BINARIES = frozenset(
|
||||
{"age", "curl", "gpg", "openssl", "sops", "vault", "wget"}
|
||||
)
|
||||
ALLOWED_BINARIES = frozenset({"kubectl", "git", "hermes", "sh", GITEA_CLIENT})
|
||||
TRUSTED_EXECUTABLE_ROOTS = (
|
||||
"/bin/",
|
||||
"/usr/bin/",
|
||||
"/usr/local/bin/",
|
||||
"/opt/data/tools/bin/",
|
||||
"/opt/hermes/.venv/bin/",
|
||||
"/opt/coordinator/",
|
||||
)
|
||||
TRUSTED_INNER_PATHS = {
|
||||
"/bin/sh": "sh",
|
||||
"/usr/bin/git": "git",
|
||||
"/usr/bin/sh": "sh",
|
||||
"/usr/local/bin/kubectl": "kubectl",
|
||||
"/opt/hermes/.venv/bin/hermes": "hermes",
|
||||
GITEA_CLIENT: GITEA_CLIENT,
|
||||
}
|
||||
|
||||
VALUE_FLAGS = frozenset(
|
||||
{
|
||||
"-C",
|
||||
"-c",
|
||||
"-l",
|
||||
"-n",
|
||||
"-o",
|
||||
"--cluster",
|
||||
"--container",
|
||||
"--context",
|
||||
"--field-selector",
|
||||
"--kubeconfig",
|
||||
"--namespace",
|
||||
"--output",
|
||||
"--request-timeout",
|
||||
"--selector",
|
||||
"--server",
|
||||
"--since",
|
||||
"--tail",
|
||||
"--user",
|
||||
}
|
||||
)
|
||||
|
||||
KUBECTL_READ_SUBCOMMANDS = frozenset(
|
||||
{
|
||||
"api-resources",
|
||||
"api-versions",
|
||||
"explain",
|
||||
"get",
|
||||
"top",
|
||||
"version",
|
||||
}
|
||||
)
|
||||
KUBECTL_EXEC_SUBCOMMANDS = frozenset({"exec"})
|
||||
KUBECTL_SUBCOMMANDS = KUBECTL_READ_SUBCOMMANDS | KUBECTL_EXEC_SUBCOMMANDS | {"auth"}
|
||||
SECRET_RESOURCES = frozenset({"secret", "secrets"})
|
||||
UNSAFE_PROJECTION_PARTS = (
|
||||
".args",
|
||||
".command",
|
||||
".data",
|
||||
".envfrom",
|
||||
".secretkeyref",
|
||||
".status.token",
|
||||
".stringdata",
|
||||
".valuefrom",
|
||||
"clientconfig",
|
||||
"password",
|
||||
"privatekey",
|
||||
)
|
||||
SAFE_ENV_VALUE = 'env[?(@.name=="hermes_auto_router_profile")].value'
|
||||
SAFE_JSON_FIELDS = frozenset(["authenticated", "transport", "state", "model", "effort", "latency_ms", "checked_at", "api_provider", "auth_method", "subscription_type"]) # fmt: skip
|
||||
|
||||
GIT_READ_SUBCOMMANDS = frozenset(
|
||||
{
|
||||
"cat-file",
|
||||
"diff",
|
||||
"for-each-ref",
|
||||
"log",
|
||||
"ls-remote",
|
||||
"ls-tree",
|
||||
"merge-base",
|
||||
"rev-list",
|
||||
"rev-parse",
|
||||
"show",
|
||||
"status",
|
||||
}
|
||||
)
|
||||
GIT_FORCE_FLAGS = frozenset(
|
||||
{"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"}
|
||||
)
|
||||
GIT_UNSAFE_FLAGS = (
|
||||
"-c",
|
||||
"--config-env",
|
||||
"--exec-path",
|
||||
"--upload-pack",
|
||||
"--receive-pack",
|
||||
)
|
||||
GITEA_READ_METHODS = frozenset({"GET"})
|
||||
HERMES_READ_COMMANDS = frozenset(
|
||||
{("kanban", "list"), ("kanban", "show"), ("sessions", "list"), ("status",)}
|
||||
)
|
||||
|
||||
# These locations contain credential bytes even when a caller claims it will
|
||||
# only inspect metadata. They are rejected for every argv, including a frozen
|
||||
# shell rendering and the inner command of kubectl exec.
|
||||
SECRET_PATH_RE = re.compile(
|
||||
r"(?:^|[\s'\"=])(?:/[^\s'\"]*)?(?:"
|
||||
r"var/run/secrets|run/secrets|vault/secrets|runtime-access|scm-access|pool-access|"
|
||||
r"\.credentials\.json|auth\.json|\.git-credentials|authorized_keys|"
|
||||
r"id_[a-z0-9]+|\.netrc|shadow|gshadow|k3s\.yaml"
|
||||
r")(?:$|[/\s'\"])",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_PARAMETER_RE = re.compile(r"[A-Za-z0-9/][A-Za-z0-9._/@:,-]{0,255}\Z")
|
||||
_PLACEHOLDER_RE = re.compile(r"\{([a-z_]+)\}")
|
||||
_SAFE_API_PATH_RE = re.compile(r"/[A-Za-z0-9][A-Za-z0-9._~:/?&=%+-]{0,511}\Z")
|
||||
|
||||
SHELL_TEMPLATES: dict[str, str] = {
|
||||
"env_names": (
|
||||
"/usr/bin/env | /usr/bin/sed -n "
|
||||
"'s/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/\\1/p' | /usr/bin/sort"
|
||||
),
|
||||
"json_fields": (
|
||||
'/opt/hermes/.venv/bin/python -c "import json,sys;d=json.load(open(sys.argv[1]));'
|
||||
'print(json.dumps({{k:d.get(k) for k in sys.argv[2].split(chr(44))}}))" '
|
||||
"{path} {fields}"
|
||||
),
|
||||
"tail_lines": "/usr/bin/tail -n {limit} {path}",
|
||||
}
|
||||
|
||||
|
||||
class PolicyError(ValueError):
|
||||
"""Raised when a requested command falls outside the safe boundary."""
|
||||
|
||||
|
||||
def template_parameters(template_name: str) -> set[str]:
|
||||
"""Return the parameter names a frozen template requires."""
|
||||
template = SHELL_TEMPLATES.get(template_name)
|
||||
if template is None:
|
||||
raise PolicyError(f"unknown shell template: {template_name}")
|
||||
return set(_PLACEHOLDER_RE.findall(template))
|
||||
|
||||
|
||||
def render_shell(template_name: str, **parameters: str) -> str:
|
||||
"""Return a frozen shell template with validated parameters substituted."""
|
||||
required = template_parameters(template_name)
|
||||
if set(parameters) != required:
|
||||
raise PolicyError(
|
||||
f"shell template {template_name} expects parameters {sorted(required)}"
|
||||
)
|
||||
for key, value in parameters.items():
|
||||
if not _PARAMETER_RE.fullmatch(value):
|
||||
raise PolicyError(f"unsafe shell parameter for {template_name}.{key}")
|
||||
if SECRET_PATH_RE.search(value):
|
||||
raise PolicyError("shell templates may not name credential paths")
|
||||
if (
|
||||
template_name == "json_fields"
|
||||
and not set(parameters["fields"].split(",")) <= SAFE_JSON_FIELDS
|
||||
):
|
||||
raise PolicyError("json_fields requested a credential-bearing or unknown field")
|
||||
return SHELL_TEMPLATES[template_name].format(**parameters)
|
||||
|
||||
|
||||
_RENDERED: set[str] = set()
|
||||
_APPROVED_PROJECTIONS: set[str] = set()
|
||||
_ARMED_REF = ""
|
||||
_ARMED_PULLS: set[int] = set()
|
||||
|
||||
|
||||
# fmt: off
|
||||
def arm_ephemeral_policy(ref: str) -> None:
|
||||
global _ARMED_REF
|
||||
if not re.fullmatch(r"ephemeral/hermes-handoff-acceptance/[a-z0-9][a-z0-9-]{7,63}", ref):
|
||||
raise PolicyError("armed policy requires one exact ephemeral ref")
|
||||
_ARMED_REF = ref
|
||||
_ARMED_PULLS.clear()
|
||||
|
||||
|
||||
def register_ephemeral_pull(number: int) -> None:
|
||||
if not _ARMED_REF or isinstance(number, bool) or not isinstance(number, int) or number <= 0:
|
||||
raise PolicyError("ephemeral pull registration is malformed")
|
||||
_ARMED_PULLS.add(number)
|
||||
|
||||
|
||||
def shell(template_name: str, **parameters: str) -> tuple[str, ...]:
|
||||
"""Return a validated ``sh -c`` argv built from a frozen template."""
|
||||
script = render_shell(template_name, **parameters)
|
||||
_RENDERED.add(script)
|
||||
return ("sh", "-c", script)
|
||||
|
||||
|
||||
def projection(value: str) -> str:
|
||||
normalized = value.lower().replace(" ", "")
|
||||
if not normalized.startswith("jsonpath=") or any(part in normalized for part in UNSAFE_PROJECTION_PARTS):
|
||||
raise PolicyError("Kubernetes projection is not credential-safe")
|
||||
if ".metadata.annotations" in normalized and not any(key in normalized for key in (
|
||||
"deployment\\.kubernetes\\.io/revision", "ai\\.bstein\\.dev/config-rev")):
|
||||
raise PolicyError("Kubernetes annotations are not on the reviewed allowlist")
|
||||
if ".metadata.labels" in normalized and "pod-template-hash" not in normalized:
|
||||
raise PolicyError("Kubernetes labels are not on the reviewed allowlist")
|
||||
if ".env" in normalized and ".value" in normalized and SAFE_ENV_VALUE not in normalized:
|
||||
raise PolicyError("Kubernetes projection may expose environment values")
|
||||
_APPROVED_PROJECTIONS.add(normalized)
|
||||
return value
|
||||
# fmt: on
|
||||
|
||||
|
||||
def positionals(argv: tuple[str, ...], start: int = 1) -> Iterator[str]:
|
||||
"""Yield positional arguments, skipping flags and their values."""
|
||||
index = start
|
||||
while index < len(argv):
|
||||
item = argv[index]
|
||||
if item == "--":
|
||||
return
|
||||
if item.startswith("-"):
|
||||
index += 2 if item in VALUE_FLAGS else 1
|
||||
continue
|
||||
yield item
|
||||
index += 1
|
||||
|
||||
|
||||
def _binary_name(value: str) -> str:
|
||||
if value == GITEA_CLIENT or (
|
||||
value.startswith("/opt/coordinator/") and value.endswith("/gitea_api.py")
|
||||
):
|
||||
return GITEA_CLIENT
|
||||
if "/" not in value:
|
||||
return value
|
||||
mapped = TRUSTED_INNER_PATHS.get(value)
|
||||
if mapped:
|
||||
return mapped
|
||||
if "/../" in value or value.endswith("/..") or "/./" in value:
|
||||
raise PolicyError(f"executable path is not normalized: {value}")
|
||||
if not value.startswith(TRUSTED_EXECUTABLE_ROOTS):
|
||||
raise PolicyError(f"executable path is not trusted: {value}")
|
||||
return value.rsplit("/", 1)[-1]
|
||||
|
||||
|
||||
def _reject_forbidden_arguments(argv: tuple[str, ...]) -> None:
|
||||
"""Refuse credential and impersonation arguments up to the ``--`` boundary.
|
||||
|
||||
Everything after ``--`` is the inner command of a ``kubectl exec``, which
|
||||
:func:`_check_kubectl` submits to :func:`check_argv` again, so it is held to
|
||||
exactly this rule rather than being exempted from it.
|
||||
"""
|
||||
boundary = argv.index("--") if "--" in argv else len(argv)
|
||||
for argument in argv[:boundary]:
|
||||
for forbidden in FORBIDDEN_ARGS:
|
||||
if argument == forbidden or argument.startswith(f"{forbidden}="):
|
||||
raise PolicyError(f"forbidden argument: {forbidden}")
|
||||
if argument == "--raw" or argument.startswith("--raw="):
|
||||
raise PolicyError("raw API reads are forbidden")
|
||||
for argument in argv:
|
||||
if argument == "--dry-run" or argument.startswith("--dry-run="):
|
||||
raise PolicyError("dry-run mutation commands are forbidden")
|
||||
|
||||
|
||||
def _reject_secret_paths(argv: tuple[str, ...]) -> None:
|
||||
if any(SECRET_PATH_RE.search(argument) for argument in argv):
|
||||
raise PolicyError("probes may not name a credential path")
|
||||
|
||||
|
||||
def _resource_name(value: str) -> str:
|
||||
return value.lower().split("/", 1)[0].split(".", 1)[0]
|
||||
|
||||
|
||||
def _output_value(argv: tuple[str, ...]) -> str:
|
||||
"""Return kubectl's output argument in a single canonical form."""
|
||||
for index, item in enumerate(argv[1:], 1):
|
||||
if item in {"-o", "--output"} and index + 1 < len(argv):
|
||||
return argv[index + 1]
|
||||
if item.startswith(("-o=", "--output=")):
|
||||
return item.split("=", 1)[1]
|
||||
return ""
|
||||
|
||||
|
||||
def _check_projection(output: str) -> None:
|
||||
"""Reject full or credential-bearing Kubernetes output projections."""
|
||||
lowered = output.lower().replace(" ", "")
|
||||
if lowered in {"json", "yaml", "wide"} or lowered.startswith("custom-columns"):
|
||||
raise PolicyError(
|
||||
"full Kubernetes object output is forbidden; use a safe projection"
|
||||
)
|
||||
if lowered and not (lowered == "name" or lowered.startswith("jsonpath=")):
|
||||
raise PolicyError(
|
||||
"Kubernetes output must be name or an approved JSONPath projection"
|
||||
)
|
||||
if lowered.startswith("jsonpath="):
|
||||
if any(part in lowered for part in UNSAFE_PROJECTION_PARTS):
|
||||
raise PolicyError("Kubernetes projection may expose runtime credentials")
|
||||
if ".env" in lowered and SAFE_ENV_VALUE not in lowered:
|
||||
# Environment names are safe; arbitrary `.value` reads are not.
|
||||
without_names = lowered.replace("{.name}", "")
|
||||
if ".value" in without_names:
|
||||
raise PolicyError("Kubernetes projection may expose environment values")
|
||||
if lowered not in _APPROVED_PROJECTIONS:
|
||||
raise PolicyError(
|
||||
"Kubernetes JSONPath projection is not on the reviewed allowlist"
|
||||
)
|
||||
|
||||
|
||||
def _check_kubectl(argv: tuple[str, ...], mode: str) -> None:
|
||||
words = list(positionals(argv))
|
||||
if not words or words[0] not in KUBECTL_SUBCOMMANDS:
|
||||
raise PolicyError("kubectl subcommand is outside the read-only boundary")
|
||||
subcommand = words[0]
|
||||
if subcommand == "auth":
|
||||
if len(words) < 2 or words[1] not in {"can-i", "whoami"}:
|
||||
raise PolicyError("kubectl auth is limited to can-i and whoami")
|
||||
return
|
||||
if subcommand == "exec":
|
||||
if "--" not in argv:
|
||||
raise PolicyError("kubectl exec requires an explicit -- command boundary")
|
||||
inner = argv[argv.index("--") + 1 :]
|
||||
if not inner:
|
||||
raise PolicyError("kubectl exec requires an inner command")
|
||||
if inner[0] not in TRUSTED_INNER_PATHS:
|
||||
raise PolicyError(
|
||||
"kubectl exec inner command must use a fixed trusted path"
|
||||
)
|
||||
check_argv(inner, mode)
|
||||
return
|
||||
resources = words[1].split(",") if len(words) > 1 else []
|
||||
if any(_resource_name(resource) in SECRET_RESOURCES for resource in resources):
|
||||
raise PolicyError("Secret objects may not be read by the harness")
|
||||
output = _output_value(argv)
|
||||
if subcommand == "get" and not output:
|
||||
raise PolicyError("kubectl get requires a reviewed output projection")
|
||||
_check_projection(output)
|
||||
|
||||
|
||||
def _check_git(argv: tuple[str, ...], mode: str) -> None:
|
||||
unsafe = False
|
||||
for item in argv[1:]:
|
||||
if item == "-C" or item.startswith(("--git-dir", "--work-tree", "--namespace")):
|
||||
unsafe = True
|
||||
if item == "-c" or (item.startswith("-c") and len(item) > 2):
|
||||
unsafe = True
|
||||
if any(
|
||||
item == flag or item.startswith(f"{flag}=") for flag in GIT_UNSAFE_FLAGS[1:]
|
||||
):
|
||||
unsafe = True
|
||||
if unsafe:
|
||||
raise PolicyError("git configuration and helper overrides are forbidden")
|
||||
subcommand = next(positionals(argv), None)
|
||||
if subcommand is None:
|
||||
raise PolicyError("git requires a subcommand")
|
||||
if subcommand in GIT_READ_SUBCOMMANDS:
|
||||
return
|
||||
if subcommand == "push" and mode == ARMED:
|
||||
if GIT_FORCE_FLAGS & set(argv):
|
||||
raise PolicyError("git push may not force or fan out across refs")
|
||||
words = list(positionals(argv))
|
||||
if len(words) != 3 or words[1] != EXPECTED_REMOTE:
|
||||
raise PolicyError("armed git push is pinned to origin and one refspec")
|
||||
destination = words[2].partition(":")[2]
|
||||
if not _ARMED_REF or destination != f"refs/heads/{_ARMED_REF}":
|
||||
raise PolicyError("armed git push is not the preflighted ephemeral ref")
|
||||
return
|
||||
raise PolicyError(f"git {subcommand} is not permitted in {mode} mode")
|
||||
|
||||
|
||||
def _check_hermes(argv: tuple[str, ...]) -> None:
|
||||
words = tuple(positionals(argv))
|
||||
if not words:
|
||||
raise PolicyError("hermes requires a subcommand")
|
||||
if words[:2] not in HERMES_READ_COMMANDS and words[:1] not in HERMES_READ_COMMANDS:
|
||||
raise PolicyError(
|
||||
f"hermes {' '.join(words[:2])} is outside the read-only boundary"
|
||||
)
|
||||
|
||||
|
||||
def _reject_dot_segments(path: str) -> None:
|
||||
"""Refuse any relative segment, including a percent-encoded one.
|
||||
|
||||
The repository pin below is a textual prefix test, so a single ``..`` would
|
||||
let a pinned path address another repository or the token endpoints once a
|
||||
router normalised it. Decode first: ``%2e%2e`` is the same segment.
|
||||
"""
|
||||
decoded = unquote(unquote(path)).split("?", 1)[0]
|
||||
if any(segment in {"", ".", ".."} for segment in decoded.split("/")[1:]):
|
||||
raise PolicyError("gitea_api.py paths may not contain relative segments")
|
||||
|
||||
|
||||
def _check_gitea(argv: tuple[str, ...], mode: str) -> None:
|
||||
if len(argv) < 3:
|
||||
raise PolicyError("gitea_api.py requires a method and a path")
|
||||
method, path = argv[1].upper(), argv[2]
|
||||
if not path.startswith("/api/v1/") or not _SAFE_API_PATH_RE.fullmatch(path):
|
||||
raise PolicyError("gitea_api.py requires a bounded API path")
|
||||
_reject_dot_segments(path)
|
||||
repo_root = f"/api/v1/repos/{EXPECTED_REPO}"
|
||||
if (
|
||||
path != "/api/v1/user"
|
||||
and path != repo_root
|
||||
and not path.startswith((f"{repo_root}/", f"{repo_root}?"))
|
||||
):
|
||||
raise PolicyError("gitea_api.py is pinned to the Atlas titan-iac repository")
|
||||
if method in GITEA_READ_METHODS:
|
||||
return
|
||||
if mode == ARMED:
|
||||
fields = [
|
||||
argv[index + 1] for index, item in enumerate(argv[:-1]) if item == "--field"
|
||||
]
|
||||
if method == "POST" and path == f"{repo_root}/pulls" and _ARMED_REF:
|
||||
required = {
|
||||
"title=WIP: Hermes handoff acceptance ephemeral probe",
|
||||
f"head={_ARMED_REF}",
|
||||
f"base={EXPECTED_BASE}",
|
||||
"body=Ephemeral acceptance probe. Closed and deleted by the harness.",
|
||||
}
|
||||
if len(fields) == len(required) and set(fields) == required:
|
||||
return
|
||||
match = re.fullmatch(rf"{re.escape(repo_root)}/pulls/([1-9][0-9]*)", path)
|
||||
if (
|
||||
method == "PATCH"
|
||||
and match
|
||||
and int(match.group(1)) in _ARMED_PULLS
|
||||
and fields == ["state=closed"]
|
||||
):
|
||||
return
|
||||
if (
|
||||
method == "DELETE"
|
||||
and _ARMED_REF
|
||||
and path == f"{repo_root}/branches/{_ARMED_REF}"
|
||||
and len(argv) == 3
|
||||
):
|
||||
return
|
||||
raise PolicyError(f"gitea_api.py {method} is not permitted in {mode} mode")
|
||||
|
||||
|
||||
def _check_shell(argv: tuple[str, ...]) -> None:
|
||||
if len(argv) != 3 or argv[1] != "-c" or argv[2] not in _RENDERED:
|
||||
raise PolicyError("sh is only reachable as sh -c <frozen template>")
|
||||
|
||||
|
||||
def check_argv(argv: tuple[str, ...], mode: str = READ_ONLY) -> None:
|
||||
"""Raise :class:`PolicyError` unless ``argv`` is safe in ``mode``."""
|
||||
if mode not in MODES:
|
||||
raise PolicyError(f"unknown mode: {mode}")
|
||||
if not argv:
|
||||
raise PolicyError("empty command")
|
||||
binary = _binary_name(argv[0])
|
||||
if binary in FORBIDDEN_BINARIES:
|
||||
raise PolicyError(f"forbidden binary: {binary}")
|
||||
if binary not in ALLOWED_BINARIES:
|
||||
raise PolicyError(f"binary is not on the harness allowlist: {binary}")
|
||||
_reject_forbidden_arguments(argv)
|
||||
_reject_secret_paths(argv)
|
||||
if binary == "sh":
|
||||
_check_shell(argv)
|
||||
elif binary == "kubectl":
|
||||
_check_kubectl(argv, mode)
|
||||
elif binary == "git":
|
||||
_check_git(argv, mode)
|
||||
elif binary == "hermes":
|
||||
_check_hermes(argv)
|
||||
else:
|
||||
_check_gitea(argv, mode)
|
||||
243
scripts/ops/hermes_handoff_redaction.py
Normal file
243
scripts/ops/hermes_handoff_redaction.py
Normal file
@ -0,0 +1,243 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Credential redaction and output bounding for the handoff acceptance harness.
|
||||
|
||||
Every string the harness records — command output, failure reasons, evidence
|
||||
values — passes through :func:`redact` before it reaches a report. Screening is
|
||||
a fail-closed accident barrier, not proof that text is secret-free: a short
|
||||
secret under an innocuous key is not reliably distinguishable from prose. The
|
||||
harness therefore also never asks for a secret in the first place. Probes read
|
||||
names, shapes, and status fields; they never read environment values, token
|
||||
files, private keys, cookies, or Vault responses.
|
||||
|
||||
Bounding is part of the same barrier. Screening always precedes truncation so a
|
||||
credential that crosses the byte boundary cannot leave a useful prefix behind.
|
||||
The process runner additionally discards all captured bytes when a live stream
|
||||
exceeds its cap; the fixed truncation marker is the only recorded output.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
REDACTED = "[redacted]"
|
||||
DEFAULT_MAX_BYTES = 64 * 1024
|
||||
TRUNCATION_NOTE = "[truncated]"
|
||||
|
||||
# Keys whose value is never safe to record, matched against a normalised
|
||||
# (lowercased, separator-stripped) form of the key.
|
||||
SENSITIVE_KEY_SUFFIXES = (
|
||||
"accesskey",
|
||||
"accesskeyid",
|
||||
"apikey",
|
||||
"authtoken",
|
||||
"clientsecret",
|
||||
"connectionstring",
|
||||
"cookie",
|
||||
"credential",
|
||||
"credentials",
|
||||
"passphrase",
|
||||
"passwd",
|
||||
"password",
|
||||
"privatekey",
|
||||
"secret",
|
||||
"secretkey",
|
||||
"sessionkey",
|
||||
"signature",
|
||||
"signingkey",
|
||||
"token",
|
||||
"webhook",
|
||||
)
|
||||
SENSITIVE_KEY_EXACT = frozenset(
|
||||
{
|
||||
"auth",
|
||||
"authorization",
|
||||
"bearer",
|
||||
"pat",
|
||||
"pw",
|
||||
"sas",
|
||||
"setcookie",
|
||||
}
|
||||
)
|
||||
# A bare `key:` is a structural field name across Kubernetes and TOML far more
|
||||
# often than it is a credential, so it is screened by value shape and by the
|
||||
# token patterns rather than by name. The compound forms above still match.
|
||||
|
||||
# Ordinary prose values shorter than this threshold are not independently
|
||||
# treated as secret-shaped. Values under sensitive keys are screened at every
|
||||
# length, including numeric values.
|
||||
MIN_SECRET_LENGTH = 8
|
||||
NON_SECRET_LITERALS = frozenset(
|
||||
{"true", "false", "null", "none", "yes", "no", "0", "1", ""}
|
||||
)
|
||||
_KEY_NOISE_RE = re.compile(r"[^a-z0-9]+")
|
||||
|
||||
# Ordered: structural matches first so a nested token is not partially rewritten
|
||||
# by a looser rule.
|
||||
_PATTERNS: tuple[re.Pattern[str], ...] = (
|
||||
re.compile(
|
||||
r"-----BEGIN[ A-Z0-9-]{0,64}-----.*?-----END[ A-Z0-9-]{0,64}-----",
|
||||
re.DOTALL,
|
||||
),
|
||||
re.compile(r"-----BEGIN[ A-Z0-9-]{0,64}-----.*", re.DOTALL),
|
||||
re.compile(r"(?i)\b(?:authorization|proxy-authorization)\s*:\s*\S.*"),
|
||||
re.compile(r"(?i)\b(?:set-)?cookie\s*:\s*\S.*"),
|
||||
re.compile(r"(?i)\b(?:bearer|basic|token)\s+[A-Za-z0-9+/_=.:~-]{8,}"),
|
||||
re.compile(r"(?i)\b[a-z][a-z0-9+.-]{1,20}://[^\s/@:]{1,128}:[^\s/@]{1,256}@"),
|
||||
re.compile(
|
||||
r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}"
|
||||
),
|
||||
re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}"),
|
||||
re.compile(r"(?<![A-Za-z0-9])gh[pousr]_[A-Za-z0-9]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])github_pat_[A-Za-z0-9_]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])glpat-[A-Za-z0-9_-]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])xox[baprs]-[A-Za-z0-9-]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])sk-(?:ant-|proj-)?[A-Za-z0-9_-]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])(?:gta|gto|gitea|forgejo)_[A-Za-z0-9_-]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])hvs\.[A-Za-z0-9_-]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])(?:AKIA|ASIA)[0-9A-Z]*"),
|
||||
re.compile(r"(?<![A-Za-z0-9])AIza[0-9A-Za-z_-]{35}"),
|
||||
re.compile(r"(?<![A-Za-z0-9])npm_[A-Za-z0-9]{16,}"),
|
||||
)
|
||||
|
||||
# `key: value` / `key=value` where the key names a credential.
|
||||
_ASSIGNMENT_RE = re.compile(
|
||||
r"""(?x)
|
||||
(?P<quote>["']?)
|
||||
(?P<key>[A-Za-z][A-Za-z0-9_.\ -]{0,63})
|
||||
(?P=quote)
|
||||
(?P<gap>\s{0,8}[:=]\s{0,8})
|
||||
(?P<value>"[^"\n]{0,4096}"|'[^'\n]{0,4096}'|[^\s,;}\n][^,;}\n]{0,4096})
|
||||
"""
|
||||
)
|
||||
|
||||
# Long, mixed-class runs. Deliberately conservative: prose and hex digests of
|
||||
# fewer than 44 characters are left alone so commit SHAs stay readable.
|
||||
_ENTROPY_RE = re.compile(
|
||||
r"(?<![A-Za-z0-9+/_=-])[A-Za-z0-9+/_=-]{44,}(?![A-Za-z0-9+/_=-])"
|
||||
)
|
||||
|
||||
|
||||
def normalise_key(key: str) -> str:
|
||||
"""Return a key lowercased with separators removed, for suffix matching."""
|
||||
return _KEY_NOISE_RE.sub("", key.lower())
|
||||
|
||||
|
||||
def is_sensitive_key(key: str) -> bool:
|
||||
"""Report whether a key name means its value must never be recorded."""
|
||||
compact = normalise_key(key)
|
||||
if not compact:
|
||||
return False
|
||||
if compact in SENSITIVE_KEY_EXACT:
|
||||
return True
|
||||
return compact.endswith(SENSITIVE_KEY_SUFFIXES)
|
||||
|
||||
|
||||
def _is_high_entropy(candidate: str) -> bool:
|
||||
"""Report whether a long run mixes enough character classes to be a token."""
|
||||
classes = sum(
|
||||
(
|
||||
any(character.islower() for character in candidate),
|
||||
any(character.isupper() for character in candidate),
|
||||
any(character.isdigit() for character in candidate),
|
||||
any(character in "+/_=-" for character in candidate),
|
||||
)
|
||||
)
|
||||
return classes >= 4
|
||||
|
||||
|
||||
def could_hold_secret(value: str) -> bool:
|
||||
"""Report whether a raw value is long and opaque enough to be a credential."""
|
||||
stripped = value.strip().strip("\"'")
|
||||
if stripped.lower() in NON_SECRET_LITERALS or len(stripped) < MIN_SECRET_LENGTH:
|
||||
return False
|
||||
try:
|
||||
float(stripped)
|
||||
except ValueError:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _redact_assignment(match: re.Match[str]) -> str:
|
||||
value = match.group("value")
|
||||
stripped = value.strip().strip("\"'").lower()
|
||||
if not is_sensitive_key(match.group("key")) or stripped in NON_SECRET_LITERALS:
|
||||
return match.group(0)
|
||||
# Preserve the original quoting so screened JSON stays parsable JSON.
|
||||
quoted = value.strip()[:1] in {'"', "'"}
|
||||
replacement = f'"{REDACTED}"' if quoted else REDACTED
|
||||
key_quote = match.group("quote")
|
||||
return (
|
||||
f"{key_quote}{match.group('key')}{key_quote}{match.group('gap')}{replacement}"
|
||||
)
|
||||
|
||||
|
||||
def _redact_entropy(match: re.Match[str]) -> str:
|
||||
return REDACTED if _is_high_entropy(match.group(0)) else match.group(0)
|
||||
|
||||
|
||||
def redact(text: str) -> str:
|
||||
"""Return ``text`` with credential-shaped substrings replaced."""
|
||||
if not text:
|
||||
return text
|
||||
for pattern in _PATTERNS:
|
||||
text = pattern.sub(REDACTED, text)
|
||||
text = _ASSIGNMENT_RE.sub(_redact_assignment, text)
|
||||
return _ENTROPY_RE.sub(_redact_entropy, text)
|
||||
|
||||
|
||||
def bound(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]:
|
||||
"""Truncate ``text`` to a UTF-8 byte budget, reporting whether it was cut."""
|
||||
if max_bytes <= 0:
|
||||
return ("", bool(text))
|
||||
encoded = text.encode("utf-8", "replace")
|
||||
if len(encoded) <= max_bytes:
|
||||
return (text, False)
|
||||
marker = TRUNCATION_NOTE.encode()[:max_bytes]
|
||||
content_budget = max(
|
||||
0, max_bytes - len(marker) - (1 if len(marker) < max_bytes else 0)
|
||||
)
|
||||
kept = encoded[:content_budget].decode("utf-8", "ignore")
|
||||
separator = "\n" if kept and len(marker) < max_bytes else ""
|
||||
return (f"{kept}{separator}{marker.decode()}", True)
|
||||
|
||||
|
||||
def safe_text(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]:
|
||||
"""Redact then bound, so truncation never exposes a credential prefix."""
|
||||
return bound(redact(text), max_bytes)
|
||||
|
||||
|
||||
def scrub(value: Any, max_bytes: int = DEFAULT_MAX_BYTES) -> Any:
|
||||
"""Recursively bound and redact a JSON-shaped value.
|
||||
|
||||
Mapping keys are themselves screened; values under a sensitive key are
|
||||
dropped outright rather than pattern-screened, because a short secret
|
||||
would survive screening.
|
||||
"""
|
||||
if isinstance(value, str):
|
||||
return safe_text(value, max_bytes)[0]
|
||||
if isinstance(value, dict):
|
||||
screened: dict[str, Any] = {}
|
||||
for key, item in value.items():
|
||||
raw_key = str(key)
|
||||
safe_key = safe_text(raw_key, max_bytes)[0]
|
||||
sensitive_value = (
|
||||
is_sensitive_key(raw_key)
|
||||
and item is not None
|
||||
and not isinstance(item, bool)
|
||||
and item != ""
|
||||
)
|
||||
screened[safe_key] = REDACTED if sensitive_value else scrub(item, max_bytes)
|
||||
return screened
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [scrub(item, max_bytes) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def contains_credential_shape(text: str) -> bool:
|
||||
"""Report whether screening would rewrite ``text``.
|
||||
|
||||
Used by the harness self-test to prove a rendered report is clean before it
|
||||
is written, and by the unit suite as an assertion helper.
|
||||
"""
|
||||
return redact(text) != text
|
||||
199
scripts/ops/hermes_handoff_rules.py
Normal file
199
scripts/ops/hermes_handoff_rules.py
Normal file
@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared primitives and dispatch for handoff acceptance evaluators.
|
||||
|
||||
Evaluators are registered by name so the check catalog stays declarative: an
|
||||
entry says *what* it asserts, not how to plumb it. The dispatch here is the one
|
||||
place that turns an unexpected exception into ``NOT_RUN`` — a harness bug must
|
||||
never be indistinguishable from a passing release.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from hermes_handoff_exec import Outcome
|
||||
from hermes_handoff_model import NOT_RUN, STATUSES, CheckSpec
|
||||
|
||||
DENIAL_MARKERS = (
|
||||
"forbidden",
|
||||
"is not allowed",
|
||||
"unauthorized",
|
||||
"permission denied",
|
||||
"cannot get",
|
||||
"cannot list",
|
||||
"cannot create",
|
||||
"cannot delete",
|
||||
"cannot patch",
|
||||
"not permitted",
|
||||
"gitea api returned http 403",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Evaluation:
|
||||
"""A classification plus the small facts that justify it."""
|
||||
|
||||
status: str
|
||||
reason: str
|
||||
evidence: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
Evaluator = Callable[[CheckSpec, dict[str, Outcome]], Evaluation]
|
||||
EVALUATORS: dict[str, Evaluator] = {}
|
||||
|
||||
|
||||
def evaluator(name: str) -> Callable[[Evaluator], Evaluator]:
|
||||
"""Register an evaluator under the rule name catalog entries use."""
|
||||
|
||||
def register(function: Evaluator) -> Evaluator:
|
||||
EVALUATORS[name] = function
|
||||
return function
|
||||
|
||||
return register
|
||||
|
||||
|
||||
def missing_steps(
|
||||
spec: CheckSpec, outcomes: Mapping[str, Outcome]
|
||||
) -> Evaluation | None:
|
||||
"""Return NOT_RUN when a required step never produced usable evidence."""
|
||||
for step in spec.steps:
|
||||
if step.optional:
|
||||
continue
|
||||
outcome = outcomes.get(step.key)
|
||||
if outcome is None:
|
||||
return Evaluation(NOT_RUN, f"step {step.key} was never executed")
|
||||
if outcome.truncated:
|
||||
return Evaluation(NOT_RUN, f"step {step.key} output was truncated")
|
||||
if not outcome.ran:
|
||||
return Evaluation(NOT_RUN, f"step {step.key} did not run: {outcome.error}")
|
||||
return None
|
||||
|
||||
|
||||
def looks_denied(outcome: Outcome) -> bool:
|
||||
"""Report whether a real request was refused by policy rather than failing."""
|
||||
if outcome.ok:
|
||||
return False
|
||||
haystack = outcome.combined.lower()
|
||||
return any(marker in haystack for marker in DENIAL_MARKERS)
|
||||
|
||||
|
||||
def step_keys(spec: CheckSpec, kind: str) -> tuple[str, ...]:
|
||||
"""Return the keys of every step of one kind."""
|
||||
return tuple(step.key for step in spec.steps if step.kind == kind)
|
||||
|
||||
|
||||
def is_list(value: Any) -> bool:
|
||||
"""Report whether a value is a JSON array rather than a string."""
|
||||
return isinstance(value, Sequence) and not isinstance(value, (str, bytes))
|
||||
|
||||
|
||||
def exact_equal(actual: Any, expected: Any) -> bool:
|
||||
"""Compare JSON values without Python's bool/int type confusion."""
|
||||
return type(actual) is type(expected) and actual == expected
|
||||
|
||||
|
||||
def dotted(payload: Any, path: str) -> Any:
|
||||
"""Return a value addressed by a dotted path.
|
||||
|
||||
``a.b[0].c`` indexes; ``a[].b`` maps the remainder of the path over a list,
|
||||
which is how the catalog records one field from every pull request or every
|
||||
Flux object without a bespoke parser per check.
|
||||
"""
|
||||
current = payload
|
||||
segments = [segment for segment in path.split(".") if segment]
|
||||
for position, raw in enumerate(segments):
|
||||
name, bracket, indexes = raw.partition("[")
|
||||
if name:
|
||||
if not isinstance(current, Mapping) or name not in current:
|
||||
raise KeyError(path)
|
||||
current = current[name]
|
||||
if not bracket:
|
||||
continue
|
||||
for chunk in indexes.rstrip("]").split("]["):
|
||||
if not is_list(current):
|
||||
raise KeyError(path)
|
||||
if chunk == "":
|
||||
remainder = ".".join(segments[position + 1 :])
|
||||
return [
|
||||
dotted(item, remainder) if remainder else item for item in current
|
||||
]
|
||||
offset = int(chunk)
|
||||
if offset >= len(current):
|
||||
raise KeyError(path)
|
||||
current = current[offset]
|
||||
return current
|
||||
|
||||
|
||||
def strict_json(text: str) -> Any:
|
||||
"""Decode strict JSON, rejecting constants and duplicate object keys."""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
raise ValueError("empty output")
|
||||
|
||||
def reject_constant(value: str) -> None:
|
||||
raise ValueError(f"non-finite JSON constant {value}")
|
||||
|
||||
def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError(f"duplicate JSON key {key!r}")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
return json.loads(
|
||||
text,
|
||||
strict=True,
|
||||
parse_constant=reject_constant,
|
||||
object_pairs_hook=unique_object,
|
||||
)
|
||||
|
||||
|
||||
def parse_json(outcome: Outcome) -> Any:
|
||||
"""Parse one outcome as strict JSON."""
|
||||
return strict_json(outcome.stdout)
|
||||
|
||||
|
||||
def lines(outcome: Outcome) -> list[str]:
|
||||
"""Return non-empty, stripped stdout lines."""
|
||||
return [line.strip() for line in outcome.stdout.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def parsed_step(
|
||||
spec: CheckSpec, outcomes: dict[str, Outcome]
|
||||
) -> tuple[Any, str] | Evaluation:
|
||||
"""Return the parsed payload of ``expect['step']`` or the NOT_RUN to report."""
|
||||
key = spec.expect["step"]
|
||||
outcome = outcomes[key]
|
||||
if not outcome.ok:
|
||||
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
|
||||
try:
|
||||
return (parse_json(outcome), key)
|
||||
except ValueError as exc:
|
||||
return Evaluation(NOT_RUN, f"step {key} did not return JSON: {exc}")
|
||||
|
||||
|
||||
def evaluate(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
|
||||
"""Dispatch to the named evaluator, converting any surprise into NOT_RUN."""
|
||||
rule = EVALUATORS.get(spec.rule)
|
||||
if rule is None:
|
||||
return Evaluation(NOT_RUN, f"unknown evaluator: {spec.rule}")
|
||||
truncated = sorted(key for key, outcome in outcomes.items() if outcome.truncated)
|
||||
if truncated:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"truncated evidence from step(s): {', '.join(truncated)}"
|
||||
)
|
||||
try:
|
||||
result = rule(spec, outcomes)
|
||||
except Exception as exc: # noqa: BLE001 - a harness bug must never read as PASS
|
||||
return Evaluation(
|
||||
NOT_RUN, f"evaluator {spec.rule} raised {type(exc).__name__}: {exc}"
|
||||
)
|
||||
if result.status not in STATUSES:
|
||||
return Evaluation(
|
||||
NOT_RUN, f"evaluator {spec.rule} returned unknown status {result.status!r}"
|
||||
)
|
||||
return result
|
||||
484
scripts/ops/hermes_handoff_run.py
Normal file
484
scripts/ops/hermes_handoff_run.py
Normal file
@ -0,0 +1,484 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Orchestration for the Hermes full-handoff acceptance harness.
|
||||
|
||||
Three things happen before any check runs, and each of them can stop the run on
|
||||
its own:
|
||||
|
||||
* the catalog is validated structurally — every rule exists, every live-denial
|
||||
evaluator carries a read-only attempt, no operator-side step can impersonate;
|
||||
* each vantage is resolved and asked who it is; and
|
||||
* the operator and in-pod identities are compared. If they are the same
|
||||
principal there is only one vantage, the dual-vantage evidence would be a
|
||||
restatement rather than a corroboration, and the run is a NO_GO.
|
||||
|
||||
Everything after that is uniform: run a check's steps against its vantage, hand
|
||||
the outcomes to the named evaluator, and record the classification. A vantage
|
||||
that could not be resolved yields ``NOT_RUN`` for every check that needs it,
|
||||
which is exactly the fail-closed answer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
|
||||
from hermes_handoff_catalog import (
|
||||
CHAT,
|
||||
NODE,
|
||||
OPERATOR,
|
||||
SELF,
|
||||
SWITCHYARD,
|
||||
VANTAGE_NAMES,
|
||||
Targets,
|
||||
)
|
||||
from hermes_handoff_checks_access import access_checks
|
||||
from hermes_handoff_checks_delivery import delivery_checks
|
||||
from hermes_handoff_checks_platform import platform_checks
|
||||
from hermes_handoff_checks_workers import worker_checks
|
||||
from hermes_handoff_evaluators import EVALUATORS, evaluate
|
||||
from hermes_handoff_exec import Outcome, Runner, Vantage, operator_vantage, pod_vantage
|
||||
from hermes_handoff_model import (
|
||||
ATTEMPT,
|
||||
EPHEMERAL,
|
||||
NOT_RUN,
|
||||
STEP_KINDS,
|
||||
CheckResult,
|
||||
CheckSpec,
|
||||
Report,
|
||||
VantageRecord,
|
||||
utc_now,
|
||||
)
|
||||
from hermes_handoff_policy import (
|
||||
EXPECTED_REMOTE,
|
||||
EXPECTED_REPO,
|
||||
IMPERSONATION_ARGS,
|
||||
PolicyError,
|
||||
check_argv,
|
||||
projection,
|
||||
)
|
||||
|
||||
UNRECORDED = "[output not recorded]"
|
||||
|
||||
|
||||
def build_catalog(targets: Targets) -> list[CheckSpec]:
|
||||
"""Return the whole acceptance catalog for a set of targets."""
|
||||
return [
|
||||
*platform_checks(targets),
|
||||
*access_checks(targets),
|
||||
*delivery_checks(targets),
|
||||
*worker_checks(targets),
|
||||
]
|
||||
|
||||
|
||||
def validate_catalog(specs: Sequence[CheckSpec]) -> list[str]:
|
||||
"""Return the structural problems that make a catalog unsafe to run."""
|
||||
problems: list[str] = []
|
||||
if not specs:
|
||||
return ["catalog is empty"]
|
||||
seen: set[str] = set()
|
||||
fake_operator = operator_vantage()
|
||||
fake_pod = pod_vantage("hermes", "acceptance-probe", "probe", fake_operator)
|
||||
for spec in specs:
|
||||
if spec.id in seen:
|
||||
problems.append(f"{spec.id}: duplicate check id")
|
||||
seen.add(spec.id)
|
||||
if spec.rule not in EVALUATORS and spec.scope != EPHEMERAL:
|
||||
problems.append(f"{spec.id}: unknown rule {spec.rule!r}")
|
||||
step_names: set[str] = set()
|
||||
for step in spec.steps:
|
||||
if step.key in step_names:
|
||||
problems.append(f"{spec.id}: duplicate step key {step.key!r}")
|
||||
step_names.add(step.key)
|
||||
if step.vantage not in VANTAGE_NAMES:
|
||||
problems.append(
|
||||
f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}"
|
||||
)
|
||||
if step.kind not in STEP_KINDS:
|
||||
problems.append(
|
||||
f"{spec.id}: step {step.key} has unknown kind {step.kind!r}"
|
||||
)
|
||||
if set(step.argv) & set(IMPERSONATION_ARGS):
|
||||
problems.append(
|
||||
f"{spec.id}: step {step.key} impersonates from the "
|
||||
f"{step.vantage} vantage"
|
||||
)
|
||||
try:
|
||||
addressed = (
|
||||
fake_operator if step.vantage == OPERATOR else fake_pod
|
||||
).wrap(step.argv)
|
||||
check_argv(addressed)
|
||||
except PolicyError as exc:
|
||||
problems.append(f"{spec.id}: step {step.key} violates policy: {exc}")
|
||||
if spec.rule == "denied" and not any(
|
||||
step.kind == ATTEMPT for step in spec.steps
|
||||
):
|
||||
problems.append(
|
||||
f"{spec.id}: a deny check needs a real attempt, not only a review"
|
||||
)
|
||||
return problems
|
||||
|
||||
|
||||
def validate_targets(targets: Targets) -> list[str]:
|
||||
"""Return release-input problems before a runner or network call exists."""
|
||||
problems: list[str] = []
|
||||
sha = re.compile(r"[0-9a-f]{40}\Z")
|
||||
image = re.compile(
|
||||
r"[^@\s]+:git-([0-9a-f]{40})-build-[1-9][0-9]*@sha256:[0-9a-f]{64}\Z"
|
||||
)
|
||||
if targets.repo != EXPECTED_REPO or targets.remote != EXPECTED_REMOTE:
|
||||
problems.append(
|
||||
"repository and remote must be the fixed Atlas titan-iac origin"
|
||||
)
|
||||
if (
|
||||
targets.reviewed_pr_number != 19
|
||||
or targets.reviewed_head_ref != "feature/hermes-full-handoff-acceptance"
|
||||
):
|
||||
problems.append(
|
||||
"reviewed PR identity must remain fixed to draft PR #19 and its existing branch"
|
||||
)
|
||||
for label, value in (
|
||||
("baseline commit", targets.baseline_commit),
|
||||
("remote main SHA", targets.remote_main_sha),
|
||||
("reviewed head SHA", targets.reviewed_head_sha),
|
||||
("build SHA", targets.build_sha),
|
||||
):
|
||||
if not sha.fullmatch(value):
|
||||
problems.append(f"{label} must be an exact lowercase 40-character SHA")
|
||||
image_match = image.fullmatch(targets.agent_image)
|
||||
if not image_match or image_match.group(1) != targets.build_sha:
|
||||
problems.append("agent image tag/digest must bind the exact build SHA")
|
||||
if targets.build_sha != targets.remote_main_sha:
|
||||
problems.append("build SHA must equal the exact release main SHA")
|
||||
if not re.fullmatch(r"[1-9][0-9]{0,8}", str(targets.deployment_revision)):
|
||||
problems.append("deployment revision must be a positive integer string")
|
||||
if targets.node_count != 3 or targets.pool_replicas != 3:
|
||||
problems.append("node and pool counts are fixed at exactly three")
|
||||
if not targets.chat_config_revision.strip():
|
||||
problems.append(
|
||||
"Telegram continuity and its chat config revision are mandatory"
|
||||
)
|
||||
if set(targets.pool_worker_env) != {"HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL"}:
|
||||
problems.append("pool assignment evidence must include worker node and ordinal")
|
||||
heads = dict(targets.dependency_heads)
|
||||
if len(heads) != len(targets.dependency_heads) or set(heads) != set(
|
||||
targets.dependency_pull_requests
|
||||
):
|
||||
problems.append("every dependency PR must have one exact current head")
|
||||
elif any(not sha.fullmatch(value) for value in heads.values()):
|
||||
problems.append("dependency heads must be exact lowercase 40-character SHAs")
|
||||
if (
|
||||
not isinstance(targets.max_evidence_age_seconds, int)
|
||||
or targets.max_evidence_age_seconds <= 0
|
||||
):
|
||||
problems.append("maximum evidence age must be a positive integer")
|
||||
return problems
|
||||
|
||||
|
||||
def resolve_identity(runner: Runner, vantage: Vantage) -> tuple[str, str]:
|
||||
"""Return the authenticated username of a vantage and any failure detail."""
|
||||
outcome = runner.run(
|
||||
(
|
||||
"kubectl",
|
||||
"auth",
|
||||
"whoami",
|
||||
"-o",
|
||||
projection("jsonpath={.status.userInfo.username}"),
|
||||
),
|
||||
vantage,
|
||||
)
|
||||
if not outcome.ok:
|
||||
return ("", outcome.error or outcome.combined[:200])
|
||||
username = outcome.stdout.strip()
|
||||
if not username or any(character.isspace() for character in username):
|
||||
return ("", "identity response carried no username")
|
||||
return (username, "")
|
||||
|
||||
|
||||
def _first_pod(
|
||||
runner: Runner, operator: Vantage, namespace: str, selector: str
|
||||
) -> tuple[str, str]:
|
||||
outcome = runner.run(
|
||||
(
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
namespace,
|
||||
"get",
|
||||
"pods",
|
||||
"--selector",
|
||||
selector,
|
||||
"--field-selector",
|
||||
"status.phase=Running",
|
||||
"-o",
|
||||
"name",
|
||||
),
|
||||
operator,
|
||||
)
|
||||
if not outcome.ok:
|
||||
return ("", outcome.error or outcome.combined[:200])
|
||||
names = [
|
||||
line.strip().removeprefix("pod/")
|
||||
for line in outcome.stdout.splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
if not names:
|
||||
return ("", f"no running pod matches {selector}")
|
||||
return (sorted(names)[0], "")
|
||||
|
||||
|
||||
def resolve_vantages(
|
||||
runner: Runner,
|
||||
targets: Targets,
|
||||
kubeconfig: str | None = None,
|
||||
context: str | None = None,
|
||||
) -> tuple[dict[str, Vantage], list[VantageRecord]]:
|
||||
"""Resolve every vantage the catalog names, recording what each one is."""
|
||||
operator = operator_vantage(kubeconfig, context)
|
||||
username, detail = resolve_identity(runner, operator)
|
||||
records = [
|
||||
VantageRecord(
|
||||
name=OPERATOR,
|
||||
description=operator.description or "external read-only operator",
|
||||
identity=username,
|
||||
available=bool(username),
|
||||
detail=detail,
|
||||
)
|
||||
]
|
||||
vantages: dict[str, Vantage] = {OPERATOR: operator}
|
||||
if not username:
|
||||
return (vantages, records)
|
||||
|
||||
for name, selector, container, description in (
|
||||
(
|
||||
SELF,
|
||||
f"app={targets.agent_deployment}",
|
||||
targets.agent_container,
|
||||
"the Hermes agent itself",
|
||||
),
|
||||
(
|
||||
SWITCHYARD,
|
||||
f"app={targets.switchyard_deployment}",
|
||||
targets.switchyard_container,
|
||||
"the Switchyard routing evidence log",
|
||||
),
|
||||
(
|
||||
NODE,
|
||||
f"app={targets.node_daemonset}",
|
||||
targets.node_container,
|
||||
"a node hardening probe",
|
||||
),
|
||||
):
|
||||
pod, failure = _first_pod(runner, operator, targets.namespace, selector)
|
||||
if not pod:
|
||||
records.append(
|
||||
VantageRecord(
|
||||
name=name, description=description, available=False, detail=failure
|
||||
)
|
||||
)
|
||||
continue
|
||||
vantage = pod_vantage(targets.namespace, pod, container, operator)
|
||||
identity, identity_detail = (
|
||||
resolve_identity(runner, vantage) if name == SELF else ("", "")
|
||||
)
|
||||
records.append(
|
||||
VantageRecord(
|
||||
name=name,
|
||||
description=f"{description} ({targets.namespace}/{pod})",
|
||||
identity=identity,
|
||||
available=True,
|
||||
detail=identity_detail,
|
||||
)
|
||||
)
|
||||
vantages[name] = vantage
|
||||
|
||||
chat_pod = f"{targets.chat_statefulset}-{targets.chat_ordinal}"
|
||||
present = runner.run(
|
||||
(
|
||||
"kubectl",
|
||||
"--namespace",
|
||||
targets.namespace,
|
||||
"get",
|
||||
f"pod/{chat_pod}",
|
||||
"-o",
|
||||
"name",
|
||||
),
|
||||
operator,
|
||||
)
|
||||
if present.ok and present.stdout.strip():
|
||||
vantages[CHAT] = pod_vantage(
|
||||
targets.namespace, chat_pod, targets.chat_container, operator
|
||||
)
|
||||
records.append(
|
||||
VantageRecord(
|
||||
name=CHAT,
|
||||
description=f"the chat surface ({targets.namespace}/{chat_pod})",
|
||||
available=True,
|
||||
)
|
||||
)
|
||||
else:
|
||||
records.append(
|
||||
VantageRecord(
|
||||
name=CHAT,
|
||||
description=f"the chat surface ({targets.namespace}/{chat_pod})",
|
||||
available=False,
|
||||
detail=present.error
|
||||
or present.combined[:200]
|
||||
or "chat tenant pod not found",
|
||||
)
|
||||
)
|
||||
return (vantages, records)
|
||||
|
||||
|
||||
def vantage_problems(records: Iterable[VantageRecord]) -> list[str]:
|
||||
"""Return the identity problems that invalidate dual-vantage evidence."""
|
||||
by_name = {record.name: record for record in records}
|
||||
operator = by_name.get(OPERATOR)
|
||||
hermes = by_name.get(SELF)
|
||||
problems: list[str] = []
|
||||
if operator is None or not operator.available:
|
||||
problems.append(
|
||||
"the operator vantage is unavailable; no evidence can be collected"
|
||||
)
|
||||
return problems
|
||||
if hermes is None or not hermes.available:
|
||||
problems.append(
|
||||
"the in-pod Hermes vantage is unavailable; self-probe evidence is missing"
|
||||
)
|
||||
return problems
|
||||
if not hermes.identity:
|
||||
problems.append("the in-pod vantage did not report an identity")
|
||||
elif operator.identity == hermes.identity:
|
||||
problems.append(
|
||||
"the operator and in-pod vantages authenticate as the same principal "
|
||||
f"({operator.identity}); run the harness with a separate operator kubeconfig"
|
||||
)
|
||||
for name, label in ((SWITCHYARD, "Switchyard"), (CHAT, "chat/Telegram")):
|
||||
record = by_name.get(name)
|
||||
if record is None or not record.available:
|
||||
problems.append(f"the required {label} vantage is unavailable")
|
||||
return problems
|
||||
|
||||
|
||||
def run_check(
|
||||
runner: Runner, spec: CheckSpec, vantages: Mapping[str, Vantage]
|
||||
) -> CheckResult:
|
||||
"""Run one check's steps and classify the result."""
|
||||
outcomes: dict[str, Outcome] = {}
|
||||
recorded: list[Outcome] = []
|
||||
for step in spec.steps:
|
||||
vantage = vantages.get(step.vantage)
|
||||
if vantage is None:
|
||||
outcome = Outcome(
|
||||
argv=step.argv,
|
||||
vantage=step.vantage,
|
||||
error=f"vantage {step.vantage} is unavailable",
|
||||
)
|
||||
else:
|
||||
outcome = runner.run(step.argv, vantage, step.max_bytes)
|
||||
outcomes[step.key] = outcome
|
||||
recorded.append(
|
||||
outcome
|
||||
if step.record
|
||||
else Outcome(
|
||||
argv=outcome.argv,
|
||||
vantage=outcome.vantage,
|
||||
returncode=outcome.returncode,
|
||||
stdout=UNRECORDED if outcome.stdout else "",
|
||||
stderr=UNRECORDED if outcome.stderr else "",
|
||||
truncated=outcome.truncated,
|
||||
duration_ms=outcome.duration_ms,
|
||||
error=outcome.error,
|
||||
executable_path=outcome.executable_path,
|
||||
executable_sha256=outcome.executable_sha256,
|
||||
)
|
||||
)
|
||||
evaluation = evaluate(spec, outcomes)
|
||||
return CheckResult(
|
||||
spec=spec,
|
||||
status=evaluation.status,
|
||||
reason=evaluation.reason,
|
||||
evidence=evaluation.evidence,
|
||||
outcomes=recorded,
|
||||
)
|
||||
|
||||
|
||||
def run_catalog(
|
||||
runner: Runner, specs: Sequence[CheckSpec], vantages: Mapping[str, Vantage]
|
||||
) -> list[CheckResult]:
|
||||
"""Run every check in the catalog.
|
||||
|
||||
Nothing is skipped once the deadline passes: the runner returns a
|
||||
deadline outcome for each remaining command, which classifies as
|
||||
``NOT_RUN``, so a truncated sweep is visible in the report rather than
|
||||
silently shorter.
|
||||
"""
|
||||
return [run_check(runner, spec, vantages) for spec in specs]
|
||||
|
||||
|
||||
def build_report(
|
||||
runner: Runner,
|
||||
targets: Targets,
|
||||
specs: Sequence[CheckSpec],
|
||||
vantages: Mapping[str, Vantage],
|
||||
records: Sequence[VantageRecord],
|
||||
mode: str,
|
||||
started_at: str,
|
||||
extra_results: Sequence[CheckResult] = (),
|
||||
) -> Report:
|
||||
"""Run the catalog and assemble the report, including structural problems."""
|
||||
problems = validate_catalog(specs) + vantage_problems(records)
|
||||
report = Report(
|
||||
mode=mode,
|
||||
started_at=started_at,
|
||||
baseline={
|
||||
"namespace": targets.namespace,
|
||||
"baseline_commit": targets.baseline_commit,
|
||||
"dependency_pull_requests": list(targets.dependency_pull_requests),
|
||||
"expected_suspensions": list(targets.expected_suspensions),
|
||||
"node_count": targets.node_count,
|
||||
"pool_replicas": targets.pool_replicas,
|
||||
},
|
||||
vantages=list(records),
|
||||
harness_errors=problems,
|
||||
)
|
||||
armed_ids = {result.spec.id for result in extra_results}
|
||||
runnable = [spec for spec in specs if spec.id not in armed_ids]
|
||||
if problems:
|
||||
report.results = [
|
||||
CheckResult(spec=spec, status=NOT_RUN, reason="harness preflight failed")
|
||||
for spec in runnable
|
||||
]
|
||||
report.results.extend(extra_results)
|
||||
else:
|
||||
report.results = [*run_catalog(runner, runnable, vantages), *extra_results]
|
||||
report.results.sort(key=lambda result: result.spec.id)
|
||||
if runner.remaining_seconds <= 0:
|
||||
report.harness_errors.append(
|
||||
"the run deadline expired before every check completed"
|
||||
)
|
||||
report.finished_at = utc_now()
|
||||
return report
|
||||
|
||||
|
||||
def unavailable_report(
|
||||
mode: str, started_at: str, reason: str, targets: Targets
|
||||
) -> Report:
|
||||
"""Return a NO_GO report for a run that could not start at all."""
|
||||
return Report(
|
||||
mode=mode,
|
||||
started_at=started_at,
|
||||
finished_at=utc_now(),
|
||||
baseline={"namespace": targets.namespace},
|
||||
harness_errors=[reason],
|
||||
results=[
|
||||
CheckResult(
|
||||
spec=CheckSpec(
|
||||
id="harness.startup",
|
||||
title="The harness could start and resolve its vantages",
|
||||
group="harness",
|
||||
rule="allowed",
|
||||
),
|
||||
status=NOT_RUN,
|
||||
reason=reason,
|
||||
)
|
||||
],
|
||||
)
|
||||
@ -25,7 +25,7 @@ spec:
|
||||
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
||||
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
||||
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
||||
ai.bstein.dev/config-rev: "20260817-cli-auto-quota-routing-v1"
|
||||
ai.bstein.dev/config-rev: "20260818-ai-usage-exporter-split-v3"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "9010"
|
||||
|
||||
@ -204,6 +204,9 @@ spec:
|
||||
- |
|
||||
/opt/hermes/.venv/bin/python /opt/coordinator/patch_api_server_sessions.py \
|
||||
/opt/hermes/gateway/platforms/api_server.py /patched/api_server.py
|
||||
grep -Fq 'conversation_history = compact_telegram_history(conversation_history)' /patched/api_server.py
|
||||
grep -Fq 'full_history = compact_telegram_history(full_history)' /patched/api_server.py
|
||||
grep -Fq 'conversation_history_snapshot = compact_telegram_history(' /patched/api_server.py
|
||||
/opt/hermes/.venv/bin/python /opt/coordinator/migrate_telegram_api_sessions.py \
|
||||
/opt/data/state.db /opt/data/response_store.db
|
||||
securityContext:
|
||||
@ -284,6 +287,7 @@ spec:
|
||||
- {name: runtime-access, mountPath: /runtime-access}
|
||||
- {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
|
||||
- {name: api-server-patch, mountPath: /opt/hermes/gateway/platforms/api_server.py, subPath: api_server.py}
|
||||
- {name: coordinator, mountPath: /opt/hermes/gateway/platforms/telegram_continuity.py, subPath: migrate_telegram_api_sessions.py, readOnly: true}
|
||||
- {name: stream-recovery-patch, mountPath: /opt/hermes/agent/conversation_loop.py, subPath: conversation_loop.py}
|
||||
- {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/environments/local.py, subPath: local.py}
|
||||
- {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py}
|
||||
|
||||
@ -97,7 +97,9 @@ configMapGenerator:
|
||||
- name: hermes-coordinator
|
||||
namespace: hermes
|
||||
files:
|
||||
- ai_usage_codex.py=scripts/ai_usage_codex.py
|
||||
- ai_usage_exporter.py=scripts/ai_usage_exporter.py
|
||||
- ai_usage_http.py=scripts/ai_usage_http.py
|
||||
- claude=scripts/claude
|
||||
- claude_command_policy.py=scripts/claude_command_policy.py
|
||||
- cli_lane_board.py=scripts/cli_lane_board.py
|
||||
|
||||
@ -27,8 +27,9 @@ type linkRecord struct {
|
||||
}
|
||||
|
||||
type telegramTopic struct {
|
||||
Label string `json:"label"`
|
||||
LastUsed int64 `json:"last_used"`
|
||||
Label string `json:"label"`
|
||||
LastUsed int64 `json:"last_used"`
|
||||
Generation int64 `json:"generation,omitempty"`
|
||||
}
|
||||
|
||||
type telegramTopicState struct {
|
||||
@ -372,7 +373,12 @@ func (router *tenantRouter) ServeHTTP(writer http.ResponseWriter, request *http.
|
||||
return
|
||||
}
|
||||
proxy := httputil.NewSingleHostReverseProxy(target)
|
||||
proxy.ModifyResponse = injectChatBridge
|
||||
proxy.ModifyResponse = func(response *http.Response) error {
|
||||
if err := boundSessionSnapshot(response); err != nil {
|
||||
return err
|
||||
}
|
||||
return injectChatBridge(response)
|
||||
}
|
||||
proxy.ErrorHandler = func(writer http.ResponseWriter, _ *http.Request, proxyErr error) {
|
||||
log.Printf("tenant slot %d unavailable", slot)
|
||||
http.Error(writer, "private chat runtime is starting", http.StatusBadGateway)
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@ -126,6 +127,9 @@ func TestRouterProxiesWebUIAndAddsTelegramShortcut(t *testing.T) {
|
||||
if !strings.Contains(response.Body.String(), "hermes-chat-bridge.js") || !strings.Contains(response.Body.String(), "hermes-chat-bridge.css") {
|
||||
t.Fatal("Telegram shortcut assets were not injected")
|
||||
}
|
||||
if !strings.Contains(response.Body.String(), "hermes-session-continuity.js") {
|
||||
t.Fatal("session continuity fallback was not injected")
|
||||
}
|
||||
assetRequest := httptest.NewRequest(http.MethodGet, "/hermes-chat-bridge.js", nil)
|
||||
assetRequest.Header.Set("X-Forwarded-User", "subject")
|
||||
assetResponse := httptest.NewRecorder()
|
||||
@ -145,6 +149,75 @@ func TestRouterProxiesWebUIAndAddsTelegramShortcut(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionContinuityAssetHasAccessiblePollFallback(t *testing.T) {
|
||||
router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(int) string { return "" })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request := httptest.NewRequest(http.MethodGet, "/hermes-session-continuity.js", nil)
|
||||
request.Header.Set("X-Forwarded-User", "subject")
|
||||
response := httptest.NewRecorder()
|
||||
router.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("got status %d", response.Code)
|
||||
}
|
||||
asset := response.Body.String()
|
||||
for _, expected := range []string{
|
||||
"aria-live", "aria-busy", "/api/sessions/", "/messages?limit=24&hermes_fallback=1",
|
||||
"Latest stored activity", "no renderable messages",
|
||||
"Session updates disconnected", "oauth2/start?rd=", "payload.session_id",
|
||||
"session.ended_at == null", "latest.observed", "addEventListener('online'",
|
||||
"addEventListener('pageshow'", "schedule(document.hidden", "fetch(",
|
||||
} {
|
||||
if !strings.Contains(asset, expected) {
|
||||
t.Fatalf("session fallback omitted %q", expected)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{
|
||||
"location.replace", "location.reload", "history.replaceState", "/api/session?", "WebSocket",
|
||||
} {
|
||||
if strings.Contains(asset, forbidden) {
|
||||
t.Fatalf("session fallback interferes with native continuity via %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionContinuityPollIsBoundedByRouter(t *testing.T) {
|
||||
backend := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
messages := make([]map[string]int, 30)
|
||||
for index := range messages {
|
||||
messages[index] = map[string]int{"index": index}
|
||||
}
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(writer).Encode(map[string]any{
|
||||
"session_id": "resolved", "messages": messages,
|
||||
})
|
||||
}))
|
||||
defer backend.Close()
|
||||
router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(int) string { return backend.URL })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request := httptest.NewRequest(
|
||||
http.MethodGet,
|
||||
"/api/sessions/root/messages?limit=24&hermes_fallback=1",
|
||||
nil,
|
||||
)
|
||||
request.Header.Set("X-Forwarded-User", "subject")
|
||||
response := httptest.NewRecorder()
|
||||
router.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("got status %d", response.Code)
|
||||
}
|
||||
var payload sessionSnapshot
|
||||
if err := json.NewDecoder(response.Body).Decode(&payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(payload.Messages) != sessionSnapshotItems || payload.TotalMessages != 30 {
|
||||
t.Fatalf("router returned unbounded fallback: %#v", payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelegramPageExplainsOneTimeOperatorSetup(t *testing.T) {
|
||||
router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(slot int) string { return "" })
|
||||
if err != nil {
|
||||
@ -294,11 +367,52 @@ func TestPrivateFileBrowserIsTenantAuthenticatedAndHidesKanban(t *testing.T) {
|
||||
assetRequest.Header.Set("X-Forwarded-User", "subject")
|
||||
assetResponse := httptest.NewRecorder()
|
||||
router.ServeHTTP(assetResponse, assetRequest)
|
||||
if !strings.Contains(assetResponse.Body.String(), "endpoint('list'") || !strings.Contains(assetResponse.Body.String(), `'[data-panel="kanban"]'`) {
|
||||
if !strings.Contains(assetResponse.Body.String(), "endpoint('list'") || !strings.Contains(assetResponse.Body.String(), `'[data-panel="' + panel + '"]'`) {
|
||||
t.Fatal("file browsing or chat-only navigation policy is missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatBridgeHidesOnlyNonChatCockpitChrome(t *testing.T) {
|
||||
router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(slot int) string { return "" })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asset := func(path string) string {
|
||||
request := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
request.Header.Set("X-Forwarded-User", "subject")
|
||||
response := httptest.NewRecorder()
|
||||
router.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("%s got status %d", path, response.Code)
|
||||
}
|
||||
return response.Body.String()
|
||||
}
|
||||
js := asset("/hermes-chat-bridge.js")
|
||||
css := asset("/hermes-chat-bridge.css")
|
||||
|
||||
// The whole cockpit (Kanban plus every non-chat rail panel) is declutter-hidden
|
||||
// both pre-JS (CSS, no flash) and by the runtime .hidden reinforcement.
|
||||
for _, panel := range []string{"kanban", "logs", "insights", "memory", "skills", "workspaces", "todos", "tasks", "profiles"} {
|
||||
if !strings.Contains(css, `[data-panel="`+panel+`"]`) {
|
||||
t.Fatalf("bridge CSS does not hide cockpit panel %q", panel)
|
||||
}
|
||||
if !strings.Contains(js, `'`+panel+`'`) {
|
||||
t.Fatalf("bridge JS does not mark cockpit panel %q hidden", panel)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(css, "display:none") || !strings.Contains(css, "#settingsModel") {
|
||||
t.Fatal("bridge CSS must hide the cockpit panels and the redundant settings model selector")
|
||||
}
|
||||
|
||||
// The clean chat experience (chat surface, history, composer, model chip)
|
||||
// must never be part of the hide set.
|
||||
for _, keep := range []string{`[data-panel="chat"]`, `[data-panel="history"]`, `[data-panel="sessions"]`, `[data-panel="composer"]`, "composerRoutingChip", "composerRoutingLabel"} {
|
||||
if strings.Contains(css, keep) || strings.Contains(js, keep) {
|
||||
t.Fatalf("declutter wrongly targets a core chat control: %q", keep)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebUIModelAndReasoningOverridesAreProxied(t *testing.T) {
|
||||
backend := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.Method != http.MethodPost {
|
||||
|
||||
97
services/hermes/router/session_continuity.go
Normal file
97
services/hermes/router/session_continuity.go
Normal file
@ -0,0 +1,97 @@
|
||||
package main
|
||||
|
||||
const sessionContinuityJS = `(() => {
|
||||
const match = location.pathname.match(/^\/session\/([^/]+)\/?$/);
|
||||
if (!match) return;
|
||||
let sessionId;
|
||||
try { sessionId = decodeURIComponent(match[1]); } catch (_) { sessionId = match[1]; }
|
||||
const started = Date.now();
|
||||
let timer = 0;
|
||||
let request = null;
|
||||
const card = document.createElement('aside');
|
||||
card.id = 'hermes-session-continuity';
|
||||
card.setAttribute('role', 'status');
|
||||
card.setAttribute('aria-live', 'polite');
|
||||
card.setAttribute('aria-atomic', 'true');
|
||||
card.hidden = true;
|
||||
card.style.cssText = 'position:fixed;right:16px;bottom:16px;z-index:1000;max-width:min(420px,calc(100vw - 32px));box-sizing:border-box;padding:10px 12px;border:1px solid #475569;border-radius:10px;background:#111827;color:#e5e7eb;font:13px/1.4 system-ui,sans-serif;box-shadow:0 8px 28px #0008';
|
||||
const message = document.createElement('span');
|
||||
const newChat = document.createElement('a');
|
||||
newChat.href = '/';
|
||||
newChat.textContent = ' Start a new chat.';
|
||||
newChat.style.color = '#7dd3fc';
|
||||
card.append(message, newChat);
|
||||
document.body.appendChild(card);
|
||||
|
||||
const show = (text, busy, link) => {
|
||||
message.textContent = text;
|
||||
card.hidden = false;
|
||||
card.setAttribute('aria-busy', busy ? 'true' : 'false');
|
||||
newChat.hidden = !link;
|
||||
};
|
||||
const hide = () => { card.hidden = true; card.setAttribute('aria-busy', 'false'); };
|
||||
const activityLabel = (messages) => {
|
||||
const latest = messages[messages.length - 1] || {};
|
||||
if (latest.tool_name) return 'tool ' + String(latest.tool_name).slice(0, 80);
|
||||
if (Array.isArray(latest.tool_calls) && latest.tool_calls.length) {
|
||||
const call = latest.tool_calls[latest.tool_calls.length - 1] || {};
|
||||
return 'tool ' + String((call.function || {}).name || 'activity').slice(0, 80);
|
||||
}
|
||||
if (latest.activity_event) return String(latest.activity_event).replace(/[._-]+/g, ' ').slice(0, 80);
|
||||
if (latest.observed && typeof latest.content === 'string') return latest.content.slice(0, 120);
|
||||
return latest.role === 'assistant' ? 'assistant update' : latest.role === 'user' ? 'request stored' : 'working';
|
||||
};
|
||||
const authOrMissing = (response) => {
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
const rd = location.pathname + location.search + location.hash;
|
||||
location.assign('/oauth2/start?rd=' + encodeURIComponent(rd));
|
||||
return 'auth';
|
||||
}
|
||||
if (response.status === 404) {
|
||||
show('This session is unavailable to this account.', false, true);
|
||||
schedule(10000);
|
||||
return 'missing';
|
||||
}
|
||||
return '';
|
||||
};
|
||||
const schedule = (delay) => {
|
||||
clearTimeout(timer);
|
||||
timer = window.setTimeout(poll, delay);
|
||||
};
|
||||
async function poll() {
|
||||
if (request) return;
|
||||
request = new AbortController();
|
||||
const timeout = window.setTimeout(() => request && request.abort(), 8000);
|
||||
try {
|
||||
const root = '/api/sessions/' + encodeURIComponent(sessionId);
|
||||
const messageResponse = await fetch(root + '/messages?limit=24&hermes_fallback=1', {cache:'no-store', credentials:'same-origin', signal:request.signal});
|
||||
if (authOrMissing(messageResponse)) return;
|
||||
if (!messageResponse.ok) throw new Error('session messages poll failed');
|
||||
const payload = await messageResponse.json();
|
||||
const messages = Array.isArray(payload.messages) ? payload.messages : [];
|
||||
const resolvedId = typeof payload.session_id === 'string' && payload.session_id ? payload.session_id : sessionId;
|
||||
const detailResponse = await fetch('/api/sessions/' + encodeURIComponent(resolvedId), {cache:'no-store', credentials:'same-origin', signal:request.signal});
|
||||
if (authOrMissing(detailResponse)) return;
|
||||
if (!detailResponse.ok) throw new Error('session detail poll failed');
|
||||
const session = await detailResponse.json();
|
||||
const lastActive = Number(session.last_active || session.started_at || 0);
|
||||
const durableWorker = session.source === 'api_server' && Boolean(session.parent_session_id);
|
||||
const recentlyInteractive = Number.isFinite(lastActive) && Date.now() / 1000 - lastActive < 300;
|
||||
const active = session.ended_at == null && messages.length > 0 && (durableWorker || recentlyInteractive);
|
||||
if (active) show('Hermes is working. Latest stored activity: ' + activityLabel(messages) + '.', true, false);
|
||||
else if (!messages.length && Date.now() - started >= 4000) show('This session has no renderable messages yet. It may be new or no longer available.', false, true);
|
||||
else hide();
|
||||
schedule(document.hidden ? 10000 : 2500);
|
||||
} catch (_) {
|
||||
show('Session updates disconnected. Retrying without changing this session…', true, false);
|
||||
schedule(document.hidden ? 10000 : 3000);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
request = null;
|
||||
}
|
||||
}
|
||||
addEventListener('online', () => schedule(0));
|
||||
addEventListener('pageshow', () => schedule(0));
|
||||
document.addEventListener('visibilitychange', () => schedule(0));
|
||||
schedule(1200);
|
||||
})();`
|
||||
65
services/hermes/router/session_snapshot.go
Normal file
65
services/hermes/router/session_snapshot.go
Normal file
@ -0,0 +1,65 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
sessionSnapshotItems = 24
|
||||
sessionSnapshotBytes = 8 << 20
|
||||
)
|
||||
|
||||
type sessionSnapshot struct {
|
||||
SessionID string `json:"session_id"`
|
||||
Messages []json.RawMessage `json:"messages"`
|
||||
TotalMessages int `json:"total_messages"`
|
||||
}
|
||||
|
||||
// boundSessionSnapshot caps only the continuity fallback response. Native
|
||||
// WebUI requests remain untouched, including when an older backend ignores
|
||||
// its optional `limit` query parameter.
|
||||
func boundSessionSnapshot(response *http.Response) error {
|
||||
request := response.Request
|
||||
if request == nil || response.StatusCode != http.StatusOK ||
|
||||
request.URL.Query().Get("hermes_fallback") != "1" ||
|
||||
!strings.HasPrefix(request.URL.Path, "/api/sessions/") ||
|
||||
!strings.HasSuffix(request.URL.Path, "/messages") {
|
||||
return nil
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, sessionSnapshotBytes+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = response.Body.Close()
|
||||
if len(body) > sessionSnapshotBytes {
|
||||
return errors.New("session snapshot exceeds safe response limit")
|
||||
}
|
||||
var payload sessionSnapshot
|
||||
if err := json.Unmarshal(body, &payload); err != nil || payload.Messages == nil {
|
||||
return errors.New("session snapshot is malformed")
|
||||
}
|
||||
total := len(payload.Messages)
|
||||
if payload.TotalMessages > total {
|
||||
total = payload.TotalMessages
|
||||
}
|
||||
if len(payload.Messages) > sessionSnapshotItems {
|
||||
payload.Messages = payload.Messages[len(payload.Messages)-sessionSnapshotItems:]
|
||||
}
|
||||
payload.TotalMessages = total
|
||||
body, err = json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response.Body = io.NopCloser(strings.NewReader(string(body)))
|
||||
response.ContentLength = int64(len(body))
|
||||
response.Header.Set("Content-Length", strconv.Itoa(len(body)))
|
||||
response.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
response.Header.Set("Cache-Control", "no-store")
|
||||
response.Header.Del("ETag")
|
||||
return nil
|
||||
}
|
||||
113
services/hermes/router/session_snapshot_test.go
Normal file
113
services/hermes/router/session_snapshot_test.go
Normal file
@ -0,0 +1,113 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
)
|
||||
|
||||
func snapshotResponse(target, body string) *http.Response {
|
||||
request, _ := http.NewRequest(http.MethodGet, target, nil)
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
Request: request,
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundSessionSnapshotKeepsOnlyRecentMessages(t *testing.T) {
|
||||
messages := make([]map[string]int, 30)
|
||||
for index := range messages {
|
||||
messages[index] = map[string]int{"index": index}
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"session_id": "resolved", "messages": messages,
|
||||
})
|
||||
response := snapshotResponse(
|
||||
"http://tenant/api/sessions/root/messages?limit=24&hermes_fallback=1",
|
||||
string(body),
|
||||
)
|
||||
response.Header.Set("ETag", "stale")
|
||||
if err := boundSessionSnapshot(response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var bounded sessionSnapshot
|
||||
if err := json.NewDecoder(response.Body).Decode(&bounded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(bounded.Messages) != 24 || bounded.TotalMessages != 30 {
|
||||
t.Fatalf("snapshot was not bounded: %#v", bounded)
|
||||
}
|
||||
var first map[string]int
|
||||
if err := json.Unmarshal(bounded.Messages[0], &first); err != nil || first["index"] != 6 {
|
||||
t.Fatalf("snapshot did not retain the recent tail: %#v, %v", first, err)
|
||||
}
|
||||
if response.Header.Get("Cache-Control") != "no-store" || response.Header.Get("ETag") != "" {
|
||||
t.Fatal("bounded snapshot retained cache metadata")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundSessionSnapshotPreservesLargerReportedTotal(t *testing.T) {
|
||||
response := snapshotResponse(
|
||||
"http://tenant/api/sessions/root/messages?hermes_fallback=1",
|
||||
`{"session_id":"leaf","messages":[],"total_messages":100}`,
|
||||
)
|
||||
if err := boundSessionSnapshot(response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var bounded sessionSnapshot
|
||||
if err := json.NewDecoder(response.Body).Decode(&bounded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bounded.SessionID != "leaf" || bounded.TotalMessages != 100 {
|
||||
t.Fatalf("reported total was lost: %#v", bounded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundSessionSnapshotFailsClosedOnMalformedOrOversizedBody(t *testing.T) {
|
||||
for name, body := range map[string]string{
|
||||
"malformed": `{`,
|
||||
"oversized": strings.Repeat("x", sessionSnapshotBytes+1),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
response := snapshotResponse(
|
||||
"http://tenant/api/sessions/root/messages?hermes_fallback=1",
|
||||
body,
|
||||
)
|
||||
if err := boundSessionSnapshot(response); err == nil {
|
||||
t.Fatal("unsafe snapshot was accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
response := snapshotResponse(
|
||||
"http://tenant/api/sessions/root/messages?hermes_fallback=1", `{}`,
|
||||
)
|
||||
response.Body = io.NopCloser(iotest.ErrReader(errors.New("read failed")))
|
||||
if err := boundSessionSnapshot(response); err == nil {
|
||||
t.Fatal("snapshot body read failure was ignored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundSessionSnapshotLeavesNativeAndErrorResponsesUntouched(t *testing.T) {
|
||||
for _, response := range []*http.Response{
|
||||
{StatusCode: http.StatusOK},
|
||||
snapshotResponse("http://tenant/api/sessions/root/messages", `{}`),
|
||||
snapshotResponse("http://tenant/api/sessions/root", `{}`),
|
||||
} {
|
||||
if err := boundSessionSnapshot(response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
errorResponse := snapshotResponse(
|
||||
"http://tenant/api/sessions/root/messages?hermes_fallback=1", `{}`,
|
||||
)
|
||||
errorResponse.StatusCode = http.StatusNotFound
|
||||
if err := boundSessionSnapshot(errorResponse); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@ -24,20 +24,21 @@ type telegramConfig struct {
|
||||
}
|
||||
|
||||
type telegramBot struct {
|
||||
config telegramConfig
|
||||
router *tenantRouter
|
||||
apiBase string
|
||||
fileBase string
|
||||
client *http.Client
|
||||
mediaClient *http.Client
|
||||
agentClient *http.Client
|
||||
mu sync.RWMutex
|
||||
botUsername string
|
||||
ready bool
|
||||
lastError string
|
||||
workLimit chan struct{}
|
||||
slotLocks map[int]*sync.Mutex
|
||||
slotLocksMux sync.Mutex
|
||||
config telegramConfig
|
||||
router *tenantRouter
|
||||
apiBase string
|
||||
fileBase string
|
||||
client *http.Client
|
||||
mediaClient *http.Client
|
||||
agentClient *http.Client
|
||||
mu sync.RWMutex
|
||||
botUsername string
|
||||
ready bool
|
||||
lastError string
|
||||
workLimit chan struct{}
|
||||
activityEvery time.Duration
|
||||
slotLocks map[int]*sync.Mutex
|
||||
slotLocksMux sync.Mutex
|
||||
}
|
||||
|
||||
type telegramUpdate struct {
|
||||
@ -96,15 +97,16 @@ func readTelegramConfig(path string) (telegramConfig, error) {
|
||||
|
||||
func newTelegramBot(config telegramConfig, router *tenantRouter) *telegramBot {
|
||||
return &telegramBot{
|
||||
config: config,
|
||||
router: router,
|
||||
apiBase: "https://api.telegram.org/bot" + config.BotToken,
|
||||
fileBase: "https://api.telegram.org/file/bot" + config.BotToken,
|
||||
client: &http.Client{Timeout: 70 * time.Second},
|
||||
mediaClient: newTenantMediaClient(),
|
||||
agentClient: &http.Client{Timeout: 15 * time.Minute},
|
||||
workLimit: make(chan struct{}, 4),
|
||||
slotLocks: map[int]*sync.Mutex{},
|
||||
config: config,
|
||||
router: router,
|
||||
apiBase: "https://api.telegram.org/bot" + config.BotToken,
|
||||
fileBase: "https://api.telegram.org/file/bot" + config.BotToken,
|
||||
client: &http.Client{Timeout: 70 * time.Second},
|
||||
mediaClient: newTenantMediaClient(),
|
||||
agentClient: &http.Client{Timeout: 15 * time.Minute},
|
||||
workLimit: make(chan struct{}, 4),
|
||||
activityEvery: 4 * time.Second,
|
||||
slotLocks: map[int]*sync.Mutex{},
|
||||
}
|
||||
}
|
||||
|
||||
@ -238,16 +240,6 @@ func (bot *telegramBot) run() {
|
||||
}
|
||||
}
|
||||
|
||||
func commandParts(text string) (string, []string) {
|
||||
fields := strings.Fields(strings.TrimSpace(text))
|
||||
if len(fields) == 0 || !strings.HasPrefix(fields[0], "/") {
|
||||
return "", nil
|
||||
}
|
||||
command := strings.TrimPrefix(strings.ToLower(fields[0]), "/")
|
||||
command, _, _ = strings.Cut(command, "@")
|
||||
return command, fields[1:]
|
||||
}
|
||||
|
||||
func (bot *telegramBot) handleUpdate(update telegramUpdate) {
|
||||
message := update.Message
|
||||
if message == nil || message.From == nil || message.Chat.Type != "private" || message.Chat.ID != message.From.ID {
|
||||
@ -276,7 +268,7 @@ func (bot *telegramBot) handleUpdate(update telegramUpdate) {
|
||||
return
|
||||
}
|
||||
if command == "help" {
|
||||
_ = bot.sendText(message.Chat.ID, "Send text or a photo to chat with Hermes. Use /topic <name> to create or switch a durable topic, /topics to list topics, or /unlink to disconnect this Telegram account. Model and intensity controls are available in Hermes WebUI.")
|
||||
_ = bot.sendText(message.Chat.ID, "Send text or a photo to chat with Hermes. Photos are available for analysis on the message that carries them. Image edits can reuse the latest image Hermes generated; inbound Telegram photos are not stored as editable artifacts. Use /topic <name> to create or switch a durable topic, /topics to list topics, or /unlink to disconnect this Telegram account. Model and intensity controls are available in Hermes WebUI.")
|
||||
return
|
||||
}
|
||||
slot, linked := bot.router.telegramSlot(userID)
|
||||
@ -332,7 +324,9 @@ func (bot *telegramBot) handleUpdate(update telegramUpdate) {
|
||||
_ = bot.sendText(message.Chat.ID, "Hermes could not save the active topic right now. Please try again shortly.")
|
||||
return
|
||||
}
|
||||
_ = bot.sendAction(message.Chat.ID, "typing")
|
||||
text = telegramInputPrompt(text, hasPhoto)
|
||||
stopActivity := bot.startTelegramActivity(message.Chat.ID)
|
||||
defer stopActivity()
|
||||
var reply string
|
||||
if hasPhoto {
|
||||
media, fetchErr := bot.fetchTelegramPhoto(message.Photo)
|
||||
@ -429,6 +423,10 @@ func (bot *telegramBot) askTenantRequest(slot int, bodyReader io.Reader, updateI
|
||||
func (bot *telegramBot) sendAction(chatID int64, action string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
return bot.sendActionContext(ctx, chatID, action)
|
||||
}
|
||||
|
||||
func (bot *telegramBot) sendActionContext(ctx context.Context, chatID int64, action string) error {
|
||||
return bot.call(ctx, "sendChatAction", url.Values{
|
||||
"chat_id": {strconv.FormatInt(chatID, 10)},
|
||||
"action": {action},
|
||||
|
||||
@ -12,6 +12,7 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@ -22,6 +23,93 @@ type telegramInputMedia struct {
|
||||
Path string
|
||||
}
|
||||
|
||||
func commandParts(text string) (string, []string) {
|
||||
fields := strings.Fields(strings.TrimSpace(text))
|
||||
if len(fields) == 0 || !strings.HasPrefix(fields[0], "/") {
|
||||
return "", nil
|
||||
}
|
||||
command := strings.TrimPrefix(strings.ToLower(fields[0]), "/")
|
||||
command, _, _ = strings.Cut(command, "@")
|
||||
return command, fields[1:]
|
||||
}
|
||||
|
||||
const inboundTelegramEditNotice = "Transport note: this Telegram photo is available for visual analysis on this turn, but it is not stored as a generated-image artifact. The image_edit_latest tools can reuse only an image Hermes generated. Do not claim an exact edit of this inbound photo; explain the limitation and offer either a new image inspired by it or an edit of the latest Hermes-generated image."
|
||||
|
||||
func looksLikeImageEdit(text string) bool {
|
||||
lower := strings.ToLower(strings.Join(strings.Fields(text), " "))
|
||||
for _, phrase := range []string{
|
||||
"edit this", "edit the image", "edit the photo", "change this image",
|
||||
"change this photo", "modify this", "retouch", "turn this into",
|
||||
"turn this cat", "turn this dog", "remove from the image",
|
||||
"replace in the image", "edit it", "change it", "modify it", "crop it",
|
||||
"resize it", "upscale it",
|
||||
} {
|
||||
if strings.Contains(lower, phrase) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
hasTarget := false
|
||||
for _, target := range []string{"image", "photo", "picture", "portrait", "selfie", "screenshot"} {
|
||||
if strings.Contains(lower, target) {
|
||||
hasTarget = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if hasTarget {
|
||||
for _, action := range []string{
|
||||
"add ", "blur", "brighten", "change", "color", "convert", "crop",
|
||||
"edit", "make ", "modify", "remove", "replace", "resize", "sharpen",
|
||||
"transform", "turn ", "upscale",
|
||||
} {
|
||||
if strings.Contains(lower, action) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func telegramInputPrompt(text string, hasPhoto bool) string {
|
||||
if hasPhoto {
|
||||
return inboundTelegramEditNotice + "\n\nUser request: " + text
|
||||
}
|
||||
if !looksLikeImageEdit(text) {
|
||||
return text
|
||||
}
|
||||
return "Transport note: no image is attached to this Telegram message. Use image_edit_latest only if Hermes previously generated an image in this private tenant. Otherwise do not claim access to an earlier inbound Telegram photo; offer a new image or ask the user to use Hermes WebUI with the source attachment.\n\nUser request: " + text
|
||||
}
|
||||
|
||||
func (bot *telegramBot) startTelegramActivity(chatID int64) func() {
|
||||
done := make(chan struct{})
|
||||
stopped := make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
every := bot.activityEvery
|
||||
if every <= 0 {
|
||||
every = 4 * time.Second
|
||||
}
|
||||
go func() {
|
||||
defer close(stopped)
|
||||
ticker := time.NewTicker(every)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
_ = bot.sendActionContext(ctx, chatID, "typing")
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}()
|
||||
var once sync.Once
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
close(done)
|
||||
cancel()
|
||||
})
|
||||
<-stopped
|
||||
}
|
||||
}
|
||||
|
||||
func largestTelegramPhoto(photos []telegramPhotoSize) (telegramPhotoSize, error) {
|
||||
var selected telegramPhotoSize
|
||||
for _, photo := range photos {
|
||||
|
||||
@ -24,7 +24,11 @@ const (
|
||||
tenantMediaLimit = 50 << 20
|
||||
)
|
||||
|
||||
var telegramMediaMarker = regexp.MustCompile(`MEDIA:([^\s\)\]]+)`)
|
||||
var (
|
||||
telegramMediaMarker = regexp.MustCompile(`(?i)MEDIA:\s*([^\s\)\]\}"']+)`)
|
||||
telegramMediaLabel = regexp.MustCompile(`(?i)MEDIA:`)
|
||||
telegramPrivatePath = regexp.MustCompile(`(?i)(?:/opt/data/(?:cache/images|workspace)|/workspace)/[^\s\)\]\}"']+`)
|
||||
)
|
||||
|
||||
type telegramReply struct {
|
||||
Text string
|
||||
@ -50,6 +54,8 @@ func parseTelegramReply(text string) telegramReply {
|
||||
}
|
||||
}
|
||||
cleaned := telegramMediaMarker.ReplaceAllString(text, "")
|
||||
cleaned = telegramPrivatePath.ReplaceAllString(cleaned, "[private attachment]")
|
||||
cleaned = telegramMediaLabel.ReplaceAllString(cleaned, "")
|
||||
lines := strings.Split(cleaned, "\n")
|
||||
compact := make([]string, 0, len(lines))
|
||||
blank := false
|
||||
|
||||
@ -124,6 +124,20 @@ func TestParseTelegramReplyRemovesOneOrMoreInternalMarkers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTelegramReplyScrubsMalformedMarkersAndBarePrivatePaths(t *testing.T) {
|
||||
reply := parseTelegramReply(
|
||||
"Delivered. MEDIA: /opt/data/cache/images/one.png and /workspace/private/two.png; MEDIA:",
|
||||
)
|
||||
if len(reply.MediaPaths) != 1 || reply.MediaPaths[0] != "/opt/data/cache/images/one.png" {
|
||||
t.Fatalf("valid spaced marker was not retained for delivery: %#v", reply)
|
||||
}
|
||||
for _, unsafe := range []string{"MEDIA:", "/opt/data/", "/workspace/"} {
|
||||
if strings.Contains(reply.Text, unsafe) {
|
||||
t.Fatalf("unsafe transport detail %q leaked in %q", unsafe, reply.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeTenantMediaPath(t *testing.T) {
|
||||
allowed := map[string]string{
|
||||
"/workspace/./renders//portrait.png": "/opt/data/workspace/renders/portrait.png",
|
||||
|
||||
@ -9,6 +9,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@ -273,6 +274,144 @@ func TestTelegramTopicValidationAndLegacyGeneralConversation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelegramTopicContinuityExpiresOnlyAfterIdleWeek(t *testing.T) {
|
||||
statePath := filepath.Join(t.TempDir(), "state.json")
|
||||
base := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
|
||||
now := base
|
||||
router, err := newTenantRouter(statePath, 1, func(int) string { return "" })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
router.now = func() time.Time { return now }
|
||||
first, err := router.selectTelegramTopic("123", "Week plan")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, elapsed := range []time.Duration{time.Hour, 24 * time.Hour} {
|
||||
now = base.Add(elapsed)
|
||||
resumed, touchErr := router.touchTelegramTopic("123")
|
||||
if touchErr != nil || resumed.Conversation != first.Conversation {
|
||||
t.Fatalf("topic changed after %s: %#v, %v", elapsed, resumed, touchErr)
|
||||
}
|
||||
}
|
||||
// Exactly seven idle days remains the same durable conversation.
|
||||
now = base.Add(8 * 24 * time.Hour)
|
||||
exactWeek, err := router.touchTelegramTopic("123")
|
||||
if err != nil || exactWeek.Conversation != first.Conversation {
|
||||
t.Fatalf("topic changed at exact idle week: %#v, %v", exactWeek, err)
|
||||
}
|
||||
reloaded, err := newTenantRouter(statePath, 1, func(int) string { return "" })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reloaded.now = func() time.Time { return now }
|
||||
if active := reloaded.activeTelegramTopic("123"); active != exactWeek {
|
||||
t.Fatalf("router restart changed topic: %#v != %#v", active, exactWeek)
|
||||
}
|
||||
now = now.Add(7*24*time.Hour + time.Second)
|
||||
rotated, err := reloaded.touchTelegramTopic("123")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rotated.Conversation == first.Conversation || !strings.HasSuffix(rotated.Conversation, "-g1") {
|
||||
t.Fatalf("expired topic did not rotate deterministically: %#v", rotated)
|
||||
}
|
||||
again, err := newTenantRouter(statePath, 1, func(int) string { return "" })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if active := again.activeTelegramTopic("123"); active != rotated {
|
||||
t.Fatalf("rotated identity changed after restart: %#v != %#v", active, rotated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelegramActivityRepeatsAndStops(t *testing.T) {
|
||||
var actions atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Path != "/sendChatAction" {
|
||||
t.Fatalf("unexpected Telegram method %s", request.URL.Path)
|
||||
}
|
||||
actions.Add(1)
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
_, _ = writer.Write([]byte(`{"ok":true,"result":true}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
bot := &telegramBot{
|
||||
apiBase: server.URL,
|
||||
client: &http.Client{Timeout: time.Second},
|
||||
activityEvery: 5 * time.Millisecond,
|
||||
}
|
||||
stop := bot.startTelegramActivity(42)
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for actions.Load() < 2 && time.Now().Before(deadline) {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
stop()
|
||||
stoppedAt := actions.Load()
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
if stoppedAt < 2 || actions.Load() != stoppedAt {
|
||||
t.Fatalf("activity did not progress and stop: before=%d after=%d", stoppedAt, actions.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelegramActivityStopCancelsInflightRequest(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) {
|
||||
close(started)
|
||||
select {
|
||||
case <-request.Context().Done():
|
||||
case <-release:
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
bot := &telegramBot{
|
||||
apiBase: server.URL,
|
||||
client: &http.Client{Timeout: time.Minute},
|
||||
activityEvery: time.Hour,
|
||||
}
|
||||
stop := bot.startTelegramActivity(42)
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("activity request did not start")
|
||||
}
|
||||
stopped := make(chan struct{})
|
||||
go func() {
|
||||
stop()
|
||||
close(stopped)
|
||||
}()
|
||||
select {
|
||||
case <-stopped:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("activity stop did not cancel the in-flight request")
|
||||
}
|
||||
close(release)
|
||||
}
|
||||
|
||||
func TestTelegramImageEditPromptStatesTransportBoundary(t *testing.T) {
|
||||
withPhoto := telegramInputPrompt("Edit this photo into a watercolor", true)
|
||||
if !strings.Contains(withPhoto, "available for visual analysis") ||
|
||||
!strings.Contains(withPhoto, "not stored as a generated-image artifact") {
|
||||
t.Fatalf("inbound attachment boundary missing: %q", withPhoto)
|
||||
}
|
||||
withoutPhoto := telegramInputPrompt("Edit this photo into a watercolor", false)
|
||||
if !strings.Contains(withoutPhoto, "no image is attached") ||
|
||||
!strings.Contains(withoutPhoto, "image_edit_latest only if Hermes previously generated") {
|
||||
t.Fatalf("prior generated image boundary missing: %q", withoutPhoto)
|
||||
}
|
||||
for _, request := range []string{"Make the photo brighter", "Crop it closer"} {
|
||||
if got := telegramInputPrompt(request, false); !strings.Contains(got, "no image is attached") {
|
||||
t.Fatalf("edit variant omitted attachment boundary: %q", got)
|
||||
}
|
||||
}
|
||||
plain := telegramInputPrompt("Explain this photograph", true)
|
||||
if !strings.Contains(plain, "available for visual analysis") ||
|
||||
!strings.Contains(plain, "User request: Explain this photograph") {
|
||||
t.Fatalf("photo analysis boundary missing: %q", plain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelegramInputRejectsUnsafePathsAndNonImages(t *testing.T) {
|
||||
for _, value := range []string{"", "../secret", "/absolute/image.jpg", `photos\image.jpg`, "photos/../../secret"} {
|
||||
if _, err := safeTelegramFilePath(value); err == nil {
|
||||
|
||||
@ -5,12 +5,16 @@ import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const telegramDefaultTopic = "General"
|
||||
const (
|
||||
telegramDefaultTopic = "General"
|
||||
telegramTopicIdleTTL = 7 * 24 * 60 * 60
|
||||
)
|
||||
|
||||
type activeTelegramTopic struct {
|
||||
Conversation string
|
||||
@ -42,12 +46,35 @@ func telegramTopicID(label string) string {
|
||||
}
|
||||
|
||||
func telegramTopicConversation(topicID string) string {
|
||||
return telegramTopicConversationGeneration(topicID, 0)
|
||||
}
|
||||
|
||||
func telegramTopicConversationGeneration(topicID string, generation int64) string {
|
||||
if topicID == "" || topicID == "general" {
|
||||
// Preserve the original named conversation so existing Telegram context
|
||||
// survives rollout into topic-aware routing.
|
||||
return "telegram"
|
||||
if generation == 0 {
|
||||
return "telegram"
|
||||
}
|
||||
topicID = "general"
|
||||
}
|
||||
return "telegram-topic-" + topicID
|
||||
conversation := "telegram-topic-" + topicID
|
||||
if generation > 0 {
|
||||
conversation += "-g" + strconv.FormatInt(generation, 10)
|
||||
}
|
||||
return conversation
|
||||
}
|
||||
|
||||
func topicExpired(topic telegramTopic, now int64) bool {
|
||||
return topic.LastUsed > 0 && now > topic.LastUsed && now-topic.LastUsed > telegramTopicIdleTTL
|
||||
}
|
||||
|
||||
func touchTopic(topic telegramTopic, now int64) telegramTopic {
|
||||
if topicExpired(topic, now) {
|
||||
topic.Generation++
|
||||
}
|
||||
topic.LastUsed = now
|
||||
return topic
|
||||
}
|
||||
|
||||
func (router *tenantRouter) activeTelegramTopic(userID string) activeTelegramTopic {
|
||||
@ -63,7 +90,10 @@ func (router *tenantRouter) activeTelegramTopic(userID string) activeTelegramTop
|
||||
if topic, found := state.Topics[topicID]; found && strings.TrimSpace(topic.Label) != "" {
|
||||
label = topic.Label
|
||||
}
|
||||
return activeTelegramTopic{Conversation: telegramTopicConversation(topicID), Label: label}
|
||||
return activeTelegramTopic{
|
||||
Conversation: telegramTopicConversationGeneration(topicID, state.Topics[topicID].Generation),
|
||||
Label: label,
|
||||
}
|
||||
}
|
||||
|
||||
func (router *tenantRouter) selectTelegramTopic(userID, rawLabel string) (activeTelegramTopic, error) {
|
||||
@ -83,12 +113,18 @@ func (router *tenantRouter) selectTelegramTopic(userID, rawLabel string) (active
|
||||
state.Topics = map[string]telegramTopic{}
|
||||
}
|
||||
state.Active = topicID
|
||||
state.Topics[topicID] = telegramTopic{Label: label, LastUsed: router.now().Unix()}
|
||||
topic := state.Topics[topicID]
|
||||
topic.Label = label
|
||||
topic = touchTopic(topic, router.now().Unix())
|
||||
state.Topics[topicID] = topic
|
||||
router.state.TelegramTopics[identity] = state
|
||||
if err := router.saveLocked(); err != nil {
|
||||
return activeTelegramTopic{}, err
|
||||
}
|
||||
return activeTelegramTopic{Conversation: telegramTopicConversation(topicID), Label: label}, nil
|
||||
return activeTelegramTopic{
|
||||
Conversation: telegramTopicConversationGeneration(topicID, topic.Generation),
|
||||
Label: label,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (router *tenantRouter) touchTelegramTopic(userID string) (activeTelegramTopic, error) {
|
||||
@ -108,13 +144,16 @@ func (router *tenantRouter) touchTelegramTopic(userID string) (activeTelegramTop
|
||||
if strings.TrimSpace(topic.Label) == "" {
|
||||
topic.Label = telegramDefaultTopic
|
||||
}
|
||||
topic.LastUsed = router.now().Unix()
|
||||
topic = touchTopic(topic, router.now().Unix())
|
||||
state.Topics[topicID] = topic
|
||||
router.state.TelegramTopics[identity] = state
|
||||
if err := router.saveLocked(); err != nil {
|
||||
return activeTelegramTopic{}, err
|
||||
}
|
||||
return activeTelegramTopic{Conversation: telegramTopicConversation(topicID), Label: topic.Label}, nil
|
||||
return activeTelegramTopic{
|
||||
Conversation: telegramTopicConversationGeneration(topicID, topic.Generation),
|
||||
Label: topic.Label,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (router *tenantRouter) telegramTopics(userID string) []telegramTopic {
|
||||
|
||||
@ -93,6 +93,8 @@ const privateFilesPage = `<!doctype html>
|
||||
</html>`
|
||||
|
||||
const bridgeCSS = `
|
||||
[data-panel="kanban"],[data-panel="logs"],[data-panel="insights"],[data-panel="memory"],[data-panel="skills"],[data-panel="workspaces"],[data-panel="todos"],[data-panel="tasks"],[data-panel="profiles"]{display:none!important}
|
||||
#settingsModel,label:has(#settingsModel),.settings-row:has(#settingsModel){display:none!important}
|
||||
#hermes-files-sidebar,#hermes-telegram-sidebar{display:flex;align-items:center}
|
||||
.hermes-link-page{margin:0;min-height:100vh;display:grid;place-items:center;background:#0f172a;color:#e2e8f0;font:16px/1.5 system-ui,sans-serif}
|
||||
.hermes-link-card{width:min(620px,calc(100% - 40px));box-sizing:border-box;padding:32px;border:1px solid #334155;border-radius:18px;background:#111827;box-shadow:0 20px 60px #0006}
|
||||
@ -105,8 +107,11 @@ const bridgeCSS = `
|
||||
const bridgeJS = `(() => {
|
||||
const page = document.querySelector('[data-telegram-page]');
|
||||
const filesPage = document.querySelector('[data-files-page]');
|
||||
const chatAdministrationPanels = ['kanban','logs','insights','memory','skills','workspaces','todos','tasks','profiles'];
|
||||
const hideChatAdministration = () => {
|
||||
document.querySelectorAll('[data-panel="kanban"]').forEach((node) => { node.hidden = true; });
|
||||
chatAdministrationPanels.forEach((panel) => {
|
||||
document.querySelectorAll('[data-panel="' + panel + '"]').forEach((node) => { node.hidden = true; });
|
||||
});
|
||||
};
|
||||
hideChatAdministration();
|
||||
if (!page && !filesPage) {
|
||||
@ -159,7 +164,7 @@ const bridgeJS = `(() => {
|
||||
const observer = new MutationObserver(() => {
|
||||
if (scheduled) return;
|
||||
scheduled = true;
|
||||
requestAnimationFrame(() => { scheduled = false; installSidebarLinks(); });
|
||||
requestAnimationFrame(() => { scheduled = false; hideChatAdministration(); installSidebarLinks(); });
|
||||
});
|
||||
installSidebarLinks();
|
||||
observer.observe(document.body, {childList:true, subtree:true});
|
||||
@ -354,6 +359,15 @@ func (router *tenantRouter) serveTelegramWeb(writer http.ResponseWriter, request
|
||||
writer.Header().Set("Cache-Control", "public, max-age=3600")
|
||||
_, _ = io.WriteString(writer, bridgeJS)
|
||||
return true
|
||||
case "/hermes-session-continuity.js":
|
||||
if request.Method != http.MethodGet {
|
||||
http.Error(writer, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return true
|
||||
}
|
||||
writer.Header().Set("Content-Type", "application/javascript; charset=utf-8")
|
||||
writer.Header().Set("Cache-Control", "public, max-age=3600")
|
||||
_, _ = io.WriteString(writer, sessionContinuityJS)
|
||||
return true
|
||||
case "/telegram":
|
||||
if request.Method != http.MethodGet {
|
||||
http.Error(writer, "method not allowed", http.StatusMethodNotAllowed)
|
||||
@ -455,6 +469,9 @@ func injectChatBridge(response *http.Response) error {
|
||||
content = strings.Replace(content, "</head>", `<link rel="stylesheet" href="/hermes-chat-bridge.css?v=20260813-telegram-readiness-v1"></head>`, 1)
|
||||
content = strings.Replace(content, "</body>", `<script src="/hermes-chat-bridge.js?v=20260813-telegram-readiness-v1" defer></script></body>`, 1)
|
||||
}
|
||||
if !strings.Contains(content, "hermes-session-continuity.js") {
|
||||
content = strings.Replace(content, "</body>", `<script src="/hermes-session-continuity.js?v=20260817-v1" defer></script></body>`, 1)
|
||||
}
|
||||
response.Body = io.NopCloser(strings.NewReader(content))
|
||||
response.ContentLength = int64(len(content))
|
||||
response.Header.Set("Content-Length", strconv.Itoa(len(content)))
|
||||
|
||||
138
services/hermes/scripts/ai_usage_codex.py
Normal file
138
services/hermes/scripts/ai_usage_codex.py
Normal file
@ -0,0 +1,138 @@
|
||||
"""Bounded Codex app-server query and child-process cleanup."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import selectors
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
|
||||
CODEX_BIN = os.environ.get("ATLAS_AI_CODEX_BIN", "/opt/data/tools/bin/codex")
|
||||
CODEX_HOME = os.environ.get("CODEX_HOME", "/runtime-access/codex")
|
||||
CODEX_CLEANUP_TIMEOUT_SECONDS = 5
|
||||
|
||||
|
||||
def _close_stream(stream: Any) -> bool:
|
||||
"""Close a subprocess pipe without allowing cleanup errors to escape."""
|
||||
if stream is None:
|
||||
return True
|
||||
try:
|
||||
stream.close()
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _cleanup_codex_process(
|
||||
process: subprocess.Popen[str], selector: selectors.BaseSelector | None
|
||||
) -> bool:
|
||||
"""Stop and reap a Codex child within two fixed waits, then close its pipes."""
|
||||
cleanup_ok = True
|
||||
if selector is not None:
|
||||
try:
|
||||
selector.close()
|
||||
except Exception:
|
||||
cleanup_ok = False
|
||||
|
||||
# Closing the request pipe first also gives a responsive app-server an EOF.
|
||||
cleanup_ok = _close_stream(getattr(process, "stdin", None)) and cleanup_ok
|
||||
try:
|
||||
process.terminate()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
except Exception:
|
||||
cleanup_ok = False
|
||||
|
||||
needs_kill = False
|
||||
try:
|
||||
process.wait(timeout=CODEX_CLEANUP_TIMEOUT_SECONDS)
|
||||
except subprocess.TimeoutExpired:
|
||||
needs_kill = True
|
||||
except Exception:
|
||||
cleanup_ok = False
|
||||
needs_kill = True
|
||||
|
||||
if needs_kill:
|
||||
try:
|
||||
process.kill()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
except Exception:
|
||||
cleanup_ok = False
|
||||
try:
|
||||
process.wait(timeout=CODEX_CLEANUP_TIMEOUT_SECONDS)
|
||||
except Exception:
|
||||
cleanup_ok = False
|
||||
|
||||
cleanup_ok = _close_stream(getattr(process, "stdout", None)) and cleanup_ok
|
||||
cleanup_ok = _close_stream(getattr(process, "stderr", None)) and cleanup_ok
|
||||
return cleanup_ok
|
||||
|
||||
|
||||
def query_codex(timeout: float = 20) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
"""Read Codex account quota and usage through its structured app-server protocol."""
|
||||
process = subprocess.Popen(
|
||||
[CODEX_BIN, "app-server", "--stdio"],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
bufsize=1,
|
||||
env={**os.environ, "CODEX_HOME": CODEX_HOME},
|
||||
)
|
||||
requests = (
|
||||
{
|
||||
"id": 1,
|
||||
"method": "initialize",
|
||||
"params": {
|
||||
"clientInfo": {
|
||||
"name": "atlas-ai-usage-exporter",
|
||||
"title": "Atlas AI Usage Exporter",
|
||||
"version": "1.0.0",
|
||||
},
|
||||
"capabilities": {"experimentalApi": True},
|
||||
},
|
||||
},
|
||||
{"method": "initialized", "params": {}},
|
||||
{"id": 2, "method": "account/rateLimits/read", "params": None},
|
||||
{"id": 3, "method": "account/usage/read", "params": None},
|
||||
)
|
||||
selector: selectors.BaseSelector | None = None
|
||||
rate_result: dict[str, Any]
|
||||
usage_result: dict[str, Any]
|
||||
try:
|
||||
if process.stdin is None or process.stdout is None:
|
||||
raise RuntimeError("Codex app-server pipes are unavailable")
|
||||
for message in requests:
|
||||
process.stdin.write(json.dumps(message, separators=(",", ":")) + "\n")
|
||||
process.stdin.flush()
|
||||
selector = selectors.DefaultSelector()
|
||||
selector.register(process.stdout, selectors.EVENT_READ)
|
||||
responses: dict[int, dict[str, Any]] = {}
|
||||
deadline = time.monotonic() + timeout
|
||||
while len(responses) < 3 and time.monotonic() < deadline:
|
||||
for key, _ in selector.select(min(1, max(0, deadline - time.monotonic()))):
|
||||
line = key.fileobj.readline()
|
||||
if not line:
|
||||
continue
|
||||
message = json.loads(line)
|
||||
if message.get("id") in (1, 2, 3):
|
||||
responses[int(message["id"])] = message
|
||||
for response_id in (1, 2, 3):
|
||||
response = responses.get(response_id)
|
||||
if (
|
||||
not response
|
||||
or "error" in response
|
||||
or not isinstance(response.get("result"), dict)
|
||||
):
|
||||
raise RuntimeError(f"Codex app-server response {response_id} failed")
|
||||
rate_result = responses[2]["result"]
|
||||
usage_result = responses[3]["result"]
|
||||
finally:
|
||||
cleanup_ok = _cleanup_codex_process(process, selector)
|
||||
if not cleanup_ok:
|
||||
raise RuntimeError("Codex app-server cleanup failed")
|
||||
return rate_result, usage_result
|
||||
@ -5,20 +5,18 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import selectors
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
import ai_usage_codex as codex_query
|
||||
import ai_usage_http as http_engine
|
||||
|
||||
|
||||
CODEX_BIN = os.environ.get("ATLAS_AI_CODEX_BIN", "/opt/data/tools/bin/codex")
|
||||
CODEX_HOME = os.environ.get("CODEX_HOME", "/runtime-access/codex")
|
||||
CLAUDE_CREDENTIALS = Path(
|
||||
os.environ.get(
|
||||
"ATLAS_AI_CLAUDE_CREDENTIALS",
|
||||
@ -48,6 +46,8 @@ METRIC_HELP = {
|
||||
"atlas_ai_quota_used_percent": "Used percentage in a first-party coding CLI quota window.",
|
||||
"atlas_ai_quota_window_duration_seconds": "Nominal duration of a coding CLI quota window.",
|
||||
}
|
||||
POLL_STARTUP_GRACE_SECONDS = 45
|
||||
POLL_PROGRESS_BUDGET_SECONDS = 60
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@ -75,7 +75,9 @@ def _number(value: Any) -> float | None:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
return None
|
||||
converted = float(value)
|
||||
return converted if converted == converted and abs(converted) != float("inf") else None
|
||||
return (
|
||||
converted if converted == converted and abs(converted) != float("inf") else None
|
||||
)
|
||||
|
||||
|
||||
def _timestamp(value: Any) -> float | None:
|
||||
@ -112,7 +114,10 @@ def _codex_limit_name(limit_id: str, snapshot: dict[str, Any]) -> str:
|
||||
|
||||
|
||||
def parse_codex_payloads(
|
||||
rate_response: dict[str, Any], usage_response: dict[str, Any], *, today: date | None = None
|
||||
rate_response: dict[str, Any],
|
||||
usage_response: dict[str, Any],
|
||||
*,
|
||||
today: date | None = None,
|
||||
) -> list[Sample]:
|
||||
"""Convert structured Codex app-server responses into bounded metrics."""
|
||||
samples: list[Sample] = []
|
||||
@ -140,15 +145,23 @@ def parse_codex_payloads(
|
||||
samples.extend(
|
||||
(
|
||||
Sample("atlas_ai_quota_used_percent", labels, used),
|
||||
Sample("atlas_ai_quota_remaining_percent", labels, max(0, 100 - used)),
|
||||
Sample(
|
||||
"atlas_ai_quota_remaining_percent", labels, max(0, 100 - used)
|
||||
),
|
||||
)
|
||||
)
|
||||
reset = _timestamp(raw_window.get("resetsAt"))
|
||||
duration = _number(raw_window.get("windowDurationMins"))
|
||||
if reset is not None:
|
||||
samples.append(Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset))
|
||||
samples.append(
|
||||
Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset)
|
||||
)
|
||||
if duration is not None:
|
||||
samples.append(Sample("atlas_ai_quota_window_duration_seconds", labels, duration * 60))
|
||||
samples.append(
|
||||
Sample(
|
||||
"atlas_ai_quota_window_duration_seconds", labels, duration * 60
|
||||
)
|
||||
)
|
||||
|
||||
summary = usage_response.get("summary")
|
||||
if isinstance(summary, dict):
|
||||
@ -189,8 +202,16 @@ def parse_codex_payloads(
|
||||
anchor_day = max((start for start, _ in parsed_daily), default=current_day)
|
||||
for period, days in (("latest_day", 1), ("seven_day", 7), ("thirty_day", 30)):
|
||||
earliest = anchor_day - timedelta(days=days - 1)
|
||||
value = sum(tokens for start, tokens in parsed_daily if earliest <= start <= anchor_day)
|
||||
samples.append(Sample("atlas_ai_account_tokens", {"provider": "openai", "period": period}, value))
|
||||
value = sum(
|
||||
tokens for start, tokens in parsed_daily if earliest <= start <= anchor_day
|
||||
)
|
||||
samples.append(
|
||||
Sample(
|
||||
"atlas_ai_account_tokens",
|
||||
{"provider": "openai", "period": period},
|
||||
value,
|
||||
)
|
||||
)
|
||||
return samples
|
||||
|
||||
|
||||
@ -213,7 +234,9 @@ def parse_claude_payload(payload: dict[str, Any]) -> list[Sample]:
|
||||
)
|
||||
reset = _timestamp(window.get("resets_at"))
|
||||
if reset is not None:
|
||||
samples.append(Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset))
|
||||
samples.append(
|
||||
Sample("atlas_ai_quota_reset_timestamp_seconds", labels, reset)
|
||||
)
|
||||
extra = payload.get("extra_usage")
|
||||
if isinstance(extra, dict):
|
||||
enabled = extra.get("is_enabled")
|
||||
@ -228,64 +251,7 @@ def parse_claude_payload(payload: dict[str, Any]) -> list[Sample]:
|
||||
return samples
|
||||
|
||||
|
||||
def query_codex(timeout: float = 20) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
"""Read Codex account quota and usage through its structured app-server protocol."""
|
||||
process = subprocess.Popen(
|
||||
[CODEX_BIN, "app-server", "--stdio"],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
bufsize=1,
|
||||
env={**os.environ, "CODEX_HOME": CODEX_HOME},
|
||||
)
|
||||
requests = (
|
||||
{
|
||||
"id": 1,
|
||||
"method": "initialize",
|
||||
"params": {
|
||||
"clientInfo": {
|
||||
"name": "atlas-ai-usage-exporter",
|
||||
"title": "Atlas AI Usage Exporter",
|
||||
"version": "1.0.0",
|
||||
},
|
||||
"capabilities": {"experimentalApi": True},
|
||||
},
|
||||
},
|
||||
{"method": "initialized", "params": {}},
|
||||
{"id": 2, "method": "account/rateLimits/read", "params": None},
|
||||
{"id": 3, "method": "account/usage/read", "params": None},
|
||||
)
|
||||
try:
|
||||
if process.stdin is None or process.stdout is None:
|
||||
raise RuntimeError("Codex app-server pipes are unavailable")
|
||||
for message in requests:
|
||||
process.stdin.write(json.dumps(message, separators=(",", ":")) + "\n")
|
||||
process.stdin.flush()
|
||||
selector = selectors.DefaultSelector()
|
||||
selector.register(process.stdout, selectors.EVENT_READ)
|
||||
responses: dict[int, dict[str, Any]] = {}
|
||||
deadline = time.monotonic() + timeout
|
||||
while len(responses) < 3 and time.monotonic() < deadline:
|
||||
for key, _ in selector.select(min(1, max(0, deadline - time.monotonic()))):
|
||||
line = key.fileobj.readline()
|
||||
if not line:
|
||||
continue
|
||||
message = json.loads(line)
|
||||
if message.get("id") in (1, 2, 3):
|
||||
responses[int(message["id"])] = message
|
||||
for response_id in (1, 2, 3):
|
||||
response = responses.get(response_id)
|
||||
if not response or "error" in response or not isinstance(response.get("result"), dict):
|
||||
raise RuntimeError(f"Codex app-server response {response_id} failed")
|
||||
return responses[2]["result"], responses[3]["result"]
|
||||
finally:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait(timeout=5)
|
||||
query_codex = codex_query.query_codex
|
||||
|
||||
|
||||
def query_claude() -> dict[str, Any]:
|
||||
@ -322,6 +288,24 @@ class Collector:
|
||||
self._lock = threading.Lock()
|
||||
self._providers = {name: ProviderState() for name in ("openai", "anthropic")}
|
||||
|
||||
def record_failure(
|
||||
self,
|
||||
provider: str,
|
||||
*,
|
||||
started: float | None = None,
|
||||
monotonic_started: float | None = None,
|
||||
) -> None:
|
||||
"""Record a failed attempt while retaining the provider's last good samples."""
|
||||
attempted_at = time.time() if started is None else started
|
||||
duration_started = (
|
||||
time.monotonic() if monotonic_started is None else monotonic_started
|
||||
)
|
||||
with self._lock:
|
||||
state = self._providers[provider]
|
||||
state.last_attempt = attempted_at
|
||||
state.duration = max(0, time.monotonic() - duration_started)
|
||||
state.fetch_success = False
|
||||
|
||||
def refresh_provider(self, provider: str) -> None:
|
||||
"""Refresh one provider while retaining the last good values on failure."""
|
||||
started = time.time()
|
||||
@ -333,19 +317,24 @@ class Collector:
|
||||
samples = parse_claude_payload(query_claude())
|
||||
else:
|
||||
raise ValueError("unknown provider")
|
||||
success = True
|
||||
except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as error:
|
||||
print(f"{provider} quota collection deferred: {type(error).__name__}", flush=True)
|
||||
samples = []
|
||||
success = False
|
||||
except Exception as error:
|
||||
print(
|
||||
f"{provider} quota collection deferred: {type(error).__name__}",
|
||||
flush=True,
|
||||
)
|
||||
self.record_failure(
|
||||
provider,
|
||||
started=started,
|
||||
monotonic_started=monotonic_started,
|
||||
)
|
||||
return
|
||||
with self._lock:
|
||||
state = self._providers[provider]
|
||||
state.last_attempt = started
|
||||
state.duration = time.monotonic() - monotonic_started
|
||||
state.fetch_success = success
|
||||
if success:
|
||||
state.samples = samples
|
||||
state.last_success = time.time()
|
||||
state.fetch_success = True
|
||||
state.samples = samples
|
||||
state.last_success = time.time()
|
||||
|
||||
def render(self) -> bytes:
|
||||
"""Render the current provider states in Prometheus text format."""
|
||||
@ -360,10 +349,24 @@ class Collector:
|
||||
samples.extend(state.samples)
|
||||
samples.extend(
|
||||
(
|
||||
Sample("atlas_ai_quota_fetch_success", labels, float(state.fetch_success)),
|
||||
Sample("atlas_ai_quota_last_attempt_timestamp_seconds", labels, state.last_attempt),
|
||||
Sample("atlas_ai_quota_last_success_timestamp_seconds", labels, state.last_success),
|
||||
Sample("atlas_ai_quota_fetch_duration_seconds", labels, state.duration),
|
||||
Sample(
|
||||
"atlas_ai_quota_fetch_success",
|
||||
labels,
|
||||
float(state.fetch_success),
|
||||
),
|
||||
Sample(
|
||||
"atlas_ai_quota_last_attempt_timestamp_seconds",
|
||||
labels,
|
||||
state.last_attempt,
|
||||
),
|
||||
Sample(
|
||||
"atlas_ai_quota_last_success_timestamp_seconds",
|
||||
labels,
|
||||
state.last_success,
|
||||
),
|
||||
Sample(
|
||||
"atlas_ai_quota_fetch_duration_seconds", labels, state.duration
|
||||
),
|
||||
)
|
||||
)
|
||||
lines: list[str] = []
|
||||
@ -381,38 +384,112 @@ class Collector:
|
||||
return ("\n".join(lines) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
class PollingEngine:
|
||||
"""Run isolated provider polls and track bounded forward progress."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
collector: Collector,
|
||||
*,
|
||||
interval: float,
|
||||
startup_grace: float = POLL_STARTUP_GRACE_SECONDS,
|
||||
progress_timeout: float | None = None,
|
||||
clock: Any = time.monotonic,
|
||||
) -> None:
|
||||
self.collector = collector
|
||||
self.interval = interval
|
||||
self.startup_grace = startup_grace
|
||||
self.progress_timeout = (
|
||||
interval + POLL_PROGRESS_BUDGET_SECONDS
|
||||
if progress_timeout is None
|
||||
else progress_timeout
|
||||
)
|
||||
self._clock = clock
|
||||
self._started_at = clock()
|
||||
self._last_progress: float | None = None
|
||||
self._progress_lock = threading.Lock()
|
||||
self._thread: threading.Thread | None = None
|
||||
|
||||
def _mark_progress(self) -> None:
|
||||
"""Record completion of one bounded provider attempt."""
|
||||
with self._progress_lock:
|
||||
self._last_progress = self._clock()
|
||||
|
||||
def poll_once(self) -> None:
|
||||
"""Refresh every provider even if another provider fails unexpectedly."""
|
||||
for provider in ("openai", "anthropic"):
|
||||
started = time.time()
|
||||
monotonic_started = time.monotonic()
|
||||
try:
|
||||
self.collector.refresh_provider(provider)
|
||||
except Exception as error:
|
||||
print(
|
||||
f"{provider} quota collection isolated: {type(error).__name__}",
|
||||
flush=True,
|
||||
)
|
||||
try:
|
||||
self.collector.record_failure(
|
||||
provider,
|
||||
started=started,
|
||||
monotonic_started=monotonic_started,
|
||||
)
|
||||
except Exception as record_error:
|
||||
print(
|
||||
f"{provider} quota failure accounting deferred: "
|
||||
f"{type(record_error).__name__}",
|
||||
flush=True,
|
||||
)
|
||||
finally:
|
||||
self._mark_progress()
|
||||
|
||||
def run(self) -> None:
|
||||
"""Poll forever without allowing a cycle-level exception to stop the thread."""
|
||||
while True:
|
||||
try:
|
||||
self.poll_once()
|
||||
except Exception as error:
|
||||
print(
|
||||
f"quota polling cycle deferred: {type(error).__name__}", flush=True
|
||||
)
|
||||
time.sleep(self.interval)
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start the daemon poller exactly once."""
|
||||
if self._thread is not None:
|
||||
return
|
||||
self._thread = threading.Thread(
|
||||
target=self.run,
|
||||
name="ai-usage-poller",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def is_healthy(self) -> bool:
|
||||
"""Report thread liveness and progress, independent of provider success."""
|
||||
thread = self._thread
|
||||
if thread is None or not thread.is_alive():
|
||||
return False
|
||||
now = self._clock()
|
||||
with self._progress_lock:
|
||||
last_progress = self._last_progress
|
||||
if last_progress is None:
|
||||
return now - self._started_at <= self.startup_grace
|
||||
return now - last_progress <= self.progress_timeout
|
||||
|
||||
|
||||
make_handler = http_engine.make_handler
|
||||
Server = http_engine.Server
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Poll quota APIs and serve only sanitized metrics and health endpoints."""
|
||||
collector = Collector()
|
||||
interval = max(60, int(os.environ.get("ATLAS_AI_USAGE_INTERVAL_SECONDS", "300")))
|
||||
|
||||
def polling_loop() -> None:
|
||||
while True:
|
||||
for provider in ("openai", "anthropic"):
|
||||
collector.refresh_provider(provider)
|
||||
time.sleep(interval)
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
if self.path == "/metrics":
|
||||
payload = collector.render()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/plain; version=0.0.4")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
elif self.path == "/healthz":
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
else:
|
||||
self.send_error(404)
|
||||
|
||||
def log_message(self, _format: str, *_args: object) -> None:
|
||||
return
|
||||
|
||||
threading.Thread(target=polling_loop, daemon=True).start()
|
||||
poller = PollingEngine(collector, interval=interval)
|
||||
poller.start()
|
||||
port = int(os.environ.get("ATLAS_AI_USAGE_PORT", "9010"))
|
||||
server = ThreadingHTTPServer(("0.0.0.0", port), Handler)
|
||||
server = Server(("0.0.0.0", port), make_handler(collector, poller))
|
||||
server.serve_forever()
|
||||
return 0
|
||||
|
||||
|
||||
49
services/hermes/scripts/ai_usage_http.py
Normal file
49
services/hermes/scripts/ai_usage_http.py
Normal file
@ -0,0 +1,49 @@
|
||||
"""Disconnect-safe HTTP serving for the AI usage exporter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from typing import Any
|
||||
|
||||
|
||||
def make_handler(collector: Any, poller: Any) -> type[BaseHTTPRequestHandler]:
|
||||
"""Build an HTTP handler bound to one collector and polling engine."""
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def _respond(self, status: int, payload: bytes, content_type: str) -> None:
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
try:
|
||||
if self.path == "/metrics":
|
||||
self._respond(
|
||||
200,
|
||||
collector.render(),
|
||||
"text/plain; version=0.0.4",
|
||||
)
|
||||
elif self.path == "/healthz":
|
||||
healthy = poller.is_healthy()
|
||||
self._respond(
|
||||
200 if healthy else 503,
|
||||
b"ok\n" if healthy else b"poller unhealthy\n",
|
||||
"text/plain; charset=utf-8",
|
||||
)
|
||||
else:
|
||||
self.send_error(404)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
return
|
||||
|
||||
def log_message(self, _format: str, *_args: object) -> None:
|
||||
return
|
||||
|
||||
return Handler
|
||||
|
||||
|
||||
class Server(ThreadingHTTPServer):
|
||||
"""Threaded metrics server that does not retain disconnected clients."""
|
||||
|
||||
daemon_threads = True
|
||||
@ -1,16 +1,236 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Backfill Telegram origin metadata for durable API conversations."""
|
||||
"""Bound Telegram context and backfill durable API-session metadata."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
MAX_RESPONSE_BYTES = 32 << 20
|
||||
MAX_HISTORY_ITEMS = 80
|
||||
MAX_HISTORY_BYTES = 192_000
|
||||
MAX_RECENT_TURNS = 12
|
||||
MAX_ITEM_CHARS = 8_000
|
||||
MAX_SUMMARY_CHARS = 12_000
|
||||
SUMMARY_PREFIX = "Telegram continuity summary (older bounded context):"
|
||||
ATTACHMENT_NOTE = (
|
||||
"[Earlier Telegram image omitted from bounded context; attach it again "
|
||||
"in Hermes WebUI if the original pixels are required.]"
|
||||
)
|
||||
_CONVERSATION = re.compile(r"^telegram(?:-topic-[a-z0-9-]{1,160})?$")
|
||||
_MEDIA_PATH = re.compile(
|
||||
r'(?:MEDIA:\s*)?(?:/opt/data/(?:cache/images|workspace)|/workspace)/[^\s\]\[)}`"]+',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_DATA_IMAGE = re.compile(r"data:image/[^;,\s]+;base64,[A-Za-z0-9+/=]+", re.IGNORECASE)
|
||||
_MESSAGE_IDENTITY_COLUMNS = (
|
||||
"session_id",
|
||||
"role",
|
||||
"content",
|
||||
"tool_call_id",
|
||||
"tool_calls",
|
||||
"tool_name",
|
||||
"timestamp",
|
||||
"token_count",
|
||||
"finish_reason",
|
||||
"reasoning",
|
||||
"reasoning_content",
|
||||
"reasoning_details",
|
||||
"codex_reasoning_items",
|
||||
"codex_message_items",
|
||||
"platform_message_id",
|
||||
"observed",
|
||||
"active",
|
||||
"compacted",
|
||||
)
|
||||
|
||||
|
||||
def _text(value: Any) -> str:
|
||||
"""Return bounded context text without reusable transport-only images."""
|
||||
if isinstance(value, str):
|
||||
text = value
|
||||
elif isinstance(value, list):
|
||||
parts: list[str] = []
|
||||
for part in value:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
kind = str(part.get("type") or "").lower()
|
||||
if "image" in kind or "image_url" in part:
|
||||
parts.append(ATTACHMENT_NOTE)
|
||||
continue
|
||||
candidate = part.get("text", part.get("content", ""))
|
||||
if isinstance(candidate, str):
|
||||
parts.append(candidate)
|
||||
text = "\n".join(parts)
|
||||
else:
|
||||
try:
|
||||
text = json.dumps(value, ensure_ascii=False, sort_keys=True)
|
||||
except (TypeError, ValueError):
|
||||
text = str(value)
|
||||
text = _DATA_IMAGE.sub(ATTACHMENT_NOTE, text)
|
||||
text = _MEDIA_PATH.sub("[private attachment]", text)
|
||||
return text[:MAX_ITEM_CHARS]
|
||||
|
||||
|
||||
def _normalized_item(raw: Any) -> dict[str, Any] | None:
|
||||
if not isinstance(raw, dict):
|
||||
return None
|
||||
role = str(raw.get("role") or "").strip().lower()
|
||||
if role not in {"assistant", "system", "tool", "user"}:
|
||||
return None
|
||||
content = _text(raw.get("content", ""))
|
||||
if not content and role != "assistant":
|
||||
return None
|
||||
item: dict[str, Any] = {"role": role, "content": content}
|
||||
if raw.get("_db_persisted") is True:
|
||||
item["_db_persisted"] = True
|
||||
for key in ("name", "tool_call_id", "tool_name"):
|
||||
value = raw.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
item[key] = value[:256]
|
||||
calls = raw.get("tool_calls")
|
||||
if isinstance(calls, list):
|
||||
bounded_calls = []
|
||||
for call in calls[:16]:
|
||||
if not isinstance(call, dict):
|
||||
continue
|
||||
function = (
|
||||
call.get("function") if isinstance(call.get("function"), dict) else {}
|
||||
)
|
||||
bounded = {
|
||||
"id": str(call.get("id") or "")[:256],
|
||||
"type": str(call.get("type") or "function")[:32],
|
||||
"function": {
|
||||
"name": str(function.get("name") or "")[:256],
|
||||
"arguments": _text(function.get("arguments", ""))[:2_000],
|
||||
},
|
||||
}
|
||||
bounded_calls.append(bounded)
|
||||
if bounded_calls:
|
||||
item["tool_calls"] = bounded_calls
|
||||
return item
|
||||
|
||||
|
||||
def _fingerprint(item: dict[str, Any]) -> str:
|
||||
try:
|
||||
visible = {key: value for key, value in item.items() if not key.startswith("_")}
|
||||
return json.dumps(
|
||||
visible, ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return repr(item)
|
||||
|
||||
|
||||
def _deduplicate(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Drop replay duplicates while preserving intentional repeated prose."""
|
||||
result: list[dict[str, Any]] = []
|
||||
stable_ids: set[tuple[str, str]] = set()
|
||||
for item in items:
|
||||
stable = ""
|
||||
if isinstance(item.get("tool_call_id"), str):
|
||||
stable = item["tool_call_id"]
|
||||
elif isinstance(item.get("id"), str):
|
||||
stable = item["id"]
|
||||
if stable:
|
||||
key = (item["role"], stable)
|
||||
if key in stable_ids:
|
||||
continue
|
||||
stable_ids.add(key)
|
||||
if result and _fingerprint(result[-1]) == _fingerprint(item):
|
||||
continue
|
||||
result.append(item)
|
||||
# Response replays can append one exact transcript to itself. Remove only
|
||||
# complete adjacent blocks, not ordinary repeated user phrases.
|
||||
changed = True
|
||||
while changed and len(result) > 1:
|
||||
changed = False
|
||||
for width in range(len(result) // 2, 0, -1):
|
||||
if [_fingerprint(x) for x in result[-2 * width : -width]] == [
|
||||
_fingerprint(x) for x in result[-width:]
|
||||
]:
|
||||
del result[-width:]
|
||||
changed = True
|
||||
break
|
||||
return result
|
||||
|
||||
|
||||
def _summary(older: list[dict[str, Any]]) -> dict[str, str] | None:
|
||||
lines: list[str] = []
|
||||
for item in older:
|
||||
role = item["role"]
|
||||
content = str(item.get("content") or "").strip()
|
||||
if role == "system" and content.startswith(SUMMARY_PREFIX):
|
||||
candidates = content[len(SUMMARY_PREFIX) :].strip().splitlines()
|
||||
elif role in {"user", "assistant"} and content:
|
||||
candidates = [f"- {role.title()}: {content[:600]}"]
|
||||
else:
|
||||
continue
|
||||
for line in candidates:
|
||||
clean = " ".join(line.split())
|
||||
if clean and clean not in lines:
|
||||
lines.append(clean)
|
||||
content = SUMMARY_PREFIX
|
||||
for line in lines[-40:]:
|
||||
candidate = content + "\n" + line
|
||||
if len(candidate) > MAX_SUMMARY_CHARS:
|
||||
break
|
||||
content = candidate
|
||||
return (
|
||||
{
|
||||
"role": "system",
|
||||
"content": content,
|
||||
"_db_persisted": True,
|
||||
"_compressed_summary": True,
|
||||
}
|
||||
if lines
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
def compact_telegram_history(history: Any) -> list[dict[str, Any]]:
|
||||
"""Return a deterministic summary plus bounded recent Telegram turns."""
|
||||
if not isinstance(history, list):
|
||||
return []
|
||||
normalized = _deduplicate(
|
||||
[item for raw in history if (item := _normalized_item(raw)) is not None]
|
||||
)
|
||||
recent: list[dict[str, Any]] = []
|
||||
recent_bytes = 2
|
||||
user_turns = 0
|
||||
split = len(normalized)
|
||||
for index in range(len(normalized) - 1, -1, -1):
|
||||
item = normalized[index]
|
||||
encoded = len(_fingerprint(item).encode("utf-8")) + 1
|
||||
next_turns = user_turns + (item["role"] == "user")
|
||||
if recent and (
|
||||
len(recent) >= MAX_HISTORY_ITEMS
|
||||
or recent_bytes + encoded > MAX_HISTORY_BYTES
|
||||
or next_turns > MAX_RECENT_TURNS
|
||||
):
|
||||
break
|
||||
recent.insert(0, item)
|
||||
recent_bytes += encoded
|
||||
user_turns = next_turns
|
||||
split = index
|
||||
summary = _summary(normalized[:split])
|
||||
result = ([summary] if summary else []) + recent
|
||||
while (
|
||||
len(json.dumps(result, ensure_ascii=False).encode("utf-8"))
|
||||
> (MAX_HISTORY_BYTES + MAX_SUMMARY_CHARS)
|
||||
and len(recent) > 1
|
||||
):
|
||||
recent.pop(0)
|
||||
result = ([summary] if summary else []) + recent
|
||||
return result
|
||||
|
||||
|
||||
def telegram_sessions(response_store: Path) -> dict[str, str]:
|
||||
"""Return session ID to stable Telegram conversation-key mappings."""
|
||||
"""Return safe session ID to stable Telegram conversation mappings."""
|
||||
if not response_store.exists():
|
||||
return {}
|
||||
connection = sqlite3.connect(f"file:{response_store}?mode=ro", uri=True)
|
||||
@ -19,21 +239,64 @@ def telegram_sessions(response_store: Path) -> dict[str, str]:
|
||||
"""SELECT c.name, r.data
|
||||
FROM conversations AS c
|
||||
JOIN responses AS r ON r.response_id = c.response_id
|
||||
WHERE c.name = 'telegram' OR c.name LIKE 'telegram-topic-%'"""
|
||||
WHERE (c.name = 'telegram' OR c.name LIKE 'telegram-topic-%')
|
||||
AND LENGTH(r.data) <= ?
|
||||
ORDER BY c.name, r.response_id""",
|
||||
(MAX_RESPONSE_BYTES,),
|
||||
).fetchall()
|
||||
except sqlite3.DatabaseError:
|
||||
return {}
|
||||
finally:
|
||||
connection.close()
|
||||
result: dict[str, str] = {}
|
||||
for conversation, raw in rows:
|
||||
try:
|
||||
session_id = str(json.loads(raw).get("session_id") or "").strip()
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
name = str(conversation or "")
|
||||
if not _CONVERSATION.fullmatch(name):
|
||||
continue
|
||||
if session_id:
|
||||
result[session_id] = str(conversation)
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
session_id = str(payload.get("session_id") or "").strip()
|
||||
except (AttributeError, TypeError, ValueError, json.JSONDecodeError):
|
||||
continue
|
||||
if (
|
||||
session_id
|
||||
and len(session_id) <= 256
|
||||
and not re.search(r"[\r\n\x00/\\]", session_id)
|
||||
):
|
||||
result.setdefault(session_id, name)
|
||||
return result
|
||||
|
||||
|
||||
def deduplicate_stored_messages(
|
||||
connection: sqlite3.Connection, session_ids: list[str]
|
||||
) -> int:
|
||||
"""Remove only byte-for-byte replay rows for known Telegram API sessions."""
|
||||
columns = {str(row[1]) for row in connection.execute("PRAGMA table_info(messages)")}
|
||||
required = {"id", "session_id", "role", "content", "timestamp"}
|
||||
if not required.issubset(columns):
|
||||
return 0
|
||||
identity = [column for column in _MESSAGE_IDENTITY_COLUMNS if column in columns]
|
||||
partition = ", ".join(identity)
|
||||
changed = 0
|
||||
for session_id in session_ids:
|
||||
query = f"""DELETE FROM messages WHERE id IN (
|
||||
SELECT id FROM (
|
||||
SELECT id, ROW_NUMBER() OVER (
|
||||
PARTITION BY {partition} ORDER BY id
|
||||
) AS replay_number
|
||||
FROM messages
|
||||
WHERE session_id = ? AND EXISTS (
|
||||
SELECT 1 FROM sessions
|
||||
WHERE sessions.id = messages.session_id
|
||||
AND sessions.source = 'api_server'
|
||||
)
|
||||
) WHERE replay_number > 1
|
||||
)""" # noqa: S608 -- identifiers come only from the fixed allowlist
|
||||
cursor = connection.execute(query, (session_id,))
|
||||
changed += max(cursor.rowcount, 0)
|
||||
return changed
|
||||
|
||||
|
||||
def migrate(state_database: Path, response_store: Path) -> int:
|
||||
"""Annotate known sessions without creating or rewriting conversations."""
|
||||
mappings = telegram_sessions(response_store)
|
||||
@ -43,12 +306,16 @@ def migrate(state_database: Path, response_store: Path) -> int:
|
||||
changed = 0
|
||||
try:
|
||||
columns = {
|
||||
str(row[1])
|
||||
for row in connection.execute("PRAGMA table_info(sessions)").fetchall()
|
||||
str(row[1]) for row in connection.execute("PRAGMA table_info(sessions)")
|
||||
}
|
||||
required = {
|
||||
"id", "source", "session_key", "chat_type", "display_name",
|
||||
"origin_json", "title",
|
||||
"id",
|
||||
"source",
|
||||
"session_key",
|
||||
"chat_type",
|
||||
"display_name",
|
||||
"origin_json",
|
||||
"title",
|
||||
}
|
||||
if not required.issubset(columns):
|
||||
return 0
|
||||
@ -57,23 +324,38 @@ def migrate(state_database: Path, response_store: Path) -> int:
|
||||
{"platform": "telegram", "session_key": conversation},
|
||||
separators=(",", ":"),
|
||||
)
|
||||
title = "Telegram · General" if conversation == "telegram" else "Telegram"
|
||||
default_title = (
|
||||
"Telegram · General" if conversation == "telegram" else "Telegram"
|
||||
)
|
||||
cursor = connection.execute(
|
||||
"""UPDATE sessions
|
||||
SET session_key = COALESCE(session_key, ?),
|
||||
chat_type = COALESCE(chat_type, 'private'),
|
||||
display_name = COALESCE(display_name, 'Telegram'),
|
||||
origin_json = COALESCE(origin_json, ?),
|
||||
title = COALESCE(title, ?)
|
||||
SET session_key = ?, chat_type = 'private',
|
||||
display_name = 'Telegram', origin_json = ?,
|
||||
title = CASE
|
||||
WHEN title IS NULL OR TRIM(title) = '' OR
|
||||
LOWER(TRIM(title)) = 'unassigned'
|
||||
THEN ? ELSE title END
|
||||
WHERE id = ? AND source = 'api_server'
|
||||
AND (
|
||||
session_key IS NULL OR chat_type IS NULL OR
|
||||
display_name IS NULL OR origin_json IS NULL OR title IS NULL
|
||||
)""",
|
||||
(conversation, origin, title, session_id),
|
||||
AND (COALESCE(session_key, '') != ? OR
|
||||
COALESCE(chat_type, '') != 'private' OR
|
||||
COALESCE(display_name, '') != 'Telegram' OR
|
||||
COALESCE(origin_json, '') != ? OR title IS NULL OR
|
||||
TRIM(title) = '' OR LOWER(TRIM(title)) = 'unassigned')""",
|
||||
(
|
||||
conversation,
|
||||
origin,
|
||||
default_title,
|
||||
session_id,
|
||||
conversation,
|
||||
origin,
|
||||
),
|
||||
)
|
||||
changed += max(cursor.rowcount, 0)
|
||||
deduplicate_stored_messages(connection, list(mappings))
|
||||
connection.commit()
|
||||
except sqlite3.DatabaseError:
|
||||
connection.rollback()
|
||||
return 0
|
||||
finally:
|
||||
connection.close()
|
||||
return changed
|
||||
|
||||
@ -6,15 +6,13 @@ from __future__ import annotations
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
BEFORE = ''' model = body.get("model") or self._model_name
|
||||
BEFORE = """ model = body.get("model") or self._model_name
|
||||
system_prompt = body.get("system_prompt")
|
||||
if system_prompt is not None and not isinstance(system_prompt, str):
|
||||
return web.json_response(_openai_error("system_prompt must be a string", code="invalid_system_prompt"), status=400)
|
||||
db.create_session(session_id, "api_server", model=str(model) if model else None, system_prompt=system_prompt)
|
||||
'''
|
||||
|
||||
AFTER = ''' model = body.get("model") or self._model_name
|
||||
"""
|
||||
AFTER = """ model = body.get("model") or self._model_name
|
||||
system_prompt = body.get("system_prompt")
|
||||
if system_prompt is not None and not isinstance(system_prompt, str):
|
||||
return web.json_response(_openai_error("system_prompt must be a string", code="invalid_system_prompt"), status=400)
|
||||
@ -46,21 +44,18 @@ AFTER = ''' model = body.get("model") or self._model_name
|
||||
system_prompt=system_prompt,
|
||||
parent_session_id=parent_session_id,
|
||||
)
|
||||
'''
|
||||
|
||||
RUNS_BEFORE = ''' run_id = f"run_{uuid.uuid4().hex}"
|
||||
"""
|
||||
RUNS_BEFORE = """ run_id = f"run_{uuid.uuid4().hex}"
|
||||
session_id = body.get("session_id") or stored_session_id or run_id
|
||||
# Approval queues gate host-side tool execution and must be isolated
|
||||
'''
|
||||
|
||||
RESPONSES_SESSION_BEFORE = ''' # Reuse session from previous_response_id chain so the dashboard
|
||||
"""
|
||||
RESPONSES_SESSION_BEFORE = """ # Reuse session from previous_response_id chain so the dashboard
|
||||
# groups the entire conversation under one session entry.
|
||||
session_id = stored_session_id or str(uuid.uuid4())
|
||||
|
||||
# Per-client model routing for /v1/responses (see model_routes).
|
||||
'''
|
||||
|
||||
RESPONSES_SESSION_AFTER = ''' # Reuse session from previous_response_id chain so the dashboard
|
||||
"""
|
||||
RESPONSES_SESSION_AFTER = """ # Reuse session from previous_response_id chain so the dashboard
|
||||
# groups the entire conversation under one session entry.
|
||||
session_id = stored_session_id or str(uuid.uuid4())
|
||||
|
||||
@ -68,9 +63,7 @@ RESPONSES_SESSION_AFTER = ''' # Reuse session from previous_response_id c
|
||||
# WebUI do not collapse it into an anonymous Api_Server session. Keep
|
||||
# the accepted vocabulary narrow: these headers are presentation and
|
||||
# routing metadata, never an authorization boundary.
|
||||
conversation_platform = request.headers.get(
|
||||
"X-Hermes-Conversation-Platform", ""
|
||||
).strip().lower()
|
||||
# X-Hermes-Conversation-Platform was parsed before history compaction.
|
||||
conversation_title = request.headers.get(
|
||||
"X-Hermes-Conversation-Title", ""
|
||||
).strip()
|
||||
@ -117,9 +110,41 @@ RESPONSES_SESSION_AFTER = ''' # Reuse session from previous_response_id c
|
||||
db.set_session_title(session_id, conversation_title)
|
||||
|
||||
# Per-client model routing for /v1/responses (see model_routes).
|
||||
'''
|
||||
"""
|
||||
TRUNCATION_BEFORE = """ # Truncation support
|
||||
if body.get("truncation") == "auto" and len(conversation_history) > 100:
|
||||
conversation_history = conversation_history[-100:]
|
||||
"""
|
||||
TRUNCATION_AFTER = """ # Telegram keeps a durable compact summary plus bounded recent turns.
|
||||
conversation_platform = request.headers.get(
|
||||
"X-Hermes-Conversation-Platform", ""
|
||||
).strip().lower()
|
||||
if conversation_platform == "telegram":
|
||||
from gateway.platforms.telegram_continuity import compact_telegram_history
|
||||
conversation_history = compact_telegram_history(conversation_history)
|
||||
elif body.get("truncation") == "auto" and len(conversation_history) > 100:
|
||||
conversation_history = conversation_history[-100:]
|
||||
"""
|
||||
HISTORY_STORE_BEFORE = """ # Build output items from the current turn only. AIAgent returns a
|
||||
"""
|
||||
HISTORY_STORE_AFTER = """ if conversation_platform == "telegram":
|
||||
full_history = compact_telegram_history(full_history)
|
||||
|
||||
RUNS_AFTER = ''' run_id = f"run_{uuid.uuid4().hex}"
|
||||
# Build output items from the current turn only. AIAgent returns a
|
||||
"""
|
||||
|
||||
SSE_STORE_BEFORE = """ self._response_store.put(response_id, {
|
||||
"""
|
||||
|
||||
SSE_STORE_AFTER = """ if gateway_session_key and gateway_session_key.startswith("telegram"):
|
||||
from gateway.platforms.telegram_continuity import compact_telegram_history
|
||||
conversation_history_snapshot = compact_telegram_history(
|
||||
conversation_history_snapshot
|
||||
)
|
||||
self._response_store.put(response_id, {
|
||||
"""
|
||||
|
||||
RUNS_AFTER = """ run_id = f"run_{uuid.uuid4().hex}"
|
||||
session_id = body.get("session_id") or stored_session_id or run_id
|
||||
|
||||
# Persist API-run lineage before the agent starts. Automated callers
|
||||
@ -183,13 +208,13 @@ RUNS_AFTER = ''' run_id = f"run_{uuid.uuid4().hex}"
|
||||
db.reopen_session(session_id)
|
||||
|
||||
# Approval queues gate host-side tool execution and must be isolated
|
||||
'''
|
||||
"""
|
||||
|
||||
RUN_CLOSE_BEFORE = ''' finally:
|
||||
RUN_CLOSE_BEFORE = """ finally:
|
||||
# If the asyncio wrapper is cancelled (for example via
|
||||
'''
|
||||
"""
|
||||
|
||||
RUN_CLOSE_AFTER = ''' finally:
|
||||
RUN_CLOSE_AFTER = """ finally:
|
||||
# Parent-linked API workers are durable dashboard sessions.
|
||||
# Close them on every terminal run path so the dashboard can
|
||||
# distinguish a long model wait from completed or failed work.
|
||||
@ -218,10 +243,10 @@ RUN_CLOSE_AFTER = ''' finally:
|
||||
)
|
||||
|
||||
# If the asyncio wrapper is cancelled (for example via
|
||||
'''
|
||||
"""
|
||||
|
||||
EVENT_CALLBACK_SIGNATURE_BEFORE = ''' def _make_run_event_callback(self, run_id: str, loop: "asyncio.AbstractEventLoop"):
|
||||
'''
|
||||
EVENT_CALLBACK_SIGNATURE_BEFORE = """ def _make_run_event_callback(self, run_id: str, loop: "asyncio.AbstractEventLoop"):
|
||||
"""
|
||||
|
||||
EVENT_CALLBACK_SIGNATURE_AFTER = ''' _RUN_ACTIVITY_BYTES = 512_000
|
||||
_RUN_ACTIVITY_FILES = 256
|
||||
@ -346,12 +371,12 @@ EVENT_CALLBACK_SIGNATURE_AFTER = ''' _RUN_ACTIVITY_BYTES = 512_000
|
||||
):
|
||||
'''
|
||||
|
||||
EVENT_CALLBACK_BODY_BEFORE = ''' def _callback(event_type: str, tool_name: str = None, preview: str = None, args=None, **kwargs):
|
||||
EVENT_CALLBACK_BODY_BEFORE = """ def _callback(event_type: str, tool_name: str = None, preview: str = None, args=None, **kwargs):
|
||||
ts = time.time()
|
||||
if event_type == "tool.started":
|
||||
'''
|
||||
"""
|
||||
|
||||
EVENT_CALLBACK_BODY_AFTER = ''' def _callback(event_type: str, tool_name: str = None, preview: str = None, args=None, **kwargs):
|
||||
EVENT_CALLBACK_BODY_AFTER = """ def _callback(event_type: str, tool_name: str = None, preview: str = None, args=None, **kwargs):
|
||||
ts = time.time()
|
||||
self._record_run_activity(
|
||||
session_id,
|
||||
@ -361,12 +386,12 @@ EVENT_CALLBACK_BODY_AFTER = ''' def _callback(event_type: str, tool_name:
|
||||
is_error=bool(kwargs.get("is_error", False)),
|
||||
)
|
||||
if event_type == "tool.started":
|
||||
'''
|
||||
"""
|
||||
|
||||
EVENT_CALLBACK_END_BEFORE = ''' # _thinking and subagent_progress are intentionally not forwarded
|
||||
'''
|
||||
EVENT_CALLBACK_END_BEFORE = """ # _thinking and subagent_progress are intentionally not forwarded
|
||||
"""
|
||||
|
||||
EVENT_CALLBACK_END_AFTER = ''' elif event_type in {
|
||||
EVENT_CALLBACK_END_AFTER = """ elif event_type in {
|
||||
"subagent.tool",
|
||||
"subagent.progress",
|
||||
"subagent_progress",
|
||||
@ -386,27 +411,27 @@ EVENT_CALLBACK_END_AFTER = ''' elif event_type in {
|
||||
"tool": tool_name if event_type == "subagent.tool" else None,
|
||||
"preview": safe_preview,
|
||||
})
|
||||
'''
|
||||
"""
|
||||
|
||||
EVENT_CALLBACK_CALL_BEFORE = ''' event_cb = self._make_run_event_callback(run_id, loop)
|
||||
'''
|
||||
EVENT_CALLBACK_CALL_BEFORE = """ event_cb = self._make_run_event_callback(run_id, loop)
|
||||
"""
|
||||
|
||||
EVENT_CALLBACK_CALL_AFTER = ''' event_cb = self._make_run_event_callback(
|
||||
EVENT_CALLBACK_CALL_AFTER = """ event_cb = self._make_run_event_callback(
|
||||
run_id,
|
||||
loop,
|
||||
session_id=session_id,
|
||||
)
|
||||
self._record_run_activity(session_id, "run.started")
|
||||
'''
|
||||
"""
|
||||
|
||||
RUN_SWEEP_BEFORE = ''' self._run_streams.pop(run_id, None)
|
||||
RUN_SWEEP_BEFORE = """ self._run_streams.pop(run_id, None)
|
||||
self._run_streams_created.pop(run_id, None)
|
||||
self._active_run_agents.pop(run_id, None)
|
||||
self._active_run_tasks.pop(run_id, None)
|
||||
self._run_approval_sessions.pop(run_id, None)
|
||||
'''
|
||||
"""
|
||||
|
||||
RUN_SWEEP_AFTER = ''' self._run_streams.pop(run_id, None)
|
||||
RUN_SWEEP_AFTER = """ self._run_streams.pop(run_id, None)
|
||||
self._run_streams_created.pop(run_id, None)
|
||||
# Stream retention and run lifetime are separate. A long run
|
||||
# can legitimately outlive its unconsumed SSE queue; keep the
|
||||
@ -416,7 +441,7 @@ RUN_SWEEP_AFTER = ''' self._run_streams.pop(run_id, None)
|
||||
self._active_run_agents.pop(run_id, None)
|
||||
self._active_run_tasks.pop(run_id, None)
|
||||
self._run_approval_sessions.pop(run_id, None)
|
||||
'''
|
||||
"""
|
||||
|
||||
|
||||
def patch(source: Path, destination: Path) -> None:
|
||||
@ -444,6 +469,9 @@ def patch(source: Path, destination: Path) -> None:
|
||||
content = content.replace(RUNS_BEFORE, RUNS_AFTER, 1)
|
||||
content = content.replace(RUN_CLOSE_BEFORE, RUN_CLOSE_AFTER, 1)
|
||||
content = content.replace(RESPONSES_SESSION_BEFORE, RESPONSES_SESSION_AFTER, 1)
|
||||
content = content.replace(TRUNCATION_BEFORE, TRUNCATION_AFTER, 1)
|
||||
content = content.replace(HISTORY_STORE_BEFORE, HISTORY_STORE_AFTER, 1)
|
||||
content = content.replace(SSE_STORE_BEFORE, SSE_STORE_AFTER, 1)
|
||||
content = content.replace(
|
||||
EVENT_CALLBACK_SIGNATURE_BEFORE,
|
||||
EVENT_CALLBACK_SIGNATURE_AFTER,
|
||||
|
||||
@ -84,7 +84,24 @@
|
||||
"services/hermes/scm-common/scripts/deadline_http.py",
|
||||
"services/hermes/scm-common/scripts/git_pack_objects.py",
|
||||
"services/hermes/scm-common/scripts/receive_pack_scan.py",
|
||||
"services/hermes/scripts/node_polkit_audit.py"
|
||||
"services/hermes/scripts/node_polkit_audit.py",
|
||||
"scripts/ops/hermes_handoff_acceptance.py",
|
||||
"scripts/ops/hermes_handoff_arming.py",
|
||||
"scripts/ops/hermes_handoff_catalog.py",
|
||||
"scripts/ops/hermes_handoff_checks_access.py",
|
||||
"scripts/ops/hermes_handoff_checks_delivery.py",
|
||||
"scripts/ops/hermes_handoff_checks_platform.py",
|
||||
"scripts/ops/hermes_handoff_checks_workers.py",
|
||||
"scripts/ops/hermes_handoff_ephemeral.py",
|
||||
"scripts/ops/hermes_handoff_evaluators.py",
|
||||
"scripts/ops/hermes_handoff_exec.py",
|
||||
"scripts/ops/hermes_handoff_json_rules.py",
|
||||
"scripts/ops/hermes_handoff_model.py",
|
||||
"scripts/ops/hermes_handoff_policy.py",
|
||||
"scripts/ops/hermes_handoff_redaction.py",
|
||||
"scripts/ops/hermes_handoff_rules.py",
|
||||
"scripts/ops/hermes_handoff_run.py",
|
||||
"testing/quality_handoff_mutation.py"
|
||||
],
|
||||
"lint_paths": [
|
||||
"ci/scripts/hermes_image_release.py",
|
||||
@ -140,7 +157,23 @@
|
||||
"services/hermes/scripts/node_account_hardening.py",
|
||||
"services/hermes/scripts/node_account_io.py",
|
||||
"services/hermes/scripts/stage_runtime_access.py",
|
||||
"services/hermes/scripts/node_polkit_audit.py"
|
||||
"services/hermes/scripts/node_polkit_audit.py",
|
||||
"scripts/ops/hermes_handoff_acceptance.py",
|
||||
"scripts/ops/hermes_handoff_arming.py",
|
||||
"scripts/ops/hermes_handoff_catalog.py",
|
||||
"scripts/ops/hermes_handoff_checks_access.py",
|
||||
"scripts/ops/hermes_handoff_checks_delivery.py",
|
||||
"scripts/ops/hermes_handoff_checks_platform.py",
|
||||
"scripts/ops/hermes_handoff_checks_workers.py",
|
||||
"scripts/ops/hermes_handoff_ephemeral.py",
|
||||
"scripts/ops/hermes_handoff_evaluators.py",
|
||||
"scripts/ops/hermes_handoff_exec.py",
|
||||
"scripts/ops/hermes_handoff_json_rules.py",
|
||||
"scripts/ops/hermes_handoff_model.py",
|
||||
"scripts/ops/hermes_handoff_policy.py",
|
||||
"scripts/ops/hermes_handoff_redaction.py",
|
||||
"scripts/ops/hermes_handoff_rules.py",
|
||||
"scripts/ops/hermes_handoff_run.py"
|
||||
],
|
||||
"pytest_suites": {
|
||||
"unit": {
|
||||
@ -160,7 +193,8 @@
|
||||
"services/mailu/scripts",
|
||||
"testing",
|
||||
"services/gitea/scripts",
|
||||
"services/hermes/scm-common/scripts"
|
||||
"services/hermes/scm-common/scripts",
|
||||
"scripts/ops"
|
||||
],
|
||||
"coverage_xml": "build/coverage-unit.xml"
|
||||
},
|
||||
@ -238,7 +272,8 @@
|
||||
"services/hermes/scripts/node_account_hardening.py",
|
||||
"services/hermes/scripts/node_account_io.py",
|
||||
"services/hermes/scripts/stage_runtime_access.py",
|
||||
"services/hermes/scripts/node_polkit_audit.py"
|
||||
"services/hermes/scripts/node_polkit_audit.py",
|
||||
"scripts/ops/hermes_handoff_*.py"
|
||||
],
|
||||
"naming_rules": [
|
||||
{
|
||||
@ -280,6 +315,28 @@
|
||||
},
|
||||
"coverage": {
|
||||
"minimum_percent": 95.0,
|
||||
"minimum_branch_percent": 95.0,
|
||||
"branch_tracked_files": [
|
||||
"scripts/ops/hermes_handoff_acceptance.py",
|
||||
"scripts/ops/hermes_handoff_arming.py",
|
||||
"scripts/ops/hermes_handoff_catalog.py",
|
||||
"scripts/ops/hermes_handoff_checks_access.py",
|
||||
"scripts/ops/hermes_handoff_checks_delivery.py",
|
||||
"scripts/ops/hermes_handoff_checks_platform.py",
|
||||
"scripts/ops/hermes_handoff_checks_workers.py",
|
||||
"scripts/ops/hermes_handoff_ephemeral.py",
|
||||
"scripts/ops/hermes_handoff_evaluators.py",
|
||||
"scripts/ops/hermes_handoff_exec.py",
|
||||
"scripts/ops/hermes_handoff_json_rules.py",
|
||||
"scripts/ops/hermes_handoff_model.py",
|
||||
"scripts/ops/hermes_handoff_policy.py",
|
||||
"scripts/ops/hermes_handoff_redaction.py",
|
||||
"scripts/ops/hermes_handoff_rules.py",
|
||||
"scripts/ops/hermes_handoff_run.py",
|
||||
"ci/scripts/semgrep_report.py",
|
||||
"testing/quality_coverage.py",
|
||||
"testing/quality_handoff_mutation.py"
|
||||
],
|
||||
"tracked_files": [
|
||||
"ci/scripts/hermes_image_release.py",
|
||||
"dockerfiles/hermes-kaniko-heredoc-runner.py",
|
||||
@ -343,7 +400,24 @@
|
||||
"services/hermes/scm-common/scripts/deadline_http.py",
|
||||
"services/hermes/scm-common/scripts/git_pack_objects.py",
|
||||
"services/hermes/scm-common/scripts/receive_pack_scan.py",
|
||||
"services/hermes/scripts/node_polkit_audit.py"
|
||||
"services/hermes/scripts/node_polkit_audit.py",
|
||||
"scripts/ops/hermes_handoff_acceptance.py",
|
||||
"scripts/ops/hermes_handoff_arming.py",
|
||||
"scripts/ops/hermes_handoff_catalog.py",
|
||||
"scripts/ops/hermes_handoff_checks_access.py",
|
||||
"scripts/ops/hermes_handoff_checks_delivery.py",
|
||||
"scripts/ops/hermes_handoff_checks_platform.py",
|
||||
"scripts/ops/hermes_handoff_checks_workers.py",
|
||||
"scripts/ops/hermes_handoff_ephemeral.py",
|
||||
"scripts/ops/hermes_handoff_evaluators.py",
|
||||
"scripts/ops/hermes_handoff_exec.py",
|
||||
"scripts/ops/hermes_handoff_json_rules.py",
|
||||
"scripts/ops/hermes_handoff_model.py",
|
||||
"scripts/ops/hermes_handoff_policy.py",
|
||||
"scripts/ops/hermes_handoff_redaction.py",
|
||||
"scripts/ops/hermes_handoff_rules.py",
|
||||
"scripts/ops/hermes_handoff_run.py",
|
||||
"testing/quality_handoff_mutation.py"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
261
testing/quality_handoff_mutation.py
Normal file
261
testing/quality_handoff_mutation.py
Normal file
@ -0,0 +1,261 @@
|
||||
"""Deterministic scored mutation gate for Hermes handoff safety invariants."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Mutation:
|
||||
"""One source mutation and the isolated probe that must kill it."""
|
||||
|
||||
name: str
|
||||
path: str
|
||||
old: str
|
||||
new: str
|
||||
probe: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MutationReport:
|
||||
"""Scored result; any invalid, surviving, or baseline-broken case is an issue."""
|
||||
|
||||
total: int
|
||||
killed: int
|
||||
issues: tuple[str, ...]
|
||||
|
||||
@property
|
||||
def score(self) -> float:
|
||||
return 100.0 if not self.total else round(self.killed * 100.0 / self.total, 2)
|
||||
|
||||
|
||||
MUTATIONS = (
|
||||
Mutation(
|
||||
"policy-allows-git-fetch",
|
||||
"hermes_handoff_policy.py",
|
||||
' "for-each-ref",\n',
|
||||
' "fetch",\n "for-each-ref",\n',
|
||||
"from hermes_handoff_policy import *\n"
|
||||
"try: check_argv(('git','fetch','origin'))\n"
|
||||
"except PolicyError: raise SystemExit(0)\n"
|
||||
"raise SystemExit(1)\n",
|
||||
),
|
||||
Mutation(
|
||||
"redaction-truncates-before-screening",
|
||||
"hermes_handoff_redaction.py",
|
||||
" return bound(redact(text), max_bytes)\n",
|
||||
" return bound(text, max_bytes)\n",
|
||||
"from hermes_handoff_redaction import safe_text\n"
|
||||
"assert 'A9' not in safe_text('password=A9secret-prefix', 23)[0]\n",
|
||||
),
|
||||
Mutation(
|
||||
"empty-report-go",
|
||||
"hermes_handoff_model.py",
|
||||
" if (\n"
|
||||
" not self.results\n"
|
||||
" or self.invalid_statuses\n"
|
||||
" or self.harness_errors\n"
|
||||
" or self.blocking\n"
|
||||
" ):\n",
|
||||
" if (\n"
|
||||
" self.invalid_statuses\n"
|
||||
" or self.harness_errors\n"
|
||||
" or self.blocking\n"
|
||||
" ):\n",
|
||||
"from hermes_handoff_model import *\nassert Report('read-only','now').decision == NO_GO\n",
|
||||
),
|
||||
Mutation(
|
||||
"generic-404-is-denial",
|
||||
"hermes_handoff_rules.py",
|
||||
"DENIAL_MARKERS = (\n",
|
||||
'DENIAL_MARKERS = (\n "404 not found",\n',
|
||||
"from hermes_handoff_rules import looks_denied\n"
|
||||
"from hermes_handoff_exec import Outcome\n"
|
||||
"assert not looks_denied(Outcome(('x',),'x',1,'','404 not found'))\n",
|
||||
),
|
||||
Mutation(
|
||||
"truncated-evidence-reaches-evaluator",
|
||||
"hermes_handoff_rules.py",
|
||||
" if truncated:\n"
|
||||
" return Evaluation(\n"
|
||||
" NOT_RUN, f\"truncated evidence from step(s): {', '.join(truncated)}\"\n"
|
||||
" )\n",
|
||||
" if False and truncated:\n return Evaluation(NOT_RUN, 'mutant')\n",
|
||||
"import hermes_handoff_evaluators\n"
|
||||
"import hermes_handoff_rules as r\n"
|
||||
"from hermes_handoff_exec import Outcome\n"
|
||||
"from hermes_handoff_model import *\n"
|
||||
"@r.evaluator('probe')\ndef probe(*_): return r.Evaluation(PASS,'bad')\n"
|
||||
"s=CheckSpec('x','x','x','probe')\n"
|
||||
"assert r.evaluate(s,{'x':Outcome(('x',),'x',0,truncated=True)}).status == NOT_RUN\n",
|
||||
),
|
||||
Mutation(
|
||||
"truncated-outcome-is-ok",
|
||||
"hermes_handoff_exec.py",
|
||||
" return self.error is None and self.returncode == 0 and not self.truncated\n",
|
||||
" return self.error is None and self.returncode == 0\n",
|
||||
"from hermes_handoff_exec import Outcome\nassert not Outcome(('x',),'x',0,truncated=True).ok\n",
|
||||
),
|
||||
Mutation(
|
||||
"nan-is-numeric",
|
||||
"hermes_handoff_json_rules.py",
|
||||
" or not math.isfinite(float(actual))\n",
|
||||
" or False\n",
|
||||
"import hermes_handoff_json_rules as j\n"
|
||||
"from hermes_handoff_model import *\nfrom hermes_handoff_exec import Outcome\n"
|
||||
"s=CheckSpec('x','x','x','json_numeric',expect={'step':'s','fields':{'x':{'min':0}}})\n"
|
||||
"j.parsed_step=lambda *_: ({'x':float('nan')},'s')\n"
|
||||
"assert j.evaluate_json_numeric(s,{'s':Outcome(('x',),'x',0)}).status == NOT_RUN\n",
|
||||
),
|
||||
Mutation(
|
||||
"cleanup-ignores-branch-delete-failure",
|
||||
"hermes_handoff_ephemeral.py",
|
||||
' if not deleted.ok:\n problems.append("ephemeral branch deletion failed")\n',
|
||||
' if False and not deleted.ok:\n problems.append("mutant")\n',
|
||||
"import hermes_handoff_ephemeral as e\nfrom hermes_handoff_exec import Outcome\n"
|
||||
"class R:\n"
|
||||
" def __init__(self): self.i=0\n"
|
||||
" def run(self,*_a,**_k):\n"
|
||||
" self.i+=1\n"
|
||||
" return Outcome(('x',),'x',1 if self.i==1 else 0)\n"
|
||||
"q=e.ArmRequest('atlas/titan-iac','origin','acceptance1',e.CONFIRMATION,expected_head='1'*40,expected_base_sha='2'*40)\n"
|
||||
"assert e._cleanup(R(),object(),q,'/api/v1/repos/atlas/titan-iac',[]).status == 'FAIL'\n",
|
||||
),
|
||||
Mutation(
|
||||
"cleanup-accepts-open-pr",
|
||||
"hermes_handoff_ephemeral.py",
|
||||
' elif "state does not match" not in problems:\n problems.append("state is not closed")\n',
|
||||
' elif False:\n problems.append("mutant")\n',
|
||||
"import hermes_handoff_ephemeral as e\n"
|
||||
"q=e.ArmRequest('atlas/titan-iac','origin','acceptance1',e.CONFIRMATION,expected_head='1'*40,expected_base_sha='2'*40)\n"
|
||||
"p={'number':1,'state':'open','draft':True,'merged':False,'base':{'ref':'main','sha':'2'*40},'head':{'ref':q.ref,'sha':'1'*40}}\n"
|
||||
"assert e._closed_pr_problems(p,q)\n",
|
||||
),
|
||||
Mutation(
|
||||
"absence-check-passes-on-zero-evidence",
|
||||
"hermes_handoff_evaluators.py",
|
||||
" if not observed:\n"
|
||||
' return Evaluation(NOT_RUN, f"step {key} exited 0 with no lines to examine")\n',
|
||||
" if False and not observed:\n return Evaluation(NOT_RUN, 'mutant')\n",
|
||||
"import hermes_handoff_evaluators as v\n"
|
||||
"from hermes_handoff_exec import Outcome\nfrom hermes_handoff_model import *\n"
|
||||
"s=CheckSpec('x','x','x','names_absent',steps=(Step('e','self',('kubectl',)),),"
|
||||
"expect={'step':'e','names':('GITEA_TOKEN',)})\n"
|
||||
"assert v.evaluate(s,{'e':Outcome(('x',),'x',0,'')}).status == NOT_RUN\n",
|
||||
),
|
||||
Mutation(
|
||||
"repository-pin-is-a-bare-prefix",
|
||||
"hermes_handoff_policy.py",
|
||||
' and not path.startswith((f"{repo_root}/", f"{repo_root}?"))\n',
|
||||
" and not path.startswith(repo_root)\n",
|
||||
"from hermes_handoff_policy import *\n"
|
||||
"try: check_argv((GITEA_CLIENT,'GET','/api/v1/repos/atlas/titan-iac-evil/pulls'))\n"
|
||||
"except PolicyError: raise SystemExit(0)\n"
|
||||
"raise SystemExit(1)\n",
|
||||
),
|
||||
Mutation(
|
||||
"exec-inner-command-is-not-rechecked",
|
||||
"hermes_handoff_policy.py",
|
||||
" check_argv(inner, mode)\n return\n",
|
||||
" return\n",
|
||||
"from hermes_handoff_policy import *\n"
|
||||
"argv=('kubectl','exec','pod','--','/usr/local/bin/kubectl','get','ns','-o','name',"
|
||||
"'--as','system:admin')\n"
|
||||
"try: check_argv(argv)\n"
|
||||
"except PolicyError: raise SystemExit(0)\n"
|
||||
"raise SystemExit(1)\n",
|
||||
),
|
||||
Mutation(
|
||||
"unrecorded-step-loses-its-binary-attestation",
|
||||
"hermes_handoff_run.py",
|
||||
" executable_path=outcome.executable_path,\n"
|
||||
" executable_sha256=outcome.executable_sha256,\n",
|
||||
"",
|
||||
"import hermes_handoff_run as r\nfrom hermes_handoff_exec import Outcome\n"
|
||||
"from hermes_handoff_model import *\n"
|
||||
"class V:\n name='self'\n"
|
||||
"class N:\n"
|
||||
" def run(self,argv,_v,_b=None): return Outcome(argv,'self',0,'locked',"
|
||||
"executable_path='/usr/bin/git',executable_sha256='a'*64)\n"
|
||||
"s=CheckSpec('x','x','x','stdout_matches',"
|
||||
"steps=(Step('s','self',('git','status'),record=False),),"
|
||||
"expect={'step':'s','equals':'locked'})\n"
|
||||
"o=r.run_check(N(),s,{'self':V()}).outcomes[0]\n"
|
||||
"assert o.executable_sha256 == 'a'*64 and o.executable_path == '/usr/bin/git'\n",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _probe(root: Path, probe: str) -> subprocess.CompletedProcess[str]:
|
||||
environment = {
|
||||
"LC_ALL": "C",
|
||||
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
|
||||
"PYTHONPATH": str(root),
|
||||
}
|
||||
return subprocess.run(
|
||||
[sys.executable, "-c", probe],
|
||||
stdin=subprocess.DEVNULL,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
check=False,
|
||||
env=environment,
|
||||
)
|
||||
|
||||
|
||||
def run_gate(
|
||||
repo_root: Path, mutations: Sequence[Mutation] = MUTATIONS
|
||||
) -> MutationReport:
|
||||
"""Run baseline and mutated probes in isolated copies and return the score."""
|
||||
source_root = repo_root / "scripts" / "ops"
|
||||
issues: list[str] = []
|
||||
killed = 0
|
||||
with tempfile.TemporaryDirectory(prefix="hermes-mutation-") as temporary:
|
||||
clean = Path(temporary) / "clean"
|
||||
shutil.copytree(source_root, clean)
|
||||
for mutation in mutations:
|
||||
source = clean / mutation.path
|
||||
original = source.read_text(encoding="utf-8")
|
||||
if original.count(mutation.old) != 1:
|
||||
issues.append(f"{mutation.name}: mutation anchor count is not one")
|
||||
continue
|
||||
baseline = _probe(clean, mutation.probe)
|
||||
if baseline.returncode != 0:
|
||||
issues.append(f"{mutation.name}: baseline probe failed")
|
||||
continue
|
||||
mutated_text = original.replace(mutation.old, mutation.new, 1)
|
||||
try:
|
||||
compile(mutated_text, str(source), "exec")
|
||||
except SyntaxError:
|
||||
issues.append(f"{mutation.name}: mutant is syntactically invalid")
|
||||
continue
|
||||
mutant = Path(temporary) / f"mutant-{mutation.name}"
|
||||
shutil.copytree(clean, mutant)
|
||||
(mutant / mutation.path).write_text(mutated_text, encoding="utf-8")
|
||||
if _probe(mutant, mutation.probe).returncode != 0:
|
||||
killed += 1
|
||||
else:
|
||||
issues.append(f"{mutation.name}: mutant survived")
|
||||
return MutationReport(len(mutations), killed, tuple(issues))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Run the repository mutation contract and print a bounded score."""
|
||||
report = run_gate(Path(__file__).resolve().parents[1])
|
||||
print(
|
||||
f"Hermes handoff mutation score: {report.killed}/{report.total} ({report.score:.2f}%)"
|
||||
)
|
||||
for issue in report.issues:
|
||||
print(issue)
|
||||
return 0 if not report.issues and report.killed == report.total else 1
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
raise SystemExit(main()) # pragma: no cover
|
||||
@ -1,12 +1,26 @@
|
||||
"""Shared pytest boundaries for repository automation tests."""
|
||||
"""Shared pytest boundaries and runtime-path fixtures for repository tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolate_coordinator_gitea_token(
|
||||
request: pytest.FixtureRequest,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Prevent coordinator tests from observing the mounted production token."""
|
||||
if request.node.path.name == "test_hermes_coordinator.py":
|
||||
monkeypatch.setenv(
|
||||
"HERMES_GITEA_TOKEN_FILE", str(tmp_path / "absent-gitea-token")
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _patched_hermes_cli_db_contract(request, monkeypatch):
|
||||
"""Keep legacy lane test doubles focused on their original safety boundary."""
|
||||
|
||||
255
testing/tests/test_hermes_ai_usage_codex.py
Normal file
255
testing/tests/test_hermes_ai_usage_codex.py
Normal file
@ -0,0 +1,255 @@
|
||||
"""Adversarial tests for bounded Codex quota-query process handling."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from io import StringIO
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "services/hermes/scripts/ai_usage_codex.py"
|
||||
|
||||
|
||||
def load_module():
|
||||
"""Load the standalone Codex query helper as a fresh test module."""
|
||||
name = "ai_usage_codex_tested"
|
||||
spec = importlib.util.spec_from_file_location(name, SCRIPT)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec and spec.loader
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_codex_query_uses_structured_app_server_protocol(tmp_path, monkeypatch):
|
||||
mod = load_module()
|
||||
request_log = tmp_path / "requests.jsonl"
|
||||
mock = tmp_path / "codex-mock"
|
||||
mock.write_text(
|
||||
"""#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
messages = [json.loads(sys.stdin.readline()) for _ in range(4)]
|
||||
with open(os.environ['REQUEST_LOG'], 'w') as handle:
|
||||
for message in messages:
|
||||
handle.write(json.dumps(message) + '\\n')
|
||||
responses = {
|
||||
1: {},
|
||||
2: {'rateLimits': {'primary': {'usedPercent': 10}}},
|
||||
3: {'summary': {}, 'dailyUsageBuckets': []},
|
||||
}
|
||||
for response_id, result in responses.items():
|
||||
print(json.dumps({'id': response_id, 'result': result}), flush=True)
|
||||
"""
|
||||
)
|
||||
mock.chmod(0o755)
|
||||
monkeypatch.setattr(mod, "CODEX_BIN", str(mock))
|
||||
monkeypatch.setenv("REQUEST_LOG", str(request_log))
|
||||
|
||||
rate, usage = mod.query_codex(timeout=5)
|
||||
requests = [json.loads(line) for line in request_log.read_text().splitlines()]
|
||||
|
||||
assert rate["rateLimits"]["primary"]["usedPercent"] == 10
|
||||
assert usage["summary"] == {}
|
||||
assert [request["method"] for request in requests] == [
|
||||
"initialize",
|
||||
"initialized",
|
||||
"account/rateLimits/read",
|
||||
"account/usage/read",
|
||||
]
|
||||
assert all("/status" not in json.dumps(request) for request in requests)
|
||||
|
||||
|
||||
def test_codex_cleanup_is_bounded_when_process_cannot_be_reaped(monkeypatch):
|
||||
mod = load_module()
|
||||
processes = []
|
||||
selectors = []
|
||||
|
||||
class StubbornProcess:
|
||||
def __init__(self, *_args, **_kwargs):
|
||||
self.stdin = StringIO()
|
||||
self.stdout = StringIO(
|
||||
"\n".join(
|
||||
json.dumps({"id": response_id, "result": result})
|
||||
for response_id, result in (
|
||||
(1, {}),
|
||||
(2, {"rateLimits": {}}),
|
||||
(3, {"summary": {}, "dailyUsageBuckets": []}),
|
||||
)
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
self.stderr = None
|
||||
self.terminate_calls = 0
|
||||
self.kill_calls = 0
|
||||
self.wait_timeouts = []
|
||||
processes.append(self)
|
||||
|
||||
def terminate(self):
|
||||
self.terminate_calls += 1
|
||||
|
||||
def kill(self):
|
||||
self.kill_calls += 1
|
||||
|
||||
def wait(self, timeout):
|
||||
self.wait_timeouts.append(timeout)
|
||||
raise subprocess.TimeoutExpired("codex-mock", timeout)
|
||||
|
||||
class TrackingSelector:
|
||||
def __init__(self):
|
||||
self.fileobj = None
|
||||
self.closed = False
|
||||
selectors.append(self)
|
||||
|
||||
def register(self, fileobj, _events):
|
||||
self.fileobj = fileobj
|
||||
|
||||
def select(self, _timeout):
|
||||
return [(SimpleNamespace(fileobj=self.fileobj), None)]
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
monkeypatch.setattr(mod.subprocess, "Popen", StubbornProcess)
|
||||
monkeypatch.setattr(mod.selectors, "DefaultSelector", TrackingSelector)
|
||||
|
||||
with pytest.raises(RuntimeError, match="^Codex app-server cleanup failed$"):
|
||||
mod.query_codex(timeout=1)
|
||||
|
||||
process = processes[0]
|
||||
assert process.terminate_calls == 1
|
||||
assert process.kill_calls == 1
|
||||
assert process.wait_timeouts == [5, 5]
|
||||
assert process.stdin.closed
|
||||
assert process.stdout.closed
|
||||
assert selectors[0].closed
|
||||
|
||||
|
||||
def test_cleanup_contains_all_pipe_and_process_api_failures():
|
||||
mod = load_module()
|
||||
|
||||
class BrokenStream:
|
||||
def close(self):
|
||||
raise OSError("private pipe detail")
|
||||
|
||||
class BrokenSelector:
|
||||
def close(self):
|
||||
raise OSError("private selector detail")
|
||||
|
||||
class BrokenProcess:
|
||||
stdin = BrokenStream()
|
||||
stdout = BrokenStream()
|
||||
stderr = BrokenStream()
|
||||
|
||||
def __init__(self):
|
||||
self.wait_calls = 0
|
||||
|
||||
def terminate(self):
|
||||
raise OSError("private terminate detail")
|
||||
|
||||
def wait(self, timeout):
|
||||
assert timeout == mod.CODEX_CLEANUP_TIMEOUT_SECONDS
|
||||
self.wait_calls += 1
|
||||
raise OSError("private wait detail")
|
||||
|
||||
def kill(self):
|
||||
raise OSError("private kill detail")
|
||||
|
||||
assert mod._close_stream(None)
|
||||
assert not mod._close_stream(BrokenStream())
|
||||
assert not mod._cleanup_codex_process(BrokenProcess(), BrokenSelector())
|
||||
|
||||
|
||||
def test_cleanup_accepts_already_exited_process_during_terminate_and_kill():
|
||||
mod = load_module()
|
||||
|
||||
class AlreadyExited:
|
||||
stdin = None
|
||||
stdout = None
|
||||
stderr = None
|
||||
|
||||
def __init__(self, *, timeout_first):
|
||||
self.timeout_first = timeout_first
|
||||
self.wait_calls = 0
|
||||
|
||||
def terminate(self):
|
||||
raise ProcessLookupError
|
||||
|
||||
def wait(self, timeout):
|
||||
self.wait_calls += 1
|
||||
if self.timeout_first and self.wait_calls == 1:
|
||||
raise subprocess.TimeoutExpired("codex", timeout)
|
||||
|
||||
def kill(self):
|
||||
raise ProcessLookupError
|
||||
|
||||
assert mod._cleanup_codex_process(AlreadyExited(timeout_first=False), None)
|
||||
assert mod._cleanup_codex_process(AlreadyExited(timeout_first=True), None)
|
||||
|
||||
|
||||
def test_query_rejects_missing_pipes_and_structured_errors(monkeypatch):
|
||||
mod = load_module()
|
||||
|
||||
class Process:
|
||||
stderr = None
|
||||
|
||||
def __init__(self, *_args, **_kwargs):
|
||||
self.stdin = None
|
||||
self.stdout = None
|
||||
|
||||
def terminate(self):
|
||||
return None
|
||||
|
||||
def wait(self, timeout):
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(mod.subprocess, "Popen", Process)
|
||||
with pytest.raises(RuntimeError, match="pipes are unavailable"):
|
||||
mod.query_codex(timeout=0)
|
||||
|
||||
class InvalidProcess(Process):
|
||||
def __init__(self, *_args, **_kwargs):
|
||||
self.stdin = StringIO()
|
||||
self.stdout = SequencedOutput()
|
||||
|
||||
class SequencedOutput:
|
||||
def __init__(self):
|
||||
self.lines = iter(
|
||||
[
|
||||
"",
|
||||
json.dumps({"id": 99, "result": {}}) + "\n",
|
||||
json.dumps({"id": 1, "result": {}}) + "\n",
|
||||
json.dumps({"id": 2, "error": {"message": "secret"}}) + "\n",
|
||||
json.dumps({"id": 3, "result": {}}) + "\n",
|
||||
]
|
||||
)
|
||||
|
||||
def readline(self):
|
||||
return next(self.lines)
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
class Selector:
|
||||
def register(self, fileobj, _events):
|
||||
self.fileobj = fileobj
|
||||
|
||||
def select(self, _timeout):
|
||||
return [(SimpleNamespace(fileobj=self.fileobj), None)]
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(mod.subprocess, "Popen", InvalidProcess)
|
||||
monkeypatch.setattr(mod.selectors, "DefaultSelector", Selector)
|
||||
with pytest.raises(
|
||||
RuntimeError, match="^Codex app-server response 2 failed$"
|
||||
) as caught:
|
||||
mod.query_codex(timeout=1)
|
||||
assert "secret" not in str(caught.value)
|
||||
@ -3,9 +3,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from datetime import date
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@ -15,6 +15,9 @@ SCRIPT = ROOT / "services/hermes/scripts/ai_usage_exporter.py"
|
||||
|
||||
def load_module():
|
||||
"""Load the standalone exporter script as a test module."""
|
||||
script_directory = str(SCRIPT.parent)
|
||||
if script_directory not in sys.path:
|
||||
sys.path.insert(0, script_directory)
|
||||
spec = importlib.util.spec_from_file_location("ai_usage_exporter", SCRIPT)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec and spec.loader
|
||||
@ -54,7 +57,9 @@ def test_codex_payloads_include_real_windows_and_bounded_token_periods():
|
||||
}
|
||||
|
||||
samples = mod.parse_codex_payloads(rate, usage, today=date(2026, 8, 16))
|
||||
values = {(item.name, tuple(sorted(item.labels.items()))): item.value for item in samples}
|
||||
values = {
|
||||
(item.name, tuple(sorted(item.labels.items()))): item.value for item in samples
|
||||
}
|
||||
|
||||
codex_labels = (
|
||||
("limit", "codex"),
|
||||
@ -69,18 +74,24 @@ def test_codex_payloads_include_real_windows_and_bounded_token_periods():
|
||||
assert values[("atlas_ai_quota_used_percent", codex_labels)] == 41
|
||||
assert values[("atlas_ai_quota_remaining_percent", codex_labels)] == 59
|
||||
assert values[("atlas_ai_quota_used_percent", spark_labels)] == 0
|
||||
assert values[
|
||||
(
|
||||
"atlas_ai_account_tokens",
|
||||
(("period", "latest_day"), ("provider", "openai")),
|
||||
)
|
||||
] == 100
|
||||
assert values[
|
||||
(
|
||||
"atlas_ai_account_tokens",
|
||||
(("period", "seven_day"), ("provider", "openai")),
|
||||
)
|
||||
] == 150
|
||||
assert (
|
||||
values[
|
||||
(
|
||||
"atlas_ai_account_tokens",
|
||||
(("period", "latest_day"), ("provider", "openai")),
|
||||
)
|
||||
]
|
||||
== 100
|
||||
)
|
||||
assert (
|
||||
values[
|
||||
(
|
||||
"atlas_ai_account_tokens",
|
||||
(("period", "seven_day"), ("provider", "openai")),
|
||||
)
|
||||
]
|
||||
== 150
|
||||
)
|
||||
|
||||
|
||||
def test_claude_payload_uses_only_supported_quota_fields():
|
||||
@ -131,45 +142,161 @@ def test_render_reports_failure_and_freshness_without_logging_credentials():
|
||||
rendered = collector.render().decode()
|
||||
|
||||
assert 'atlas_ai_quota_fetch_success{provider="openai"} 0' in rendered
|
||||
assert 'atlas_ai_quota_last_success_timestamp_seconds{provider="openai"} 100' in rendered
|
||||
assert 'atlas_ai_quota_used_percent{limit="codex",provider="openai",window="seven_day"} 42' in rendered
|
||||
assert (
|
||||
'atlas_ai_quota_last_success_timestamp_seconds{provider="openai"} 100'
|
||||
in rendered
|
||||
)
|
||||
assert (
|
||||
'atlas_ai_quota_used_percent{limit="codex",provider="openai",window="seven_day"} 42'
|
||||
in rendered
|
||||
)
|
||||
assert "accessToken" not in rendered
|
||||
assert "refreshToken" not in rendered
|
||||
|
||||
|
||||
def test_codex_query_uses_structured_app_server_protocol(tmp_path, monkeypatch):
|
||||
def test_unexpected_provider_failure_is_sanitized_and_preserves_last_good(
|
||||
monkeypatch, capsys
|
||||
):
|
||||
mod = load_module()
|
||||
request_log = tmp_path / "requests.jsonl"
|
||||
mock = tmp_path / "codex-mock"
|
||||
mock.write_text(
|
||||
"""#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
messages = [json.loads(sys.stdin.readline()) for _ in range(4)]
|
||||
with open(os.environ['REQUEST_LOG'], 'w') as handle:
|
||||
for message in messages:
|
||||
handle.write(json.dumps(message) + '\\n')
|
||||
responses = {
|
||||
1: {},
|
||||
2: {'rateLimits': {'primary': {'usedPercent': 10}}},
|
||||
3: {'summary': {}, 'dailyUsageBuckets': []},
|
||||
}
|
||||
for response_id, result in responses.items():
|
||||
print(json.dumps({'id': response_id, 'result': result}), flush=True)
|
||||
"""
|
||||
collector = mod.Collector()
|
||||
previous = mod.Sample(
|
||||
"atlas_ai_quota_used_percent",
|
||||
{"provider": "openai", "limit": "codex", "window": "seven_day"},
|
||||
42,
|
||||
)
|
||||
mock.chmod(0o755)
|
||||
monkeypatch.setattr(mod, "CODEX_BIN", str(mock))
|
||||
monkeypatch.setenv("REQUEST_LOG", str(request_log))
|
||||
collector._providers["openai"].samples = [previous]
|
||||
secret = "provider-body-must-not-appear"
|
||||
|
||||
rate, usage = mod.query_codex(timeout=5)
|
||||
requests = [json.loads(line) for line in request_log.read_text().splitlines()]
|
||||
class UnexpectedProviderFailure(Exception):
|
||||
pass
|
||||
|
||||
assert rate["rateLimits"]["primary"]["usedPercent"] == 10
|
||||
assert usage["summary"] == {}
|
||||
assert [request["method"] for request in requests] == [
|
||||
"initialize",
|
||||
"initialized",
|
||||
"account/rateLimits/read",
|
||||
"account/usage/read",
|
||||
]
|
||||
assert all("/status" not in json.dumps(request) for request in requests)
|
||||
def fail():
|
||||
raise UnexpectedProviderFailure(secret)
|
||||
|
||||
monkeypatch.setattr(mod, "query_codex", fail)
|
||||
collector.refresh_provider("openai")
|
||||
|
||||
output = capsys.readouterr().out
|
||||
state = collector._providers["openai"]
|
||||
assert state.samples == [previous]
|
||||
assert state.fetch_success is False
|
||||
assert state.last_attempt > 0
|
||||
assert "UnexpectedProviderFailure" in output
|
||||
assert secret not in output
|
||||
|
||||
|
||||
def test_polling_engine_isolates_providers_and_future_cycles(monkeypatch, capsys):
|
||||
mod = load_module()
|
||||
collector = mod.Collector()
|
||||
calls = []
|
||||
secret = "cleanup-detail-must-not-appear"
|
||||
|
||||
class UnexpectedCleanupFailure(Exception):
|
||||
pass
|
||||
|
||||
def refresh(provider):
|
||||
calls.append(provider)
|
||||
if provider == "openai" and calls.count("openai") == 1:
|
||||
raise UnexpectedCleanupFailure(secret)
|
||||
|
||||
monkeypatch.setattr(collector, "refresh_provider", refresh)
|
||||
engine = mod.PollingEngine(collector, interval=60)
|
||||
|
||||
engine.poll_once()
|
||||
engine.poll_once()
|
||||
|
||||
output = capsys.readouterr().out
|
||||
assert calls == ["openai", "anthropic", "openai", "anthropic"]
|
||||
assert collector._providers["openai"].fetch_success is False
|
||||
assert collector._providers["openai"].last_attempt > 0
|
||||
assert "UnexpectedCleanupFailure" in output
|
||||
assert secret not in output
|
||||
|
||||
|
||||
def test_health_tracks_poller_liveness_and_progress_not_provider_success():
|
||||
mod = load_module()
|
||||
now = [100.0]
|
||||
collector = mod.Collector()
|
||||
collector._providers["openai"].fetch_success = False
|
||||
engine = mod.PollingEngine(
|
||||
collector,
|
||||
interval=60,
|
||||
startup_grace=10,
|
||||
progress_timeout=75,
|
||||
clock=lambda: now[0],
|
||||
)
|
||||
|
||||
class ThreadState:
|
||||
alive = True
|
||||
|
||||
def is_alive(self):
|
||||
return self.alive
|
||||
|
||||
thread = ThreadState()
|
||||
engine._thread = thread
|
||||
assert engine.is_healthy()
|
||||
|
||||
now[0] = 111.0
|
||||
assert not engine.is_healthy()
|
||||
|
||||
engine._mark_progress()
|
||||
assert engine.is_healthy()
|
||||
now[0] = 187.0
|
||||
assert not engine.is_healthy()
|
||||
|
||||
thread.alive = False
|
||||
now[0] = 100.0
|
||||
assert not engine.is_healthy()
|
||||
|
||||
|
||||
def test_health_endpoint_uses_poller_state_and_ignores_provider_failure():
|
||||
mod = load_module()
|
||||
collector = mod.Collector()
|
||||
|
||||
class Poller:
|
||||
healthy = False
|
||||
|
||||
def is_healthy(self):
|
||||
return self.healthy
|
||||
|
||||
poller = Poller()
|
||||
handler_class = mod.make_handler(collector, poller)
|
||||
handler = object.__new__(handler_class)
|
||||
handler.path = "/healthz"
|
||||
statuses = []
|
||||
handler.send_response = statuses.append
|
||||
handler.send_header = lambda _name, _value: None
|
||||
handler.end_headers = lambda: None
|
||||
handler.wfile = BytesIO()
|
||||
|
||||
handler.do_GET()
|
||||
assert statuses == [503]
|
||||
|
||||
poller.healthy = True
|
||||
statuses.clear()
|
||||
collector._providers["openai"].fetch_success = False
|
||||
handler.do_GET()
|
||||
assert statuses == [200]
|
||||
|
||||
|
||||
def test_metrics_response_tolerates_disconnected_client():
|
||||
mod = load_module()
|
||||
collector = mod.Collector()
|
||||
|
||||
class HealthyPoller:
|
||||
def is_healthy(self):
|
||||
return True
|
||||
|
||||
class Disconnected(BytesIO):
|
||||
def write(self, _body):
|
||||
raise BrokenPipeError
|
||||
|
||||
handler_class = mod.make_handler(collector, HealthyPoller())
|
||||
handler = object.__new__(handler_class)
|
||||
handler.path = "/metrics"
|
||||
handler.send_response = lambda _status: None
|
||||
handler.send_header = lambda _name, _value: None
|
||||
handler.end_headers = lambda: None
|
||||
handler.wfile = Disconnected()
|
||||
|
||||
handler.do_GET()
|
||||
|
||||
275
testing/tests/test_hermes_ai_usage_exporter_coverage.py
Normal file
275
testing/tests/test_hermes_ai_usage_exporter_coverage.py
Normal file
@ -0,0 +1,275 @@
|
||||
"""Focused edge coverage for AI usage exporter provider and polling boundaries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "services/hermes/scripts/ai_usage_exporter.py"
|
||||
|
||||
|
||||
def load_module():
|
||||
"""Load a fresh exporter while making its sibling modules importable."""
|
||||
script_directory = str(SCRIPT.parent)
|
||||
if script_directory not in sys.path:
|
||||
sys.path.insert(0, script_directory)
|
||||
name = "ai_usage_exporter_coverage"
|
||||
spec = importlib.util.spec_from_file_location(name, SCRIPT)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec and spec.loader
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_parser_rejects_nonfinite_and_malformed_optional_fields():
|
||||
mod = load_module()
|
||||
|
||||
assert mod._number(True) is None
|
||||
assert mod._number("1") is None
|
||||
assert mod._number(float("nan")) is None
|
||||
assert mod._number(float("inf")) is None
|
||||
assert mod._timestamp(None) is None
|
||||
assert mod._timestamp("not-a-time") is None
|
||||
assert mod._window_name({}, "fallback") == "fallback"
|
||||
assert mod._codex_limit_name("other", {}) == "additional"
|
||||
|
||||
samples = mod.parse_codex_payloads(
|
||||
{
|
||||
"rateLimitsByLimitId": {
|
||||
1: {},
|
||||
"not-a-snapshot": "invalid",
|
||||
"unnamed": {
|
||||
"primary": {"usedPercent": None},
|
||||
"secondary": {
|
||||
"usedPercent": 120,
|
||||
"resetsAt": "invalid",
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
{
|
||||
"summary": {
|
||||
"lifetimeTokens": None,
|
||||
"currentStreakDays": 2,
|
||||
"longestStreakDays": 3,
|
||||
"longestRunningTurnSec": 4,
|
||||
},
|
||||
"dailyUsageBuckets": [
|
||||
None,
|
||||
{"startDate": "invalid", "tokens": 2},
|
||||
{"startDate": "2026-08-17", "tokens": None},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert any(sample.value == 120 for sample in samples)
|
||||
assert not any(
|
||||
sample.name == "atlas_ai_quota_reset_timestamp_seconds" for sample in samples
|
||||
)
|
||||
assert mod.parse_codex_payloads({"rateLimits": None}, {"summary": None})
|
||||
|
||||
|
||||
def test_claude_parser_ignores_unbounded_fields_and_invalid_extra_usage():
|
||||
mod = load_module()
|
||||
|
||||
assert mod.parse_claude_payload(
|
||||
{
|
||||
"five_hour": {"utilization": None},
|
||||
"seven_day": {"utilization": 10, "resets_at": None},
|
||||
"extra_usage": {"is_enabled": "yes"},
|
||||
}
|
||||
)
|
||||
assert mod.parse_claude_payload({"extra_usage": "invalid"}) == []
|
||||
|
||||
|
||||
def test_provider_queries_validate_credentials_and_response_shape(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
mod = load_module()
|
||||
credentials = tmp_path / "credentials.json"
|
||||
credentials.write_text("{}", encoding="utf-8")
|
||||
monkeypatch.setattr(mod, "CLAUDE_CREDENTIALS", credentials)
|
||||
|
||||
with pytest.raises(RuntimeError, match="credentials are incomplete"):
|
||||
mod.query_claude()
|
||||
|
||||
credentials.write_text(
|
||||
json.dumps({"claudeAiOauth": {"accessToken": "runtime-only-token"}}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
bodies = iter([{"five_hour": {}}, []])
|
||||
requests = []
|
||||
|
||||
class Response:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
def read(self, maximum):
|
||||
assert maximum == 1 << 20
|
||||
return json.dumps(next(bodies)).encode()
|
||||
|
||||
def open_request(request, timeout):
|
||||
requests.append(request)
|
||||
assert timeout == 15
|
||||
return Response()
|
||||
|
||||
monkeypatch.setattr(mod, "urlopen", open_request)
|
||||
assert mod.query_claude() == {"five_hour": {}}
|
||||
assert requests[0].get_header("Authorization") == "Bearer runtime-only-token"
|
||||
with pytest.raises(RuntimeError, match="response is not an object"):
|
||||
mod.query_claude()
|
||||
|
||||
assert mod.query_codex is mod.codex_query.query_codex
|
||||
|
||||
|
||||
def test_collector_refreshes_both_providers_and_contains_unknown_provider(
|
||||
monkeypatch, capsys
|
||||
):
|
||||
mod = load_module()
|
||||
collector = mod.Collector()
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"query_codex",
|
||||
lambda: ({"rateLimits": {"primary": {"usedPercent": 1}}}, {}),
|
||||
)
|
||||
monkeypatch.setattr(mod, "query_claude", lambda: {"five_hour": {"utilization": 2}})
|
||||
|
||||
collector.refresh_provider("openai")
|
||||
collector.refresh_provider("anthropic")
|
||||
assert collector._providers["openai"].fetch_success
|
||||
assert collector._providers["anthropic"].fetch_success
|
||||
assert collector._providers["openai"].last_success > 0
|
||||
|
||||
collector.record_failure("openai")
|
||||
assert not collector._providers["openai"].fetch_success
|
||||
with pytest.raises(KeyError, match="unknown"):
|
||||
collector.refresh_provider("unknown")
|
||||
assert "unknown quota collection deferred: ValueError" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_polling_failure_accounting_and_cycle_errors_are_sanitized(monkeypatch, capsys):
|
||||
mod = load_module()
|
||||
secret = "must-not-leak"
|
||||
|
||||
class Failure(Exception):
|
||||
pass
|
||||
|
||||
class Collector:
|
||||
def refresh_provider(self, _provider):
|
||||
raise Failure(secret)
|
||||
|
||||
def record_failure(self, *_args, **_kwargs):
|
||||
raise Failure(secret)
|
||||
|
||||
engine = mod.PollingEngine(Collector(), interval=60)
|
||||
engine.poll_once()
|
||||
output = capsys.readouterr().out
|
||||
assert "quota failure accounting deferred: Failure" in output
|
||||
assert secret not in output
|
||||
|
||||
monkeypatch.setattr(
|
||||
engine, "poll_once", lambda: (_ for _ in ()).throw(Failure(secret))
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
mod.time, "sleep", lambda _interval: (_ for _ in ()).throw(StopIteration)
|
||||
)
|
||||
with pytest.raises(StopIteration):
|
||||
engine.run()
|
||||
output = capsys.readouterr().out
|
||||
assert "quota polling cycle deferred: Failure" in output
|
||||
assert secret not in output
|
||||
|
||||
|
||||
def test_poller_start_is_idempotent_and_main_wires_server(monkeypatch):
|
||||
mod = load_module()
|
||||
threads = []
|
||||
|
||||
class Thread:
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
self.started = False
|
||||
threads.append(self)
|
||||
|
||||
def start(self):
|
||||
self.started = True
|
||||
|
||||
monkeypatch.setattr(mod.threading, "Thread", Thread)
|
||||
engine = mod.PollingEngine(mod.Collector(), interval=60)
|
||||
engine.start()
|
||||
engine.start()
|
||||
assert len(threads) == 1
|
||||
assert threads[0].started
|
||||
assert threads[0].kwargs["daemon"] is True
|
||||
|
||||
events = []
|
||||
|
||||
class Poller:
|
||||
def __init__(self, collector, *, interval):
|
||||
events.append(("poller", collector, interval))
|
||||
|
||||
def start(self):
|
||||
events.append("started")
|
||||
|
||||
class Server:
|
||||
def __init__(self, address, handler):
|
||||
events.append(("server", address, handler))
|
||||
|
||||
def serve_forever(self):
|
||||
events.append("served")
|
||||
|
||||
monkeypatch.setattr(mod, "PollingEngine", Poller)
|
||||
monkeypatch.setattr(mod, "Server", Server)
|
||||
monkeypatch.setattr(
|
||||
mod, "make_handler", lambda collector, poller: (collector, poller)
|
||||
)
|
||||
monkeypatch.setenv("ATLAS_AI_USAGE_INTERVAL_SECONDS", "1")
|
||||
monkeypatch.setenv("ATLAS_AI_USAGE_PORT", "19010")
|
||||
|
||||
assert mod.main() == 0
|
||||
assert events[0][0] == "poller"
|
||||
assert events[0][2] == 60
|
||||
assert events[2][1][1] == 19010
|
||||
assert events[-1] == "served"
|
||||
|
||||
|
||||
def test_http_handler_covers_not_found_logging_and_connection_reset():
|
||||
mod = load_module()
|
||||
|
||||
class Collector:
|
||||
def render(self):
|
||||
return b"metrics\n"
|
||||
|
||||
class Poller:
|
||||
def is_healthy(self):
|
||||
return True
|
||||
|
||||
handler_class = mod.make_handler(Collector(), Poller())
|
||||
handler = object.__new__(handler_class)
|
||||
handler.path = "/missing"
|
||||
statuses = []
|
||||
handler.send_error = statuses.append
|
||||
handler.do_GET()
|
||||
assert statuses == [404]
|
||||
assert handler.log_message("ignored") is None
|
||||
|
||||
class Reset(BytesIO):
|
||||
def write(self, _body):
|
||||
raise ConnectionResetError
|
||||
|
||||
handler.path = "/metrics"
|
||||
handler.send_response = lambda _status: None
|
||||
handler.send_header = lambda _name, _value: None
|
||||
handler.end_headers = lambda: None
|
||||
handler.wfile = Reset()
|
||||
handler.do_GET()
|
||||
@ -255,7 +255,12 @@ def test_provider_status_separates_observed_activity_from_plan_quota(monkeypatch
|
||||
monkeypatch.setattr(module, "_claude_account", lambda: {
|
||||
"authenticated": True, "plan": "max", "quota_reported": False,
|
||||
})
|
||||
monkeypatch.setattr(module, "_fresh_health", lambda _path: {})
|
||||
monkeypatch.setattr(module, "_fresh_health", lambda _path, maximum_age=86400.0: {})
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"_configured_models",
|
||||
lambda provider: ["gpt-5.6-sol"] if provider == "codex" else ["claude-sonnet-5"],
|
||||
)
|
||||
|
||||
payload = module.provider_status_payload()
|
||||
|
||||
|
||||
@ -305,7 +305,7 @@ def test_webui_recovers_auth_and_labels_session_scoped_controls():
|
||||
assert "session_key.startswith('telegram-topic-')" in telegram_project_patch
|
||||
|
||||
router = (ROOT / "dockerfiles" / "hermes-webui-router.js").read_text()
|
||||
assert "'atlas/auto/fast':'AUTO · Fast'" in router
|
||||
assert "'atlas/auto/fast':'Automatic · Fast'" in router
|
||||
assert "'atlas/manual/codex/sol':'Codex · SOL'" in router
|
||||
assert "'atlas/manual/claude/opus':'Claude · Opus'" in router
|
||||
assert "watchModelOptions('modelSelect')" in router
|
||||
|
||||
493
testing/tests/test_hermes_chat_smoothness.py
Normal file
493
testing/tests/test_hermes_chat_smoothness.py
Normal file
@ -0,0 +1,493 @@
|
||||
"""Adversarial contracts for chat-only continuity and truthful rendering."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import runpy
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
HERMES = ROOT / "services" / "hermes"
|
||||
|
||||
|
||||
def _load(name: str, path: Path):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def continuity():
|
||||
return _load(
|
||||
"telegram_continuity",
|
||||
HERMES / "scripts" / "migrate_telegram_api_sessions.py",
|
||||
)
|
||||
|
||||
|
||||
def test_context_is_deterministic_bounded_deduplicated_and_truthful(continuity):
|
||||
turns = []
|
||||
for index in range(30):
|
||||
user = {"role": "user", "content": f"question {index}", "_db_persisted": True}
|
||||
assistant = {
|
||||
"role": "assistant",
|
||||
"content": f"answer {index}",
|
||||
"_db_persisted": True,
|
||||
}
|
||||
turns.extend((user, assistant))
|
||||
if index == 4:
|
||||
# Simulate one complete transcript replay.
|
||||
turns.extend((user.copy(), assistant.copy()))
|
||||
turns.extend(
|
||||
[
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call-1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "browser",
|
||||
"arguments": "x" * 40_000,
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call-1",
|
||||
"tool_name": "browser",
|
||||
"content": "tool output " * 4_000,
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "edit this"},
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64," + "a" * 80_000,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Created it MEDIA:/opt/data/cache/images/private.png",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
compacted = continuity.compact_telegram_history(turns)
|
||||
|
||||
assert compacted == continuity.compact_telegram_history(turns)
|
||||
assert compacted[0]["role"] == "system"
|
||||
assert compacted[0]["_db_persisted"] is True
|
||||
assert compacted[0]["content"].startswith(continuity.SUMMARY_PREFIX)
|
||||
assert sum(item["role"] == "user" for item in compacted[1:]) <= 12
|
||||
assert len(compacted[1:]) <= continuity.MAX_HISTORY_ITEMS
|
||||
assert len(json.dumps(compacted).encode()) <= (
|
||||
continuity.MAX_HISTORY_BYTES + continuity.MAX_SUMMARY_CHARS
|
||||
)
|
||||
encoded = json.dumps(compacted)
|
||||
assert "data:image" not in encoded
|
||||
assert "MEDIA:" not in encoded
|
||||
assert "/opt/data/" not in encoded
|
||||
assert continuity.ATTACHMENT_NOTE in encoded
|
||||
assert "question 29" in encoded
|
||||
assert (
|
||||
next(item for item in compacted if item.get("content") == "question 29")[
|
||||
"_db_persisted"
|
||||
]
|
||||
is True
|
||||
)
|
||||
assert "x" * 3_000 not in encoded
|
||||
fingerprints = [continuity._fingerprint(item) for item in compacted]
|
||||
assert all(
|
||||
left != right
|
||||
for left, right in zip(fingerprints, fingerprints[1:], strict=False)
|
||||
)
|
||||
|
||||
|
||||
def test_context_rejects_malformed_history_and_bounds_single_large_item(continuity):
|
||||
assert continuity.compact_telegram_history(None) == []
|
||||
assert continuity.compact_telegram_history({"role": "user"}) == []
|
||||
malformed = [
|
||||
None,
|
||||
"text",
|
||||
{"role": "unknown", "content": "skip"},
|
||||
{"role": "tool", "content": ""},
|
||||
{"role": "user", "content": {"nested": "value"}},
|
||||
{"role": "assistant", "content": "z" * 2_000_000},
|
||||
]
|
||||
compacted = continuity.compact_telegram_history(malformed)
|
||||
assert len(compacted) == 2
|
||||
assert compacted[0]["content"] == '{"nested": "value"}'
|
||||
assert len(compacted[1]["content"]) == continuity.MAX_ITEM_CHARS
|
||||
|
||||
|
||||
def test_context_defensive_normalization_and_replay_edges(continuity, monkeypatch):
|
||||
assert continuity._text(["skip", {"type": "text", "text": "keep"}]) == "keep"
|
||||
assert continuity._text({"not-json-serializable"}) == "{'not-json-serializable'}"
|
||||
normalized = continuity._normalized_item(
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
None,
|
||||
{"id": "call", "function": {"name": "browser", "arguments": {1}}},
|
||||
],
|
||||
}
|
||||
)
|
||||
assert normalized and normalized["tool_calls"][0]["id"] == "call"
|
||||
assert continuity._fingerprint({"role": "user", "content": {1}}).startswith(
|
||||
"{'role':"
|
||||
)
|
||||
|
||||
user = {"role": "user", "content": "same"}
|
||||
assistant = {"role": "assistant", "content": "reply"}
|
||||
assert continuity._deduplicate(
|
||||
[user, assistant, user.copy(), assistant.copy()]
|
||||
) == [
|
||||
user,
|
||||
assistant,
|
||||
]
|
||||
assert continuity._deduplicate(
|
||||
[
|
||||
{"role": "tool", "content": "first", "id": "stable"},
|
||||
{"role": "tool", "content": "replay", "id": "stable"},
|
||||
]
|
||||
) == [{"role": "tool", "content": "first", "id": "stable"}]
|
||||
|
||||
summary = continuity._summary(
|
||||
[
|
||||
{
|
||||
"role": "system",
|
||||
"content": continuity.SUMMARY_PREFIX + "\n- existing fact",
|
||||
},
|
||||
{"role": "tool", "content": "not durable prose"},
|
||||
]
|
||||
)
|
||||
assert summary and "existing fact" in summary["content"]
|
||||
monkeypatch.setattr(
|
||||
continuity, "MAX_SUMMARY_CHARS", len(continuity.SUMMARY_PREFIX) + 1
|
||||
)
|
||||
assert (
|
||||
continuity._summary([{"role": "user", "content": "too long"}])["content"]
|
||||
== continuity.SUMMARY_PREFIX
|
||||
)
|
||||
|
||||
|
||||
def test_context_final_serialized_size_guard(continuity, monkeypatch):
|
||||
monkeypatch.setattr(continuity, "MAX_HISTORY_ITEMS", 2)
|
||||
monkeypatch.setattr(continuity, "MAX_HISTORY_BYTES", 1_000)
|
||||
monkeypatch.setattr(
|
||||
continuity,
|
||||
"_summary",
|
||||
lambda _older: {
|
||||
"role": "system",
|
||||
"content": "s" * 20_000,
|
||||
"_db_persisted": True,
|
||||
},
|
||||
)
|
||||
compacted = continuity.compact_telegram_history(
|
||||
[
|
||||
{"role": "user", "content": "older"},
|
||||
{"role": "assistant", "content": "recent one"},
|
||||
{"role": "assistant", "content": "recent two"},
|
||||
]
|
||||
)
|
||||
assert [item["content"] for item in compacted[1:]] == ["recent two"]
|
||||
|
||||
|
||||
def _create_session_databases(tmp_path: Path):
|
||||
state = tmp_path / "state.db"
|
||||
responses = tmp_path / "response_store.db"
|
||||
with sqlite3.connect(state) as connection:
|
||||
connection.execute(
|
||||
"""CREATE TABLE sessions (
|
||||
id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT,
|
||||
display_name TEXT, origin_json TEXT, title TEXT
|
||||
)"""
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO sessions VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
(
|
||||
"legacy",
|
||||
"api_server",
|
||||
"wrong",
|
||||
"group",
|
||||
"Unassigned",
|
||||
"{}",
|
||||
"Unassigned",
|
||||
),
|
||||
("named", "api_server", None, None, None, None, "Keep this title"),
|
||||
("other", "cli", None, None, "Unassigned", None, None),
|
||||
),
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE TABLE messages (id INTEGER PRIMARY KEY, session_id TEXT, role TEXT, content TEXT, timestamp REAL)"
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO messages VALUES (?, 'legacy', 'user', 'same', 1)", ((1,), (2,))
|
||||
)
|
||||
with sqlite3.connect(responses) as connection:
|
||||
connection.execute(
|
||||
"CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)"
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)"
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO responses VALUES (?, ?, 0)",
|
||||
(
|
||||
("general", json.dumps({"session_id": "legacy"})),
|
||||
("topic", json.dumps({"session_id": "named"})),
|
||||
("bad-json", "{"),
|
||||
("bad-id", json.dumps({"session_id": "../unsafe"})),
|
||||
(
|
||||
"oversized",
|
||||
json.dumps({"session_id": "other", "padding": "x" * 500}),
|
||||
),
|
||||
),
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO conversations VALUES (?, ?)",
|
||||
(
|
||||
("telegram", "general"),
|
||||
("telegram-topic-plans-g1", "topic"),
|
||||
("telegram-topic-malformed", "bad-json"),
|
||||
("telegram-topic-unsafe", "bad-id"),
|
||||
("telegram-topic-oversized", "oversized"),
|
||||
),
|
||||
)
|
||||
return state, responses
|
||||
|
||||
|
||||
def test_existing_session_migration_repairs_labels_and_is_idempotent(
|
||||
tmp_path: Path, continuity, monkeypatch
|
||||
):
|
||||
state, responses = _create_session_databases(tmp_path)
|
||||
monkeypatch.setattr(continuity, "MAX_RESPONSE_BYTES", 300)
|
||||
|
||||
assert continuity.migrate(state, responses) == 2
|
||||
assert continuity.migrate(state, responses) == 0
|
||||
|
||||
with sqlite3.connect(state) as connection:
|
||||
rows = {
|
||||
row[0]: row[1:]
|
||||
for row in connection.execute(
|
||||
"SELECT id, session_key, chat_type, display_name, origin_json, title "
|
||||
"FROM sessions ORDER BY id"
|
||||
)
|
||||
}
|
||||
assert rows["legacy"][:3] == ("telegram", "private", "Telegram")
|
||||
assert json.loads(rows["legacy"][3]) == {
|
||||
"platform": "telegram",
|
||||
"session_key": "telegram",
|
||||
}
|
||||
assert rows["legacy"][4] == "Telegram · General"
|
||||
assert rows["named"][:3] == (
|
||||
"telegram-topic-plans-g1",
|
||||
"private",
|
||||
"Telegram",
|
||||
)
|
||||
assert rows["named"][4] == "Keep this title"
|
||||
assert rows["other"] == (None, None, "Unassigned", None, None)
|
||||
with sqlite3.connect(state) as connection:
|
||||
assert connection.execute("SELECT COUNT(*) FROM messages").fetchone() == (1,)
|
||||
|
||||
|
||||
def test_migration_fails_closed_on_missing_or_malformed_databases(
|
||||
tmp_path: Path, continuity
|
||||
):
|
||||
assert continuity.telegram_sessions(tmp_path / "missing.db") == {}
|
||||
assert (
|
||||
continuity.migrate(tmp_path / "missing-state.db", tmp_path / "missing.db") == 0
|
||||
)
|
||||
broken = tmp_path / "broken.db"
|
||||
broken.write_text("not sqlite", encoding="utf-8")
|
||||
assert continuity.telegram_sessions(broken) == {}
|
||||
|
||||
state = tmp_path / "state.db"
|
||||
responses = tmp_path / "responses.db"
|
||||
with sqlite3.connect(state) as connection:
|
||||
connection.execute("CREATE TABLE sessions (id TEXT PRIMARY KEY)")
|
||||
with sqlite3.connect(responses) as connection:
|
||||
connection.execute("CREATE TABLE conversations (name TEXT, response_id TEXT)")
|
||||
connection.execute("CREATE TABLE responses (response_id TEXT, data TEXT)")
|
||||
connection.execute(
|
||||
"INSERT INTO responses VALUES ('valid', '{\"session_id\": \"known\"}')"
|
||||
)
|
||||
connection.execute("INSERT INTO conversations VALUES ('telegram', 'valid')")
|
||||
assert continuity.migrate(state, responses) == 0
|
||||
|
||||
|
||||
def test_migration_skips_invalid_conversation_and_rolls_back_database_errors(
|
||||
tmp_path: Path, continuity
|
||||
):
|
||||
state = tmp_path / "state.db"
|
||||
responses = tmp_path / "responses.db"
|
||||
with sqlite3.connect(state) as connection:
|
||||
connection.execute(
|
||||
"""CREATE TABLE sessions (
|
||||
id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT,
|
||||
display_name TEXT, origin_json TEXT, title TEXT
|
||||
)"""
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO sessions VALUES ('known', 'api_server', NULL, NULL, NULL, NULL, NULL)"
|
||||
)
|
||||
connection.execute(
|
||||
"""CREATE TRIGGER reject_updates BEFORE UPDATE ON sessions
|
||||
BEGIN SELECT RAISE(FAIL, 'read only for test'); END"""
|
||||
)
|
||||
with sqlite3.connect(responses) as connection:
|
||||
connection.execute(
|
||||
"CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)"
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL)"
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO responses VALUES (?, ?)",
|
||||
(
|
||||
("valid", json.dumps({"session_id": "known"})),
|
||||
("invalid", json.dumps({"session_id": "ignored"})),
|
||||
),
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO conversations VALUES (?, ?)",
|
||||
(("telegram", "valid"), ("telegram-topic-BAD", "invalid")),
|
||||
)
|
||||
|
||||
assert continuity.telegram_sessions(responses) == {"known": "telegram"}
|
||||
assert continuity.migrate(state, responses) == 0
|
||||
with sqlite3.connect(state) as connection:
|
||||
assert connection.execute(
|
||||
"SELECT session_key FROM sessions WHERE id = 'known'"
|
||||
).fetchone() == (None,)
|
||||
|
||||
|
||||
def test_api_patch_compacts_input_batch_and_disconnect_snapshots(tmp_path: Path):
|
||||
patcher = _load(
|
||||
"chat_api_patch",
|
||||
HERMES / "scripts" / "patch_api_server_sessions.py",
|
||||
)
|
||||
source = tmp_path / "api_server.py"
|
||||
destination = tmp_path / "patched.py"
|
||||
source.write_text(
|
||||
patcher.BEFORE
|
||||
+ patcher.RUNS_BEFORE
|
||||
+ patcher.RUN_CLOSE_BEFORE
|
||||
+ patcher.TRUNCATION_BEFORE
|
||||
+ patcher.RESPONSES_SESSION_BEFORE
|
||||
+ patcher.EVENT_CALLBACK_SIGNATURE_BEFORE
|
||||
+ patcher.EVENT_CALLBACK_BODY_BEFORE
|
||||
+ patcher.EVENT_CALLBACK_END_BEFORE
|
||||
+ patcher.EVENT_CALLBACK_CALL_BEFORE
|
||||
+ patcher.RUN_SWEEP_BEFORE
|
||||
+ patcher.SSE_STORE_BEFORE
|
||||
+ patcher.HISTORY_STORE_BEFORE,
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
patcher.patch(source, destination)
|
||||
patched = destination.read_text(encoding="utf-8")
|
||||
|
||||
assert (
|
||||
"conversation_history = compact_telegram_history(conversation_history)"
|
||||
in patched
|
||||
)
|
||||
assert "conversation_history_snapshot = compact_telegram_history(" in patched
|
||||
assert "full_history = compact_telegram_history(full_history)" in patched
|
||||
assert patched.count("from gateway.platforms.telegram_continuity import") == 2
|
||||
|
||||
|
||||
def _patch_source(patcher) -> str:
|
||||
return (
|
||||
patcher.BEFORE
|
||||
+ patcher.RUNS_BEFORE
|
||||
+ patcher.RUN_CLOSE_BEFORE
|
||||
+ patcher.RESPONSES_SESSION_BEFORE
|
||||
+ patcher.EVENT_CALLBACK_SIGNATURE_BEFORE
|
||||
+ patcher.EVENT_CALLBACK_BODY_BEFORE
|
||||
+ patcher.EVENT_CALLBACK_END_BEFORE
|
||||
+ patcher.EVENT_CALLBACK_CALL_BEFORE
|
||||
+ patcher.RUN_SWEEP_BEFORE
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"marker_name",
|
||||
[
|
||||
"BEFORE",
|
||||
"RUNS_BEFORE",
|
||||
"RUN_CLOSE_BEFORE",
|
||||
"RESPONSES_SESSION_BEFORE",
|
||||
"EVENT_CALLBACK_SIGNATURE_BEFORE",
|
||||
],
|
||||
)
|
||||
def test_api_patch_fails_closed_on_upstream_drift(tmp_path: Path, marker_name: str):
|
||||
patcher = _load(
|
||||
"chat_api_patch_drift",
|
||||
HERMES / "scripts" / "patch_api_server_sessions.py",
|
||||
)
|
||||
source = tmp_path / "api_server.py"
|
||||
marker = getattr(patcher, marker_name)
|
||||
source.write_text(_patch_source(patcher).replace(marker, "", 1), encoding="utf-8")
|
||||
with pytest.raises(RuntimeError, match="patch context changed"):
|
||||
patcher.patch(source, tmp_path / "patched.py")
|
||||
|
||||
|
||||
def test_script_entrypoints(tmp_path: Path, monkeypatch):
|
||||
continuity_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py"
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
str(continuity_path),
|
||||
str(tmp_path / "state.db"),
|
||||
str(tmp_path / "responses.db"),
|
||||
],
|
||||
)
|
||||
with pytest.raises(SystemExit, match="0"):
|
||||
runpy.run_path(str(continuity_path), run_name="__main__")
|
||||
|
||||
patch_path = HERMES / "scripts" / "patch_api_server_sessions.py"
|
||||
patcher = _load("chat_api_patch_cli", patch_path)
|
||||
source = tmp_path / "api_server.py"
|
||||
destination = tmp_path / "patched.py"
|
||||
source.write_text(_patch_source(patcher), encoding="utf-8")
|
||||
monkeypatch.setattr(sys, "argv", [str(patch_path), str(source), str(destination)])
|
||||
with pytest.raises(SystemExit, match="0"):
|
||||
runpy.run_path(str(patch_path), run_name="__main__")
|
||||
assert destination.exists()
|
||||
|
||||
|
||||
def test_chat_mounts_continuity_only_into_isolated_tenants():
|
||||
statefulset = yaml.safe_load((HERMES / "chat-statefulset.yaml").read_text())
|
||||
pod = statefulset["spec"]["template"]["spec"]
|
||||
hermes = next(item for item in pod["containers"] if item["name"] == "hermes")
|
||||
assert {
|
||||
"name": "coordinator",
|
||||
"mountPath": "/opt/hermes/gateway/platforms/telegram_continuity.py",
|
||||
"subPath": "migrate_telegram_api_sessions.py",
|
||||
"readOnly": True,
|
||||
} in hermes["volumeMounts"]
|
||||
patch_init = next(
|
||||
item
|
||||
for item in pod["initContainers"]
|
||||
if item["name"] == "patch-api-server-sessions"
|
||||
)
|
||||
command = patch_init["args"][0]
|
||||
assert command.count("grep -Fq") == 3
|
||||
assert "compact_telegram_history" in command
|
||||
assert "agent-deployment" not in (HERMES / "chat-statefulset.yaml").read_text()
|
||||
460
testing/tests/test_hermes_handoff_acceptance.py
Normal file
460
testing/tests/test_hermes_handoff_acceptance.py
Normal file
@ -0,0 +1,460 @@
|
||||
"""End-to-end contracts for the Hermes full-handoff acceptance harness CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
from testing.tests.test_hermes_handoff_support import FakeClock, load_handoff_module
|
||||
|
||||
acceptance = load_handoff_module("hermes_handoff_acceptance")
|
||||
catalog = load_handoff_module("hermes_handoff_catalog")
|
||||
exec_module = load_handoff_module("hermes_handoff_exec")
|
||||
model = load_handoff_module("hermes_handoff_model")
|
||||
policy = load_handoff_module("hermes_handoff_policy")
|
||||
harness_run = load_handoff_module("hermes_handoff_run")
|
||||
|
||||
|
||||
def completed(stdout: str = "", stderr: str = "", returncode: int = 0):
|
||||
return subprocess.CompletedProcess(
|
||||
args=[], returncode=returncode, stdout=stdout, stderr=stderr
|
||||
)
|
||||
|
||||
|
||||
class ScriptedCluster:
|
||||
"""A ``subprocess.run`` stand-in that answers by command shape.
|
||||
|
||||
Real runs issue a couple of hundred commands; scripting them positionally
|
||||
would make every test a brittle ordering puzzle. Matching on the command
|
||||
line keeps each test about the behaviour it is checking.
|
||||
"""
|
||||
|
||||
def __init__(self, rules: list[tuple[str, object]], default=None) -> None:
|
||||
self.rules = rules
|
||||
self.default = default or completed(stderr="not found", returncode=1)
|
||||
self.calls: list[list[str]] = []
|
||||
|
||||
def __call__(self, argv, **_kwargs):
|
||||
self.calls.append(list(argv))
|
||||
line = " ".join(argv)
|
||||
for fragment, response in self.rules:
|
||||
if fragment in line:
|
||||
return response
|
||||
return self.default
|
||||
|
||||
|
||||
def whoami(username: str):
|
||||
return completed(stdout=username)
|
||||
|
||||
|
||||
REVIEWED = "8f005458282269ba5c07941814e4237f5d4cf3ac"
|
||||
SHA = "1" * 40
|
||||
VALID_ARGS = [
|
||||
"--remote-main-sha",
|
||||
SHA,
|
||||
"--reviewed-head-sha",
|
||||
REVIEWED,
|
||||
"--agent-image",
|
||||
f"registry.example/hermes:git-{'1' * 40}-build-7@sha256:{'2' * 64}",
|
||||
"--build-sha",
|
||||
SHA,
|
||||
"--deployment-revision",
|
||||
"1",
|
||||
"--chat-config-revision",
|
||||
"chat-rev",
|
||||
*[
|
||||
item
|
||||
for number in catalog.Targets().dependency_pull_requests
|
||||
for item in ("--dependency-head", f"{number}={SHA}")
|
||||
],
|
||||
]
|
||||
|
||||
|
||||
HEALTHY_CLUSTER = [
|
||||
(
|
||||
"exec --namespace hermes hermes-agent-1",
|
||||
whoami("system:serviceaccount:hermes:hermes-agent"),
|
||||
),
|
||||
("auth whoami", whoami("kubernetes-admin")),
|
||||
("get pods --selector app=hermes-agent", completed(stdout="pod/hermes-agent-1\n")),
|
||||
("get pods --selector", completed(stdout="pod/other-1\n")),
|
||||
("get pod/hermes-chat-tenant-0", completed(stdout="pod/hermes-chat-tenant-0\n")),
|
||||
]
|
||||
|
||||
|
||||
def run_cli(monkeypatch, argv, rules):
|
||||
cluster = ScriptedCluster(rules)
|
||||
clock = FakeClock()
|
||||
|
||||
def build_runner(**kwargs):
|
||||
return exec_module.Runner(
|
||||
clock=clock,
|
||||
spawn=cluster,
|
||||
environ={},
|
||||
attestor=lambda command, _environment: exec_module.Attestation(
|
||||
command, "a" * 64
|
||||
),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(acceptance, "Runner", build_runner)
|
||||
return (acceptance.main([*VALID_ARGS, *argv]), cluster)
|
||||
|
||||
|
||||
def read_report(path):
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def test_the_parser_exposes_the_documented_controls() -> None:
|
||||
arguments = acceptance.build_parser().parse_args(
|
||||
[
|
||||
"--namespace",
|
||||
"hermes",
|
||||
"--context",
|
||||
"atlas-operator",
|
||||
"--dependency-pr",
|
||||
"14",
|
||||
"--dependency-pr",
|
||||
"15",
|
||||
"--expected-suspension",
|
||||
"flux-system/parked",
|
||||
"--node-count",
|
||||
"21",
|
||||
"--pool-worker-env",
|
||||
"HERMES_POOL_LEASE_SECONDS",
|
||||
]
|
||||
)
|
||||
|
||||
targets = acceptance.targets_from_args(arguments, None)
|
||||
|
||||
assert targets.dependency_pull_requests == (14, 15)
|
||||
assert targets.expected_suspensions == ("flux-system/parked",)
|
||||
assert targets.node_count == 21
|
||||
assert targets.pool_worker_env == ("HERMES_POOL_LEASE_SECONDS",)
|
||||
assert not hasattr(arguments, "concurrency")
|
||||
assert not hasattr(arguments, "expect_telegram_sessions")
|
||||
|
||||
|
||||
def test_defaults_fall_back_to_the_shipped_targets() -> None:
|
||||
arguments = acceptance.build_parser().parse_args([])
|
||||
targets = acceptance.targets_from_args(arguments, None)
|
||||
|
||||
assert targets.dependency_pull_requests == catalog.Targets.dependency_pull_requests
|
||||
assert targets.expected_suspensions == ()
|
||||
assert targets.repo == catalog.Targets.repo
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["bad", "x=" + "1" * 40, "0=" + "1" * 40, "1=BAD"])
|
||||
def test_dependency_heads_require_a_positive_pr_and_lowercase_sha(value: str) -> None:
|
||||
with pytest.raises(Exception):
|
||||
acceptance.dependency_head(value)
|
||||
|
||||
|
||||
def test_a_default_run_is_read_only_and_never_pushes_or_posts(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
output = tmp_path / "report.json"
|
||||
status, cluster = run_cli(
|
||||
monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER
|
||||
)
|
||||
report = read_report(output)
|
||||
|
||||
assert status == acceptance.EXIT_NO_GO
|
||||
assert report["mode"] == policy.READ_ONLY
|
||||
assert not any("push" in call for call in cluster.calls)
|
||||
assert not any("POST" in call for call in cluster.calls)
|
||||
assert not any("DELETE" in call for call in cluster.calls)
|
||||
writes = [
|
||||
call
|
||||
for call in cluster.calls
|
||||
if {"patch", "create", "delete", "apply", "replace"} & set(call)
|
||||
and "can-i" not in call
|
||||
]
|
||||
assert writes == []
|
||||
|
||||
|
||||
def test_mutating_checks_report_not_run_until_the_mode_is_armed(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
output = tmp_path / "report.json"
|
||||
run_cli(monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER)
|
||||
|
||||
ephemeral = {
|
||||
check["id"]: check
|
||||
for check in read_report(output)["checks"]
|
||||
if check["group"] == "ephemeral"
|
||||
}
|
||||
|
||||
assert len(ephemeral) == 4
|
||||
for check in ephemeral.values():
|
||||
assert check["status"] == model.NOT_RUN
|
||||
assert check["mandatory"] is False
|
||||
assert check["scope"] == model.EPHEMERAL
|
||||
|
||||
|
||||
def test_a_zero_state_cluster_is_no_go_with_every_check_accounted_for(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
"""Nothing observed must never read as nothing wrong."""
|
||||
output = tmp_path / "report.json"
|
||||
status, _ = run_cli(
|
||||
monkeypatch,
|
||||
["--output", str(output), "--json-only"],
|
||||
[("auth whoami", whoami("kubernetes-admin"))],
|
||||
)
|
||||
report = read_report(output)
|
||||
|
||||
assert status == acceptance.EXIT_NO_GO
|
||||
assert report["decision"] == model.NO_GO
|
||||
assert report["counts"][model.PASS] == 0
|
||||
assert sum(report["counts"].values()) == len(report["checks"])
|
||||
assert any(
|
||||
"in-pod Hermes vantage is unavailable" in error
|
||||
for error in report["harness_errors"]
|
||||
)
|
||||
|
||||
|
||||
def test_partial_failure_is_reported_per_check_rather_than_aborting(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
output = tmp_path / "report.json"
|
||||
rules = [
|
||||
*HEALTHY_CLUSTER,
|
||||
("auth can-i", completed(stdout="no\n", returncode=1)),
|
||||
("get namespaces", completed(stdout="namespace/hermes\n")),
|
||||
("merge-base", completed()),
|
||||
]
|
||||
status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules)
|
||||
report = read_report(output)
|
||||
|
||||
statuses = {check["status"] for check in report["checks"]}
|
||||
|
||||
assert status == acceptance.EXIT_NO_GO
|
||||
assert model.PASS in statuses and model.NOT_RUN in statuses
|
||||
assert report["blocking"]
|
||||
|
||||
|
||||
def test_a_vantage_collision_is_a_no_go_even_with_clean_checks(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
output = tmp_path / "report.json"
|
||||
rules = [
|
||||
(
|
||||
"exec --namespace hermes hermes-agent-1",
|
||||
whoami("system:serviceaccount:hermes:hermes-agent"),
|
||||
),
|
||||
("auth whoami", whoami("system:serviceaccount:hermes:hermes-agent")),
|
||||
(
|
||||
"get pods --selector app=hermes-agent",
|
||||
completed(stdout="pod/hermes-agent-1\n"),
|
||||
),
|
||||
("get pods --selector", completed(stdout="pod/other-1\n")),
|
||||
(
|
||||
"get pod/hermes-chat-tenant-0",
|
||||
completed(stdout="pod/hermes-chat-tenant-0\n"),
|
||||
),
|
||||
]
|
||||
status, _ = run_cli(monkeypatch, ["--output", str(output), "--json-only"], rules)
|
||||
report = read_report(output)
|
||||
|
||||
assert status == acceptance.EXIT_NO_GO
|
||||
assert any("same principal" in error for error in report["harness_errors"])
|
||||
|
||||
|
||||
def test_an_invalid_deadline_stops_before_any_subprocess(monkeypatch, tmp_path) -> None:
|
||||
output = tmp_path / "report.json"
|
||||
status, _ = run_cli(
|
||||
monkeypatch,
|
||||
["--output", str(output), "--json-only", "--deadline", "0"],
|
||||
HEALTHY_CLUSTER,
|
||||
)
|
||||
report = read_report(output)
|
||||
|
||||
assert status == acceptance.EXIT_USAGE
|
||||
assert report["counts"][model.NOT_RUN] > 0
|
||||
assert any("invalid execution bound" in error for error in report["harness_errors"])
|
||||
|
||||
|
||||
def test_arming_with_a_wrong_confirmation_stops_before_anything_runs(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
output = tmp_path / "report.json"
|
||||
status, cluster = run_cli(
|
||||
monkeypatch,
|
||||
[
|
||||
"--output",
|
||||
str(output),
|
||||
"--json-only",
|
||||
"--arm-ephemeral-push",
|
||||
"--confirm",
|
||||
"please",
|
||||
"--ephemeral-token",
|
||||
"acceptance-20260817a",
|
||||
],
|
||||
HEALTHY_CLUSTER,
|
||||
)
|
||||
report = read_report(output)
|
||||
|
||||
assert status == acceptance.EXIT_NO_GO
|
||||
assert "arming refused" in report["harness_errors"][0]
|
||||
assert not any("push" in call for call in cluster.calls)
|
||||
|
||||
|
||||
def test_arming_against_a_protected_push_target_is_refused(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
output = tmp_path / "report.json"
|
||||
status, cluster = run_cli(
|
||||
monkeypatch,
|
||||
[
|
||||
"--output",
|
||||
str(output),
|
||||
"--json-only",
|
||||
"--arm-ephemeral-push",
|
||||
"--confirm",
|
||||
acceptance.CONFIRMATION,
|
||||
"--ephemeral-token",
|
||||
"main",
|
||||
],
|
||||
HEALTHY_CLUSTER,
|
||||
)
|
||||
|
||||
assert status == acceptance.EXIT_NO_GO
|
||||
assert "arming refused" in read_report(output)["harness_errors"][0]
|
||||
assert not any("push" in call for call in cluster.calls)
|
||||
|
||||
|
||||
def test_an_armed_run_does_not_mutate_until_read_only_acceptance_is_go(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
output = tmp_path / "report.json"
|
||||
monkeypatch.setattr(acceptance, "preflight", lambda *_args: None)
|
||||
rules = [
|
||||
("git ls-remote origin refs/heads/ephemeral", completed(stdout="")),
|
||||
("git push", completed()),
|
||||
(
|
||||
"POST /api/v1/repos/atlas/titan-iac/pulls",
|
||||
completed(stdout=json.dumps({"number": 9})),
|
||||
),
|
||||
("PATCH", completed()),
|
||||
("DELETE", completed()),
|
||||
(
|
||||
"GET /api/v1/repos/atlas/titan-iac/pulls/9",
|
||||
completed(stdout='{"state": "closed"}'),
|
||||
),
|
||||
*HEALTHY_CLUSTER,
|
||||
]
|
||||
status, cluster = run_cli(
|
||||
monkeypatch,
|
||||
[
|
||||
"--output",
|
||||
str(output),
|
||||
"--json-only",
|
||||
"--arm-ephemeral-push",
|
||||
"--confirm",
|
||||
acceptance.CONFIRMATION,
|
||||
"--ephemeral-token",
|
||||
"acceptance-20260817a",
|
||||
],
|
||||
rules,
|
||||
)
|
||||
report = read_report(output)
|
||||
assert report["mode"] == policy.ARMED
|
||||
assert status == acceptance.EXIT_NO_GO
|
||||
assert any(
|
||||
"read-only acceptance did not GO" in error for error in report["harness_errors"]
|
||||
)
|
||||
pushes = [call for call in cluster.calls if "push" in call]
|
||||
assert pushes == []
|
||||
assert not any("DELETE" in call for call in cluster.calls)
|
||||
|
||||
|
||||
def test_the_report_is_written_to_stdout_when_no_output_path_is_given(
|
||||
monkeypatch, capsys
|
||||
) -> None:
|
||||
status, _ = run_cli(monkeypatch, ["--json-only"], HEALTHY_CLUSTER)
|
||||
payload = json.loads(capsys.readouterr().out)
|
||||
|
||||
assert status == acceptance.EXIT_NO_GO
|
||||
assert payload["harness"] == model.HARNESS
|
||||
|
||||
|
||||
def test_the_human_summary_accompanies_the_json_unless_suppressed(
|
||||
monkeypatch, capsys, tmp_path
|
||||
) -> None:
|
||||
output = tmp_path / "report.json"
|
||||
run_cli(monkeypatch, ["--output", str(output)], HEALTHY_CLUSTER)
|
||||
captured = capsys.readouterr()
|
||||
|
||||
assert model.HARNESS in captured.err
|
||||
assert "vantage operator" in captured.err
|
||||
assert captured.out == ""
|
||||
|
||||
|
||||
def test_a_report_that_fails_its_own_screening_is_replaced_not_published(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
"""The last barrier: never write an artifact nobody re-read."""
|
||||
output = tmp_path / "report.json"
|
||||
arguments = acceptance.build_parser().parse_args(
|
||||
["--output", str(output), "--json-only"]
|
||||
)
|
||||
report = model.Report(mode="read-only", started_at="2026-08-17T00:00:00Z")
|
||||
report.harness_errors = ["leaked ghp_ABCDEFGHIJKLMNOPQRSTUV1234"]
|
||||
|
||||
monkey = report.as_dict
|
||||
report.as_dict = lambda *_args, **_kwargs: { # type: ignore[method-assign]
|
||||
**monkey(),
|
||||
"leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234",
|
||||
}
|
||||
offenders = acceptance.emit(report, arguments)
|
||||
payload = read_report(output)
|
||||
|
||||
assert offenders == ["$.leak"]
|
||||
assert payload["decision"] == model.NO_GO
|
||||
assert "leak" not in payload
|
||||
assert "failed its own credential screening" in payload["harness_errors"][0]
|
||||
|
||||
|
||||
def test_a_screening_failure_forces_no_go_from_the_entry_point(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
"""The exit status must follow the refusal to publish, not the check tally."""
|
||||
output = tmp_path / "report.json"
|
||||
monkeypatch.setattr(
|
||||
acceptance, "unscreened_fields", lambda _payload: ["$.checks[0].stdout"]
|
||||
)
|
||||
monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: [])
|
||||
monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: [])
|
||||
|
||||
status, _ = run_cli(
|
||||
monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER
|
||||
)
|
||||
|
||||
assert status == acceptance.EXIT_NO_GO
|
||||
assert read_report(output)["decision"] == model.NO_GO
|
||||
|
||||
|
||||
def test_no_recorded_command_output_carries_a_credential_shape(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
output = tmp_path / "report.json"
|
||||
leaky = completed(stdout="ANTHROPIC_API_KEY=sk-ant-abcdefghijklmnopqrstuvwxyz\n")
|
||||
run_cli(
|
||||
monkeypatch,
|
||||
["--output", str(output), "--json-only"],
|
||||
[*HEALTHY_CLUSTER, ("exec", leaky)],
|
||||
)
|
||||
|
||||
assert model.unscreened_fields(read_report(output)) == []
|
||||
assert "sk-ant-abcdefghijklmnopqrstuvwxyz" not in output.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", ["--help"])
|
||||
def test_the_help_text_describes_the_read_only_default(flag: str, capsys) -> None:
|
||||
with pytest.raises(SystemExit):
|
||||
acceptance.main([flag])
|
||||
|
||||
assert "Read-only by default" in capsys.readouterr().out
|
||||
120
testing/tests/test_hermes_handoff_acceptance_armed.py
Normal file
120
testing/tests/test_hermes_handoff_acceptance_armed.py
Normal file
@ -0,0 +1,120 @@
|
||||
"""Static and armed-transition regressions for the handoff CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from testing.tests.test_hermes_handoff_acceptance import (
|
||||
HEALTHY_CLUSTER,
|
||||
REVIEWED,
|
||||
VALID_ARGS,
|
||||
acceptance,
|
||||
harness_run,
|
||||
model,
|
||||
policy,
|
||||
read_report,
|
||||
run_cli,
|
||||
)
|
||||
|
||||
|
||||
def test_an_empty_catalog_fails_before_any_subprocess(monkeypatch, tmp_path) -> None:
|
||||
output = tmp_path / "report.json"
|
||||
monkeypatch.setattr(harness_run, "build_catalog", lambda _targets: [])
|
||||
monkeypatch.setattr(acceptance, "build_catalog", lambda _targets: [])
|
||||
monkeypatch.setattr(harness_run, "vantage_problems", lambda _records: [])
|
||||
|
||||
status, _ = run_cli(
|
||||
monkeypatch, ["--output", str(output), "--json-only"], HEALTHY_CLUSTER
|
||||
)
|
||||
|
||||
assert status == acceptance.EXIT_NO_GO
|
||||
assert read_report(output)["decision"] == model.NO_GO
|
||||
|
||||
|
||||
def test_armed_success_path_replaces_placeholders_only_after_a_go(monkeypatch) -> None:
|
||||
base = model.CheckSpec(
|
||||
id="release.pass", title="pass", group="release", rule="allowed"
|
||||
)
|
||||
placeholder = model.CheckSpec(
|
||||
id="ephemeral.cleanup-verified",
|
||||
title="pending",
|
||||
group="ephemeral",
|
||||
rule="not_armed",
|
||||
mandatory=False,
|
||||
scope=model.EPHEMERAL,
|
||||
)
|
||||
report = model.Report(
|
||||
mode=policy.ARMED,
|
||||
started_at="2026-08-17T00:00:00Z",
|
||||
results=[
|
||||
model.CheckResult(base, model.PASS),
|
||||
model.CheckResult(placeholder, model.NOT_RUN),
|
||||
],
|
||||
)
|
||||
armed_spec = model.CheckSpec(
|
||||
id="ephemeral.cleanup-verified",
|
||||
title="clean",
|
||||
group="ephemeral",
|
||||
rule="armed",
|
||||
scope=model.EPHEMERAL,
|
||||
)
|
||||
armed = model.CheckResult(armed_spec, model.PASS)
|
||||
monkeypatch.setattr(
|
||||
acceptance, "resolve_vantages", lambda *_args: ({"operator": object()}, [])
|
||||
)
|
||||
monkeypatch.setattr(acceptance, "preflight", lambda *_args: None)
|
||||
monkeypatch.setattr(acceptance, "build_report", lambda *_args: report)
|
||||
monkeypatch.setattr(acceptance, "_armed_results", lambda *_args: [armed])
|
||||
monkeypatch.setattr(acceptance, "emit", lambda *_args: [])
|
||||
status = acceptance.main(
|
||||
[
|
||||
*VALID_ARGS,
|
||||
"--arm-ephemeral-push",
|
||||
"--confirm",
|
||||
acceptance.CONFIRMATION,
|
||||
"--ephemeral-token",
|
||||
"acceptance-20260817a",
|
||||
]
|
||||
)
|
||||
assert status == acceptance.EXIT_GO
|
||||
assert (
|
||||
next(
|
||||
result
|
||||
for result in report.results
|
||||
if result.spec.id.startswith("ephemeral")
|
||||
).status
|
||||
== model.PASS
|
||||
)
|
||||
monkeypatch.setattr(acceptance, "emit", lambda *_args: ["screening-failure"])
|
||||
assert (
|
||||
acceptance.main(
|
||||
[
|
||||
*VALID_ARGS,
|
||||
"--arm-ephemeral-push",
|
||||
"--confirm",
|
||||
acceptance.CONFIRMATION,
|
||||
"--ephemeral-token",
|
||||
"acceptance-20260817b",
|
||||
]
|
||||
)
|
||||
== acceptance.EXIT_NO_GO
|
||||
)
|
||||
|
||||
|
||||
def test_armed_results_are_bound_to_operator_and_exact_reviewed_head(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
arguments = acceptance.build_parser().parse_args(
|
||||
[
|
||||
*VALID_ARGS,
|
||||
"--confirm",
|
||||
acceptance.CONFIRMATION,
|
||||
"--ephemeral-token",
|
||||
"acceptance-20260817a",
|
||||
]
|
||||
)
|
||||
calls = []
|
||||
monkeypatch.setattr(acceptance, "preflight", lambda *args: calls.append(args))
|
||||
monkeypatch.setattr(acceptance, "run_armed", lambda *args: [args])
|
||||
operator = object()
|
||||
result = acceptance._armed_results(arguments, object(), {"operator": operator})
|
||||
assert calls[0][0].expected_head == REVIEWED
|
||||
assert result[0][1] is operator
|
||||
195
testing/tests/test_hermes_handoff_catalog.py
Normal file
195
testing/tests/test_hermes_handoff_catalog.py
Normal file
@ -0,0 +1,195 @@
|
||||
"""Structural contracts for the handoff acceptance catalog."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
|
||||
import pytest
|
||||
|
||||
from testing.tests.test_hermes_handoff_support import load_handoff_module
|
||||
|
||||
catalog = load_handoff_module("hermes_handoff_catalog")
|
||||
harness_run = load_handoff_module("hermes_handoff_run")
|
||||
evaluators = load_handoff_module("hermes_handoff_evaluators")
|
||||
model = load_handoff_module("hermes_handoff_model")
|
||||
policy = load_handoff_module("hermes_handoff_policy")
|
||||
exec_module = load_handoff_module("hermes_handoff_exec")
|
||||
|
||||
NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc)
|
||||
TARGETS = catalog.Targets(now=NOW)
|
||||
SPECS = harness_run.build_catalog(TARGETS)
|
||||
BY_ID = {check.id: check for check in SPECS}
|
||||
|
||||
VANTAGES = {
|
||||
catalog.OPERATOR: exec_module.operator_vantage(context="atlas-operator"),
|
||||
catalog.SELF: exec_module.pod_vantage("hermes", "hermes-agent-1", "hermes"),
|
||||
catalog.SWITCHYARD: exec_module.pod_vantage(
|
||||
"hermes", "hermes-switchyard-1", "switchyard"
|
||||
),
|
||||
catalog.NODE: exec_module.pod_vantage(
|
||||
"hermes", "hermes-node-ssh-access-1", "key-reconciler"
|
||||
),
|
||||
catalog.CHAT: exec_module.pod_vantage("hermes", "hermes-chat-tenant-0", "hermes"),
|
||||
}
|
||||
|
||||
|
||||
def test_the_catalog_is_structurally_valid() -> None:
|
||||
assert harness_run.validate_catalog(SPECS) == []
|
||||
assert len(SPECS) >= 60
|
||||
|
||||
|
||||
def test_every_step_is_accepted_by_the_read_only_policy_from_its_vantage() -> None:
|
||||
"""A catalog entry that cannot run under policy is a bug found before a run."""
|
||||
for check in SPECS:
|
||||
for step in check.steps:
|
||||
addressed = VANTAGES[step.vantage].wrap(step.argv)
|
||||
policy.check_argv(addressed, policy.READ_ONLY)
|
||||
|
||||
|
||||
def test_no_step_constructs_a_mutation_or_dry_run() -> None:
|
||||
"""Authorization review may name a verb; no executed subcommand mutates."""
|
||||
for check in SPECS:
|
||||
for step in check.steps:
|
||||
subcommand = next(policy.positionals(step.argv), "")
|
||||
assert subcommand not in {"apply", "create", "delete", "patch", "replace"}
|
||||
assert not any(argument.startswith("--dry-run") for argument in step.argv)
|
||||
|
||||
|
||||
def test_no_catalog_step_constructs_impersonation() -> None:
|
||||
operator_steps = [
|
||||
step
|
||||
for check in SPECS
|
||||
for step in check.steps
|
||||
if step.vantage == catalog.OPERATOR
|
||||
]
|
||||
assert operator_steps
|
||||
assert not any(
|
||||
set(step.argv) & set(policy.IMPERSONATION_ARGS) for step in operator_steps
|
||||
)
|
||||
|
||||
assert not any(
|
||||
set(step.argv) & set(policy.IMPERSONATION_ARGS)
|
||||
for check in SPECS
|
||||
for step in check.steps
|
||||
)
|
||||
|
||||
|
||||
def test_credential_or_mutation_denials_are_review_only() -> None:
|
||||
deny_checks = [check for check in SPECS if check.rule == "review_denied"]
|
||||
assert len(deny_checks) >= 5
|
||||
for check in deny_checks:
|
||||
assert check.steps and all(step.kind == model.REVIEW for step in check.steps), (
|
||||
check.id
|
||||
)
|
||||
|
||||
|
||||
def test_both_vantages_are_represented_across_the_catalog() -> None:
|
||||
used = {step.vantage for check in SPECS for step in check.steps}
|
||||
assert catalog.OPERATOR in used and catalog.SELF in used
|
||||
assert used <= set(catalog.VANTAGE_NAMES)
|
||||
|
||||
|
||||
def test_every_rule_named_by_a_check_is_registered() -> None:
|
||||
for check in SPECS:
|
||||
if check.scope == model.EPHEMERAL:
|
||||
assert check.rule == "not_armed"
|
||||
continue
|
||||
assert check.rule in evaluators.EVALUATORS, check.id
|
||||
|
||||
|
||||
def test_every_evidence_step_referenced_by_expect_exists() -> None:
|
||||
for check in SPECS:
|
||||
keys = {step.key for step in check.steps}
|
||||
named = [check.expect.get("step")] + list(check.expect.get("steps", ()))
|
||||
for key in filter(None, named):
|
||||
assert key in keys, f"{check.id} names missing step {key}"
|
||||
|
||||
|
||||
def test_the_default_catalog_reports_every_mutating_check_as_not_armed() -> None:
|
||||
ephemeral = [check for check in SPECS if check.scope == model.EPHEMERAL]
|
||||
assert len(ephemeral) == 4
|
||||
for check in ephemeral:
|
||||
assert check.mandatory is False
|
||||
assert evaluators.evaluate(check, {}).status == model.NOT_RUN
|
||||
|
||||
|
||||
def test_dependency_pull_requests_each_get_their_own_merge_gate() -> None:
|
||||
for number in TARGETS.dependency_pull_requests:
|
||||
check = BY_ID[f"baseline.dependency-pr-{number}-merged"]
|
||||
assert check.mandatory
|
||||
assert check.expect["fields"]["merged"] is True
|
||||
assert check.expect["fields"]["base.ref"] == "main"
|
||||
assert "head.sha" in check.expect["fields"]
|
||||
|
||||
|
||||
def test_the_baseline_commit_gate_names_the_configured_commit() -> None:
|
||||
check = BY_ID["baseline.origin-main-descends-merged-work"]
|
||||
assert TARGETS.baseline_commit in check.steps[0].argv
|
||||
assert "--is-ancestor" in check.steps[0].argv
|
||||
|
||||
|
||||
def test_pool_assignment_and_chat_continuity_are_unconditional() -> None:
|
||||
assert "pool.assignment-safety-knobs-are-configured" in BY_ID
|
||||
assert "pool.assignment-fieldrefs-are-bound" in BY_ID
|
||||
assert "surfaces.chat-runs-the-telegram-topic-revision" in BY_ID
|
||||
assert "surfaces.chat-telegram-sessions-are-continuous" in BY_ID
|
||||
assert TARGETS.node_count == TARGETS.pool_replicas == 3
|
||||
|
||||
|
||||
def test_bulk_evidence_is_parsed_but_not_pasted_into_the_report() -> None:
|
||||
routing = BY_ID["routing.provider-effort-and-fallback-evidence"]
|
||||
assert routing.steps[0].record is False
|
||||
assert routing.steps[0].max_bytes == TARGETS.routing_tail_bytes
|
||||
|
||||
kanban = BY_ID["surfaces.kanban-activity-is-visible"]
|
||||
assert kanban.steps[0].record is False
|
||||
|
||||
|
||||
def test_freshness_gates_carry_the_run_clock() -> None:
|
||||
for identifier in (
|
||||
"identity.codex-evidence-is-fresh",
|
||||
"identity.claude-evidence-is-fresh",
|
||||
):
|
||||
assert BY_ID[identifier].expect["now"] == NOW
|
||||
routing = BY_ID["routing.provider-effort-and-fallback-evidence"]
|
||||
assert routing.expect["now"] == NOW
|
||||
assert routing.expect["max_age_seconds"] == TARGETS.max_evidence_age_seconds
|
||||
|
||||
|
||||
def test_check_identifiers_are_unique_and_grouped() -> None:
|
||||
identifiers = [check.id for check in SPECS]
|
||||
assert len(identifiers) == len(set(identifiers))
|
||||
for check in SPECS:
|
||||
assert check.id.split(".")[0]
|
||||
assert check.group
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"identifier",
|
||||
[
|
||||
"identity.no-provider-api-key-in-pod",
|
||||
"identity.no-provider-api-key-in-manifest",
|
||||
"forge.worker-holds-no-git-credential-variable",
|
||||
"access.secrets-are-denied",
|
||||
"access.service-account-tokens-are-denied",
|
||||
"access.workload-mutation-is-denied",
|
||||
"access.pod-exec-is-denied",
|
||||
"access.required-reads-are-allowed",
|
||||
"access.flux-and-helm-status-are-allowed",
|
||||
"access.namespace-view-agrees-across-vantages",
|
||||
"nodes.hardening-covers-every-node",
|
||||
"build.builder-service-account-is-tokenless",
|
||||
"build.harbor-immutability-rule-is-applied",
|
||||
"release.exact-lineage-is-running",
|
||||
"reliability.finalization-and-replay-patches-are-deployed",
|
||||
"pool.three-workers-have-distinct-nodes-and-volumes",
|
||||
"pool.coordinator-retains-sole-state-ownership",
|
||||
"surfaces.kanban-activity-is-visible",
|
||||
"surfaces.chat-telegram-topic-state-is-durable",
|
||||
"gitops.kustomizations-reconcile-cleanly",
|
||||
"routing.provider-effort-and-fallback-evidence",
|
||||
"scopes.agent-profile-is-distinct",
|
||||
],
|
||||
)
|
||||
def test_the_handoff_requirements_each_have_a_mandatory_check(identifier: str) -> None:
|
||||
assert BY_ID[identifier].mandatory is True
|
||||
438
testing/tests/test_hermes_handoff_ephemeral.py
Normal file
438
testing/tests/test_hermes_handoff_ephemeral.py
Normal file
@ -0,0 +1,438 @@
|
||||
"""Contracts for the strictly bounded, self-cleaning armed mode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from testing.tests.test_hermes_handoff_support import (
|
||||
FakeClock,
|
||||
FakeSpawn,
|
||||
load_handoff_module,
|
||||
)
|
||||
|
||||
ephemeral = load_handoff_module("hermes_handoff_ephemeral")
|
||||
arming = load_handoff_module("hermes_handoff_arming")
|
||||
exec_module = load_handoff_module("hermes_handoff_exec")
|
||||
policy = load_handoff_module("hermes_handoff_policy")
|
||||
model = load_handoff_module("hermes_handoff_model")
|
||||
|
||||
TOKEN = "acceptance-20260817a"
|
||||
HEAD = "8f005458282269ba5c07941814e4237f5d4cf3ac"
|
||||
MAIN = "1" * 40
|
||||
VANTAGE = exec_module.operator_vantage()
|
||||
|
||||
|
||||
def request(**overrides):
|
||||
fields = {
|
||||
"repo": policy.EXPECTED_REPO,
|
||||
"remote": policy.EXPECTED_REMOTE,
|
||||
"token": TOKEN,
|
||||
"confirmation": ephemeral.CONFIRMATION,
|
||||
"base": policy.EXPECTED_BASE,
|
||||
"expected_head": HEAD,
|
||||
"expected_base_sha": MAIN,
|
||||
}
|
||||
fields.update(overrides)
|
||||
return ephemeral.ArmRequest(**fields)
|
||||
|
||||
|
||||
def completed(stdout: str = "", stderr: str = "", returncode: int = 0):
|
||||
return subprocess.CompletedProcess(
|
||||
args=[], returncode=returncode, stdout=stdout, stderr=stderr
|
||||
)
|
||||
|
||||
|
||||
def armed_runner(results: list, mode=policy.ARMED):
|
||||
spawn = FakeSpawn(results)
|
||||
runner = exec_module.Runner(
|
||||
mode=mode,
|
||||
clock=FakeClock(),
|
||||
spawn=spawn,
|
||||
environ={},
|
||||
deadline_seconds=3600,
|
||||
attestor=lambda command, _environment: exec_module.Attestation(
|
||||
command, "a" * 64
|
||||
),
|
||||
)
|
||||
return runner, spawn
|
||||
|
||||
|
||||
def by_id(results: list) -> dict:
|
||||
return {result.spec.id: result for result in results}
|
||||
|
||||
|
||||
def pr_payload(state="open", number=42):
|
||||
return {
|
||||
"number": number,
|
||||
"state": state,
|
||||
"draft": True,
|
||||
"merged": False,
|
||||
"base": {"ref": "main", "sha": MAIN},
|
||||
"head": {"ref": request().ref, "sha": HEAD},
|
||||
}
|
||||
|
||||
|
||||
def linked_worktree(tmp_path: Path) -> Path:
|
||||
root = tmp_path / "worktree"
|
||||
common = tmp_path / "repo.git"
|
||||
gitdir = common / "worktrees" / "acceptance"
|
||||
gitdir.mkdir(parents=True)
|
||||
(root).mkdir()
|
||||
(root / ".git").write_text(f"gitdir: {gitdir}\n", encoding="utf-8")
|
||||
(gitdir / "commondir").write_text("../..\n", encoding="utf-8")
|
||||
(gitdir / "HEAD").write_text(
|
||||
f"ref: refs/heads/{arming.EXPECTED_HEAD_REF}\n", encoding="utf-8"
|
||||
)
|
||||
ref = common / "refs" / "heads" / arming.EXPECTED_HEAD_REF
|
||||
ref.parent.mkdir(parents=True)
|
||||
ref.write_text(f"{HEAD}\n", encoding="utf-8")
|
||||
remote_ref = common / "refs" / "remotes" / "origin" / "main"
|
||||
remote_ref.parent.mkdir(parents=True)
|
||||
remote_ref.write_text(f"{MAIN}\n", encoding="utf-8")
|
||||
(common / "config").write_text(
|
||||
f'[remote "origin"]\n\turl = {arming.EXPECTED_REMOTE_URL}\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
return root
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"ref", ["main", "master", "refs/heads/main", "HEAD", "feature/x", ""]
|
||||
)
|
||||
def test_only_the_unique_ephemeral_ref_is_allowed(ref: str) -> None:
|
||||
with pytest.raises(ephemeral.ArmingError):
|
||||
ephemeral.assert_push_target_allowed(ref)
|
||||
ephemeral.assert_push_target_allowed(request().ref)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides",
|
||||
[
|
||||
{"confirmation": "wrong"},
|
||||
{"repo": "atlas/other"},
|
||||
{"remote": "upstream"},
|
||||
{"base": "develop"},
|
||||
{"token": "short"},
|
||||
{"expected_head": "bad"},
|
||||
{"expected_base_sha": "bad"},
|
||||
],
|
||||
)
|
||||
def test_preflight_rejects_bad_inputs_before_any_runner_exists(
|
||||
tmp_path: Path, overrides: dict
|
||||
) -> None:
|
||||
with pytest.raises(ephemeral.ArmingError):
|
||||
ephemeral.preflight(
|
||||
request(**overrides), policy.EXPECTED_REPO, linked_worktree(tmp_path)
|
||||
)
|
||||
|
||||
|
||||
def test_preflight_attests_the_actual_linked_worktree(tmp_path: Path) -> None:
|
||||
ephemeral.preflight(request(), policy.EXPECTED_REPO, linked_worktree(tmp_path))
|
||||
root = linked_worktree(tmp_path / "wrong")
|
||||
(Path((root / ".git").read_text().split(": ", 1)[1].strip()) / "HEAD").write_text(
|
||||
"ref: refs/heads/other\n", encoding="utf-8"
|
||||
)
|
||||
with pytest.raises(ephemeral.ArmingError, match="existing PR #19"):
|
||||
ephemeral.preflight(request(), policy.EXPECTED_REPO, root)
|
||||
|
||||
|
||||
def test_worktree_attestation_rejects_each_malformed_local_source(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
roots = []
|
||||
empty = tmp_path / "empty"
|
||||
empty.mkdir()
|
||||
roots.append(empty)
|
||||
bad_marker = linked_worktree(tmp_path / "marker")
|
||||
(bad_marker / ".git").write_text("not-a-gitdir\n", encoding="utf-8")
|
||||
roots.append(bad_marker)
|
||||
wrong_sha = linked_worktree(tmp_path / "sha")
|
||||
gitdir = Path((wrong_sha / ".git").read_text().split(": ", 1)[1].strip())
|
||||
common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve()
|
||||
(common / "refs" / "heads" / arming.EXPECTED_HEAD_REF).write_text("0" * 40)
|
||||
roots.append(wrong_sha)
|
||||
wrong_main = linked_worktree(tmp_path / "main")
|
||||
gitdir = Path((wrong_main / ".git").read_text().split(": ", 1)[1].strip())
|
||||
common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve()
|
||||
(common / "refs" / "remotes" / "origin" / "main").write_text("0" * 40)
|
||||
roots.append(wrong_main)
|
||||
bad_config = linked_worktree(tmp_path / "config")
|
||||
gitdir = Path((bad_config / ".git").read_text().split(": ", 1)[1].strip())
|
||||
common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve()
|
||||
(common / "config").write_text("[broken\n", encoding="utf-8")
|
||||
roots.append(bad_config)
|
||||
wrong_remote = linked_worktree(tmp_path / "remote")
|
||||
gitdir = Path((wrong_remote / ".git").read_text().split(": ", 1)[1].strip())
|
||||
common = (gitdir / (gitdir / "commondir").read_text().strip()).resolve()
|
||||
(common / "config").write_text('[remote "origin"]\nurl=https://example.invalid/x\n')
|
||||
roots.append(wrong_remote)
|
||||
for root in roots:
|
||||
with pytest.raises((ephemeral.ArmingError, FileNotFoundError)):
|
||||
ephemeral.preflight(request(), policy.EXPECTED_REPO, root)
|
||||
oversized = tmp_path / "oversized"
|
||||
oversized.write_text("x" * 5, encoding="utf-8")
|
||||
with pytest.raises(ephemeral.ArmingError):
|
||||
ephemeral.bounded_read(oversized, 4)
|
||||
|
||||
|
||||
def test_guard_failure_stops_before_runner_calls(monkeypatch) -> None:
|
||||
monkeypatch.setattr(ephemeral, "assert_push_target_allowed", lambda _ref: None)
|
||||
runner, spawn = armed_runner([])
|
||||
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
||||
assert results["ephemeral.protected-branch-refusal"].status == model.FAIL
|
||||
assert all(
|
||||
result.status in {model.FAIL, model.NOT_RUN} for result in results.values()
|
||||
)
|
||||
assert spawn.calls == []
|
||||
|
||||
|
||||
def test_full_armed_run_validates_and_removes_exact_artifacts() -> None:
|
||||
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
|
||||
runner, spawn = armed_runner(
|
||||
[
|
||||
completed(),
|
||||
completed(stdout=f"{HEAD}\n"),
|
||||
completed(),
|
||||
completed(stdout=ref_line),
|
||||
completed(stdout=json.dumps(pr_payload())),
|
||||
completed(stdout=json.dumps([pr_payload()])),
|
||||
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
||||
completed(),
|
||||
completed(),
|
||||
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
||||
]
|
||||
)
|
||||
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
||||
assert all(result.status == model.PASS for result in results.values())
|
||||
push = next(call for call in spawn.calls if call["argv"][1] == "push")
|
||||
assert push["argv"][-1] == f"HEAD:refs/heads/{request().ref}"
|
||||
assert all(result.spec.mandatory for result in results.values())
|
||||
|
||||
|
||||
def test_existing_or_unverifiable_ref_never_reaches_a_write() -> None:
|
||||
for first in (completed(stdout="abc\trefs/heads/x\n"), completed(returncode=128)):
|
||||
runner, spawn = armed_runner([first])
|
||||
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
||||
assert results["ephemeral.feature-branch-push"].status == model.FAIL
|
||||
assert not any(
|
||||
call["argv"][1] in {"push", "POST", "PATCH", "DELETE"}
|
||||
for call in spawn.calls
|
||||
)
|
||||
|
||||
|
||||
def test_failed_or_ambiguous_create_is_discovered_and_cleanup_still_runs() -> None:
|
||||
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
|
||||
runner, _ = armed_runner(
|
||||
[
|
||||
completed(),
|
||||
completed(stdout=f"{HEAD}\n"),
|
||||
completed(),
|
||||
completed(stdout=ref_line),
|
||||
completed(returncode=1),
|
||||
completed(stdout=json.dumps([pr_payload()])),
|
||||
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
||||
completed(),
|
||||
completed(),
|
||||
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
||||
]
|
||||
)
|
||||
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
||||
assert results["ephemeral.draft-pull-request"].status == model.PASS
|
||||
assert results["ephemeral.cleanup-verified"].status == model.PASS
|
||||
|
||||
|
||||
def test_pull_request_parsers_reject_every_malformed_or_ambiguous_shape() -> None:
|
||||
assert ephemeral._pr_problems([], request()) == ["pull request is not an object"]
|
||||
malformed = pr_payload()
|
||||
malformed.update({"number": True, "draft": "yes", "base": None})
|
||||
malformed["head"] = {"ref": "other", "sha": "bad"}
|
||||
problems = ephemeral._pr_problems(malformed, request())
|
||||
assert {"number is malformed", "draft does not match", "base is malformed"} <= set(
|
||||
problems
|
||||
)
|
||||
assert ephemeral._created_number(
|
||||
exec_module.Outcome((), "x", returncode=1), request()
|
||||
) == (0, "pull-request creation failed: exit status 1")
|
||||
for body in ("not-json", "{}"):
|
||||
number, error = ephemeral._created_number(
|
||||
exec_module.Outcome((), "x", 0, body), request()
|
||||
)
|
||||
assert number == 0 and error
|
||||
assert ephemeral._created_number(
|
||||
exec_module.Outcome((), "x", 0, json.dumps(pr_payload())), request()
|
||||
) == (42, "")
|
||||
for rows in ([None], [{"head": {"ref": "other"}, "base": {}}]):
|
||||
numbers, error = ephemeral._match_pulls(rows, request())
|
||||
assert numbers == [] and error
|
||||
invalid = pr_payload()
|
||||
invalid["draft"] = False
|
||||
assert ephemeral._match_pulls([invalid], request())[1]
|
||||
assert "discovered 0" in ephemeral._match_pulls([], request())[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"delete_result,remaining,closed",
|
||||
[
|
||||
(completed(returncode=1), completed(), pr_payload(state="closed")),
|
||||
(completed(), completed(returncode=128), pr_payload(state="closed")),
|
||||
(completed(), completed(), {}),
|
||||
(completed(), completed(), pr_payload(state="open")),
|
||||
],
|
||||
)
|
||||
def test_cleanup_fails_closed_on_partial_or_malformed_evidence(
|
||||
delete_result, remaining, closed
|
||||
) -> None:
|
||||
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
|
||||
runner, _ = armed_runner(
|
||||
[
|
||||
completed(),
|
||||
completed(stdout=f"{HEAD}\n"),
|
||||
completed(),
|
||||
completed(stdout=ref_line),
|
||||
completed(),
|
||||
completed(stdout=json.dumps([pr_payload()])),
|
||||
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
||||
delete_result,
|
||||
remaining,
|
||||
completed(stdout=json.dumps(closed)),
|
||||
]
|
||||
)
|
||||
cleanup = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[
|
||||
"ephemeral.cleanup-verified"
|
||||
]
|
||||
assert cleanup.status == model.FAIL
|
||||
|
||||
|
||||
def test_cleanup_fails_when_close_or_verification_commands_are_bad_json_or_failures() -> (
|
||||
None
|
||||
):
|
||||
api = "/api/v1/repos/atlas/titan-iac"
|
||||
for scripted in (
|
||||
[completed(returncode=1), completed(), completed(), completed(returncode=1)],
|
||||
[completed(stdout="bad"), completed(), completed(), completed(stdout="bad")],
|
||||
[
|
||||
completed(stdout=json.dumps(pr_payload())),
|
||||
completed(),
|
||||
completed(),
|
||||
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
||||
],
|
||||
):
|
||||
runner, _ = armed_runner(scripted)
|
||||
result = ephemeral._cleanup(runner, VANTAGE, request(), api, [42])
|
||||
assert result.status == model.FAIL
|
||||
|
||||
|
||||
def test_read_only_runner_cannot_reach_push() -> None:
|
||||
runner, spawn = armed_runner(
|
||||
[completed(), completed(stdout=f"{HEAD}\n"), completed(), completed()],
|
||||
mode=policy.READ_ONLY,
|
||||
)
|
||||
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
||||
assert results["ephemeral.feature-branch-push"].status == model.FAIL
|
||||
assert not any(call["argv"][1] == "push" for call in spawn.calls)
|
||||
|
||||
|
||||
def push_prelude(ref_line: str) -> list:
|
||||
"""The four scripted results that get an armed run to a verified push."""
|
||||
return [
|
||||
completed(),
|
||||
completed(stdout=f"{HEAD}\n"),
|
||||
completed(),
|
||||
completed(stdout=ref_line),
|
||||
]
|
||||
|
||||
|
||||
def test_pull_request_discovery_reads_every_page() -> None:
|
||||
"""A full first page must not end discovery: the match can be on page two."""
|
||||
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
|
||||
filler = [
|
||||
{
|
||||
"number": index,
|
||||
"state": "open",
|
||||
"draft": True,
|
||||
"merged": False,
|
||||
"base": {"ref": "main", "sha": MAIN},
|
||||
"head": {"ref": "feature/other", "sha": HEAD},
|
||||
}
|
||||
for index in range(1, ephemeral.DISCOVERY_PAGE_SIZE + 1)
|
||||
]
|
||||
runner, spawn = armed_runner(
|
||||
[
|
||||
*push_prelude(ref_line),
|
||||
completed(stdout=json.dumps(pr_payload())),
|
||||
completed(stdout=json.dumps(filler)),
|
||||
completed(stdout=json.dumps([pr_payload()])),
|
||||
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
||||
completed(),
|
||||
completed(),
|
||||
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
||||
]
|
||||
)
|
||||
|
||||
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
||||
|
||||
assert results["ephemeral.draft-pull-request"].status == model.PASS
|
||||
pages = [call["argv"][2] for call in spawn.calls if "pulls?state=all" in str(call)]
|
||||
assert any("page=1" in page for page in pages)
|
||||
assert any("page=2" in page for page in pages)
|
||||
|
||||
|
||||
def test_uncertain_discovery_still_closes_the_created_pull_and_names_residue() -> None:
|
||||
"""The create response names the pull, so an unusable index cannot orphan it."""
|
||||
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
|
||||
runner, spawn = armed_runner(
|
||||
[
|
||||
*push_prelude(ref_line),
|
||||
completed(stdout=json.dumps(pr_payload())),
|
||||
completed(returncode=1),
|
||||
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
||||
completed(),
|
||||
completed(),
|
||||
completed(stdout=json.dumps(pr_payload(state="closed"))),
|
||||
]
|
||||
)
|
||||
|
||||
results = by_id(ephemeral.run_armed(runner, VANTAGE, request()))
|
||||
draft = results["ephemeral.draft-pull-request"]
|
||||
|
||||
assert draft.status == model.FAIL
|
||||
assert draft.evidence["pull_requests"] == [42]
|
||||
assert draft.evidence["residue_ref"] == request().ref
|
||||
assert any("pulls/42" in line for line in draft.evidence["manual_cleanup"])
|
||||
assert any("branches/" in line for line in draft.evidence["manual_cleanup"])
|
||||
# The pull the harness created is closed, not left behind.
|
||||
assert any(
|
||||
call["argv"][1] == "PATCH" and call["argv"][2].endswith("/pulls/42")
|
||||
for call in spawn.calls
|
||||
)
|
||||
assert results["ephemeral.cleanup-verified"].status == model.FAIL
|
||||
|
||||
|
||||
def test_a_wholly_unidentifiable_create_names_the_ref_for_manual_cleanup() -> None:
|
||||
"""When neither source names a pull, the operator gets the exact ref to sweep."""
|
||||
ref_line = f"{HEAD}\trefs/heads/{request().ref}\n"
|
||||
runner, _ = armed_runner(
|
||||
[
|
||||
*push_prelude(ref_line),
|
||||
completed(stdout="not-json"),
|
||||
completed(returncode=1),
|
||||
completed(),
|
||||
completed(),
|
||||
]
|
||||
)
|
||||
|
||||
draft = by_id(ephemeral.run_armed(runner, VANTAGE, request()))[
|
||||
"ephemeral.draft-pull-request"
|
||||
]
|
||||
|
||||
assert draft.status == model.FAIL
|
||||
assert draft.evidence["pull_requests"] == []
|
||||
assert draft.evidence["residue_ref"] == request().ref
|
||||
assert any(
|
||||
request().ref in line and "close by hand" in line
|
||||
for line in draft.evidence["manual_cleanup"]
|
||||
)
|
||||
457
testing/tests/test_hermes_handoff_evaluators.py
Normal file
457
testing/tests/test_hermes_handoff_evaluators.py
Normal file
@ -0,0 +1,457 @@
|
||||
"""Contracts for the text-shaped handoff acceptance evaluators."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from testing.tests.test_hermes_handoff_support import (
|
||||
load_handoff_module,
|
||||
outcome,
|
||||
spec,
|
||||
step,
|
||||
)
|
||||
|
||||
evaluators = load_handoff_module("hermes_handoff_evaluators")
|
||||
model = load_handoff_module("hermes_handoff_model")
|
||||
|
||||
REVIEW = model.REVIEW
|
||||
ATTEMPT = model.ATTEMPT
|
||||
FORBIDDEN = outcome(
|
||||
stderr="Error from server (Forbidden): pods is forbidden", returncode=1
|
||||
)
|
||||
|
||||
|
||||
def deny_spec() -> object:
|
||||
return spec(
|
||||
"denied",
|
||||
steps=(step("review", kind=REVIEW), step("attempt", kind=ATTEMPT)),
|
||||
)
|
||||
|
||||
|
||||
def evaluate(built, outcomes):
|
||||
return evaluators.evaluate(built, outcomes)
|
||||
|
||||
|
||||
def test_a_deny_check_passes_only_on_a_review_no_and_a_live_refusal() -> None:
|
||||
evaluation = evaluate(
|
||||
deny_spec(), {"review": outcome(stdout="no"), "attempt": FORBIDDEN}
|
||||
)
|
||||
|
||||
assert evaluation.status == model.PASS
|
||||
assert evaluation.evidence == {"review": "no", "attempt": "refused"}
|
||||
|
||||
|
||||
def test_a_deny_check_fails_when_the_review_says_the_authority_exists() -> None:
|
||||
evaluation = evaluate(
|
||||
deny_spec(), {"review": outcome(stdout="yes"), "attempt": FORBIDDEN}
|
||||
)
|
||||
|
||||
assert evaluation.status == model.FAIL
|
||||
assert "reported yes" in evaluation.reason
|
||||
|
||||
|
||||
def test_a_deny_check_fails_when_the_forbidden_request_actually_succeeds() -> None:
|
||||
evaluation = evaluate(
|
||||
deny_spec(), {"review": outcome(stdout="no"), "attempt": outcome()}
|
||||
)
|
||||
|
||||
assert evaluation.status == model.FAIL
|
||||
assert "must be refused" in evaluation.reason
|
||||
|
||||
|
||||
def test_a_request_that_merely_broke_is_not_evidence_of_a_refusal() -> None:
|
||||
broken = outcome(stderr="connection refused", returncode=7)
|
||||
evaluation = evaluate(
|
||||
deny_spec(), {"review": outcome(stdout="no"), "attempt": broken}
|
||||
)
|
||||
|
||||
assert evaluation.status == model.NOT_RUN
|
||||
assert "recognisable refusal" in evaluation.reason
|
||||
|
||||
|
||||
def test_a_deny_check_without_a_live_attempt_refuses_to_classify() -> None:
|
||||
"""An authorization review states policy; only an attempt proves enforcement."""
|
||||
review_only = spec("denied", steps=(step("review", kind=REVIEW),))
|
||||
|
||||
evaluation = evaluate(review_only, {"review": outcome(stdout="no")})
|
||||
|
||||
assert evaluation.status == model.NOT_RUN
|
||||
assert "require a real attempt" in evaluation.reason
|
||||
|
||||
|
||||
def test_an_allow_check_needs_every_step_to_succeed_and_every_review_to_say_yes() -> (
|
||||
None
|
||||
):
|
||||
built = spec("allowed", steps=(step("review", kind=REVIEW), step("read")))
|
||||
|
||||
assert (
|
||||
evaluate(built, {"review": outcome(stdout="yes"), "read": outcome()}).status
|
||||
== model.PASS
|
||||
)
|
||||
|
||||
denied = evaluate(built, {"review": outcome(stdout="no"), "read": outcome()})
|
||||
assert denied.status == model.FAIL and "reported no" in denied.reason
|
||||
|
||||
failed = evaluate(
|
||||
built, {"review": outcome(stdout="yes"), "read": outcome(returncode=1)}
|
||||
)
|
||||
assert failed.status == model.FAIL and "step read failed" in failed.reason
|
||||
|
||||
|
||||
def test_names_absent_flags_exact_names_and_substrings() -> None:
|
||||
built = spec(
|
||||
"names_absent",
|
||||
{"step": "env", "names": ("GITEA_TOKEN",), "contains": ("API_KEY",)},
|
||||
steps=(step("env"),),
|
||||
)
|
||||
|
||||
clean = evaluate(built, {"env": outcome(stdout="PATH\nHOME\n")})
|
||||
assert clean.status == model.PASS and clean.evidence["observed_count"] == 2
|
||||
|
||||
leaked = evaluate(
|
||||
built, {"env": outcome(stdout="PATH\nGITEA_TOKEN\nOPENAI_API_KEY\n")}
|
||||
)
|
||||
assert leaked.status == model.FAIL
|
||||
assert leaked.evidence["offenders"] == ["GITEA_TOKEN", "OPENAI_API_KEY"]
|
||||
|
||||
unreadable = evaluate(built, {"env": outcome(returncode=1)})
|
||||
assert unreadable.status == model.NOT_RUN
|
||||
|
||||
|
||||
def test_names_present_requires_names_and_substrings_and_rejects_a_silent_empty() -> (
|
||||
None
|
||||
):
|
||||
built = spec(
|
||||
"names_present",
|
||||
{
|
||||
"step": "init",
|
||||
"names": ("patch-web-session-activity",),
|
||||
"contains": ("@sha256:",),
|
||||
},
|
||||
steps=(step("init"),),
|
||||
)
|
||||
|
||||
good = outcome(stdout="patch-web-session-activity\nimage@sha256:abc\n")
|
||||
assert evaluate(built, {"init": good}).status == model.PASS
|
||||
|
||||
partial = evaluate(built, {"init": outcome(stdout="patch-web-session-activity\n")})
|
||||
assert partial.status == model.FAIL and "@sha256:" in partial.reason
|
||||
|
||||
empty = spec("names_present", {"step": "init"}, steps=(step("init"),))
|
||||
assert evaluate(empty, {"init": outcome(stdout="")}).status == model.NOT_RUN
|
||||
|
||||
|
||||
def test_stdout_matches_accepts_a_value_or_a_set_of_values() -> None:
|
||||
single = spec(
|
||||
"stdout_matches", {"step": "s", "equals": "locked"}, steps=(step("s"),)
|
||||
)
|
||||
assert evaluate(single, {"s": outcome(stdout=" locked \n")}).status == model.PASS
|
||||
|
||||
mismatch = evaluate(single, {"s": outcome(stdout="unlocked")})
|
||||
assert mismatch.status == model.FAIL and mismatch.evidence["observed"] == "unlocked"
|
||||
|
||||
several = spec(
|
||||
"stdout_matches",
|
||||
{"step": "s", "equals": ["600 root:root", "absent"]},
|
||||
steps=(step("s"),),
|
||||
)
|
||||
assert evaluate(several, {"s": outcome(stdout="absent")}).status == model.PASS
|
||||
|
||||
assert evaluate(single, {"s": outcome(returncode=1)}).status == model.NOT_RUN
|
||||
|
||||
|
||||
def test_distinct_count_separates_replica_count_from_actual_spread() -> None:
|
||||
built = spec(
|
||||
"distinct_count",
|
||||
{"step": "n", "minimum": 3, "total_equals": 3},
|
||||
steps=(step("n"),),
|
||||
)
|
||||
|
||||
spread = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-06\ntitan-07\n")})
|
||||
assert spread.status == model.PASS and spread.evidence["distinct"] == 3
|
||||
|
||||
stacked = evaluate(built, {"n": outcome(stdout="titan-05\ntitan-05\ntitan-05\n")})
|
||||
assert stacked.status == model.FAIL and "distinct values" in stacked.reason
|
||||
|
||||
extra = spec(
|
||||
"distinct_count",
|
||||
{"step": "n", "minimum": 1, "total_equals": 2},
|
||||
steps=(step("n"),),
|
||||
)
|
||||
over = evaluate(extra, {"n": outcome(stdout="a\nb\nc\n")})
|
||||
assert over.status == model.FAIL and "expected exactly 2" in over.reason
|
||||
|
||||
assert evaluate(built, {"n": outcome(stdout="")}).status == model.NOT_RUN
|
||||
assert evaluate(built, {"n": outcome(returncode=1)}).status == model.NOT_RUN
|
||||
|
||||
|
||||
def test_vantages_agree_reports_a_mismatch_as_its_own_finding() -> None:
|
||||
built = spec(
|
||||
"vantages_agree",
|
||||
{"steps": ("operator", "self")},
|
||||
steps=(step("operator", vantage="operator"), step("self")),
|
||||
)
|
||||
|
||||
agree = evaluate(
|
||||
built,
|
||||
{
|
||||
"operator": outcome(stdout="ns/a\nns/b\n"),
|
||||
"self": outcome(stdout="ns/b\nns/a\n"),
|
||||
},
|
||||
)
|
||||
assert agree.status == model.PASS
|
||||
|
||||
differ = evaluate(
|
||||
built,
|
||||
{"operator": outcome(stdout="ns/a\nns/b\n"), "self": outcome(stdout="ns/a\n")},
|
||||
)
|
||||
assert differ.status == model.FAIL and "disagree" in differ.reason
|
||||
|
||||
empty = evaluate(
|
||||
built, {"operator": outcome(stdout=""), "self": outcome(stdout="")}
|
||||
)
|
||||
assert empty.status == model.NOT_RUN
|
||||
|
||||
broken = evaluate(built, {"operator": outcome(returncode=1), "self": outcome()})
|
||||
assert broken.status == model.NOT_RUN
|
||||
|
||||
|
||||
def test_steps_agree_is_the_same_rule_under_a_neutral_name() -> None:
|
||||
assert (
|
||||
evaluators.EVALUATORS["steps_agree"] is evaluators.EVALUATORS["vantages_agree"]
|
||||
)
|
||||
|
||||
|
||||
def test_lines_match_catches_a_blank_leading_column() -> None:
|
||||
built = spec(
|
||||
"lines_match",
|
||||
{"step": "s", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
|
||||
steps=(step("s"),),
|
||||
)
|
||||
table = "Preview Src ID\n-------\nwork on task cli 2026\n"
|
||||
|
||||
assert evaluate(built, {"s": outcome(stdout=table)}).status == model.PASS
|
||||
|
||||
blank = "Preview Src ID\n-------\n cli 2026\n"
|
||||
failed = evaluate(built, {"s": outcome(stdout=blank)})
|
||||
assert failed.status == model.FAIL and failed.evidence["offenders"]
|
||||
|
||||
assert (
|
||||
evaluate(built, {"s": outcome(stdout="No sessions found.\n")}).status
|
||||
== model.NOT_RUN
|
||||
)
|
||||
assert evaluate(built, {"s": outcome(returncode=1)}).status == model.NOT_RUN
|
||||
|
||||
|
||||
def test_not_armed_is_the_fixed_skip_a_default_run_must_produce() -> None:
|
||||
built = spec("not_armed", {"reason": "ephemeral mutation mode is not armed"})
|
||||
|
||||
evaluation = evaluate(built, {})
|
||||
|
||||
assert evaluation.status == model.NOT_RUN
|
||||
assert evaluation.evidence == {"armed": False}
|
||||
assert evaluation.reason == "ephemeral mutation mode is not armed"
|
||||
assert (
|
||||
evaluate(spec("not_armed"), {}).reason == "ephemeral mutation mode is not armed"
|
||||
)
|
||||
|
||||
|
||||
def test_truncation_fails_closed_before_every_registered_evaluator() -> None:
|
||||
"""No evaluator may reinterpret a clipped required capture as PASS."""
|
||||
clipped = outcome(stdout="apparently good", truncated=True)
|
||||
for name in evaluators.EVALUATORS:
|
||||
built = spec(name, {"step": "s"}, steps=(step("s"),))
|
||||
evaluation = evaluate(built, {"s": clipped})
|
||||
assert evaluation.status == model.NOT_RUN, name
|
||||
assert "truncated" in evaluation.reason, name
|
||||
|
||||
|
||||
def test_missing_required_steps_fail_closed_inside_every_step_evaluator() -> None:
|
||||
for name in evaluators.EVALUATORS:
|
||||
if name == "not_armed":
|
||||
continue
|
||||
built = spec(name, {"step": "s"}, steps=(step("s"),))
|
||||
assert evaluate(built, {}).status == model.NOT_RUN, name
|
||||
|
||||
|
||||
def test_pool_topology_requires_three_ordinals_nodes_claims_and_ready_state() -> None:
|
||||
built = spec(
|
||||
"pool_topology",
|
||||
{
|
||||
"state_step": "state",
|
||||
"worker_step": "workers",
|
||||
"replicas": 3,
|
||||
"name": "worker",
|
||||
},
|
||||
steps=(step("state"), step("workers")),
|
||||
)
|
||||
state = outcome(stdout="3\t3\t3\tfalse")
|
||||
workers = outcome(
|
||||
stdout=(
|
||||
"worker-0\tnode-a\tRunning\tTrue\tclaim-0,\n"
|
||||
"worker-1\tnode-b\tRunning\tTrue\tclaim-1,\n"
|
||||
"worker-2\tnode-c\tRunning\tTrue\tclaim-2,\n"
|
||||
)
|
||||
)
|
||||
assert evaluate(built, {"state": state, "workers": workers}).status == model.PASS
|
||||
stacked = outcome(stdout=workers.stdout.replace("node-b", "node-a"))
|
||||
assert evaluate(built, {"state": state, "workers": stacked}).status == model.FAIL
|
||||
assert (
|
||||
evaluate(built, {"state": outcome(stdout="3\t3"), "workers": workers}).status
|
||||
== model.NOT_RUN
|
||||
)
|
||||
assert (
|
||||
evaluate(
|
||||
built, {"state": outcome(stdout="x\t3\t3\tfalse"), "workers": workers}
|
||||
).status
|
||||
== model.NOT_RUN
|
||||
)
|
||||
assert (
|
||||
evaluate(
|
||||
built, {"state": outcome(stdout="3\t2\t3\tfalse"), "workers": workers}
|
||||
).status
|
||||
== model.FAIL
|
||||
)
|
||||
malformed = outcome(stdout="worker-0\tnode-a\n")
|
||||
assert (
|
||||
evaluate(built, {"state": state, "workers": malformed}).status == model.NOT_RUN
|
||||
)
|
||||
bad_worker = outcome(stdout=workers.stdout.replace("Running", "Pending", 1))
|
||||
assert evaluate(built, {"state": state, "workers": bad_worker}).status == model.FAIL
|
||||
duplicate = outcome(stdout=workers.stdout.replace("worker-1", "worker-0"))
|
||||
assert evaluate(built, {"state": state, "workers": duplicate}).status == model.FAIL
|
||||
wrong_ordinals = outcome(stdout=workers.stdout.replace("worker-2", "worker-3"))
|
||||
assert (
|
||||
evaluate(built, {"state": state, "workers": wrong_ordinals}).status
|
||||
== model.FAIL
|
||||
)
|
||||
|
||||
|
||||
MAIN = "1" * 40
|
||||
HEAD = "2" * 40
|
||||
BUILD = MAIN
|
||||
DIGEST = "sha256:" + "4" * 64
|
||||
IMAGE = f"registry.example/hermes:git-{BUILD}-build-7@{DIGEST}"
|
||||
|
||||
|
||||
def lineage_spec():
|
||||
keys = (
|
||||
"remote-main",
|
||||
"reviewed-pr",
|
||||
"build-source",
|
||||
"deployment",
|
||||
"pods",
|
||||
"replicasets",
|
||||
"flux",
|
||||
)
|
||||
return spec(
|
||||
"release_lineage",
|
||||
{
|
||||
"main_sha": MAIN,
|
||||
"head_sha": HEAD,
|
||||
"head_ref": "feature/hermes-full-handoff-acceptance",
|
||||
"pr_number": 19,
|
||||
"image": IMAGE,
|
||||
"build_sha": BUILD,
|
||||
"deployment_revision": "7",
|
||||
},
|
||||
steps=tuple(step(key) for key in keys),
|
||||
)
|
||||
|
||||
|
||||
def lineage_outcomes():
|
||||
pull = {
|
||||
"number": 19,
|
||||
"state": "open",
|
||||
"draft": True,
|
||||
"merged": False,
|
||||
"base": {"ref": "main", "sha": MAIN},
|
||||
"head": {"ref": "feature/hermes-full-handoff-acceptance", "sha": HEAD},
|
||||
}
|
||||
return {
|
||||
"remote-main": outcome(stdout=f"{MAIN}\trefs/heads/main\n"),
|
||||
"reviewed-pr": outcome(stdout=json.dumps(pull)),
|
||||
"build-source": outcome(stdout=BUILD),
|
||||
"deployment": outcome(stdout=f"5\t5\t2\t2\t7\t{IMAGE}"),
|
||||
"replicasets": outcome(
|
||||
stdout=f"rs-old\t6\t0\t0\t{IMAGE}\told\nrs-live\t7\t2\t2\t{IMAGE}\thash7\n"
|
||||
),
|
||||
"pods": outcome(
|
||||
stdout=(
|
||||
f"pod-a\tRunning\tTrue\t{IMAGE}\tdocker-pullable://hermes@{DIGEST}\thash7\n"
|
||||
f"pod-b\tRunning\tTrue\t{IMAGE}\tcontainerd://hermes@{DIGEST}\thash7\n"
|
||||
)
|
||||
),
|
||||
"flux": outcome(stdout=f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{MAIN}"),
|
||||
}
|
||||
|
||||
|
||||
def test_release_lineage_atomically_binds_every_release_identity() -> None:
|
||||
assert evaluate(lineage_spec(), lineage_outcomes()).status == model.PASS
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("key", "bad"),
|
||||
[
|
||||
("remote-main", ""),
|
||||
("reviewed-pr", "{}"),
|
||||
("build-source", HEAD),
|
||||
("deployment", "5\t4\t2\t2\t7\tbad"),
|
||||
("replicasets", "malformed"),
|
||||
("pods", "malformed"),
|
||||
("flux", "hermes\t9\t9\tfalse\tUnknown\tmain@sha1:" + MAIN),
|
||||
],
|
||||
)
|
||||
def test_release_lineage_fails_closed_on_each_malformed_or_mismatched_source(
|
||||
key, bad
|
||||
) -> None:
|
||||
outcomes = lineage_outcomes()
|
||||
outcomes[key] = outcome(stdout=bad)
|
||||
assert evaluate(lineage_spec(), outcomes).status in {model.FAIL, model.NOT_RUN}
|
||||
|
||||
|
||||
def test_release_lineage_rejects_structural_partial_failure_variants() -> None:
|
||||
variants = []
|
||||
malformed_expect = lineage_spec()
|
||||
malformed_expect = model.CheckSpec(
|
||||
**{
|
||||
**malformed_expect.__dict__,
|
||||
"expect": {**malformed_expect.expect, "main_sha": "bad"},
|
||||
}
|
||||
)
|
||||
variants.append((malformed_expect, lineage_outcomes()))
|
||||
malformed_image = lineage_spec()
|
||||
malformed_image = model.CheckSpec(
|
||||
**{
|
||||
**malformed_image.__dict__,
|
||||
"expect": {
|
||||
**malformed_image.expect,
|
||||
"image": f"registry.example/hermes@{DIGEST}",
|
||||
},
|
||||
}
|
||||
)
|
||||
variants.append((malformed_image, lineage_outcomes()))
|
||||
for key, value in (
|
||||
("reviewed-pr", "[]"),
|
||||
("deployment", f"x\tx\t2\t2\t7\t{IMAGE}"),
|
||||
("replicasets", f"rs\t7\t1\t2\t{IMAGE}\thash7"),
|
||||
("replicasets", f"a\t7\t2\t2\t{IMAGE}\th1\nb\t7\t2\t2\t{IMAGE}\th2"),
|
||||
("replicasets", f"rs\t6\t2\t2\t{IMAGE}\thash7"),
|
||||
("pods", f"pod-a\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7"),
|
||||
(
|
||||
"pods",
|
||||
f"pod-a\tPending\tFalse\t{IMAGE}\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7",
|
||||
),
|
||||
(
|
||||
"pods",
|
||||
f"pod-a\tRunning\tTrue\tbad\tx@{DIGEST}\thash7\npod-b\tRunning\tTrue\t{IMAGE}\tx@{DIGEST}\thash7",
|
||||
),
|
||||
("flux", f"hermes\t9\t9\tfalse\tTrue\tmain@sha1:{HEAD}"),
|
||||
):
|
||||
outcomes = lineage_outcomes()
|
||||
outcomes[key] = outcome(stdout=value)
|
||||
variants.append((lineage_spec(), outcomes))
|
||||
for built, outcomes in variants:
|
||||
assert evaluate(built, outcomes).status in {model.FAIL, model.NOT_RUN}
|
||||
360
testing/tests/test_hermes_handoff_exec.py
Normal file
360
testing/tests/test_hermes_handoff_exec.py
Normal file
@ -0,0 +1,360 @@
|
||||
"""Contracts for bounded, screened command execution in the handoff harness."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from testing.tests.test_hermes_handoff_support import (
|
||||
FakeClock,
|
||||
FakeSpawn,
|
||||
load_handoff_module,
|
||||
)
|
||||
|
||||
runner_module = load_handoff_module("hermes_handoff_exec")
|
||||
policy = load_handoff_module("hermes_handoff_policy")
|
||||
|
||||
|
||||
def completed(stdout: str = "", stderr: str = "", returncode: int = 0):
|
||||
return subprocess.CompletedProcess(
|
||||
args=[], returncode=returncode, stdout=stdout, stderr=stderr
|
||||
)
|
||||
|
||||
|
||||
def build(results: list, **kwargs) -> tuple:
|
||||
clock = FakeClock()
|
||||
spawn = FakeSpawn(results)
|
||||
runner = runner_module.Runner(
|
||||
clock=clock,
|
||||
spawn=spawn,
|
||||
environ={},
|
||||
attestor=lambda command, _environment: runner_module.Attestation(
|
||||
command, "a" * 64
|
||||
),
|
||||
**kwargs,
|
||||
)
|
||||
return (runner, spawn, clock)
|
||||
|
||||
|
||||
OPERATOR = runner_module.operator_vantage()
|
||||
SAFE_GET = ("kubectl", "get", "pods", "-o", "name")
|
||||
|
||||
|
||||
def test_a_successful_command_is_captured_and_screened() -> None:
|
||||
runner, spawn, _ = build([completed(stdout="password=a-long-enough-secret\n")])
|
||||
|
||||
outcome = runner.run(SAFE_GET, OPERATOR)
|
||||
|
||||
assert outcome.ok and outcome.ran
|
||||
assert "[redacted]" in outcome.stdout
|
||||
assert outcome.returncode == 0
|
||||
assert spawn.calls[0]["argv"] == list(SAFE_GET)
|
||||
assert runner.commands_run == 1
|
||||
|
||||
|
||||
def test_a_failing_command_is_not_ok_but_still_ran() -> None:
|
||||
runner, _, _ = build(
|
||||
[completed(stderr="Error from server (Forbidden)", returncode=1)]
|
||||
)
|
||||
|
||||
outcome = runner.run(SAFE_GET, OPERATOR)
|
||||
|
||||
assert outcome.ran and not outcome.ok
|
||||
assert "Forbidden" in outcome.combined
|
||||
|
||||
|
||||
def test_a_policy_violation_never_reaches_a_subprocess() -> None:
|
||||
runner, spawn, _ = build([])
|
||||
|
||||
outcome = runner.run(("kubectl", "delete", "pods", "--all"), OPERATOR)
|
||||
|
||||
assert outcome.error and outcome.error.startswith("policy:")
|
||||
assert not outcome.ran
|
||||
assert spawn.calls == []
|
||||
assert runner.commands_run == 0
|
||||
|
||||
|
||||
def test_a_timeout_is_recorded_rather_than_raised() -> None:
|
||||
runner, _, _ = build([subprocess.TimeoutExpired(cmd="kubectl", timeout=1.0)])
|
||||
|
||||
outcome = runner.run(SAFE_GET, OPERATOR)
|
||||
|
||||
assert outcome.error == runner_module.TIMEOUT_ERROR
|
||||
assert not outcome.ran
|
||||
|
||||
|
||||
def test_a_missing_binary_is_recorded_rather_than_raised() -> None:
|
||||
runner, _, _ = build([FileNotFoundError(2, "No such file or directory")])
|
||||
|
||||
outcome = runner.run(SAFE_GET, OPERATOR)
|
||||
|
||||
assert outcome.error and outcome.error.startswith("spawn:")
|
||||
|
||||
|
||||
def test_the_run_deadline_stops_further_probes_without_spawning() -> None:
|
||||
runner, spawn, clock = build([completed(stdout="first")], deadline_seconds=10.0)
|
||||
|
||||
assert runner.run(SAFE_GET, OPERATOR).ok
|
||||
clock.advance(11.0)
|
||||
second = runner.run(("kubectl", "get", "nodes", "-o", "name"), OPERATOR)
|
||||
|
||||
assert second.error == runner_module.DEADLINE_ERROR
|
||||
assert len(spawn.calls) == 1
|
||||
assert runner.remaining_seconds < 0
|
||||
|
||||
|
||||
def test_the_command_timeout_is_clamped_by_the_remaining_deadline() -> None:
|
||||
runner, spawn, clock = build(
|
||||
[completed()], command_timeout=30.0, deadline_seconds=10.0
|
||||
)
|
||||
clock.advance(6.0)
|
||||
|
||||
runner.run(SAFE_GET, OPERATOR)
|
||||
|
||||
assert spawn.calls[0]["timeout"] == 4.0
|
||||
|
||||
|
||||
def test_output_is_bounded_per_command_and_can_be_raised_per_call() -> None:
|
||||
runner, _, _ = build(
|
||||
[completed(stdout="y" * 200), completed(stdout="y" * 200)], max_bytes=50
|
||||
)
|
||||
|
||||
small = runner.run(SAFE_GET, OPERATOR)
|
||||
large = runner.run(SAFE_GET, OPERATOR, max_bytes=4096)
|
||||
|
||||
assert small.truncated
|
||||
assert not large.truncated
|
||||
|
||||
|
||||
def test_the_child_environment_is_rebuilt_from_an_allowlist() -> None:
|
||||
source = {
|
||||
"PATH": "/bin",
|
||||
"KUBECONFIG": "/tmp/kc",
|
||||
"ANTHROPIC_API_KEY": "leak",
|
||||
"HOME": "/root",
|
||||
}
|
||||
environment = runner_module.build_environment(OPERATOR, source)
|
||||
|
||||
assert environment["PATH"] == runner_module.SAFE_PATH
|
||||
assert environment["KUBECONFIG"] == "/tmp/kc"
|
||||
assert environment["GITEA_BASE_URL"] == "https://scm.bstein.dev"
|
||||
assert environment["GIT_CONFIG_GLOBAL"] == "/dev/null"
|
||||
assert "HOME" not in environment
|
||||
assert "ANTHROPIC_API_KEY" not in environment
|
||||
|
||||
|
||||
def test_a_vantage_can_override_the_environment_it_needs() -> None:
|
||||
vantage = runner_module.operator_vantage(kubeconfig="/tmp/operator.yaml")
|
||||
environment = runner_module.build_environment(vantage, {"KUBECONFIG": "/tmp/other"})
|
||||
|
||||
assert environment["KUBECONFIG"] == "/tmp/operator.yaml"
|
||||
assert "kubeconfig-pinned" in vantage.description
|
||||
|
||||
|
||||
def test_the_operator_context_is_inserted_only_for_context_aware_tools() -> None:
|
||||
vantage = runner_module.operator_vantage(context="atlas-operator")
|
||||
|
||||
assert vantage.wrap(("kubectl", "get", "pods"))[:3] == (
|
||||
"kubectl",
|
||||
"--context",
|
||||
"atlas-operator",
|
||||
)
|
||||
assert vantage.wrap(("git", "status")) == ("git", "status")
|
||||
assert vantage.wrap(()) == ()
|
||||
|
||||
|
||||
def test_the_pod_vantage_execs_into_the_container_and_drops_the_context() -> None:
|
||||
operator = runner_module.operator_vantage(context="atlas-operator")
|
||||
vantage = runner_module.pod_vantage("hermes", "hermes-agent-1", "hermes", operator)
|
||||
|
||||
wrapped = vantage.wrap(SAFE_GET)
|
||||
|
||||
assert wrapped[:3] == ("kubectl", "--context", "atlas-operator")
|
||||
assert wrapped[3] == "exec"
|
||||
assert wrapped[-6:] == (
|
||||
"--",
|
||||
"/usr/local/bin/kubectl",
|
||||
"get",
|
||||
"pods",
|
||||
"-o",
|
||||
"name",
|
||||
)
|
||||
assert vantage.description == "in-pod hermes/hermes-agent-1[hermes]"
|
||||
policy.check_argv(wrapped)
|
||||
|
||||
|
||||
def test_a_pod_vantage_without_an_operator_still_addresses_the_pod() -> None:
|
||||
vantage = runner_module.pod_vantage("hermes", "pod-a", "hermes")
|
||||
|
||||
assert vantage.wrap(("kubectl", "version"))[0] == "kubectl"
|
||||
assert vantage.env == {}
|
||||
|
||||
|
||||
def test_outcome_serialisation_screens_the_command_line() -> None:
|
||||
outcome = runner_module.Outcome(
|
||||
argv=("git", "clone", "https://user:a-long-enough-secret@scm.example.dev/x"),
|
||||
vantage="operator",
|
||||
returncode=0,
|
||||
stdout="ok",
|
||||
)
|
||||
|
||||
payload = outcome.as_dict()
|
||||
|
||||
assert "a-long-enough-secret" not in payload["command"]
|
||||
assert payload["vantage"] == "operator"
|
||||
assert payload["stdout"] == "ok"
|
||||
|
||||
|
||||
def test_combined_output_joins_only_the_streams_that_carry_text() -> None:
|
||||
assert (
|
||||
runner_module.Outcome(argv=(), vantage="x", stdout="a", stderr="b").combined
|
||||
== "a\nb"
|
||||
)
|
||||
assert runner_module.Outcome(argv=(), vantage="x", stdout="a").combined == "a"
|
||||
assert runner_module.Outcome(argv=(), vantage="x").combined == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs",
|
||||
[
|
||||
{"command_timeout": float("nan")},
|
||||
{"command_timeout": float("inf")},
|
||||
{"command_timeout": 0},
|
||||
{"deadline_seconds": -1},
|
||||
{"max_bytes": -1},
|
||||
{"max_bytes": 10**100},
|
||||
],
|
||||
)
|
||||
def test_execution_bounds_reject_nonfinite_nonpositive_or_unbounded_inputs(
|
||||
kwargs,
|
||||
) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
runner_module.Runner(**kwargs)
|
||||
|
||||
|
||||
def test_execution_bounds_reject_boolean_and_nonnumeric_inputs() -> None:
|
||||
for value in (True, "1", None):
|
||||
with pytest.raises(ValueError, match="numeric"):
|
||||
runner_module.Runner(command_timeout=value)
|
||||
|
||||
|
||||
def test_every_reviewer_unsafe_argv_is_refused_without_spawning() -> None:
|
||||
runner, spawn, _ = build([])
|
||||
commands = [
|
||||
("kubectl", "auth", "reconcile", "-f", "x"),
|
||||
("kubectl", "config", "view", "--raw"),
|
||||
("kubectl", "get", "secret/x", "-o", "name"),
|
||||
("kubectl", "get", "--raw=/api/v1/secrets"),
|
||||
("kubectl", "delete", "pod/x", "--dry-run=client"),
|
||||
("kubectl", "delete", "pod/x", "--", "--dry-run=server"),
|
||||
("git", "fetch", "origin"),
|
||||
("git", "config", "core.sshCommand", "x"),
|
||||
("git", "-ccore.sshCommand=x", "ls-remote", "origin"),
|
||||
]
|
||||
for command in commands:
|
||||
assert runner.run(command, OPERATOR).error.startswith("policy:")
|
||||
assert spawn.calls == []
|
||||
assert runner.commands_run == 0
|
||||
|
||||
|
||||
def test_live_capture_bounds_concurrent_stdout_and_stderr() -> None:
|
||||
capture = runner_module.execute_bounded(
|
||||
(
|
||||
"/usr/bin/python3",
|
||||
"-c",
|
||||
"import os;os.write(1,b'x'*2000000);os.write(2,b'y'*2000000)",
|
||||
),
|
||||
{"LC_ALL": "C"},
|
||||
timeout=5,
|
||||
max_bytes=1024,
|
||||
)
|
||||
assert capture.returncode == 0
|
||||
assert capture.stdout_truncated and capture.stderr_truncated
|
||||
assert len(capture.stdout) == len(capture.stderr) == 1024
|
||||
|
||||
|
||||
def test_absolute_timeout_kills_the_process_group_and_reaps_quickly() -> None:
|
||||
started = time.monotonic()
|
||||
capture = runner_module.execute_bounded(
|
||||
(
|
||||
"/usr/bin/python3",
|
||||
"-c",
|
||||
"import os,signal,time;"
|
||||
"p=os.fork();"
|
||||
"signal.signal(signal.SIGTERM,signal.SIG_IGN);"
|
||||
"time.sleep(30)",
|
||||
),
|
||||
{"LC_ALL": "C"},
|
||||
timeout=0.15,
|
||||
max_bytes=1024,
|
||||
)
|
||||
assert capture.timed_out
|
||||
assert time.monotonic() - started < 1.5
|
||||
|
||||
|
||||
def test_descendant_holding_pipes_is_bounded_after_the_leader_exits() -> None:
|
||||
started = time.monotonic()
|
||||
capture = runner_module.execute_bounded(
|
||||
(
|
||||
"/usr/bin/python3",
|
||||
"-c",
|
||||
"import os,signal,time;"
|
||||
"p=os.fork();"
|
||||
"(os._exit(0) if p else None);"
|
||||
"signal.signal(signal.SIGTERM,signal.SIG_IGN);"
|
||||
"time.sleep(30)",
|
||||
),
|
||||
{"LC_ALL": "C"},
|
||||
timeout=2,
|
||||
max_bytes=1024,
|
||||
)
|
||||
assert not capture.timed_out
|
||||
assert time.monotonic() - started < 1.5
|
||||
|
||||
|
||||
def test_attestation_rejects_a_reviewer_authored_executable(tmp_path) -> None:
|
||||
fake = tmp_path / "kubectl"
|
||||
fake.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
|
||||
fake.chmod(0o755)
|
||||
with pytest.raises(OSError, match="trusted paths"):
|
||||
runner_module.attest_executable(str(fake), {"PATH": str(tmp_path)})
|
||||
|
||||
|
||||
def test_attestation_checks_presence_mode_digest_and_success(
|
||||
tmp_path, monkeypatch
|
||||
) -> None:
|
||||
fake = tmp_path / "tool"
|
||||
fake.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
|
||||
fake.chmod(0o755)
|
||||
digest = hashlib.sha256(fake.read_bytes()).hexdigest()
|
||||
monkeypatch.setitem(runner_module.EXPECTED_PATHS, "tool", {str(fake)})
|
||||
monkeypatch.setitem(runner_module.EXPECTED_SHA256, "tool", {digest})
|
||||
assert (
|
||||
runner_module.attest_executable("tool", {"PATH": str(tmp_path)}).sha256
|
||||
== digest
|
||||
)
|
||||
with pytest.raises(OSError, match="unavailable"):
|
||||
runner_module.attest_executable("missing", {"PATH": str(tmp_path)})
|
||||
fake.chmod(0o777)
|
||||
with pytest.raises(OSError, match="unsafe ownership mode"):
|
||||
runner_module.attest_executable("tool", {"PATH": str(tmp_path)})
|
||||
fake.chmod(0o755)
|
||||
monkeypatch.setitem(runner_module.EXPECTED_SHA256, "tool", {"0" * 64})
|
||||
with pytest.raises(OSError, match="digest"):
|
||||
runner_module.attest_executable("tool", {"PATH": str(tmp_path)})
|
||||
|
||||
|
||||
def test_runner_attestation_or_resolved_policy_failure_never_spawns() -> None:
|
||||
runner, spawn, _ = build([])
|
||||
runner._attestor = lambda _command, _environment: (_ for _ in ()).throw(
|
||||
OSError("no")
|
||||
)
|
||||
assert runner.run(SAFE_GET, OPERATOR).error.startswith("attestation:")
|
||||
runner._attestor = lambda _command, _environment: runner_module.Attestation(
|
||||
"/tmp/reviewer-authored/kubectl", "a" * 64
|
||||
)
|
||||
assert runner.run(SAFE_GET, OPERATOR).error.startswith("policy:")
|
||||
assert spawn.calls == []
|
||||
413
testing/tests/test_hermes_handoff_json_rules.py
Normal file
413
testing/tests/test_hermes_handoff_json_rules.py
Normal file
@ -0,0 +1,413 @@
|
||||
"""Contracts for the JSON-shaped handoff acceptance evaluators."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from testing.tests.test_hermes_handoff_support import (
|
||||
load_handoff_module,
|
||||
outcome,
|
||||
spec,
|
||||
step,
|
||||
)
|
||||
|
||||
evaluators = load_handoff_module("hermes_handoff_evaluators")
|
||||
model = load_handoff_module("hermes_handoff_model")
|
||||
|
||||
NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc)
|
||||
|
||||
|
||||
def evaluate(built, payload):
|
||||
body = payload if isinstance(payload, str) else json.dumps(payload)
|
||||
return evaluators.evaluate(built, {"s": outcome(stdout=body)})
|
||||
|
||||
|
||||
def one(rule: str, expect: dict):
|
||||
return spec(rule, {"step": "s", **expect}, steps=(step("s"),))
|
||||
|
||||
|
||||
def test_json_field_compares_paths_against_expected_values() -> None:
|
||||
built = one(
|
||||
"json_field", {"fields": {"authenticated": True, "transport": ["a", "b"]}}
|
||||
)
|
||||
|
||||
assert (
|
||||
evaluate(built, {"authenticated": True, "transport": "b"}).status == model.PASS
|
||||
)
|
||||
|
||||
wrong = evaluate(built, {"authenticated": False, "transport": "c"})
|
||||
assert wrong.status == model.FAIL
|
||||
assert "authenticated=False" in wrong.reason and "transport='c'" in wrong.reason
|
||||
|
||||
assert evaluate(built, {"authenticated": True}).status == model.NOT_RUN
|
||||
|
||||
|
||||
def test_json_field_skips_rather_than_guesses_on_unusable_output() -> None:
|
||||
built = one("json_field", {"fields": {"a": 1}})
|
||||
|
||||
assert evaluate(built, "not json").status == model.NOT_RUN
|
||||
failed = evaluators.evaluate(built, {"s": outcome(returncode=1)})
|
||||
assert failed.status == model.NOT_RUN
|
||||
assert evaluate(built, '{"a": 1, "a": 1}').status == model.NOT_RUN
|
||||
assert evaluate(built, '{"a": NaN}').status == model.NOT_RUN
|
||||
|
||||
|
||||
def test_json_equality_does_not_confuse_booleans_and_integers() -> None:
|
||||
built = one("json_field", {"fields": {"enabled": True}})
|
||||
assert evaluate(built, {"enabled": 1}).status == model.FAIL
|
||||
array = one("all_of_field", {"fields": {"enabled": False}})
|
||||
assert evaluate(array, [{"enabled": 0}]).status == model.FAIL
|
||||
|
||||
|
||||
def test_json_numeric_enforces_inclusive_bounds_and_rejects_non_numbers() -> None:
|
||||
built = one("json_numeric", {"fields": {"latency_ms": {"min": 1, "max": 1000}}})
|
||||
|
||||
assert evaluate(built, {"latency_ms": 500}).status == model.PASS
|
||||
assert evaluate(built, {"latency_ms": 1}).status == model.PASS
|
||||
|
||||
low = evaluate(built, {"latency_ms": 0})
|
||||
assert low.status == model.FAIL and "below minimum" in low.reason
|
||||
|
||||
high = evaluate(built, {"latency_ms": 5000})
|
||||
assert high.status == model.FAIL and "above maximum" in high.reason
|
||||
|
||||
assert evaluate(built, {"latency_ms": True}).status == model.NOT_RUN
|
||||
assert evaluate(built, {"latency_ms": "fast"}).status == model.NOT_RUN
|
||||
assert evaluate(built, {}).status == model.NOT_RUN
|
||||
|
||||
|
||||
def test_json_recent_treats_stale_evidence_as_a_failure() -> None:
|
||||
built = one("json_recent", {"now": NOW, "fields": {"checked_at": 3600}})
|
||||
|
||||
fresh = evaluate(built, {"checked_at": "2026-08-17T11:30:00Z"})
|
||||
assert (
|
||||
fresh.status == model.PASS
|
||||
and fresh.evidence["checked_at"]["age_seconds"] == 1800
|
||||
)
|
||||
|
||||
stale = evaluate(built, {"checked_at": "2026-08-16T00:00:00Z"})
|
||||
assert stale.status == model.FAIL and "limit 3600s" in stale.reason
|
||||
|
||||
naive = evaluate(built, {"checked_at": "2026-08-17T11:30:00"})
|
||||
assert naive.status == model.NOT_RUN
|
||||
assert evaluate(built, {"checked_at": "2026-08-17T12:00:01Z"}).status == model.FAIL
|
||||
|
||||
assert evaluate(built, {"checked_at": "yesterday"}).status == model.NOT_RUN
|
||||
assert evaluate(built, {}).status == model.NOT_RUN
|
||||
invalid_clock = one("json_recent", {"now": "now", "fields": {"checked_at": 1}})
|
||||
assert (
|
||||
evaluate(invalid_clock, {"checked_at": "2026-08-17T11:30:00Z"}).status
|
||||
== model.NOT_RUN
|
||||
)
|
||||
|
||||
|
||||
def test_all_of_field_selects_keys_and_reports_every_mismatch() -> None:
|
||||
built = one(
|
||||
"all_of_field",
|
||||
{"key_field": "number", "keys": [14, 15], "fields": {"merged": True}},
|
||||
)
|
||||
payload = [
|
||||
{"number": 14, "merged": True},
|
||||
{"number": 15, "merged": False},
|
||||
{"number": 16, "merged": False},
|
||||
]
|
||||
|
||||
evaluation = evaluate(built, payload)
|
||||
|
||||
assert evaluation.status == model.FAIL
|
||||
assert "15: merged=False" in evaluation.reason
|
||||
assert "16" not in evaluation.reason
|
||||
assert evaluation.evidence["examined"] == [14, 15]
|
||||
|
||||
|
||||
def test_all_of_field_skips_when_an_expected_key_is_absent_or_nothing_matched() -> None:
|
||||
keyed = one(
|
||||
"all_of_field",
|
||||
{"key_field": "number", "keys": [14], "fields": {"merged": True}},
|
||||
)
|
||||
absent = evaluate(keyed, [{"number": 99, "merged": True}])
|
||||
assert absent.status == model.NOT_RUN and "expected keys absent" in absent.reason
|
||||
|
||||
unkeyed = one("all_of_field", {"fields": {"status": ["done"]}})
|
||||
assert evaluate(unkeyed, []).status == model.NOT_RUN
|
||||
assert evaluate(unkeyed, {"not": "an array"}).status == model.NOT_RUN
|
||||
|
||||
|
||||
def test_all_of_field_skips_when_the_named_array_path_is_absent() -> None:
|
||||
built = spec(
|
||||
"all_of_field",
|
||||
{"step": "s", "array": "items", "fields": {"a": 1}},
|
||||
steps=(step("s"),),
|
||||
)
|
||||
|
||||
evaluation = evaluate(built, {"other": []})
|
||||
|
||||
assert evaluation.status == model.NOT_RUN
|
||||
assert "array path is absent" in evaluation.reason
|
||||
|
||||
|
||||
def test_all_of_field_treats_an_absent_field_as_a_mismatch_not_a_crash() -> None:
|
||||
built = one("all_of_field", {"key_field": "name", "fields": {"suspended": False}})
|
||||
|
||||
evaluation = evaluate(built, [{"name": "a"}])
|
||||
|
||||
assert evaluation.status == model.FAIL
|
||||
assert "suspended=None" in evaluation.reason
|
||||
|
||||
|
||||
def test_all_of_field_honours_exemptions_and_non_empty_paths() -> None:
|
||||
exempted = one(
|
||||
"all_of_field",
|
||||
{
|
||||
"key_field": "name",
|
||||
"exempt_keys": ["parked"],
|
||||
"fields": {"suspended": False},
|
||||
},
|
||||
)
|
||||
payload = [
|
||||
{"name": "parked", "suspended": True},
|
||||
{"name": "live", "suspended": False},
|
||||
]
|
||||
assert evaluate(exempted, payload).status == model.PASS
|
||||
|
||||
non_empty = one("all_of_field", {"key_field": "id", "non_empty": ["title"]})
|
||||
assert evaluate(non_empty, [{"id": "a", "title": "work"}]).status == model.PASS
|
||||
blank = evaluate(non_empty, [{"id": "a", "title": ""}, {"id": "b"}])
|
||||
assert blank.status == model.FAIL and "is empty" in blank.reason
|
||||
|
||||
|
||||
def test_json_record_captures_named_paths_or_skips() -> None:
|
||||
built = one(
|
||||
"json_record", {"record": {"numbers": "[].number", "refs": "[].head.ref"}}
|
||||
)
|
||||
payload = [
|
||||
{"number": 14, "head": {"ref": "a"}},
|
||||
{"number": 15, "head": {"ref": "b"}},
|
||||
]
|
||||
|
||||
evaluation = evaluate(built, payload)
|
||||
assert evaluation.status == model.PASS
|
||||
assert evaluation.evidence == {"numbers": [14, 15], "refs": ["a", "b"]}
|
||||
|
||||
missing = evaluate(built, [{"number": 14}])
|
||||
assert missing.status == model.NOT_RUN and "absent" in missing.reason
|
||||
|
||||
|
||||
def routing_log(*records: dict) -> str:
|
||||
return "\n".join(json.dumps(record) for record in records)
|
||||
|
||||
|
||||
def routing_spec(**extra):
|
||||
return spec(
|
||||
"routing_evidence",
|
||||
{
|
||||
"step": "s",
|
||||
"providers": ("codex", "claude"),
|
||||
"efforts": ("high", "xhigh"),
|
||||
"lanes": ("route", "worker"),
|
||||
"require_fallback_evidence": True,
|
||||
**extra,
|
||||
},
|
||||
steps=(step("s"),),
|
||||
)
|
||||
|
||||
|
||||
FULL_LOG = routing_log(
|
||||
{"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"},
|
||||
{"ts": "2026-08-17T11:30:00Z", "model": "route/claude/opus/xhigh"},
|
||||
{
|
||||
"ts": "2026-08-17T11:45:00Z",
|
||||
"model": "worker/codex/sol/xhigh",
|
||||
"fallback_reason": "unavailable",
|
||||
},
|
||||
{"ts": "2026-08-17T11:50:00Z", "model": "qwen2.5:14b", "tier": "classifier"},
|
||||
)
|
||||
|
||||
|
||||
def test_routing_evidence_summarises_lanes_providers_efforts_and_fallbacks() -> None:
|
||||
evaluation = evaluate(routing_spec(), FULL_LOG)
|
||||
|
||||
assert evaluation.status == model.PASS
|
||||
assert evaluation.evidence["providers"] == ["claude", "codex"]
|
||||
assert evaluation.evidence["efforts"] == ["high", "xhigh"]
|
||||
assert evaluation.evidence["lanes"] == ["route", "worker"]
|
||||
assert evaluation.evidence["fallback_reasons"] == ["unavailable"]
|
||||
assert evaluation.evidence["records"] == 4 and evaluation.evidence["routed"] == 3
|
||||
|
||||
|
||||
def test_routing_evidence_names_exactly_what_the_window_did_not_show() -> None:
|
||||
partial = routing_log(
|
||||
{"ts": "2026-08-17T11:00:00Z", "model": "route/codex/sol/high"}
|
||||
)
|
||||
|
||||
evaluation = evaluate(routing_spec(), partial)
|
||||
|
||||
assert evaluation.status == model.FAIL
|
||||
assert "provider:claude" in evaluation.reason
|
||||
assert "effort:xhigh" in evaluation.reason
|
||||
assert "lane:worker" in evaluation.reason
|
||||
assert "fallback:none-observed" in evaluation.reason
|
||||
|
||||
|
||||
def test_routing_evidence_rejects_a_clipped_tail_and_empty_input() -> None:
|
||||
clipped = '{"ts": "2026-08-17T11:0\n' + FULL_LOG
|
||||
|
||||
assert evaluate(routing_spec(), clipped).status == model.NOT_RUN
|
||||
assert evaluate(routing_spec(), "").status == model.NOT_RUN
|
||||
assert evaluate(routing_spec(), "garbage\n").status == model.NOT_RUN
|
||||
|
||||
|
||||
def test_routing_evidence_can_require_freshness() -> None:
|
||||
fresh = evaluate(routing_spec(max_age_seconds=3600, now=NOW), FULL_LOG)
|
||||
assert fresh.status == model.PASS
|
||||
assert max(fresh.evidence["age_seconds"].values()) <= 3600
|
||||
|
||||
stale = evaluate(
|
||||
routing_spec(max_age_seconds=60, now=NOW + dt.timedelta(days=2)), FULL_LOG
|
||||
)
|
||||
assert stale.status == model.FAIL and "freshness" in stale.reason
|
||||
|
||||
undated = routing_log(
|
||||
{"model": "route/codex/sol/high"}, {"model": "route/claude/opus/xhigh"}
|
||||
)
|
||||
unknown = evaluate(routing_spec(max_age_seconds=60, now=NOW), undated)
|
||||
assert unknown.status == model.NOT_RUN
|
||||
|
||||
|
||||
def test_routing_freshness_is_required_per_provider_not_only_globally() -> None:
|
||||
rows = routing_log(
|
||||
{"ts": "2026-08-15T11:00:00Z", "model": "route/claude/opus/high"},
|
||||
{"ts": "2026-08-17T11:59:00Z", "model": "route/codex/sol/xhigh"},
|
||||
{
|
||||
"ts": "2026-08-17T11:59:00Z",
|
||||
"model": "worker/codex/sol/high",
|
||||
"fallback_reason": "down",
|
||||
},
|
||||
)
|
||||
evaluation = evaluate(routing_spec(max_age_seconds=3600, now=NOW), rows)
|
||||
assert evaluation.status == model.FAIL
|
||||
assert "freshness:provider:claude" in evaluation.reason
|
||||
missing_dimension = evaluate(
|
||||
routing_spec(max_age_seconds=3600, now=NOW),
|
||||
routing_log(
|
||||
{"ts": "2026-08-17T11:59:00Z", "model": "route/codex/sol/high"},
|
||||
{
|
||||
"ts": "2026-08-17T11:59:00Z",
|
||||
"model": "worker/codex/sol/xhigh",
|
||||
"fallback_reason": "down",
|
||||
},
|
||||
),
|
||||
)
|
||||
assert missing_dimension.status == model.FAIL
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body",
|
||||
[
|
||||
'["not-an-object"]',
|
||||
'{"ts":"2026-08-17T12:00:01Z","model":"route/codex/sol/high"}',
|
||||
'{"ts":"2026-08-17T11:00:00Z"}',
|
||||
'{"ts":"2026-08-17T11:00:00Z","model":"bad"}',
|
||||
'{"ts":"2026-08-17T11:00:00Z","model":"route/codex/sol/high","fallback_reason":0}',
|
||||
],
|
||||
)
|
||||
def test_routing_rows_fail_closed_on_structural_defects(body: str) -> None:
|
||||
assert evaluate(routing_spec(now=NOW), body).status in {model.FAIL, model.NOT_RUN}
|
||||
invalid_clock = routing_spec(now="now")
|
||||
assert evaluate(invalid_clock, FULL_LOG).status == model.NOT_RUN
|
||||
|
||||
|
||||
def flux_spec(*expected: str):
|
||||
return spec(
|
||||
"flux_health",
|
||||
{"step": "s", "expected_suspensions": expected},
|
||||
steps=(step("s"),),
|
||||
)
|
||||
|
||||
|
||||
def test_flux_health_separates_parked_objects_from_broken_ones() -> None:
|
||||
rows = (
|
||||
"flux-system/core\t1\t1\tfalse\tReady=True,Healthy=True,\n"
|
||||
"flux-system/parked\t1\t1\ttrue\tReady=False,\n"
|
||||
"flux-system/broken\t1\t1\tfalse\tReady=False,Healthy=False,\n"
|
||||
)
|
||||
|
||||
evaluation = evaluators.evaluate(
|
||||
flux_spec("flux-system/parked"), {"s": outcome(stdout=rows)}
|
||||
)
|
||||
|
||||
assert evaluation.status == model.FAIL
|
||||
assert evaluation.evidence["unhealthy"] == ["flux-system/broken"]
|
||||
assert evaluation.evidence["unexpected_suspensions"] == []
|
||||
assert "unhealthy: flux-system/broken" in evaluation.reason
|
||||
|
||||
|
||||
def test_flux_health_flags_a_suspension_nobody_declared() -> None:
|
||||
rows = "flux-system/core\t1\t1\tfalse\tReady=True,\nflux-system/parked\t1\t1\ttrue\tReady=False,\n"
|
||||
|
||||
evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)})
|
||||
|
||||
assert evaluation.status == model.FAIL
|
||||
assert evaluation.evidence["unexpected_suspensions"] == ["flux-system/parked"]
|
||||
|
||||
|
||||
def test_flux_health_fails_closed_on_in_flight_or_unknown_conditions() -> None:
|
||||
rows = (
|
||||
"flux-system/reconciling\t1\t1\tfalse\tReconciling=True,Ready=False,\n"
|
||||
"flux-system/degraded\t1\t1\tfalse\tReady=False,Healthy=True,\n"
|
||||
"flux-system/waiting\t1\t1\tfalse\tReady=Unknown,\n"
|
||||
)
|
||||
|
||||
evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)})
|
||||
|
||||
assert evaluation.status == model.FAIL
|
||||
assert evaluation.evidence["total"] == 3
|
||||
|
||||
|
||||
def test_blank_rows_in_a_projection_are_ignored_by_both_log_parsers() -> None:
|
||||
padded = "\n\n" + FULL_LOG + "\n\n"
|
||||
assert evaluate(routing_spec(), padded).status == model.PASS
|
||||
|
||||
rows = "\n\nflux-system/core\t1\t1\tfalse\tReady=True,\n\n"
|
||||
evaluation = evaluators.evaluate(flux_spec(), {"s": outcome(stdout=rows)})
|
||||
assert evaluation.status == model.PASS and evaluation.evidence["total"] == 1
|
||||
|
||||
|
||||
def test_flux_health_skips_on_an_empty_or_failed_projection() -> None:
|
||||
assert (
|
||||
evaluators.evaluate(flux_spec(), {"s": outcome(stdout="")}).status
|
||||
== model.NOT_RUN
|
||||
)
|
||||
assert (
|
||||
evaluators.evaluate(flux_spec(), {"s": outcome(returncode=1)}).status
|
||||
== model.NOT_RUN
|
||||
)
|
||||
malformed = "flux-system/core\t1\t0\tfalse\tReady=True,\n"
|
||||
assert (
|
||||
evaluators.evaluate(flux_spec(), {"s": outcome(stdout=malformed)}).status
|
||||
== model.FAIL
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"row",
|
||||
[
|
||||
"too\tfew",
|
||||
"BAD NAME\t1\t1\tfalse\tReady=True,",
|
||||
"flux-system/core\tx\t1\tfalse\tReady=True,",
|
||||
"flux-system/core\t1\t1\tmaybe\tReady=True,",
|
||||
"flux-system/core\t1\t1\tfalse\tbroken,",
|
||||
"flux-system/core\t1\t1\tfalse\tReady=Yes,",
|
||||
"flux-system/core\t1\t1\tfalse\tReady=True,Ready=True,",
|
||||
],
|
||||
)
|
||||
def test_flux_rows_reject_malformed_columns_names_generations_and_conditions(
|
||||
row: str,
|
||||
) -> None:
|
||||
assert (
|
||||
evaluators.evaluate(flux_spec(), {"s": outcome(stdout=row)}).status
|
||||
== model.NOT_RUN
|
||||
)
|
||||
206
testing/tests/test_hermes_handoff_model.py
Normal file
206
testing/tests/test_hermes_handoff_model.py
Normal file
@ -0,0 +1,206 @@
|
||||
"""Contracts for the handoff harness result model, verdict, and rendering."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
|
||||
import pytest
|
||||
|
||||
from testing.tests.test_hermes_handoff_support import (
|
||||
load_handoff_module,
|
||||
outcome,
|
||||
spec,
|
||||
step,
|
||||
)
|
||||
|
||||
model = load_handoff_module("hermes_handoff_model")
|
||||
|
||||
|
||||
def result(
|
||||
status: str,
|
||||
identifier: str = "test.check",
|
||||
mandatory: bool = True,
|
||||
group: str = "g",
|
||||
):
|
||||
return model.CheckResult(
|
||||
spec=model.CheckSpec(
|
||||
id=identifier, title="t", group=group, rule="allowed", mandatory=mandatory
|
||||
),
|
||||
status=status,
|
||||
reason=f"{status} reason",
|
||||
)
|
||||
|
||||
|
||||
def report(*results, errors: tuple[str, ...] = ()) -> model.Report:
|
||||
return model.Report(
|
||||
mode="read-only",
|
||||
started_at="2026-08-17T00:00:00Z",
|
||||
results=list(results),
|
||||
harness_errors=list(errors),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("status", "mandatory", "blocking"),
|
||||
[
|
||||
(model.PASS, True, False),
|
||||
(model.NOT_APPLICABLE, True, False),
|
||||
(model.FAIL, True, True),
|
||||
(model.NOT_RUN, True, True),
|
||||
(model.FAIL, False, False),
|
||||
(model.NOT_RUN, False, False),
|
||||
],
|
||||
)
|
||||
def test_only_a_mandatory_failure_or_skip_blocks(status, mandatory, blocking) -> None:
|
||||
assert result(status, mandatory=mandatory).blocking is blocking
|
||||
|
||||
|
||||
def test_a_mandatory_skip_is_a_no_go_exactly_like_a_failure() -> None:
|
||||
"""The whole point of the four-valued classification."""
|
||||
assert report(result(model.NOT_RUN)).decision == model.NO_GO
|
||||
assert report(result(model.FAIL)).decision == model.NO_GO
|
||||
assert report(result(model.PASS), result(model.NOT_APPLICABLE)).decision == model.GO
|
||||
|
||||
|
||||
def test_a_harness_error_is_a_no_go_even_when_every_check_passed() -> None:
|
||||
clean = report(
|
||||
result(model.PASS), errors=("the two vantages are the same principal",)
|
||||
)
|
||||
assert clean.decision == model.NO_GO
|
||||
|
||||
|
||||
def test_a_zero_state_or_unknown_status_report_is_no_go() -> None:
|
||||
"""Absence or an unrecognised classification can never round up to GO."""
|
||||
assert report().decision == model.NO_GO
|
||||
assert report(errors=("no checks were built",)).decision == model.NO_GO
|
||||
unknown = result("MAYBE")
|
||||
assert unknown.blocking is True
|
||||
built = report(unknown)
|
||||
assert built.decision == model.NO_GO
|
||||
assert built.counts == dict.fromkeys(model.STATUSES, 0)
|
||||
assert "groups:" in report(result(model.PASS)).render_summary()
|
||||
|
||||
|
||||
def test_blocking_results_are_ordered_failures_before_skips() -> None:
|
||||
ordered = report(
|
||||
result(model.NOT_RUN, "b.skip"),
|
||||
result(model.FAIL, "z.fail"),
|
||||
result(model.PASS, "a.pass"),
|
||||
).blocking
|
||||
|
||||
assert [item.spec.id for item in ordered] == ["z.fail", "b.skip"]
|
||||
|
||||
|
||||
def test_counts_and_group_summary_tally_every_status() -> None:
|
||||
built = report(
|
||||
result(model.PASS, "a", group="alpha"),
|
||||
result(model.FAIL, "b", group="alpha"),
|
||||
result(model.NOT_RUN, "c", group="beta"),
|
||||
)
|
||||
|
||||
assert built.counts == {
|
||||
model.PASS: 1,
|
||||
model.FAIL: 1,
|
||||
model.NOT_RUN: 1,
|
||||
model.NOT_APPLICABLE: 0,
|
||||
}
|
||||
assert built.group_summary()["alpha"][model.PASS] == 1
|
||||
assert built.group_summary()["beta"][model.NOT_RUN] == 1
|
||||
|
||||
|
||||
def test_the_serialised_report_carries_the_verdict_and_its_reasons() -> None:
|
||||
built = report(result(model.FAIL, "x.fail"), result(model.PASS, "y.pass"))
|
||||
built.vantages = [
|
||||
model.VantageRecord(
|
||||
name="operator", description="d", identity="u", available=True
|
||||
)
|
||||
]
|
||||
|
||||
payload = built.as_dict()
|
||||
|
||||
assert payload["decision"] == model.NO_GO
|
||||
assert payload["blocking"] == ["x.fail"]
|
||||
assert payload["harness"] == model.HARNESS
|
||||
assert payload["schema_version"] == model.SCHEMA_VERSION
|
||||
assert payload["vantages"][0]["identity"] == "u"
|
||||
assert {check["id"] for check in payload["checks"]} == {"x.fail", "y.pass"}
|
||||
|
||||
|
||||
def test_check_serialisation_screens_evidence_and_can_omit_commands() -> None:
|
||||
built = model.CheckResult(
|
||||
spec=spec("allowed", identifier="x", steps=(step("a"),)),
|
||||
status=model.PASS,
|
||||
evidence={"token": "a-long-enough-secret", "count": 3},
|
||||
outcomes=[outcome(stdout="fine")],
|
||||
)
|
||||
|
||||
with_commands = built.as_dict()
|
||||
without = built.as_dict(include_outcomes=False)
|
||||
|
||||
assert with_commands["evidence"]["token"] == "[redacted]"
|
||||
assert with_commands["evidence"]["count"] == 3
|
||||
assert len(with_commands["commands"]) == 1
|
||||
assert "commands" not in without
|
||||
assert with_commands["vantages"] == ["self"]
|
||||
|
||||
|
||||
def test_rationale_is_carried_only_when_a_check_has_one() -> None:
|
||||
without = model.CheckResult(spec=spec("allowed"), status=model.PASS).as_dict()
|
||||
assert "rationale" not in without
|
||||
|
||||
annotated = model.CheckSpec(
|
||||
id="x", title="t", group="g", rule="allowed", rationale="because"
|
||||
)
|
||||
with_reason = model.CheckResult(spec=annotated, status=model.PASS).as_dict()
|
||||
assert with_reason["rationale"] == "because"
|
||||
|
||||
|
||||
def test_distinct_vantages_are_reported_in_first_use_order() -> None:
|
||||
built = spec(
|
||||
"allowed",
|
||||
steps=(
|
||||
step("a", vantage="operator"),
|
||||
step("b", vantage="self"),
|
||||
step("c", vantage="operator"),
|
||||
),
|
||||
)
|
||||
assert built.vantages == ("operator", "self")
|
||||
|
||||
|
||||
def test_the_human_summary_names_the_verdict_vantages_and_blockers() -> None:
|
||||
built = report(result(model.FAIL, "x.fail", group="alpha"), errors=("collision",))
|
||||
built.vantages = [
|
||||
model.VantageRecord(
|
||||
name="operator", description="d", identity="u", available=True
|
||||
),
|
||||
model.VantageRecord(name="self", description="d", available=False),
|
||||
]
|
||||
|
||||
summary = built.render_summary()
|
||||
|
||||
assert summary.splitlines()[0] == f"{model.HARNESS}: {model.NO_GO}"
|
||||
assert "vantage operator: available identity=u" in summary
|
||||
assert "vantage self: unavailable identity=unknown" in summary
|
||||
assert "harness error: collision" in summary
|
||||
assert "x.fail" in summary
|
||||
assert "alpha" in summary
|
||||
|
||||
|
||||
def test_utc_now_renders_a_zulu_timestamp_and_accepts_an_injected_clock() -> None:
|
||||
fixed = dt.datetime(2026, 8, 17, 9, 30, 15, 123456, tzinfo=dt.timezone.utc)
|
||||
assert model.utc_now(lambda: fixed) == "2026-08-17T09:30:15Z"
|
||||
assert model.utc_now().endswith("Z")
|
||||
|
||||
|
||||
def test_unscreened_fields_walks_the_whole_payload() -> None:
|
||||
offenders = model.unscreened_fields(
|
||||
{
|
||||
"clean": "hermes",
|
||||
"nested": {"leak": "token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234"},
|
||||
"items": [1, "Authorization: Bearer abcdefghijklmnop"],
|
||||
}
|
||||
)
|
||||
|
||||
assert offenders == ["$.nested.leak", "$.items[1]"]
|
||||
assert model.unscreened_fields({"clean": "hermes"}) == []
|
||||
assert model.unscreened_fields(None) == []
|
||||
60
testing/tests/test_hermes_handoff_mutation_gate.py
Normal file
60
testing/tests/test_hermes_handoff_mutation_gate.py
Normal file
@ -0,0 +1,60 @@
|
||||
"""Tests for the deterministic Hermes handoff mutation gate."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from testing.quality_handoff_mutation import (
|
||||
MUTATIONS,
|
||||
Mutation,
|
||||
MutationReport,
|
||||
main,
|
||||
run_gate,
|
||||
)
|
||||
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
|
||||
|
||||
def test_every_security_mutant_is_killed() -> None:
|
||||
report = run_gate(ROOT)
|
||||
assert report.issues == ()
|
||||
assert report.score == 100.0
|
||||
assert report.killed == report.total >= 9
|
||||
|
||||
|
||||
def test_invalid_anchor_baseline_syntax_and_survivor_are_reported() -> None:
|
||||
path = "hermes_handoff_model.py"
|
||||
cases = (
|
||||
Mutation("missing", path, "not present", "x", "pass"),
|
||||
Mutation(
|
||||
"baseline",
|
||||
path,
|
||||
"SCHEMA_VERSION = 1",
|
||||
"SCHEMA_VERSION = 2",
|
||||
"raise SystemExit(1)",
|
||||
),
|
||||
Mutation("syntax", path, "SCHEMA_VERSION = 1", "if", "pass"),
|
||||
Mutation("survivor", path, "SCHEMA_VERSION = 1", "SCHEMA_VERSION = 2", "pass"),
|
||||
)
|
||||
report = run_gate(ROOT, cases)
|
||||
assert report.killed == 0
|
||||
assert len(report.issues) == 4
|
||||
assert report.score == 0.0
|
||||
|
||||
|
||||
def test_main_reports_a_perfect_score(capsys) -> None:
|
||||
"""Every mutant must die, and the count must track the declared set."""
|
||||
assert main() == 0
|
||||
total = len(MUTATIONS)
|
||||
assert total >= 13
|
||||
assert f"{total}/{total} (100.00%)" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_main_reports_mutation_issues(monkeypatch, capsys) -> None:
|
||||
monkeypatch.setattr(
|
||||
"testing.quality_handoff_mutation.run_gate",
|
||||
lambda _root: MutationReport(1, 0, ("survivor",)),
|
||||
)
|
||||
assert main() == 1
|
||||
assert "survivor" in capsys.readouterr().out
|
||||
367
testing/tests/test_hermes_handoff_policy.py
Normal file
367
testing/tests/test_hermes_handoff_policy.py
Normal file
@ -0,0 +1,367 @@
|
||||
"""Contracts for the fail-closed command policy of the handoff harness."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from testing.tests.test_hermes_handoff_support import load_handoff_module
|
||||
|
||||
policy = load_handoff_module("hermes_handoff_policy")
|
||||
|
||||
|
||||
def check(*argv: str, mode: str | None = None) -> None:
|
||||
policy.check_argv(argv, mode or policy.READ_ONLY)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"argv",
|
||||
[
|
||||
("kubectl", "--namespace", "hermes", "get", "pods", "-o", "name"),
|
||||
("kubectl", "auth", "can-i", "get", "secrets", "--all-namespaces"),
|
||||
("kubectl", "get", "nodes", "-o", "name"),
|
||||
(
|
||||
"kubectl",
|
||||
"get",
|
||||
"pods",
|
||||
"-o",
|
||||
policy.projection("jsonpath={.status.phase}"),
|
||||
),
|
||||
("git", "merge-base", "--is-ancestor", "a", "b"),
|
||||
("git", "ls-remote", "origin", "refs/heads/main"),
|
||||
("hermes", "kanban", "list", "--json"),
|
||||
("hermes", "sessions", "list", "--source", "telegram"),
|
||||
("hermes", "status"),
|
||||
(policy.GITEA_CLIENT, "GET", "/api/v1/user"),
|
||||
],
|
||||
)
|
||||
def test_read_only_commands_are_permitted(argv: tuple[str, ...]) -> None:
|
||||
check(*argv)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"argv",
|
||||
[
|
||||
(),
|
||||
("vault", "read", "kv/x"),
|
||||
("curl", "https://example.dev"),
|
||||
("rm", "-rf", "/"),
|
||||
("kubectl", "get", "pods", "--token", "abc"),
|
||||
("kubectl", "get", "pods"),
|
||||
("kubectl", "get", "pods", "--as=system:admin"),
|
||||
("kubectl", "get", "secrets", "-o", "name"),
|
||||
("kubectl", "get", "pods,secrets/x", "-o", "name"),
|
||||
("kubectl", "describe", "pod/x"),
|
||||
("kubectl", "config", "view", "--raw"),
|
||||
("kubectl", "auth", "reconcile", "-f", "x"),
|
||||
("kubectl", "delete", "pod/x", "--dry-run=client"),
|
||||
("kubectl", "delete", "pod/x", "--", "--dry-run=server"),
|
||||
("kubectl", "get", "pods", "-o=json"),
|
||||
("kubectl", "get", "pods", "-o", "jsonpath={.spec.containers[*].env[*].value}"),
|
||||
("kubectl", "port-forward", "svc/x", "80"),
|
||||
("kubectl", "exec", "pod"),
|
||||
("kubectl", "exec", "pod", "--"),
|
||||
("kubectl",),
|
||||
("flux", "get", "kustomizations", "--all-namespaces"),
|
||||
("flux", "reconcile", "kustomization", "x"),
|
||||
("flux",),
|
||||
("helm", "list", "--all-namespaces"),
|
||||
("helm", "upgrade", "x"),
|
||||
("helm", "get", "values", "x"),
|
||||
("git", "commit", "-m", "x"),
|
||||
("git", "push", "origin", "HEAD:refs/heads/x"),
|
||||
("git", "fetch", "origin"),
|
||||
("git", "config", "core.sshCommand", "x"),
|
||||
("git", "-ccore.sshCommand=x", "ls-remote", "origin"),
|
||||
("git", "--git-dir=/tmp/x", "status"),
|
||||
("hermes", "kanban", "complete", "t_1"),
|
||||
("hermes",),
|
||||
(policy.GITEA_CLIENT, "POST", "/api/v1/x"),
|
||||
(policy.GITEA_CLIENT, "GET"),
|
||||
("sh", "-c", "rm -rf /"),
|
||||
("sh", "echo"),
|
||||
("git", "log", "/etc/shadow"),
|
||||
],
|
||||
)
|
||||
def test_unsafe_commands_are_refused(argv: tuple[str, ...]) -> None:
|
||||
with pytest.raises(policy.PolicyError):
|
||||
check(*argv)
|
||||
|
||||
|
||||
def test_unknown_mode_is_refused() -> None:
|
||||
with pytest.raises(policy.PolicyError, match="unknown mode"):
|
||||
policy.check_argv(("kubectl", "get", "pods"), "whatever")
|
||||
|
||||
|
||||
def test_dry_run_mutation_commands_are_never_constructible() -> None:
|
||||
with pytest.raises(policy.PolicyError, match="dry-run"):
|
||||
check("kubectl", "patch", "deploy/x", "--dry-run=server")
|
||||
|
||||
|
||||
def test_a_resource_named_after_a_subcommand_cannot_smuggle_a_delete() -> None:
|
||||
"""`-n get` must not make a delete look like a read."""
|
||||
with pytest.raises(policy.PolicyError):
|
||||
check("kubectl", "delete", "-n", "get", "configmap", "x")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", ["--as", "--as-group", "--as-uid"])
|
||||
@pytest.mark.parametrize("mode", ["read-only", "ephemeral-armed"])
|
||||
def test_impersonation_is_refused_from_every_vantage_and_mode(
|
||||
flag: str, mode: str
|
||||
) -> None:
|
||||
"""No vantage may impersonate: not the operator, and not inside a pod.
|
||||
|
||||
The inner command of ``kubectl exec`` used to be granted impersonation
|
||||
unconditionally, so a self/switchyard/node/chat step carrying ``--as`` was
|
||||
accepted. The inner argv is now re-checked under the same rule.
|
||||
"""
|
||||
for argv in (
|
||||
("kubectl", "get", "namespaces", "-o", "name", flag, "system:admin"),
|
||||
("kubectl", "get", "ns", "-o", "name", f"{flag}=system:admin"),
|
||||
):
|
||||
with pytest.raises(policy.PolicyError, match="forbidden argument"):
|
||||
check(*argv, mode=mode)
|
||||
inner = (
|
||||
"kubectl",
|
||||
"exec",
|
||||
"pod",
|
||||
"--",
|
||||
"/usr/local/bin/kubectl",
|
||||
"get",
|
||||
"ns",
|
||||
"-o",
|
||||
"name",
|
||||
flag,
|
||||
"system:admin",
|
||||
)
|
||||
with pytest.raises(policy.PolicyError, match="forbidden argument"):
|
||||
check(*inner, mode=mode)
|
||||
|
||||
|
||||
def test_exec_validates_its_inner_command() -> None:
|
||||
script = policy.render_shell("env_names")
|
||||
policy._RENDERED.add(script)
|
||||
check(
|
||||
"kubectl", "exec", "--namespace", "hermes", "pod", "--", "/bin/sh", "-c", script
|
||||
)
|
||||
check(
|
||||
"kubectl",
|
||||
"exec",
|
||||
"pod",
|
||||
"--",
|
||||
"/usr/local/bin/kubectl",
|
||||
"get",
|
||||
"ns",
|
||||
"-o",
|
||||
"name",
|
||||
)
|
||||
with pytest.raises(policy.PolicyError):
|
||||
check("kubectl", "exec", "pod", "--", "bash", "-c", "id")
|
||||
|
||||
|
||||
def test_the_repository_pin_has_an_exact_boundary_and_rejects_dot_segments() -> None:
|
||||
"""A prefix match without a boundary let a look-alike repository through."""
|
||||
for path in (
|
||||
"/api/v1/repos/atlas/titan-iac-evil/pulls",
|
||||
"/api/v1/repos/atlas/titan-iacx",
|
||||
"/api/v1/repos/atlas/titan-iac../pulls",
|
||||
):
|
||||
with pytest.raises(policy.PolicyError):
|
||||
check(policy.GITEA_CLIENT, "GET", path)
|
||||
for path in (
|
||||
"/api/v1/repos/atlas/titan-iac/../../user/tokens",
|
||||
"/api/v1/repos/atlas/titan-iac/%2e%2e/%2e%2e/user/tokens",
|
||||
"/api/v1/repos/atlas/titan-iac/%252e%252e/user/tokens",
|
||||
"/api/v1/repos/atlas/titan-iac/./pulls",
|
||||
"/api/v1/repos/atlas/titan-iac//pulls",
|
||||
):
|
||||
with pytest.raises(policy.PolicyError, match="relative segments"):
|
||||
check(policy.GITEA_CLIENT, "GET", path)
|
||||
check(policy.GITEA_CLIENT, "GET", "/api/v1/repos/atlas/titan-iac")
|
||||
check(policy.GITEA_CLIENT, "GET", "/api/v1/repos/atlas/titan-iac/pulls?state=all")
|
||||
|
||||
|
||||
def test_every_allowlisted_binary_is_release_attestable() -> None:
|
||||
"""An allowlisted binary with no pinned digest is dead, misleading config."""
|
||||
exec_module = load_handoff_module("hermes_handoff_exec")
|
||||
assert set(exec_module.EXPECTED_SHA256) == set(policy.ALLOWED_BINARIES)
|
||||
assert set(exec_module.EXPECTED_PATHS) == set(policy.ALLOWED_BINARIES)
|
||||
assert set(exec_module.POD_COMMAND_PATHS) == set(policy.ALLOWED_BINARIES)
|
||||
assert set(exec_module.CONTEXT_AWARE_BINARIES) <= set(policy.ALLOWED_BINARIES)
|
||||
|
||||
|
||||
def test_shell_templates_reject_credential_roots() -> None:
|
||||
with pytest.raises(policy.PolicyError, match="credential paths"):
|
||||
policy.shell("json_fields", path="/runtime-access/token", fields="state")
|
||||
with pytest.raises(policy.PolicyError, match="credential-bearing"):
|
||||
policy.shell("json_fields", path="/tmp/health.json", fields="state,token")
|
||||
|
||||
|
||||
def test_armed_mode_opens_exactly_the_mutations_it_should() -> None:
|
||||
ref = "ephemeral/hermes-handoff-acceptance/acceptance1"
|
||||
policy.arm_ephemeral_policy(ref)
|
||||
policy.register_ephemeral_pull(9)
|
||||
check("git", "push", "origin", f"HEAD:refs/heads/{ref}", mode=policy.ARMED)
|
||||
root = "/api/v1/repos/atlas/titan-iac"
|
||||
check(
|
||||
policy.GITEA_CLIENT,
|
||||
"POST",
|
||||
f"{root}/pulls",
|
||||
"--field",
|
||||
"title=WIP: Hermes handoff acceptance ephemeral probe",
|
||||
"--field",
|
||||
f"head={ref}",
|
||||
"--field",
|
||||
"base=main",
|
||||
"--field",
|
||||
"body=Ephemeral acceptance probe. Closed and deleted by the harness.",
|
||||
mode=policy.ARMED,
|
||||
)
|
||||
check(
|
||||
policy.GITEA_CLIENT,
|
||||
"PATCH",
|
||||
f"{root}/pulls/9",
|
||||
"--field",
|
||||
"state=closed",
|
||||
mode=policy.ARMED,
|
||||
)
|
||||
check(policy.GITEA_CLIENT, "DELETE", f"{root}/branches/{ref}", mode=policy.ARMED)
|
||||
with pytest.raises(policy.PolicyError, match="may not force"):
|
||||
check(
|
||||
"git",
|
||||
"push",
|
||||
"--force",
|
||||
"origin",
|
||||
f"HEAD:refs/heads/{ref}",
|
||||
mode=policy.ARMED,
|
||||
)
|
||||
with pytest.raises(policy.PolicyError, match="may not force"):
|
||||
check("git", "push", "--all", "origin", mode=policy.ARMED)
|
||||
with pytest.raises(policy.PolicyError, match="not permitted"):
|
||||
check(policy.GITEA_CLIENT, "PUT", f"{root}/pulls/9", mode=policy.ARMED)
|
||||
|
||||
|
||||
def test_arming_and_projection_registration_fail_closed() -> None:
|
||||
with pytest.raises(policy.PolicyError, match="exact ephemeral ref"):
|
||||
policy.arm_ephemeral_policy("main")
|
||||
policy._ARMED_REF = ""
|
||||
with pytest.raises(policy.PolicyError, match="registration is malformed"):
|
||||
policy.register_ephemeral_pull(9)
|
||||
policy.arm_ephemeral_policy("ephemeral/hermes-handoff-acceptance/acceptance1")
|
||||
|
||||
for candidate in (
|
||||
"wide",
|
||||
"jsonpath={.status.token}",
|
||||
"jsonpath={.metadata.annotations}",
|
||||
"jsonpath={.metadata.labels.other}",
|
||||
"jsonpath={.spec.containers[*].env[*].value}",
|
||||
):
|
||||
with pytest.raises(policy.PolicyError):
|
||||
policy.projection(candidate)
|
||||
|
||||
with pytest.raises(policy.PolicyError):
|
||||
check("/usr/bin/awk", "x")
|
||||
with pytest.raises(policy.PolicyError):
|
||||
check("git", "--exec-path=/tmp", "status")
|
||||
|
||||
|
||||
def test_every_frozen_template_renders_and_is_accepted() -> None:
|
||||
parameters = {
|
||||
"account": "hermes-agent",
|
||||
"askpass": "/opt/coordinator/gitea_askpass.sh",
|
||||
"fields": "state,model",
|
||||
"limit": "200",
|
||||
"path": "/host-etc/passwd",
|
||||
"url": "https://scm.example.dev/atlas/repo.git",
|
||||
}
|
||||
for name in policy.SHELL_TEMPLATES:
|
||||
required = policy.template_parameters(name)
|
||||
argv = policy.shell(name, **{key: parameters[key] for key in required})
|
||||
policy.check_argv(argv)
|
||||
assert argv[0] == "sh"
|
||||
|
||||
|
||||
def test_template_rendering_rejects_unknown_names_and_unsafe_parameters() -> None:
|
||||
with pytest.raises(policy.PolicyError, match="unknown shell template"):
|
||||
policy.render_shell("nope")
|
||||
with pytest.raises(policy.PolicyError, match="expects parameters"):
|
||||
policy.render_shell("json_fields")
|
||||
with pytest.raises(policy.PolicyError, match="unsafe shell parameter"):
|
||||
policy.render_shell("json_fields", path="/tmp;rm", fields="state")
|
||||
with pytest.raises(policy.PolicyError, match="unsafe shell parameter"):
|
||||
policy.render_shell("json_fields", path="$(id)", fields="state")
|
||||
|
||||
|
||||
def test_positionals_skip_flags_and_stop_at_the_boundary() -> None:
|
||||
argv = ("kubectl", "--namespace", "hermes", "get", "pods", "--", "ignored")
|
||||
assert list(policy.positionals(argv)) == ["get", "pods"]
|
||||
assert list(policy.positionals(("kubectl", "--all-namespaces", "get"))) == ["get"]
|
||||
|
||||
|
||||
def test_a_subcommand_must_precede_its_resource_arguments() -> None:
|
||||
with pytest.raises(policy.PolicyError):
|
||||
check("kubectl", "pods", "get")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"argv",
|
||||
[
|
||||
("/usr/bin/../../tmp/kubectl", "get", "pods"),
|
||||
("/tmp/reviewer/kubectl", "get", "pods"),
|
||||
("kubectl", "get", "pods", "-o", "go-template=x"),
|
||||
("kubectl", "get", "pods", "-o", "jsonpath={.status.token}"),
|
||||
("kubectl", "get", "pods", "-o", "jsonpath={.metadata.annotations}"),
|
||||
("kubectl", "get", "pods", "-o", "jsonpath={.spec.containers[*].env[*].value}"),
|
||||
("git", "-C", "/tmp", "status"),
|
||||
("git", "--work-tree=/tmp", "status"),
|
||||
(policy.GITEA_CLIENT, "GET", "not-an-api-path"),
|
||||
(policy.GITEA_CLIENT, "GET", "/api/v1/repos/atlas/other"),
|
||||
],
|
||||
)
|
||||
def test_provenance_projection_and_repository_bypasses_are_rejected(argv) -> None:
|
||||
with pytest.raises(policy.PolicyError):
|
||||
check(*argv)
|
||||
|
||||
|
||||
def test_armed_writes_are_pinned_to_exact_remote_ref_and_endpoints() -> None:
|
||||
ref = "ephemeral/hermes-handoff-acceptance/acceptance1"
|
||||
policy.arm_ephemeral_policy(ref)
|
||||
policy.register_ephemeral_pull(9)
|
||||
with pytest.raises(policy.PolicyError, match="pinned to origin"):
|
||||
check(
|
||||
"git",
|
||||
"push",
|
||||
"upstream",
|
||||
"HEAD:refs/heads/ephemeral/hermes-handoff-acceptance/acceptance1",
|
||||
mode=policy.ARMED,
|
||||
)
|
||||
with pytest.raises(policy.PolicyError, match="preflighted ephemeral ref"):
|
||||
check("git", "push", "origin", "HEAD:refs/heads/other", mode=policy.ARMED)
|
||||
with pytest.raises(policy.PolicyError):
|
||||
check(
|
||||
policy.GITEA_CLIENT,
|
||||
"DELETE",
|
||||
"/api/v1/repos/atlas/titan-iac",
|
||||
mode=policy.ARMED,
|
||||
)
|
||||
with pytest.raises(policy.PolicyError):
|
||||
check(
|
||||
policy.GITEA_CLIENT,
|
||||
"PATCH",
|
||||
"/api/v1/repos/atlas/titan-iac/pulls/8",
|
||||
"--field",
|
||||
"state=closed",
|
||||
mode=policy.ARMED,
|
||||
)
|
||||
with pytest.raises(policy.PolicyError):
|
||||
check(
|
||||
policy.GITEA_CLIENT,
|
||||
"POST",
|
||||
"/api/v1/repos/atlas/titan-iac/pulls",
|
||||
"--field",
|
||||
"title=WIP: wrong branch",
|
||||
"--field",
|
||||
"head=main",
|
||||
"--field",
|
||||
"base=main",
|
||||
mode=policy.ARMED,
|
||||
)
|
||||
192
testing/tests/test_hermes_handoff_redaction.py
Normal file
192
testing/tests/test_hermes_handoff_redaction.py
Normal file
@ -0,0 +1,192 @@
|
||||
"""Contracts for the handoff harness credential screen and output bounding."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from testing.tests.test_hermes_handoff_support import load_handoff_module
|
||||
|
||||
redaction = load_handoff_module("hermes_handoff_redaction")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"token: ghp_ABCDEFGHIJKLMNOPQRSTUV1234",
|
||||
"Authorization: Bearer abcdefghijklmnop",
|
||||
"cookie: session=abcdefghijklmnop",
|
||||
"password=hunter2000secret",
|
||||
"-----BEGIN OPENSSH PRIVATE KEY-----\nabc\n-----END OPENSSH PRIVATE KEY-----",
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI",
|
||||
"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0.SflKxwRJ",
|
||||
"https://user:supersecretvalue@scm.example.dev/x",
|
||||
"vault token hvs.CAESIJabcdefghijklmnop",
|
||||
"aws AKIAIOSFODNN7EXAMPLE",
|
||||
],
|
||||
)
|
||||
def test_credential_shapes_are_screened(text: str) -> None:
|
||||
assert redaction.contains_credential_shape(text)
|
||||
assert redaction.REDACTED in redaction.redact(text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7",
|
||||
"ab346f55509d584e457fe26cf90be3078f7a375c",
|
||||
'"automountServiceAccountToken": false',
|
||||
'"key": "kubernetes.io/arch"',
|
||||
"namespace: hermes",
|
||||
"replicas: 3",
|
||||
"checked_at: 2026-08-17T09:17:07.224400+00:00",
|
||||
],
|
||||
)
|
||||
def test_ordinary_evidence_survives_screening(text: str) -> None:
|
||||
assert redaction.redact(text) == text
|
||||
assert not redaction.contains_credential_shape(text)
|
||||
|
||||
|
||||
def test_screened_json_is_still_parsable_json() -> None:
|
||||
"""A screen that corrupts structure would break every downstream evaluator."""
|
||||
payload = {
|
||||
"spec": {"automountServiceAccountToken": True, "key": "kubernetes.io/arch"},
|
||||
"data": {"password": "a-long-enough-secret-value"},
|
||||
}
|
||||
screened = redaction.redact(json.dumps(payload, indent=2))
|
||||
reparsed = json.loads(screened)
|
||||
|
||||
assert reparsed["spec"]["automountServiceAccountToken"] is True
|
||||
assert reparsed["spec"]["key"] == "kubernetes.io/arch"
|
||||
assert reparsed["data"]["password"] == redaction.REDACTED
|
||||
|
||||
|
||||
def test_high_entropy_runs_are_screened_but_hex_digests_are_not() -> None:
|
||||
assert (
|
||||
redaction.redact("Zm9vYmFyYmF6cXV4-Quux_1234567890ABCDEFghij")
|
||||
!= redaction.REDACTED
|
||||
)
|
||||
long_mixed = "aB3" + "x9Y2z_" * 10
|
||||
assert redaction.redact(long_mixed) == redaction.REDACTED
|
||||
assert redaction.redact("f" * 64) == "f" * 64
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("key", "sensitive"),
|
||||
[
|
||||
("ANTHROPIC_API_KEY", True),
|
||||
("gitea_token", True),
|
||||
("clientSecret", True),
|
||||
("Set-Cookie", True),
|
||||
("automountServiceAccountToken", True),
|
||||
("key", False),
|
||||
("namespace", False),
|
||||
("", False),
|
||||
],
|
||||
)
|
||||
def test_key_sensitivity_classification(key: str, sensitive: bool) -> None:
|
||||
assert redaction.is_sensitive_key(key) is sensitive
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "possible"),
|
||||
[
|
||||
("true", False),
|
||||
("False", False),
|
||||
("3", False),
|
||||
("12.5", False),
|
||||
("short", False),
|
||||
("", False),
|
||||
("a-long-enough-secret", True),
|
||||
('"another-long-secret"', True),
|
||||
("1234567890123", False),
|
||||
("-12.5e9", False),
|
||||
],
|
||||
)
|
||||
def test_value_shape_gates_screening(value: str, possible: bool) -> None:
|
||||
assert redaction.could_hold_secret(value) is possible
|
||||
|
||||
|
||||
def test_bound_truncates_to_a_byte_budget_and_reports_it() -> None:
|
||||
text, truncated = redaction.bound("x" * 100, max_bytes=10)
|
||||
assert truncated
|
||||
assert len(text.encode()) <= 10
|
||||
assert text == redaction.TRUNCATION_NOTE[:10]
|
||||
|
||||
text, truncated = redaction.bound("short", max_bytes=64)
|
||||
assert (text, truncated) == ("short", False)
|
||||
|
||||
|
||||
def test_bound_with_no_budget_keeps_nothing() -> None:
|
||||
assert redaction.bound("content", max_bytes=0) == ("", True)
|
||||
assert redaction.bound("", max_bytes=0) == ("", False)
|
||||
|
||||
|
||||
def test_bound_never_splits_a_multibyte_character() -> None:
|
||||
text, truncated = redaction.bound("é" * 10, max_bytes=5)
|
||||
assert truncated
|
||||
assert len(text.encode()) <= 5
|
||||
|
||||
|
||||
def test_safe_text_screens_before_it_bounds() -> None:
|
||||
"""A boundary must never preserve a useful credential prefix."""
|
||||
payload = "password=" + "a" * 200
|
||||
text, truncated = redaction.safe_text(payload, max_bytes=20)
|
||||
assert not truncated
|
||||
assert redaction.REDACTED in text
|
||||
|
||||
|
||||
def test_no_byte_budget_exposes_a_credential_prefix() -> None:
|
||||
payload = "password=A9secret-prefix-that-must-never-survive"
|
||||
for budget in range(1, 64):
|
||||
screened, _ = redaction.safe_text(payload, max_bytes=budget)
|
||||
assert "A9" not in screened
|
||||
assert len(screened.encode()) <= budget
|
||||
|
||||
|
||||
def test_scrub_walks_nested_structures_and_keeps_useful_types() -> None:
|
||||
scrubbed = redaction.scrub(
|
||||
{
|
||||
"token": "a-long-enough-secret",
|
||||
"automountServiceAccountToken": False,
|
||||
"items": [{"password": "another-long-secret"}, 3, None],
|
||||
"note": ("tuple", "values"),
|
||||
}
|
||||
)
|
||||
|
||||
assert scrubbed["token"] == redaction.REDACTED
|
||||
assert scrubbed["automountServiceAccountToken"] is False
|
||||
assert scrubbed["items"][0]["password"] == redaction.REDACTED
|
||||
assert scrubbed["items"][1] == 3
|
||||
assert scrubbed["items"][2] is None
|
||||
assert scrubbed["note"] == ["tuple", "values"]
|
||||
|
||||
|
||||
def test_scrub_screens_short_numeric_nested_values_and_mapping_keys() -> None:
|
||||
scrubbed = redaction.scrub(
|
||||
{
|
||||
"token": 1234,
|
||||
"credentials": {"x": "y"},
|
||||
"ghp_ABCDEF1234567890": "mapping-key",
|
||||
"private_key": "x",
|
||||
}
|
||||
)
|
||||
assert scrubbed["token"] == redaction.REDACTED
|
||||
assert scrubbed["credentials"] == redaction.REDACTED
|
||||
assert scrubbed["private_key"] == redaction.REDACTED
|
||||
assert not any("ghp_" in key for key in scrubbed)
|
||||
|
||||
|
||||
def test_unterminated_private_key_blocks_are_screened() -> None:
|
||||
assert "PRIVATE KEY" not in redaction.redact("-----BEGIN PRIVATE KEY-----\nabc")
|
||||
|
||||
|
||||
def test_screening_is_idempotent() -> None:
|
||||
once = redaction.redact("api_key=AKIAIOSFODNN7EXAMPLE")
|
||||
assert redaction.redact(once) == once
|
||||
assert redaction.redact("") == ""
|
||||
|
||||
|
||||
def test_normalise_key_strips_separators() -> None:
|
||||
assert redaction.normalise_key("Client-Secret_v2") == "clientsecretv2"
|
||||
166
testing/tests/test_hermes_handoff_rules.py
Normal file
166
testing/tests/test_hermes_handoff_rules.py
Normal file
@ -0,0 +1,166 @@
|
||||
"""Contracts for the shared evaluator primitives and dispatch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from testing.tests.test_hermes_handoff_support import (
|
||||
load_handoff_module,
|
||||
outcome,
|
||||
spec,
|
||||
step,
|
||||
)
|
||||
|
||||
rules = load_handoff_module("hermes_handoff_rules")
|
||||
model = load_handoff_module("hermes_handoff_model")
|
||||
load_handoff_module("hermes_handoff_evaluators")
|
||||
|
||||
|
||||
PAYLOAD = {
|
||||
"a": {"b": [{"c": 1}, {"c": 2}]},
|
||||
"items": [{"number": 14, "merged": True}, {"number": 15, "merged": False}],
|
||||
"flag": True,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("path", "expected"),
|
||||
[
|
||||
("", PAYLOAD),
|
||||
("flag", True),
|
||||
("a.b[0].c", 1),
|
||||
("a.b[].c", [1, 2]),
|
||||
("a.b[]", [{"c": 1}, {"c": 2}]),
|
||||
("items[].number", [14, 15]),
|
||||
],
|
||||
)
|
||||
def test_dotted_paths_index_and_map(path: str, expected) -> None:
|
||||
assert rules.dotted(PAYLOAD, path) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path", ["missing", "a.missing", "a.b[9].c", "flag[0]", "flag.b"]
|
||||
)
|
||||
def test_an_absent_path_raises_rather_than_returning_none(path: str) -> None:
|
||||
with pytest.raises(KeyError):
|
||||
rules.dotted(PAYLOAD, path)
|
||||
|
||||
|
||||
def test_parse_json_rejects_raw_control_characters_and_empty_output() -> None:
|
||||
"""Only RFC-compliant, nonempty JSON is accepted as release evidence."""
|
||||
with pytest.raises(ValueError):
|
||||
rules.parse_json(outcome(stdout='{"title": "a\nb"}'))
|
||||
with pytest.raises(ValueError, match="empty output"):
|
||||
rules.parse_json(outcome(stdout=" "))
|
||||
with pytest.raises(ValueError):
|
||||
rules.parse_json(outcome(stdout="not json"))
|
||||
|
||||
|
||||
def test_lines_drops_blank_rows_and_trims() -> None:
|
||||
assert rules.lines(outcome(stdout=" a \n\n b\n")) == ["a", "b"]
|
||||
|
||||
|
||||
def test_missing_steps_reports_the_first_step_that_produced_nothing() -> None:
|
||||
built = spec("allowed", steps=(step("a"), step("b")))
|
||||
|
||||
never_run = rules.missing_steps(built, {})
|
||||
assert never_run.status == model.NOT_RUN
|
||||
assert "never executed" in never_run.reason
|
||||
|
||||
errored = rules.missing_steps(
|
||||
built, {"a": outcome(error="timeout"), "b": outcome()}
|
||||
)
|
||||
assert errored.status == model.NOT_RUN
|
||||
assert "did not run" in errored.reason
|
||||
|
||||
truncated = rules.missing_steps(
|
||||
built, {"a": outcome(truncated=True), "b": outcome()}
|
||||
)
|
||||
assert truncated.status == model.NOT_RUN and "truncated" in truncated.reason
|
||||
|
||||
assert rules.missing_steps(built, {"a": outcome(), "b": outcome()}) is None
|
||||
|
||||
|
||||
def test_an_optional_step_may_be_absent() -> None:
|
||||
built = spec("allowed", steps=(step("a"), step("b", optional=True)))
|
||||
assert rules.missing_steps(built, {"a": outcome()}) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("stderr", "returncode", "denied"),
|
||||
[
|
||||
("Error from server (Forbidden): pods is forbidden", 1, True),
|
||||
("User cannot list resource", 1, True),
|
||||
("Gitea API returned HTTP 403", 1, True),
|
||||
("connection refused", 1, False),
|
||||
("", 0, False),
|
||||
],
|
||||
)
|
||||
def test_denial_detection_needs_a_refusal_not_merely_a_failure(
|
||||
stderr, returncode, denied
|
||||
) -> None:
|
||||
assert rules.looks_denied(outcome(stderr=stderr, returncode=returncode)) is denied
|
||||
|
||||
|
||||
def test_step_keys_selects_by_kind() -> None:
|
||||
built = spec(
|
||||
"denied",
|
||||
steps=(step("r", kind=model.REVIEW), step("a", kind=model.ATTEMPT), step("x")),
|
||||
)
|
||||
assert rules.step_keys(built, model.REVIEW) == ("r",)
|
||||
assert rules.step_keys(built, model.ATTEMPT) == ("a",)
|
||||
assert rules.step_keys(built, model.READ) == ("x",)
|
||||
|
||||
|
||||
def test_parsed_step_returns_the_payload_or_the_skip_to_report() -> None:
|
||||
built = spec("json_field", {"step": "s"}, steps=(step("s"),))
|
||||
|
||||
payload, key = rules.parsed_step(built, {"s": outcome(stdout='{"a": 1}')})
|
||||
assert (payload, key) == ({"a": 1}, "s")
|
||||
|
||||
failed = rules.parsed_step(built, {"s": outcome(returncode=1)})
|
||||
assert failed.status == model.NOT_RUN and "failed" in failed.reason
|
||||
|
||||
unparsable = rules.parsed_step(built, {"s": outcome(stdout="nope")})
|
||||
assert (
|
||||
unparsable.status == model.NOT_RUN
|
||||
and "did not return JSON" in unparsable.reason
|
||||
)
|
||||
|
||||
|
||||
def test_an_unknown_rule_is_not_run_rather_than_a_crash() -> None:
|
||||
evaluation = rules.evaluate(spec("no-such-rule"), {})
|
||||
assert evaluation.status == model.NOT_RUN
|
||||
assert "unknown evaluator" in evaluation.reason
|
||||
|
||||
|
||||
def test_an_evaluator_that_raises_is_not_run_rather_than_a_pass() -> None:
|
||||
"""A harness bug must never be indistinguishable from a clean release."""
|
||||
|
||||
@rules.evaluator("explodes-for-test")
|
||||
def _explode(_spec, _outcomes):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
evaluation = rules.evaluate(spec("explodes-for-test"), {})
|
||||
|
||||
assert evaluation.status == model.NOT_RUN
|
||||
assert "RuntimeError" in evaluation.reason and "boom" in evaluation.reason
|
||||
del rules.EVALUATORS["explodes-for-test"]
|
||||
|
||||
|
||||
def test_an_evaluator_returning_an_unknown_status_is_not_run() -> None:
|
||||
@rules.evaluator("unknown-status-for-test")
|
||||
def _unknown(_spec, _outcomes):
|
||||
return rules.Evaluation("MAYBE", "bad")
|
||||
|
||||
evaluation = rules.evaluate(spec("unknown-status-for-test"), {})
|
||||
assert evaluation.status == model.NOT_RUN
|
||||
assert "unknown status" in evaluation.reason
|
||||
del rules.EVALUATORS["unknown-status-for-test"]
|
||||
|
||||
|
||||
def test_is_list_distinguishes_arrays_from_strings() -> None:
|
||||
assert rules.is_list([1]) and rules.is_list(())
|
||||
assert not rules.is_list("text")
|
||||
assert not rules.is_list(b"bytes")
|
||||
assert not rules.is_list({"a": 1})
|
||||
496
testing/tests/test_hermes_handoff_run.py
Normal file
496
testing/tests/test_hermes_handoff_run.py
Normal file
@ -0,0 +1,496 @@
|
||||
"""Contracts for handoff acceptance orchestration: vantages, catalog, report."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import subprocess
|
||||
from dataclasses import replace
|
||||
|
||||
from testing.tests.test_hermes_handoff_support import (
|
||||
FakeClock,
|
||||
FakeSpawn,
|
||||
load_handoff_module,
|
||||
step,
|
||||
)
|
||||
|
||||
harness_run = load_handoff_module("hermes_handoff_run")
|
||||
catalog = load_handoff_module("hermes_handoff_catalog")
|
||||
exec_module = load_handoff_module("hermes_handoff_exec")
|
||||
model = load_handoff_module("hermes_handoff_model")
|
||||
|
||||
NOW = dt.datetime(2026, 8, 17, 12, 0, 0, tzinfo=dt.timezone.utc)
|
||||
TARGETS = catalog.Targets(now=NOW)
|
||||
|
||||
|
||||
def completed(stdout: str = "", stderr: str = "", returncode: int = 0):
|
||||
return subprocess.CompletedProcess(
|
||||
args=[], returncode=returncode, stdout=stdout, stderr=stderr
|
||||
)
|
||||
|
||||
|
||||
def whoami(username: str) -> object:
|
||||
return completed(stdout=username)
|
||||
|
||||
|
||||
def runner_for(results: list, **kwargs):
|
||||
return exec_module.Runner(
|
||||
clock=FakeClock(),
|
||||
spawn=FakeSpawn(results),
|
||||
environ={},
|
||||
deadline_seconds=3600,
|
||||
attestor=lambda command, _environment: exec_module.Attestation(
|
||||
command, "a" * 64
|
||||
),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def check(
|
||||
rule: str,
|
||||
expect: dict | None = None,
|
||||
steps=(),
|
||||
identifier="test.check",
|
||||
mandatory=True,
|
||||
):
|
||||
return model.CheckSpec(
|
||||
id=identifier,
|
||||
title="t",
|
||||
group="g",
|
||||
rule=rule,
|
||||
steps=tuple(steps),
|
||||
expect=expect or {},
|
||||
mandatory=mandatory,
|
||||
)
|
||||
|
||||
|
||||
def test_the_shipped_catalog_is_valid_and_covers_every_group() -> None:
|
||||
specs = harness_run.build_catalog(TARGETS)
|
||||
assert harness_run.validate_catalog(specs) == []
|
||||
groups = {spec.group for spec in specs}
|
||||
assert {
|
||||
"baseline",
|
||||
"identity",
|
||||
"routing",
|
||||
"scopes",
|
||||
"access-denied",
|
||||
"gitops",
|
||||
} <= groups
|
||||
assert {
|
||||
"forge",
|
||||
"nodes",
|
||||
"build",
|
||||
"reliability",
|
||||
"pool",
|
||||
"surfaces",
|
||||
"ephemeral",
|
||||
} <= groups
|
||||
|
||||
|
||||
def test_catalog_validation_names_every_structural_problem() -> None:
|
||||
broken = [
|
||||
check("no-such-rule", identifier="a.dup"),
|
||||
check("allowed", identifier="a.dup"),
|
||||
check("allowed", steps=(step("s", vantage="nowhere"),), identifier="b.vantage"),
|
||||
check("allowed", steps=(step("s", kind="guess"),), identifier="c.kind"),
|
||||
check(
|
||||
"denied", steps=(step("s", kind=model.REVIEW),), identifier="d.review-only"
|
||||
),
|
||||
model.CheckSpec(
|
||||
id="e.impersonation",
|
||||
title="t",
|
||||
group="g",
|
||||
rule="allowed",
|
||||
steps=(
|
||||
model.Step(
|
||||
key="s",
|
||||
vantage=catalog.OPERATOR,
|
||||
argv=("kubectl", "get", "ns", "--as", "x"),
|
||||
),
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
problems = " | ".join(harness_run.validate_catalog(broken))
|
||||
|
||||
assert "duplicate check id" in problems
|
||||
assert "unknown rule" in problems
|
||||
assert "unknown vantage" in problems
|
||||
assert "unknown kind" in problems
|
||||
assert "needs a real attempt" in problems
|
||||
assert "impersonates from the operator vantage" in problems
|
||||
|
||||
|
||||
def test_target_validation_rejects_every_vacuous_or_unbound_release_input() -> None:
|
||||
broken = replace(
|
||||
TARGETS,
|
||||
repo="atlas/other",
|
||||
remote="upstream",
|
||||
reviewed_pr_number=20,
|
||||
reviewed_head_ref="other",
|
||||
baseline_commit="bad",
|
||||
remote_main_sha="bad",
|
||||
reviewed_head_sha="bad",
|
||||
build_sha="bad",
|
||||
agent_image="latest",
|
||||
deployment_revision="0",
|
||||
node_count=1,
|
||||
pool_replicas=1,
|
||||
chat_config_revision="",
|
||||
pool_worker_env=(),
|
||||
dependency_heads=((14, "bad"), (14, "bad")),
|
||||
max_evidence_age_seconds=0,
|
||||
)
|
||||
problems = harness_run.validate_targets(broken)
|
||||
assert len(problems) >= 12
|
||||
sha = "1" * 40
|
||||
valid = replace(
|
||||
TARGETS,
|
||||
remote_main_sha=sha,
|
||||
reviewed_head_sha=sha,
|
||||
build_sha=sha,
|
||||
agent_image=f"registry.example/hermes:git-{sha}-build-7@sha256:{'2' * 64}",
|
||||
deployment_revision="1",
|
||||
chat_config_revision="rev",
|
||||
dependency_heads=tuple(
|
||||
(number, sha) for number in TARGETS.dependency_pull_requests
|
||||
),
|
||||
)
|
||||
assert harness_run.validate_targets(valid) == []
|
||||
wrong_image_source = replace(
|
||||
valid,
|
||||
agent_image=f"registry.example/hermes:git-{'2' * 40}-build-7@sha256:{'2' * 64}",
|
||||
)
|
||||
assert "image tag/digest" in " ".join(
|
||||
harness_run.validate_targets(wrong_image_source)
|
||||
)
|
||||
wrong_build = replace(valid, build_sha="2" * 40)
|
||||
assert "build SHA" in " ".join(harness_run.validate_targets(wrong_build))
|
||||
malformed_heads = replace(
|
||||
valid,
|
||||
dependency_heads=tuple(
|
||||
(number, "bad") for number in valid.dependency_pull_requests
|
||||
),
|
||||
)
|
||||
assert "dependency heads" in " ".join(harness_run.validate_targets(malformed_heads))
|
||||
|
||||
|
||||
def test_identity_resolution_extracts_only_the_username() -> None:
|
||||
runner = runner_for([whoami("system:serviceaccount:hermes:hermes-agent")])
|
||||
username, detail = harness_run.resolve_identity(
|
||||
runner, exec_module.operator_vantage()
|
||||
)
|
||||
assert (username, detail) == ("system:serviceaccount:hermes:hermes-agent", "")
|
||||
|
||||
failed = harness_run.resolve_identity(
|
||||
runner_for([completed(stderr="no access", returncode=1)]),
|
||||
exec_module.operator_vantage(),
|
||||
)
|
||||
assert failed[0] == "" and "no access" in failed[1]
|
||||
|
||||
unparsable = harness_run.resolve_identity(
|
||||
runner_for([completed(stdout="two names")]), exec_module.operator_vantage()
|
||||
)
|
||||
assert unparsable[0] == "" and "no username" in unparsable[1]
|
||||
|
||||
nameless = harness_run.resolve_identity(
|
||||
runner_for([completed(stdout="")]), exec_module.operator_vantage()
|
||||
)
|
||||
assert nameless[0] == "" and "no username" in nameless[1]
|
||||
|
||||
|
||||
def test_vantages_resolve_to_running_pods_and_record_what_is_missing() -> None:
|
||||
runner = runner_for(
|
||||
[
|
||||
whoami("kubernetes-admin"),
|
||||
completed(stdout="pod/hermes-agent-2\npod/hermes-agent-1\n"),
|
||||
whoami("system:serviceaccount:hermes:hermes-agent"),
|
||||
completed(stdout="pod/hermes-switchyard-1\n"),
|
||||
completed(stderr="no pods", returncode=1),
|
||||
completed(stdout="pod/hermes-chat-tenant-0\n"),
|
||||
]
|
||||
)
|
||||
|
||||
vantages, records = harness_run.resolve_vantages(runner, TARGETS)
|
||||
by_name = {record.name: record for record in records}
|
||||
|
||||
assert set(vantages) == {
|
||||
catalog.OPERATOR,
|
||||
catalog.SELF,
|
||||
catalog.SWITCHYARD,
|
||||
catalog.CHAT,
|
||||
}
|
||||
assert by_name[catalog.SELF].identity == "system:serviceaccount:hermes:hermes-agent"
|
||||
assert "hermes-agent-1" in by_name[catalog.SELF].description # deterministic pick
|
||||
assert by_name[catalog.NODE].available is False
|
||||
assert by_name[catalog.CHAT].available is True
|
||||
|
||||
|
||||
def test_an_unreachable_operator_stops_vantage_resolution_immediately() -> None:
|
||||
runner = runner_for([completed(stderr="connection refused", returncode=1)])
|
||||
|
||||
vantages, records = harness_run.resolve_vantages(runner, TARGETS)
|
||||
|
||||
assert set(vantages) == {catalog.OPERATOR}
|
||||
assert records[0].available is False
|
||||
assert harness_run.vantage_problems(records)[0].startswith(
|
||||
"the operator vantage is unavailable"
|
||||
)
|
||||
|
||||
|
||||
def test_a_missing_chat_pod_is_recorded_rather_than_assumed_present() -> None:
|
||||
runner = runner_for(
|
||||
[
|
||||
whoami("kubernetes-admin"),
|
||||
completed(stdout="pod/hermes-agent-1\n"),
|
||||
whoami("system:serviceaccount:hermes:hermes-agent"),
|
||||
completed(stdout="pod/hermes-switchyard-1\n"),
|
||||
completed(stdout="pod/hermes-node-1\n"),
|
||||
completed(stderr="pods 'hermes-chat-tenant-0' not found", returncode=1),
|
||||
]
|
||||
)
|
||||
|
||||
vantages, records = harness_run.resolve_vantages(runner, TARGETS)
|
||||
chat = next(record for record in records if record.name == catalog.CHAT)
|
||||
|
||||
assert catalog.CHAT not in vantages
|
||||
assert chat.available is False and "not found" in chat.detail
|
||||
|
||||
|
||||
def test_a_selector_that_matches_no_running_pod_leaves_the_vantage_absent() -> None:
|
||||
runner = runner_for(
|
||||
[
|
||||
whoami("kubernetes-admin"),
|
||||
completed(stdout="\n"),
|
||||
completed(stdout="\n"),
|
||||
completed(stdout="\n"),
|
||||
completed(stdout="\n"),
|
||||
]
|
||||
)
|
||||
|
||||
vantages, records = harness_run.resolve_vantages(runner, TARGETS)
|
||||
by_name = {record.name: record for record in records}
|
||||
|
||||
assert set(vantages) == {catalog.OPERATOR}
|
||||
assert "no running pod matches" in by_name[catalog.SELF].detail
|
||||
|
||||
|
||||
def test_two_vantages_that_are_the_same_principal_invalidate_the_evidence() -> None:
|
||||
"""Corroboration needs two identities; one identity twice is a restatement."""
|
||||
records = [
|
||||
model.VantageRecord(
|
||||
name=catalog.OPERATOR, description="d", identity="same", available=True
|
||||
),
|
||||
model.VantageRecord(
|
||||
name=catalog.SELF, description="d", identity="same", available=True
|
||||
),
|
||||
model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True),
|
||||
model.VantageRecord(name=catalog.CHAT, description="d", available=True),
|
||||
]
|
||||
|
||||
problems = harness_run.vantage_problems(records)
|
||||
|
||||
assert len(problems) == 1
|
||||
assert "same principal" in problems[0]
|
||||
|
||||
|
||||
def test_vantage_problems_flag_a_missing_or_anonymous_self_probe() -> None:
|
||||
operator = model.VantageRecord(
|
||||
name=catalog.OPERATOR, description="d", identity="admin", available=True
|
||||
)
|
||||
|
||||
absent = harness_run.vantage_problems([operator])
|
||||
assert "in-pod Hermes vantage is unavailable" in absent[0]
|
||||
|
||||
anonymous = harness_run.vantage_problems(
|
||||
[
|
||||
operator,
|
||||
model.VantageRecord(name=catalog.SELF, description="d", available=True),
|
||||
]
|
||||
)
|
||||
assert "did not report an identity" in anonymous[0]
|
||||
|
||||
clean = harness_run.vantage_problems(
|
||||
[
|
||||
operator,
|
||||
model.VantageRecord(
|
||||
name=catalog.SELF, description="d", identity="agent", available=True
|
||||
),
|
||||
model.VantageRecord(
|
||||
name=catalog.SWITCHYARD, description="d", available=True
|
||||
),
|
||||
model.VantageRecord(name=catalog.CHAT, description="d", available=True),
|
||||
]
|
||||
)
|
||||
assert clean == []
|
||||
|
||||
|
||||
def test_a_check_whose_vantage_is_absent_is_not_run() -> None:
|
||||
spec = check("allowed", steps=(step("s", vantage=catalog.NODE),))
|
||||
|
||||
result = harness_run.run_check(runner_for([]), spec, {})
|
||||
|
||||
assert result.status == model.NOT_RUN
|
||||
assert "vantage node is unavailable" in result.reason
|
||||
assert result.outcomes[0].error == "vantage node is unavailable"
|
||||
|
||||
|
||||
def test_validate_catalog_rejects_impersonation_from_every_vantage() -> None:
|
||||
"""The operator vantage was the only one guarded; now none is exempt."""
|
||||
for vantage in (catalog.OPERATOR, catalog.SELF, catalog.CHAT):
|
||||
spec = check(
|
||||
"stdout_matches",
|
||||
{"step": "s", "equals": "x"},
|
||||
steps=(
|
||||
model.Step(
|
||||
key="s",
|
||||
vantage=vantage,
|
||||
argv=("kubectl", "get", "ns", "-o", "name", "--as", "system:admin"),
|
||||
),
|
||||
),
|
||||
)
|
||||
problems = harness_run.validate_catalog([spec])
|
||||
assert any("impersonates" in problem for problem in problems)
|
||||
assert any(vantage in problem for problem in problems)
|
||||
|
||||
|
||||
def test_bulk_evidence_is_evaluated_but_replaced_before_it_is_recorded() -> None:
|
||||
spec = check(
|
||||
"stdout_matches",
|
||||
{"step": "s", "equals": "locked"},
|
||||
steps=(
|
||||
model.Step(
|
||||
key="s",
|
||||
vantage=catalog.OPERATOR,
|
||||
argv=("kubectl", "version"),
|
||||
record=False,
|
||||
),
|
||||
),
|
||||
)
|
||||
runner = runner_for([completed(stdout="locked", stderr="noise")])
|
||||
|
||||
result = harness_run.run_check(
|
||||
runner, spec, {catalog.OPERATOR: exec_module.operator_vantage()}
|
||||
)
|
||||
|
||||
assert result.status == model.PASS
|
||||
assert result.outcomes[0].stdout == harness_run.UNRECORDED
|
||||
assert result.outcomes[0].stderr == harness_run.UNRECORDED
|
||||
# Withholding bulk bytes must not withhold the binary that produced them.
|
||||
recorded = result.outcomes[0]
|
||||
assert recorded.executable_path
|
||||
assert recorded.executable_sha256
|
||||
assert recorded.executable_path == "kubectl"
|
||||
assert recorded.executable_sha256 == "a" * 64
|
||||
assert recorded.as_dict()["executable_sha256"] == recorded.executable_sha256
|
||||
|
||||
|
||||
def test_an_expired_deadline_leaves_the_report_saying_so() -> None:
|
||||
clock = FakeClock()
|
||||
runner = exec_module.Runner(
|
||||
clock=clock,
|
||||
spawn=FakeSpawn([]),
|
||||
environ={},
|
||||
deadline_seconds=1.0,
|
||||
attestor=lambda command, _environment: exec_module.Attestation(
|
||||
command, "a" * 64
|
||||
),
|
||||
)
|
||||
clock.advance(5.0)
|
||||
spec = check("allowed", steps=(step("s", vantage=catalog.OPERATOR),))
|
||||
|
||||
report = harness_run.build_report(
|
||||
runner,
|
||||
TARGETS,
|
||||
[spec],
|
||||
{catalog.OPERATOR: exec_module.operator_vantage()},
|
||||
[],
|
||||
"read-only",
|
||||
"2026-08-17T00:00:00Z",
|
||||
)
|
||||
|
||||
assert report.results[0].status == model.NOT_RUN
|
||||
assert any("deadline expired" in error for error in report.harness_errors)
|
||||
assert report.decision == model.NO_GO
|
||||
|
||||
|
||||
def test_the_report_carries_the_configured_baseline_and_sorts_its_checks() -> None:
|
||||
runner = runner_for([completed(stdout="ok"), completed(stdout="ok")])
|
||||
specs = [
|
||||
check(
|
||||
"allowed", steps=(step("s", vantage=catalog.OPERATOR),), identifier="z.last"
|
||||
),
|
||||
check(
|
||||
"allowed",
|
||||
steps=(step("s", vantage=catalog.OPERATOR),),
|
||||
identifier="a.first",
|
||||
),
|
||||
]
|
||||
records = [
|
||||
model.VantageRecord(
|
||||
name=catalog.OPERATOR, description="d", identity="admin", available=True
|
||||
),
|
||||
model.VantageRecord(
|
||||
name=catalog.SELF, description="d", identity="agent", available=True
|
||||
),
|
||||
model.VantageRecord(name=catalog.SWITCHYARD, description="d", available=True),
|
||||
model.VantageRecord(name=catalog.CHAT, description="d", available=True),
|
||||
]
|
||||
|
||||
report = harness_run.build_report(
|
||||
runner,
|
||||
TARGETS,
|
||||
specs,
|
||||
{catalog.OPERATOR: exec_module.operator_vantage()},
|
||||
records,
|
||||
"read-only",
|
||||
"2026-08-17T00:00:00Z",
|
||||
)
|
||||
|
||||
assert [result.spec.id for result in report.results] == ["a.first", "z.last"]
|
||||
assert report.baseline["baseline_commit"] == TARGETS.baseline_commit
|
||||
assert report.baseline["dependency_pull_requests"] == list(
|
||||
TARGETS.dependency_pull_requests
|
||||
)
|
||||
assert report.decision == model.GO
|
||||
assert report.finished_at
|
||||
|
||||
|
||||
def test_armed_results_replace_the_matching_catalog_placeholders() -> None:
|
||||
placeholder = check(
|
||||
"not_armed", identifier="ephemeral.feature-branch-push", mandatory=False
|
||||
)
|
||||
armed = model.CheckResult(
|
||||
spec=model.CheckSpec(
|
||||
id="ephemeral.feature-branch-push",
|
||||
title="t",
|
||||
group="ephemeral",
|
||||
rule="armed",
|
||||
),
|
||||
status=model.PASS,
|
||||
reason="pushed",
|
||||
)
|
||||
|
||||
report = harness_run.build_report(
|
||||
runner_for([]),
|
||||
TARGETS,
|
||||
[placeholder],
|
||||
{},
|
||||
[],
|
||||
"ephemeral-armed",
|
||||
"2026-08-17T00:00:00Z",
|
||||
[armed],
|
||||
)
|
||||
|
||||
assert len(report.results) == 1
|
||||
assert report.results[0].status == model.PASS
|
||||
|
||||
|
||||
def test_an_unstartable_run_reports_no_go_with_a_named_reason() -> None:
|
||||
report = harness_run.unavailable_report(
|
||||
"read-only", "2026-08-17T00:00:00Z", "arming refused: bad phrase", TARGETS
|
||||
)
|
||||
|
||||
assert report.decision == model.NO_GO
|
||||
assert report.results[0].spec.id == "harness.startup"
|
||||
assert report.results[0].status == model.NOT_RUN
|
||||
assert report.harness_errors == ["arming refused: bad phrase"]
|
||||
114
testing/tests/test_hermes_handoff_support.py
Normal file
114
testing/tests/test_hermes_handoff_support.py
Normal file
@ -0,0 +1,114 @@
|
||||
"""Shared loader and fixtures for the handoff acceptance harness tests.
|
||||
|
||||
The harness lives in ``scripts/ops`` as sibling modules rather than an installed
|
||||
package, so tests import it the way the entry point does: by putting that
|
||||
directory on the path once and importing by name.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
OPS = REPO_ROOT / "scripts" / "ops"
|
||||
|
||||
if str(OPS) not in sys.path:
|
||||
sys.path.insert(0, str(OPS))
|
||||
|
||||
|
||||
def load_handoff_module(name: str) -> Any:
|
||||
"""Import one harness module by name."""
|
||||
return importlib.import_module(name)
|
||||
|
||||
|
||||
exec_module = load_handoff_module("hermes_handoff_exec")
|
||||
model = load_handoff_module("hermes_handoff_model")
|
||||
|
||||
|
||||
def outcome(
|
||||
stdout: str = "",
|
||||
stderr: str = "",
|
||||
returncode: int | None = 0,
|
||||
error: str | None = None,
|
||||
truncated: bool = False,
|
||||
vantage: str = "self",
|
||||
argv: tuple[str, ...] = ("kubectl", "get", "pods"),
|
||||
) -> Any:
|
||||
"""Return a synthetic command outcome for evaluator tests."""
|
||||
return exec_module.Outcome(
|
||||
argv=argv,
|
||||
vantage=vantage,
|
||||
returncode=returncode,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
error=error,
|
||||
truncated=truncated,
|
||||
)
|
||||
|
||||
|
||||
def spec(
|
||||
rule: str,
|
||||
expect: dict | None = None,
|
||||
steps: tuple = (),
|
||||
identifier: str = "test.check",
|
||||
mandatory: bool = True,
|
||||
scope: str | None = None,
|
||||
) -> Any:
|
||||
"""Return a synthetic check specification."""
|
||||
return model.CheckSpec(
|
||||
id=identifier,
|
||||
title="test check",
|
||||
group="test",
|
||||
rule=rule,
|
||||
steps=steps,
|
||||
expect=expect or {},
|
||||
mandatory=mandatory,
|
||||
scope=scope or model.ALWAYS,
|
||||
)
|
||||
|
||||
|
||||
def step(
|
||||
key: str, kind: str | None = None, vantage: str = "self", optional: bool = False
|
||||
) -> Any:
|
||||
"""Return a synthetic step."""
|
||||
return model.Step(
|
||||
key=key,
|
||||
vantage=vantage,
|
||||
argv=("kubectl", "get", "pods", "-o", "name"),
|
||||
kind=kind or model.READ,
|
||||
optional=optional,
|
||||
)
|
||||
|
||||
|
||||
class FakeClock:
|
||||
"""A monotonic clock the tests advance by hand."""
|
||||
|
||||
def __init__(self, start: float = 0.0) -> None:
|
||||
self.now = start
|
||||
|
||||
def __call__(self) -> float:
|
||||
return self.now
|
||||
|
||||
def advance(self, seconds: float) -> None:
|
||||
"""Move the clock forward."""
|
||||
self.now += seconds
|
||||
|
||||
|
||||
class FakeSpawn:
|
||||
"""A ``subprocess.run`` stand-in driven by a queue of scripted results."""
|
||||
|
||||
def __init__(self, results: list) -> None:
|
||||
self.results = list(results)
|
||||
self.calls: list[dict] = []
|
||||
|
||||
def __call__(self, argv, **kwargs):
|
||||
self.calls.append({"argv": list(argv), **kwargs})
|
||||
if not self.results:
|
||||
raise AssertionError(f"unexpected command: {argv}")
|
||||
outcome_or_error = self.results.pop(0)
|
||||
if isinstance(outcome_or_error, Exception):
|
||||
raise outcome_or_error
|
||||
return outcome_or_error
|
||||
243
testing/tests/test_hermes_handoff_zero_evidence.py
Normal file
243
testing/tests/test_hermes_handoff_zero_evidence.py
Normal file
@ -0,0 +1,243 @@
|
||||
"""Regressions for the zero-evidence fail-open in the absence checks.
|
||||
|
||||
``evaluate_names_absent`` used to return ``PASS`` when its step exited 0 with no
|
||||
output, so five mandatory checks — the ones asserting that provider API keys,
|
||||
forge credentials, cluster-admin bindings, and shared coordinator state are
|
||||
*absent* — could report a pass on no evidence at all and turn a ``NO_GO`` into a
|
||||
``GO``. These tests pin the real catalog specs, not synthetic look-alikes, and
|
||||
reproduce both ways silence actually arrives: a POSIX pipeline whose exit status
|
||||
comes from its last stage, and a ``kubectl -o jsonpath`` over drifted structure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
from testing.tests.test_hermes_handoff_support import load_handoff_module, outcome
|
||||
|
||||
catalog = load_handoff_module("hermes_handoff_catalog")
|
||||
evaluators = load_handoff_module("hermes_handoff_evaluators")
|
||||
harness_run = load_handoff_module("hermes_handoff_run")
|
||||
model = load_handoff_module("hermes_handoff_model")
|
||||
policy = load_handoff_module("hermes_handoff_policy")
|
||||
|
||||
# Every mandatory check whose whole claim is that something is not there.
|
||||
ABSENCE_CHECKS = (
|
||||
"access.no-cluster-admin-binding-for-agent",
|
||||
"forge.worker-holds-no-git-credential-variable",
|
||||
"identity.no-provider-api-key-in-manifest",
|
||||
"identity.no-provider-api-key-in-pod",
|
||||
"pool.coordinator-retains-sole-state-ownership",
|
||||
)
|
||||
|
||||
|
||||
def real_catalog() -> dict:
|
||||
targets = catalog.Targets(now=datetime.datetime.now(datetime.UTC))
|
||||
return {spec.id: spec for spec in harness_run.build_catalog(targets)}
|
||||
|
||||
|
||||
def test_the_five_absence_checks_are_exactly_the_mandatory_names_absent_set() -> None:
|
||||
"""If a sixth absence check appears it must be covered here too."""
|
||||
specs = real_catalog()
|
||||
found = {
|
||||
identifier
|
||||
for identifier, spec in specs.items()
|
||||
if spec.rule == "names_absent" and spec.mandatory
|
||||
}
|
||||
assert found == set(ABSENCE_CHECKS)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("identifier", ABSENCE_CHECKS)
|
||||
def test_a_real_absence_check_never_passes_on_zero_observations(
|
||||
identifier: str,
|
||||
) -> None:
|
||||
"""rc=0 with no lines is drift, so the verdict must be NOT_RUN, not PASS."""
|
||||
spec = real_catalog()[identifier]
|
||||
step_key = spec.expect["step"]
|
||||
|
||||
silent = evaluators.evaluate(spec, {step_key: outcome(stdout="")})
|
||||
assert silent.status == model.NOT_RUN
|
||||
assert "no lines" in silent.reason
|
||||
assert silent.status != model.PASS
|
||||
|
||||
whitespace = evaluators.evaluate(spec, {step_key: outcome(stdout="\n \n\t\n")})
|
||||
assert whitespace.status == model.NOT_RUN
|
||||
|
||||
|
||||
@pytest.mark.parametrize("identifier", ABSENCE_CHECKS)
|
||||
def test_a_real_absence_check_still_passes_on_real_evidence(identifier: str) -> None:
|
||||
"""The guard must not turn a healthy observation into a false NOT_RUN."""
|
||||
spec = real_catalog()[identifier]
|
||||
step_key = spec.expect["step"]
|
||||
benign = "clusterrolebinding.rbac.authorization.k8s.io/system:basic-user\nPATH\n"
|
||||
|
||||
passed = evaluators.evaluate(spec, {step_key: outcome(stdout=benign)})
|
||||
assert passed.status == model.PASS
|
||||
assert passed.evidence["observed_count"] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("identifier", ABSENCE_CHECKS)
|
||||
def test_a_real_absence_check_still_fails_on_a_forbidden_name(identifier: str) -> None:
|
||||
spec = real_catalog()[identifier]
|
||||
step_key = spec.expect["step"]
|
||||
offender = next(
|
||||
iter(spec.expect.get("names", ()) or spec.expect.get("contains", ()))
|
||||
)
|
||||
|
||||
verdict = evaluators.evaluate(
|
||||
spec, {step_key: outcome(stdout=f"OTHER\n{offender}")}
|
||||
)
|
||||
assert verdict.status == model.FAIL
|
||||
assert offender in verdict.reason
|
||||
|
||||
|
||||
def test_zero_evidence_on_a_mandatory_absence_check_forces_no_go() -> None:
|
||||
"""A NOT_RUN on a mandatory check must not be rounded up to a GO.
|
||||
|
||||
Every other check is held at PASS so the verdict turns on the five absence
|
||||
checks alone: silence must cost the release its GO.
|
||||
"""
|
||||
specs = real_catalog()
|
||||
silent, green = [], []
|
||||
for identifier, spec in specs.items():
|
||||
status = model.PASS
|
||||
if identifier in ABSENCE_CHECKS:
|
||||
evaluation = evaluators.evaluate(
|
||||
spec, {spec.expect["step"]: outcome(stdout="")}
|
||||
)
|
||||
assert evaluation.status == model.NOT_RUN
|
||||
status = evaluation.status
|
||||
silent.append(model.CheckResult(spec=spec, status=status))
|
||||
green.append(model.CheckResult(spec=spec, status=model.PASS))
|
||||
|
||||
assert all(result.blocking for result in silent if result.status == model.NOT_RUN)
|
||||
assert model.Report(mode="read-only", started_at="t", results=silent).decision == (
|
||||
model.NO_GO
|
||||
)
|
||||
assert (
|
||||
model.Report(mode="read-only", started_at="t", results=green).decision
|
||||
== model.GO
|
||||
)
|
||||
|
||||
|
||||
def test_a_broken_shell_pipeline_exits_zero_with_no_output() -> None:
|
||||
"""The harness' own frozen template is reachable with rc=0 and 0 bytes.
|
||||
|
||||
A POSIX pipeline reports the status of its *last* stage, so a missing or
|
||||
renamed upstream binary produces success-with-no-evidence rather than an
|
||||
error. This is one of the two live paths that made the fail-open reachable.
|
||||
"""
|
||||
script = policy.render_shell("env_names").replace("/usr/bin/env", "/usr/bin/absent")
|
||||
|
||||
completed = subprocess.run( # noqa: S603
|
||||
["/bin/sh", "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert completed.returncode == 0
|
||||
assert completed.stdout == ""
|
||||
|
||||
spec = real_catalog()["identity.no-provider-api-key-in-pod"]
|
||||
verdict = evaluators.evaluate(
|
||||
spec, {"env": outcome(stdout=completed.stdout, returncode=completed.returncode)}
|
||||
)
|
||||
assert verdict.status == model.NOT_RUN
|
||||
|
||||
|
||||
DRIFTED_JSONPATH_OUTPUT = {
|
||||
# A renamed parent — `containerz` — makes kubectl print nothing at all.
|
||||
"renamed parent": "",
|
||||
# A renamed leaf — `.nam` — makes it print only the separators.
|
||||
"renamed leaf": "\n" * 6,
|
||||
# A container filter that matches nothing behaves the same way.
|
||||
"unmatched filter": "",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", sorted(DRIFTED_JSONPATH_OUTPUT))
|
||||
def test_drifted_kubectl_jsonpath_output_is_not_run(shape: str) -> None:
|
||||
"""The exact bytes a drifted ``-o jsonpath`` emits must never read as a pass.
|
||||
|
||||
``kubectl -o jsonpath`` exits 0 when the structure it walks is missing, so a
|
||||
renamed field, a renamed leaf, or an unmatched container filter all deliver
|
||||
success with no usable evidence. Pinned as literal output so the regression
|
||||
holds with no cluster and no network; the live rc=0 behaviour is asserted
|
||||
separately by :func:`test_a_live_drifted_projection_still_exits_zero`.
|
||||
"""
|
||||
spec = real_catalog()["identity.no-provider-api-key-in-manifest"]
|
||||
|
||||
verdict = evaluators.evaluate(
|
||||
spec, {"env": outcome(stdout=DRIFTED_JSONPATH_OUTPUT[shape])}
|
||||
)
|
||||
|
||||
assert verdict.status == model.NOT_RUN
|
||||
assert "no lines" in verdict.reason
|
||||
|
||||
|
||||
def test_a_live_drifted_projection_still_exits_zero() -> None:
|
||||
"""Confirm against the real tool that drift is success, not an error.
|
||||
|
||||
kubectl has no offline JSONPath renderer, so this reads the live agent
|
||||
workload — a plain, read-only ``get`` with a name-only projection — and skips
|
||||
when no cluster is reachable. The hermetic assertion above does not skip.
|
||||
"""
|
||||
kubectl = shutil.which("kubectl")
|
||||
if kubectl is None: # pragma: no cover - kubectl ships with the release image
|
||||
pytest.skip("kubectl is not installed in this environment")
|
||||
base = ["--namespace", "hermes", "get", "deploy/hermes-agent", "-o"]
|
||||
healthy = 'jsonpath={range .spec.template.spec.containers[*]}{.name}{"\\n"}{end}'
|
||||
probe = subprocess.run( # noqa: S603
|
||||
[kubectl, *base, healthy],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
if probe.returncode != 0:
|
||||
pytest.skip("no read-only cluster access in this environment")
|
||||
assert probe.stdout.strip()
|
||||
|
||||
for drifted in (
|
||||
'jsonpath={range .spec.template.spec.containerz[*]}{.name}{"\\n"}{end}',
|
||||
'jsonpath={range .spec.template.spec.containers[*]}{.nam}{"\\n"}{end}',
|
||||
):
|
||||
result = subprocess.run( # noqa: S603
|
||||
[kubectl, *base, drifted],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0
|
||||
assert result.stdout.strip() == ""
|
||||
spec = real_catalog()["identity.no-provider-api-key-in-manifest"]
|
||||
verdict = evaluators.evaluate(spec, {"env": outcome(stdout=result.stdout)})
|
||||
assert verdict.status == model.NOT_RUN
|
||||
|
||||
|
||||
def test_every_absence_probe_yields_one_line_per_observed_item() -> None:
|
||||
"""A healthy probe must produce evidence, or the guard would misfire.
|
||||
|
||||
The pool claim projection emits ``<volume>=<claim>`` so a template volume
|
||||
without a PVC still counts as an observation; an empty result therefore only
|
||||
happens when the object or the projection has drifted.
|
||||
"""
|
||||
spec = real_catalog()["pool.coordinator-retains-sole-state-ownership"]
|
||||
argv = spec.steps[0].argv
|
||||
assert '{.name}{"="}' in argv[-1]
|
||||
|
||||
verdict = evaluators.evaluate(spec, {"claims": outcome(stdout="tmp=\nconfig=\n")})
|
||||
assert verdict.status == model.PASS
|
||||
assert verdict.evidence["observed_count"] == 2
|
||||
|
||||
leaked = evaluators.evaluate(
|
||||
spec, {"claims": outcome(stdout="state=hermes-agent-home\n")}
|
||||
)
|
||||
assert leaked.status == model.FAIL
|
||||
@ -18,6 +18,21 @@ def test_bundled_contract_exposes_local_and_jenkins_profiles():
|
||||
assert contract["pytest_suites"]["unit"]["paths"]
|
||||
|
||||
|
||||
def test_handoff_modules_are_exactly_managed_linted_and_covered():
|
||||
"""The canonical contract must not silently drop a handoff module."""
|
||||
contract = load_contract()
|
||||
expected = {
|
||||
path.as_posix() for path in Path("scripts/ops").glob("hermes_handoff_*.py")
|
||||
}
|
||||
assert len(expected) == 16
|
||||
assert expected <= set(contract["managed_modules"])
|
||||
assert expected <= set(contract["lint_paths"])
|
||||
assert expected <= set(contract["coverage"]["tracked_files"])
|
||||
assert expected <= set(contract["coverage"]["branch_tracked_files"])
|
||||
assert "scripts/ops" in contract["pytest_suites"]["unit"]["coverage_sources"]
|
||||
assert "scripts/ops/hermes_handoff_*.py" in contract["hygiene"]["line_limit_globs"]
|
||||
|
||||
|
||||
def test_docs_check_reports_missing_docstring_and_missing_path(tmp_path: Path):
|
||||
module_path = tmp_path / "managed.py"
|
||||
module_path.write_text("value = 1\n", encoding="utf-8")
|
||||
@ -43,7 +58,10 @@ def test_docs_check_reports_missing_required_doc_metadata(tmp_path: Path):
|
||||
(tmp_path / "README.md").write_text("", encoding="utf-8")
|
||||
|
||||
contract = {
|
||||
"required_docs": [{"path": "README.md", "description": ""}, {"path": "missing.md", "description": "Missing"}],
|
||||
"required_docs": [
|
||||
{"path": "README.md", "description": ""},
|
||||
{"path": "missing.md", "description": "Missing"},
|
||||
],
|
||||
"managed_modules": [],
|
||||
"lint_paths": [],
|
||||
"pytest_suites": {"unit": {"description": "", "paths": []}},
|
||||
@ -62,6 +80,7 @@ def test_docs_check_reports_missing_required_doc_metadata(tmp_path: Path):
|
||||
def test_hygiene_check_enforces_line_limit_and_name_rules(tmp_path: Path):
|
||||
tests_dir = tmp_path / "tests"
|
||||
tests_dir.mkdir()
|
||||
(tests_dir / "conftest.py").write_text("value = 1\n", encoding="utf-8")
|
||||
bad_name = tests_dir / "bad-name.py"
|
||||
bad_name.write_text("x = 1\n", encoding="utf-8")
|
||||
long_file = tests_dir / "test_too_long.py"
|
||||
@ -84,7 +103,9 @@ def test_hygiene_check_enforces_line_limit_and_name_rules(tmp_path: Path):
|
||||
issues = run_hygiene_check(contract, tmp_path)
|
||||
|
||||
assert any("file exceeds 3 LOC" in issue for issue in issues)
|
||||
assert any("naming rule failed" in issue and "bad-name.py" in issue for issue in issues)
|
||||
assert any(
|
||||
"naming rule failed" in issue and "bad-name.py" in issue for issue in issues
|
||||
)
|
||||
|
||||
|
||||
def test_coverage_check_enforces_per_file_floor(tmp_path: Path):
|
||||
@ -126,9 +147,9 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path):
|
||||
|
||||
def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Path):
|
||||
missing_xml = tmp_path / "missing.xml"
|
||||
assert run_coverage_check({"coverage": {"tracked_files": []}}, tmp_path, missing_xml) == [
|
||||
"coverage xml missing: missing.xml"
|
||||
]
|
||||
assert run_coverage_check(
|
||||
{"coverage": {"tracked_files": []}}, tmp_path, missing_xml
|
||||
) == ["coverage xml missing: missing.xml"]
|
||||
|
||||
source_dir = tmp_path / "pkg"
|
||||
source_dir.mkdir()
|
||||
|
||||
@ -12,10 +12,15 @@ def test_compute_workspace_line_coverage_handles_missing_xml(tmp_path: Path) ->
|
||||
"""Missing coverage XML should produce a zero workspace coverage score."""
|
||||
|
||||
contract = {"coverage": {"tracked_files": ["managed.py"]}}
|
||||
assert compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") == 0.0
|
||||
assert (
|
||||
compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml")
|
||||
== 0.0
|
||||
)
|
||||
|
||||
|
||||
def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path: Path) -> None:
|
||||
def test_compute_workspace_line_coverage_averages_present_tracked_files(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Workspace coverage should average only tracked files that appear in the report."""
|
||||
|
||||
coverage_xml = tmp_path / "coverage.xml"
|
||||
@ -41,7 +46,9 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path
|
||||
assert compute_workspace_line_coverage(contract, tmp_path, coverage_xml) == 75.0
|
||||
|
||||
|
||||
def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: Path) -> None:
|
||||
def test_run_check_keeps_relative_names_when_source_roots_do_not_match(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Relative filenames should remain relative when no declared source root contains them."""
|
||||
|
||||
coverage_xml = tmp_path / "coverage.xml"
|
||||
@ -76,6 +83,19 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path:
|
||||
assert issues == ["line coverage below 95.0%: relative.py (80.0%)"]
|
||||
|
||||
|
||||
def test_workspace_coverage_returns_zero_when_xml_has_no_tracked_samples(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
xml = tmp_path / "coverage.xml"
|
||||
xml.write_text("<coverage><packages /></coverage>", encoding="utf-8")
|
||||
assert (
|
||||
compute_workspace_line_coverage(
|
||||
{"coverage": {"tracked_files": ["missing.py"]}}, tmp_path, xml
|
||||
)
|
||||
== 0.0
|
||||
)
|
||||
|
||||
|
||||
def test_run_check_fails_a_file_below_the_branch_floor(tmp_path: Path) -> None:
|
||||
"""Full line coverage must never hide a file whose branch coverage is weak."""
|
||||
|
||||
|
||||
@ -100,6 +100,14 @@ def test_build_sonar_issues_uses_safe_defaults_for_sparse_findings() -> None:
|
||||
assert issue["primaryLocation"]["textRange"] == {"startLine": 1, "endLine": 1}
|
||||
|
||||
|
||||
def test_sonar_issue_never_ends_before_its_start_line() -> None:
|
||||
issue = semgrep_report._sonar_issue_from_finding(
|
||||
{"path": "app.py", "start": {"line": 10}, "end": {"line": 2}}
|
||||
)
|
||||
assert issue is not None
|
||||
assert issue["primaryLocation"]["textRange"] == {"startLine": 10, "endLine": 10}
|
||||
|
||||
|
||||
def test_build_sonar_issues_clamps_inverted_line_ranges() -> None:
|
||||
"""An end line before the start line should collapse to a valid range."""
|
||||
|
||||
@ -131,7 +139,9 @@ def test_read_json_handles_invalid_json_and_non_object(tmp_path: Path) -> None:
|
||||
list_json = tmp_path / "list.json"
|
||||
list_json.write_text("[]", encoding="utf-8")
|
||||
|
||||
assert semgrep_report._read_json(bad_json)["errors"][0]["message"].startswith("invalid JSON:")
|
||||
assert semgrep_report._read_json(bad_json)["errors"][0]["message"].startswith(
|
||||
"invalid JSON:"
|
||||
)
|
||||
assert semgrep_report._read_json(list_json) == {
|
||||
"errors": [{"message": "report payload is not an object"}],
|
||||
"results": [],
|
||||
@ -160,16 +170,25 @@ def test_main_handles_missing_or_invalid_input(tmp_path: Path) -> None:
|
||||
payload = json.loads(output.read_text(encoding="utf-8"))
|
||||
assert payload["status"] == "failed"
|
||||
assert payload["errors_total"] == 1
|
||||
sonar_payload = json.loads((tmp_path / "build" / "sonar-issues.json").read_text(encoding="utf-8"))
|
||||
sonar_payload = json.loads(
|
||||
(tmp_path / "build" / "sonar-issues.json").read_text(encoding="utf-8")
|
||||
)
|
||||
assert sonar_payload == {"issues": []}
|
||||
|
||||
|
||||
def test_main_honors_custom_blocking_severity_without_sonar_output(tmp_path: Path) -> None:
|
||||
def test_main_honors_custom_blocking_severity_without_sonar_output(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""CLI callers can choose stricter blocking severities without writing Sonar issues."""
|
||||
|
||||
semgrep_json = tmp_path / "semgrep.json"
|
||||
semgrep_json.write_text(
|
||||
json.dumps({"results": [{"path": "app.py", "extra": {"severity": "WARNING"}}], "errors": []}),
|
||||
json.dumps(
|
||||
{
|
||||
"results": [{"path": "app.py", "extra": {"severity": "WARNING"}}],
|
||||
"errors": [],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
output = tmp_path / "semgrep-report.json"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user