Compare commits

...

54 Commits

Author SHA1 Message Date
jenkins
a9ee1070be Merge remote-tracking branch 'origin/feature/hermes-full-handoff-acceptance' into wt/t_1e95ea6d
# Conflicts:
#	services/hermes/agent-deployment.yaml
#	testing/tests/test_hermes_auto_router.py
2026-08-18 06:41:13 -03:00
jenkins
943ecfd15b Merge remote-tracking branch 'origin/feature/hermes-distributed-worker-pool' into feature/hermes-full-handoff-acceptance
# Conflicts:
#	ci/scripts/semgrep_report.py
#	testing/quality_contract.json
#	testing/quality_coverage.py
#	testing/tests/conftest.py
#	testing/tests/test_hermes_auto_router.py
#	testing/tests/test_quality_contract.py
#	testing/tests/test_quality_coverage_helpers.py
#	testing/tests/test_semgrep_report.py
2026-08-18 06:28:18 -03:00
jenkins
5f27e50c74 hermes: reconcile pool tests and staging hygiene with the train API
Post-merge fixes after rebasing the distributed worker pool onto the
review-goal-semantics train tip:

- Pool SCM submission tests target the train's relocated receive-pack
  scanner: FEATURE_REF_RE now lives in receive_pack_scan, bodies are
  built via the shared _receive_command helper (valid pack), and the
  update-rejection assertion matches the train's message.
- Take the train's canonical test_hermes_scm_broker,
  test_hermes_cli_dispatch_runtime and test_hermes_cli_execution_edges,
  which exercise the train's broker/dispatch/execution behavior.
- Runtime staging tests patch os.fchown alongside os.chown so the
  UID-10000 _write_secret path passes under a non-root gate runner
  (production ownership behavior unchanged).
- Split the jenkins build-evidence contracts out of
  test_hermes_runtime_access into test_hermes_runtime_evidence to keep
  both files under the 500-LOC hygiene ceiling after the merge.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-08-18 02:00:19 -03:00
jenkins
8036101f0b Merge remote-tracking branch 'origin/feature/hermes-review-goal-semantics' into feature/hermes-distributed-worker-pool
# Conflicts:
#	scripts/tests/test_dashboards_render_atlas_drilldowns.py
#	scripts/tests/test_dashboards_render_jobs.py
#	services/hermes/scm-common/scripts/scm_broker.py
#	services/hermes/scripts/cli_lane_dispatch.py
#	services/hermes/scripts/cli_lane_execution.py
#	testing/quality_contract.json
#	testing/tests/test_hermes_agent_access.py
#	testing/tests/test_hermes_agent_security.py
#	testing/tests/test_hermes_chat_config.py
#	testing/tests/test_hermes_chat_images.py
#	testing/tests/test_hermes_chat_provider_auth.py
#	testing/tests/test_hermes_chat_quality.py
#	testing/tests/test_hermes_chat_support.py
#	testing/tests/test_hermes_chat_voice.py
#	testing/tests/test_hermes_cli_finalization_edges.py
#	testing/tests/test_hermes_cli_foundation_coverage.py
#	testing/tests/test_hermes_cli_lanes_configuration.py
#	testing/tests/test_hermes_cli_recovery_edges.py
#	testing/tests/test_hermes_cli_retention_edges.py
#	testing/tests/test_hermes_coordinator.py
#	testing/tests/test_hermes_coordinator_boards.py
#	testing/tests/test_hermes_coordinator_support.py
2026-08-18 01:43:39 -03:00
jenkins
7db9f61659 Merge remote-tracking branch 'origin/feature/hermes-cli-auto-quota-routing' into feature/hermes-review-goal-semantics
# Conflicts:
#	services/hermes/scripts/cli_lane_runner.py
2026-08-18 01:18:06 -03:00
jenkins
603524ba6a Merge remote-tracking branch 'origin/fix/hermes-result-decomposition-reliability' into feature/hermes-cli-auto-quota-routing
# Conflicts:
#	services/hermes/networkpolicy.yaml
#	testing/quality_contract.json
#	testing/tests/test_hermes_agent_security.py
2026-08-18 00:57:54 -03:00
jenkins
a3077224e0 Merge origin/main (#14 + #16) into PR #15 decomposition reliability
Reconcile two independent test/gate reorganizations:
- Gate/semgrep/mailu: keep main's #16 dual-metric implementation.
- quality_contract.json: union #16 image-builder + #14 scm/node + #15 cli_lane.
- agent-deployment.yaml: keep #14 gitea removal + #16 image-build-token + #15 probe.
- Test splits: main's chat/coordinator/agent organization is authoritative;
  drop #15's redundant competing splits and #14's stale cli-lane duplicates;
  keep #15's cli-lane decomposition suite and port the execution-safety test.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-18 00:42:53 -03:00
be1ff33d4b Merge pull request 'Bound Hermes Atlas pull request authority' (#14) from feature/hermes-safe-gitea-pr-client into main
Reviewed-on: atlas/titan-iac#14
2026-08-18 03:17:20 +00:00
flux-bot
8c7074ae16 chore(maintenance): automated image update 2026-08-18 01:48:20 +00:00
flux-bot
6b11331520 chore(maintenance): automated image update 2026-08-18 01:47:20 +00:00
flux-bot
376cc14dd1 chore(maintenance): automated image update 2026-08-18 01:44:19 +00:00
flux-bot
77df54e3ba chore(maintenance): automated image update 2026-08-18 01:39:17 +00:00
jenkins
9d42ff33f5 Merge origin/main (#16 image-builder) into hermes-safe-gitea-pr
Resolve 8 conflicts, uniting #16's daemonless image-release lane with
#14's SCM broker + node-audit boundary. Nothing dropped from either side.

- quality_coverage.py / test_quality_coverage_helpers.py: take main's
  dual-metric gate (line+branch enforced per file at minimum_percent);
  drop #14's now-dead branch_tracked_files/minimum_branch_percent keys.
- quality_contract.json: union all lists (managed_modules, lint_paths,
  coverage_sources, tracked_files, hygiene globs) so #14's scm-broker,
  git_pack_objects, receive_pack_scan, deadline_http and node_polkit_audit
  are branch-checked alongside #16's image-builder modules.
- Vault auth (vault_k8s_auth_configure.sh): coexist both role sets. Keep
  #16's hermes-switchyard split + hermes-jenkins-token-seed; keep #14's
  hermes-scm-broker role. Preserve #14's security property: hermes-agent
  no longer holds developer-gitea (broker role carries it).
- agent-deployment.yaml / stage_runtime_access.py: keep #16's
  jenkins-image-build-token injection/staging; keep #14's removal of the
  agent's gitea-token/gitea-username.
- Bump vault-k8s-auth-hermes job -9 -> -10 (and its health check + test)
  so the merged auth config re-applies over any -9 already in-cluster.
- flux hermes dependsOn: union jenkins + hermes-scm-broker + observer-rbac.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-17 21:07:06 -03:00
jenkins
a5d0abb678 hermes: align capacity gate with auth classifier
CAPACITY_PATTERN (the gate that sets result.capacity_failure) lacked the
bare unauthorized/forbidden/401/403 signals that classify_capacity_failure
already recognizes, so an auth blip surfacing only as "403 Forbidden"
was blocked as capability instead of transient: no auto failover, no
health cooldown. Add 401|403|unauthorized|forbidden to the gate so it
matches the classifier; reason classification still distinguishes auth
from quota/rate-limit/transport.

Tests: an auto card failing with only "403 Forbidden" now fails over to
the other provider, records an auth cooldown (authenticated:false), and
classifies the fallback reason as auth in metrics; a manual card with the
same failure still fails closed as transient. Router-outage tests moved to
test_hermes_cli_router_outage.py to keep both files <500 LOC.

Based on PR #15 (fix/hermes-result-decomposition-reliability).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-08-17 20:46:00 -03:00
jenkins
31439f096a hermes: bound streaming git DNS resolution
The watchdog can only close a live socket, but getaddrinfo runs before
any socket exists and ignores socket timeouts, so a slow resolver
outlived the stream deadline. Resolve under the same absolute deadline
in a joinable worker before the real connect.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-17 20:37:30 -03:00
jenkins
6c1123201e hermes: audit every sudo include spelling
A quoted, space-bearing #include path failed the include regex and fell
through to the comment branch, leaving a second authority file
unenumerated. Detect any include directive before the comment rule and
fail closed on every form except the exact bare includedir into the
audited /etc/sudoers.d.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-17 20:37:30 -03:00
jenkins
6e9e8c6ab5 hermes: reject regex literals in polkit grants
A regex literal embedding a quote desynced the string tokenizer and
swallowed an unconditional Result.YES, so the grant read as unscoped and
slipped past the literal-identity check. Fail closed on any slash outside
a string or comment; neither regex nor division belongs in an
identity-scoped grant.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-17 20:37:30 -03:00
41d10ad3f7 Merge pull request 'hermes: add daemonless agent image release lane' (#16) from feature/hermes-unprivileged-image-builder into main
Reviewed-on: atlas/titan-iac#16
2026-08-17 23:36:50 +00:00
jenkins
6df688adcc hermes: test quota gating, cooldown recovery, and failover policy
Deterministic coverage for the quota-aware lane: threshold boundaries
(14.9/15/15.1), both-below preference, fetch-failure fail-open, cooldown
elapsed-vs-not hysteresis, quota-reset recovery (never for auth),
explicit fail-closed in both directions, bounded double-failure block,
failure-reason classification, metrics emission, and worker env key
stripping. Based on PR #15 (fix/hermes-result-decomposition-reliability).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-08-17 20:31:12 -03:00
jenkins
034c8372c7 hermes: enforce quota-aware fail-closed cli-auto provider routing
Based on PR #15 (fix/hermes-result-decomposition-reliability); stacked
on the decomposed cli_lane modules.

- cli_lane_quota: soft-exclude a provider from NEW cli-auto work below
  the remaining-quota threshold (both-below prefers more remaining;
  fetch failure fails open with a metric).
- cli_lane_health: lane now writes provider health (G7) with classified
  failure reasons splitting the capacity conflation (quota/auth/
  rate-limit/transport) and cooldown hysteresis; re-admission only on
  full cooldown expiry, passed quota reset, or fresh success (G4).
- cli_lane_routing: capacity-limited health now excludes a provider
  (G3); cooldown/reset-aware re-admission.
- cli_lane_failover: explicit cli-codex-*/cli-claude-* assignees fail
  closed as transient instead of switching providers (G5); fallback
  depth stays bounded at two hosted providers (G1) with effort
  preserved; Switchyard outages block transient, not capability (G9).
- cli_lane_metrics: route-decision/fallback counters, quota and
  soft-exclusion gauges, pod-local scrape server (G6).
- cli_lane_provider: worker env drops ANTHROPIC_API_KEY, CLAUDE_API_KEY,
  OPENAI_API_KEY, API_SERVER_KEY so no metered path exists (G10).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-08-17 20:31:05 -03:00
jenkins
62d8cd984b hermes: wire quota-aware lane routing config and metrics scrape
Add kanban routing keys (provider_quota_min_remaining_percent: 15,
provider_capacity_cooldown_seconds: 300, provider_auth_cooldown_seconds:
3600), a lane-metrics port/Service on 9011 with service-annotation
scraping, monitoring ingress for the new port, and the lane's quota
metrics URL env. Based on PR #15 (fix/hermes-result-decomposition-
reliability); stacked because this work rides on the decomposed lane.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-08-17 20:30:53 -03:00
jenkins
6f993c049e hermes: keep agent web reachable during lane deferral
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-17 15:20:43 -03:00
jenkins
5020e49057 hermes: track new boundary modules in gate
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-17 15:15:47 -03:00
jenkins
b8f2163af0 hermes: prove sudo and polkit denial closed
Expand sudoers User_Alias chains so aliases, wildcards, netgroups, and
undefined names cannot smuggle authority to the Hermes account, and
require polkit grants to scope through exact literal identity
comparisons: computed strings, bracket lookups, subject aliasing,
operator-built values, and unconditional or wildcard grants fail closed.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-17 15:15:47 -03:00
jenkins
340ec58b68 hermes: quarantine-scan pushed git objects
Inflate every receive-pack object under strict pack, size, and checksum
bounds, resolve deltas against in-pack bases only, and scan the real
decompressed payloads for runtime-token forms, private keys, SSH key
material, and known provider token formats. Thin packs are rejected so
no pushed content escapes the scan, and upstream Git exchanges now run
under one absolute stream deadline.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-17 15:15:47 -03:00
jenkins
578239a496 hermes: bound SCM control calls with hard deadlines
Run every Gitea API and broker control exchange inside a killable helper
process whose connect, send, and read share one absolute wall-clock
deadline, and add a watchdog that force-closes streaming connections at
expiry. Redirects are rejected before authentication headers can move.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-17 15:15:35 -03:00
jenkins
2019b08276 gitea: require literal branch protection rules
Replace the reimplemented gobwas/glob matcher with exact-literal rule
matching that fails closed on any special or malformed pattern, accepts
legacy zero priorities, and rejects duplicate primary-branch rules.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-17 15:15:35 -03:00
jenkins
f77119b238 hermes: enforce per-file branch coverage in the quality gate
The coverage check read only line-rate, so a file could pass with weak
branch coverage. Load both Cobertura rates and fail any tracked file
below the 95% floor on either metric, failing closed when branch
evidence is absent. Prove enforcement end to end with a synthetic
fully-lined but branch-weak file failing run_profile.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-17 15:10:17 -03:00
jenkins
64272f52d2 hermes: lift remaining tracked modules to the branch floor
Exercise the mailu sync retry, attribute, and skip branches, the
listener non-object JSON path, and the hygiene conftest skip; drop the
unreachable inverted-range clamp in the semgrep report (the line helper
already floors the end line) and pin that behavior with a test. Exclude
the mailu __main__ guards from measurement.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-17 15:10:17 -03:00
jenkins
3a57371989 hermes: close release-lane branch coverage gaps
Cover the uncovered branches in the image release script (redirect
handler, policy-read credential and size bounds, count and shape
rejections, EOF manifest scan) and in the Harbor immutability helper
(scope-failure verification, absent robot duration, stale-read retries,
created-rule ID binding, bounded startup retries). Exclude the
__main__ guard like the sibling release script.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-17 15:10:17 -03:00
jenkins
df75479069 hermes: close lane branch coverage gaps
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-17 15:03:59 -03:00
jenkins
e3ecc18d0e hermes: refresh capabilities and readiness per loop
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-17 15:03:50 -03:00
Hermes Agent
360d8aad8f hermes: close the review-role gaps found reviewing PR 22
Repairs the blockers from the independent review of the previous head. The
role-aware verdict contract was correct but reachable only through one call
site and only on cards written with real newlines, so most real review cards
never used it.

* The role-blind call is no longer a weaker classifier that can reject before
  the role-aware judge runs. Without a card there is no defensible
  role-dependent judgement, so `unfinished_result_reason()` applies only the
  card-independent checks. That closes the short-circuit at every call site,
  including the one PR15 moves to `cli_lane_execution`, and it makes a
  journalled terminal record accepted under one version of these semantics
  re-validate under any other instead of being quarantined into a re-dispatch
  of an already-accepted task.
* The lane resolves the role from the card and passes it, and records it in
  the Kanban metadata. The verdict contract binds only where the lane can buy
  another turn: in single-shot mode a rejection discards the worker's real
  result, so review cards keep the relaxation without gaining any rejection
  single-shot mode did not already have.
* Card scope expands the literal \n escapes the board stores in one-line
  bodies, so explicit `Hermes-Task-Role` / `Hermes-Expected-Output` directives
  are honoured on the 6 of 78 live cards that carry no real newline, and the
  read-only, verdict and mutation heuristics stop being cut apart by them.
* Card scope now ends at the first non-card H2 and at the runner's controller
  evidence, which is emitted under its own heading. Goal-controller rejection
  history can no longer sit inside the card, and an upstream heading rename
  fails closed instead of admitting history into role resolution.
* The inference recognises the SHIP/BLOCK-shaped deliverables real cards
  actually use: 21 of 78 live cards resolve to review, up from 10, with no
  implementation card misclassified. Cards asking for a findings list rather
  than a verdict deliberately stay on the model judge, since the verdict is
  the review contract's only gate.
* A declared review role no longer outranks mutation evidence: a report that
  changed files falls back to the implementation regime.
* Judge reasons go through the agent runtime's canonical redactor, extended
  for the two shapes it deliberately passes through and this lane handles -
  `scheme://user:secret@host` and a credential named in prose - while a 40-hex
  commit SHA survives as evidence.

Regressions cover the recovered t_dbdcd739 incident, a verbatim snapshot of
every live card on three boards with a hand-labelled expected role, the
upgrade and single-shot properties against the previous gate, the upstream
context-heading contract, and the end-to-end `execute_claim` shape that used
to burn every goal turn.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 17:27:11 +00:00
Hermes Agent
54156f839e docs(hermes): record the repaired pool operational boundaries
Document what an operator needs to know about the repairs: why the ordinal
workspace claim is ReadWriteMany and why colocation is only a preference, that
`lease_failed` is retryable rather than terminal and what a persistent deferred
park means, why a retry publishes an attempt-scoped ref and what to expect from
the extra drafts, and why the best-effort pool is deliberately absent from the
hermes Kustomization health checks.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:47:24 +00:00
Hermes Agent
5d963c9354 hermes: make pool lease recovery and release isolation safe
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.

`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.

The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.

Also closes three consequential handoff risks the same review raised:

* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
  claim, so a drain or preemption that moved only the lower-priority worker
  deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
  shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
  already are on the same class), colocation is a preference, and the mediator
  shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
  never submit and the run's work was discarded with the failure. Submission
  now targets a fresh attempt- or content-scoped ref in the same reviewed
  namespace -- never an update -- and is idempotent under replay. A refused
  submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
  the 10m Flux health window for the whole hermes app. They now install once
  per pinned version onto a durable volume, re-verified against the real
  binaries and time-bounded, and the best-effort pool no longer gates the
  health of the app its dependents wait on.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
Hermes Agent
7a55b259bf hermes: add the fenced three-node distributed execution pool
Three fenced worker Pods claim Hermes Kanban runs through a coordinator that
owns every state transition, with per-ordinal HMAC authority, a mediated
broker-only SCM path, and durable per-ordinal workspaces.

Content is the reviewed head of PR #18 (689bcb6e) with PR 16's and PR 19's
contributions removed: they were merged in only to validate co-existence and are
not prerequisites, so this branch no longer carries them as ancestors. Only PR 14
and PR 15 remain, because the broker boundary and the cli_lane_* decomposition
are load-bearing for two of the fixed P0 boundaries.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00
Hermes Agent
03b10ff8b1 Merge PR 15 (CLI lane decomposition) on top of PR 14 (Atlas SCM broker)
The distributed worker pool stacks on exactly two open pull requests, in this
order: PR 14 supplies the broker-only SCM boundary the mediators route through
and the hermes-scm-boundary-v2 ConfigMap they mount, and PR 15 supplies the
cli_lane_* decomposition -- including canonical_run_id and the eligibility
predicate on claim_ready -- that the coordinator depends on. Neither can be
dropped without breaking a fixed P0 boundary, so both are carried here as
prerequisites and this branch must not merge before them.

PR 16 (agent image release lane) and PR 19 (full-handoff acceptance harness)
are NOT prerequisites and are deliberately absent, so reviewing this branch no
longer means approving them.

PR 14 and PR 15 conflict with each other in nine paths. Each is resolved to the
resolution already reviewed on this branch at 4d4cf1bd.
2026-08-17 16:29:29 +00:00
Hermes Agent
02e559d7ac docs(hermes): record the recomputed conflict graph and head SHAs
Refresh #12 and #18 to their live Gitea heads, record #19's replaced head
exactly, and replace the pairwise conflict set with a freshly computed one
over all 28 combinations. #18+#19 is now clean because #19 no longer
duplicates the repo-wide hygiene legacy-exception mechanism that #18 also
carries; #12+#19 is new because #12 advanced. Name the exact overlapping
files per conflicting pair and how to resolve each additively.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:27:28 +00:00
Hermes Agent
d208a89d9d fix(hermes): close the zero-evidence fail-open in absence checks
evaluate_names_absent returned PASS when its step exited 0 with no output,
so five mandatory checks - the ones asserting that provider API keys, forge
credentials, a cluster-admin binding, and shared coordinator state are
absent - could report a pass on no evidence and turn a NO_GO into a GO.
Both name rules now resolve their step through one guard in _line_step, so
zero observations are NOT_RUN. Regressions pin all five real catalog specs
plus both reachable silence paths: a POSIX pipeline whose status comes from
its last stage, and a drifted kubectl -o jsonpath. The pool claim projection
emits one <volume>=<claim> line per template volume so a volume without a
PVC still counts as an observation rather than reading as drift.

Also closes the review's reachable hardening and evidence defects:

- pin Gitea paths to atlas/titan-iac on an exact segment boundary and
  reject relative segments, including percent-encoded ones
- forbid impersonation structurally in every mode and vantage; the inner
  command of kubectl exec is re-checked rather than exempted, and
  validate_catalog no longer guards only the operator vantage
- drop flux and helm from the binary allowlist; they had no pinned release
  digest, so no allowlisted binary can now be admitted that the executor
  would refuse to attest
- remove the inert --concurrency and --expect-telegram-sessions flags and
  the dead concurrency bound; Telegram continuity stays mandatory
- read the ephemeral pull index page by page, treat the create response as
  an authoritative source for the pull number, close every number either
  source names, and surface residue_ref plus exact manual_cleanup commands
  when creation is uncertain
- keep executable_path and executable_sha256 on unrecorded bulk-evidence
  steps so withholding bytes never withholds binary attestation
- revert the repo-wide hygiene legacy-exception mechanism; the contract
  change here is purely additive and the four pre-existing over-cap files
  are left to the canonical contract change in PR #14/#15
- correct the runbook ruff format scope so the documented command passes

Split hermes_handoff_arming.py out of hermes_handoff_ephemeral.py to keep
both modules under the 500-line cap. All 16 handoff modules hold at least
95% line and branch coverage; the mutation gate is 13/13.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:24:37 +00:00
Hermes Agent
b080b5f622 hermes: finalize completed review verdicts in the goal judge
The local goal judge scored every worker report against "did the reviewed
implementation reach a shippable state". A read-only reviewer that returned a
completed BLOCK verdict with findings was therefore resumed turn after turn with
an instruction to repair code it was forbidden to touch (observed live on
t_dbdcd739), burning subscription capacity and risking an unbounded loop.

Completion is now judged against the action the card assigned:

* Cards declare their role explicitly with Hermes-Task-Role / Hermes-Expected-
  Output metadata. Pre-contract cards fall back to a narrow inference that needs
  a read-only scope, a requested verdict, no requested mutation deliverable, and
  a report that changed no files.
* Role resolution reads only the card itself. Prior attempts, parent results,
  cross-task history and comments appended to the worker context can no longer
  reassign the role.
* Review, audit and diagnostic cards finalize deterministically on a truthful
  SHIP or BLOCK verdict with evidence, and fail closed on a missing,
  unrecognized or self-contradictory verdict, on a BLOCK without findings, and
  on a verdict without evidence. Every rejection reason carries the read-only
  guard, so a resumed review is never told to edit the reviewed implementation.
* Implementation cards keep the fail-closed model judge unchanged, including the
  unfinished-work heuristic and judge-unavailable rejection.
* All judge reasons are bounded, single-line and secret-redacted before they
  reach Kanban metadata, comments and continuation prompts.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 14:54:36 +00:00
Hermes Agent
c808baff40 docs(hermes): record repaired conflict graph 2026-08-17 13:12:36 +00:00
Hermes Agent
9d5214f809 fix(hermes): close full-handoff acceptance blockers 2026-08-17 13:11:34 +00:00
jenkins
e8c26ecf85 hermes: add daemonless agent image release lane 2026-08-17 08:25:52 -03:00
jenkins
73fefbb5d9 hermes: gate mixed image lane APIs 2026-08-17 08:16:35 -03:00
jenkins
8c6e3acdac hermes: isolate Atlas SCM write authority 2026-08-17 07:58:44 -03:00
jenkins
2c1d97e551 hermes: match canonical Forgejo PR URL 2026-08-17 07:22:26 -03:00
jenkins
cf3e46c2b6 hermes: harden Forgejo client boundary 2026-08-17 07:22:26 -03:00
jenkins
762784da6b hermes: bound Atlas pull request client 2026-08-17 07:22:26 -03:00
jenkins
1ba9f13959 hermes: bind terminal commit to journal inode 2026-08-17 07:19:10 -03:00
jenkins
8c71d55853 hermes: harden journal quarantine races 2026-08-17 07:15:31 -03:00
jenkins
c20bc263e0 hermes: close terminal replay edge cases 2026-08-17 07:15:31 -03:00
jenkins
64ea6d685c hermes: harden replay and decomposition races 2026-08-17 07:15:31 -03:00
jenkins
f04a524738 hermes: preserve terminal results and prevent duplicate fanout 2026-08-17 07:15:31 -03:00
8f00545828 hermes: add a fail-closed full-handoff acceptance harness
Decides whether the Hermes platform handoff is fit to release, and refuses
to round an absence of evidence up to a pass.

The harness is read-only by default and classifies 71 checks PASS / FAIL /
NOT_RUN / NOT_APPLICABLE. Any mandatory FAIL or NOT_RUN is NO_GO, and so is a
harness-level problem: an unreachable vantage, a catalog entry whose evidence
no longer exists, an expired deadline, or an evaluator that raised.

Evidence comes from two vantages that cannot cover for each other: an external
read-only operator kubeconfig, and the Hermes agent probing itself from inside
its own pod. Before any check runs, the harness asks each vantage who it is and
stops if they are the same principal, because dual-vantage evidence from one
identity is a restatement rather than a corroboration. `--as` is rejected for
every operator-side command and reachable only as the inner command of a
`kubectl exec`, so impersonation can never stand in for a real self-probe. A
deny check needs a live refused request, not only an authorization review.

Two safety properties are structural rather than conventional, enforced where
an argv becomes a subprocess: the default mode mutates nothing (mutating verbs
require a server dry run; there is deliberately no live TokenRequest probe,
because a successful one would mint a real credential), and no probe can pull a
credential value into a report (no vault/sops/curl, secrets readable only with
-o name, environment probes list names, shell only through frozen reviewed
templates). Captures are bounded before they are screened, and the rendered
report is re-screened before it is written.

Mutation lives behind a separate arming flag with an exact confirmation phrase,
a caller-supplied unique ref, a preflight that refuses a protected push target
before any network call, and a cleanup whose verification is itself mandatory.
A default run reports those four checks NOT_RUN.

The catalog is declarative so a reviewer reads what is asserted rather than how
it is plumbed, and so structural properties can be proven over every entry
before a run. Catalog drift surfaces as NOT_RUN, which stops the release.

docs/hermes_full_handoff_acceptance.md carries the merge order for PRs #14-#18
on top of the merged #13 baseline, the image build and Flux rollout, the
rollback point for each step, the go/no-go checklist, and the limits that are
asserted rather than exercised.

Validation: 295 handoff tests pass with 100% line coverage on all 15 new
modules; the full unit suite is 647 passed with two failures that reproduce
unchanged on origin/main; Ruff, py_compile, kustomize render, and a diff
credential screen are clean; a live read-only run against Atlas returns NO_GO
for the pre-merge cluster with no unscreened fields in the report.
2026-08-17 10:14:17 +00:00
296 changed files with 58492 additions and 6120 deletions

View File

@ -0,0 +1,288 @@
pipeline {
agent {
kubernetes {
defaultContainer 'python'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
atlas.bstein.dev/workload: hermes-agent-image-builder
spec:
serviceAccountName: hermes-image-builder
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
securityContext:
fsGroup: 1000
fsGroupChangePolicy: OnRootMismatch
nodeSelector:
kubernetes.io/arch: arm64
node-role.kubernetes.io/worker: "true"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-04
- titan-14
- titan-18
- titan-19
- titan-22
- titan-24
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: hardware
operator: In
values:
- rpi5
imagePullSecrets:
- name: harbor-bstein-robot
containers:
- name: jnlp
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 25m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
- name: python
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
command: ["sleep"]
args: ["99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 25m
memory: 64Mi
limits:
cpu: 250m
memory: 256Mi
- name: kaniko
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
command: ["/busybox/sh", "-c"]
args: ["/busybox/sleep 99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
privileged: false
runAsUser: 0
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 250m
memory: 1Gi
limits:
cpu: "2"
memory: 4Gi
"""
}
}
parameters {
booleanParam(
name: 'PUBLISH_IMAGE',
defaultValue: false,
description: 'Publish the reviewed main revision to Harbor.'
)
string(
name: 'EXPECTED_SOURCE_REVISION',
defaultValue: '',
description: 'Exact 40-character commit on atlas/titan-iac main.'
)
string(
name: 'CONFIRM_PUBLISH',
defaultValue: '',
description: 'Enter PUBLISH HERMES AGENT to confirm the release.'
)
}
environment {
HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent'
}
options {
disableConcurrentBuilds()
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
skipDefaultCheckout(true)
timeout(time: 90, unit: 'MINUTES')
}
stages {
stage('Checkout reviewed source') {
steps {
checkout scm
}
}
stage('Enforce release boundary') {
steps {
sh '''
set -eu
mkdir -p build
test "${PUBLISH_IMAGE}" = "true"
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
case "${EXPECTED_SOURCE_REVISION}" in
*[!0-9a-f]*|'')
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
exit 2
;;
esac
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
actual_revision="$(git rev-parse HEAD)"
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
test "${actual_revision}" = "$(git rev-parse origin/main)"
test -z "$(git status --porcelain)"
test -f dockerfiles/Dockerfile.hermes-agent
case "${BUILD_NUMBER}" in
''|0*|*[!0-9]*)
echo "BUILD_NUMBER must be a positive decimal integer" >&2
exit 2
;;
esac
printf '%s\n' \
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
> build/hermes-agent.destination
'''
}
}
stage('Reject replay before publish') {
steps {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-agent.destination)"
python3 ci/scripts/hermes_image_release.py assert-absent \
--source-revision "${EXPECTED_SOURCE_REVISION}" \
--build-number "${BUILD_NUMBER}" \
--destination "${destination}"
'''
}
}
}
stage('Build and publish without a daemon') {
steps {
container('kaniko') {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''#!/busybox/sh
set -eu
set +x
config_path=/kaniko/.docker/config.json
destination="$(cat build/hermes-agent.destination)"
umask 077
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
/busybox/mkdir -p /kaniko/.docker
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
unset HARBOR_USER HARBOR_PASSWORD auth
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
/kaniko/executor \
--context="dir://${WORKSPACE}" \
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
--destination="${destination}" \
--digest-file="${WORKSPACE}/build/hermes-agent.digest" \
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \
--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
--label="org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" \
--cleanup \
--push-retry=3
'''
}
}
}
}
stage('Render reviewed Flux handoff') {
steps {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-agent.destination)"
python3 ci/scripts/hermes_image_release.py render \
--digest-file build/hermes-agent.digest \
--image-file build/hermes-agent.image \
--source-revision "${EXPECTED_SOURCE_REVISION}" \
--build-number "${BUILD_NUMBER}" \
--destination "${destination}" \
--kustomization services/hermes/kustomization.yaml \
--output-dir build/hermes-agent-release
test -s build/hermes-agent-release/hermes-image-update.patch
test -s build/hermes-agent-release/hermes-agent-image.json
'''
}
}
}
}
post {
success {
sh '''
set -eu
expected_files="$(printf '%s\n' \
build/hermes-agent.destination \
build/hermes-agent.digest \
build/hermes-agent.image \
build/hermes-agent-release/hermes-agent-image.json \
build/hermes-agent-release/hermes-image-update.patch \
build/hermes-agent-release/hermes-kustomization.yaml \
| LC_ALL=C sort)"
actual_files="$(find build -type f -print | LC_ALL=C sort)"
test "${actual_files}" = "${expected_files}"
destination="$(cat build/hermes-agent.destination)"
python3 ci/scripts/hermes_image_release.py verify-evidence \
--digest-file build/hermes-agent.digest \
--image-file build/hermes-agent.image \
--source-revision "${EXPECTED_SOURCE_REVISION}" \
--build-number "${BUILD_NUMBER}" \
--destination "${destination}" \
--kustomization services/hermes/kustomization.yaml \
--output-dir build/hermes-agent-release
'''
archiveArtifacts(
artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
allowEmptyArchive: false,
fingerprint: true
)
}
cleanup {
container('kaniko') {
sh '''#!/busybox/sh
/busybox/rm -f /kaniko/.docker/config.json
'''
}
}
}
}

View File

@ -0,0 +1,500 @@
#!/usr/bin/env python3
"""Verify and render a reviewable Hermes agent image release."""
from __future__ import annotations
import argparse
import base64
import difflib
import json
import os
import re
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Any, Callable
DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-agent"
HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0"
HARBOR_PROJECT = "bstein"
HARBOR_REPOSITORY = "hermes-agent"
IMMUTABLE_REPOSITORY_PATTERN = "hermes-agent"
IMMUTABLE_TAG_PATTERN = "git-*-build-*"
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$")
DESTINATION_PATTERN = re.compile(
r"^registry\.bstein\.dev/bstein/hermes-agent:"
r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$"
)
class _NoRedirect(urllib.request.HTTPRedirectHandler):
"""Never send registry credentials to a redirect target."""
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
return None
def _validated(value: str, pattern: re.Pattern[str], label: str) -> str:
"""Return a normalized value when it matches the release contract."""
normalized = value.strip()
if not pattern.fullmatch(normalized):
raise ValueError(f"invalid {label}: expected {pattern.pattern}")
return normalized
def validate_destination(
destination: str, source_revision: str, build_number: str
) -> tuple[str, str]:
"""Bind one unique build tag to the reviewed revision and Jenkins build."""
revision = _validated(source_revision, REVISION_PATTERN, "source revision")
build = _validated(build_number, BUILD_PATTERN, "build number")
match = DESTINATION_PATTERN.fullmatch(destination.strip())
if not match or match.groups() != (revision, build):
raise ValueError(
"destination must bind the reviewed revision and unique Jenkins build"
)
return revision, build
def validate_kaniko_evidence(
*, digest_text: str, image_text: str, destination: str
) -> str:
"""Cross-check both independent Kaniko output files against the destination."""
digest_lines = digest_text.splitlines()
image_lines = image_text.splitlines()
if len(digest_lines) != 1:
raise ValueError("Kaniko digest evidence must contain exactly one line")
if len(image_lines) != 1:
raise ValueError("Kaniko image evidence must contain exactly one line")
digest = _validated(digest_lines[0], DIGEST_PATTERN, "image digest")
if image_lines[0].strip() != f"{destination}@{digest}":
raise ValueError("Kaniko image evidence does not match destination and digest")
return digest
def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
"""Make a registry request without following redirects."""
opener = urllib.request.build_opener(_NoRedirect())
try:
return opener.open(request, timeout=timeout)
except urllib.error.HTTPError as exc:
return exc
def _artifact_response(
destination: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> tuple[int, bytes]:
"""Read one exact Harbor artifact by tag with bounded response size."""
match = DESTINATION_PATTERN.fullmatch(destination)
if not match:
raise ValueError("invalid destination")
if not username or not password:
raise RuntimeError("Harbor credentials are empty")
tag = destination.rsplit(":", 1)[1]
encoded_tag = urllib.parse.quote(tag, safe="")
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
request = urllib.request.Request(
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/"
f"{HARBOR_REPOSITORY}/artifacts/{encoded_tag}"
"?with_immutable_status=true",
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
method="GET",
)
with opener(request, 20) as response:
body = response.read(1_048_577)
if len(body) > 1_048_576:
raise RuntimeError("Harbor artifact response exceeded the size limit")
return int(response.status), body
def _immutable_rules_response(
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> tuple[int, bytes, dict[str, str]]:
"""Read the project policy with the same least-privilege publish identity."""
if not username or not password:
raise RuntimeError("Harbor credentials are empty")
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
request = urllib.request.Request(
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules"
"?page=1&page_size=100",
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
method="GET",
)
with opener(request, 20) as response:
body = response.read(1_048_577)
if len(body) > 1_048_576:
raise RuntimeError("Harbor immutable rule response exceeded the size limit")
return int(response.status), body, dict(response.headers)
def _require_complete_rule_page(
rules: list[dict[str, Any]], headers: dict[str, str]
) -> None:
"""Require proof that the bounded first page contains every rule."""
raw_total = next(
(value for key, value in headers.items() if key.lower() == "x-total-count"),
None,
)
if raw_total is None or not str(raw_total).isdecimal():
raise RuntimeError("Harbor immutable rule list omitted a valid total count")
if int(raw_total) != len(rules):
raise RuntimeError("Harbor immutable rule list was truncated")
def _normalized_immutable_rule(rule: dict[str, Any]) -> dict[str, Any]:
"""Select only fields that bind the server-side build-tag policy."""
return {
"disabled": bool(rule.get("disabled", False)),
"action": rule.get("action"),
"template": rule.get("template"),
"tag_selectors": [
{
"kind": item.get("kind"),
"decoration": item.get("decoration"),
"pattern": item.get("pattern"),
}
for item in rule.get("tag_selectors") or []
if isinstance(item, dict)
],
"scope_selectors": {
"repository": [
{
"kind": item.get("kind"),
"decoration": item.get("decoration"),
"pattern": item.get("pattern"),
}
for item in (rule.get("scope_selectors") or {}).get(
"repository", []
)
if isinstance(item, dict)
]
},
}
def verify_immutable_policy(
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> None:
"""Fail closed before build unless the exact Harbor rule is active."""
status, body, headers = _immutable_rules_response(
username=username, password=password, opener=opener
)
if status != 200:
raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}")
try:
rules = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc
if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules):
raise RuntimeError("Harbor immutable rule list has an invalid shape")
_require_complete_rule_page(rules, headers)
expected = {
"disabled": False,
"action": "immutable",
"template": "immutable_template",
"tag_selectors": [
{
"kind": "doublestar",
"decoration": "matches",
"pattern": IMMUTABLE_TAG_PATTERN,
}
],
"scope_selectors": {
"repository": [
{
"kind": "doublestar",
"decoration": "repoMatches",
"pattern": IMMUTABLE_REPOSITORY_PATTERN,
}
]
},
}
matches = [
_normalized_immutable_rule(item)
for item in rules
if _normalized_immutable_rule(item)["tag_selectors"]
== expected["tag_selectors"]
and _normalized_immutable_rule(item)["scope_selectors"]
== expected["scope_selectors"]
]
if matches != [expected]:
raise RuntimeError("Harbor immutable build-tag policy is absent or not exact")
def assert_tag_absent(
destination: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> None:
"""Reject replay before Kaniko can push an already-used immutable identity."""
status, _body = _artifact_response(
destination, username=username, password=password, opener=opener
)
if status == 404:
return
if status == 200:
raise RuntimeError("Harbor destination tag already exists; refusing overwrite")
raise RuntimeError(f"Harbor destination preflight returned HTTP {status}")
def verify_registry_digest(
destination: str,
digest: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> None:
"""Verify Harbor independently resolves the pushed tag to Kaniko's digest."""
digest = _validated(digest, DIGEST_PATTERN, "image digest")
status, body = _artifact_response(
destination, username=username, password=password, opener=opener
)
if status != 200:
raise RuntimeError(f"Harbor manifest verification returned HTTP {status}")
try:
artifact = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError("Harbor returned invalid artifact JSON") from exc
harbor_digest = str(artifact.get("digest") or "").strip()
if not DIGEST_PATTERN.fullmatch(harbor_digest):
raise RuntimeError("Harbor response omitted a valid artifact digest")
if harbor_digest != digest:
raise RuntimeError("Harbor digest does not match Kaniko evidence")
expected_tag = destination.rsplit(":", 1)[1]
matching_tags = [
item
for item in artifact.get("tags") or []
if isinstance(item, dict) and item.get("name") == expected_tag
]
if len(matching_tags) != 1:
raise RuntimeError("Harbor artifact does not contain the expected tag")
if matching_tags[0].get("immutable") is not True:
raise RuntimeError("Harbor did not enforce the expected tag as immutable")
def render_kustomization(source: str, digest: str, image: str = DEFAULT_IMAGE) -> str:
"""Replace exactly one matching Kustomize image digest without reformatting."""
digest = _validated(digest, DIGEST_PATTERN, "image digest")
lines = source.splitlines(keepends=True)
matches: list[int] = []
for index, line in enumerate(lines):
if line.strip() != f"- name: {image}":
continue
name_indent = len(line) - len(line.lstrip())
for candidate_index in range(index + 1, len(lines)):
candidate = lines[candidate_index]
stripped = candidate.strip()
candidate_indent = len(candidate) - len(candidate.lstrip())
if stripped.startswith("- name:") and candidate_indent == name_indent:
break
if stripped.startswith("digest:") and candidate_indent > name_indent:
matches.append(candidate_index)
break
if len(matches) != 1:
raise ValueError(
f"expected exactly one digest for image {image!r}; found {len(matches)}"
)
index = matches[0]
newline = "\n" if lines[index].endswith("\n") else ""
prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())]
lines[index] = f"{prefix}digest: {digest}{newline}"
return "".join(lines)
def write_release_artifacts(
*,
digest: str,
source_revision: str,
build_number: str,
destination: str,
kustomization: Path,
output_dir: Path,
) -> dict[str, str]:
"""Write a rendered manifest, patch, and credential-free release metadata."""
digest = _validated(digest, DIGEST_PATTERN, "image digest")
source_revision, build_number = validate_destination(
destination, source_revision, build_number
)
source = kustomization.read_text(encoding="utf-8")
rendered = render_kustomization(source, digest)
relative_name = kustomization.name
patch = "".join(
difflib.unified_diff(
source.splitlines(keepends=True),
rendered.splitlines(keepends=True),
fromfile=f"a/services/hermes/{relative_name}",
tofile=f"b/services/hermes/{relative_name}",
)
)
if not patch:
raise ValueError("published digest already matches the Flux manifest")
output_dir.mkdir(parents=True, exist_ok=True)
(output_dir / "hermes-kustomization.yaml").write_text(rendered, encoding="utf-8")
(output_dir / "hermes-image-update.patch").write_text(patch, encoding="utf-8")
metadata = {
"build_number": build_number,
"digest": digest,
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
"image": DEFAULT_IMAGE,
"published_tag": destination,
"source_revision": source_revision,
}
(output_dir / "hermes-agent-image.json").write_text(
json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
return metadata
def validate_release_artifacts(
*,
digest_file: Path,
image_file: Path,
source_revision: str,
build_number: str,
destination: str,
kustomization: Path,
output_dir: Path,
) -> None:
"""Revalidate the exact successful-build evidence without rewriting it."""
digest = validate_kaniko_evidence(
digest_text=digest_file.read_text(encoding="utf-8"),
image_text=image_file.read_text(encoding="utf-8"),
destination=destination,
)
source_revision, build_number = validate_destination(
destination, source_revision, build_number
)
expected_names = {
"hermes-agent-image.json",
"hermes-image-update.patch",
"hermes-kustomization.yaml",
}
entries = list(output_dir.iterdir())
if {entry.name for entry in entries} != expected_names or not all(
entry.is_file() and not entry.is_symlink() for entry in entries
):
raise ValueError("release output must contain exactly three evidence files")
source = kustomization.read_text(encoding="utf-8")
rendered = render_kustomization(source, digest)
relative_name = kustomization.name
patch = "".join(
difflib.unified_diff(
source.splitlines(keepends=True),
rendered.splitlines(keepends=True),
fromfile=f"a/services/hermes/{relative_name}",
tofile=f"b/services/hermes/{relative_name}",
)
)
metadata = {
"build_number": build_number,
"digest": digest,
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
"image": DEFAULT_IMAGE,
"published_tag": destination,
"source_revision": source_revision,
}
expected = {
"hermes-agent-image.json": json.dumps(metadata, indent=2, sort_keys=True) + "\n",
"hermes-image-update.patch": patch,
"hermes-kustomization.yaml": rendered,
}
for name, expected_text in expected.items():
if (output_dir / name).read_text(encoding="utf-8") != expected_text:
raise ValueError(f"release evidence is incomplete or mismatched: {name}")
def _credentials() -> tuple[str, str]:
"""Read the masked, runtime-only Jenkins credential environment."""
username = os.environ.get("HARBOR_USER", "")
password = os.environ.get("HARBOR_PASSWORD", "")
if not username or not password:
raise RuntimeError("Harbor credentials are unavailable")
return username, password
def _common_arguments(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--source-revision", required=True)
parser.add_argument("--build-number", required=True)
parser.add_argument("--destination", required=True)
def main() -> int:
"""Fail closed around the unique tag, then verify and render after push."""
parser = argparse.ArgumentParser()
commands = parser.add_subparsers(dest="command", required=True)
absent = commands.add_parser("assert-absent")
_common_arguments(absent)
render = commands.add_parser("render")
_common_arguments(render)
render.add_argument("--digest-file", required=True, type=Path)
render.add_argument("--image-file", required=True, type=Path)
render.add_argument("--kustomization", required=True, type=Path)
render.add_argument("--output-dir", required=True, type=Path)
verify = commands.add_parser("verify-evidence")
_common_arguments(verify)
verify.add_argument("--digest-file", required=True, type=Path)
verify.add_argument("--image-file", required=True, type=Path)
verify.add_argument("--kustomization", required=True, type=Path)
verify.add_argument("--output-dir", required=True, type=Path)
args = parser.parse_args()
validate_destination(args.destination, args.source_revision, args.build_number)
if args.command == "verify-evidence":
validate_release_artifacts(
digest_file=args.digest_file,
image_file=args.image_file,
source_revision=args.source_revision,
build_number=args.build_number,
destination=args.destination,
kustomization=args.kustomization,
output_dir=args.output_dir,
)
return 0
username, password = _credentials()
if args.command == "assert-absent":
verify_immutable_policy(username=username, password=password)
assert_tag_absent(args.destination, username=username, password=password)
return 0
digest = validate_kaniko_evidence(
digest_text=args.digest_file.read_text(encoding="utf-8"),
image_text=args.image_file.read_text(encoding="utf-8"),
destination=args.destination,
)
verify_registry_digest(
args.destination, digest, username=username, password=password
)
write_release_artifacts(
digest=digest,
source_revision=args.source_revision,
build_number=args.build_number,
destination=args.destination,
kustomization=args.kustomization,
output_dir=args.output_dir,
)
return 0
if __name__ == "__main__": # pragma: no cover - exercised through main()
raise SystemExit(main())

View File

@ -52,7 +52,7 @@ def build_report(
results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else []
severity_counts = {severity: 0 for severity in SEVERITIES}
severity_counts = dict.fromkeys(SEVERITIES, 0)
blocking_findings = 0
for finding in results:
severity = _finding_severity(finding)
@ -93,9 +93,8 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None:
start = finding.get("start") if isinstance(finding.get("start"), dict) else {}
end = finding.get("end") if isinstance(finding.get("end"), dict) else {}
start_line = _line_number(start.get("line") if isinstance(start, dict) else None)
# _line_number floors the end line at start_line, so the range never inverts.
end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line)
if end_line < start_line:
end_line = start_line
return {
"engineId": "semgrep",
"ruleId": str(finding.get("check_id") or "semgrep.unknown"),

View File

@ -15,7 +15,13 @@ spec:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: false
wait: true
timeout: 10m
healthChecks:
- apiVersion: batch/v1
kind: Job
name: harbor-hermes-agent-immutability-ensure-1
namespace: harbor
dependsOn:
- name: core
- name: longhorn

View File

@ -0,0 +1,19 @@
# clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-observer-bindings
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes-observer-bindings
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 5m
dependsOn:
- name: hermes
- name: hermes-observer-rbac

View File

@ -0,0 +1,16 @@
# clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-observer-rbac
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes-observer-rbac
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 5m

View File

@ -0,0 +1,19 @@
# clusters/atlas/flux-system/applications/hermes-scm-broker-code/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-scm-broker-code
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes/scm-common
targetNamespace: hermes-scm
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 5m
dependsOn:
- name: hermes-scm-namespace

View File

@ -0,0 +1,27 @@
# clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-scm-broker
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes-scm-broker
targetNamespace: hermes-scm
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 10m
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: hermes-scm-broker
namespace: hermes-scm
dependsOn:
- name: vault
- name: gitea
- name: hermes-scm-namespace
- name: hermes-scm-broker-code

View File

@ -0,0 +1,16 @@
# clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-scm-namespace
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes-scm-namespace
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 5m

View File

@ -30,6 +30,12 @@ spec:
kind: Deployment
name: hermes-agent
namespace: hermes
# hermes-execution-worker and its mediators are deliberately absent. They run
# at scavenger priority on a best-effort pool whose first start installs the
# provider CLIs, so gating this Kustomization's 10m health window on them
# would stall hermes-chat and hermes-observer-bindings, which dependsOn
# hermes. The pool reports its own health through the worker readiness probe,
# the per-ordinal mediator /ready endpoint, and the coordinator on :9007.
- apiVersion: apps/v1
kind: DaemonSet
name: hermes-node-ssh-access
@ -60,3 +66,6 @@ spec:
- name: keycloak
- name: longhorn
- name: vault
- name: jenkins
- name: hermes-scm-broker
- name: hermes-observer-rbac

View File

@ -6,10 +6,9 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "CI controller changes are applied only during planned maintenance."
spec:
interval: 10m
suspend: true
suspend: false
path: ./services/jenkins
prune: true
sourceRef:
@ -18,6 +17,8 @@ spec:
targetNamespace: jenkins
dependsOn:
- name: helm
- name: harbor
- name: vault-hermes-jenkins-token-seed
healthChecks:
- apiVersion: apps/v1
kind: Deployment

View File

@ -4,6 +4,7 @@ kind: Kustomization
resources:
- gitea/kustomization.yaml
- vault/kustomization.yaml
- vault-hermes-jenkins-token-seed/kustomization.yaml
- vaultwarden/kustomization.yaml
- comms/kustomization.yaml
- crypto/kustomization.yaml
@ -27,7 +28,12 @@ resources:
- jenkins/kustomization.yaml
- ai-llm/kustomization.yaml
- openclaw/kustomization.yaml
- hermes-scm-namespace/kustomization.yaml
- hermes-scm-broker-code/kustomization.yaml
- hermes/kustomization.yaml
- hermes-observer-rbac/kustomization.yaml
- hermes-observer-bindings/kustomization.yaml
- hermes-scm-broker/kustomization.yaml
- hermes-chat/kustomization.yaml
- hermes-triage-demo/kustomization.yaml
- game-stream/kustomization.yaml

View File

@ -0,0 +1,26 @@
# clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: vault-hermes-jenkins-token-seed
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
path: ./services/vault-hermes-jenkins-token-seed
targetNamespace: vault
prune: true
wait: true
timeout: 5m
dependsOn:
- name: vault
healthChecks:
- apiVersion: batch/v1
kind: Job
name: vault-hermes-jenkins-build-token-seed-2
namespace: vault

View File

@ -16,6 +16,11 @@ spec:
targetNamespace: vault
prune: true
wait: true
healthChecks:
- apiVersion: batch/v1
kind: Job
name: vault-k8s-auth-hermes-10
namespace: vault
dependsOn:
- name: longhorn
- name: helm

View File

@ -15,6 +15,10 @@ RUN apt-get update \
# Paid/provider-backed search remains selectable through normal Hermes config.
RUN uv pip install --python /opt/hermes/.venv/bin/python ddgs==9.14.4
ARG HERMES_KANIKO_HEREDOC_COMPAT=0
COPY dockerfiles/Dockerfile.hermes-agent /tmp/hermes-agent.Dockerfile
COPY dockerfiles/hermes-kaniko-heredoc-runner.py /tmp/hermes-kaniko-heredoc-runner.py
# Keep dashboard chat sockets tied to the intended React mount and conversation.
# A resumed conversation needs a different PTY attachment key from a fresh chat;
# reconnects to that same conversation must keep using the same key.
@ -252,6 +256,7 @@ for (const [before, after, label] of [
}
fs.writeFileSync(sidebarPath, sidebar);
NODE
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 1 ;; *) exit 2 ;; esac
# The upstream OIDC gate authenticates users but deliberately treats the
# dashboard as one shared workstation. Allow a deployment to narrow that
@ -324,6 +329,7 @@ for before, after, label in (
source = source.replace(before, after, 1)
path.write_text(source)
PY
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 2 ;; *) exit 2 ;; esac
# Give trusted plugins a pre-turn routing hook. It runs after fallback runtime
# restoration but before Hermes builds its provider-specific system prompt.
@ -493,6 +499,7 @@ if oneshot.count(oneshot_before) != 1:
)
oneshot_path.write_text(oneshot.replace(oneshot_before, oneshot_after, 1))
PY
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 3 ;; *) exit 2 ;; esac
# Hermes WebUI sends its model/provider/reasoning selection on /v1/runs.
# Upstream currently applies only statically declared model_routes there, so
@ -628,6 +635,7 @@ runs_source = runs_source.replace(runs_agent_before, runs_agent_after, 1)
source = source[:runs_start] + runs_source
path.write_text(source)
PY
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 4 ;; *) exit 2 ;; esac
# ``create_task(initial_status="blocked")`` is an explicit operator park, but
# upstream only treats later block_task() events as sticky. Recognize the
@ -779,6 +787,7 @@ if cli_source.count(help_before) != 1:
)
cli_path.write_text(cli_source.replace(help_before, help_after, 1))
PY
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 5 ;; *) exit 2 ;; esac
COPY dockerfiles/hermes-kanban-blocked-regression.py /tmp/hermes-kanban-blocked-regression.py
RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py \
@ -816,6 +825,7 @@ if source.count(before) != 1:
)
path.write_text(source.replace(before, after, 1))
PY
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 6 ;; *) exit 2 ;; esac
# Keep the dashboard's account cards aligned with the managed provider lanes.
# Switchyard receives Claude Code subscription OAuth through the environment;
@ -882,6 +892,7 @@ for before, after, label in replacements:
path.write_text(source)
PY
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 7 ;; *) exit 2 ;; esac
# Make parent-linked API workers first-class live dashboard sessions. They stay
# active until the API runner closes them, even during a long final model call;
@ -975,6 +986,7 @@ source = source.replace(row_before, row_after)
path.write_text(source)
PY
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 8 ;; *) exit 2 ;; esac
# Keep the browser terminal reliable across GPU context loss, make durable
# worker lineage visible, expose an accessible API-worker transcript, and
@ -1543,9 +1555,42 @@ function RootRedirect() {
fs.writeFileSync(path, source);
}
NODE
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 9 ;; *) exit 2 ;; esac
COPY dockerfiles/patch-hermes-execution-safety.py /tmp/patch-hermes-execution-safety.py
COPY dockerfiles/hermes-execution-safety-regression.py /tmp/hermes-execution-safety-regression.py
COPY dockerfiles/hermes_execution_patch_support.py /tmp/hermes_execution_patch_support.py
COPY dockerfiles/patch_hermes_run_safety.py /tmp/patch_hermes_run_safety.py
COPY dockerfiles/patch_hermes_decomposition_safety.py /tmp/patch_hermes_decomposition_safety.py
COPY dockerfiles/hermes_execution_regression_support.py /tmp/hermes_execution_regression_support.py
COPY dockerfiles/hermes_run_safety_regression.py /tmp/hermes_run_safety_regression.py
COPY dockerfiles/hermes_decomposition_safety_regression.py /tmp/hermes_decomposition_safety_regression.py
COPY dockerfiles/hermes_lane_compatibility_regression.py /tmp/hermes_lane_compatibility_regression.py
COPY services/hermes/scripts/cli_lane_*.py /tmp/hermes-lane-regression/
RUN HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
HERMES_COMPATIBILITY_MODE=legacy \
/opt/hermes/.venv/bin/python /tmp/hermes_lane_compatibility_regression.py \
&& /opt/hermes/.venv/bin/python /tmp/patch-hermes-execution-safety.py \
&& HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
HERMES_COMPATIBILITY_MODE=patched \
/opt/hermes/.venv/bin/python /tmp/hermes_lane_compatibility_regression.py \
&& HERMES_CLI_LANE_SOURCE=/tmp/hermes-lane-regression \
/opt/hermes/.venv/bin/python /tmp/hermes-execution-safety-regression.py \
&& rm /tmp/patch-hermes-execution-safety.py \
/tmp/hermes-execution-safety-regression.py \
/tmp/hermes_execution_patch_support.py \
/tmp/patch_hermes_run_safety.py \
/tmp/patch_hermes_decomposition_safety.py \
/tmp/hermes_execution_regression_support.py \
/tmp/hermes_run_safety_regression.py \
/tmp/hermes_decomposition_safety_regression.py \
/tmp/hermes_lane_compatibility_regression.py \
&& find /tmp/hermes-lane-regression -depth -delete
COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate
RUN rm -f /tmp/hermes-agent.Dockerfile /tmp/hermes-kaniko-heredoc-runner.py
RUN cd /opt/hermes/web \
&& npm run build \
&& grep -Fq 'await api.getSessions(1, 0' src/pages/ChatPage.tsx \
@ -1602,6 +1647,8 @@ RUN cd /opt/hermes/web \
/opt/hermes/agent/turn_context.py \
/opt/hermes/agent/conversation_loop.py \
/opt/hermes/hermes_cli/oneshot.py \
/opt/hermes/hermes_cli/kanban_decompose.py \
/opt/hermes/gateway/kanban_watchers.py \
/opt/hermes/tools/delegate_tool.py \
/opt/hermes/tools/web_tools.py \
/opt/hermes/tools/python_sandbox_tool.py \

View File

@ -5,3 +5,18 @@
!dockerfiles/hermes-public-extract/**
!dockerfiles/hermes-session-activity-panel.tsx
!dockerfiles/hermes-session-migrate.py
!dockerfiles/Dockerfile.hermes-agent
!dockerfiles/hermes-kaniko-heredoc-runner.py
!dockerfiles/patch-hermes-execution-safety.py
!dockerfiles/hermes-execution-safety-regression.py
!dockerfiles/hermes_execution_patch_support.py
!dockerfiles/patch_hermes_run_safety.py
!dockerfiles/patch_hermes_decomposition_safety.py
!dockerfiles/hermes_execution_regression_support.py
!dockerfiles/hermes_run_safety_regression.py
!dockerfiles/hermes_decomposition_safety_regression.py
!dockerfiles/hermes_lane_compatibility_regression.py
!services/
!services/hermes/
!services/hermes/scripts/
!services/hermes/scripts/cli_lane_*.py

View File

@ -0,0 +1,23 @@
"""Run the composable Hermes execution-safety build regressions."""
import unittest
from hermes_decomposition_safety_regression import DecompositionSafetyTests
from hermes_run_safety_regression import ExactRunSafetyTests
def main() -> int:
"""Run both transaction domains and return a build-friendly status."""
loader = unittest.defaultTestLoader
suite = unittest.TestSuite(
(
loader.loadTestsFromTestCase(ExactRunSafetyTests),
loader.loadTestsFromTestCase(DecompositionSafetyTests),
)
)
result = unittest.TextTestRunner(verbosity=2).run(suite)
return 0 if result.wasSuccessful() else 1
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,146 @@
#!/usr/bin/env python3
"""Replay the reviewed Hermes Dockerfile heredocs for pinned Kaniko v1.23.2.
Kaniko v1.23.2 parses shell heredoc bodies into ``RunCommand.Files`` but its RUN
implementation executes only ``CmdLine``. The interpreters therefore receive
empty stdin and exit successfully. Docker and BuildKit execute these blocks
normally, so this compatibility runner is enabled only by the Kaniko pipeline.
"""
from __future__ import annotations
import argparse
import re
import subprocess
from pathlib import Path
MAX_DOCKERFILE_BYTES = 2_000_000
BLOCKS = (
("RUN node <<'NODE'", "NODE", ("node",)),
("RUN python - <<'PY'", "PY", ("python", "-")),
)
EXPECTED_COMMANDS = ("node", *("python",) * 7, "node")
PARSER_DIRECTIVE = re.compile(
r"^\s*#\s*([a-z]+)\s*=\s*(.*?)\s*$", re.IGNORECASE
)
def _escape_character(lines: list[str]) -> str:
"""Return the one Dockerfile parser escape directive, or its default."""
escape = "\\"
directives: set[str] = set()
for line in lines:
if not line.strip():
break
match = PARSER_DIRECTIVE.fullmatch(line)
if not match:
break
name, value = match.group(1).lower(), match.group(2)
if name in directives:
raise ValueError(f"multiple Dockerfile {name} directives")
directives.add(name)
if name == "escape":
if value not in {"\\", "`"}:
raise ValueError("unsupported Dockerfile escape directive")
escape = value
return escape
def _continued(line: str, escape: str) -> bool:
"""Match Docker's unescaped continuation marker at physical line end."""
stripped = line.rstrip()
count = len(stripped) - len(stripped.rstrip(escape))
return count % 2 == 1
def _logical_instruction(
lines: list[str], index: int, escape: str
) -> tuple[str, int]:
"""Normalize escape-newline pairs, including split opcodes/operators."""
logical = lines[index]
while _continued(logical.split("\n")[-1], escape):
logical = logical.rstrip()
logical = logical[:-1]
index += 1
while index < len(lines) and lines[index].lstrip().startswith("#"):
index += 1
if index >= len(lines) or not lines[index].strip():
raise ValueError("unterminated Dockerfile line continuation")
logical += lines[index].lstrip()
return logical, index
def extract_blocks(source: str) -> list[tuple[tuple[str, ...], str]]:
"""Inventory every RUN heredoc, then return only the reviewed contract."""
markers = {start: (end, command) for start, end, command in BLOCKS}
lines = source.splitlines()
escape = _escape_character(lines)
blocks: list[tuple[tuple[str, ...], str]] = []
index = 0
while index < len(lines):
marker = markers.get(lines[index])
if marker is None:
if not lines[index].strip() or lines[index].lstrip().startswith("#"):
index += 1
continue
logical, index = _logical_instruction(lines, index, escape)
stripped = logical.lstrip()
if (
stripped[:3].upper() == "RUN"
and (len(stripped) == 3 or stripped[3].isspace())
and "<<" in stripped
):
raise ValueError(
"unsupported RUN heredoc outside the exact reviewed contract"
)
index += 1
continue
end, command = marker
body_start = index + 1
index = body_start
while index < len(lines) and lines[index] != end:
index += 1
if index == len(lines):
raise ValueError(f"unterminated {command[0]} heredoc")
body = "\n".join(lines[body_start:index]) + "\n"
if not body.strip():
raise ValueError(f"empty {command[0]} heredoc")
blocks.append((command, body))
index += 1
commands = tuple(command[0] for command, _body in blocks)
if commands != EXPECTED_COMMANDS:
raise ValueError(
"Hermes Dockerfile heredoc contract changed: "
f"expected {EXPECTED_COMMANDS!r}, received {commands!r}"
)
return blocks
def replay(dockerfile: Path, block_index: int) -> None:
"""Execute one exact heredoc after inventorying the complete Dockerfile."""
size = dockerfile.stat().st_size
if size < 1 or size > MAX_DOCKERFILE_BYTES:
raise ValueError("Hermes Dockerfile size is outside the reviewed boundary")
source = dockerfile.read_text(encoding="utf-8")
blocks = extract_blocks(source)
if block_index < 1 or block_index > len(blocks):
raise ValueError("heredoc block index is outside the reviewed contract")
command, body = blocks[block_index - 1]
print(f"replaying reviewed Dockerfile heredoc {block_index}/{len(blocks)}")
subprocess.run(command, input=body, text=True, check=True)
def main() -> int:
"""Parse the one explicit Dockerfile path and replay its reviewed patches."""
parser = argparse.ArgumentParser()
parser.add_argument("--dockerfile", required=True, type=Path)
parser.add_argument("--block-index", required=True, type=int)
args = parser.parse_args()
replay(args.dockerfile, args.block_index)
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,120 @@
"""Regress automatic decomposition against existing execution history."""
from __future__ import annotations
from hermes_cli import kanban_db
from hermes_cli import kanban_decompose
from hermes_execution_regression_support import ExecutionSafetyTestCase
class DecompositionSafetyTests(ExecutionSafetyTestCase):
"""Verify only unexecuted triage roots may decompose automatically."""
def test_automatic_decomposer_skips_executed_triage_task(self) -> None:
task_id = self._route_executed_task_to_triage()
completions = self._client(
{"fanout": False, "title": "must not run", "body": "must not run"}
)
before = len(kanban_db.list_tasks(self.connection))
outcome = kanban_decompose.decompose_task(task_id, automatic=True)
self.assertFalse(outcome.ok)
self.assertIn("execution history", outcome.reason)
self.assertEqual(completions.calls, 0)
self.assertEqual(self._status(task_id), "triage")
self.assertEqual(len(kanban_db.list_tasks(self.connection)), before)
def test_manual_decomposition_remains_available_after_execution(self) -> None:
task_id = self._route_executed_task_to_triage()
completions = self._client(
{
"fanout": False,
"title": "operator-approved retry",
"body": "Retain one bounded task.",
"assignee": "default",
}
)
outcome = kanban_decompose.decompose_task(task_id, author="operator")
self.assertTrue(outcome.ok)
self.assertFalse(outcome.fanout)
self.assertEqual(completions.calls, 1)
self.assertEqual(self._status(task_id), "ready")
def test_fresh_triage_task_still_auto_promotes(self) -> None:
task_id = kanban_db.create_task(
self.connection,
title="fresh objective",
triage=True,
)
completions = self._client(
{
"fanout": False,
"title": "specified objective",
"body": "One bounded task.",
"assignee": "default",
}
)
outcome = kanban_decompose.decompose_task(task_id, automatic=True)
self.assertTrue(outcome.ok)
self.assertEqual(completions.calls, 1)
self.assertEqual(self._status(task_id), "ready")
def test_execution_history_gained_during_single_llm_call_blocks_commit(self) -> None:
task_id = kanban_db.create_task(
self.connection,
title="concurrent single objective",
triage=True,
)
completions = self._client(
{
"fanout": False,
"title": "redundant rewrite",
"body": "must not be committed",
"assignee": "default",
},
before_response=lambda: self._insert_run(task_id),
)
outcome = kanban_decompose.decompose_task(task_id, automatic=True)
self.assertFalse(outcome.ok)
self.assertIn("gained execution history", outcome.reason)
self.assertEqual(completions.calls, 1)
self.assertEqual(self._status(task_id), "triage")
def test_execution_history_gained_during_fanout_llm_call_blocks_commit(self) -> None:
task_id = kanban_db.create_task(
self.connection,
title="concurrent triage objective",
triage=True,
)
completions = self._client(
{
"fanout": True,
"tasks": [
{
"title": "redundant child",
"body": "must not be created",
"assignee": "default",
"parents": [],
}
],
},
before_response=lambda: self._insert_run(task_id),
)
before = len(kanban_db.list_tasks(self.connection))
outcome = kanban_decompose.decompose_task(task_id, automatic=True)
self.assertFalse(outcome.ok)
self.assertIn("gained execution history", outcome.reason)
self.assertEqual(completions.calls, 1)
self.assertEqual(self._status(task_id), "triage")
self.assertEqual(len(kanban_db.list_tasks(self.connection)), before)

View File

@ -0,0 +1,12 @@
"""Fail-closed helpers shared by Hermes source patches."""
def replace_once(source: str, before: str, after: str, label: str) -> str:
"""Replace one anchored upstream fragment and fail closed on source drift."""
count = source.count(before)
if count != 1:
raise SystemExit(
f"Hermes {label} patch context changed: expected 1, found {count}"
)
return source.replace(before, after, 1)

View File

@ -0,0 +1,132 @@
"""Shared isolated database fixture for Hermes execution-safety regressions."""
from __future__ import annotations
import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest import mock
_HERMES_HOME = tempfile.TemporaryDirectory(prefix="hermes-execution-test-home-")
os.environ["HERMES_HOME"] = _HERMES_HOME.name
_LANE_SOURCE = Path(
os.environ.get("HERMES_CLI_LANE_SOURCE", "/tmp/hermes-lane-regression")
)
sys.path.insert(0, str(_LANE_SOURCE))
from agent import auxiliary_client # noqa: E402
from cli_lane_config import TerminalIdentity # noqa: E402
from cli_lane_finalization import _finalize_document_db # noqa: E402
from hermes_cli import kanban_db # noqa: E402
from hermes_cli import kanban_decompose # noqa: E402
__all__ = ("ExecutionSafetyTestCase", "TerminalIdentity", "_finalize_document_db")
class _FakeCompletions:
def __init__(self, payload: dict, before_response=None) -> None:
self.payload = payload
self.before_response = before_response
self.calls = 0
def create(self, **_kwargs):
self.calls += 1
if self.before_response is not None:
self.before_response()
return SimpleNamespace(
choices=[
SimpleNamespace(
message=SimpleNamespace(content=json.dumps(self.payload))
)
]
)
class _FakeClient:
def __init__(self, completions: _FakeCompletions) -> None:
self.chat = SimpleNamespace(completions=completions)
class ExecutionSafetyTestCase(unittest.TestCase):
"""Exercise patched upstream APIs against an isolated real database."""
def setUp(self) -> None:
self.connection = kanban_db.connect()
self.patches = [
mock.patch.object(
kanban_decompose,
"_build_roster",
return_value=([], {"default"}),
),
mock.patch.object(
kanban_decompose,
"_load_config",
return_value={"kanban": {"auto_promote_children": True}},
),
mock.patch.object(
auxiliary_client,
"get_auxiliary_extra_body",
return_value={},
),
]
for patch in self.patches:
patch.start()
def tearDown(self) -> None:
for patch in reversed(self.patches):
patch.stop()
self.connection.close()
def _client(self, payload: dict, before_response=None) -> _FakeCompletions:
completions = _FakeCompletions(payload, before_response)
client = _FakeClient(completions)
patch = mock.patch.object(
auxiliary_client,
"get_text_auxiliary_client",
return_value=(client, "test-decomposer"),
)
patch.start()
self.addCleanup(patch.stop)
return completions
def _status(self, task_id: str) -> str:
task = kanban_db.get_task(self.connection, task_id)
self.assertIsNotNone(task)
return task.status
def _route_executed_task_to_triage(self) -> str:
task_id = kanban_db.create_task(
self.connection,
title="already implemented parent",
)
for attempt in range(kanban_db.BLOCK_RECURRENCE_LIMIT):
if attempt:
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
self.assertTrue(
kanban_db.block_task(
self.connection,
task_id,
reason="same unavailable capability",
kind="capability",
)
)
self.assertEqual(self._status(task_id), "triage")
self.assertEqual(
len(kanban_db.list_runs(self.connection, task_id)),
kanban_db.BLOCK_RECURRENCE_LIMIT,
)
return task_id
def _insert_run(self, task_id: str) -> None:
with kanban_db.connect_closing() as connection:
kanban_db._synthesize_ended_run(
connection,
task_id,
outcome="reclaimed",
summary="concurrent execution evidence",
)

View File

@ -0,0 +1,120 @@
"""Exercise mounted CLI scripts against the exact image Kanban API."""
from __future__ import annotations
import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from unittest import mock
_HOME = tempfile.TemporaryDirectory(prefix="hermes-lane-compatibility-")
os.environ["HERMES_HOME"] = _HOME.name
os.environ["HERMES_CLI_LANE_CONCURRENCY"] = "1"
lane_source = Path(os.environ["HERMES_CLI_LANE_SOURCE"])
sys.path.insert(0, str(lane_source))
from cli_lane_capabilities import ( # noqa: E402
detect_kanban_capabilities,
initialize_kanban_capabilities,
runtime_health,
)
from cli_lane_config import TerminalIdentity # noqa: E402
from cli_lane_dispatch import recover_orphans # noqa: E402
from cli_lane_finalization import _finalize_document_db # noqa: E402
from hermes_cli import kanban_db # noqa: E402
class MountedLaneCompatibilityTests(unittest.TestCase):
"""Verify safe behavior before and after the image API patch."""
def setUp(self) -> None:
self.connection = kanban_db.connect()
def tearDown(self) -> None:
self.connection.close()
@staticmethod
def _document(summary: str) -> dict:
return {
"result": json.dumps(
{
"status": "completed",
"summary": summary,
"changed_files": [],
"tests_run": ["mixed source compatibility"],
"artifacts": [],
"findings": [],
"blockers": [],
},
sort_keys=True,
),
"summary": summary,
"metadata": {"matrix": os.environ["HERMES_COMPATIBILITY_MODE"]},
}
def test_exact_image_api_matrix(self) -> None:
mode = os.environ["HERMES_COMPATIBILITY_MODE"]
capabilities = initialize_kanban_capabilities(kanban_db)
self.assertTrue(capabilities.exact_run_completion)
self.assertEqual(capabilities.ready, mode == "patched")
self.assertEqual(runtime_health()["state"], "ready" if mode == "patched" else "deferred")
self.assertLess(Path(_HOME.name, "cli-lanes/runtime-health.json").stat().st_size, 2048)
def test_current_active_run_completion_stays_safe(self) -> None:
task_id = kanban_db.create_task(self.connection, title="active accepted result")
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
run_id = kanban_db.get_task(self.connection, task_id).current_run_id
outcome = _finalize_document_db(
kanban_db,
TerminalIdentity("default", task_id, run_id, "pending"),
self._document("active result"),
)
self.assertEqual(outcome, "committed")
self.assertEqual(kanban_db.get_task(self.connection, task_id).status, "done")
def test_ended_run_replay_is_enabled_only_after_patch(self) -> None:
mode = os.environ["HERMES_COMPATIBILITY_MODE"]
task_id = kanban_db.create_task(self.connection, title="ended accepted result")
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
run_id = kanban_db.get_task(self.connection, task_id).current_run_id
self.assertTrue(
kanban_db.block_task(
self.connection,
task_id,
reason="simulate post-journal error",
expected_run_id=run_id,
)
)
outcome = _finalize_document_db(
kanban_db,
TerminalIdentity("default", task_id, run_id, "pending"),
self._document("ended result"),
)
self.assertEqual(outcome, "committed" if mode == "patched" else "deferred")
def test_old_reclaim_api_is_never_called_unguarded(self) -> None:
capabilities = detect_kanban_capabilities(kanban_db)
if capabilities.exact_run_reclaim:
self.skipTest("patched API exercises guarded reclaim in the exact-run suite")
with mock.patch.object(
kanban_db,
"reclaim_task",
side_effect=AssertionError("legacy unguarded reclaim called"),
), mock.patch("cli_lane_dispatch.recover_pending_finalizations", return_value=0):
recover_orphans()
def main() -> int:
"""Run the compatibility matrix for the selected exact image source."""
result = unittest.TextTestRunner(verbosity=2).run(
unittest.defaultTestLoader.loadTestsFromTestCase(MountedLaneCompatibilityTests)
)
return 0 if result.wasSuccessful() else 1
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,467 @@
"""Regress exact-run replay, provenance, and reclaim safety."""
from __future__ import annotations
import threading
from unittest import mock
from hermes_cli import kanban_db
from hermes_execution_regression_support import (
ExecutionSafetyTestCase,
TerminalIdentity,
_finalize_document_db,
)
from cli_lane_files import state_path
from cli_lane_records import _write_terminal_record
from cli_lane_recovery import recover_pending_finalizations
class ExactRunSafetyTests(ExecutionSafetyTestCase):
"""Verify exact-run transactions reject every stale concurrent actor."""
def _lane_document(self, summary: str) -> dict:
"""Return the DB-bearing portion of one accepted lane journal."""
return {
"result": summary,
"summary": summary,
"metadata": {"source": "exact-run-regression"},
}
def test_lane_marks_replaced_ended_run_stale_without_mutation(self) -> None:
task_id = kanban_db.create_task(self.connection, title="replaced journal")
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
first_run = kanban_db.get_task(self.connection, task_id).current_run_id
self.assertTrue(
kanban_db.block_task(
self.connection,
task_id,
reason="first run ended after persistence",
expected_run_id=first_run,
)
)
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
second_run = kanban_db.get_task(self.connection, task_id).current_run_id
self.assertTrue(
kanban_db.block_task(
self.connection,
task_id,
reason="replacement run also ended",
expected_run_id=second_run,
)
)
before = kanban_db.get_task(self.connection, task_id)
outcome = _finalize_document_db(
kanban_db,
TerminalIdentity("default", task_id, first_run, "pending"),
self._lane_document("stale first result"),
)
after = kanban_db.get_task(self.connection, task_id)
self.assertEqual(outcome, "stale")
self.assertEqual(after.status, before.status)
self.assertEqual(after.current_run_id, before.current_run_id)
self.assertEqual(after.result, before.result)
self.assertEqual(after.completed_run_id, before.completed_run_id)
def test_lane_marks_nonexistent_ended_run_stale_without_mutation(self) -> None:
task_id = kanban_db.create_task(self.connection, title="no such run")
before = kanban_db.get_task(self.connection, task_id)
outcome = _finalize_document_db(
kanban_db,
TerminalIdentity("default", task_id, 42, "pending"),
self._lane_document("orphan result"),
)
after = kanban_db.get_task(self.connection, task_id)
self.assertEqual(outcome, "stale")
self.assertEqual(after.status, before.status)
self.assertEqual(after.result, before.result)
self.assertEqual(kanban_db.list_runs(self.connection, task_id), [])
def test_full_lane_recovery_conflicts_a_replaced_ended_run(self) -> None:
task_id = kanban_db.create_task(self.connection, title="recovery conflict")
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
first_run = kanban_db.get_task(self.connection, task_id).current_run_id
self.assertTrue(
kanban_db.block_task(
self.connection,
task_id,
reason="first accepted run ended",
expected_run_id=first_run,
)
)
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
second_run = kanban_db.get_task(self.connection, task_id).current_run_id
self.assertTrue(
kanban_db.block_task(
self.connection,
task_id,
reason="replacement ended",
expected_run_id=second_run,
)
)
before = kanban_db.get_task(self.connection, task_id)
state_file = state_path("default", task_id)
pending, _record = _write_terminal_record(
state_file,
board="default",
task_id=task_id,
run_id=first_run,
structured={
"status": "completed",
"summary": "accepted stale result",
"changed_files": [],
"tests_run": ["real SQLite recovery regression"],
"artifacts": [],
"findings": [],
"blockers": [],
},
summary="accepted stale result",
metadata={"source": "exact-run-regression"},
)
self.assertEqual(recover_pending_finalizations(), 0)
after = kanban_db.get_task(self.connection, task_id)
self.assertEqual(after.status, before.status)
self.assertEqual(after.current_run_id, before.current_run_id)
self.assertEqual(after.result, before.result)
self.assertFalse(pending.exists())
conflicts = list(
state_file.parent.glob(
f"{state_file.stem}.run-{first_run}.terminal.conflict-*.json"
)
)
self.assertEqual(len(conflicts), 1)
def test_latest_ended_run_can_replay_a_durable_completion(self) -> None:
task_id = kanban_db.create_task(self.connection, title="journaled result")
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
run_id = kanban_db.get_task(self.connection, task_id).current_run_id
self.assertTrue(
kanban_db.block_task(
self.connection,
task_id,
reason="legacy post-journal failure",
kind="capability",
expected_run_id=run_id,
)
)
with mock.patch.object(
kanban_db, "_fire_kanban_lifecycle_hook"
) as lifecycle:
self.assertTrue(
kanban_db.complete_task(
self.connection,
task_id,
result="durable terminal result",
summary="durable terminal result",
replay_ended_run_id=run_id,
)
)
lifecycle.assert_called_once()
self.assertEqual(lifecycle.call_args.args[:2], ("kanban_task_completed", task_id))
self.assertEqual(lifecycle.call_args.kwargs["run_id"], run_id)
task = kanban_db.get_task(self.connection, task_id)
self.assertEqual(task.status, "done")
self.assertEqual(task.completed_run_id, run_id)
runs = kanban_db.list_runs(self.connection, task_id)
self.assertEqual(len(runs), 1)
self.assertEqual(runs[0].id, run_id)
self.assertEqual(runs[0].status, "done")
self.assertEqual(runs[0].outcome, "completed")
self.assertEqual(runs[0].summary, "durable terminal result")
completed_events = self.connection.execute(
"SELECT run_id FROM task_events "
"WHERE task_id = ? AND kind = 'completed' ORDER BY id",
(task_id,),
).fetchall()
self.assertEqual([row["run_id"] for row in completed_events], [run_id])
def test_unchanged_triage_can_replay_its_exact_latest_ended_run(self) -> None:
task_id = self._route_executed_task_to_triage()
runs = kanban_db.list_runs(self.connection, task_id)
run_id = runs[-1].id
self.assertTrue(
kanban_db.complete_task(
self.connection,
task_id,
result="accepted before the loop breaker routed triage",
summary="accepted before the loop breaker routed triage",
replay_ended_run_id=run_id,
)
)
task = kanban_db.get_task(self.connection, task_id)
self.assertEqual(task.status, "done")
self.assertEqual(task.completed_run_id, run_id)
updated_runs = kanban_db.list_runs(self.connection, task_id)
self.assertEqual(len(updated_runs), kanban_db.BLOCK_RECURRENCE_LIMIT)
self.assertEqual(updated_runs[-1].id, run_id)
self.assertEqual(updated_runs[-1].outcome, "completed")
def test_triage_status_transition_prevents_stale_exact_run_replay(self) -> None:
task_id = self._route_executed_task_to_triage()
run_id = kanban_db.list_runs(self.connection, task_id)[-1].id
self.assertTrue(
kanban_db.specify_triage_task(
self.connection,
task_id,
title="operator changed the objective",
author="operator",
)
)
self.assertFalse(
kanban_db.complete_task(
self.connection,
task_id,
result="stale accepted result",
replay_ended_run_id=run_id,
)
)
self.assertNotEqual(self._status(task_id), "done")
def test_decomposition_lineage_prevents_replay_even_if_status_returns(self) -> None:
task_id = self._route_executed_task_to_triage()
run_id = kanban_db.list_runs(self.connection, task_id)[-1].id
children = kanban_db.decompose_triage_task(
self.connection,
task_id,
root_assignee="default",
children=[{"title": "new child objective"}],
author="operator",
auto_promote=False,
)
self.assertEqual(len(children or []), 1)
with kanban_db.write_txn(self.connection):
self.connection.execute(
"UPDATE tasks SET status = 'ready' WHERE id = ?",
(task_id,),
)
self.assertFalse(
kanban_db.complete_task(
self.connection,
task_id,
result="pre-decomposition accepted result",
replay_ended_run_id=run_id,
)
)
self.assertEqual(self._status(task_id), "ready")
def test_older_ended_run_cannot_complete_over_a_replacement(self) -> None:
task_id = kanban_db.create_task(self.connection, title="replacement guard")
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
old_run = kanban_db.get_task(self.connection, task_id).current_run_id
self.assertTrue(
kanban_db.block_task(
self.connection,
task_id,
reason="first run",
kind="capability",
expected_run_id=old_run,
)
)
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
replacement_run = kanban_db.get_task(self.connection, task_id).current_run_id
self.assertNotEqual(old_run, replacement_run)
self.assertTrue(
kanban_db.block_task(
self.connection,
task_id,
reason="replacement run",
kind="capability",
expected_run_id=replacement_run,
)
)
self.assertFalse(
kanban_db.complete_task(
self.connection,
task_id,
result="stale result",
replay_ended_run_id=old_run,
)
)
self.assertIn(self._status(task_id), {"blocked", "triage"})
def test_equal_result_bytes_remain_bound_to_the_completing_run(self) -> None:
task_id = kanban_db.create_task(self.connection, title="same result retries")
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
first_run = kanban_db.get_task(self.connection, task_id).current_run_id
self.assertTrue(
kanban_db.block_task(
self.connection,
task_id,
reason="first attempt ended",
expected_run_id=first_run,
)
)
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
second_run = kanban_db.get_task(self.connection, task_id).current_run_id
self.assertTrue(
kanban_db.complete_task(
self.connection,
task_id,
result="identical result bytes",
summary="second run wins",
expected_run_id=second_run,
)
)
task = kanban_db.get_task(self.connection, task_id)
self.assertEqual(task.completed_run_id, second_run)
self.assertNotEqual(task.completed_run_id, first_run)
runs = {run.id: run for run in kanban_db.list_runs(self.connection, task_id)}
self.assertEqual(runs[first_run].outcome, "blocked")
self.assertEqual(runs[second_run].outcome, "completed")
def test_exact_run_reclaim_succeeds_for_the_authoritative_run(self) -> None:
task_id = kanban_db.create_task(self.connection, title="recover exact run")
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
run_id = kanban_db.get_task(self.connection, task_id).current_run_id
self.assertTrue(
kanban_db.reclaim_task(
self.connection,
task_id,
reason="invalid exact journal",
expected_run_id=run_id,
)
)
task = kanban_db.get_task(self.connection, task_id)
self.assertEqual(task.status, "ready")
self.assertIsNone(task.current_run_id)
def test_stale_reclaim_before_transaction_preserves_replacement_run(self) -> None:
task_id = kanban_db.create_task(self.connection, title="replacement before txn")
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
old_run = kanban_db.get_task(self.connection, task_id).current_run_id
self.assertTrue(
kanban_db.block_task(
self.connection,
task_id,
reason="old recovery run",
expected_run_id=old_run,
)
)
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
replacement = kanban_db.get_task(self.connection, task_id)
replacement_run = replacement.current_run_id
self.assertNotEqual(old_run, replacement_run)
signals = []
self.assertFalse(
kanban_db.reclaim_task(
self.connection,
task_id,
reason="stale journal",
expected_run_id=old_run,
signal_fn=lambda *args: signals.append(args),
)
)
latest = kanban_db.get_task(self.connection, task_id)
self.assertEqual(latest.status, "running")
self.assertEqual(latest.current_run_id, replacement_run)
self.assertEqual(latest.claim_lock, replacement.claim_lock)
self.assertEqual(signals, [])
def test_reclaim_update_guard_preserves_run_changed_inside_transaction(self) -> None:
task_id = kanban_db.create_task(self.connection, title="replacement in txn")
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
old_run = kanban_db.get_task(self.connection, task_id).current_run_id
replacement = {}
def install_replacement(*_args, **_kwargs):
cursor = self.connection.execute(
"INSERT INTO task_runs (task_id, status, claim_lock, started_at) "
"VALUES (?, 'running', ?, strftime('%s','now'))",
(task_id, "replacement-lock"),
)
replacement["run_id"] = int(cursor.lastrowid)
self.connection.execute(
"UPDATE tasks SET current_run_id = ?, claim_lock = ? WHERE id = ?",
(replacement["run_id"], "replacement-lock", task_id),
)
return {}
with mock.patch.object(
kanban_db,
"_terminate_reclaimed_worker",
side_effect=install_replacement,
):
self.assertFalse(
kanban_db.reclaim_task(
self.connection,
task_id,
reason="journal for old run",
expected_run_id=old_run,
)
)
latest = kanban_db.get_task(self.connection, task_id)
self.assertEqual(latest.status, "running")
self.assertEqual(latest.current_run_id, replacement["run_id"])
self.assertEqual(latest.claim_lock, "replacement-lock")
def test_concurrent_exact_reclaim_and_completion_have_one_winner(self) -> None:
task_id = kanban_db.create_task(self.connection, title="concurrent finalizer")
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
run_id = kanban_db.get_task(self.connection, task_id).current_run_id
barrier = threading.Barrier(2)
outcomes = {}
errors = []
def reclaim() -> None:
try:
with kanban_db.connect_closing() as connection:
barrier.wait()
outcomes["reclaim"] = kanban_db.reclaim_task(
connection,
task_id,
reason="concurrent invalid journal",
expected_run_id=run_id,
)
except BaseException as error: # pragma: no cover - assertion relay
errors.append(error)
def finalize() -> None:
try:
with kanban_db.connect_closing() as connection:
barrier.wait()
outcomes["complete"] = kanban_db.complete_task(
connection,
task_id,
result="durable winner",
summary="durable winner",
expected_run_id=run_id,
)
except BaseException as error: # pragma: no cover - assertion relay
errors.append(error)
threads = [threading.Thread(target=reclaim), threading.Thread(target=finalize)]
for thread in threads:
thread.start()
for thread in threads:
thread.join(timeout=10)
self.assertFalse(thread.is_alive())
self.assertEqual(errors, [])
self.assertEqual(set(outcomes), {"reclaim", "complete"})
self.assertEqual(sum(bool(value) for value in outcomes.values()), 1)
latest = kanban_db.get_task(self.connection, task_id)
self.assertIn(latest.status, {"done", "ready"})
self.assertIsNone(latest.current_run_id)
runs = kanban_db.list_runs(self.connection, task_id)
self.assertEqual(len(runs), 1)
self.assertIsNotNone(runs[0].ended_at)

View File

@ -0,0 +1,6 @@
"""Apply the composable Hermes execution-safety source patches."""
# Import order is intentional: the decomposition DB contract builds on the
# exact-run schema and transaction changes. Each module fails closed on drift.
import patch_hermes_run_safety # noqa: F401
import patch_hermes_decomposition_safety # noqa: F401

View File

@ -0,0 +1,262 @@
"""Patch transactional automatic-decomposition execution-history guards."""
import os
from pathlib import Path
from hermes_execution_patch_support import replace_once
source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes"))
db_path = source_root / "hermes_cli/kanban_db.py"
db = db_path.read_text(encoding="utf-8")
specify_signature_before = '''def specify_triage_task(
conn: sqlite3.Connection,
task_id: str,
*,
title: Optional[str] = None,
body: Optional[str] = None,
assignee: Optional[str] = None,
author: Optional[str] = None,
) -> bool:
'''
specify_signature_after = '''def specify_triage_task(
conn: sqlite3.Connection,
task_id: str,
*,
title: Optional[str] = None,
body: Optional[str] = None,
assignee: Optional[str] = None,
author: Optional[str] = None,
require_no_runs: bool = False,
) -> bool:
'''
db = replace_once(
db,
specify_signature_before,
specify_signature_after,
"transactional specify signature",
)
specify_guard_before = ''' if existing is None:
return False
sets: list[str] = ["status = 'todo'"]
'''
specify_guard_after = ''' if existing is None:
return False
if require_no_runs and conn.execute(
"SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1",
(task_id,),
).fetchone() is not None:
return False
sets: list[str] = ["status = 'todo'"]
'''
db = replace_once(
db,
specify_guard_before,
specify_guard_after,
"transactional specify execution-history guard",
)
decompose_signature_before = '''def decompose_triage_task(
conn: sqlite3.Connection,
task_id: str,
*,
root_assignee: Optional[str],
children: list[dict],
author: Optional[str] = None,
auto_promote: bool = True,
) -> Optional[list[str]]:
'''
decompose_signature_after = '''def decompose_triage_task(
conn: sqlite3.Connection,
task_id: str,
*,
root_assignee: Optional[str],
children: list[dict],
author: Optional[str] = None,
auto_promote: bool = True,
require_no_runs: bool = False,
) -> Optional[list[str]]:
'''
db = replace_once(
db,
decompose_signature_before,
decompose_signature_after,
"transactional decomposition signature",
)
decompose_guard_before = ''' if root_row["status"] != "triage":
return None
tenant = root_row["tenant"]
'''
decompose_guard_after = ''' if root_row["status"] != "triage":
return None
if require_no_runs and conn.execute(
"SELECT 1 FROM task_runs WHERE task_id = ? LIMIT 1",
(task_id,),
).fetchone() is not None:
return None
tenant = root_row["tenant"]
'''
db = replace_once(
db,
decompose_guard_before,
decompose_guard_after,
"transactional decomposition execution-history guard",
)
db_path.write_text(db, encoding="utf-8")
decompose_path = source_root / "hermes_cli/kanban_decompose.py"
decompose = decompose_path.read_text(encoding="utf-8")
helper_anchor = ''' if chosen not in valid_names:
return default_assignee
return chosen
'''
helper_replacement = helper_anchor + '''def _has_execution_history(task_id: str) -> bool:
"""Return whether a task has ever entered a worker run."""
with kb.connect_closing() as conn:
return bool(kb.list_runs(conn, task_id))
'''
decompose = replace_once(
decompose,
helper_anchor,
helper_replacement,
"auto-decompose execution-history helper",
)
signature_before = '''def decompose_task(
task_id: str,
*,
author: Optional[str] = None,
timeout: Optional[int] = None,
) -> DecomposeOutcome:
'''
signature_after = '''def decompose_task(
task_id: str,
*,
author: Optional[str] = None,
timeout: Optional[int] = None,
automatic: bool = False,
) -> DecomposeOutcome:
'''
decompose = replace_once(
decompose,
signature_before,
signature_after,
"auto-decompose function signature",
)
status_before = ''' if task.status != "triage":
return DecomposeOutcome(
task_id, False, f"task is not in triage (status={task.status!r})"
)
cfg = _load_config()
'''
status_after = ''' if task.status != "triage":
return DecomposeOutcome(
task_id, False, f"task is not in triage (status={task.status!r})"
)
if automatic and _has_execution_history(task_id):
return DecomposeOutcome(
task_id,
False,
"task has execution history and requires deliberate manual triage",
)
cfg = _load_config()
'''
decompose = replace_once(
decompose,
status_before,
status_after,
"auto-decompose preflight guard",
)
single_before = ''' author=audit_author,
)
if not ok:
'''
single_after = ''' author=audit_author,
require_no_runs=automatic,
)
if not ok:
if automatic and _has_execution_history(task_id):
return DecomposeOutcome(
task_id,
False,
"task gained execution history and requires deliberate manual triage",
)
'''
decompose = replace_once(
decompose,
single_before,
single_after,
"auto-decompose single-task commit guard",
)
fanout_before = ''' author=audit_author,
auto_promote=auto_promote,
)
'''
fanout_after = ''' author=audit_author,
auto_promote=auto_promote,
require_no_runs=automatic,
)
'''
decompose = replace_once(
decompose,
fanout_before,
fanout_after,
"auto-decompose transactional fanout guard",
)
fanout_outcome_before = ''' if child_ids is None:
return DecomposeOutcome(
task_id, False, "task moved out of triage before decomposition",
)
'''
fanout_outcome_after = ''' if child_ids is None:
if automatic and _has_execution_history(task_id):
return DecomposeOutcome(
task_id,
False,
"task gained execution history and requires deliberate manual triage",
)
return DecomposeOutcome(
task_id, False, "task moved out of triage before decomposition",
)
'''
decompose = replace_once(
decompose,
fanout_outcome_before,
fanout_outcome_after,
"auto-decompose fanout rejection reason",
)
decompose_path.write_text(decompose, encoding="utf-8")
watcher_path = source_root / "gateway/kanban_watchers.py"
watcher = watcher_path.read_text(encoding="utf-8")
call_before = ''' outcome = _decomp.decompose_task(
tid, author="auto-decomposer",
)
'''
call_after = ''' outcome = _decomp.decompose_task(
tid,
author="auto-decomposer",
automatic=True,
)
'''
watcher = replace_once(
watcher,
call_before,
call_after,
"gateway automatic-decomposition call",
)
watcher_path.write_text(watcher, encoding="utf-8")

View File

@ -0,0 +1,480 @@
"""Patch exact-run completion, provenance, and reclaim transactions."""
import os
from pathlib import Path
from hermes_execution_patch_support import replace_once
source_root = Path(os.environ.get("HERMES_SOURCE_ROOT", "/opt/hermes"))
db_path = source_root / "hermes_cli/kanban_db.py"
db = db_path.read_text(encoding="utf-8")
complete_signature_before = '''def complete_task(
conn: sqlite3.Connection,
task_id: str,
*,
result: Optional[str] = None,
summary: Optional[str] = None,
metadata: Optional[dict] = None,
created_cards: Optional[Iterable[str]] = None,
expected_run_id: Optional[int] = None,
) -> bool:
'''
complete_signature_after = '''def complete_task(
conn: sqlite3.Connection,
task_id: str,
*,
result: Optional[str] = None,
summary: Optional[str] = None,
metadata: Optional[dict] = None,
created_cards: Optional[Iterable[str]] = None,
expected_run_id: Optional[int] = None,
replay_ended_run_id: Optional[int] = None,
) -> bool:
'''
db = replace_once(
db,
complete_signature_before,
complete_signature_after,
"exact ended-run completion signature",
)
complete_branch_before = ''' with write_txn(conn):
if expected_run_id is None:
cur = conn.execute(
"""
UPDATE tasks
SET status = 'done',
result = ?,
completed_at = ?,
claim_lock = NULL,
claim_expires= NULL,
worker_pid = NULL,
block_kind = NULL,
block_recurrences = 0
WHERE id = ?
AND status IN ('running', 'ready', 'blocked', 'scheduled')
""",
(result, now, task_id),
)
else:
'''
complete_branch_after = ''' # Journal recovery may arrive after a legacy error path ended the
# accepted run. Verify that it is still the latest run in this same
# transaction; a replacement attempt must make the replay fail closed.
with write_txn(conn):
replayed_run_id = None
if expected_run_id is not None and replay_ended_run_id is not None:
raise ValueError("expected_run_id and replay_ended_run_id are mutually exclusive")
if replay_ended_run_id is not None:
replayed_run_id = int(replay_ended_run_id)
# Replay is permitted only while the exact run-ending transition
# remains the latest status mutation. Comments and diagnostics do
# not invalidate durable accepted work, but an unblock, explicit
# triage specification, decomposition, reclaim, or newer attempt
# does. BEGIN IMMEDIATE keeps this lineage check and the guarded
# task/run updates in one transaction.
last_transition = conn.execute(
"""
SELECT kind, run_id
FROM task_events
WHERE task_id = ?
AND kind IN (
'archived', 'block_loop_detected', 'blocked', 'claimed',
'completed', 'decomposed', 'dependency_wait', 'gave_up',
'promoted', 'promoted_manual', 'reclaimed', 'scheduled',
'specified', 'stale', 'timed_out', 'unblocked'
)
ORDER BY id DESC
LIMIT 1
""",
(task_id,),
).fetchone()
if (
last_transition is None
or last_transition["run_id"] != replayed_run_id
or last_transition["kind"] not in {
"block_loop_detected",
"blocked",
"reclaimed",
"stale",
"timed_out",
"gave_up",
}
):
return False
cur = conn.execute(
"""
UPDATE tasks
SET status = 'done',
result = ?,
completed_at = ?,
claim_lock = NULL,
claim_expires= NULL,
worker_pid = NULL,
block_kind = NULL,
block_recurrences = 0
WHERE id = ?
AND status IN ('ready', 'blocked', 'triage')
AND current_run_id IS NULL
AND ? = (
SELECT id FROM task_runs
WHERE task_id = ?
ORDER BY id DESC
LIMIT 1
)
AND EXISTS (
SELECT 1 FROM task_runs
WHERE id = ?
AND task_id = ?
AND ended_at IS NOT NULL
)
""",
(
result,
now,
task_id,
replayed_run_id,
task_id,
replayed_run_id,
task_id,
),
)
elif expected_run_id is None:
cur = conn.execute(
"""
UPDATE tasks
SET status = 'done',
result = ?,
completed_at = ?,
claim_lock = NULL,
claim_expires= NULL,
worker_pid = NULL,
block_kind = NULL,
block_recurrences = 0
WHERE id = ?
AND status IN ('running', 'ready', 'blocked', 'scheduled')
""",
(result, now, task_id),
)
else:
'''
db = replace_once(
db,
complete_branch_before,
complete_branch_after,
"transactional exact ended-run completion",
)
run_completion_before = ''' run_id = _end_run(
conn, task_id,
outcome="completed", status="done",
summary=summary if summary is not None else result,
metadata=metadata,
)
# If complete_task was called on a never-claimed task (ready or
# blocked → done with no run in flight), synthesize a
# zero-duration run so the handoff fields are persisted in
# attempt history instead of silently lost.
if run_id is None and (summary or metadata or result):
run_id = _synthesize_ended_run(
conn, task_id,
outcome="completed",
summary=summary if summary is not None else result,
metadata=metadata,
)
'''
run_completion_after = ''' if replayed_run_id is None:
run_id = _end_run(
conn, task_id,
outcome="completed", status="done",
summary=summary if summary is not None else result,
metadata=metadata,
)
# If complete_task was called on a never-claimed task (ready or
# blocked → done with no run in flight), synthesize a
# zero-duration run so the handoff fields are persisted in
# attempt history instead of silently lost.
if run_id is None and (summary or metadata or result):
run_id = _synthesize_ended_run(
conn, task_id,
outcome="completed",
summary=summary if summary is not None else result,
metadata=metadata,
)
else:
run_id = replayed_run_id
replayed = conn.execute(
"""
UPDATE task_runs
SET status = 'done',
outcome = 'completed',
summary = ?,
error = NULL,
metadata = ?,
claim_lock = NULL,
claim_expires = NULL,
worker_pid = NULL
WHERE id = ?
AND task_id = ?
AND ended_at IS NOT NULL
""",
(
summary if summary is not None else result,
json.dumps(metadata, ensure_ascii=False) if metadata else None,
run_id,
task_id,
),
)
if replayed.rowcount != 1:
raise RuntimeError("exact ended-run completion lost its run identity")
conn.execute(
"UPDATE tasks SET completed_run_id = ? WHERE id = ?",
(run_id, task_id),
)
'''
db = replace_once(
db,
run_completion_before,
run_completion_after,
"exact completion run provenance",
)
reclaim_before = '''def reclaim_task(
conn: sqlite3.Connection,
task_id: str,
*,
reason: Optional[str] = None,
signal_fn=None,
) -> bool:
"""Operator-driven reclaim: release the claim and reset to ``ready``.
Unlike :func:`release_stale_claims` which only acts on tasks whose
``claim_expires`` has passed, this function reclaims immediately
regardless of TTL. Intended for the dashboard/CLI recovery flow
when an operator wants to abort a running worker without waiting
for the TTL to expire (e.g. after seeing a hallucination warning).
Returns True if a reclaim happened, False if the task isn't in a
reclaimable state (not running, or doesn't exist).
"""
row = conn.execute(
"SELECT status, claim_lock, worker_pid FROM tasks WHERE id = ?",
(task_id,),
).fetchone()
if not row:
return False
if row["status"] != "running" and row["claim_lock"] is None:
# Nothing to reclaim — already ready / blocked / done.
return False
prev_lock = row["claim_lock"]
termination = _terminate_reclaimed_worker(
row["worker_pid"], prev_lock, signal_fn=signal_fn,
)
with write_txn(conn):
cur = conn.execute(
"UPDATE tasks SET status = 'ready', claim_lock = NULL, "
"claim_expires = NULL, worker_pid = NULL "
"WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
"AND claim_lock IS ?",
(task_id, prev_lock),
)
if cur.rowcount != 1:
return False
run_id = _end_run(
conn, task_id,
outcome="reclaimed", status="reclaimed",
error=(
f"manual_reclaim: {reason}" if reason
else f"manual_reclaim lock={prev_lock}"
),
metadata=termination,
)
payload = {
"manual": True,
"reason": reason,
"prev_lock": prev_lock,
}
payload.update(termination)
_append_event(
conn, task_id, "reclaimed",
payload,
run_id=run_id,
)
# Operator intervention — they've looked at the task, so the
# consecutive-failures counter is now stale. Give the next retry
# a fresh budget. (_clear_failure_counter opens its own write_txn,
# so it runs after the enclosing one commits.)
_clear_failure_counter(conn, task_id)
return True
'''
reclaim_after = '''def reclaim_task(
conn: sqlite3.Connection,
task_id: str,
*,
reason: Optional[str] = None,
signal_fn=None,
expected_run_id: Optional[int] = None,
) -> bool:
"""Operator-driven reclaim: release the claim and reset to ``ready``.
Unlike :func:`release_stale_claims` which only acts on tasks whose
``claim_expires`` has passed, this function reclaims immediately
regardless of TTL. Intended for the dashboard/CLI recovery flow
when an operator wants to abort a running worker without waiting
for the TTL to expire (e.g. after seeing a hallucination warning).
When ``expected_run_id`` is supplied, the run identity is checked after
``BEGIN IMMEDIATE`` and included in the guarded update. A replacement run
is therefore neither signalled nor reclaimed by stale recovery evidence.
Returns True if a reclaim happened, False if the task isn't in a
reclaimable state (not running, or doesn't exist), or its run changed.
"""
with write_txn(conn):
row = conn.execute(
"SELECT status, claim_lock, worker_pid, current_run_id "
"FROM tasks WHERE id = ?",
(task_id,),
).fetchone()
if not row:
return False
if expected_run_id is not None and row["current_run_id"] != expected_run_id:
return False
if row["status"] != "running" and row["claim_lock"] is None:
# Nothing to reclaim — already ready / blocked / done.
return False
prev_lock = row["claim_lock"]
termination = _terminate_reclaimed_worker(
row["worker_pid"], prev_lock, signal_fn=signal_fn,
)
if expected_run_id is None:
cur = conn.execute(
"UPDATE tasks SET status = 'ready', claim_lock = NULL, "
"claim_expires = NULL, worker_pid = NULL "
"WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
"AND claim_lock IS ?",
(task_id, prev_lock),
)
else:
cur = conn.execute(
"UPDATE tasks SET status = 'ready', claim_lock = NULL, "
"claim_expires = NULL, worker_pid = NULL "
"WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
"AND claim_lock IS ? AND current_run_id = ?",
(task_id, prev_lock, int(expected_run_id)),
)
if cur.rowcount != 1:
return False
run_id = _end_run(
conn, task_id,
outcome="reclaimed", status="reclaimed",
error=(
f"manual_reclaim: {reason}" if reason
else f"manual_reclaim lock={prev_lock}"
),
metadata=termination,
)
payload = {
"manual": True,
"reason": reason,
"prev_lock": prev_lock,
}
payload.update(termination)
_append_event(
conn, task_id, "reclaimed",
payload,
run_id=run_id,
)
# Operator intervention — they've looked at the task, so the
# consecutive-failures counter is now stale. Give the next retry
# a fresh budget. (_clear_failure_counter opens its own write_txn,
# so it runs after the enclosing one commits.)
_clear_failure_counter(conn, task_id)
return True
'''
db = replace_once(
db,
reclaim_before,
reclaim_after,
"transactional exact-run reclaim",
)
task_field_before = ''' current_run_id: Optional[int] = None
workflow_template_id: Optional[str] = None
'''
task_field_after = ''' current_run_id: Optional[int] = None
# Durable provenance for the run that authoritatively completed this task.
completed_run_id: Optional[int] = None
workflow_template_id: Optional[str] = None
'''
db = replace_once(
db,
task_field_before,
task_field_after,
"completed-run task field",
)
task_row_before = ''' current_run_id=(
row["current_run_id"] if "current_run_id" in keys else None
),
workflow_template_id=(
'''
task_row_after = ''' current_run_id=(
row["current_run_id"] if "current_run_id" in keys else None
),
completed_run_id=(
row["completed_run_id"] if "completed_run_id" in keys else None
),
workflow_template_id=(
'''
db = replace_once(
db,
task_row_before,
task_row_after,
"completed-run row mapping",
)
task_schema_before = ''' -- run is in-flight). Denormalised for cheap reads.
current_run_id INTEGER,
-- Forward-compat for v2 workflow routing. In v1 the kernel writes
'''
task_schema_after = ''' -- run is in-flight). Denormalised for cheap reads.
current_run_id INTEGER,
-- Immutable provenance for the task result's authoritative run.
completed_run_id INTEGER,
-- Forward-compat for v2 workflow routing. In v1 the kernel writes
'''
db = replace_once(
db,
task_schema_before,
task_schema_after,
"completed-run schema",
)
task_migration_before = ''' if "current_run_id" not in cols:
_add_column_if_missing(
conn, "tasks", "current_run_id", "current_run_id INTEGER"
)
if "workflow_template_id" not in cols:
'''
task_migration_after = ''' if "current_run_id" not in cols:
_add_column_if_missing(
conn, "tasks", "current_run_id", "current_run_id INTEGER"
)
if "completed_run_id" not in cols:
_add_column_if_missing(
conn, "tasks", "completed_run_id", "completed_run_id INTEGER"
)
if "workflow_template_id" not in cols:
'''
db = replace_once(
db,
task_migration_before,
task_migration_after,
"completed-run migration",
)
db_path.write_text(db, encoding="utf-8")

View File

@ -0,0 +1,341 @@
# Hermes full-handoff acceptance and release runbook
`scripts/ops/hermes_handoff_acceptance.py` is a fail-closed release gate. It
contains 63 checks in 15 groups. An empty catalog, an unknown result status, a
mandatory `FAIL` or `NOT_RUN`, a harness error, or incomplete evidence makes the
decision `NO_GO`. The four ephemeral checks are non-mandatory `NOT_RUN` in the
default mode and are replaced with mandatory results only in armed mode.
This runbook describes evidence collection. It does not authorize a merge,
publish, deployment, Flux reconcile, credential read, or live cluster mutation.
Human review of PR #19 and of the resulting report is required.
## Required release inputs
Every value that can go stale must be supplied explicitly. Resolve `main`, PR
heads, the image digest, build source SHA, deployment revision, and chat config
revision from their authoritative read-only sources immediately before the run.
Do not copy the snapshot in the dependency section into a later run.
```sh
scripts/ops/hermes_handoff_acceptance.py \
--context atlas-operator \
--remote-main-sha "$REMOTE_MAIN_SHA" \
--reviewed-head-sha "$REVIEWED_PR19_HEAD" \
--agent-image "$AGENT_GIT_SHA_BUILD_TAG_AT_SHA256" \
--build-sha "$IMAGE_BUILD_SOURCE_SHA" \
--deployment-revision "$RUNNING_DEPLOYMENT_REVISION" \
--chat-config-revision "$RUNNING_CHAT_CONFIG_REVISION" \
--dependency-head "12=$PR12_HEAD" \
--dependency-head "14=$PR14_HEAD" \
--dependency-head "15=$PR15_HEAD" \
--dependency-head "16=$PR16_HEAD" \
--dependency-head "17=$PR17_HEAD" \
--dependency-head "18=$PR18_HEAD" \
--dependency-head "20=$PR20_HEAD" \
--output build/hermes-handoff-acceptance.json
```
The fixed topology is three pool workers on three distinct nodes. `--node-count`
and `--pool-replicas` cannot lower that requirement. Pool node/ordinal assignment,
Telegram session continuity, and the chat revision are mandatory. Deliberate Flux
suspensions must be named with repeatable `--expected-suspension` arguments.
Exit status is `0` only for `GO`, `1` for `NO_GO`, and `2` for invalid execution
bounds. JSON is written to `--output` or stdout; the human summary goes to stderr
unless `--json-only` is passed.
## Structural safety boundary
The catalog is validated before a `Runner` exists. Invalid inputs, an empty or
unsafe catalog, and an invalid armed preflight therefore create no subprocess.
The command policy is checked both before and after executable resolution.
Default mode permits a narrow read grammar:
- Kubernetes reads use `name` or reviewed JSONPath projections. Secret resources,
raw API reads, full JSON/YAML, `describe`, `cluster-info dump`, credential-bearing
projections, `auth reconcile`, config operations, all mutations, and every
dry-run mutation trick are rejected before spawn.
- Git is read-only. `fetch`, configuration/helper overrides, alternate worktrees,
external diff helpers, and push are rejected.
- Gitea paths are pinned to `atlas/titan-iac` on an exact segment boundary, so a
look-alike repository such as `atlas/titan-iac-evil` is refused, and any
relative segment — including a percent-encoded `%2e%2e` — is rejected before
the request is built.
- Shell commands must be rendered from frozen templates, use fixed executable
paths, and cannot name service-account, Vault, runtime-access, SSH, or other
credential roots.
- Impersonation is structurally impossible. `--as`, `--as-group`, and `--as-uid`
are refused for every argv, in every mode, from every vantage, and the inner
command of a `kubectl exec` is re-checked under the same rule rather than being
granted an exemption. `validate_catalog` refuses an impersonating step from any
vantage, not only the operator. A future audited self-probe that genuinely needs
impersonation is a reviewed code change, not a configuration flag.
- The binary allowlist is exactly the set with pinned release digests:
`kubectl`, `git`, `sh`, `hermes`, and the Gitea client. `flux` and `helm` are
not on it; Flux and Helm status is read through `kubectl` instead, so no
allowlisted binary can be admitted that the executor would refuse to attest.
- `PATH` is replaced with a fixed path. Every outer executable is resolved to an
allowed real path, checked for unsafe write mode, and compared with a pinned
SHA-256 digest. A caller-authored `kubectl`, `git`, or helper cannot fabricate
evidence.
The executor uses a new process group, drains stdout and stderr concurrently,
and retains at most the configured bytes while the child is running. A monotonic
absolute timeout terminates and reaps the group, including descendants that keep
pipes open. Timeout, deadline, and output-byte values reject booleans,
zero/negative values, non-finite numbers, and values outside fixed bounds. There
is no concurrency knob: `run_catalog` is strictly sequential and says so rather
than accepting a bound it would ignore.
Truncation discards the affected stream and makes every evaluator `NOT_RUN`.
Bulk-evidence steps have their bytes replaced before they are recorded, but they
keep the resolved executable path and its attested SHA-256, so withholding output
never withholds the provenance of the binary that produced it.
Credential screening runs before truncation. Command, output, error, identity,
reason, evidence, JSON, and human-summary fields are screened recursively,
including mapping keys and short/numeric sensitive values. The complete rendered
report is screened again; a remaining credential shape is replaced by a bounded
`NO_GO` stub rather than being written. No credential prefix is retained at a
truncation boundary.
## Vantages and evidence
The operator must use an external read-only kubeconfig. The self probe runs in
the actual Hermes agent pod. Both identities are obtained with
`kubectl auth whoami`; equal or missing principals make the run `NO_GO`. Required
Switchyard and chat/Telegram pod vantages must also resolve. Node coverage is
derived from all daemonset pod node names, not one selected pod.
Dangerous Kubernetes permissions are checked only with side-effect-free
`kubectl auth can-i` reviews. The default catalog never constructs a Secret,
TokenRequest, mutation, attach, port-forward, or arbitrary exec request. The one
forge administrative denial check performs a read-only GET and accepts only an
explicit authorization refusal; generic `404`, `not found`, and `no route` text
are not denial evidence.
An absence is only evidence when the probe observed something. Both name rules
resolve their step through one guard, so a step that exits `0` with no lines — or
with only separators — is `NOT_RUN`, never `PASS`. This is reachable in practice
two ways, and both are pinned by regressions: a POSIX pipeline reports the status
of its *last* stage, so a missing upstream binary in the frozen `env_names`
template exits `0` with no bytes; and `kubectl -o jsonpath` exits `0` when the
structure it walks has drifted, whether a parent field, a leaf field, or a
container filter. Because all five absence checks are mandatory, silence costs
the release its `GO`. The pool claim projection emits one `<volume>=<claim>` line
per template volume so a volume without a PVC still counts as an observation.
The strict evaluators reject duplicate-key JSON, raw control characters,
`NaN`/infinity, boolean/number equality coercion, future timestamps, stale
provider evidence, and every malformed routing or Flux row. Routing freshness is
required separately for each provider, lane, and effort. Flux objects must be
non-suspended, current at `observedGeneration`, and explicitly `Ready=True`;
unknown or reconciling state is not healthy.
`release.exact-lineage-is-running` atomically binds:
- fixed remote `main` SHA and local `origin/main`;
- open draft PR #19, base `main`, existing feature ref, and exact reviewed head;
- the exact `:git-<source-SHA>-build-<n>@sha256:<digest>` image identity,
with the source SHA equal to the release `main` SHA;
- current Deployment generation and revision;
- the one active ReplicaSet revision/hash; every Ready pod image and imageID;
- the Flux-applied `main` revision.
The pool evaluator requires exactly ordinals 0, 1, and 2, all Ready, on three
distinct nodes and claims, with service-account token automount disabled. The
chat checks always require durable Telegram sessions and the exact running chat
configuration revision.
## Armed ephemeral mode
Armed mode is the only write path and runs only after the complete default
catalog reports `GO`.
```sh
scripts/ops/hermes_handoff_acceptance.py \
<all required release inputs above> \
--arm-ephemeral-push \
--confirm "ARM EPHEMERAL HERMES HANDOFF PUSH" \
--ephemeral-token acceptance-20260817a \
--output build/hermes-handoff-acceptance-armed.json
```
Before any runner, network request, or cluster vantage is created, preflight
binds the working directory to the existing linked PR #19 worktree, the exact
feature branch and reviewed local HEAD, the fixed HTTPS Atlas origin, base
`main`, confirmation phrase, and one unique ephemeral ref. The command policy is
then pinned to that exact ref. It allows only its push, one exact draft-create
payload, cleanup of the exactly discovered PR number, and deletion of that ref.
The run reserves at least 300 seconds (up to 6000) for cleanup. It verifies the
pushed SHA, then identifies the pull request from two independent sources: the
create response itself, and the pull index read page by page to its last page
(50 per page, 20 pages maximum) rather than a single truncated first page. Either
source alone names a pull request this run definitely made, because preflight
proved the ephemeral ref did not exist. Every number found from either source is
registered and closed, so a failed or unusable index can no longer leave an
orphan open. A `GO` still requires exactly one open draft with the expected base,
head ref, base SHA, and head SHA, and the two sources must agree.
Cleanup requires successful PR close, successful branch deletion, a successful
empty `ls-remote`, and a strict re-read of the same closed PR. Malformed or
partial cleanup is `FAIL`; it is never inferred from an error. When anything is
uncertain the `draft-pull-request` evidence carries `residue_ref` and an exact
`manual_cleanup` command list — the `PATCH .../pulls/<n> --field state=closed`
calls for every number found, the `DELETE .../branches/<ref>` call, and, when no
number could be determined at all, the exact ref to sweep the index for by hand.
Force, force-with-lease, fan-out, protected refs, other repositories, other PR
numbers, arbitrary PATCH payloads, and repository administration remain denied.
## Dependency, merge, and conflict graph
Authoritative snapshot observed 2026-08-17 after refreshing Gitea and Git against
remote `main` `d8f2d818b9a552ea6c2d7fe86554be829bd5ffff`:
| PR | Head SHA | Draft | Base | Role |
| --- | --- | --- | --- | --- |
| #12 | `ed43dbaa7f094e1841aa760862e80015deb9d80c` | no | `main` | AI usage exporter repair |
| #14 | `8c6e3acdacb223f68b1716f0a74bd543e18890e6` | yes | `main` | bounded forge client and least-privilege RBAC |
| #15 | `73fefbb5d92ef07371a9a997f22d848acd2d06ed` | yes | `main` | result-decomposition reliability |
| #16 | `e8c26ecf85a0c802d378fffffb98b7df43b78242` | no | `main` | unprivileged image builder |
| #17 | `48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c` | yes | `main` | three-lane placement |
| #18 | `689bcb6e48414854447988ba174143a307d308b9` | yes | `main` | distributed three-worker pool |
| #19 | `c808baff40a12a90f43c2e5e39b94d139f88ea29` | yes | `main` | head this repair replaces on the same branch |
| #20 | `a242dcc786576ae1a18000cb4941836ff610dff2` | yes | `main` | CLI process-group reaping |
The repaired #19 SHA cannot be embedded in its own commit without changing that
SHA. Resolve it from Gitea after push and require it as `--reviewed-head-sha`;
the PR handoff records the exact value. All eight heads in this table, including
the repaired #19, merge cleanly with `main` individually. Pairwise
`git merge-tree --write-tree` over all 28 combinations found conflicts for:
#12+#17, #12+#18, #12+#19, #14+#15, #14+#16, #14+#17, #14+#19, #14+#20, #15+#16,
#15+#17, #15+#19, #16+#19, #16+#20, #17+#18, and #17+#20. Gitea's
`mergeable=false` on a draft is not treated as conflict evidence.
This graph differs from the previous snapshot in exactly two ways. #12 and #18
both advanced, which adds #12+#18, #17+#18, and #12+#19. And #19 no longer
duplicates the repo-wide hygiene legacy-exception mechanism that #18 also
carries, which makes #18+#19 clean. Recompute before every integration step;
nothing here is durable.
Use this dependency order, refreshing every head and re-running merge-tree at
each step:
1. #14 first so the release never restores or prolongs cluster-admin authority.
2. #16, then #15, resolving their Dockerfile and quality-contract changes
additively. Never take one PR's quality contract wholesale.
3. #20 after #15; it depends on the CLI lifecycle work and must be rebased across
the current #14/#16 contract and helper changes.
4. #12 before #17, resolving their Deployment overlap once.
5. #17, then #18, after the authority, image, reliability, and reaping work.
6. #19 last as the integration/acceptance change, only after #12/#14-#18/#20 are
merged and their exact supplied heads are ancestors of refreshed `main`.
An exact post-repair merge-tree check reports #19 clean with current `main`, #17,
#18, and #20, and conflicting with #12, #14, #15, and #16 on these files:
| Pair | Overlapping files |
| --- | --- |
| #12+#19 | `testing/tests/test_hermes_auto_router.py` |
| #14+#19 | `testing/quality_contract.json`, `testing/quality_coverage.py`, `testing/tests/test_quality_coverage_helpers.py` |
| #15+#19 | `testing/tests/conftest.py` |
| #16+#19 | `testing/quality_contract.json` |
Every one of those overlaps is additive on the #19 side. Resolve the contract by
union — take every PR's managed modules, lint paths, tracked files, and branch
tracked files, and never take one PR's quality contract wholesale. #19 adds two
fixture-isolation fixtures to `conftest.py` and two lines to the auto-router test;
keep both sides. Preserve branch coverage, every tracked module, fixture
isolation, and all desired-state changes during conflict resolution.
Recompute this graph from the Gitea API immediately before integration; do not
infer conflict status from Gitea's draft `mergeable` field.
The #16/#19 integration must keep both the immutable digest and the source tag
(`newTag: git-<main-SHA>-build-<n>` plus the digest) in rendered image identity.
Digest-only rendering loses the attested build-source link and therefore
correctly remains `NO_GO` under the exact-lineage evaluator.
## Safe rollback guidance
Every rollback is a reviewed forward fix or selective Git revert followed by
the normal Flux workflow. Never restore `hermes-agent-cluster-admin`, even when
reverting unrelated portions of #14. If a rollback would widen RBAC, stop and
write a forward least-privilege fix instead.
Reverting #18 removes the StatefulSet desired state, but its
`volumeClaimTemplates` PVCs are retained by the Kubernetes default because no
delete retention policy is set. Do not claim Flux pruning deletes those PVCs.
Any PVC deletion is a separate, explicit, human-reviewed data-destruction task
with exact targets and recovery implications.
Image rollback means a reviewed commit restoring a previously attested digest;
never retag an immutable image. Do not use direct `kubectl` edits as a durable
rollback.
## Quality and release evidence
The canonical quality contract explicitly manages, lints, and tracks all 16
`scripts/ops/hermes_handoff_*.py` modules. It enforces at least 95% line and
branch coverage per tracked file and the 500-physical-line source cap. Contract
tests derive the module set so a future file cannot silently escape the gate.
This change is purely additive to the contract: it registers its own modules and
changes no repo-wide hygiene semantics.
The gate's `hygiene` step is red on `main` and stays red here. Exactly four
pre-existing test files exceed the 500-line cap —
`scripts/tests/test_dashboards_render_atlas.py` (516),
`testing/tests/test_hermes_chat_quality.py` (2242),
`testing/tests/test_hermes_cli_lanes.py` (1971), and
`testing/tests/test_hermes_coordinator.py` (510). All four are byte-identical at
merge-base `30259b52` and at current `origin/main`, none is touched by this
change, and the same four are the only issues a pristine `main` reports. Splitting
them, or grandfathering them, is a repo-wide contract decision that belongs to the
canonical contract change carried by PR #14/#15, not to this acceptance harness.
Every other gate step — docs, smell, unit, coverage — is green, and every file
this change adds or edits is under the cap.
```sh
python3 -m testing.quality_gate --profile local --build-dir build
python3 -m testing.quality_handoff_mutation
python3 -m ruff check --select F,B,SIM,C4,UP --ignore B017,UP015,UP035 \
scripts/ops/hermes_handoff_*.py testing/quality_coverage.py \
testing/quality_handoff_mutation.py testing/tests/test_hermes_handoff_*.py
python3 -m ruff format --check \
scripts/ops/hermes_handoff_*.py testing/quality_coverage.py \
testing/quality_handoff_mutation.py testing/tests/test_hermes_handoff_*.py
python3 -m compileall -q scripts/ops testing
kustomize build services/hermes | kubectl apply --dry-run=client --validate=false -f -
git diff --check origin/main...HEAD
```
The format scope names the files this change owns. A wider `testing/quality_*.py`
glob picks up `testing/quality_gate.py`, which is unmodified here and not
format-clean, so the documented command would fail for a reason this change did
not cause. `ruff format` is evidence, not a gate step: the canonical gate runs
`ruff check` only.
The deterministic mutation gate must kill every policy, redaction, verdict,
numeric, truncation, provenance, and cleanup mutant, including one that restores
the zero-evidence pass, one that restores the bare-prefix repository pin, one
that stops re-checking a `kubectl exec` inner command, and one that drops binary
attestation from an unrecorded step. The adversarial suite includes large
concurrent stdout/stderr, malformed JSON and bytes, zero state, partial failure,
absolute timeout, descendant-held pipes, malformed cleanup, ambiguous creates,
and no-subprocess/no-mutation spies. Render/client dry-run, exact conflict checks,
and bounded diff/full-tree secret scans must also pass.
The provider-health and Gitea-token-sensitive tests use hermetic fixtures. Their
result must be identical whether synthetic runtime files exist or are absent;
tests must never read the mounted production token.
## Release decision
The handoff remains `NO_GO` until all dependency heads are merged, the exact
lineage check passes from an external operator, both default and armed reports
are complete, the branch and PR cleanup is verified, the canonical and
adversarial gates are green, and a human has reviewed PR #19 plus the artifacts.
The harness never merges, deploys, reconciles, publishes, or approves anything.

View File

@ -0,0 +1,354 @@
#!/usr/bin/env python3
"""Fail-closed acceptance harness for the Hermes full handoff.
Read-only by default. The harness collects evidence from two genuinely separate
places — an external operator vantage and the Hermes agent probing itself from
inside its own pod — classifies every check PASS / FAIL / NOT_RUN /
NOT_APPLICABLE, and emits machine-readable JSON alongside a short human summary.
Any mandatory FAIL or NOT_RUN is a NO_GO, and so is a harness-level problem: an
unreachable vantage, a catalog whose evidence no longer exists, an expired
deadline, or two vantages that turn out to be the same principal. Nothing here
rounds an absence of evidence up to a pass.
scripts/ops/hermes_handoff_acceptance.py --help
The release runbook lists every required freshness and lineage input.
Mutation lives behind a separate arming flag, needs an exact confirmation
phrase and a unique ref, refuses protected push targets before any network
call, and verifies its own cleanup. A default run reports those checks NOT_RUN.
"""
from __future__ import annotations
import argparse
import datetime as dt
import json
import sys
from collections.abc import Sequence
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from hermes_handoff_catalog import Targets # noqa: E402
from hermes_handoff_ephemeral import ( # noqa: E402
CONFIRMATION,
ArmingError,
ArmRequest,
preflight,
run_armed,
)
from hermes_handoff_exec import ( # noqa: E402
DEFAULT_COMMAND_TIMEOUT,
DEFAULT_RUN_DEADLINE,
Runner,
)
from hermes_handoff_model import GO, Report, unscreened_fields, utc_now # noqa: E402
from hermes_handoff_policy import ( # noqa: E402
ARMED,
EXPECTED_BASE,
EXPECTED_REMOTE,
EXPECTED_REPO,
READ_ONLY,
)
from hermes_handoff_redaction import DEFAULT_MAX_BYTES # noqa: E402
from hermes_handoff_run import ( # noqa: E402
build_catalog,
build_report,
resolve_vantages,
unavailable_report,
validate_catalog,
validate_targets,
)
EXIT_GO = 0
EXIT_NO_GO = 1
EXIT_USAGE = 2
def dependency_head(value: str) -> tuple[int, str]:
"""Parse one `PR=SHA` dependency-head binding."""
try:
number, sha = value.split("=", 1)
parsed = int(number)
except (TypeError, ValueError) as exc:
raise argparse.ArgumentTypeError(
"dependency head must be PR=40-character-SHA"
) from exc
if (
parsed <= 0
or len(sha) != 40
or any(character not in "0123456789abcdef" for character in sha)
):
raise argparse.ArgumentTypeError(
"dependency head must be PR=lowercase-40-character-SHA"
)
return (parsed, sha)
def build_parser() -> argparse.ArgumentParser:
"""Return the command-line interface for the harness."""
parser = argparse.ArgumentParser(
prog="hermes_handoff_acceptance",
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument("--namespace", default="hermes", help="Hermes namespace.")
parser.add_argument(
"--context", help="Kubernetes context for the operator vantage."
)
parser.add_argument("--kubeconfig", help="Kubeconfig for the operator vantage.")
parser.add_argument(
"--output", type=Path, help="Write the JSON report here instead of stdout."
)
parser.add_argument(
"--json-only", action="store_true", help="Suppress the human summary."
)
parser.add_argument(
"--command-timeout",
type=float,
default=DEFAULT_COMMAND_TIMEOUT,
help="Per-command seconds.",
)
parser.add_argument(
"--deadline",
type=float,
default=DEFAULT_RUN_DEADLINE,
help="Whole-run seconds.",
)
parser.add_argument(
"--max-output-bytes",
type=int,
default=DEFAULT_MAX_BYTES,
help="Per-capture byte budget.",
)
parser.add_argument(
"--baseline-commit",
default=Targets.baseline_commit,
help="Commit main must descend from.",
)
parser.add_argument(
"--dependency-pr",
type=int,
action="append",
default=None,
help="Pull request that must be merged before this release is certifiable (repeatable).",
)
parser.add_argument(
"--dependency-head",
type=dependency_head,
action="append",
default=None,
metavar="PR=SHA",
help="Exact current dependency PR head (repeat for every dependency).",
)
parser.add_argument(
"--remote-main-sha", default="", help="Exact observed origin/main SHA."
)
parser.add_argument(
"--reviewed-head-sha", default="", help="Exact reviewed PR #19 head SHA."
)
parser.add_argument(
"--agent-image",
default="",
help="Exact agent :git-<source>-build-<n>@sha256 identity.",
)
parser.add_argument(
"--build-sha", default="", help="Exact source SHA used for the image build."
)
parser.add_argument(
"--deployment-revision", default="", help="Exact running Deployment revision."
)
parser.add_argument(
"--expected-suspension",
action="append",
default=None,
metavar="NAMESPACE/NAME",
help="Flux object allowed to be suspended (repeatable).",
)
parser.add_argument(
"--node-count",
type=int,
default=3,
help="Nodes the fleet must cover (fixed at 3).",
)
parser.add_argument(
"--pool-replicas", type=int, default=3, help="Expected pool worker count."
)
parser.add_argument(
"--chat-config-revision",
default="",
help="Expected chat pod-template config revision.",
)
parser.add_argument(
"--chat-ordinal",
type=int,
default=0,
help="Chat tenant ordinal that carries the Telegram identity.",
)
parser.add_argument(
"--pool-worker-env",
action="append",
default=None,
help="Environment variable the pool workers must declare (repeatable).",
)
parser.add_argument(
"--max-evidence-age",
type=int,
default=86_400,
help="Seconds before provider and routing evidence is treated as stale.",
)
armed = parser.add_argument_group("ephemeral mutation mode")
armed.add_argument(
"--arm-ephemeral-push",
action="store_true",
help="Permit one ephemeral branch push and draft pull request, then remove both.",
)
armed.add_argument(
"--confirm", default="", help=f"Exact confirmation phrase: {CONFIRMATION!r}."
)
armed.add_argument(
"--ephemeral-token", default="", help="Unique 8-64 character ref suffix."
)
return parser
def targets_from_args(arguments: argparse.Namespace, now: dt.datetime) -> Targets:
"""Return the catalog targets a parsed command line describes."""
defaults = Targets()
return Targets(
namespace=arguments.namespace,
baseline_commit=arguments.baseline_commit,
dependency_pull_requests=tuple(
arguments.dependency_pr or defaults.dependency_pull_requests
),
dependency_heads=tuple(arguments.dependency_head or ()),
remote_main_sha=arguments.remote_main_sha,
reviewed_head_sha=arguments.reviewed_head_sha,
agent_image=arguments.agent_image,
build_sha=arguments.build_sha,
deployment_revision=arguments.deployment_revision,
expected_suspensions=tuple(arguments.expected_suspension or ()),
node_count=arguments.node_count,
pool_replicas=arguments.pool_replicas,
chat_config_revision=arguments.chat_config_revision,
chat_ordinal=arguments.chat_ordinal,
pool_worker_env=tuple(arguments.pool_worker_env or defaults.pool_worker_env),
max_evidence_age_seconds=arguments.max_evidence_age,
now=now,
)
def emit(report: Report, arguments: argparse.Namespace, stream=None) -> list[str]:
"""Write the report, refusing to publish one that still looks credential-shaped."""
payload = report.as_dict()
offenders = unscreened_fields(payload)
if offenders:
payload = {
"harness": payload["harness"],
"decision": "NO_GO",
"harness_errors": [
"the rendered report failed its own credential screening at "
+ ", ".join(offenders[:8])
],
}
rendered = json.dumps(payload, indent=2, sort_keys=True, default=str)
if arguments.output:
arguments.output.parent.mkdir(parents=True, exist_ok=True)
arguments.output.write_text(rendered + "\n", encoding="utf-8")
else:
print(rendered, file=stream or sys.stdout)
if not arguments.json_only:
print(report.render_summary(), file=sys.stderr)
return offenders
def _armed_results(arguments: argparse.Namespace, runner: Runner, vantages) -> list:
request = ArmRequest(
repo=EXPECTED_REPO,
remote=EXPECTED_REMOTE,
token=arguments.ephemeral_token,
confirmation=arguments.confirm,
base=EXPECTED_BASE,
expected_head=arguments.reviewed_head_sha,
expected_base_sha=arguments.remote_main_sha,
)
preflight(request, EXPECTED_REPO, Path.cwd())
vantage = vantages["operator"]
return run_armed(runner, vantage, request)
def main(argv: Sequence[str] | None = None) -> int:
"""Run the acceptance sweep and return the process exit status."""
arguments = build_parser().parse_args(argv)
now = dt.datetime.now(dt.timezone.utc)
started_at = utc_now(lambda: now)
targets = targets_from_args(arguments, now)
mode = ARMED if arguments.arm_ephemeral_push else READ_ONLY
specs = build_catalog(targets)
startup_problems = [*validate_targets(targets), *validate_catalog(specs)]
if arguments.arm_ephemeral_push:
try:
preflight(
ArmRequest(
repo=EXPECTED_REPO,
remote=EXPECTED_REMOTE,
token=arguments.ephemeral_token,
confirmation=arguments.confirm,
base=EXPECTED_BASE,
expected_head=arguments.reviewed_head_sha,
expected_base_sha=arguments.remote_main_sha,
),
EXPECTED_REPO,
Path.cwd(),
)
except ArmingError as exc:
startup_problems.append(f"arming refused: {exc}")
if startup_problems:
report = unavailable_report(
mode, started_at, "; ".join(startup_problems), targets
)
emit(report, arguments)
return EXIT_NO_GO
try:
runner = Runner(
mode=mode,
command_timeout=arguments.command_timeout,
deadline_seconds=arguments.deadline,
max_bytes=arguments.max_output_bytes,
)
except ValueError as exc:
report = unavailable_report(
mode, started_at, f"invalid execution bound: {exc}", targets
)
emit(report, arguments)
return EXIT_USAGE
vantages, records = resolve_vantages(
runner, targets, arguments.kubeconfig, arguments.context
)
report = build_report(runner, targets, specs, vantages, records, mode, started_at)
if arguments.arm_ephemeral_push:
if report.decision != GO:
report.harness_errors.append(
"armed mutation was not started because read-only acceptance did not GO"
)
else:
armed_results = _armed_results(arguments, runner, vantages)
armed_ids = {result.spec.id for result in armed_results}
report.results = [
result for result in report.results if result.spec.id not in armed_ids
]
report.results.extend(armed_results)
report.results.sort(key=lambda result: result.spec.id)
report.finished_at = utc_now()
offenders = emit(report, arguments)
if offenders:
return EXIT_NO_GO
return EXIT_GO if report.decision == GO else EXIT_NO_GO
if __name__ == "__main__": # pragma: no cover - exercised through main()
raise SystemExit(main())

View File

@ -0,0 +1,154 @@
#!/usr/bin/env python3
"""Arming, provenance, and push-target rules for the armed ephemeral mode.
Nothing here touches the network or spawns a process. It is the half of armed
mode that must be true before a single command exists: the confirmation phrase,
the fixed repository and base, the one ephemeral ref grammar, and a local
worktree whose HEAD and ``origin/main`` are byte-exact with the reviewed release.
"""
from __future__ import annotations
import configparser
import re
from dataclasses import dataclass
from pathlib import Path
from hermes_handoff_policy import (
EXPECTED_BASE,
EXPECTED_REMOTE,
EXPECTED_REPO,
arm_ephemeral_policy,
)
CONFIRMATION = "ARM EPHEMERAL HERMES HANDOFF PUSH"
REF_PREFIX = "ephemeral/hermes-handoff-acceptance"
REF_RE = re.compile(rf"{REF_PREFIX}/[a-z0-9][a-z0-9-]{{7,63}}\Z")
TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9-]{7,63}\Z")
SHA_RE = re.compile(r"[0-9a-f]{40}\Z")
EXPECTED_HEAD_REF = "feature/hermes-full-handoff-acceptance"
EXPECTED_REMOTE_URL = "https://scm.bstein.dev/atlas/titan-iac.git"
PROTECTED_NAMES = frozenset(
{
"default",
"develop",
"head",
"main",
"master",
"prod",
"production",
"release",
"stable",
"trunk",
}
)
class ArmingError(ValueError):
"""Raised when an ephemeral run is not safe to start."""
@dataclass(frozen=True)
class ArmRequest:
"""Every fixed and caller-confirmed input to one armed run."""
repo: str
remote: str
token: str
confirmation: str
base: str = EXPECTED_BASE
expected_head: str = ""
expected_base_sha: str = ""
@property
def ref(self) -> str:
return f"{REF_PREFIX}/{self.token}"
def normalise_branch(name: str) -> str:
"""Return a branch name stripped of ref decoration, for comparison."""
return name.strip().removeprefix("refs/heads/").strip("/").lower()
def assert_push_target_allowed(ref: str) -> None:
"""Raise unless ``ref`` is the one ephemeral branch grammar."""
if normalise_branch(ref) in PROTECTED_NAMES:
raise ArmingError(f"refusing to push to protected branch {ref!r}")
if not REF_RE.fullmatch(ref):
raise ArmingError(f"push target {ref!r} is not an ephemeral acceptance ref")
def bounded_read(path: Path, limit: int = 1024 * 1024) -> str:
data = path.read_bytes()
if not data or len(data) > limit:
raise ArmingError(f"local Git metadata is empty or exceeds {limit} bytes")
return data.decode("utf-8", "strict")
def _attest_worktree(root: Path, expected_head: str, expected_base_sha: str) -> None:
"""Bind cwd, common Git metadata, branch, remote, and exact local HEAD."""
resolved = root.resolve(strict=True)
marker = resolved / ".git"
if not marker.is_file():
raise ArmingError(
"armed mode must run at the root of the existing linked worktree"
)
line = bounded_read(marker, 4096).strip()
if not line.startswith("gitdir: "):
raise ArmingError("linked-worktree Git metadata is malformed")
gitdir = Path(line.removeprefix("gitdir: ")).resolve(strict=True)
common = (gitdir / bounded_read(gitdir / "commondir", 4096).strip()).resolve(
strict=True
)
head = bounded_read(gitdir / "HEAD", 4096).strip()
expected_ref = f"refs/heads/{EXPECTED_HEAD_REF}"
if head != f"ref: {expected_ref}":
raise ArmingError("armed mode must run from the existing PR #19 feature branch")
ref_path = common / expected_ref
actual_head = bounded_read(ref_path, 4096).strip() if ref_path.is_file() else ""
if actual_head != expected_head:
raise ArmingError(
"local worktree HEAD does not match the exact reviewed PR head"
)
remote_ref = common / "refs" / "remotes" / EXPECTED_REMOTE / EXPECTED_BASE
remote_main = bounded_read(remote_ref, 4096).strip() if remote_ref.is_file() else ""
if remote_main != expected_base_sha:
raise ArmingError("local origin/main does not match the exact release base SHA")
parser = configparser.RawConfigParser(interpolation=None)
try:
parser.read_string(bounded_read(common / "config"))
remote_url = parser.get(f'remote "{EXPECTED_REMOTE}"', "url")
except (configparser.Error, KeyError) as exc:
raise ArmingError("local Git remote metadata is malformed") from exc
if remote_url != EXPECTED_REMOTE_URL:
raise ArmingError("origin is not the fixed Atlas titan-iac HTTPS repository")
def preflight(
request: ArmRequest, expected_repo: str, worktree: Path | None = None
) -> None:
"""Validate arming and local provenance before any subprocess/network call."""
if request.confirmation != CONFIRMATION:
raise ArmingError("confirmation phrase does not match")
if expected_repo != EXPECTED_REPO or request.repo != EXPECTED_REPO:
raise ArmingError(
"armed repository is not the fixed Atlas titan-iac repository"
)
if (
request.remote != EXPECTED_REMOTE
or normalise_branch(request.base) != EXPECTED_BASE
):
raise ArmingError("armed remote and pull-request base are fixed to origin/main")
if not TOKEN_RE.fullmatch(request.token):
raise ArmingError(
"ephemeral token must be 8-64 lowercase alphanumeric or dash characters"
)
if not SHA_RE.fullmatch(request.expected_head) or not SHA_RE.fullmatch(
request.expected_base_sha
):
raise ArmingError("exact reviewed head and release base SHAs are required")
assert_push_target_allowed(request.ref)
_attest_worktree(
worktree or Path.cwd(), request.expected_head, request.expected_base_sha
)
arm_ephemeral_policy(request.ref)

View File

@ -0,0 +1,202 @@
#!/usr/bin/env python3
"""Declarative acceptance catalog: baseline, identity, routing, scopes, access.
The catalog is data on purpose. A reviewer should be able to read what the
release is asserted to satisfy without reading any plumbing, and the harness
should be able to prove structural properties over every entry — both vantages
represented, no impersonation from outside a pod, and no mutation command of
any kind — before it runs a single command.
Catalog drift is a NO_GO, not a pass. If a manifest, a field name, or a workload
is renamed, the affected checks report ``NOT_RUN`` because the evidence they
name is not there, and the release stops until either the platform or the
catalog is corrected.
"""
from __future__ import annotations
from collections.abc import Sequence
from dataclasses import dataclass, field
from typing import Any
from hermes_handoff_model import ALWAYS, READ, CheckSpec, Step
from hermes_handoff_policy import projection
OPERATOR = "operator"
SELF = "self"
SWITCHYARD = "switchyard"
NODE = "node"
CHAT = "chat"
VANTAGE_NAMES = (OPERATOR, SELF, SWITCHYARD, NODE, CHAT)
# `name<TAB>generation<TAB>observedGeneration<TAB>suspend<TAB>conditions`.
FLUX_PROJECTION = projection(
(
"jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{'\\t'}"
"{.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}"
"{range .status.conditions[*]}{.type}={.status},{end}{'\\n'}{end}"
).replace("'", '"')
)
ENV_NAME_PROJECTION = projection(
(
"jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{'\\n'}{end}{end}"
).replace("'", '"')
)
INIT_NAME_PROJECTION = projection(
(
"jsonpath={range .spec.template.spec.initContainers[*]}{.name}{'\\n'}{end}"
).replace("'", '"')
)
NODE_NAME_PROJECTION = projection(
"jsonpath={range .items[*]}{.spec.nodeName}{'\\n'}{end}".replace("'", '"')
)
POOL_PROJECTION = projection(
(
"jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.spec.nodeName}{'\\t'}"
"{.status.phase}{'\\t'}{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}"
"{'\\t'}{range .spec.volumes[*]}{.persistentVolumeClaim.claimName}{','}{end}{'\\n'}{end}"
).replace("'", '"')
)
PROVIDER_API_KEY_NAMES = (
"ANTHROPIC_API_KEY",
"ANTHROPIC_AUTH_TOKEN",
"ANTHROPIC_BASE_URL",
"AZURE_OPENAI_API_KEY",
"CLAUDE_API_KEY",
"CLAUDE_CODE_OAUTH_TOKEN",
"GEMINI_API_KEY",
"GOOGLE_API_KEY",
"OPENAI_API_KEY",
"OPENAI_BASE_URL",
)
FORGE_CREDENTIAL_NAMES = (
"GITEA_PASSWORD",
"GITEA_TOKEN",
"GITEA_USERNAME",
"GIT_PASSWORD",
)
CODEX_HEALTH = "/opt/data/provider-health/codex.json"
CLAUDE_HEALTH = "/opt/data/provider-health/claude.json"
CODEX_HEALTH_FIELDS = "authenticated,transport,state,model,effort,latency_ms,checked_at"
CLAUDE_HEALTH_FIELDS = (
"authenticated,api_provider,auth_method,transport,state,subscription_type,"
"model,effort,latency_ms,checked_at"
)
ROUTING_LOG = "/var/lib/switchyard/routing.jsonl"
@dataclass(frozen=True)
class Targets:
"""Runtime names and expected values the catalog is written against.
Everything a site can legitimately rename lives here so the assertions stay
readable and a rename is a configuration change rather than a code change.
"""
namespace: str = "hermes"
agent_deployment: str = "hermes-agent"
agent_container: str = "hermes"
chat_statefulset: str = "hermes-chat-tenant"
chat_container: str = "hermes"
chat_ordinal: int = 0
chat_router_deployment: str = "hermes-chat-router"
triage_deployment: str = "hermes"
triage_container: str = "hermes"
switchyard_deployment: str = "hermes-switchyard"
switchyard_container: str = "switchyard"
node_daemonset: str = "hermes-node-ssh-access"
node_container: str = "key-reconciler"
pool_statefulset: str = "hermes-execution-worker"
pool_replicas: int = 3
pool_selector: str = "app=hermes-execution-worker"
coordinator_claims: tuple[str, ...] = (
"hermes-agent-home",
"hermes-agent-workspace",
)
node_account: str = "hermes-agent"
node_account_identity: str = "1200:1200:/home/hermes-agent:/bin/bash"
node_count: int = 3
repo: str = "atlas/titan-iac"
remote: str = "origin"
baseline_commit: str = "ab346f55509d584e457fe26cf90be3078f7a375c"
dependency_pull_requests: tuple[int, ...] = (12, 14, 15, 16, 17, 18, 20)
dependency_heads: tuple[tuple[int, str], ...] = ()
remote_main_sha: str = ""
reviewed_pr_number: int = 19
reviewed_head_ref: str = "feature/hermes-full-handoff-acceptance"
reviewed_head_sha: str = ""
agent_image: str = ""
build_sha: str = ""
deployment_revision: str = ""
expected_suspensions: tuple[str, ...] = ()
harbor_namespace: str = "harbor"
harbor_immutability_job: str = "hermes-agent-immutability"
builder_namespace: str = "jenkins"
builder_serviceaccount: str = "hermes-image-builder"
builder_pipeline: str = "ci/Jenkinsfile.hermes-agent-image"
builder_capabilities: str = (
'add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]'
)
agent_init_containers: tuple[str, ...] = ("patch-web-session-activity",)
chat_init_containers: tuple[str, ...] = ("patch-api-server-sessions",)
chat_config_revision: str = ""
pool_worker_env: tuple[str, ...] = ("HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL")
max_evidence_age_seconds: int = 86_400
routing_tail_lines: int = 600
routing_tail_bytes: int = 512 * 1024
listing_bytes: int = 512 * 1024
git_askpass: str = "/opt/coordinator/gitea_askpass.sh"
now: Any = None
extra: dict[str, Any] = field(default_factory=dict)
def step(
key: str,
vantage: str,
*argv: str,
kind: str = READ,
optional: bool = False,
record: bool = True,
max_bytes: int | None = None,
) -> Step:
"""Return a probe step addressed to one vantage."""
return Step(
key=key,
vantage=vantage,
argv=tuple(argv),
kind=kind,
optional=optional,
record=record,
max_bytes=max_bytes,
)
def check(
identifier: str,
title: str,
group: str,
rule: str,
steps: Sequence[Step],
expect: dict[str, Any] | None = None,
mandatory: bool = True,
scope: str = ALWAYS,
rationale: str = "",
) -> CheckSpec:
"""Return one catalog entry."""
return CheckSpec(
id=identifier,
title=title,
group=group,
rule=rule,
steps=tuple(steps),
expect=expect or {},
mandatory=mandatory,
scope=scope,
rationale=rationale,
)

View File

@ -0,0 +1,339 @@
#!/usr/bin/env python3
"""Acceptance checks for what Hermes may and may not do to the cluster.
Denials use Kubernetes authorization reviews only. Live denied attempts were
removed because a policy regression could turn a supposed probe into a real
credential read or mutation; the default harness never constructs such argv.
"""
from __future__ import annotations
from hermes_handoff_catalog import (
FLUX_PROJECTION,
OPERATOR,
SELF,
Targets,
check,
step,
)
from hermes_handoff_model import REVIEW, CheckSpec
def _access(targets: Targets) -> list[CheckSpec]:
namespace = targets.namespace
return [
check(
"access.no-cluster-admin-binding-for-agent",
"No ClusterRoleBinding grants the agent service account cluster-admin",
"access-denied",
"names_absent",
[
step(
"bindings",
OPERATOR,
"kubectl",
"get",
"clusterrolebindings",
"-o",
"name",
)
],
{"step": "bindings", "contains": ("hermes-agent-cluster-admin",)},
rationale="The binding inventory and in-pod authorization reviews are independent desired/enforced views.",
),
check(
"access.secrets-are-denied",
"Hermes cannot read Secrets cluster-wide",
"access-denied",
"review_denied",
[
step(
"review",
SELF,
"kubectl",
"auth",
"can-i",
"get",
"secrets",
"--all-namespaces",
kind=REVIEW,
)
],
rationale="A live Secret request is never constructed, even when an RBAC regression would allow it.",
),
check(
"access.service-account-tokens-are-denied",
"Hermes cannot mint a service-account token",
"access-denied",
"review_denied",
[
step(
"review-token",
SELF,
"kubectl",
"auth",
"can-i",
"create",
"serviceaccounts/token",
"--namespace",
namespace,
kind=REVIEW,
),
step(
"review-serviceaccount",
SELF,
"kubectl",
"auth",
"can-i",
"create",
"serviceaccounts",
"--namespace",
namespace,
kind=REVIEW,
),
],
rationale="Token creation has no credential-safe live probe, so both relevant grants must be denied by review.",
),
check(
"access.impersonation-is-denied",
"Hermes cannot impersonate another identity",
"access-denied",
"review_denied",
[
step(
"review",
SELF,
"kubectl",
"auth",
"can-i",
"impersonate",
"users",
"--all-namespaces",
kind=REVIEW,
)
],
rationale="The default harness never constructs an impersonated request.",
),
check(
"access.workload-mutation-is-denied",
"Hermes cannot mutate its own workload",
"access-denied",
"review_denied",
[
step(
"review",
SELF,
"kubectl",
"auth",
"can-i",
"patch",
"deployments",
"--namespace",
namespace,
kind=REVIEW,
)
],
rationale="Client-side and server-side dry-run mutation tricks are excluded structurally.",
),
check(
"access.pod-exec-is-denied",
"Hermes cannot exec into an arbitrary pod",
"access-denied",
"review_denied",
[
step(
"review",
SELF,
"kubectl",
"auth",
"can-i",
"create",
"pods/exec",
"--all-namespaces",
kind=REVIEW,
)
],
rationale="No cross-pod exec request is created by a read-only run.",
),
check(
"access.attach-portforward-and-kube-system-writes-are-denied",
"Hermes holds no attach, port-forward, or control-plane write authority",
"access-denied",
"review_denied",
[
step(
"review-attach",
SELF,
"kubectl",
"auth",
"can-i",
"create",
"pods/attach",
"--all-namespaces",
kind=REVIEW,
),
step(
"review-portforward",
SELF,
"kubectl",
"auth",
"can-i",
"create",
"pods/portforward",
"--all-namespaces",
kind=REVIEW,
),
step(
"review-create",
SELF,
"kubectl",
"auth",
"can-i",
"create",
"configmaps",
"--namespace",
"kube-system",
kind=REVIEW,
),
],
rationale="Every authority is reviewed without issuing the request it would authorize.",
),
check(
"access.required-reads-are-allowed",
"Hermes retains the reads its operator role depends on",
"access-allowed",
"allowed",
[
step(
"watch-review",
SELF,
"kubectl",
"auth",
"can-i",
"watch",
"pods",
"--namespace",
namespace,
kind=REVIEW,
),
step(
"pods",
SELF,
"kubectl",
"--namespace",
namespace,
"get",
"pods",
"-o",
"name",
),
step("namespaces", SELF, "kubectl", "get", "namespaces", "-o", "name"),
step(
"deployments",
SELF,
"kubectl",
"--namespace",
namespace,
"get",
"deployments",
"-o",
"name",
),
step("nodes", SELF, "kubectl", "get", "nodes", "-o", "name"),
],
),
check(
"access.flux-and-helm-status-are-allowed",
"Hermes can read Flux and Helm reconciliation status",
"access-allowed",
"allowed",
[
step(
"kustomizations",
SELF,
"kubectl",
"get",
"kustomizations.kustomize.toolkit.fluxcd.io",
"--all-namespaces",
"-o",
"name",
record=False,
),
step(
"helmreleases",
SELF,
"kubectl",
"get",
"helmreleases.helm.toolkit.fluxcd.io",
"--all-namespaces",
"-o",
"name",
record=False,
),
],
),
check(
"access.namespace-view-agrees-across-vantages",
"The operator and in-pod vantages see the same namespace inventory",
"access-allowed",
"vantages_agree",
[
step(
"operator",
OPERATOR,
"kubectl",
"get",
"namespaces",
"-o",
"name",
record=False,
),
step(
"self",
SELF,
"kubectl",
"get",
"namespaces",
"-o",
"name",
record=False,
),
],
{"steps": ("operator", "self")},
rationale="A disagreement here means one vantage is not seeing the cluster the other is certifying.",
),
]
def _gitops(targets: Targets) -> list[CheckSpec]:
return [
check(
f"gitops.{kind.split('.')[0]}-reconcile-cleanly",
f"No unexpected {label} suspension or unhealthy reconciliation",
"gitops",
"flux_health",
[
step(
"objects",
OPERATOR,
"kubectl",
"get",
kind,
"--all-namespaces",
"-o",
FLUX_PROJECTION,
record=False,
)
],
{"step": "objects", "expected_suspensions": targets.expected_suspensions},
)
for kind, label in (
("kustomizations.kustomize.toolkit.fluxcd.io", "Kustomization"),
("helmreleases.helm.toolkit.fluxcd.io", "HelmRelease"),
)
]
def access_checks(targets: Targets) -> list[CheckSpec]:
"""Return the access-denied, access-allowed, and GitOps health checks."""
return [*_access(targets), *_gitops(targets)]

View File

@ -0,0 +1,374 @@
#!/usr/bin/env python3
"""Acceptance checks for forge authority, node accounts, images, and regressions.
What the platform is allowed to do outside its own namespace: which forge
authority a worker holds, what a node account is and is not, how an image is
built and pinned, and whether the fixes the release depends on are actually
running rather than merely merged.
Where a property has no side-effect-free live probe — a merge that must never be
attempted, a branch protection the harness identity cannot read — the entry says
so in its rationale and asserts the nearest fact that *is* observable, rather
than quietly asserting nothing.
"""
from __future__ import annotations
from hermes_handoff_catalog import (
FORGE_CREDENTIAL_NAMES,
INIT_NAME_PROJECTION,
NODE_NAME_PROJECTION,
OPERATOR,
SELF,
Targets,
check,
step,
)
from hermes_handoff_model import ATTEMPT, CheckSpec
from hermes_handoff_policy import GITEA_CLIENT, projection, shell
def _forge(targets: Targets) -> list[CheckSpec]:
api = f"/api/v1/repos/{targets.repo}"
return [
check(
"forge.worker-holds-no-git-credential-variable",
"No forge credential is exported into the worker process",
"forge",
"names_absent",
[step("env", SELF, *shell("env_names"))],
{"step": "env", "names": FORGE_CREDENTIAL_NAMES},
rationale="Authentication reaches Git through an askpass helper that reads runtime state, never through the environment.",
),
check(
"forge.identity-is-not-an-administrator",
"The forge identity the worker uses holds no administrative rights",
"forge",
"json_field",
[step("user", SELF, GITEA_CLIENT, "GET", "/api/v1/user")],
{"step": "user", "fields": {"is_admin": False}},
),
check(
"forge.repository-authority-is-push-only",
"The worker can push a branch but cannot administer the repository",
"forge",
"json_field",
[step("repo", SELF, GITEA_CLIENT, "GET", api)],
{
"step": "repo",
"fields": {
"permissions.admin": False,
"permissions.push": True,
"permissions.pull": True,
},
},
rationale="Merge, approve, close, and protection changes all require authority this identity is proven not to hold.",
),
check(
"forge.administrative-route-is-refused",
"An administrative forge route is refused for this identity",
"forge",
"denied",
[
step(
"attempt",
SELF,
GITEA_CLIENT,
"GET",
f"{api}/branch_protections",
kind=ATTEMPT,
)
],
rationale="A live refusal, not a configuration reading. The same lack of authority is what blocks a merge route.",
),
]
def _nodes(targets: Targets) -> list[CheckSpec]:
return [
check(
"nodes.hardening-covers-every-node",
"The node account audit runs on every node in the cluster",
"nodes",
"distinct_count",
[
step(
"coverage",
OPERATOR,
"kubectl",
"--namespace",
targets.namespace,
"get",
"pods",
"--selector",
f"app={targets.node_daemonset}",
"-o",
NODE_NAME_PROJECTION,
)
],
{
"step": "coverage",
"minimum": targets.node_count,
"total_equals": targets.node_count,
},
rationale="A hardened fleet with one unenrolled node is an unhardened fleet.",
),
]
def _build(targets: Targets) -> list[CheckSpec]:
pipeline = f"{targets.remote}/main:{targets.builder_pipeline}"
return [
check(
"build.builder-service-account-is-tokenless",
"The image builder service account mounts no API token",
"build",
"stdout_matches",
[
step(
"serviceaccount",
OPERATOR,
"kubectl",
"--namespace",
targets.builder_namespace,
"get",
"serviceaccount",
targets.builder_serviceaccount,
"-o",
projection("jsonpath={.automountServiceAccountToken}"),
)
],
{"step": "serviceaccount", "equals": "false"},
),
check(
"build.builder-runs-unprivileged-with-exact-capabilities",
"The build pod is unprivileged and adds only the expected capabilities",
"build",
"names_present",
[step("pipeline", OPERATOR, "git", "show", pipeline, record=False)],
{
"step": "pipeline",
"names": (
targets.builder_capabilities,
"privileged: false",
"allowPrivilegeEscalation: false",
"automountServiceAccountToken: false",
'drop: ["ALL"]',
),
},
rationale="Exact lines, not a substring sweep: a widened capability set has to change one of these lines to take effect.",
),
check(
"build.harbor-immutability-rule-is-applied",
"The Harbor immutable-tag rule for the agent image has been applied",
"build",
"lines_match",
[
step(
"job",
OPERATOR,
"kubectl",
"--namespace",
targets.harbor_namespace,
"get",
"job",
targets.harbor_immutability_job,
"-o",
projection("jsonpath={.status.succeeded}"),
)
],
{"step": "job", "pattern": r"^[1-9][0-9]*$", "minimum": 1},
rationale="The job verifies the rule it creates, so a completed run is evidence the rule matched the expected contract.",
),
_release_lineage(targets),
]
def _release_lineage(targets: Targets) -> CheckSpec:
"""Bind the exact reviewed release from Git through the Ready pods."""
container = targets.agent_container
namespace = targets.namespace
deployment = f"deploy/{targets.agent_deployment}"
deployment_projection = projection(
(
"jsonpath={.metadata.generation}{'\\t'}{.status.observedGeneration}{'\\t'}"
"{.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}"
"{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}"
f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}'
).replace("'", '"')
)
pod_projection = projection(
(
"jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.status.phase}{'\\t'}"
"{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}"
f'{{.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}'
f'{{.status.containerStatuses[?(@.name=="{container}")].imageID}}{{"\\t"}}'
"{.metadata.labels.pod-template-hash}{'\\n'}{end}"
).replace("'", '"')
)
replicaset_projection = projection(
(
"jsonpath={range .items[*]}{.metadata.name}{'\\t'}"
"{.metadata.annotations.deployment\\.kubernetes\\.io/revision}{'\\t'}"
"{.status.readyReplicas}{'\\t'}{.status.availableReplicas}{'\\t'}"
f'{{.spec.template.spec.containers[?(@.name=="{container}")].image}}{{"\\t"}}'
"{.metadata.labels.pod-template-hash}{'\\n'}{end}"
).replace("'", '"')
)
flux_projection = projection(
(
"jsonpath={.metadata.name}{'\\t'}{.metadata.generation}{'\\t'}"
"{.status.observedGeneration}{'\\t'}{.spec.suspend}{'\\t'}"
"{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\t'}"
"{.status.lastAppliedRevision}"
).replace("'", '"')
)
return check(
"release.exact-lineage-is-running",
"Remote main, reviewed PR, image, build, Flux, and running revision are identical",
"release",
"release_lineage",
[
step(
"remote-main",
OPERATOR,
"git",
"ls-remote",
targets.remote,
"refs/heads/main",
),
step(
"reviewed-pr",
OPERATOR,
GITEA_CLIENT,
"GET",
f"/api/v1/repos/{targets.repo}/pulls/{targets.reviewed_pr_number}",
),
step(
"build-source",
OPERATOR,
"git",
"rev-parse",
f"{targets.remote}/main",
),
step(
"deployment",
OPERATOR,
"kubectl",
"--namespace",
namespace,
"get",
deployment,
"-o",
deployment_projection,
),
step(
"pods",
OPERATOR,
"kubectl",
"--namespace",
namespace,
"get",
"pods",
"--selector",
f"app={targets.agent_deployment}",
"-o",
pod_projection,
),
step(
"replicasets",
OPERATOR,
"kubectl",
"--namespace",
namespace,
"get",
"replicasets",
"--selector",
f"app={targets.agent_deployment}",
"-o",
replicaset_projection,
),
step(
"flux",
OPERATOR,
"kubectl",
"--namespace",
"flux-system",
"get",
"kustomization/hermes",
"-o",
flux_projection,
),
],
{
"main_sha": targets.remote_main_sha,
"head_sha": targets.reviewed_head_sha,
"head_ref": targets.reviewed_head_ref,
"pr_number": targets.reviewed_pr_number,
"image": targets.agent_image,
"build_sha": targets.build_sha,
"deployment_revision": targets.deployment_revision,
},
rationale="A PASS requires one exact release identity across every source and running object.",
)
def _reliability(targets: Targets) -> list[CheckSpec]:
return [
check(
"reliability.finalization-and-replay-patches-are-deployed",
"The agent workload runs the finalization, replay, and session patches",
"reliability",
"names_present",
[
step(
"init",
OPERATOR,
"kubectl",
"--namespace",
targets.namespace,
"get",
f"deploy/{targets.agent_deployment}",
"-o",
INIT_NAME_PROJECTION,
)
],
{"step": "init", "names": targets.agent_init_containers},
rationale="The regression suites live in the repository; this asserts the fixes they cover are actually running.",
),
check(
"reliability.regression-suites-are-present-on-main",
"The decomposition and worker-recovery regression suites are on main",
"reliability",
"names_present",
[
step(
"suites",
OPERATOR,
"git",
"ls-tree",
"--name-only",
f"{targets.remote}/main",
"testing/tests/",
)
],
{
"step": "suites",
"names": (
"testing/tests/test_hermes_cli_lanes.py",
"testing/tests/test_hermes_coordinator.py",
"testing/tests/test_hermes_worker_recovery.py",
),
},
),
]
def delivery_checks(targets: Targets) -> list[CheckSpec]:
"""Return the forge, node, build, and reliability checks."""
return [
*_forge(targets),
*_nodes(targets),
*_build(targets),
*_reliability(targets),
]

View File

@ -0,0 +1,387 @@
#!/usr/bin/env python3
"""Acceptance checks for the release baseline, provider identity, and routing.
These are the facts a handoff rests on before any authority question is asked:
that the tree under test is the merged one, that both providers authenticate
through the subscriptions they are supposed to, that Switchyard actually decided
the routes it claims to, and that chat, agent, and triage are still three
distinct scopes rather than one surface wearing three hostnames.
"""
from __future__ import annotations
from hermes_handoff_catalog import (
CLAUDE_HEALTH,
CLAUDE_HEALTH_FIELDS,
CODEX_HEALTH,
CODEX_HEALTH_FIELDS,
ENV_NAME_PROJECTION,
OPERATOR,
PROVIDER_API_KEY_NAMES,
ROUTING_LOG,
SELF,
SWITCHYARD,
Targets,
check,
step,
)
from hermes_handoff_model import CheckSpec
from hermes_handoff_policy import GITEA_CLIENT, projection, shell
def _baseline(targets: Targets) -> list[CheckSpec]:
api = f"/api/v1/repos/{targets.repo}"
dependency_heads = dict(targets.dependency_heads)
checks = [
check(
"baseline.local-origin-main-is-current",
"The local origin/main ref equals the fixed remote-main SHA",
"baseline",
"stdout_matches",
[step("main", OPERATOR, "git", "rev-parse", f"{targets.remote}/main")],
{"step": "main", "equals": targets.remote_main_sha},
rationale="The ancestry check is meaningful only after the local tracking ref is bound to the remote observation.",
),
check(
"baseline.origin-main-descends-merged-work",
"origin/main descends from the merged Hermes isolation baseline",
"baseline",
"allowed",
[
step(
"ancestor",
OPERATOR,
"git",
"merge-base",
"--is-ancestor",
targets.baseline_commit,
f"{targets.remote}/main",
)
],
rationale="Everything downstream assumes the merged worker-isolation baseline is present.",
),
check(
"baseline.pending-pull-requests-recorded",
"Exact open pull-request heads are recorded, not assumed",
"baseline",
"json_record",
[
step(
"open",
OPERATOR,
GITEA_CLIENT,
"GET",
f"{api}/pulls?state=open&limit=50",
)
],
{
"step": "open",
"record": {
"numbers": "[].number",
"head_refs": "[].head.ref",
"head_shas": "[].head.sha",
"base_shas": "[].base.sha",
"drafts": "[].draft",
"mergeable": "[].mergeable",
},
},
rationale="The runbook has to name the heads it was verified against; stale heads are how a release verifies the wrong tree.",
),
check(
"baseline.operator-checkout-clean",
"The operator checkout has no uncommitted drift",
"baseline",
"stdout_matches",
[step("status", OPERATOR, "git", "status", "--porcelain")],
{"step": "status", "equals": ""},
mandatory=False,
rationale="Advisory: a dirty checkout does not invalidate cluster evidence, but it does muddy what was compared.",
),
]
checks += [
check(
f"baseline.dependency-pr-{number}-merged",
f"Pull request #{number} is merged into main",
"baseline",
"json_field",
[step("pr", OPERATOR, GITEA_CLIENT, "GET", f"{api}/pulls/{number}")],
{
"step": "pr",
"fields": {
"merged": True,
"base.ref": "main",
"head.sha": dependency_heads.get(number, ""),
},
},
rationale="This harness certifies the post-merge platform; an unmerged dependency means it is measuring something else.",
)
for number in targets.dependency_pull_requests
]
return checks
def _identity(targets: Targets) -> list[CheckSpec]:
codex = shell("json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS)
claude = shell("json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS)
agent = f"deploy/{targets.agent_deployment}"
return [
check(
"identity.codex-is-chatgpt-subscription",
"Codex authenticates through the ChatGPT subscription, not an API key",
"identity",
"json_field",
[step("health", SELF, *codex)],
{
"step": "health",
"fields": {
"authenticated": True,
"transport": "codex-chatgpt-subscription",
"state": "available",
},
},
rationale="Only allow-listed status fields are read; the credential itself is never opened.",
),
check(
"identity.codex-evidence-is-fresh",
"Codex authentication evidence is current",
"identity",
"json_recent",
[step("health", SELF, *codex)],
{
"step": "health",
"now": targets.now,
"fields": {"checked_at": targets.max_evidence_age_seconds},
},
rationale="Stale provider evidence describes a state the release no longer has.",
),
check(
"identity.claude-is-firstparty-subscription",
"Claude authenticates as a claude.ai first-party subscription",
"identity",
"json_field",
[step("health", SELF, *claude)],
{
"step": "health",
"fields": {
"authenticated": True,
"api_provider": "firstParty",
"auth_method": "claude.ai",
"transport": "claude-code-cli-subscription",
"state": "available",
},
},
),
check(
"identity.claude-evidence-is-fresh",
"Claude authentication evidence is current",
"identity",
"json_recent",
[step("health", SELF, *claude)],
{
"step": "health",
"now": targets.now,
"fields": {"checked_at": targets.max_evidence_age_seconds},
},
),
check(
"identity.no-provider-api-key-in-pod",
"No provider API-key variable is set in the running agent process",
"identity",
"names_absent",
[step("env", SELF, *shell("env_names"))],
{
"step": "env",
"names": PROVIDER_API_KEY_NAMES,
"contains": ("API_KEY",),
},
rationale="Names only. The probe lists variable names and never their values.",
),
check(
"identity.no-provider-api-key-in-manifest",
"No provider API-key variable is declared on the agent workload",
"identity",
"names_absent",
[
step(
"env",
OPERATOR,
"kubectl",
"--namespace",
targets.namespace,
"get",
agent,
"-o",
ENV_NAME_PROJECTION,
)
],
{"step": "env", "names": PROVIDER_API_KEY_NAMES, "contains": ("API_KEY",)},
rationale="The operator counterpart to the in-pod probe: desired state and running state are asserted separately.",
),
]
def _routing(targets: Targets) -> list[CheckSpec]:
switchyard = f"deploy/{targets.switchyard_deployment}"
tail = shell("tail_lines", path=ROUTING_LOG, limit=str(targets.routing_tail_lines))
routing_step = step(
"routing", SWITCHYARD, *tail, record=False, max_bytes=targets.routing_tail_bytes
)
return [
check(
"routing.switchyard-is-available",
"Switchyard has a ready replica serving every model-call boundary",
"routing",
"lines_match",
[
step(
"deployment",
OPERATOR,
"kubectl",
"--namespace",
targets.namespace,
"get",
switchyard,
"-o",
projection("jsonpath={.status.readyReplicas}"),
)
],
{"step": "deployment", "pattern": r"^[1-9][0-9]*$", "minimum": 1},
),
check(
"routing.provider-effort-and-fallback-evidence",
"Routing evidence covers both providers, all graded efforts, and a real fallback",
"routing",
"routing_evidence",
[routing_step],
{
"step": "routing",
"providers": ("codex", "claude"),
"efforts": ("medium", "high", "xhigh"),
"lanes": ("route", "worker"),
"require_fallback_evidence": True,
"max_age_seconds": targets.max_evidence_age_seconds,
"now": targets.now,
},
rationale="The routing log records the decided route per boundary; the tail is parsed here and never pasted into the report.",
),
check(
"routing.codex-latency-recorded",
"Codex boundary latency is measured, not assumed",
"routing",
"json_numeric",
[
step(
"health",
SELF,
*shell(
"json_fields", path=CODEX_HEALTH, fields=CODEX_HEALTH_FIELDS
),
)
],
{"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}},
),
check(
"routing.claude-latency-recorded",
"Claude boundary latency is measured, not assumed",
"routing",
"json_numeric",
[
step(
"health",
SELF,
*shell(
"json_fields", path=CLAUDE_HEALTH, fields=CLAUDE_HEALTH_FIELDS
),
)
],
{"step": "health", "fields": {"latency_ms": {"min": 1, "max": 600_000}}},
),
]
def _scope_profile(
targets: Targets, name: str, workload: str, container: str, expected: str
) -> CheckSpec:
profile_projection = projection(
f'jsonpath={{.spec.template.spec.containers[?(@.name=="{container}")]'
'.env[?(@.name=="HERMES_AUTO_ROUTER_PROFILE")].value}'
)
return check(
f"scopes.{name}-profile-is-distinct",
f"The {name} surface runs under its own routing profile",
"scopes",
"stdout_matches",
[
step(
"profile",
OPERATOR,
"kubectl",
"--namespace",
targets.namespace,
"get",
workload,
"-o",
profile_projection,
)
],
{"step": "profile", "equals": expected},
rationale="Chat, agent, and triage must not collapse into one scope; each states its own profile.",
)
def _scopes(targets: Targets) -> list[CheckSpec]:
return [
_scope_profile(
targets,
"agent",
f"deploy/{targets.agent_deployment}",
targets.agent_container,
"agent",
),
_scope_profile(
targets,
"chat",
f"statefulset/{targets.chat_statefulset}",
targets.chat_container,
"chat",
),
_scope_profile(
targets,
"triage",
f"deploy/{targets.triage_deployment}",
targets.triage_container,
"triage",
),
check(
"scopes.surfaces-have-distinct-hosts",
"Chat, agent, and triage are published on distinct hostnames",
"scopes",
"distinct_count",
[
step(
"hosts",
OPERATOR,
"kubectl",
"--namespace",
targets.namespace,
"get",
"ingress",
"-o",
projection(
'jsonpath={range .items[*]}{range .spec.rules[*]}{.host}{"\\n"}{end}{end}'
),
)
],
{"step": "hosts", "minimum": 3},
),
]
def platform_checks(targets: Targets) -> list[CheckSpec]:
"""Return the baseline, identity, routing, and scope checks."""
return [
*_baseline(targets),
*_identity(targets),
*_routing(targets),
*_scopes(targets),
]

View File

@ -0,0 +1,376 @@
#!/usr/bin/env python3
"""Acceptance checks for the distributed worker pool and the visible surfaces.
A pool is only distributed if its workers occupy distinct nodes and distinct
volumes and leave state ownership with the coordinator; a replica count alone
satisfies none of that. A surface is only working if the work it did is visible
in it, which is why the session and Kanban checks look at the same data the UI
renders rather than at whether a pod is Running.
The ephemeral entries are placeholders in a default run. They report NOT_RUN
until the mutation mode is separately armed, and the armed mode replaces them
with real, mandatory results.
"""
from __future__ import annotations
from hermes_handoff_catalog import (
CHAT,
INIT_NAME_PROJECTION,
OPERATOR,
POOL_PROJECTION,
SELF,
Targets,
check,
)
from hermes_handoff_catalog import step as step
from hermes_handoff_model import EPHEMERAL, CheckSpec
from hermes_handoff_policy import projection
KANBAN_STATUSES = [
"archived",
"blocked",
"done",
"ready",
"review",
"running",
"scheduled",
"todo",
"triage",
]
def _pool(targets: Targets) -> list[CheckSpec]:
selector = targets.pool_selector
return [
check(
"pool.three-workers-have-distinct-nodes-and-volumes",
"Exactly three Ready ordinal workers occupy distinct nodes and claims",
"pool",
"pool_topology",
[
step(
"statefulset",
OPERATOR,
"kubectl",
"--namespace",
targets.namespace,
"get",
f"statefulset/{targets.pool_statefulset}",
"-o",
projection(
"jsonpath={.spec.replicas}{'\\t'}{.status.readyReplicas}{'\\t'}{.status.currentReplicas}{'\\t'}{.spec.template.spec.automountServiceAccountToken}".replace(
"'", '"'
)
),
),
step(
"workers",
OPERATOR,
"kubectl",
"--namespace",
targets.namespace,
"get",
"pods",
"--selector",
selector,
"-o",
POOL_PROJECTION,
),
],
{
"state_step": "statefulset",
"worker_step": "workers",
"replicas": targets.pool_replicas,
"name": targets.pool_statefulset,
},
rationale="Replica count, Ready state, ordinal coverage, node placement, PVC isolation, and tokenlessness are one fail-closed assertion.",
),
check(
"pool.coordinator-retains-sole-state-ownership",
"No pool worker mounts the coordinator's durable state",
"pool",
"names_absent",
[
step(
"claims",
OPERATOR,
"kubectl",
"--namespace",
targets.namespace,
"get",
f"statefulset/{targets.pool_statefulset}",
"-o",
projection(
"jsonpath={range .spec.template.spec.volumes[*]}{.name}{'='}"
"{.persistentVolumeClaim.claimName}{'\\n'}{end}".replace(
"'", '"'
)
),
)
],
{"step": "claims", "contains": targets.coordinator_claims},
rationale="Two writers on one SQLite state file is the duplicate-work and lost-result failure this pool exists to avoid. One line per template volume keeps a drifted or empty projection NOT_RUN instead of a silent pass.",
),
*_pool_env(targets),
]
def _pool_env(targets: Targets) -> list[CheckSpec]:
"""Return the mandatory assignment-safety configuration check."""
return [
check(
"pool.assignment-safety-knobs-are-configured",
"Pool workers carry the lease and de-duplication configuration",
"pool",
"names_present",
[
step(
"env",
OPERATOR,
"kubectl",
"--namespace",
targets.namespace,
"get",
f"statefulset/{targets.pool_statefulset}",
"-o",
projection(
'jsonpath={range .spec.template.spec.containers[*]}{range .env[*]}{.name}{"\\n"}{end}{end}'
),
)
],
{"step": "env", "names": targets.pool_worker_env},
rationale="Deployed configuration only; the stale-claim and duplicate-dispatch behaviour itself is covered by the pool's own regression suite.",
),
check(
"pool.assignment-fieldrefs-are-bound",
"Worker node and ordinal identities come from fixed downward-API fields",
"pool",
"names_present",
[
step(
"source",
OPERATOR,
"git",
"show",
f"{targets.remote}/main:services/hermes/execution-worker-statefulset.yaml",
record=False,
)
],
{
"step": "source",
"names": (
"name: HERMES_WORKER_ORDINAL",
"fieldPath: metadata.labels['apps.kubernetes.io/pod-index']",
"name: HERMES_WORKER_NODE",
"fieldPath: spec.nodeName",
),
},
rationale="This prevents caller-authored ordinal/node values from impersonating another worker assignment.",
),
]
def _surfaces(targets: Targets) -> list[CheckSpec]:
checks = [
check(
"surfaces.kanban-activity-is-visible",
"Durable Kanban activity is visible to the agent surface",
"surfaces",
"all_of_field",
[
step(
"tasks",
SELF,
"hermes",
"kanban",
"list",
"--json",
record=False,
max_bytes=targets.listing_bytes,
)
],
{"step": "tasks", "key_field": "id", "fields": {"status": KANBAN_STATUSES}},
rationale="Non-empty and well-formed. An empty board is NOT_RUN, because it cannot distinguish a quiet week from a broken read.",
),
check(
"surfaces.durable-sessions-render-without-blank-rows",
"Durable sessions are listed with a preview, a source, and an id",
"surfaces",
"lines_match",
[
step(
"sessions",
SELF,
"hermes",
"sessions",
"list",
"--limit",
"20",
record=False,
)
],
{"step": "sessions", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
rationale="A row whose leading column is blank is the session-render regression seen from the data side.",
),
check(
"surfaces.agent-ui-session-patches-are-deployed",
"The agent surface runs the session-activity rendering patches",
"surfaces",
"names_present",
[
step(
"init",
OPERATOR,
"kubectl",
"--namespace",
targets.namespace,
"get",
f"deploy/{targets.agent_deployment}",
"-o",
INIT_NAME_PROJECTION,
)
],
{
"step": "init",
"names": ("patch-web-session-activity", "patch-api-server-sessions"),
},
rationale="Blank session rows were a rendering regression; this asserts the patches that fixed it are in the running pod.",
),
check(
"surfaces.chat-session-patches-are-deployed",
"The chat surface runs the API-session continuity patches",
"surfaces",
"names_present",
[
step(
"init",
OPERATOR,
"kubectl",
"--namespace",
targets.namespace,
"get",
f"statefulset/{targets.chat_statefulset}",
"-o",
INIT_NAME_PROJECTION,
)
],
{"step": "init", "names": targets.chat_init_containers},
),
check(
"surfaces.chat-telegram-topic-state-is-durable",
"Telegram topic and tenant state is held on durable storage",
"surfaces",
"names_present",
[
step(
"claims",
OPERATOR,
"kubectl",
"--namespace",
targets.namespace,
"get",
f"deploy/{targets.chat_router_deployment}",
"-o",
projection(
'jsonpath={range .spec.template.spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\\n"}{end}'
),
)
],
{"step": "claims", "contains": ("state",)},
rationale="Topic selection survives a router restart only if the tenant state file is on a claim rather than in the pod.",
),
]
checks.append(
check(
"surfaces.chat-telegram-sessions-are-continuous",
"The selected chat tenant still holds durable Telegram-sourced sessions",
"surfaces",
"lines_match",
[
step(
"telegram",
CHAT,
"hermes",
"sessions",
"list",
"--source",
"telegram",
"--limit",
"20",
record=False,
)
],
{"step": "telegram", "pattern": r"^\S.*\S$", "skip": 2, "minimum": 1},
rationale="Session continuity is per tenant and depends on there having been Telegram traffic, so it is asserted only when the operator names the tenant that carries the identity.",
)
)
checks.append(
check(
"surfaces.chat-runs-the-telegram-topic-revision",
"The chat surface runs the durable Telegram topic configuration",
"surfaces",
"stdout_matches",
[
step(
"revision",
OPERATOR,
"kubectl",
"--namespace",
targets.namespace,
"get",
f"statefulset/{targets.chat_statefulset}",
"-o",
projection(
"jsonpath={.spec.template.metadata.annotations.ai\\.bstein\\.dev/config-rev}"
),
)
],
{"step": "revision", "equals": targets.chat_config_revision},
rationale="Telegram topic continuity is carried by the config revision the pod template pins.",
)
)
return checks
def _ephemeral() -> list[CheckSpec]:
return [
check(
f"ephemeral.{name}",
title,
"ephemeral",
"not_armed",
[],
{"reason": reason},
mandatory=False,
scope=EPHEMERAL,
rationale="Mutating checks are separately armed. A default run reports them NOT_RUN and never touches the forge.",
)
for name, title, reason in (
(
"feature-branch-push",
"A unique ephemeral feature branch can be pushed and removed",
"ephemeral mutation mode is not armed",
),
(
"draft-pull-request",
"A draft pull request can be opened and closed against the ephemeral branch",
"ephemeral mutation mode is not armed",
),
(
"protected-branch-refusal",
"Arming against a protected branch is refused before any network call",
"ephemeral mutation mode is not armed",
),
(
"cleanup-verified",
"Every ephemeral ref and pull request is removed and the removal is verified",
"ephemeral mutation mode is not armed",
),
)
]
def worker_checks(targets: Targets) -> list[CheckSpec]:
"""Return the pool, surface, and ephemeral-placeholder checks."""
return [*_pool(targets), *_surfaces(targets), *_ephemeral()]

View File

@ -0,0 +1,439 @@
#!/usr/bin/env python3
"""Discoverable, residue-surfacing, self-cleaning armed ephemeral run.
The create response — not the index — is the authoritative name of a pull
request this harness made, and the index is read to its last page. When either
source is uncertain the run fails closed and reports the exact residue plus the
exact commands that remove it, so an operator is never left guessing what was
created.
Arming, provenance, and push-target rules live in :mod:`hermes_handoff_arming`
and are re-exported here for callers that treat armed mode as one surface.
"""
from __future__ import annotations
from collections.abc import Sequence
from typing import Any
from hermes_handoff_arming import (
CONFIRMATION,
REF_PREFIX,
ArmingError,
ArmRequest,
assert_push_target_allowed,
bounded_read,
normalise_branch,
preflight,
)
from hermes_handoff_exec import Outcome, Runner, Vantage
from hermes_handoff_model import EPHEMERAL, FAIL, NOT_RUN, PASS, CheckResult, CheckSpec
from hermes_handoff_policy import (
EXPECTED_BASE,
GITEA_CLIENT,
register_ephemeral_pull,
)
from hermes_handoff_rules import strict_json
__all__ = [
"CONFIRMATION",
"DISCOVERY_PAGE_BUDGET",
"DISCOVERY_PAGE_SIZE",
"REF_PREFIX",
"ArmRequest",
"ArmingError",
"assert_push_target_allowed",
"bounded_read",
"normalise_branch",
"preflight",
"run_armed",
]
DRAFT_TITLE_PREFIX = "WIP: "
DISCOVERY_PAGE_SIZE = 50
DISCOVERY_PAGE_BUDGET = 20
def _spec(identifier: str, title: str) -> CheckSpec:
return CheckSpec(
id=f"ephemeral.{identifier}",
title=title,
group="ephemeral",
rule="armed",
steps=(),
mandatory=True,
scope=EPHEMERAL,
)
def _result(
identifier: str,
title: str,
status: str,
reason: str,
outcomes: Sequence[Outcome] = (),
evidence: dict[str, Any] | None = None,
) -> CheckResult:
return CheckResult(
spec=_spec(identifier, title),
status=status,
reason=reason,
evidence=evidence or {},
outcomes=list(outcomes),
)
def _protected_refusal_result() -> CheckResult:
accepted: list[str] = []
for candidate in ("main", "master", "refs/heads/main", "HEAD", f"{REF_PREFIX}/x"):
try:
assert_push_target_allowed(candidate)
except ArmingError:
continue
accepted.append(candidate)
return _result(
"protected-branch-refusal",
"Arming against a protected branch is refused before any network call",
FAIL if accepted else PASS,
f"guard accepted {accepted}"
if accepted
else "every protected and malformed target was refused",
evidence={"accepted": accepted},
)
def _detail(outcome: Outcome) -> str:
return (
outcome.error or outcome.combined[:200] or f"exit status {outcome.returncode}"
)
def _pr_problems(payload: Any, request: ArmRequest) -> list[str]:
if not isinstance(payload, dict):
return ["pull request is not an object"]
wanted = {
"state": "open",
"draft": True,
"merged": False,
"base": {"ref": EXPECTED_BASE, "sha": request.expected_base_sha},
"head": {"ref": request.ref, "sha": request.expected_head},
}
problems: list[str] = []
if (
not isinstance(payload.get("number"), int)
or isinstance(payload.get("number"), bool)
or payload["number"] <= 0
):
problems.append("number is malformed")
for field in ("state", "draft", "merged"):
if (
type(payload.get(field)) is not type(wanted[field])
or payload.get(field) != wanted[field]
):
problems.append(f"{field} does not match")
for group, values in (("base", wanted["base"]), ("head", wanted["head"])):
observed = payload.get(group)
if not isinstance(observed, dict):
problems.append(f"{group} is malformed")
continue
for field, value in values.items():
if observed.get(field) != value:
problems.append(f"{group}.{field} does not match")
return problems
def _created_number(outcome: Outcome, request: ArmRequest) -> tuple[int, str]:
"""Return the number the create response itself reports.
The create response is the only source that names a pull request the harness
definitely made. Relying on the index alone means an index that fails or
answers ambiguously leaves an unnamed, unclosed pull request behind.
"""
if not outcome.ok:
return (0, f"pull-request creation failed: {_detail(outcome)}")
try:
payload = strict_json(outcome.stdout)
except ValueError as exc:
return (0, f"pull-request creation returned malformed JSON: {exc}")
problems = _pr_problems(payload, request)
if problems:
return (0, f"created pull request is invalid: {', '.join(problems)}")
return (payload["number"], "")
def _index_pages(
runner: Runner, vantage: Vantage, api: str, reserve: float
) -> tuple[list[Outcome], list[Any], str]:
"""Read every page of the pull index; a truncated page can hide a match."""
outcomes: list[Outcome] = []
rows: list[Any] = []
for page in range(1, DISCOVERY_PAGE_BUDGET + 1):
outcome = runner.run(
(
GITEA_CLIENT,
"GET",
f"{api}/pulls?state=all&limit={DISCOVERY_PAGE_SIZE}&page={page}",
),
vantage,
reserve_seconds=reserve,
)
outcomes.append(outcome)
if not outcome.ok:
return (outcomes, [], f"discovery page {page} failed: {_detail(outcome)}")
try:
payload = strict_json(outcome.stdout)
except ValueError as exc:
return (outcomes, [], f"discovery page {page} is malformed JSON: {exc}")
if not isinstance(payload, list):
return (outcomes, [], f"discovery page {page} did not return an array")
rows.extend(payload)
if len(payload) < DISCOVERY_PAGE_SIZE:
return (outcomes, rows, "")
return (outcomes, [], "pull-request discovery exceeded its page budget")
def _residue(request: ArmRequest, numbers: Sequence[int], api: str) -> dict[str, Any]:
"""Name the exact residue and the exact commands that remove it by hand."""
manual = [
f"{GITEA_CLIENT} PATCH {api}/pulls/{number} --field state=closed"
for number in numbers
]
if not numbers:
manual.append(
f"{GITEA_CLIENT} GET {api}/pulls?state=all&limit={DISCOVERY_PAGE_SIZE}"
f" then close by hand every pull whose head ref is {request.ref}"
)
manual.append(f"{GITEA_CLIENT} DELETE {api}/branches/{request.ref}")
return {"residue_ref": request.ref, "manual_cleanup": manual}
def _match_pulls(rows: Sequence[Any], request: ArmRequest) -> tuple[list[int], str]:
matches: list[int] = []
for index, item in enumerate(rows):
if (
not isinstance(item, dict)
or not isinstance(item.get("head"), dict)
or not isinstance(item.get("base"), dict)
):
return ([], f"pull-request discovery row {index + 1} is malformed")
if item["head"].get("ref") != request.ref:
continue
problems = _pr_problems(item, request)
if problems:
return ([], f"discovered pull request is invalid: {', '.join(problems)}")
matches.append(item["number"])
if len(matches) != 1:
return (
matches,
f"expected one exact draft pull request, discovered {len(matches)}",
)
return (matches, "")
def _closed_pr_problems(payload: Any, request: ArmRequest) -> list[str]:
"""Validate the same exact PR identity with state changed to closed."""
problems = _pr_problems(payload, request)
if isinstance(payload, dict) and payload.get("state") == "closed":
problems = [
problem for problem in problems if problem != "state does not match"
]
elif "state does not match" not in problems:
problems.append("state is not closed")
return problems
def _cleanup(
runner: Runner,
vantage: Vantage,
request: ArmRequest,
api: str,
numbers: Sequence[int],
discovery_error: str = "",
) -> CheckResult:
outcomes: list[Outcome] = []
problems = [discovery_error] if discovery_error else []
for number in sorted(set(numbers)):
closed = runner.run(
(GITEA_CLIENT, "PATCH", f"{api}/pulls/{number}", "--field", "state=closed"),
vantage,
)
outcomes.append(closed)
if not closed.ok:
problems.append(f"pull request #{number} close failed")
continue
try:
payload = strict_json(closed.stdout)
except ValueError:
problems.append(f"pull request #{number} close response is malformed")
continue
expected = _closed_pr_problems(payload, request)
if expected:
problems.append(f"pull request #{number} close response does not match")
deleted = runner.run(
(GITEA_CLIENT, "DELETE", f"{api}/branches/{request.ref}"), vantage
)
outcomes.append(deleted)
if not deleted.ok:
problems.append("ephemeral branch deletion failed")
remaining = runner.run(
("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"), vantage
)
outcomes.append(remaining)
if not remaining.ok or remaining.stdout.strip():
problems.append("ephemeral branch absence could not be verified")
for number in sorted(set(numbers)):
observed = runner.run((GITEA_CLIENT, "GET", f"{api}/pulls/{number}"), vantage)
outcomes.append(observed)
if not observed.ok:
problems.append(f"pull request #{number} closure could not be verified")
continue
try:
payload = strict_json(observed.stdout)
except ValueError:
problems.append(f"pull request #{number} verification is malformed")
continue
expected = _closed_pr_problems(payload, request)
if expected:
problems.append(f"pull request #{number} is not exactly closed")
return _result(
"cleanup-verified",
"Every ephemeral ref and pull request is removed and the removal is verified",
FAIL if problems else PASS,
"; ".join(problems)
if problems
else "the exact branch and draft pull request are gone",
outcomes,
{"ref": request.ref, "pull_requests": sorted(set(numbers))},
)
def _nothing_created(branch: CheckResult, draft_reason: str) -> list[CheckResult]:
"""Report the two results that follow a run which created no artifact."""
return [
branch,
_result(
"draft-pull-request",
"A draft pull request can be opened and closed",
NOT_RUN,
draft_reason,
),
_result(
"cleanup-verified",
"Every ephemeral artifact is removed",
NOT_RUN,
"nothing was created",
),
]
def run_armed(
runner: Runner, vantage: Vantage, request: ArmRequest
) -> list[CheckResult]:
"""Create, discover, validate, and clean one exact ephemeral branch/PR."""
guard = _protected_refusal_result()
branch_title = "A unique ephemeral feature branch can be pushed and removed"
if guard.status != PASS:
unrun = _result("feature-branch-push", branch_title, NOT_RUN, "guard failed")
return [guard, *_nothing_created(unrun, "protected-ref guard failed")]
api = f"/api/v1/repos/{request.repo}"
reserve = min(6000.0, max(300.0, runner.command_timeout * 10))
existing = runner.run(
("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"),
vantage,
reserve_seconds=reserve,
)
if not existing.ok or existing.stdout.strip():
reason = (
_detail(existing) if not existing.ok else "the ephemeral ref already exists"
)
refused = _result("feature-branch-push", branch_title, FAIL, reason, [existing])
return [guard, *_nothing_created(refused, "branch preflight failed")]
local_head = runner.run(
("git", "rev-parse", "HEAD"), vantage, reserve_seconds=reserve
)
pushed = (
runner.run(
("git", "push", request.remote, f"HEAD:refs/heads/{request.ref}"),
vantage,
reserve_seconds=reserve,
)
if local_head.ok and local_head.stdout.strip() == request.expected_head
else Outcome(
argv=("git", "push"),
vantage=vantage.name,
error="local HEAD changed after static preflight",
)
)
verified = runner.run(
("git", "ls-remote", request.remote, f"refs/heads/{request.ref}"),
vantage,
reserve_seconds=reserve,
)
branch_ok = verified.ok and verified.stdout.split() == [
request.expected_head,
f"refs/heads/{request.ref}",
]
branch_result = _result(
"feature-branch-push",
branch_title,
PASS if branch_ok else FAIL,
"the exact reviewed HEAD was pushed"
if branch_ok
else "push could not be verified exactly",
[local_head, pushed, verified],
{"ref": request.ref, "head": request.expected_head},
)
numbers: list[int] = []
problems: list[str] = ["branch creation was not verified"]
create_outcomes: list[Outcome] = []
if branch_ok:
created = runner.run(
(
GITEA_CLIENT,
"POST",
f"{api}/pulls",
"--field",
f"title={DRAFT_TITLE_PREFIX}Hermes handoff acceptance ephemeral probe",
"--field",
f"head={request.ref}",
"--field",
f"base={EXPECTED_BASE}",
"--field",
"body=Ephemeral acceptance probe. Closed and deleted by the harness.",
),
vantage,
reserve_seconds=reserve,
)
number, create_error = _created_number(created, request)
pages, rows, index_error = _index_pages(runner, vantage, api, reserve)
found, match_error = (
_match_pulls(rows, request) if not index_error else ([], "")
)
create_outcomes = [created, *pages]
# Either source alone names a pull request this run definitely made: the
# ephemeral ref was proven absent in preflight, so any pull on it is ours.
numbers = sorted({item for item in (number, *found) if item})
problems = [text for text in (index_error, match_error) if text]
if number and found and found != [number]:
problems.append(f"created #{number} is not the discovered {found}")
if not numbers:
problems.append(
create_error or "no exact draft pull request was identified"
)
for item in numbers:
register_ephemeral_pull(item)
exact = len(numbers) == 1 and not problems
draft_result = _result(
"draft-pull-request",
"A draft pull request can be opened and closed against the ephemeral branch",
PASS if exact else FAIL,
f"discovered exact draft pull request #{numbers[0]}"
if exact
else "; ".join(problems),
create_outcomes,
{"pull_requests": numbers}
if exact
else {"pull_requests": numbers, **_residue(request, numbers, api)},
)
cleanup = _cleanup(runner, vantage, request, api, numbers, "; ".join(problems))
return [guard, branch_result, draft_result, cleanup]

View File

@ -0,0 +1,497 @@
#!/usr/bin/env python3
"""Text-shaped evaluators for the handoff acceptance harness.
These rules cover evidence that arrives as a refusal, an exit status, or a list
of names. The ``denied`` rule is the load-bearing one: it refuses to classify on
an authorization review alone — ``kubectl auth can-i`` reports what RBAC says,
not what the API server does — so a real request must have been made and
refused before it will pass.
An absence is only evidence when the probe observed something, so a name step
that exits 0 with no lines is ``NOT_RUN``: drift produces silence, not a pass.
Importing this module also registers the JSON-shaped rules, so a caller only has
to import one place to have the full catalog vocabulary available.
"""
from __future__ import annotations
import re
from typing import Any
from hermes_handoff_exec import Outcome
from hermes_handoff_json_rules import evaluate_flux_health # noqa: F401 (registers rules)
from hermes_handoff_model import ATTEMPT, FAIL, NOT_RUN, PASS, REVIEW, CheckSpec
from hermes_handoff_rules import (
EVALUATORS,
Evaluation,
dotted,
evaluate,
evaluator,
exact_equal,
lines,
looks_denied,
missing_steps,
step_keys,
strict_json,
)
__all__ = ["EVALUATORS", "Evaluation", "evaluate", "evaluator"]
@evaluator("denied")
def evaluate_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
"""Pass only when the review says no and a real request was refused."""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
attempts = step_keys(spec, ATTEMPT)
if not attempts:
return Evaluation(
NOT_RUN, "deny checks require a real attempt, not only a review"
)
evidence: dict[str, Any] = {}
for key in step_keys(spec, REVIEW):
verdict = outcomes[key].stdout.strip().lower()
evidence[key] = verdict
if verdict != "no":
return Evaluation(
FAIL,
f"authorization review {key} reported {verdict or 'nothing'}",
evidence,
)
for key in attempts:
outcome = outcomes[key]
if outcome.ok:
return Evaluation(
FAIL, f"attempt {key} succeeded but must be refused", evidence
)
if not looks_denied(outcome):
return Evaluation(
NOT_RUN,
f"attempt {key} failed without a recognisable refusal (rc={outcome.returncode})",
evidence,
)
evidence[key] = "refused"
return Evaluation(PASS, "review and live attempt were both refused", evidence)
@evaluator("review_denied")
def evaluate_review_denied(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
"""Pass only when every side-effect-free authorization review says no."""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
reviews = step_keys(spec, REVIEW)
if not reviews:
return Evaluation(NOT_RUN, "a denial check requires an authorization review")
evidence: dict[str, str] = {}
for key in reviews:
verdict = outcomes[key].stdout.strip().lower()
evidence[key] = verdict
if verdict != "no":
return Evaluation(
FAIL,
f"authorization review {key} reported {verdict or 'nothing'}",
evidence,
)
return Evaluation(PASS, "every authorization review was denied", evidence)
@evaluator("allowed")
def evaluate_allowed(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
"""Pass when every required step succeeded and every review said yes."""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
evidence: dict[str, Any] = {}
for step in spec.steps:
outcome = outcomes[step.key]
if step.kind == REVIEW:
verdict = outcome.stdout.strip().lower()
evidence[step.key] = verdict
if verdict != "yes":
return Evaluation(
FAIL,
f"authorization review {step.key} reported {verdict or 'nothing'}",
evidence,
)
continue
if not outcome.ok:
return Evaluation(
FAIL, f"step {step.key} failed (rc={outcome.returncode})", evidence
)
evidence[step.key] = "succeeded"
return Evaluation(PASS, "required access is present", evidence)
def _matched(observed: set[str], patterns: tuple[str, ...]) -> set[str]:
return {name for pattern in patterns for name in observed if pattern in name}
def _line_step(
spec: CheckSpec, outcomes: dict[str, Outcome]
) -> tuple[set[str], str] | Evaluation:
"""Return a step's distinct lines, or the NOT_RUN that silence must produce."""
key = spec.expect["step"]
outcome = outcomes[key]
if not outcome.ok:
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
observed = set(lines(outcome))
if not observed:
return Evaluation(NOT_RUN, f"step {key} exited 0 with no lines to examine")
return (observed, key)
@evaluator("names_absent")
def evaluate_names_absent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
"""Pass when a step observed names and none of them is forbidden."""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
resolved = _line_step(spec, outcomes)
if isinstance(resolved, Evaluation):
return resolved
observed, _ = resolved
offenders = sorted(
(observed & set(spec.expect.get("names", ())))
| _matched(observed, tuple(spec.expect.get("contains", ())))
)
evidence = {"observed_count": len(observed), "offenders": offenders}
if offenders:
return Evaluation(
FAIL, f"forbidden names present: {', '.join(offenders)}", evidence
)
return Evaluation(PASS, "no forbidden names are present", evidence)
@evaluator("names_present")
def evaluate_names_present(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
"""Pass when every required name, and every required substring, appears."""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
resolved = _line_step(spec, outcomes)
if isinstance(resolved, Evaluation):
return resolved
observed, _ = resolved
missing = sorted(set(spec.expect.get("names", ())) - observed)
missing += sorted(
pattern
for pattern in spec.expect.get("contains", ())
if not any(pattern in name for name in observed)
)
evidence = {"observed_count": len(observed), "missing": missing}
if missing:
return Evaluation(
FAIL, f"required names absent: {', '.join(missing)}", evidence
)
return Evaluation(PASS, "every required name is present", evidence)
@evaluator("stdout_matches")
def evaluate_stdout_matches(
spec: CheckSpec, outcomes: dict[str, Outcome]
) -> Evaluation:
"""Compare a step's trimmed stdout against an expected value or set."""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
key = spec.expect["step"]
outcome = outcomes[key]
if not outcome.ok:
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
actual = outcome.stdout.strip()
expected = spec.expect["equals"]
allowed = expected if isinstance(expected, list) else [expected]
evidence = {"observed": actual}
if actual not in allowed:
return Evaluation(
FAIL, f"observed {actual!r}, expected one of {allowed!r}", evidence
)
return Evaluation(PASS, "output matches the expected value", evidence)
@evaluator("distinct_count")
def evaluate_distinct_count(
spec: CheckSpec, outcomes: dict[str, Outcome]
) -> Evaluation:
"""Pass when a step yields enough distinct rows — and, optionally, exactly N.
Distinctness is the point for placement checks: three worker pods pinned to
one node satisfy a replica count and defeat the isolation the replica count
was standing in for.
"""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
key = spec.expect["step"]
outcome = outcomes[key]
if not outcome.ok:
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
values = lines(outcome)
distinct = sorted(set(values))
minimum = spec.expect["minimum"]
evidence = {
"total": len(values),
"distinct": len(distinct),
"values": distinct[:16],
}
if not values:
return Evaluation(NOT_RUN, f"step {key} produced no rows to count", evidence)
if len(distinct) < minimum:
return Evaluation(
FAIL,
f"{len(distinct)} distinct values, expected at least {minimum}",
evidence,
)
total_equals = spec.expect.get("total_equals")
if total_equals is not None and len(values) != total_equals:
return Evaluation(
FAIL, f"{len(values)} rows, expected exactly {total_equals}", evidence
)
return Evaluation(PASS, f"{len(distinct)} distinct values observed", evidence)
@evaluator("lines_match")
def evaluate_lines_match(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
"""Pass when every data row of a tabular listing matches a shape.
Some surfaces only speak tables. A blank leading column is exactly what the
session-render regression looked like, so asserting the shape of each row is
a real check rather than a proxy for one.
"""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
key = spec.expect["step"]
outcome = outcomes[key]
if not outcome.ok:
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
pattern = re.compile(spec.expect["pattern"])
rows = outcome.stdout.splitlines()[spec.expect.get("skip", 0) :]
rows = [row for row in rows if row.strip()]
offenders = [row[:60] for row in rows if not pattern.search(row)]
evidence = {"rows": len(rows), "offenders": offenders[:8]}
if len(rows) < spec.expect.get("minimum", 1):
return Evaluation(
NOT_RUN,
f"step {key} produced {len(rows)} data rows, expected at least "
f"{spec.expect.get('minimum', 1)}",
evidence,
)
if offenders:
return Evaluation(
FAIL, f"{len(offenders)} row(s) do not match the expected shape", evidence
)
return Evaluation(PASS, f"all {len(rows)} rows match the expected shape", evidence)
@evaluator("vantages_agree")
def evaluate_vantages_agree(
spec: CheckSpec, outcomes: dict[str, Outcome]
) -> Evaluation:
"""Pass only when two independent observations report the same fact.
A disagreement is a finding in its own right: it usually means the manifest
on record and the workload actually running have drifted apart, and either
one alone would have looked clean.
"""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
left_key, right_key = spec.expect["steps"]
observations: dict[str, list[str]] = {}
for key in (left_key, right_key):
outcome = outcomes[key]
if not outcome.ok:
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
observations[key] = sorted(lines(outcome))
evidence = {key: value[:16] for key, value in observations.items()}
if observations[left_key] != observations[right_key]:
return Evaluation(
FAIL, f"vantages disagree: {left_key} and {right_key} differ", evidence
)
if not observations[left_key]:
return Evaluation(
NOT_RUN, "both vantages returned nothing to compare", evidence
)
return Evaluation(PASS, "both vantages report the same state", evidence)
@evaluator("not_armed")
def evaluate_not_armed(spec: CheckSpec, _outcomes: dict[str, Outcome]) -> Evaluation:
"""Report the fixed NOT_RUN a default, read-only run must produce."""
reason = spec.expect.get("reason", "ephemeral mutation mode is not armed")
return Evaluation(NOT_RUN, reason, {"armed": False})
# This validator is one compact, linear state machine so every exit observes
# only fully validated state.
# fmt: off
@evaluator("pool_topology")
def evaluate_pool_topology(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
"""Require exactly three Ready ordinal workers on distinct nodes and claims."""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
state = outcomes[spec.expect["state_step"]].stdout.strip().split("\t")
if len(state) != 4:
return Evaluation(NOT_RUN, "StatefulSet projection is malformed")
expected = spec.expect.get("replicas", 3)
try:
replicas, ready, current = (int(value) for value in state[:3])
except ValueError:
return Evaluation(NOT_RUN, "StatefulSet replica fields are malformed")
evidence: dict[str, Any] = {"replicas": replicas, "ready": ready, "current": current,
"automountServiceAccountToken": state[3]}
if (replicas, ready, current, state[3].lower()) != (expected, expected, expected, "false"):
return Evaluation(FAIL, "worker replica or tokenless state does not match the contract", evidence)
workers: dict[int, tuple[str, str]] = {}
names: list[str] = []
for number, row in enumerate(lines(outcomes[spec.expect["worker_step"]]), 1):
columns = row.split("\t")
if len(columns) != 5:
return Evaluation(NOT_RUN, f"worker row {number} is malformed", evidence)
name, node, phase, ready_state, claims = columns
match = re.fullmatch(rf"{re.escape(spec.expect['name'])}-(\d+)", name)
claim_list = [claim for claim in claims.split(",") if claim]
if (not match or not node or phase != "Running" or ready_state != "True"
or len(claim_list) != 1):
return Evaluation(FAIL, f"worker row {number} is not a Ready isolated worker", evidence)
ordinal = int(match.group(1))
if ordinal in workers:
return Evaluation(FAIL, f"worker ordinal {ordinal} is duplicated", evidence)
workers[ordinal] = (node, claim_list[0])
names.append(name)
evidence.update({"workers": sorted(names), "nodes": sorted({v[0] for v in workers.values()}),
"claims": sorted({v[1] for v in workers.values()})})
if set(workers) != set(range(expected)):
return Evaluation(FAIL, f"worker ordinals are not exactly 0..{expected - 1}", evidence)
if (len({v[0] for v in workers.values()}) != expected
or len({v[1] for v in workers.values()}) != expected):
return Evaluation(FAIL, "workers do not occupy distinct nodes and claims", evidence)
return Evaluation(PASS, "three Ready workers have distinct nodes, ordinals, and claims", evidence)
# fmt: on
_SHA_RE = re.compile(r"[0-9a-f]{40}\Z")
_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}\Z")
_IMAGE_RE = re.compile(
r"[^@\s]+:git-([0-9a-f]{40})-build-([1-9][0-9]*)@(sha256:[0-9a-f]{64})\Z"
)
def _strict_object(outcome: Outcome, label: str) -> dict[str, Any] | Evaluation:
try:
payload = strict_json(outcome.stdout)
except ValueError as exc:
return Evaluation(NOT_RUN, f"{label} JSON is malformed: {exc}")
if not isinstance(payload, dict):
return Evaluation(NOT_RUN, f"{label} JSON is not an object")
return payload
# fmt: off
@evaluator("release_lineage")
def evaluate_release_lineage(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
"""Atomically bind Git, PR, image, build, Flux, and running revision evidence."""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
expected = spec.expect
main_sha = expected["main_sha"]
head_sha = expected["head_sha"]
image = expected["image"]
build_sha = expected["build_sha"]
revision = str(expected["deployment_revision"])
digest = image.rsplit("@", 1)[-1]
image_match = _IMAGE_RE.fullmatch(image)
if not (_SHA_RE.fullmatch(main_sha) and _SHA_RE.fullmatch(head_sha)
and _SHA_RE.fullmatch(build_sha) and _DIGEST_RE.fullmatch(digest)
and image_match and image_match.group(1) == build_sha and build_sha == main_sha):
return Evaluation(NOT_RUN, "release lineage expectations are malformed")
remote = outcomes["remote-main"].stdout.strip().split()
if remote != [main_sha, "refs/heads/main"]:
return Evaluation(FAIL, "remote main does not match the fixed expected SHA",
{"remote_main": remote[:2]})
pr = _strict_object(outcomes["reviewed-pr"], "reviewed PR")
if isinstance(pr, Evaluation):
return pr
fields = {
"number": expected["pr_number"],
"state": "open",
"draft": True,
"merged": False,
"base.ref": "main",
"base.sha": main_sha,
"head.ref": expected["head_ref"],
"head.sha": head_sha,
}
for path, wanted in fields.items():
try:
actual = dotted(pr, path)
except (KeyError, ValueError, IndexError):
return Evaluation(NOT_RUN, f"reviewed PR is missing {path}")
if not exact_equal(actual, wanted):
return Evaluation(FAIL, f"reviewed PR {path} does not match", {path: actual})
if outcomes["build-source"].stdout.strip() != build_sha:
return Evaluation(FAIL, "build source SHA does not match the fixed release build SHA")
deployment = outcomes["deployment"].stdout.strip().split("\t")
if len(deployment) != 6:
return Evaluation(NOT_RUN, "Deployment projection is malformed")
generation, observed, replicas, ready, deployed_revision, deployed_image = deployment
try:
current = (int(generation) > 0 and generation == observed
and int(replicas) > 0 and replicas == ready)
except ValueError:
return Evaluation(NOT_RUN, "Deployment numeric fields are malformed")
if not current or (deployed_revision, deployed_image) != (revision, image):
return Evaluation(FAIL, "Deployment lineage does not match the release contract")
replica_hash = ""
for number, row in enumerate(lines(outcomes["replicasets"]), 1):
columns = row.split("\t")
if len(columns) != 6:
return Evaluation(NOT_RUN, f"ReplicaSet row {number} is malformed")
_, rs_revision, rs_ready, rs_available, rs_image, rs_hash = columns
if rs_revision != revision:
continue
if (rs_ready != replicas or rs_available != replicas or rs_image != image or not rs_hash):
return Evaluation(FAIL, "active ReplicaSet does not match the Deployment lineage")
if replica_hash:
return Evaluation(FAIL, "more than one ReplicaSet claims the running revision")
replica_hash = rs_hash
if not replica_hash:
return Evaluation(FAIL, "running Deployment revision has no matching ReplicaSet")
pod_rows = lines(outcomes["pods"])
if len(pod_rows) != int(replicas):
return Evaluation(FAIL, "running pod count does not match the Deployment")
pods: list[str] = []
for number, row in enumerate(pod_rows, 1):
columns = row.split("\t")
if len(columns) != 6:
return Evaluation(NOT_RUN, f"pod row {number} is malformed")
name, phase, ready_state, pod_image, image_id, pod_hash = columns
if phase != "Running" or ready_state != "True":
return Evaluation(FAIL, f"pod {name or number} is not Ready")
if (pod_image, pod_hash) != (image, replica_hash) or not image_id.endswith(f"@{digest}"):
return Evaluation(FAIL, f"pod {name or number} lineage does not match")
pods.append(name)
flux = outcomes["flux"].stdout.strip().split("\t")
if len(flux) != 6:
return Evaluation(NOT_RUN, "Flux lineage projection is malformed")
name, generation, observed, suspended, ready_state, applied = flux
if (not name or generation != observed or suspended.lower() == "true" or ready_state != "True"):
return Evaluation(FAIL, "Flux source is not current and Ready")
match = re.search(r"sha1:([0-9a-f]{40})\Z", applied)
if not match or match.group(1) != main_sha:
return Evaluation(FAIL, "Flux applied revision is not the expected remote main")
return Evaluation(PASS, "Git, PR, image, build, Flux, and running revision are bound",
{"main_sha": main_sha, "head_sha": head_sha, "image_digest": digest,
"build_sha": build_sha, "deployment_revision": revision, "pods": pods})
# fmt: on
# Two observations of the same fact agree whether or not they came from
# different vantages; the dual-vantage case is just the important one.
EVALUATORS["steps_agree"] = evaluate_vantages_agree

View File

@ -0,0 +1,482 @@
#!/usr/bin/env python3
"""Bounded, screened, provenance-attested command execution.
It bounds pipes, times out and reaps the process group, and discards truncated output.
"""
from __future__ import annotations
import hashlib
import math
import os
import selectors
import shlex
import shutil
import signal
import stat
import subprocess
import time
from collections.abc import Callable, Mapping, Sequence
from contextlib import suppress
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from hermes_handoff_policy import GITEA_CLIENT, READ_ONLY, PolicyError, check_argv
from hermes_handoff_redaction import (
DEFAULT_MAX_BYTES,
TRUNCATION_NOTE,
redact,
safe_text,
scrub,
)
DEFAULT_COMMAND_TIMEOUT = 25.0
DEFAULT_RUN_DEADLINE = 900.0
DEFAULT_CLEANUP_RESERVE = 90.0
MAX_COMMAND_TIMEOUT = 600.0
MAX_RUN_DEADLINE = 7200.0
MAX_OUTPUT_BYTES = 8 * 1024 * 1024
READ_CHUNK = 64 * 1024
PIPE_DRAIN_SECONDS = 0.25
SAFE_PATH = "/opt/data/tools/bin:/opt/hermes/.venv/bin:/opt/coordinator:/usr/local/bin:/usr/bin:/bin"
# These exact allowlists are intentionally dense: reviewers compare them as a
# single provenance boundary rather than as extensible application config.
# fmt: off
ENV_PASSTHROUGH = ("KUBECONFIG", "LANG", "LC_ALL", "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE")
FIXED_ENVIRONMENT = {"GIT_ASKPASS": "/opt/coordinator/gitea_askpass.sh",
"GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_NOSYSTEM": "1",
"GIT_TERMINAL_PROMPT": "0", "GITEA_BASE_URL": "https://scm.bstein.dev",
"LC_ALL": "C", "PATH": SAFE_PATH}
EXPECTED_PATHS = {
"kubectl": {"/opt/data/tools/bin/kubectl", "/usr/local/bin/kubectl", "/usr/bin/kubectl"},
"git": {"/usr/bin/git", "/bin/git"},
"hermes": {"/opt/hermes/.venv/bin/hermes"},
"sh": {"/usr/bin/sh", "/bin/sh"},
GITEA_CLIENT: {GITEA_CLIENT},
}
EXPECTED_SHA256 = {
"kubectl": {"3d514dbae5dc8c09f773df0ef0f5d449dfad05b3aca5c96b13565f886df345fd"},
"git": {"a0e562e4bd3c4c79379e91d8c07a10104b2cefe8fac966dc6bd4874a57a807f3"},
"sh": {"367967c823a0c391e5049b15a67c6a0a629c88b9b6dcdca75ef13ac9d65334b1"},
"hermes": {"c8419290ef7f1a95f59eadcea9f58f446b1e6535a7f4ffee9f1c0cb1172335e4"},
GITEA_CLIENT: {"f0943db4e81ce96908e94b0f0e78aa4b5ef95d6c4f7803f9f74a43e761576d85"},
}
DEADLINE_ERROR = "deadline-exceeded"
TIMEOUT_ERROR = "timeout"
OPERATOR = "operator"
SELF = "self"
CONTEXT_AWARE_BINARIES = ("kubectl",)
POD_COMMAND_PATHS = {"kubectl": "/usr/local/bin/kubectl", "hermes": "/opt/hermes/.venv/bin/hermes",
"git": "/usr/bin/git", "sh": "/bin/sh", GITEA_CLIENT: GITEA_CLIENT}
# fmt: on
@dataclass(frozen=True)
class Attestation:
"""The exact outer executable used for a probe."""
path: str
sha256: str
@dataclass(frozen=True)
class Capture:
"""Raw bounded process result returned by an executor implementation."""
returncode: int
stdout: bytes = b""
stderr: bytes = b""
stdout_truncated: bool = False
stderr_truncated: bool = False
timed_out: bool = False
@dataclass(frozen=True)
class Outcome:
"""The screened, bounded result of one probe command."""
argv: tuple[str, ...]
vantage: str
returncode: int | None = None
stdout: str = ""
stderr: str = ""
truncated: bool = False
duration_ms: int = 0
error: str | None = None
executable_path: str = ""
executable_sha256: str = ""
@property
def ok(self) -> bool:
return self.error is None and self.returncode == 0 and not self.truncated
@property
def ran(self) -> bool:
return self.error is None
@property
def combined(self) -> str:
return "\n".join(part for part in (self.stdout, self.stderr) if part)
def as_dict(self) -> dict[str, object]:
return scrub(
{
"command": redact(shlex.join(self.argv)),
"vantage": self.vantage,
"returncode": self.returncode,
"stdout": self.stdout,
"stderr": self.stderr,
"truncated": self.truncated,
"duration_ms": self.duration_ms,
"error": self.error,
"executable_path": self.executable_path,
"executable_sha256": self.executable_sha256,
}
)
@dataclass(frozen=True)
class Vantage:
"""A named identity and transport used to observe a probe."""
name: str
prefix: tuple[str, ...] = ()
inserts: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
env: Mapping[str, str] = field(default_factory=dict)
command_paths: Mapping[str, str] = field(default_factory=dict)
description: str = ""
def wrap(self, argv: Sequence[str]) -> tuple[str, ...]:
command = tuple(argv)
if command:
replacement = self.command_paths.get(command[0])
if replacement:
command = (replacement, *command[1:])
extra = self.inserts.get(command[0].rsplit("/", 1)[-1])
if extra:
command = (command[0], *extra, *command[1:])
return (*self.prefix, *command)
def _finite_number(name: str, value: Any, minimum: float, maximum: float) -> float:
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ValueError(f"{name} must be numeric")
converted = float(value)
if not math.isfinite(converted) or not minimum <= converted <= maximum:
raise ValueError(
f"{name} must be finite and between {minimum:g} and {maximum:g}"
)
return converted
def _byte_budget(value: Any) -> int:
if (
isinstance(value, bool)
or not isinstance(value, int)
or not 1 <= value <= MAX_OUTPUT_BYTES
):
raise ValueError(
f"max_bytes must be an integer between 1 and {MAX_OUTPUT_BYTES}"
)
return value
def build_environment(
vantage: Vantage, source: Mapping[str, str] | None = None
) -> dict[str, str]:
"""Build a deterministic child environment without caller-controlled PATH."""
origin = os.environ if source is None else source
environment = {name: origin[name] for name in ENV_PASSTHROUGH if name in origin}
environment.update(FIXED_ENVIRONMENT)
environment.update(vantage.env)
environment["PATH"] = SAFE_PATH
environment["GITEA_BASE_URL"] = FIXED_ENVIRONMENT["GITEA_BASE_URL"]
environment["GIT_ASKPASS"] = FIXED_ENVIRONMENT["GIT_ASKPASS"]
return environment
# fmt: off
def operator_vantage(kubeconfig: str | None = None, context: str | None = None) -> Vantage:
inserts = dict.fromkeys(CONTEXT_AWARE_BINARIES, ("--context", context)) if context else {}
detail = " ".join(part for part in (f"context={context}" if context else "",
"kubeconfig-pinned" if kubeconfig else "") if part)
return Vantage(name=OPERATOR, inserts=inserts,
env={"KUBECONFIG": kubeconfig} if kubeconfig else {},
description=f"operator {detail}".strip())
def pod_vantage(namespace: str, pod: str, container: str, operator: Vantage | None = None) -> Vantage:
reach = operator.inserts.get("kubectl", ()) if operator else ()
return Vantage(name=SELF, prefix=("kubectl", *reach, "exec", "--namespace", namespace,
pod, "--container", container, "--"),
env=dict(operator.env) if operator else {}, command_paths=POD_COMMAND_PATHS,
description=f"in-pod {namespace}/{pod}[{container}]")
# fmt: on
def attest_executable(command: str, environment: Mapping[str, str]) -> Attestation:
"""Resolve and hash an executable from a fixed, trusted path set."""
name = command if command == GITEA_CLIENT else command.rsplit("/", 1)[-1]
candidate = (
command if "/" in command else shutil.which(command, path=environment["PATH"])
)
if not candidate:
raise OSError(f"trusted executable is unavailable: {name}")
resolved = str(Path(candidate).resolve(strict=True))
allowed = {
str(Path(item).resolve(strict=False))
for item in EXPECTED_PATHS.get(name, set())
}
if resolved not in allowed:
raise OSError(f"executable resolved outside its trusted paths: {name}")
details = os.stat(resolved)
if (
not stat.S_ISREG(details.st_mode)
or details.st_mode & 0o022
or not details.st_mode & 0o111
):
raise OSError(f"trusted executable has unsafe ownership mode: {name}")
digest = hashlib.sha256()
with open(resolved, "rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
observed = digest.hexdigest()
expected = EXPECTED_SHA256.get(name)
if not expected or observed not in expected:
raise OSError(f"executable digest is not release-attested: {name}")
return Attestation(resolved, observed)
def _append_bounded(buffer: bytearray, chunk: bytes, limit: int) -> bool:
remaining = limit + 1 - len(buffer)
if remaining > 0:
buffer.extend(chunk[:remaining])
return len(buffer) > limit or len(chunk) > remaining
def _signal_group(process: subprocess.Popen[bytes], signum: int) -> None:
with suppress(ProcessLookupError):
os.killpg(process.pid, signum)
def execute_bounded(
argv: Sequence[str],
environment: Mapping[str, str],
timeout: float,
max_bytes: int,
clock: Callable[[], float] = time.monotonic,
popen: Callable[..., subprocess.Popen[bytes]] = subprocess.Popen,
) -> Capture:
"""Execute one process with bounded live pipe capture and group cleanup."""
process = popen(
list(argv),
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=dict(environment),
text=False,
close_fds=True,
start_new_session=True,
)
streams = {"stdout": process.stdout, "stderr": process.stderr}
buffers = {"stdout": bytearray(), "stderr": bytearray()}
truncated = {"stdout": False, "stderr": False}
selector = selectors.DefaultSelector()
for name, stream in streams.items():
if (
stream is not None
): # pragma: no branch - PIPE was requested for both streams
os.set_blocking(stream.fileno(), False)
selector.register(stream, selectors.EVENT_READ, name)
expires = clock() + timeout
timed_out = False
try:
while process.poll() is None:
remaining = expires - clock()
if remaining <= 0:
timed_out = True
break
for key, _ in selector.select(min(remaining, 0.1)):
try:
chunk = os.read(key.fileobj.fileno(), READ_CHUNK)
except BlockingIOError: # pragma: no cover - readiness race, retry only
continue
if not chunk:
selector.unregister(key.fileobj)
continue
name = key.data
truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes)
_signal_group(process, signal.SIGKILL if timed_out else signal.SIGTERM)
drain_until = clock() + PIPE_DRAIN_SECONDS
while selector.get_map() and clock() < drain_until:
for key, _ in selector.select(max(0.0, drain_until - clock())):
try:
chunk = os.read(key.fileobj.fileno(), READ_CHUNK)
except BlockingIOError: # pragma: no cover - readiness race, retry only
continue
if not chunk:
selector.unregister(key.fileobj)
continue
name = key.data
truncated[name] |= _append_bounded(buffers[name], chunk, max_bytes)
_signal_group(process, signal.SIGKILL)
returncode = process.wait(timeout=PIPE_DRAIN_SECONDS)
finally:
selector.close()
for stream in streams.values():
if (
stream is not None
): # pragma: no branch - PIPE was requested for both streams
stream.close()
if (
process.poll() is None
): # pragma: no branch - fail-safe for wait/selector exceptions
_signal_group(process, signal.SIGKILL)
process.wait()
return Capture(
returncode=returncode,
stdout=bytes(buffers["stdout"][:max_bytes]),
stderr=bytes(buffers["stderr"][:max_bytes]),
stdout_truncated=truncated["stdout"],
stderr_truncated=truncated["stderr"],
timed_out=timed_out,
)
def _injected_executor(
spawn: Callable[..., subprocess.CompletedProcess[Any]],
) -> Callable[..., Capture]:
"""Adapt deterministic test doubles without weakening the production path."""
def execute(
argv: Sequence[str],
environment: Mapping[str, str],
timeout: float,
max_bytes: int,
) -> Capture:
completed = spawn(
list(argv),
capture_output=True,
text=True,
timeout=timeout,
check=False,
env=dict(environment),
)
stdout = str(completed.stdout or "").encode()
stderr = str(completed.stderr or "").encode()
return Capture(
completed.returncode,
stdout[:max_bytes],
stderr[:max_bytes],
len(stdout) > max_bytes,
len(stderr) > max_bytes,
)
return execute
def _screen_capture(data: bytes, truncated: bool, budget: int) -> str:
if truncated:
return TRUNCATION_NOTE.encode()[:budget].decode()
return safe_text(data.decode("utf-8", "replace"), budget)[0]
class Runner:
"""Run probes under policy, provenance, timeout, deadline, and byte budgets."""
def __init__(
self,
mode: str = READ_ONLY,
command_timeout: float = DEFAULT_COMMAND_TIMEOUT,
deadline_seconds: float = DEFAULT_RUN_DEADLINE,
max_bytes: int = DEFAULT_MAX_BYTES,
clock: Callable[[], float] = time.monotonic,
executor: Callable[..., Capture] | None = None,
spawn: Callable[..., subprocess.CompletedProcess[Any]] | None = None,
environ: Mapping[str, str] | None = None,
attestor: Callable[[str, Mapping[str, str]], Attestation] = attest_executable,
) -> None:
self.mode = mode
self.command_timeout = _finite_number(
"command_timeout", command_timeout, 0.05, MAX_COMMAND_TIMEOUT
)
deadline = _finite_number(
"deadline_seconds", deadline_seconds, 0.05, MAX_RUN_DEADLINE
)
self.max_bytes = _byte_budget(max_bytes)
self._clock = clock
self._executor = executor or (
_injected_executor(spawn) if spawn else execute_bounded
)
self._environ = environ
self._attestor = attestor
self._deadline_at = clock() + deadline
self.commands_run = 0
@property
def remaining_seconds(self) -> float:
return self._deadline_at - self._clock()
def run(
self,
argv: Sequence[str],
vantage: Vantage,
max_bytes: int | None = None,
reserve_seconds: float = 0.0,
) -> Outcome:
budget_bytes = self.max_bytes if max_bytes is None else _byte_budget(max_bytes)
reserve = _finite_number(
"reserve_seconds", reserve_seconds, 0.0, MAX_RUN_DEADLINE
)
addressed = vantage.wrap(argv)
try:
check_argv(addressed, self.mode)
except PolicyError as exc:
return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}")
budget = min(self.command_timeout, self.remaining_seconds - reserve)
if budget <= 0:
return Outcome(argv=addressed, vantage=vantage.name, error=DEADLINE_ERROR)
environment = build_environment(vantage, self._environ)
try:
attestation = self._attestor(addressed[0], environment)
except (OSError, ValueError) as exc:
return Outcome(
argv=addressed, vantage=vantage.name, error=f"attestation: {exc}"
)
executed = (attestation.path, *addressed[1:])
try:
check_argv(executed, self.mode)
except PolicyError as exc:
return Outcome(argv=addressed, vantage=vantage.name, error=f"policy: {exc}")
started = self._clock()
self.commands_run += 1
try:
capture = self._executor(executed, environment, budget, budget_bytes)
except subprocess.TimeoutExpired:
capture = Capture(-signal.SIGKILL, timed_out=True)
except OSError as exc:
return Outcome(
argv=addressed,
vantage=vantage.name,
error=f"spawn: {exc.strerror or type(exc).__name__}",
executable_path=attestation.path,
executable_sha256=attestation.sha256,
)
was_truncated = capture.stdout_truncated or capture.stderr_truncated
stdout = _screen_capture(capture.stdout, capture.stdout_truncated, budget_bytes)
stderr = _screen_capture(capture.stderr, capture.stderr_truncated, budget_bytes)
return Outcome(
argv=addressed,
vantage=vantage.name,
returncode=capture.returncode,
stdout=stdout,
stderr=stderr,
truncated=was_truncated,
duration_ms=int((self._clock() - started) * 1000),
error=TIMEOUT_ERROR if capture.timed_out else None,
executable_path=attestation.path,
executable_sha256=attestation.sha256,
)

View File

@ -0,0 +1,474 @@
#!/usr/bin/env python3
"""JSON-shaped evaluators for the handoff acceptance harness.
These rules cover the evidence that arrives as structured output — Kubernetes
objects, forge pull requests, provider health files, and the two append-only
evidence logs the platform keeps. They share the fail-closed contract of every
other rule: an absent path, an unparsable payload, or an empty result set is
``NOT_RUN``, never a quiet pass.
"""
from __future__ import annotations
import datetime as dt
import math
import re
from typing import Any
from hermes_handoff_exec import Outcome
from hermes_handoff_model import FAIL, NOT_RUN, PASS, CheckSpec
from hermes_handoff_rules import (
Evaluation,
dotted,
evaluator,
exact_equal,
is_list,
missing_steps,
parsed_step,
strict_json,
)
_ROUTE_PART_RE = re.compile(r"[a-z0-9][a-z0-9_.-]{0,63}\Z")
_FLUX_NAME_RE = re.compile(
r"[a-z0-9]([-a-z0-9.]*[a-z0-9])?/[a-z0-9]([-a-z0-9.]*[a-z0-9])?\Z"
)
def _timestamp(value: Any) -> dt.datetime:
if not isinstance(value, str) or not value.strip():
raise ValueError("timestamp is not a non-empty string")
observed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
if observed.tzinfo is None:
raise ValueError("timestamp has no timezone")
return observed.astimezone(dt.timezone.utc)
def _now(value: Any) -> dt.datetime:
if not isinstance(value, dt.datetime) or value.tzinfo is None:
raise ValueError("run clock is not timezone-aware")
return value.astimezone(dt.timezone.utc)
@evaluator("json_field")
def evaluate_json_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
"""Compare dotted JSON paths of one step's output against expected values."""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
parsed = parsed_step(spec, outcomes)
if isinstance(parsed, Evaluation):
return parsed
payload, key = parsed
evidence: dict[str, Any] = {}
mismatches: list[str] = []
for path, expected in spec.expect["fields"].items():
try:
actual = dotted(payload, path)
except (KeyError, ValueError, IndexError):
return Evaluation(
NOT_RUN, f"path {path} is absent from step {key}", evidence
)
evidence[path] = actual
allowed = expected if isinstance(expected, list) else [expected]
if not any(exact_equal(actual, item) for item in allowed):
mismatches.append(f"{path}={actual!r} expected one of {allowed!r}")
if mismatches:
return Evaluation(FAIL, "; ".join(mismatches), evidence)
return Evaluation(PASS, "observed fields match the release contract", evidence)
@evaluator("json_numeric")
def evaluate_json_numeric(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
"""Compare numeric JSON paths against inclusive minimum/maximum bounds."""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
parsed = parsed_step(spec, outcomes)
if isinstance(parsed, Evaluation):
return parsed
payload, key = parsed
evidence: dict[str, Any] = {}
problems: list[str] = []
for path, bounds in spec.expect["fields"].items():
try:
actual = dotted(payload, path)
except (KeyError, ValueError, IndexError):
return Evaluation(
NOT_RUN, f"path {path} is absent from step {key}", evidence
)
if (
not isinstance(actual, (int, float))
or isinstance(actual, bool)
or not math.isfinite(float(actual))
):
return Evaluation(
NOT_RUN, f"path {path} is not numeric in step {key}", evidence
)
evidence[path] = actual
if "min" in bounds and actual < bounds["min"]:
problems.append(f"{path}={actual} below minimum {bounds['min']}")
if "max" in bounds and actual > bounds["max"]:
problems.append(f"{path}={actual} above maximum {bounds['max']}")
if problems:
return Evaluation(FAIL, "; ".join(problems), evidence)
return Evaluation(PASS, "numeric fields are within the expected bounds", evidence)
@evaluator("json_recent")
def evaluate_json_recent(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
"""Pass when a timestamp field is fresh enough to be evidence about now.
Stale provider or routing evidence is worse than none: it describes a state
the release no longer has. A timestamp older than the allowed age is a FAIL,
and one the harness cannot parse is NOT_RUN.
"""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
parsed = parsed_step(spec, outcomes)
if isinstance(parsed, Evaluation):
return parsed
payload, key = parsed
try:
now = _now(spec.expect["now"])
except (TypeError, ValueError) as exc:
return Evaluation(NOT_RUN, f"invalid run clock: {exc}")
evidence: dict[str, Any] = {}
stale: list[str] = []
for path, max_age in spec.expect["fields"].items():
try:
raw = dotted(payload, path)
except (KeyError, ValueError, IndexError):
return Evaluation(
NOT_RUN, f"path {path} is absent from step {key}", evidence
)
try:
observed = _timestamp(raw)
except (TypeError, ValueError):
return Evaluation(
NOT_RUN, f"path {path} is not a timezone-aware ISO timestamp", evidence
)
age = (now - observed).total_seconds()
evidence[path] = {"age_seconds": int(age)}
if age < 0:
stale.append(f"{path} is {abs(int(age))}s in the future")
elif age > max_age:
stale.append(f"{path} is {int(age)}s old, limit {max_age}s")
if stale:
return Evaluation(FAIL, "; ".join(stale), evidence)
return Evaluation(PASS, "observed timestamps are fresh", evidence)
@evaluator("all_of_field")
def evaluate_all_of_field(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
"""Pass when every selected element of a JSON array matches expected fields."""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
parsed = parsed_step(spec, outcomes)
if isinstance(parsed, Evaluation):
return parsed
payload, key = parsed
try:
elements = dotted(payload, spec.expect.get("array", ""))
except (KeyError, ValueError, IndexError):
return Evaluation(NOT_RUN, f"array path is absent from step {key}")
if not is_list(elements):
return Evaluation(NOT_RUN, f"step {key} did not return an array")
key_field = spec.expect.get("key_field", "")
wanted = spec.expect.get("keys")
exempt = set(spec.expect.get("exempt_keys", ()))
seen: list[Any] = []
problems: list[str] = []
evidence: dict[str, Any] = {}
for element in elements:
identity = dotted(element, key_field) if key_field else None
if wanted is not None and identity not in wanted:
continue
if identity in exempt:
continue
seen.append(identity)
for path in spec.expect.get("non_empty", ()):
try:
value = dotted(element, path)
except (KeyError, ValueError, IndexError):
value = None
if value in (None, "", [], {}):
problems.append(f"{identity}: {path} is empty")
for path, expected in spec.expect.get("fields", {}).items():
try:
actual = dotted(element, path)
except (KeyError, ValueError, IndexError):
actual = None
allowed = expected if isinstance(expected, list) else [expected]
if not any(exact_equal(actual, item) for item in allowed):
problems.append(
f"{identity}: {path}={actual!r} expected one of {allowed!r}"
)
evidence["examined"] = seen[:32]
if wanted is not None:
absent = [item for item in wanted if item not in seen]
if absent:
return Evaluation(
NOT_RUN, f"expected keys absent from step {key}: {absent}", evidence
)
elif not seen:
return Evaluation(NOT_RUN, f"step {key} returned nothing to examine", evidence)
if problems:
return Evaluation(FAIL, "; ".join(problems[:8]), evidence)
return Evaluation(
PASS, f"{len(seen)} element(s) match the release contract", evidence
)
@evaluator("json_record")
def evaluate_json_record(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
"""Capture named JSON paths as evidence; pass when all of them resolve.
Recording is a release requirement in its own right — the runbook has to
name the exact heads it was verified against — so an unreachable source is a
NOT_RUN rather than an empty section.
"""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
parsed = parsed_step(spec, outcomes)
if isinstance(parsed, Evaluation):
return parsed
payload, key = parsed
evidence: dict[str, Any] = {}
for label, path in spec.expect["record"].items():
try:
evidence[label] = dotted(payload, path)
except (KeyError, ValueError, IndexError):
return Evaluation(
NOT_RUN, f"path {path} is absent from step {key}", evidence
)
return Evaluation(PASS, "state recorded", evidence)
@evaluator("routing_evidence")
def evaluate_routing_evidence(
spec: CheckSpec, outcomes: dict[str, Outcome]
) -> Evaluation:
"""Summarise a Switchyard routing log tail into provider/effort coverage.
Route identifiers are ``<lane>/<provider>/<family>/<effort>``, so provider,
model family, effort tier, and lane all come from the same field. Fallback
and failure evidence is a separate ``fallback_reason`` on the record that
was retried. Every row must be complete strict JSON; a clipped or malformed
line makes the evidence unusable rather than being silently discarded.
"""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
key = spec.expect["step"]
outcome = outcomes[key]
if not outcome.ok:
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
records: list[dict[str, Any]] = []
routes: list[list[str]] = []
stamps_by_dimension: dict[tuple[str, str], dt.datetime] = {}
now_value = spec.expect.get("now")
try:
now = _now(now_value) if now_value is not None else None
except (TypeError, ValueError) as exc:
return Evaluation(NOT_RUN, f"invalid routing run clock: {exc}")
for row_number, line in enumerate(outcome.stdout.splitlines(), 1):
line = line.strip()
if not line:
continue
try:
record = strict_json(line)
except ValueError as exc:
return Evaluation(
NOT_RUN, f"routing row {row_number} is malformed JSON: {exc}"
)
if not isinstance(record, dict):
return Evaluation(NOT_RUN, f"routing row {row_number} is not an object")
try:
observed = _timestamp(record.get("ts"))
except (TypeError, ValueError) as exc:
return Evaluation(
NOT_RUN, f"routing row {row_number} has invalid timestamp: {exc}"
)
if now is not None and observed > now:
return Evaluation(FAIL, f"routing row {row_number} has a future timestamp")
model = record.get("model")
if not isinstance(model, str) or not model:
return Evaluation(NOT_RUN, f"routing row {row_number} has no model")
route = model.split("/")
if len(route) == 4 and all(_ROUTE_PART_RE.fullmatch(part) for part in route):
routes.append(route)
for label, value in (
("provider", route[1]),
("effort", route[3]),
("lane", route[0]),
):
key_dimension = (label, value)
stamps_by_dimension[key_dimension] = max(
observed, stamps_by_dimension.get(key_dimension, observed)
)
elif record.get("tier") != "classifier":
return Evaluation(
NOT_RUN, f"routing row {row_number} has malformed route {model!r}"
)
fallback = record.get("fallback_reason")
if fallback is not None and (
not isinstance(fallback, str) or not fallback.strip()
):
return Evaluation(
NOT_RUN, f"routing row {row_number} has malformed fallback_reason"
)
record["_observed"] = observed
records.append(record)
fallbacks = [record for record in records if record.get("fallback_reason")]
stamps = sorted(record["_observed"] for record in records)
evidence: dict[str, Any] = {
"records": len(records),
"routed": len(routes),
"lanes": sorted({route[0] for route in routes}),
"providers": sorted({route[1] for route in routes}),
"families": sorted({route[2] for route in routes}),
"efforts": sorted({route[3] for route in routes}),
"fallback_events": len(fallbacks),
"fallback_reasons": sorted(
{str(record["fallback_reason"]) for record in fallbacks}
),
"latest_ts": stamps[-1].isoformat() if stamps else "",
}
if not records:
return Evaluation(
NOT_RUN, "routing log tail contained no parsable records", evidence
)
missing = [
f"{label}:{item}"
for label, wanted in (
("provider", spec.expect.get("providers", ())),
("effort", spec.expect.get("efforts", ())),
("lane", spec.expect.get("lanes", ())),
)
for item in wanted
if item not in evidence[f"{label}s"]
]
if spec.expect.get("require_fallback_evidence") and not fallbacks:
missing.append("fallback:none-observed")
max_age = spec.expect.get("max_age_seconds")
if max_age and now:
ages: dict[str, int] = {}
for label, wanted in (
("provider", spec.expect.get("providers", ())),
("effort", spec.expect.get("efforts", ())),
("lane", spec.expect.get("lanes", ())),
):
for item in wanted:
stamp = stamps_by_dimension.get((label, item))
if stamp is None:
continue
age = int((now - stamp).total_seconds())
ages[f"{label}:{item}"] = age
if age > max_age:
missing.append(
f"freshness:{label}:{item}:{age}s-exceeds-{max_age}s"
)
evidence["age_seconds"] = ages
if missing:
return Evaluation(
FAIL, f"routing evidence missing {', '.join(missing)}", evidence
)
return Evaluation(PASS, "routing evidence covers the required lanes", evidence)
@evaluator("flux_health")
def evaluate_flux_health(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
"""Classify a compact Flux projection into suspensions and unhealthy objects.
Each row carries name, generation, observed generation, suspension, and all
condition states. A non-suspended object must be current, explicitly
``Ready=True``, not ``Healthy=False/Unknown``, and not reconciling. Suspended
objects are judged against the exact expected-suspension list.
"""
blocked = missing_steps(spec, outcomes)
if blocked:
return blocked
key = spec.expect["step"]
outcome = outcomes[key]
if not outcome.ok:
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
expected = set(spec.expect.get("expected_suspensions", ()))
suspended: list[str] = []
unhealthy: list[str] = []
total = 0
for row_number, line in enumerate(outcome.stdout.splitlines(), 1):
if not line.strip():
continue
columns = line.split("\t")
if len(columns) != 5:
return Evaluation(
NOT_RUN, f"Flux row {row_number} has {len(columns)} columns"
)
name, generation, observed_generation, suspend, conditions = columns
name = name.strip()
if not _FLUX_NAME_RE.fullmatch(name):
return Evaluation(
NOT_RUN, f"Flux row {row_number} has malformed object name"
)
try:
current_generation = int(generation)
reconciled_generation = int(observed_generation)
except ValueError:
return Evaluation(
NOT_RUN, f"Flux row {row_number} has malformed generations"
)
if current_generation <= 0 or current_generation != reconciled_generation:
return Evaluation(
FAIL, f"Flux row {row_number} has stale observedGeneration"
)
if suspend.strip().lower() not in {"", "false", "true"}:
return Evaluation(
NOT_RUN, f"Flux row {row_number} has malformed suspension state"
)
total += 1
if suspend.strip().lower() == "true":
suspended.append(name)
continue
states: dict[str, str] = {}
for pair in filter(None, conditions.strip().strip(",").split(",")):
if pair.count("=") != 1:
return Evaluation(
NOT_RUN, f"Flux row {row_number} has malformed condition"
)
condition, state = pair.split("=", 1)
if (
not condition
or state not in {"True", "False", "Unknown"}
or condition in states
):
return Evaluation(
NOT_RUN, f"Flux row {row_number} has malformed condition state"
)
states[condition] = state
healthy = states.get("Healthy")
if (
states.get("Ready") != "True"
or healthy in {"False", "Unknown"}
or states.get("Reconciling") == "True"
):
unhealthy.append(name)
unexpected = sorted(set(suspended) - expected)
evidence = {
"total": total,
"suspended": sorted(suspended),
"unexpected_suspensions": unexpected,
"unhealthy": sorted(unhealthy),
}
if not total:
return Evaluation(NOT_RUN, f"step {key} returned no Flux objects", evidence)
problems = []
if unexpected:
problems.append(f"unexpected suspensions: {', '.join(unexpected)}")
if unhealthy:
problems.append(f"unhealthy: {', '.join(sorted(unhealthy)[:8])}")
if problems:
return Evaluation(FAIL, "; ".join(problems), evidence)
return Evaluation(PASS, f"{total} Flux objects reconciled as expected", evidence)

View File

@ -0,0 +1,290 @@
#!/usr/bin/env python3
"""Result model, aggregation, and rendering for the handoff acceptance harness.
The classification is deliberately four-valued. ``NOT_APPLICABLE`` is the only
way a check can be excused, and a check can only reach it by matching a scope
rule that a reviewer wrote down; anything the harness merely failed to observe
is ``NOT_RUN``. Because a mandatory ``NOT_RUN`` is a NO_GO exactly like a
``FAIL``, an unreachable cluster, an exhausted deadline, or a probe nobody
maintained can never round up to a release.
"""
from __future__ import annotations
import datetime as dt
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from typing import Any
from hermes_handoff_exec import Outcome
from hermes_handoff_redaction import contains_credential_shape, safe_text, scrub
PASS = "PASS"
FAIL = "FAIL"
NOT_RUN = "NOT_RUN"
NOT_APPLICABLE = "NOT_APPLICABLE"
STATUSES = (PASS, FAIL, NOT_RUN, NOT_APPLICABLE)
GO = "GO"
NO_GO = "NO_GO"
ALWAYS = "always"
EPHEMERAL = "ephemeral"
HARNESS = "hermes-handoff-acceptance"
SCHEMA_VERSION = 1
REVIEW = "review"
ATTEMPT = "attempt"
READ = "read"
STEP_KINDS = (REVIEW, ATTEMPT, READ)
@dataclass(frozen=True)
class Step:
"""One command a check needs, and the vantage it must be observed from.
``kind`` separates a self access review (``review``) from a real request the
identity is expected to be refused (``attempt``). A deny check needs both:
an authorization review states policy, only an attempt proves the API server
enforces it.
``record`` is cleared for bulk evidence — a routing log tail, a large object
listing — which the harness parses but must not paste into the artifact.
``max_bytes`` raises the capture budget for that same bulk evidence, where a
default-sized truncation would silently narrow the window being examined.
"""
key: str
vantage: str
argv: tuple[str, ...]
kind: str = READ
optional: bool = False
record: bool = True
max_bytes: int | None = None
@dataclass(frozen=True)
class CheckSpec:
"""A declarative acceptance check.
``rule`` names an evaluator; ``expect`` carries its parameters. Keeping the
catalog declarative means a reviewer reads what is asserted, not how it is
plumbed, and the harness can prove structural properties over every check
(no impersonation, no mutation, both vantages present) before running any.
"""
id: str
title: str
group: str
rule: str
steps: tuple[Step, ...] = ()
expect: Mapping[str, Any] = field(default_factory=dict)
mandatory: bool = True
scope: str = ALWAYS
rationale: str = ""
@property
def vantages(self) -> tuple[str, ...]:
"""Return the distinct vantages this check draws evidence from."""
seen: list[str] = []
for step in self.steps:
if step.vantage not in seen:
seen.append(step.vantage)
return tuple(seen)
@dataclass
class CheckResult:
"""The classified outcome of one acceptance check."""
spec: CheckSpec
status: str
reason: str = ""
evidence: dict[str, Any] = field(default_factory=dict)
outcomes: list[Outcome] = field(default_factory=list)
@property
def blocking(self) -> bool:
"""Report whether this result alone forces NO_GO."""
if self.status not in STATUSES:
return True
return self.spec.mandatory and self.status in {FAIL, NOT_RUN}
def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]:
"""Return a JSON-serialisable, screened view of the result."""
payload: dict[str, Any] = {
"id": self.spec.id,
"title": self.spec.title,
"group": self.spec.group,
"rule": self.spec.rule,
"scope": self.spec.scope,
"mandatory": self.spec.mandatory,
"vantages": list(self.spec.vantages),
"status": self.status,
"reason": self.reason,
"blocking": self.blocking,
"evidence": self.evidence,
}
if self.spec.rationale:
payload["rationale"] = self.spec.rationale
if include_outcomes:
payload["commands"] = [outcome.as_dict() for outcome in self.outcomes]
return scrub(payload)
@dataclass
class VantageRecord:
"""What the harness proved about one vantage before trusting its evidence."""
name: str
description: str
identity: str = ""
available: bool = False
detail: str = ""
def as_dict(self) -> dict[str, Any]:
"""Return a JSON-serialisable view of the vantage."""
return scrub(
{
"name": self.name,
"description": self.description,
"identity": self.identity,
"available": self.available,
"detail": self.detail,
}
)
@dataclass
class Report:
"""The whole acceptance run: metadata, per-check results, and a verdict."""
mode: str
started_at: str
baseline: dict[str, Any] = field(default_factory=dict)
vantages: list[VantageRecord] = field(default_factory=list)
results: list[CheckResult] = field(default_factory=list)
harness_errors: list[str] = field(default_factory=list)
finished_at: str = ""
@property
def counts(self) -> dict[str, int]:
"""Return the number of results in each status."""
tally = dict.fromkeys(STATUSES, 0)
for result in self.results:
if result.status in tally:
tally[result.status] += 1
return tally
@property
def invalid_statuses(self) -> list[str]:
"""Return check ids whose status is outside the report schema."""
return [
result.spec.id for result in self.results if result.status not in STATUSES
]
@property
def blocking(self) -> list[CheckResult]:
"""Return every result that forces NO_GO, most severe first."""
order = {FAIL: 0, NOT_RUN: 1}
return sorted(
(result for result in self.results if result.blocking),
key=lambda result: (order.get(result.status, 2), result.spec.id),
)
@property
def decision(self) -> str:
"""Return GO only when nothing mandatory failed, skipped, or errored."""
if (
not self.results
or self.invalid_statuses
or self.harness_errors
or self.blocking
):
return NO_GO
return GO
def group_summary(self) -> dict[str, dict[str, int]]:
"""Return per-group status counts, for the human summary."""
groups: dict[str, dict[str, int]] = {}
for result in self.results:
bucket = groups.setdefault(result.spec.group, dict.fromkeys(STATUSES, 0))
bucket[result.status] = bucket.get(result.status, 0) + 1
return groups
def as_dict(self, include_outcomes: bool = True) -> dict[str, Any]:
"""Return the machine-readable report."""
return scrub(
{
"harness": HARNESS,
"schema_version": SCHEMA_VERSION,
"mode": self.mode,
"started_at": self.started_at,
"finished_at": self.finished_at,
"decision": self.decision,
"counts": self.counts,
"baseline": scrub(self.baseline),
"vantages": [vantage.as_dict() for vantage in self.vantages],
"harness_errors": [scrub(error) for error in self.harness_errors],
"invalid_statuses": self.invalid_statuses,
"blocking": [result.spec.id for result in self.blocking],
"checks": [result.as_dict(include_outcomes) for result in self.results],
}
)
def render_summary(self, width: int = 88) -> str:
"""Return the concise human summary printed alongside the JSON."""
counts = self.counts
lines = [
f"{HARNESS}: {self.decision}",
f" mode={self.mode} started={self.started_at} checks={len(self.results)}",
" " + " ".join(f"{status}={counts[status]}" for status in STATUSES),
]
for vantage in self.vantages:
state = "available" if vantage.available else "unavailable"
identity = vantage.identity or "unknown"
lines.append(f" vantage {vantage.name}: {state} identity={identity}")
for error in self.harness_errors:
lines.append(f" harness error: {error}")
if self.blocking:
lines.append(" blocking:")
for result in self.blocking:
lines.append(f" {result.status:<15}{result.spec.id}")
lines.append(f" {'':<15}{result.reason[:width]}")
groups = self.group_summary()
lines.append(" groups:")
for name in sorted(groups):
bucket = groups[name]
detail = " ".join(
f"{status}={bucket[status]}" for status in STATUSES if bucket[status]
)
lines.append(f" {name:<18}{detail}")
return safe_text("\n".join(lines), 32 * 1024)[0]
def utc_now(clock: Any = None) -> str:
"""Return an ISO-8601 UTC timestamp, injectable for deterministic tests."""
now = clock() if clock else dt.datetime.now(dt.timezone.utc)
return now.replace(microsecond=0).isoformat().replace("+00:00", "Z")
def unscreened_fields(payload: Any, path: str = "$") -> list[str]:
"""Return JSON pointers whose text still looks credential-shaped.
The report is already screened field by field; this is the invariant check
that runs before anything is written, so a screening gap fails the run
instead of publishing an artifact nobody re-read.
"""
offenders: list[str] = []
if isinstance(payload, str):
if contains_credential_shape(payload):
offenders.append(path)
elif isinstance(payload, Mapping):
for key, value in payload.items():
offenders.extend(unscreened_fields(value, f"{path}.{key}"))
elif isinstance(payload, Sequence) and not isinstance(payload, (str, bytes)):
for index, value in enumerate(payload):
offenders.extend(unscreened_fields(value, f"{path}[{index}]"))
return offenders

View File

@ -0,0 +1,493 @@
#!/usr/bin/env python3
"""Fail-closed command policy for the Hermes handoff acceptance harness.
Before spawn it admits only credential-safe, structurally read-only grammar;
shell is limited to frozen templates with absolute tool paths.
Impersonation is structurally impossible: ``--as`` and its siblings are refused
for every argv, in every mode, from every vantage, and the inner command of a
``kubectl exec`` is re-checked under the same rule rather than being trusted.
Only binaries with a pinned release digest are on the allowlist, so nothing can
be admitted here that the executor would then refuse to attest.
"""
from __future__ import annotations
import re
from collections.abc import Iterator
from urllib.parse import unquote
READ_ONLY = "read-only"
ARMED = "ephemeral-armed"
MODES = (READ_ONLY, ARMED)
GITEA_CLIENT = "/opt/coordinator/gitea_api.py"
EXPECTED_REPO = "atlas/titan-iac"
EXPECTED_REMOTE = "origin"
EXPECTED_BASE = "main"
# fmt: off
FORBIDDEN_ARGS = ("--as", "--as-group", "--as-uid", "--client-certificate",
"--client-key", "--password", "--token", "--username")
# fmt: on
IMPERSONATION_ARGS = ("--as", "--as-group", "--as-uid")
FORBIDDEN_BINARIES = frozenset(
{"age", "curl", "gpg", "openssl", "sops", "vault", "wget"}
)
ALLOWED_BINARIES = frozenset({"kubectl", "git", "hermes", "sh", GITEA_CLIENT})
TRUSTED_EXECUTABLE_ROOTS = (
"/bin/",
"/usr/bin/",
"/usr/local/bin/",
"/opt/data/tools/bin/",
"/opt/hermes/.venv/bin/",
"/opt/coordinator/",
)
TRUSTED_INNER_PATHS = {
"/bin/sh": "sh",
"/usr/bin/git": "git",
"/usr/bin/sh": "sh",
"/usr/local/bin/kubectl": "kubectl",
"/opt/hermes/.venv/bin/hermes": "hermes",
GITEA_CLIENT: GITEA_CLIENT,
}
VALUE_FLAGS = frozenset(
{
"-C",
"-c",
"-l",
"-n",
"-o",
"--cluster",
"--container",
"--context",
"--field-selector",
"--kubeconfig",
"--namespace",
"--output",
"--request-timeout",
"--selector",
"--server",
"--since",
"--tail",
"--user",
}
)
KUBECTL_READ_SUBCOMMANDS = frozenset(
{
"api-resources",
"api-versions",
"explain",
"get",
"top",
"version",
}
)
KUBECTL_EXEC_SUBCOMMANDS = frozenset({"exec"})
KUBECTL_SUBCOMMANDS = KUBECTL_READ_SUBCOMMANDS | KUBECTL_EXEC_SUBCOMMANDS | {"auth"}
SECRET_RESOURCES = frozenset({"secret", "secrets"})
UNSAFE_PROJECTION_PARTS = (
".args",
".command",
".data",
".envfrom",
".secretkeyref",
".status.token",
".stringdata",
".valuefrom",
"clientconfig",
"password",
"privatekey",
)
SAFE_ENV_VALUE = 'env[?(@.name=="hermes_auto_router_profile")].value'
SAFE_JSON_FIELDS = frozenset(["authenticated", "transport", "state", "model", "effort", "latency_ms", "checked_at", "api_provider", "auth_method", "subscription_type"]) # fmt: skip
GIT_READ_SUBCOMMANDS = frozenset(
{
"cat-file",
"diff",
"for-each-ref",
"log",
"ls-remote",
"ls-tree",
"merge-base",
"rev-list",
"rev-parse",
"show",
"status",
}
)
GIT_FORCE_FLAGS = frozenset(
{"-f", "--force", "--force-with-lease", "--mirror", "--all", "--tags"}
)
GIT_UNSAFE_FLAGS = (
"-c",
"--config-env",
"--exec-path",
"--upload-pack",
"--receive-pack",
)
GITEA_READ_METHODS = frozenset({"GET"})
HERMES_READ_COMMANDS = frozenset(
{("kanban", "list"), ("kanban", "show"), ("sessions", "list"), ("status",)}
)
# These locations contain credential bytes even when a caller claims it will
# only inspect metadata. They are rejected for every argv, including a frozen
# shell rendering and the inner command of kubectl exec.
SECRET_PATH_RE = re.compile(
r"(?:^|[\s'\"=])(?:/[^\s'\"]*)?(?:"
r"var/run/secrets|run/secrets|vault/secrets|runtime-access|scm-access|pool-access|"
r"\.credentials\.json|auth\.json|\.git-credentials|authorized_keys|"
r"id_[a-z0-9]+|\.netrc|shadow|gshadow|k3s\.yaml"
r")(?:$|[/\s'\"])",
re.IGNORECASE,
)
_PARAMETER_RE = re.compile(r"[A-Za-z0-9/][A-Za-z0-9._/@:,-]{0,255}\Z")
_PLACEHOLDER_RE = re.compile(r"\{([a-z_]+)\}")
_SAFE_API_PATH_RE = re.compile(r"/[A-Za-z0-9][A-Za-z0-9._~:/?&=%+-]{0,511}\Z")
SHELL_TEMPLATES: dict[str, str] = {
"env_names": (
"/usr/bin/env | /usr/bin/sed -n "
"'s/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/\\1/p' | /usr/bin/sort"
),
"json_fields": (
'/opt/hermes/.venv/bin/python -c "import json,sys;d=json.load(open(sys.argv[1]));'
'print(json.dumps({{k:d.get(k) for k in sys.argv[2].split(chr(44))}}))" '
"{path} {fields}"
),
"tail_lines": "/usr/bin/tail -n {limit} {path}",
}
class PolicyError(ValueError):
"""Raised when a requested command falls outside the safe boundary."""
def template_parameters(template_name: str) -> set[str]:
"""Return the parameter names a frozen template requires."""
template = SHELL_TEMPLATES.get(template_name)
if template is None:
raise PolicyError(f"unknown shell template: {template_name}")
return set(_PLACEHOLDER_RE.findall(template))
def render_shell(template_name: str, **parameters: str) -> str:
"""Return a frozen shell template with validated parameters substituted."""
required = template_parameters(template_name)
if set(parameters) != required:
raise PolicyError(
f"shell template {template_name} expects parameters {sorted(required)}"
)
for key, value in parameters.items():
if not _PARAMETER_RE.fullmatch(value):
raise PolicyError(f"unsafe shell parameter for {template_name}.{key}")
if SECRET_PATH_RE.search(value):
raise PolicyError("shell templates may not name credential paths")
if (
template_name == "json_fields"
and not set(parameters["fields"].split(",")) <= SAFE_JSON_FIELDS
):
raise PolicyError("json_fields requested a credential-bearing or unknown field")
return SHELL_TEMPLATES[template_name].format(**parameters)
_RENDERED: set[str] = set()
_APPROVED_PROJECTIONS: set[str] = set()
_ARMED_REF = ""
_ARMED_PULLS: set[int] = set()
# fmt: off
def arm_ephemeral_policy(ref: str) -> None:
global _ARMED_REF
if not re.fullmatch(r"ephemeral/hermes-handoff-acceptance/[a-z0-9][a-z0-9-]{7,63}", ref):
raise PolicyError("armed policy requires one exact ephemeral ref")
_ARMED_REF = ref
_ARMED_PULLS.clear()
def register_ephemeral_pull(number: int) -> None:
if not _ARMED_REF or isinstance(number, bool) or not isinstance(number, int) or number <= 0:
raise PolicyError("ephemeral pull registration is malformed")
_ARMED_PULLS.add(number)
def shell(template_name: str, **parameters: str) -> tuple[str, ...]:
"""Return a validated ``sh -c`` argv built from a frozen template."""
script = render_shell(template_name, **parameters)
_RENDERED.add(script)
return ("sh", "-c", script)
def projection(value: str) -> str:
normalized = value.lower().replace(" ", "")
if not normalized.startswith("jsonpath=") or any(part in normalized for part in UNSAFE_PROJECTION_PARTS):
raise PolicyError("Kubernetes projection is not credential-safe")
if ".metadata.annotations" in normalized and not any(key in normalized for key in (
"deployment\\.kubernetes\\.io/revision", "ai\\.bstein\\.dev/config-rev")):
raise PolicyError("Kubernetes annotations are not on the reviewed allowlist")
if ".metadata.labels" in normalized and "pod-template-hash" not in normalized:
raise PolicyError("Kubernetes labels are not on the reviewed allowlist")
if ".env" in normalized and ".value" in normalized and SAFE_ENV_VALUE not in normalized:
raise PolicyError("Kubernetes projection may expose environment values")
_APPROVED_PROJECTIONS.add(normalized)
return value
# fmt: on
def positionals(argv: tuple[str, ...], start: int = 1) -> Iterator[str]:
"""Yield positional arguments, skipping flags and their values."""
index = start
while index < len(argv):
item = argv[index]
if item == "--":
return
if item.startswith("-"):
index += 2 if item in VALUE_FLAGS else 1
continue
yield item
index += 1
def _binary_name(value: str) -> str:
if value == GITEA_CLIENT or (
value.startswith("/opt/coordinator/") and value.endswith("/gitea_api.py")
):
return GITEA_CLIENT
if "/" not in value:
return value
mapped = TRUSTED_INNER_PATHS.get(value)
if mapped:
return mapped
if "/../" in value or value.endswith("/..") or "/./" in value:
raise PolicyError(f"executable path is not normalized: {value}")
if not value.startswith(TRUSTED_EXECUTABLE_ROOTS):
raise PolicyError(f"executable path is not trusted: {value}")
return value.rsplit("/", 1)[-1]
def _reject_forbidden_arguments(argv: tuple[str, ...]) -> None:
"""Refuse credential and impersonation arguments up to the ``--`` boundary.
Everything after ``--`` is the inner command of a ``kubectl exec``, which
:func:`_check_kubectl` submits to :func:`check_argv` again, so it is held to
exactly this rule rather than being exempted from it.
"""
boundary = argv.index("--") if "--" in argv else len(argv)
for argument in argv[:boundary]:
for forbidden in FORBIDDEN_ARGS:
if argument == forbidden or argument.startswith(f"{forbidden}="):
raise PolicyError(f"forbidden argument: {forbidden}")
if argument == "--raw" or argument.startswith("--raw="):
raise PolicyError("raw API reads are forbidden")
for argument in argv:
if argument == "--dry-run" or argument.startswith("--dry-run="):
raise PolicyError("dry-run mutation commands are forbidden")
def _reject_secret_paths(argv: tuple[str, ...]) -> None:
if any(SECRET_PATH_RE.search(argument) for argument in argv):
raise PolicyError("probes may not name a credential path")
def _resource_name(value: str) -> str:
return value.lower().split("/", 1)[0].split(".", 1)[0]
def _output_value(argv: tuple[str, ...]) -> str:
"""Return kubectl's output argument in a single canonical form."""
for index, item in enumerate(argv[1:], 1):
if item in {"-o", "--output"} and index + 1 < len(argv):
return argv[index + 1]
if item.startswith(("-o=", "--output=")):
return item.split("=", 1)[1]
return ""
def _check_projection(output: str) -> None:
"""Reject full or credential-bearing Kubernetes output projections."""
lowered = output.lower().replace(" ", "")
if lowered in {"json", "yaml", "wide"} or lowered.startswith("custom-columns"):
raise PolicyError(
"full Kubernetes object output is forbidden; use a safe projection"
)
if lowered and not (lowered == "name" or lowered.startswith("jsonpath=")):
raise PolicyError(
"Kubernetes output must be name or an approved JSONPath projection"
)
if lowered.startswith("jsonpath="):
if any(part in lowered for part in UNSAFE_PROJECTION_PARTS):
raise PolicyError("Kubernetes projection may expose runtime credentials")
if ".env" in lowered and SAFE_ENV_VALUE not in lowered:
# Environment names are safe; arbitrary `.value` reads are not.
without_names = lowered.replace("{.name}", "")
if ".value" in without_names:
raise PolicyError("Kubernetes projection may expose environment values")
if lowered not in _APPROVED_PROJECTIONS:
raise PolicyError(
"Kubernetes JSONPath projection is not on the reviewed allowlist"
)
def _check_kubectl(argv: tuple[str, ...], mode: str) -> None:
words = list(positionals(argv))
if not words or words[0] not in KUBECTL_SUBCOMMANDS:
raise PolicyError("kubectl subcommand is outside the read-only boundary")
subcommand = words[0]
if subcommand == "auth":
if len(words) < 2 or words[1] not in {"can-i", "whoami"}:
raise PolicyError("kubectl auth is limited to can-i and whoami")
return
if subcommand == "exec":
if "--" not in argv:
raise PolicyError("kubectl exec requires an explicit -- command boundary")
inner = argv[argv.index("--") + 1 :]
if not inner:
raise PolicyError("kubectl exec requires an inner command")
if inner[0] not in TRUSTED_INNER_PATHS:
raise PolicyError(
"kubectl exec inner command must use a fixed trusted path"
)
check_argv(inner, mode)
return
resources = words[1].split(",") if len(words) > 1 else []
if any(_resource_name(resource) in SECRET_RESOURCES for resource in resources):
raise PolicyError("Secret objects may not be read by the harness")
output = _output_value(argv)
if subcommand == "get" and not output:
raise PolicyError("kubectl get requires a reviewed output projection")
_check_projection(output)
def _check_git(argv: tuple[str, ...], mode: str) -> None:
unsafe = False
for item in argv[1:]:
if item == "-C" or item.startswith(("--git-dir", "--work-tree", "--namespace")):
unsafe = True
if item == "-c" or (item.startswith("-c") and len(item) > 2):
unsafe = True
if any(
item == flag or item.startswith(f"{flag}=") for flag in GIT_UNSAFE_FLAGS[1:]
):
unsafe = True
if unsafe:
raise PolicyError("git configuration and helper overrides are forbidden")
subcommand = next(positionals(argv), None)
if subcommand is None:
raise PolicyError("git requires a subcommand")
if subcommand in GIT_READ_SUBCOMMANDS:
return
if subcommand == "push" and mode == ARMED:
if GIT_FORCE_FLAGS & set(argv):
raise PolicyError("git push may not force or fan out across refs")
words = list(positionals(argv))
if len(words) != 3 or words[1] != EXPECTED_REMOTE:
raise PolicyError("armed git push is pinned to origin and one refspec")
destination = words[2].partition(":")[2]
if not _ARMED_REF or destination != f"refs/heads/{_ARMED_REF}":
raise PolicyError("armed git push is not the preflighted ephemeral ref")
return
raise PolicyError(f"git {subcommand} is not permitted in {mode} mode")
def _check_hermes(argv: tuple[str, ...]) -> None:
words = tuple(positionals(argv))
if not words:
raise PolicyError("hermes requires a subcommand")
if words[:2] not in HERMES_READ_COMMANDS and words[:1] not in HERMES_READ_COMMANDS:
raise PolicyError(
f"hermes {' '.join(words[:2])} is outside the read-only boundary"
)
def _reject_dot_segments(path: str) -> None:
"""Refuse any relative segment, including a percent-encoded one.
The repository pin below is a textual prefix test, so a single ``..`` would
let a pinned path address another repository or the token endpoints once a
router normalised it. Decode first: ``%2e%2e`` is the same segment.
"""
decoded = unquote(unquote(path)).split("?", 1)[0]
if any(segment in {"", ".", ".."} for segment in decoded.split("/")[1:]):
raise PolicyError("gitea_api.py paths may not contain relative segments")
def _check_gitea(argv: tuple[str, ...], mode: str) -> None:
if len(argv) < 3:
raise PolicyError("gitea_api.py requires a method and a path")
method, path = argv[1].upper(), argv[2]
if not path.startswith("/api/v1/") or not _SAFE_API_PATH_RE.fullmatch(path):
raise PolicyError("gitea_api.py requires a bounded API path")
_reject_dot_segments(path)
repo_root = f"/api/v1/repos/{EXPECTED_REPO}"
if (
path != "/api/v1/user"
and path != repo_root
and not path.startswith((f"{repo_root}/", f"{repo_root}?"))
):
raise PolicyError("gitea_api.py is pinned to the Atlas titan-iac repository")
if method in GITEA_READ_METHODS:
return
if mode == ARMED:
fields = [
argv[index + 1] for index, item in enumerate(argv[:-1]) if item == "--field"
]
if method == "POST" and path == f"{repo_root}/pulls" and _ARMED_REF:
required = {
"title=WIP: Hermes handoff acceptance ephemeral probe",
f"head={_ARMED_REF}",
f"base={EXPECTED_BASE}",
"body=Ephemeral acceptance probe. Closed and deleted by the harness.",
}
if len(fields) == len(required) and set(fields) == required:
return
match = re.fullmatch(rf"{re.escape(repo_root)}/pulls/([1-9][0-9]*)", path)
if (
method == "PATCH"
and match
and int(match.group(1)) in _ARMED_PULLS
and fields == ["state=closed"]
):
return
if (
method == "DELETE"
and _ARMED_REF
and path == f"{repo_root}/branches/{_ARMED_REF}"
and len(argv) == 3
):
return
raise PolicyError(f"gitea_api.py {method} is not permitted in {mode} mode")
def _check_shell(argv: tuple[str, ...]) -> None:
if len(argv) != 3 or argv[1] != "-c" or argv[2] not in _RENDERED:
raise PolicyError("sh is only reachable as sh -c <frozen template>")
def check_argv(argv: tuple[str, ...], mode: str = READ_ONLY) -> None:
"""Raise :class:`PolicyError` unless ``argv`` is safe in ``mode``."""
if mode not in MODES:
raise PolicyError(f"unknown mode: {mode}")
if not argv:
raise PolicyError("empty command")
binary = _binary_name(argv[0])
if binary in FORBIDDEN_BINARIES:
raise PolicyError(f"forbidden binary: {binary}")
if binary not in ALLOWED_BINARIES:
raise PolicyError(f"binary is not on the harness allowlist: {binary}")
_reject_forbidden_arguments(argv)
_reject_secret_paths(argv)
if binary == "sh":
_check_shell(argv)
elif binary == "kubectl":
_check_kubectl(argv, mode)
elif binary == "git":
_check_git(argv, mode)
elif binary == "hermes":
_check_hermes(argv)
else:
_check_gitea(argv, mode)

View File

@ -0,0 +1,243 @@
#!/usr/bin/env python3
"""Credential redaction and output bounding for the handoff acceptance harness.
Every string the harness records — command output, failure reasons, evidence
values — passes through :func:`redact` before it reaches a report. Screening is
a fail-closed accident barrier, not proof that text is secret-free: a short
secret under an innocuous key is not reliably distinguishable from prose. The
harness therefore also never asks for a secret in the first place. Probes read
names, shapes, and status fields; they never read environment values, token
files, private keys, cookies, or Vault responses.
Bounding is part of the same barrier. Screening always precedes truncation so a
credential that crosses the byte boundary cannot leave a useful prefix behind.
The process runner additionally discards all captured bytes when a live stream
exceeds its cap; the fixed truncation marker is the only recorded output.
"""
from __future__ import annotations
import re
from typing import Any
REDACTED = "[redacted]"
DEFAULT_MAX_BYTES = 64 * 1024
TRUNCATION_NOTE = "[truncated]"
# Keys whose value is never safe to record, matched against a normalised
# (lowercased, separator-stripped) form of the key.
SENSITIVE_KEY_SUFFIXES = (
"accesskey",
"accesskeyid",
"apikey",
"authtoken",
"clientsecret",
"connectionstring",
"cookie",
"credential",
"credentials",
"passphrase",
"passwd",
"password",
"privatekey",
"secret",
"secretkey",
"sessionkey",
"signature",
"signingkey",
"token",
"webhook",
)
SENSITIVE_KEY_EXACT = frozenset(
{
"auth",
"authorization",
"bearer",
"pat",
"pw",
"sas",
"setcookie",
}
)
# A bare `key:` is a structural field name across Kubernetes and TOML far more
# often than it is a credential, so it is screened by value shape and by the
# token patterns rather than by name. The compound forms above still match.
# Ordinary prose values shorter than this threshold are not independently
# treated as secret-shaped. Values under sensitive keys are screened at every
# length, including numeric values.
MIN_SECRET_LENGTH = 8
NON_SECRET_LITERALS = frozenset(
{"true", "false", "null", "none", "yes", "no", "0", "1", ""}
)
_KEY_NOISE_RE = re.compile(r"[^a-z0-9]+")
# Ordered: structural matches first so a nested token is not partially rewritten
# by a looser rule.
_PATTERNS: tuple[re.Pattern[str], ...] = (
re.compile(
r"-----BEGIN[ A-Z0-9-]{0,64}-----.*?-----END[ A-Z0-9-]{0,64}-----",
re.DOTALL,
),
re.compile(r"-----BEGIN[ A-Z0-9-]{0,64}-----.*", re.DOTALL),
re.compile(r"(?i)\b(?:authorization|proxy-authorization)\s*:\s*\S.*"),
re.compile(r"(?i)\b(?:set-)?cookie\s*:\s*\S.*"),
re.compile(r"(?i)\b(?:bearer|basic|token)\s+[A-Za-z0-9+/_=.:~-]{8,}"),
re.compile(r"(?i)\b[a-z][a-z0-9+.-]{1,20}://[^\s/@:]{1,128}:[^\s/@]{1,256}@"),
re.compile(
r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}"
),
re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}"),
re.compile(r"(?<![A-Za-z0-9])gh[pousr]_[A-Za-z0-9]*"),
re.compile(r"(?<![A-Za-z0-9])github_pat_[A-Za-z0-9_]*"),
re.compile(r"(?<![A-Za-z0-9])glpat-[A-Za-z0-9_-]*"),
re.compile(r"(?<![A-Za-z0-9])xox[baprs]-[A-Za-z0-9-]*"),
re.compile(r"(?<![A-Za-z0-9])sk-(?:ant-|proj-)?[A-Za-z0-9_-]*"),
re.compile(r"(?<![A-Za-z0-9])(?:gta|gto|gitea|forgejo)_[A-Za-z0-9_-]*"),
re.compile(r"(?<![A-Za-z0-9])hvs\.[A-Za-z0-9_-]*"),
re.compile(r"(?<![A-Za-z0-9])(?:AKIA|ASIA)[0-9A-Z]*"),
re.compile(r"(?<![A-Za-z0-9])AIza[0-9A-Za-z_-]{35}"),
re.compile(r"(?<![A-Za-z0-9])npm_[A-Za-z0-9]{16,}"),
)
# `key: value` / `key=value` where the key names a credential.
_ASSIGNMENT_RE = re.compile(
r"""(?x)
(?P<quote>["']?)
(?P<key>[A-Za-z][A-Za-z0-9_.\ -]{0,63})
(?P=quote)
(?P<gap>\s{0,8}[:=]\s{0,8})
(?P<value>"[^"\n]{0,4096}"|'[^'\n]{0,4096}'|[^\s,;}\n][^,;}\n]{0,4096})
"""
)
# Long, mixed-class runs. Deliberately conservative: prose and hex digests of
# fewer than 44 characters are left alone so commit SHAs stay readable.
_ENTROPY_RE = re.compile(
r"(?<![A-Za-z0-9+/_=-])[A-Za-z0-9+/_=-]{44,}(?![A-Za-z0-9+/_=-])"
)
def normalise_key(key: str) -> str:
"""Return a key lowercased with separators removed, for suffix matching."""
return _KEY_NOISE_RE.sub("", key.lower())
def is_sensitive_key(key: str) -> bool:
"""Report whether a key name means its value must never be recorded."""
compact = normalise_key(key)
if not compact:
return False
if compact in SENSITIVE_KEY_EXACT:
return True
return compact.endswith(SENSITIVE_KEY_SUFFIXES)
def _is_high_entropy(candidate: str) -> bool:
"""Report whether a long run mixes enough character classes to be a token."""
classes = sum(
(
any(character.islower() for character in candidate),
any(character.isupper() for character in candidate),
any(character.isdigit() for character in candidate),
any(character in "+/_=-" for character in candidate),
)
)
return classes >= 4
def could_hold_secret(value: str) -> bool:
"""Report whether a raw value is long and opaque enough to be a credential."""
stripped = value.strip().strip("\"'")
if stripped.lower() in NON_SECRET_LITERALS or len(stripped) < MIN_SECRET_LENGTH:
return False
try:
float(stripped)
except ValueError:
return True
return False
def _redact_assignment(match: re.Match[str]) -> str:
value = match.group("value")
stripped = value.strip().strip("\"'").lower()
if not is_sensitive_key(match.group("key")) or stripped in NON_SECRET_LITERALS:
return match.group(0)
# Preserve the original quoting so screened JSON stays parsable JSON.
quoted = value.strip()[:1] in {'"', "'"}
replacement = f'"{REDACTED}"' if quoted else REDACTED
key_quote = match.group("quote")
return (
f"{key_quote}{match.group('key')}{key_quote}{match.group('gap')}{replacement}"
)
def _redact_entropy(match: re.Match[str]) -> str:
return REDACTED if _is_high_entropy(match.group(0)) else match.group(0)
def redact(text: str) -> str:
"""Return ``text`` with credential-shaped substrings replaced."""
if not text:
return text
for pattern in _PATTERNS:
text = pattern.sub(REDACTED, text)
text = _ASSIGNMENT_RE.sub(_redact_assignment, text)
return _ENTROPY_RE.sub(_redact_entropy, text)
def bound(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]:
"""Truncate ``text`` to a UTF-8 byte budget, reporting whether it was cut."""
if max_bytes <= 0:
return ("", bool(text))
encoded = text.encode("utf-8", "replace")
if len(encoded) <= max_bytes:
return (text, False)
marker = TRUNCATION_NOTE.encode()[:max_bytes]
content_budget = max(
0, max_bytes - len(marker) - (1 if len(marker) < max_bytes else 0)
)
kept = encoded[:content_budget].decode("utf-8", "ignore")
separator = "\n" if kept and len(marker) < max_bytes else ""
return (f"{kept}{separator}{marker.decode()}", True)
def safe_text(text: str, max_bytes: int = DEFAULT_MAX_BYTES) -> tuple[str, bool]:
"""Redact then bound, so truncation never exposes a credential prefix."""
return bound(redact(text), max_bytes)
def scrub(value: Any, max_bytes: int = DEFAULT_MAX_BYTES) -> Any:
"""Recursively bound and redact a JSON-shaped value.
Mapping keys are themselves screened; values under a sensitive key are
dropped outright rather than pattern-screened, because a short secret
would survive screening.
"""
if isinstance(value, str):
return safe_text(value, max_bytes)[0]
if isinstance(value, dict):
screened: dict[str, Any] = {}
for key, item in value.items():
raw_key = str(key)
safe_key = safe_text(raw_key, max_bytes)[0]
sensitive_value = (
is_sensitive_key(raw_key)
and item is not None
and not isinstance(item, bool)
and item != ""
)
screened[safe_key] = REDACTED if sensitive_value else scrub(item, max_bytes)
return screened
if isinstance(value, (list, tuple)):
return [scrub(item, max_bytes) for item in value]
return value
def contains_credential_shape(text: str) -> bool:
"""Report whether screening would rewrite ``text``.
Used by the harness self-test to prove a rendered report is clean before it
is written, and by the unit suite as an assertion helper.
"""
return redact(text) != text

View File

@ -0,0 +1,199 @@
#!/usr/bin/env python3
"""Shared primitives and dispatch for handoff acceptance evaluators.
Evaluators are registered by name so the check catalog stays declarative: an
entry says *what* it asserts, not how to plumb it. The dispatch here is the one
place that turns an unexpected exception into ``NOT_RUN`` — a harness bug must
never be indistinguishable from a passing release.
"""
from __future__ import annotations
import json
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, field
from typing import Any
from hermes_handoff_exec import Outcome
from hermes_handoff_model import NOT_RUN, STATUSES, CheckSpec
DENIAL_MARKERS = (
"forbidden",
"is not allowed",
"unauthorized",
"permission denied",
"cannot get",
"cannot list",
"cannot create",
"cannot delete",
"cannot patch",
"not permitted",
"gitea api returned http 403",
)
@dataclass
class Evaluation:
"""A classification plus the small facts that justify it."""
status: str
reason: str
evidence: dict[str, Any] = field(default_factory=dict)
Evaluator = Callable[[CheckSpec, dict[str, Outcome]], Evaluation]
EVALUATORS: dict[str, Evaluator] = {}
def evaluator(name: str) -> Callable[[Evaluator], Evaluator]:
"""Register an evaluator under the rule name catalog entries use."""
def register(function: Evaluator) -> Evaluator:
EVALUATORS[name] = function
return function
return register
def missing_steps(
spec: CheckSpec, outcomes: Mapping[str, Outcome]
) -> Evaluation | None:
"""Return NOT_RUN when a required step never produced usable evidence."""
for step in spec.steps:
if step.optional:
continue
outcome = outcomes.get(step.key)
if outcome is None:
return Evaluation(NOT_RUN, f"step {step.key} was never executed")
if outcome.truncated:
return Evaluation(NOT_RUN, f"step {step.key} output was truncated")
if not outcome.ran:
return Evaluation(NOT_RUN, f"step {step.key} did not run: {outcome.error}")
return None
def looks_denied(outcome: Outcome) -> bool:
"""Report whether a real request was refused by policy rather than failing."""
if outcome.ok:
return False
haystack = outcome.combined.lower()
return any(marker in haystack for marker in DENIAL_MARKERS)
def step_keys(spec: CheckSpec, kind: str) -> tuple[str, ...]:
"""Return the keys of every step of one kind."""
return tuple(step.key for step in spec.steps if step.kind == kind)
def is_list(value: Any) -> bool:
"""Report whether a value is a JSON array rather than a string."""
return isinstance(value, Sequence) and not isinstance(value, (str, bytes))
def exact_equal(actual: Any, expected: Any) -> bool:
"""Compare JSON values without Python's bool/int type confusion."""
return type(actual) is type(expected) and actual == expected
def dotted(payload: Any, path: str) -> Any:
"""Return a value addressed by a dotted path.
``a.b[0].c`` indexes; ``a[].b`` maps the remainder of the path over a list,
which is how the catalog records one field from every pull request or every
Flux object without a bespoke parser per check.
"""
current = payload
segments = [segment for segment in path.split(".") if segment]
for position, raw in enumerate(segments):
name, bracket, indexes = raw.partition("[")
if name:
if not isinstance(current, Mapping) or name not in current:
raise KeyError(path)
current = current[name]
if not bracket:
continue
for chunk in indexes.rstrip("]").split("]["):
if not is_list(current):
raise KeyError(path)
if chunk == "":
remainder = ".".join(segments[position + 1 :])
return [
dotted(item, remainder) if remainder else item for item in current
]
offset = int(chunk)
if offset >= len(current):
raise KeyError(path)
current = current[offset]
return current
def strict_json(text: str) -> Any:
"""Decode strict JSON, rejecting constants and duplicate object keys."""
text = text.strip()
if not text:
raise ValueError("empty output")
def reject_constant(value: str) -> None:
raise ValueError(f"non-finite JSON constant {value}")
def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
result: dict[str, Any] = {}
for key, value in pairs:
if key in result:
raise ValueError(f"duplicate JSON key {key!r}")
result[key] = value
return result
return json.loads(
text,
strict=True,
parse_constant=reject_constant,
object_pairs_hook=unique_object,
)
def parse_json(outcome: Outcome) -> Any:
"""Parse one outcome as strict JSON."""
return strict_json(outcome.stdout)
def lines(outcome: Outcome) -> list[str]:
"""Return non-empty, stripped stdout lines."""
return [line.strip() for line in outcome.stdout.splitlines() if line.strip()]
def parsed_step(
spec: CheckSpec, outcomes: dict[str, Outcome]
) -> tuple[Any, str] | Evaluation:
"""Return the parsed payload of ``expect['step']`` or the NOT_RUN to report."""
key = spec.expect["step"]
outcome = outcomes[key]
if not outcome.ok:
return Evaluation(NOT_RUN, f"step {key} failed (rc={outcome.returncode})")
try:
return (parse_json(outcome), key)
except ValueError as exc:
return Evaluation(NOT_RUN, f"step {key} did not return JSON: {exc}")
def evaluate(spec: CheckSpec, outcomes: dict[str, Outcome]) -> Evaluation:
"""Dispatch to the named evaluator, converting any surprise into NOT_RUN."""
rule = EVALUATORS.get(spec.rule)
if rule is None:
return Evaluation(NOT_RUN, f"unknown evaluator: {spec.rule}")
truncated = sorted(key for key, outcome in outcomes.items() if outcome.truncated)
if truncated:
return Evaluation(
NOT_RUN, f"truncated evidence from step(s): {', '.join(truncated)}"
)
try:
result = rule(spec, outcomes)
except Exception as exc: # noqa: BLE001 - a harness bug must never read as PASS
return Evaluation(
NOT_RUN, f"evaluator {spec.rule} raised {type(exc).__name__}: {exc}"
)
if result.status not in STATUSES:
return Evaluation(
NOT_RUN, f"evaluator {spec.rule} returned unknown status {result.status!r}"
)
return result

View File

@ -0,0 +1,484 @@
#!/usr/bin/env python3
"""Orchestration for the Hermes full-handoff acceptance harness.
Three things happen before any check runs, and each of them can stop the run on
its own:
* the catalog is validated structurally — every rule exists, every live-denial
evaluator carries a read-only attempt, no operator-side step can impersonate;
* each vantage is resolved and asked who it is; and
* the operator and in-pod identities are compared. If they are the same
principal there is only one vantage, the dual-vantage evidence would be a
restatement rather than a corroboration, and the run is a NO_GO.
Everything after that is uniform: run a check's steps against its vantage, hand
the outcomes to the named evaluator, and record the classification. A vantage
that could not be resolved yields ``NOT_RUN`` for every check that needs it,
which is exactly the fail-closed answer.
"""
from __future__ import annotations
import re
from collections.abc import Iterable, Mapping, Sequence
from hermes_handoff_catalog import (
CHAT,
NODE,
OPERATOR,
SELF,
SWITCHYARD,
VANTAGE_NAMES,
Targets,
)
from hermes_handoff_checks_access import access_checks
from hermes_handoff_checks_delivery import delivery_checks
from hermes_handoff_checks_platform import platform_checks
from hermes_handoff_checks_workers import worker_checks
from hermes_handoff_evaluators import EVALUATORS, evaluate
from hermes_handoff_exec import Outcome, Runner, Vantage, operator_vantage, pod_vantage
from hermes_handoff_model import (
ATTEMPT,
EPHEMERAL,
NOT_RUN,
STEP_KINDS,
CheckResult,
CheckSpec,
Report,
VantageRecord,
utc_now,
)
from hermes_handoff_policy import (
EXPECTED_REMOTE,
EXPECTED_REPO,
IMPERSONATION_ARGS,
PolicyError,
check_argv,
projection,
)
UNRECORDED = "[output not recorded]"
def build_catalog(targets: Targets) -> list[CheckSpec]:
"""Return the whole acceptance catalog for a set of targets."""
return [
*platform_checks(targets),
*access_checks(targets),
*delivery_checks(targets),
*worker_checks(targets),
]
def validate_catalog(specs: Sequence[CheckSpec]) -> list[str]:
"""Return the structural problems that make a catalog unsafe to run."""
problems: list[str] = []
if not specs:
return ["catalog is empty"]
seen: set[str] = set()
fake_operator = operator_vantage()
fake_pod = pod_vantage("hermes", "acceptance-probe", "probe", fake_operator)
for spec in specs:
if spec.id in seen:
problems.append(f"{spec.id}: duplicate check id")
seen.add(spec.id)
if spec.rule not in EVALUATORS and spec.scope != EPHEMERAL:
problems.append(f"{spec.id}: unknown rule {spec.rule!r}")
step_names: set[str] = set()
for step in spec.steps:
if step.key in step_names:
problems.append(f"{spec.id}: duplicate step key {step.key!r}")
step_names.add(step.key)
if step.vantage not in VANTAGE_NAMES:
problems.append(
f"{spec.id}: step {step.key} names unknown vantage {step.vantage!r}"
)
if step.kind not in STEP_KINDS:
problems.append(
f"{spec.id}: step {step.key} has unknown kind {step.kind!r}"
)
if set(step.argv) & set(IMPERSONATION_ARGS):
problems.append(
f"{spec.id}: step {step.key} impersonates from the "
f"{step.vantage} vantage"
)
try:
addressed = (
fake_operator if step.vantage == OPERATOR else fake_pod
).wrap(step.argv)
check_argv(addressed)
except PolicyError as exc:
problems.append(f"{spec.id}: step {step.key} violates policy: {exc}")
if spec.rule == "denied" and not any(
step.kind == ATTEMPT for step in spec.steps
):
problems.append(
f"{spec.id}: a deny check needs a real attempt, not only a review"
)
return problems
def validate_targets(targets: Targets) -> list[str]:
"""Return release-input problems before a runner or network call exists."""
problems: list[str] = []
sha = re.compile(r"[0-9a-f]{40}\Z")
image = re.compile(
r"[^@\s]+:git-([0-9a-f]{40})-build-[1-9][0-9]*@sha256:[0-9a-f]{64}\Z"
)
if targets.repo != EXPECTED_REPO or targets.remote != EXPECTED_REMOTE:
problems.append(
"repository and remote must be the fixed Atlas titan-iac origin"
)
if (
targets.reviewed_pr_number != 19
or targets.reviewed_head_ref != "feature/hermes-full-handoff-acceptance"
):
problems.append(
"reviewed PR identity must remain fixed to draft PR #19 and its existing branch"
)
for label, value in (
("baseline commit", targets.baseline_commit),
("remote main SHA", targets.remote_main_sha),
("reviewed head SHA", targets.reviewed_head_sha),
("build SHA", targets.build_sha),
):
if not sha.fullmatch(value):
problems.append(f"{label} must be an exact lowercase 40-character SHA")
image_match = image.fullmatch(targets.agent_image)
if not image_match or image_match.group(1) != targets.build_sha:
problems.append("agent image tag/digest must bind the exact build SHA")
if targets.build_sha != targets.remote_main_sha:
problems.append("build SHA must equal the exact release main SHA")
if not re.fullmatch(r"[1-9][0-9]{0,8}", str(targets.deployment_revision)):
problems.append("deployment revision must be a positive integer string")
if targets.node_count != 3 or targets.pool_replicas != 3:
problems.append("node and pool counts are fixed at exactly three")
if not targets.chat_config_revision.strip():
problems.append(
"Telegram continuity and its chat config revision are mandatory"
)
if set(targets.pool_worker_env) != {"HERMES_WORKER_NODE", "HERMES_WORKER_ORDINAL"}:
problems.append("pool assignment evidence must include worker node and ordinal")
heads = dict(targets.dependency_heads)
if len(heads) != len(targets.dependency_heads) or set(heads) != set(
targets.dependency_pull_requests
):
problems.append("every dependency PR must have one exact current head")
elif any(not sha.fullmatch(value) for value in heads.values()):
problems.append("dependency heads must be exact lowercase 40-character SHAs")
if (
not isinstance(targets.max_evidence_age_seconds, int)
or targets.max_evidence_age_seconds <= 0
):
problems.append("maximum evidence age must be a positive integer")
return problems
def resolve_identity(runner: Runner, vantage: Vantage) -> tuple[str, str]:
"""Return the authenticated username of a vantage and any failure detail."""
outcome = runner.run(
(
"kubectl",
"auth",
"whoami",
"-o",
projection("jsonpath={.status.userInfo.username}"),
),
vantage,
)
if not outcome.ok:
return ("", outcome.error or outcome.combined[:200])
username = outcome.stdout.strip()
if not username or any(character.isspace() for character in username):
return ("", "identity response carried no username")
return (username, "")
def _first_pod(
runner: Runner, operator: Vantage, namespace: str, selector: str
) -> tuple[str, str]:
outcome = runner.run(
(
"kubectl",
"--namespace",
namespace,
"get",
"pods",
"--selector",
selector,
"--field-selector",
"status.phase=Running",
"-o",
"name",
),
operator,
)
if not outcome.ok:
return ("", outcome.error or outcome.combined[:200])
names = [
line.strip().removeprefix("pod/")
for line in outcome.stdout.splitlines()
if line.strip()
]
if not names:
return ("", f"no running pod matches {selector}")
return (sorted(names)[0], "")
def resolve_vantages(
runner: Runner,
targets: Targets,
kubeconfig: str | None = None,
context: str | None = None,
) -> tuple[dict[str, Vantage], list[VantageRecord]]:
"""Resolve every vantage the catalog names, recording what each one is."""
operator = operator_vantage(kubeconfig, context)
username, detail = resolve_identity(runner, operator)
records = [
VantageRecord(
name=OPERATOR,
description=operator.description or "external read-only operator",
identity=username,
available=bool(username),
detail=detail,
)
]
vantages: dict[str, Vantage] = {OPERATOR: operator}
if not username:
return (vantages, records)
for name, selector, container, description in (
(
SELF,
f"app={targets.agent_deployment}",
targets.agent_container,
"the Hermes agent itself",
),
(
SWITCHYARD,
f"app={targets.switchyard_deployment}",
targets.switchyard_container,
"the Switchyard routing evidence log",
),
(
NODE,
f"app={targets.node_daemonset}",
targets.node_container,
"a node hardening probe",
),
):
pod, failure = _first_pod(runner, operator, targets.namespace, selector)
if not pod:
records.append(
VantageRecord(
name=name, description=description, available=False, detail=failure
)
)
continue
vantage = pod_vantage(targets.namespace, pod, container, operator)
identity, identity_detail = (
resolve_identity(runner, vantage) if name == SELF else ("", "")
)
records.append(
VantageRecord(
name=name,
description=f"{description} ({targets.namespace}/{pod})",
identity=identity,
available=True,
detail=identity_detail,
)
)
vantages[name] = vantage
chat_pod = f"{targets.chat_statefulset}-{targets.chat_ordinal}"
present = runner.run(
(
"kubectl",
"--namespace",
targets.namespace,
"get",
f"pod/{chat_pod}",
"-o",
"name",
),
operator,
)
if present.ok and present.stdout.strip():
vantages[CHAT] = pod_vantage(
targets.namespace, chat_pod, targets.chat_container, operator
)
records.append(
VantageRecord(
name=CHAT,
description=f"the chat surface ({targets.namespace}/{chat_pod})",
available=True,
)
)
else:
records.append(
VantageRecord(
name=CHAT,
description=f"the chat surface ({targets.namespace}/{chat_pod})",
available=False,
detail=present.error
or present.combined[:200]
or "chat tenant pod not found",
)
)
return (vantages, records)
def vantage_problems(records: Iterable[VantageRecord]) -> list[str]:
"""Return the identity problems that invalidate dual-vantage evidence."""
by_name = {record.name: record for record in records}
operator = by_name.get(OPERATOR)
hermes = by_name.get(SELF)
problems: list[str] = []
if operator is None or not operator.available:
problems.append(
"the operator vantage is unavailable; no evidence can be collected"
)
return problems
if hermes is None or not hermes.available:
problems.append(
"the in-pod Hermes vantage is unavailable; self-probe evidence is missing"
)
return problems
if not hermes.identity:
problems.append("the in-pod vantage did not report an identity")
elif operator.identity == hermes.identity:
problems.append(
"the operator and in-pod vantages authenticate as the same principal "
f"({operator.identity}); run the harness with a separate operator kubeconfig"
)
for name, label in ((SWITCHYARD, "Switchyard"), (CHAT, "chat/Telegram")):
record = by_name.get(name)
if record is None or not record.available:
problems.append(f"the required {label} vantage is unavailable")
return problems
def run_check(
runner: Runner, spec: CheckSpec, vantages: Mapping[str, Vantage]
) -> CheckResult:
"""Run one check's steps and classify the result."""
outcomes: dict[str, Outcome] = {}
recorded: list[Outcome] = []
for step in spec.steps:
vantage = vantages.get(step.vantage)
if vantage is None:
outcome = Outcome(
argv=step.argv,
vantage=step.vantage,
error=f"vantage {step.vantage} is unavailable",
)
else:
outcome = runner.run(step.argv, vantage, step.max_bytes)
outcomes[step.key] = outcome
recorded.append(
outcome
if step.record
else Outcome(
argv=outcome.argv,
vantage=outcome.vantage,
returncode=outcome.returncode,
stdout=UNRECORDED if outcome.stdout else "",
stderr=UNRECORDED if outcome.stderr else "",
truncated=outcome.truncated,
duration_ms=outcome.duration_ms,
error=outcome.error,
executable_path=outcome.executable_path,
executable_sha256=outcome.executable_sha256,
)
)
evaluation = evaluate(spec, outcomes)
return CheckResult(
spec=spec,
status=evaluation.status,
reason=evaluation.reason,
evidence=evaluation.evidence,
outcomes=recorded,
)
def run_catalog(
runner: Runner, specs: Sequence[CheckSpec], vantages: Mapping[str, Vantage]
) -> list[CheckResult]:
"""Run every check in the catalog.
Nothing is skipped once the deadline passes: the runner returns a
deadline outcome for each remaining command, which classifies as
``NOT_RUN``, so a truncated sweep is visible in the report rather than
silently shorter.
"""
return [run_check(runner, spec, vantages) for spec in specs]
def build_report(
runner: Runner,
targets: Targets,
specs: Sequence[CheckSpec],
vantages: Mapping[str, Vantage],
records: Sequence[VantageRecord],
mode: str,
started_at: str,
extra_results: Sequence[CheckResult] = (),
) -> Report:
"""Run the catalog and assemble the report, including structural problems."""
problems = validate_catalog(specs) + vantage_problems(records)
report = Report(
mode=mode,
started_at=started_at,
baseline={
"namespace": targets.namespace,
"baseline_commit": targets.baseline_commit,
"dependency_pull_requests": list(targets.dependency_pull_requests),
"expected_suspensions": list(targets.expected_suspensions),
"node_count": targets.node_count,
"pool_replicas": targets.pool_replicas,
},
vantages=list(records),
harness_errors=problems,
)
armed_ids = {result.spec.id for result in extra_results}
runnable = [spec for spec in specs if spec.id not in armed_ids]
if problems:
report.results = [
CheckResult(spec=spec, status=NOT_RUN, reason="harness preflight failed")
for spec in runnable
]
report.results.extend(extra_results)
else:
report.results = [*run_catalog(runner, runnable, vantages), *extra_results]
report.results.sort(key=lambda result: result.spec.id)
if runner.remaining_seconds <= 0:
report.harness_errors.append(
"the run deadline expired before every check completed"
)
report.finished_at = utc_now()
return report
def unavailable_report(
mode: str, started_at: str, reason: str, targets: Targets
) -> Report:
"""Return a NO_GO report for a run that could not start at all."""
return Report(
mode=mode,
started_at=started_at,
finished_at=utc_now(),
baseline={"namespace": targets.namespace},
harness_errors=[reason],
results=[
CheckResult(
spec=CheckSpec(
id="harness.startup",
title="The harness could start and resolve its vantages",
group="harness",
rule="allowed",
),
status=NOT_RUN,
reason=reason,
)
],
)

View File

@ -0,0 +1,25 @@
import importlib.util
import pathlib
def load_module():
path = pathlib.Path(__file__).resolve().parents[1] / "render" / "dashboards_render_atlas.py"
spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path)
module = importlib.util.module_from_spec(spec)
assert spec.loader is not None
spec.loader.exec_module(module)
return module
def flatten_panels(panels):
flat = []
for panel in panels:
flat.append(panel)
flat.extend(panel.get("panels", []))
return flat
__all__ = (
"flatten_panels",
"importlib",
"load_module",
"pathlib",
)

View File

@ -0,0 +1,127 @@
"""Detailed Atlas Jobs dashboard collapse and drilldown contracts."""
from scripts.tests.test_dashboard_render_support import load_module
def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
mod = load_module()
dashboard = mod.build_jobs_dashboard()
panels = dashboard["panels"]
rows = [panel for panel in panels if panel["type"] == "row"]
visible_query_panels = [panel for panel in panels if panel["type"] != "row"]
nested_panels_by_title = {
child["title"]: child for row in rows for child in row.get("panels", [])
}
assert len(panels) == 18
assert len(visible_query_panels) == 12
assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12
assert all(
panel["title"] != "Coverage Gap to 95% by Suite"
for panel in visible_query_panels
)
assert [row["title"] for row in rows] == [
"CI Runs And Test Result History",
"Check Failure Rates By Suite",
"Check Healthy Rates By Suite",
"Test Drilldowns And Problem Tests",
"Telemetry Completeness And Branches",
"SonarQube Project Health",
]
assert all(row["collapsed"] for row in rows)
assert "Coverage Failure Rate" in nested_panels_by_title
assert "Supply Chain Healthy Rate" in nested_panels_by_title
assert "Test Category Health History" in nested_panels_by_title
assert "Selected Test Pass Rate History" in nested_panels_by_title
assert "Coverage Metrics Present by Suite" in nested_panels_by_title
assert "SonarQube API Up" in nested_panels_by_title
failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"]
assert failure_rate_panel["type"] == "state-timeline"
assert failure_rate_panel["fieldConfig"]["defaults"]["unit"] == "percent"
assert failure_rate_panel["fieldConfig"]["defaults"]["max"] == 100
assert (
failure_rate_panel["fieldConfig"]["defaults"]["thresholds"]["steps"][0]["color"]
== "dark-blue"
)
assert "increase(" not in failure_rate_panel["targets"][0]["expr"]
assert (
"platform_quality:check_failed_flag:present_1h"
in failure_rate_panel["targets"][0]["expr"]
)
assert (
"platform_quality:check_seen_flag:present_1h"
in failure_rate_panel["targets"][0]["expr"]
)
assert (
"platform_quality:check_status:present_1h"
not in failure_rate_panel["targets"][0]["expr"]
)
assert (
".*_quality_gate_checks_total" not in failure_rate_panel["targets"][0]["expr"]
)
assert "last_over_time" not in failure_rate_panel["targets"][0]["expr"]
assert "label_replace" not in failure_rate_panel["targets"][0]["expr"]
assert "0 *" in failure_rate_panel["targets"][0]["expr"]
assert "and on(suite)" not in failure_rate_panel["targets"][0]["expr"]
pass_rate_panel = nested_panels_by_title["Selected Test Pass Rate History"]
assert pass_rate_panel["type"] == "state-timeline"
assert (
"platform_quality:test_case_pass_rate:percent_1h"
in pass_rate_panel["targets"][0]["expr"]
)
assert (
"platform_quality_gate_test_case_result"
not in pass_rate_panel["targets"][0]["expr"]
)
pass_fail_panel = nested_panels_by_title["Selected Test Pass/Fail History"]
assert pass_fail_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
assert all(
"platform_quality:test_case_status:count_1h" in target["expr"]
for target in pass_fail_panel["targets"]
)
problematic_panel = nested_panels_by_title[
"Problematic Tests Over Time (Top failures)"
]
assert problematic_panel["type"] == "state-timeline"
assert problematic_panel["gridPos"]["w"] == 24
assert 'test!=""' in problematic_panel["targets"][0]["expr"]
assert "vector(0)" not in problematic_panel["targets"][0]["expr"]
assert problematic_panel["fieldConfig"]["defaults"]["thresholds"]["steps"] == [
{"color": "dark-blue", "value": None},
{"color": "dark-green", "value": 2},
{"color": "dark-yellow", "value": 3},
{"color": "dark-orange", "value": 5},
{"color": "dark-red", "value": 8},
]
assert "rolling 24h failure count" in problematic_panel["description"]
assert "at least two recent failures" in problematic_panel["description"]
sonar_mix_panel = nested_panels_by_title["Sonar Gate Status Mix (Selected)"]
sonar_health_panel = nested_panels_by_title["Sonar Gate Health by Project"]
assert sonar_mix_panel["gridPos"]["w"] == 4
assert sonar_health_panel["gridPos"]["w"] == 8
assert sonar_health_panel["type"] == "state-timeline"
assert (
"platform_quality:sonar_gate_health_percent:latest_1h"
in sonar_health_panel["targets"][0]["expr"]
)
assert (
"sonarqube_project_quality_gate_pass"
not in sonar_health_panel["targets"][0]["expr"]
)
branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"]
recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"]
assert branch_panel["gridPos"]["x"] == 12
assert recent_branch_panel["gridPos"]["x"] == 18
assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"]
assert "[7d:1h]" in branch_panel["targets"][0]["expr"]
assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent"
assert "unless on(suite)" in branch_panel["targets"][0]["expr"]
assert "> bool 0" in branch_panel["targets"][0]["expr"]
assert branch_panel["targets"][0]["expr"].startswith("sort(")

View File

@ -1,241 +1,11 @@
import importlib.util
import pathlib
"""Atlas testing and jobs dashboard rendering contracts."""
from __future__ import annotations
def load_module():
path = pathlib.Path(__file__).resolve().parents[1] / "render" / "dashboards_render_atlas.py"
spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path)
module = importlib.util.module_from_spec(spec)
assert spec.loader is not None
spec.loader.exec_module(module)
return module
def flatten_panels(panels):
flat = []
for panel in panels:
flat.append(panel)
flat.extend(panel.get("panels", []))
return flat
def test_table_panel_options_and_filterable():
mod = load_module()
panel = mod.table_panel(
1,
"test",
"metric",
{"h": 1, "w": 1, "x": 0, "y": 0},
unit="percent",
transformations=[{"id": "labelsToFields", "options": {}}],
instant=True,
options={"showColumnFilters": False},
filterable=False,
footer={"show": False, "fields": "", "calcs": []},
format="table",
)
assert panel["fieldConfig"]["defaults"]["unit"] == "percent"
assert panel["fieldConfig"]["defaults"]["custom"]["filterable"] is False
assert panel["options"]["showHeader"] is True
assert panel["targets"][0]["format"] == "table"
def test_node_filter_and_expr_helpers():
mod = load_module()
expr = mod.node_filter("titan-.*")
assert "label_replace" in expr
cpu_expr = mod.node_cpu_expr("titan-.*")
mem_expr = mod.node_mem_expr("titan-.*")
assert "node_cpu_seconds_total" in cpu_expr
assert "clamp_max(clamp_min(" in cpu_expr
assert "* 100, 0), 100)" in cpu_expr
assert "node_memory_MemAvailable_bytes" in mem_expr
def test_overview_availability_panel_uses_recorded_365d_rollup():
mod = load_module()
dashboard = mod.build_overview()
panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27)
assert panel["title"] == "Atlas Availability (365d)"
availability_expr = panel["targets"][0]["expr"]
assert (
'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])'
in availability_expr
)
assert 'code=~"5.."' in availability_expr
assert 'code=~"[1-5].."' in availability_expr
assert "atlas:availability:failures_1d" not in availability_expr
assert "atlas:availability:requests_1d" not in availability_expr
assert "sum_over_time" not in availability_expr
assert "kube_node_status_condition" not in availability_expr
assert "kube_deployment_status_replicas_available" not in availability_expr
assert panel["targets"][0]["instant"] is True
assert "Every server-side 5xx" in panel["description"]
assert "Replica counts, Grafana health" in panel["description"]
assert "daily rollup job publishes one annual sample" in panel["description"]
assert "keeps it for up to 48 hours" in panel["description"]
def test_overview_uses_readable_quality_power_and_gitops_panels():
mod = load_module()
dashboard = mod.build_overview()
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
assert dashboard["links"] == [
{"title": "Atlas Testing", "url": "/d/atlas-testing", "targetBlank": True}
]
assert "atlas-jobs" not in repr(dashboard)
assert "Platform Test Success Rate" not in panels_by_title
assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
assert panels_by_title["Test Category Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 13}
assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
assert panels_by_title["Test Category Health"]["targets"][0]["format"] == "time_series"
assert panels_by_title["Test Category Health"]["targets"][0]["range"] is True
assert "${overview_suite:regex}" not in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert panels_by_title["Test Category Health"]["timeFrom"] == "24h"
assert f'category=~"{mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert "manual" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
assert "unit" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
assert panels_by_title["UPS History (Power Draw)"]["gridPos"] == {"h": 6, "w": 6, "x": 3, "y": 7}
assert panels_by_title["Ariadne Run Volume"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 7}
assert panels_by_title["Pyrphoros UPS Current"]["gridPos"]["w"] == 3
assert panels_by_title["Current Enclosure Climate"]["gridPos"]["w"] == 3
assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["placement"] == "bottom"
assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["displayMode"] == "list"
assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "line"
assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["fillOpacity"] == 22
assert all(target["expr"].startswith("max(") for target in panels_by_title["UPS History (Power Draw)"]["targets"])
ups_overrides = panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["overrides"]
ups_override_by_name = {override["matcher"]["options"]: override for override in ups_overrides}
assert ups_override_by_name["Pyrphoros"]["properties"] == [
{"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-blue"}},
]
assert ups_override_by_name["Statera"]["properties"] == [
{"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-yellow"}},
]
assert panels_by_title["Ariadne Run Volume"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["placement"] == "bottom"
assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["displayMode"] == "list"
assert "Fan History (0-10)" not in panels_by_title
assert panels_by_title["Fan Intensity History"]["type"] == "state-timeline"
assert panels_by_title["Fan Intensity History"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 13}
assert panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["max"] == 10
assert panels_by_title["Fan Intensity History"]["targets"][0]["legendFormat"] == "{{fan}}"
fan_steps = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["thresholds"]["steps"]
assert [step["value"] for step in fan_steps] == [None, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
assert fan_steps[0]["color"] == "#8f1d1d"
assert fan_steps[5]["color"] == "#d4b106"
assert fan_steps[-1]["color"] == "#1f60c4"
fan_mappings = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["mappings"][0]["options"]
assert fan_mappings["0"]["text"] == "Off"
assert fan_mappings["10"]["text"] == "10"
assert "Tent Interior" in panels_by_title["Fan Intensity History"]["targets"][0]["expr"]
assert panels_by_title["Fan Intensity History"]["options"]["legend"]["displayMode"] == "list"
assert panels_by_title["Fan Intensity History"]["options"]["legend"]["showLegend"] is False
assert panels_by_title["Fan Intensity History"]["options"]["mergeValues"] is False
assert panels_by_title["Fan Intensity History"]["options"]["showValue"] == "auto"
assert panels_by_title["Flux Source"]["type"] == "stat"
assert panels_by_title["Flux Source"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 7}
assert panels_by_title["Flux Source"]["targets"][0]["legendFormat"] == "{{branch}}"
assert panels_by_title["Current Gate Health"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 9}
assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Current Gate Health"]["targets"][0]["expr"]
assert panels_by_title["CI Run Success (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 11}
assert panels_by_title["Suites With Runs (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 15}
suites_reporting_expr = panels_by_title["Suites With Runs (24h)"]["targets"][0]["expr"]
assert "> bool 0" in suites_reporting_expr
assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in suites_reporting_expr
assert "bstein-home" not in suites_reporting_expr
assert "published quality-gate run" in panels_by_title["Suites With Runs (24h)"]["description"]
assert panels_by_title["Avg Coverage"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 17}
assert "LOC Clean Suites" not in panels_by_title
assert panels_by_title["GitOps Health"]["type"] == "state-timeline"
assert panels_by_title["GitOps Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 7}
gitops_expr = panels_by_title["GitOps Health"]["targets"][0]["expr"]
assert "Kustomizations Not Suspended" in gitops_expr
assert "HelmReleases Not Suspended" in gitops_expr
assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
assert panels_by_title["Test Category Health"]["options"]["legend"]["showLegend"] is False
assert panels_by_title["Test Category Health"]["options"]["mergeValues"] is False
assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto"
assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9
assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"])
pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"]
assert "backup-telemetry-missing" in pvc_backup_expr
assert 'pvc_backup_(count|last_success_timestamp_seconds|health_reason)' in pvc_backup_expr
gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
assert "nvidia_gpu_device_utilization_percent" in gpu_expr
assert "jetson_gr3d_active_seconds_total" in gpu_expr
assert "increase" in gpu_expr
assert "sum_over_time" not in gpu_expr
assert "count_over_time" not in gpu_expr
assert "avg_over_time" in gpu_expr
assert "$__range" in gpu_expr
assert "$__range_s" in gpu_expr
assert "sum by (namespace)" in gpu_expr
assert 'namespace", "shared"' not in gpu_expr
assert "kube_pod_container_resource_requests" in gpu_expr
assert mod.GPU_RESOURCE_REGEX in gpu_expr
assert '"gpu_source", "nvidia"' in gpu_expr
assert '"gpu_source", "jetson"' in gpu_expr
assert "100 *" in gpu_expr
assert "100 -" not in gpu_expr
assert 'namespace", "unattributed"' in gpu_expr
assert 'namespace", "idle"' in gpu_expr
assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
assert "selected time range" in panels_by_title["Namespace GPU Utilization"]["description"]
assert "continuously sampled" in panels_by_title["Namespace GPU Utilization"]["description"]
assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
mod = load_module()
dashboard = mod.build_gpu_dashboard()
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"]
assert "nvidia_gpu_device_utilization_percent" in node_expr
assert "DCGM_FI_DEV_GPU_UTIL" in node_expr
assert "jetson_gr3d_freq_percent" in node_expr
assert "unless on(node)" in node_expr
assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"]
assert "nvidia_process_gpu_sm_util_percent" in process_expr
assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
def test_overview_and_testing_panels_all_have_concise_descriptions():
mod = load_module()
for dashboard in [mod.build_overview(), mod.build_jobs_dashboard(), mod.build_testing_dashboard()]:
panels = flatten_panels(dashboard["panels"])
assert all(panel.get("description") for panel in panels if panel["type"] != "row")
assert "85.7" not in repr(dashboard)
def test_render_configmap_writes(tmp_path):
mod = load_module()
mod.DASHBOARD_DIR = tmp_path / "dash"
mod.ROOT = tmp_path
uid = "atlas-test"
info = {"configmap": tmp_path / "cm.yaml"}
data = {"title": "Atlas Test"}
mod.write_json(uid, data)
mod.render_configmap(uid, info)
json_path = mod.DASHBOARD_DIR / f"{uid}.json"
assert json_path.exists()
content = (tmp_path / "cm.yaml").read_text()
assert "kind: ConfigMap" in content
assert f"{uid}.json" in content
from test_dashboard_render_support import (
flatten_panels,
load_module,
)
def test_testing_suite_variable_uses_configured_suite_values_only():
@ -257,7 +27,6 @@ def test_testing_suite_variable_uses_configured_suite_values_only():
option["value"] in out_of_scope_names for option in variable["options"]
)
def test_testing_dashboard_is_public_but_jobs_dashboard_remains_internal():
mod = load_module()
jobs = mod.build_jobs_dashboard()
@ -269,7 +38,6 @@ def test_testing_dashboard_is_public_but_jobs_dashboard_remains_internal():
assert testing["folderUid"] == mod.PUBLIC_DASHBOARD_FOLDER
assert testing["editable"] is False
def test_jobs_dashboard_separates_current_gate_health_from_reliability():
mod = load_module()
dashboard = mod.build_jobs_dashboard()
@ -395,7 +163,6 @@ def test_jobs_dashboard_separates_current_gate_health_from_reliability():
assert run_volume_panel["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
assert "[$__interval]" not in run_volume_panel["targets"][0]["expr"]
def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors():
mod = load_module()
dashboard = mod.build_jobs_dashboard()
@ -416,8 +183,6 @@ def test_jobs_dashboard_bar_gauges_use_solid_threshold_colors():
assert {"color": "dark-yellow", "value": 93} in threshold_steps
assert {"color": "dark-blue", "value": 100} in threshold_steps
def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
mod = load_module()
dashboard = mod.build_jobs_dashboard()

View File

@ -0,0 +1,221 @@
"""Atlas overview dashboard rendering contracts."""
from __future__ import annotations
from test_dashboard_render_support import (
flatten_panels,
load_module,
)
def test_table_panel_options_and_filterable():
mod = load_module()
panel = mod.table_panel(
1,
"test",
"metric",
{"h": 1, "w": 1, "x": 0, "y": 0},
unit="percent",
transformations=[{"id": "labelsToFields", "options": {}}],
instant=True,
options={"showColumnFilters": False},
filterable=False,
footer={"show": False, "fields": "", "calcs": []},
format="table",
)
assert panel["fieldConfig"]["defaults"]["unit"] == "percent"
assert panel["fieldConfig"]["defaults"]["custom"]["filterable"] is False
assert panel["options"]["showHeader"] is True
assert panel["targets"][0]["format"] == "table"
def test_node_filter_and_expr_helpers():
mod = load_module()
expr = mod.node_filter("titan-.*")
assert "label_replace" in expr
cpu_expr = mod.node_cpu_expr("titan-.*")
mem_expr = mod.node_mem_expr("titan-.*")
assert "node_cpu_seconds_total" in cpu_expr
assert "clamp_max(clamp_min(" in cpu_expr
assert "* 100, 0), 100)" in cpu_expr
assert "node_memory_MemAvailable_bytes" in mem_expr
def test_overview_availability_panel_uses_recorded_365d_rollup():
mod = load_module()
dashboard = mod.build_overview()
panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27)
assert panel["title"] == "Atlas Availability (365d)"
availability_expr = panel["targets"][0]["expr"]
assert (
'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])'
in availability_expr
)
assert 'code=~"5.."' in availability_expr
assert 'code=~"[1-5].."' in availability_expr
assert "atlas:availability:failures_1d" not in availability_expr
assert "atlas:availability:requests_1d" not in availability_expr
assert "sum_over_time" not in availability_expr
assert "kube_node_status_condition" not in availability_expr
assert "kube_deployment_status_replicas_available" not in availability_expr
assert panel["targets"][0]["instant"] is True
assert "Every server-side 5xx" in panel["description"]
assert "Replica counts, Grafana health" in panel["description"]
assert "daily rollup job publishes one annual sample" in panel["description"]
assert "keeps it for up to 48 hours" in panel["description"]
def test_overview_uses_readable_quality_power_and_gitops_panels():
mod = load_module()
dashboard = mod.build_overview()
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
assert dashboard["links"] == [
{"title": "Atlas Testing", "url": "/d/atlas-testing", "targetBlank": True}
]
assert "atlas-jobs" not in repr(dashboard)
assert "Platform Test Success Rate" not in panels_by_title
assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
assert panels_by_title["Test Category Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 13}
assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
assert panels_by_title["Test Category Health"]["targets"][0]["format"] == "time_series"
assert panels_by_title["Test Category Health"]["targets"][0]["range"] is True
assert "${overview_suite:regex}" not in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert panels_by_title["Test Category Health"]["timeFrom"] == "24h"
assert f'category=~"{mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert "manual" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
assert "unit" not in mod.PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX
assert panels_by_title["UPS History (Power Draw)"]["gridPos"] == {"h": 6, "w": 6, "x": 3, "y": 7}
assert panels_by_title["Ariadne Run Volume"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 7}
assert panels_by_title["Pyrphoros UPS Current"]["gridPos"]["w"] == 3
assert panels_by_title["Current Enclosure Climate"]["gridPos"]["w"] == 3
assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["placement"] == "bottom"
assert panels_by_title["UPS History (Power Draw)"]["options"]["legend"]["displayMode"] == "list"
assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "line"
assert panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["defaults"]["custom"]["fillOpacity"] == 22
assert all(target["expr"].startswith("max(") for target in panels_by_title["UPS History (Power Draw)"]["targets"])
ups_overrides = panels_by_title["UPS History (Power Draw)"]["fieldConfig"]["overrides"]
ups_override_by_name = {override["matcher"]["options"]: override for override in ups_overrides}
assert ups_override_by_name["Pyrphoros"]["properties"] == [
{"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-blue"}},
]
assert ups_override_by_name["Statera"]["properties"] == [
{"id": "color", "value": {"mode": "fixed", "fixedColor": "dark-yellow"}},
]
assert panels_by_title["Ariadne Run Volume"]["fieldConfig"]["defaults"]["custom"]["drawStyle"] == "bars"
assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["placement"] == "bottom"
assert panels_by_title["Ariadne Run Volume"]["options"]["legend"]["displayMode"] == "list"
assert "Fan History (0-10)" not in panels_by_title
assert panels_by_title["Fan Intensity History"]["type"] == "state-timeline"
assert panels_by_title["Fan Intensity History"]["gridPos"] == {"h": 6, "w": 6, "x": 9, "y": 13}
assert panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["max"] == 10
assert panels_by_title["Fan Intensity History"]["targets"][0]["legendFormat"] == "{{fan}}"
fan_steps = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["thresholds"]["steps"]
assert [step["value"] for step in fan_steps] == [None, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
assert fan_steps[0]["color"] == "#8f1d1d"
assert fan_steps[5]["color"] == "#d4b106"
assert fan_steps[-1]["color"] == "#1f60c4"
fan_mappings = panels_by_title["Fan Intensity History"]["fieldConfig"]["defaults"]["mappings"][0]["options"]
assert fan_mappings["0"]["text"] == "Off"
assert fan_mappings["10"]["text"] == "10"
assert "Tent Interior" in panels_by_title["Fan Intensity History"]["targets"][0]["expr"]
assert panels_by_title["Fan Intensity History"]["options"]["legend"]["displayMode"] == "list"
assert panels_by_title["Fan Intensity History"]["options"]["legend"]["showLegend"] is False
assert panels_by_title["Fan Intensity History"]["options"]["mergeValues"] is False
assert panels_by_title["Fan Intensity History"]["options"]["showValue"] == "auto"
assert panels_by_title["Flux Source"]["type"] == "stat"
assert panels_by_title["Flux Source"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 7}
assert panels_by_title["Flux Source"]["targets"][0]["legendFormat"] == "{{branch}}"
assert panels_by_title["Current Gate Health"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 9}
assert "platform_quality:test_category_health_rate:percent_1h" in panels_by_title["Current Gate Health"]["targets"][0]["expr"]
assert panels_by_title["CI Run Success (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 11}
assert panels_by_title["Suites With Runs (24h)"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 15}
suites_reporting_expr = panels_by_title["Suites With Runs (24h)"]["targets"][0]["expr"]
assert "> bool 0" in suites_reporting_expr
assert mod.PLATFORM_TEST_SUITE_CANONICAL_MATCHER in suites_reporting_expr
assert "bstein-home" not in suites_reporting_expr
assert "published quality-gate run" in panels_by_title["Suites With Runs (24h)"]["description"]
assert panels_by_title["Avg Coverage"]["gridPos"] == {"h": 2, "w": 3, "x": 21, "y": 17}
assert "LOC Clean Suites" not in panels_by_title
assert panels_by_title["GitOps Health"]["type"] == "state-timeline"
assert panels_by_title["GitOps Health"]["gridPos"] == {"h": 6, "w": 6, "x": 15, "y": 7}
gitops_expr = panels_by_title["GitOps Health"]["targets"][0]["expr"]
assert "Kustomizations Not Suspended" in gitops_expr
assert "HelmReleases Not Suspended" in gitops_expr
assert panels_by_title["Test Category Health"]["type"] == "state-timeline"
assert panels_by_title["Test Category Health"]["options"]["legend"]["showLegend"] is False
assert panels_by_title["Test Category Health"]["options"]["mergeValues"] is False
assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto"
assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9
assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"])
pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"]
assert "backup-telemetry-missing" in pvc_backup_expr
assert 'pvc_backup_(count|last_success_timestamp_seconds|health_reason)' in pvc_backup_expr
gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
assert "nvidia_gpu_device_utilization_percent" in gpu_expr
assert "jetson_gr3d_active_seconds_total" in gpu_expr
assert "increase" in gpu_expr
assert "sum_over_time" not in gpu_expr
assert "count_over_time" not in gpu_expr
assert "avg_over_time" in gpu_expr
assert "$__range" in gpu_expr
assert "$__range_s" in gpu_expr
assert "sum by (namespace)" in gpu_expr
assert 'namespace", "shared"' not in gpu_expr
assert "kube_pod_container_resource_requests" in gpu_expr
assert mod.GPU_RESOURCE_REGEX in gpu_expr
assert '"gpu_source", "nvidia"' in gpu_expr
assert '"gpu_source", "jetson"' in gpu_expr
assert "100 *" in gpu_expr
assert "100 -" not in gpu_expr
assert 'namespace", "unattributed"' in gpu_expr
assert 'namespace", "idle"' in gpu_expr
assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
assert "selected time range" in panels_by_title["Namespace GPU Utilization"]["description"]
assert "continuously sampled" in panels_by_title["Namespace GPU Utilization"]["description"]
assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
mod = load_module()
dashboard = mod.build_gpu_dashboard()
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"]
assert "nvidia_gpu_device_utilization_percent" in node_expr
assert "DCGM_FI_DEV_GPU_UTIL" in node_expr
assert "jetson_gr3d_freq_percent" in node_expr
assert "unless on(node)" in node_expr
assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"]
assert "nvidia_process_gpu_sm_util_percent" in process_expr
assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
def test_overview_and_testing_panels_all_have_concise_descriptions():
mod = load_module()
for dashboard in [mod.build_overview(), mod.build_jobs_dashboard(), mod.build_testing_dashboard()]:
panels = flatten_panels(dashboard["panels"])
assert all(panel.get("description") for panel in panels if panel["type"] != "row")
assert "85.7" not in repr(dashboard)
def test_render_configmap_writes(tmp_path):
mod = load_module()
mod.DASHBOARD_DIR = tmp_path / "dash"
mod.ROOT = tmp_path
uid = "atlas-test"
info = {"configmap": tmp_path / "cm.yaml"}
data = {"title": "Atlas Test"}
mod.write_json(uid, data)
mod.render_configmap(uid, info)
json_path = mod.DASHBOARD_DIR / f"{uid}.json"
assert json_path.exists()
content = (tmp_path / "cm.yaml").read_text()
assert "kind: ConfigMap" in content
assert f"{uid}.json" in content

View File

@ -232,6 +232,24 @@ def test_retry_db_connect_reraises_final_error(monkeypatch):
sync.retry_db_connect(attempts=1)
def test_retry_helpers_return_none_without_attempts(monkeypatch):
sync = load_sync_module(monkeypatch)
assert sync.retry_request("request", lambda: "never", attempts=0) is None
assert sync.retry_db_connect(attempts=0) is None
def test_kc_update_attributes_replaces_non_dict_attributes(monkeypatch):
sync = load_sync_module(monkeypatch)
current_resp = _FakeResponse({"attributes": "not-a-dict"})
ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}})
sync.SESSION = _FakeSession(_FakeResponse({}), [current_resp, ok_resp])
sync.kc_update_attributes("token", {"id": "u1", "username": "u1"}, {"mailu_app_password": "abc"})
assert sync.SESSION.put_called
def test_ensure_mailu_user_skips_foreign_domain(monkeypatch):
sync = load_sync_module(monkeypatch)
executed = []
@ -405,3 +423,62 @@ def test_main_generates_password_and_upserts(monkeypatch):
# Only mail-enabled users (or legacy users with a mailbox) are synced and backfilled.
assert len(updated) == 3
assert conns and len(conns[0]._cursor.executions) == 3
def test_main_skips_disabled_users_and_provisioned_users_without_update(monkeypatch):
sync = load_sync_module(monkeypatch)
monkeypatch.setattr(sync.bcrypt_sha256, "hash", lambda password: f"hash:{password}")
users = [
{
"id": "u1",
"username": "disabled",
"email": "disabled@example.com",
"enabled": False,
"attributes": {"mailu_enabled": ["true"]},
},
{
"id": "u2",
"username": "settled",
"email": "settled@example.com",
"attributes": {
"mailu_enabled": ["true"],
"mailu_email": ["settled@example.com"],
"mailu_app_password": ["existing"],
},
},
]
updated = []
class _Cursor:
def __init__(self):
self.executions = []
def execute(self, sql, params):
self.executions.append(params)
def close(self):
return None
class _Conn:
def __init__(self):
self.autocommit = False
self._cursor = _Cursor()
def cursor(self, cursor_factory=None):
return self._cursor
def close(self):
return None
conn = _Conn()
monkeypatch.setattr(sync, "get_kc_token", lambda: "tok")
monkeypatch.setattr(sync, "kc_get_users", lambda token: users)
monkeypatch.setattr(sync, "kc_update_attributes", lambda token, user, attrs: updated.append(user["id"]))
monkeypatch.setattr(sync.psycopg2, "connect", lambda **kwargs: conn)
sync.main()
# The already-provisioned user is synced without a Keycloak write; the
# disabled user never reaches the mailbox upsert.
assert updated == []
assert len(conn._cursor.executions) == 1

View File

@ -0,0 +1,96 @@
"""Retry, attribute-corruption, and skip-path edges for the Mailu sync job."""
from test_mailu_sync import _FakeResponse, _FakeSession, load_sync_module
def test_retry_helpers_with_zero_attempts_do_nothing(monkeypatch):
sync = load_sync_module(monkeypatch)
monkeypatch.setattr(
sync.psycopg2,
"connect",
lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not connect")),
)
assert sync.retry_request(
"request",
lambda: (_ for _ in ()).throw(AssertionError("must not run")),
attempts=0,
) is None
assert sync.retry_db_connect(attempts=0) is None
def test_kc_update_attributes_replaces_a_corrupt_attribute_map(monkeypatch):
sync = load_sync_module(monkeypatch)
current_resp = _FakeResponse({"username": "u1", "attributes": "corrupt"})
ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}})
sync.SESSION = _FakeSession(_FakeResponse({}), [current_resp, ok_resp])
sync.kc_update_attributes(
"token",
{"id": "u1", "username": "u1"},
{"mailu_app_password": "abc"},
)
assert sync.SESSION.put_called and sync.SESSION.get_calls == 2
def test_main_skips_disabled_users_and_settled_attribute_sets(monkeypatch):
sync = load_sync_module(monkeypatch)
monkeypatch.setattr(sync.bcrypt_sha256, "hash", lambda password: f"hash:{password}")
users = [
{
"id": "u_disabled",
"username": "disabled",
"email": "disabled@example.com",
"enabled": False,
"attributes": {"mailu_enabled": ["true"]},
},
{
"id": "u_settled",
"username": "settled",
"email": "settled@example.com",
"attributes": {
"mailu_enabled": ["true"],
"mailu_email": ["settled@example.com"],
"mailu_app_password": ["keepme"],
},
},
]
updated = []
class _Cursor:
def __init__(self):
self.executions = []
def execute(self, sql, params):
self.executions.append(params)
def close(self):
return None
class _Conn:
def __init__(self):
self.autocommit = False
self._cursor = _Cursor()
def cursor(self, cursor_factory=None):
return self._cursor
def close(self):
return None
conn = _Conn()
monkeypatch.setattr(sync, "get_kc_token", lambda: "tok")
monkeypatch.setattr(sync, "kc_get_users", lambda token: users)
monkeypatch.setattr(
sync,
"kc_update_attributes",
lambda token, user, attrs: updated.append(user["id"]),
)
monkeypatch.setattr(sync.psycopg2, "connect", lambda **kwargs: conn)
sync.main()
assert updated == []
assert [params["email"] for params in conn._cursor.executions] == [
"settled@example.com"
]

View File

@ -127,8 +127,64 @@ def test_listener_post_wait_keeps_running_request_successful(monkeypatch):
assert handler.responses == [200]
def test_listener_post_treats_non_object_json_as_plain_trigger(monkeypatch):
listener = load_listener_module(monkeypatch)
called = []
monkeypatch.setattr(listener, "_trigger_sync_async", lambda force=False: called.append(force) or True)
handler = _handler_for(listener, "[1, 2]")
handler.do_POST()
assert called == [False]
assert handler.responses == [202]
def test_listener_log_message_is_quiet(monkeypatch):
listener = load_listener_module(monkeypatch)
handler = listener.Handler.__new__(listener.Handler)
assert handler.log_message("ignored %s", "value") is None
def test_listener_post_treats_non_object_payloads_as_plain_triggers(monkeypatch):
listener = load_listener_module(monkeypatch)
called = []
monkeypatch.setattr(
listener,
"_trigger_sync_async",
lambda force=False: called.append(force) or True,
)
handler = _handler_for(listener, "[1, 2, 3]")
handler.do_POST()
assert called == [False]
assert handler.responses == [202]
def test_listener_module_serves_forever_when_run_directly(monkeypatch):
import http.server
import runpy
served = []
class _FakeServer:
def __init__(self, address, handler):
served.append((address, handler.__name__))
def serve_forever(self):
served.append("serving")
monkeypatch.setenv("MAILU_SYNC_WAIT_TIMEOUT_SEC", "0")
monkeypatch.setattr(http.server, "ThreadingHTTPServer", _FakeServer)
module_path = (
pathlib.Path(__file__).resolve().parents[2]
/ "services"
/ "mailu"
/ "scripts"
/ "mailu_sync_listener.py"
)
runpy.run_path(str(module_path), run_name="__main__")
assert served == [(("", 8080), "Handler"), "serving"]

View File

@ -2,7 +2,7 @@
apiVersion: batch/v1
kind: Job
metadata:
name: gitea-atlas-identity-bootstrap-3
name: gitea-atlas-identity-bootstrap-5
namespace: gitea
labels:
app.kubernetes.io/name: gitea
@ -34,6 +34,20 @@ spec:
nodeSelector:
node-role.kubernetes.io/worker: "true"
hardware: rpi5
initContainers:
- name: python-runtime
image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df
command: [/bin/sh, -ec]
args: ["cp -R /usr/local/. /python/"]
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
volumeMounts:
- {name: python-runtime, mountPath: /python}
containers:
- name: bootstrap
image: gitea/gitea:1.23
@ -80,6 +94,7 @@ spec:
- name: bootstrap
mountPath: /opt/bootstrap
readOnly: true
- {name: python-runtime, mountPath: /opt/python, readOnly: true}
volumes:
- name: data
persistentVolumeClaim:
@ -88,3 +103,6 @@ spec:
configMap:
name: gitea-atlas-identity-bootstrap
defaultMode: 0555
- name: python-runtime
emptyDir:
sizeLimit: 128Mi

View File

@ -15,6 +15,7 @@ configMapGenerator:
- name: gitea-atlas-identity-bootstrap
files:
- gitea_atlas_identity_ensure.sh=scripts/gitea_atlas_identity_ensure.sh
- gitea_branch_protection_check.py=scripts/gitea_branch_protection_check.py
generatorOptions:
disableNameSuffixHash: true

View File

@ -19,6 +19,7 @@ source_owner="${GITEA_SOURCE_OWNER:-bstein}"
managed_repositories="${GITEA_MANAGED_REPOSITORIES:-hermes-code-demo cassandra soteria pegasus metis ananke ariadne typhon titan-iac}"
contributor_team=Contributors
contributor_units='["repo.actions","repo.packages","repo.code","repo.issues","repo.wiki","repo.pulls","repo.releases","repo.projects"]'
protected_reviewer=bstein
die() {
echo "Gitea Atlas identity bootstrap failed: $*" >&2
@ -272,6 +273,46 @@ transfer_managed_repositories() {
trap - EXIT HUP INT TERM
}
ensure_default_branch_protection() {
repository=$1
repo_file=$(mktemp)
protections_file=$(mktemp)
response_file=$(mktemp)
status=$(api_request GET "/repos/${organization}/${repository}" '' "${repo_file}")
expect_status "${status}" 200 "read ${repository} metadata for branch protection"
default_branch=$(sed -n 's/.*"default_branch":"\([^"]*\)".*/\1/p' "${repo_file}")
case "${default_branch}" in
main|master) ;;
*) die "${repository} has unsupported default branch ${default_branch:-missing}" ;;
esac
status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}")
expect_status "${status}" 200 "list ${repository} branch protections"
if ! protection_state=$(
/opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \
"${protections_file}" "${default_branch}" "${protected_reviewer}"
); then
die "${repository} ${default_branch} protection differs from the human-review policy"
fi
if [ "${protection_state}" = ABSENT ]; then
payload="{\"rule_name\":\"${default_branch}\",\"enable_push\":true,\"enable_push_whitelist\":true,\"push_whitelist_usernames\":[\"${protected_reviewer}\"],\"push_whitelist_teams\":[],\"push_whitelist_deploy_keys\":false,\"enable_force_push\":false,\"enable_force_push_allowlist\":false,\"force_push_allowlist_usernames\":[],\"force_push_allowlist_teams\":[],\"force_push_allowlist_deploy_keys\":false,\"enable_merge_whitelist\":true,\"merge_whitelist_usernames\":[\"${protected_reviewer}\"],\"merge_whitelist_teams\":[],\"enable_approvals_whitelist\":true,\"approvals_whitelist_username\":[\"${protected_reviewer}\"],\"approvals_whitelist_teams\":[],\"required_approvals\":1,\"block_on_rejected_reviews\":true,\"block_on_outdated_branch\":true,\"dismiss_stale_approvals\":true,\"block_admin_merge_override\":true}"
status=$(api_request POST "/repos/${organization}/${repository}/branch_protections" "${payload}" "${response_file}")
expect_status "${status}" 201 "protect ${repository} ${default_branch}"
status=$(api_request GET "/repos/${organization}/${repository}/branch_protections" '' "${protections_file}")
expect_status "${status}" 200 "read back ${repository} branch protections"
protection_state=$(
/opt/python/bin/python3 /opt/bootstrap/gitea_branch_protection_check.py \
"${protections_file}" "${default_branch}" "${protected_reviewer}"
) || die "${repository} ${default_branch} protection differs from the human-review policy"
fi
[ "${protection_state}" = PRESENT ] || \
die "${repository} ${default_branch} protection did not become effective"
rm -f "${repo_file}" "${protections_file}" "${response_file}"
}
wait_for_gitea
ensure_user "${reconciler_user}" "${reconciler_email}" true
ensure_user "${hermes_user}" "${hermes_email}" false
@ -318,6 +359,10 @@ expect_status "${status}" 204 "add Hermes to Atlas contributors"
transfer_managed_repositories
for repository in ${managed_repositories}; do
ensure_default_branch_protection "${repository}"
done
vault_write gitea/atlas-reconciler "{\"data\":{\"username\":\"${reconciler_user}\",\"token\":\"${reconciler_token}\",\"base_url\":\"${public_url}\"}}"
vault_write hermes/developer-gitea "{\"data\":{\"username\":\"${hermes_user}\",\"email\":\"${hermes_email}\",\"token\":\"${hermes_token}\",\"base_url\":\"${public_url}\"}}"

View File

@ -0,0 +1,146 @@
#!/usr/bin/env python3
"""Validate the first effective Gitea protection for an Atlas primary branch."""
from __future__ import annotations
import argparse
import datetime
import json
import os
import stat
from pathlib import Path
MAX_INPUT = 1024 * 1024
MAX_RULES = 100
MAX_RULE_NAME = 255
SPECIAL = frozenset("*?\\[]{}")
class PolicyError(RuntimeError):
"""The protection response is ambiguous or violates human review policy."""
def _read_bounded(path: Path) -> bytes:
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
try:
metadata = os.fstat(descriptor)
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > MAX_INPUT:
raise PolicyError("branch protection response exceeds the safe limit")
value = os.read(descriptor, MAX_INPUT + 1)
finally:
os.close(descriptor)
if len(value) != metadata.st_size:
raise PolicyError("branch protection response changed while reading")
return value
def _is_plain(pattern: str) -> bool:
return not any(character in SPECIAL for character in pattern)
def _literal_rule(pattern: str) -> str:
"""Return one exact rule name or reject Gitea glob interpretation.
Gitea 1.23.8 delegates matching to gobwas/glob. Reimplementing that
grammar and its error quoting rules here would create a second policy
engine. The reconciler instead accepts only plain ASCII literals. This
deliberately rejects braces, ranges, escapes, malformed globs, and even
special rules that appear unrelated: an operator must remove ambiguity
before this job may attest the primary-branch boundary.
"""
if not pattern.isascii() or not 1 <= len(pattern) <= MAX_RULE_NAME:
raise PolicyError("branch protection rule name is invalid")
if not _is_plain(pattern):
raise PolicyError("glob branch protection rules require human review")
return pattern
def _created(value: object) -> datetime.datetime:
if not isinstance(value, str) or len(value) > 64:
raise PolicyError("branch protection creation time is invalid")
try:
result = datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError as exc:
raise PolicyError("branch protection creation time is invalid") from exc
if result.tzinfo is None:
raise PolicyError("branch protection creation time is invalid")
return result
def _required(reviewer: str) -> dict[str, object]:
return {
"enable_push": True,
"enable_push_whitelist": True,
"push_whitelist_usernames": [reviewer],
"push_whitelist_deploy_keys": False,
"enable_force_push": False,
"enable_merge_whitelist": True,
"merge_whitelist_usernames": [reviewer],
"enable_approvals_whitelist": True,
"approvals_whitelist_username": [reviewer],
"required_approvals": 1,
"block_on_rejected_reviews": True,
"block_on_outdated_branch": True,
"dismiss_stale_approvals": True,
"block_admin_merge_override": True,
}
def evaluate(value: bytes, branch: str, reviewer: str) -> str:
"""Return PRESENT/ABSENT or fail on ambiguous/effective policy drift."""
if len(value) > MAX_INPUT or branch not in {"main", "master"}:
raise PolicyError("branch protection input is invalid")
data = json.loads(value)
if not isinstance(data, list) or len(data) > MAX_RULES:
raise PolicyError("branch protection response is invalid")
ordered: list[tuple[int, datetime.datetime, int, dict[str, object]]] = []
for index, raw in enumerate(data):
if not isinstance(raw, dict):
raise PolicyError("branch protection entry is invalid")
priority = raw.get("priority")
rule_name = raw.get("rule_name")
if (
not isinstance(priority, int)
or isinstance(priority, bool)
or not 0 <= priority <= 1_000_000
or not isinstance(rule_name, str)
):
raise PolicyError("branch protection priority is ambiguous")
created = _created(raw.get("created_at"))
literal = _literal_rule(rule_name)
if literal == branch:
# With every rule constrained to a plain literal, Gitea's glob
# comparator cannot affect this result. Preserve API ordering:
# priority first (including legacy zero), creation time, then the
# stable response order for an otherwise exact tie.
ordered.append((priority, created, index, raw))
if not ordered:
return "ABSENT"
if len(ordered) != 1:
raise PolicyError("duplicate primary-branch protections are ambiguous")
effective = ordered[0][3]
for key, expected in _required(reviewer).items():
if effective.get(key) != expected:
raise PolicyError(
f"effective {branch} protection differs from human-review policy"
)
return "PRESENT"
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("response", type=Path)
parser.add_argument("branch", choices=("main", "master"))
parser.add_argument("reviewer")
args = parser.parse_args()
try:
value = _read_bounded(args.response)
print(evaluate(value, args.branch, args.reviewer))
except (OSError, json.JSONDecodeError, PolicyError) as exc:
print(f"branch protection check failed: {exc}", file=__import__("sys").stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,79 @@
# services/harbor/hermes-agent-immutability-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
name: harbor-hermes-agent-immutability-ensure-1
namespace: harbor
spec:
backoffLimit: 2
activeDeadlineSeconds: 600
template:
metadata:
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-run-as-user: "65532"
vault.hashicorp.com/agent-run-as-group: "65532"
vault.hashicorp.com/role: harbor-policy-bootstrap
vault.hashicorp.com/agent-inject-secret-harbor-admin-password: kv/data/atlas/harbor/harbor-core
vault.hashicorp.com/agent-inject-template-harbor-admin-password: |
{{- with secret "kv/data/atlas/harbor/harbor-core" -}}
{{ .Data.data.harbor_admin_password }}
{{- end -}}
spec:
serviceAccountName: harbor-policy-bootstrap
enableServiceLinks: false
restartPolicy: Never
nodeSelector:
hardware: rpi5
kubernetes.io/arch: arm64
node-role.kubernetes.io/worker: "true"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values: [titan-04, titan-14, titan-18, titan-19, titan-24]
securityContext:
fsGroup: 65532
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
containers:
- name: ensure
image: docker.io/library/python@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757
imagePullPolicy: IfNotPresent
command: [python3, /scripts/harbor_hermes_agent_immutability_ensure.py]
env:
- name: HARBOR_API_ORIGIN
value: https://registry.bstein.dev/api/v2.0
- name: HARBOR_ADMIN_PASSWORD_FILE
value: /vault/secrets/harbor-admin-password
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
readOnlyRootFilesystem: true
runAsGroup: 65532
runAsNonRoot: true
runAsUser: 65532
seccompProfile:
type: RuntimeDefault
volumeMounts:
- name: scripts
mountPath: /scripts
readOnly: true
- name: tmp
mountPath: /tmp
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: 250m, memory: 128Mi}
volumes:
- name: scripts
configMap:
name: harbor-hermes-agent-immutability-script
defaultMode: 0555
- name: tmp
emptyDir: {}

View File

@ -12,9 +12,14 @@ resources:
- certificate.yaml
- helmrelease.yaml
- vault-sync-deployment.yaml
- policy-bootstrap-serviceaccount.yaml
- hermes-agent-immutability-job.yaml
- bootstrap-jobs/cassandra-registry-ensure-job.yaml
- image.yaml
configMapGenerator:
- name: harbor-vault-entrypoint
files:
- scripts/vault-entrypoint.sh
- name: harbor-hermes-agent-immutability-script
files:
- harbor_hermes_agent_immutability_ensure.py=scripts/harbor_hermes_agent_immutability_ensure.py

View File

@ -0,0 +1,7 @@
# services/harbor/policy-bootstrap-serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: harbor-policy-bootstrap
namespace: harbor
automountServiceAccountToken: true

View File

@ -0,0 +1,378 @@
#!/usr/bin/env python3
"""Create and verify the narrowly scoped Hermes agent immutable-tag rule."""
from __future__ import annotations
import base64
import json
import os
import time
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Any
PROJECT = "bstein"
REPOSITORY_PATTERN = "hermes-agent"
TAG_PATTERN = "git-*-build-*"
PUBLISH_ROBOT = "robot$jenkins-pipelines"
EXPECTED_ORIGIN = "https://registry.bstein.dev/api/v2.0"
MAX_RESPONSE = 1_048_576
TRANSIENT_STATUSES = {429, 502, 503, 504}
EXPECTED_RULE = {
"disabled": False,
"action": "immutable",
"template": "immutable_template",
"tag_selectors": [
{
"kind": "doublestar",
"decoration": "matches",
"pattern": TAG_PATTERN,
}
],
"scope_selectors": {
"repository": [
{
"kind": "doublestar",
"decoration": "repoMatches",
"pattern": REPOSITORY_PATTERN,
}
]
},
}
class NoRedirect(urllib.request.HTTPRedirectHandler):
"""Prevent Basic credentials from following an unexpected redirect."""
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
return None
class HarborUnavailable(RuntimeError):
"""Harbor is not ready yet, rather than returning a policy decision."""
def normalized_rule(rule: dict[str, Any]) -> dict[str, Any]:
"""Return only the immutable contract fields Harbor must preserve."""
return {
"disabled": bool(rule.get("disabled", False)),
"action": rule.get("action"),
"template": rule.get("template"),
"tag_selectors": [
{
"kind": selector.get("kind"),
"decoration": selector.get("decoration"),
"pattern": selector.get("pattern"),
}
for selector in rule.get("tag_selectors") or []
if isinstance(selector, dict)
],
"scope_selectors": {
"repository": [
{
"kind": selector.get("kind"),
"decoration": selector.get("decoration"),
"pattern": selector.get("pattern"),
}
for selector in (rule.get("scope_selectors") or {}).get(
"repository", []
)
if isinstance(selector, dict)
]
},
}
def targets_hermes_builds(rule: dict[str, Any]) -> bool:
"""Detect a rule that claims this exact repository and tag selector."""
normalized = normalized_rule(rule)
return (
normalized["tag_selectors"] == EXPECTED_RULE["tag_selectors"]
and normalized["scope_selectors"] == EXPECTED_RULE["scope_selectors"]
)
class HarborClient:
"""Bounded same-origin client for Harbor's immutable-tag API."""
def __init__(self, origin: str, username: str, password: str) -> None:
normalized_origin = origin.rstrip("/")
if normalized_origin != EXPECTED_ORIGIN:
raise ValueError("Harbor API origin is not the pinned production API")
self.origin = normalized_origin
token = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
self.headers = {"Authorization": f"Basic {token}"}
self.opener = urllib.request.build_opener(NoRedirect())
def request(
self, method: str, path: str, payload: dict[str, Any] | None = None
) -> tuple[int, bytes, dict[str, str]]:
"""Issue one request, returning even non-2xx responses for strict checks."""
data = None
headers = dict(self.headers)
if payload is not None:
data = json.dumps(payload, separators=(",", ":")).encode()
headers["Content-Type"] = "application/json"
request = urllib.request.Request(
f"{self.origin}{path}", data=data, headers=headers, method=method
)
try:
response = self.opener.open(request, timeout=20)
except urllib.error.HTTPError as exc:
response = exc
except (urllib.error.URLError, TimeoutError) as exc:
raise HarborUnavailable("Harbor policy API is unavailable") from exc
with response:
body = response.read(MAX_RESPONSE + 1)
if len(body) > MAX_RESPONSE:
raise RuntimeError("Harbor response exceeded the size limit")
return int(response.status), body, dict(response.headers)
def list_rules(client: HarborClient) -> list[dict[str, Any]]:
"""Read and validate the complete small rule set for the project."""
path = f"/projects/{PROJECT}/immutabletagrules?page=1&page_size=100"
status, body, headers = client.request("GET", path)
if status in TRANSIENT_STATUSES:
raise HarborUnavailable(f"Harbor immutable rule list returned HTTP {status}")
if status != 200:
raise RuntimeError(f"Harbor immutable rule list returned HTTP {status}")
values = _json_list(body, "immutable rule")
_require_complete_page(values, headers, "immutable rule")
return values
def _json_list(body: bytes, label: str) -> list[dict[str, Any]]:
"""Decode one bounded Harbor list without accepting a partial shape."""
try:
value = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError(f"Harbor returned invalid {label} JSON") from exc
if not isinstance(value, list) or not all(isinstance(item, dict) for item in value):
raise RuntimeError(f"Harbor {label} list has an invalid shape")
return value
def _require_complete_page(
values: list[dict[str, Any]], headers: dict[str, str], label: str
) -> None:
"""Reject a truncated first page or a proxy that strips count evidence."""
raw_total = next(
(value for key, value in headers.items() if key.lower() == "x-total-count"),
None,
)
if raw_total is None or not str(raw_total).isdecimal():
raise RuntimeError(f"Harbor {label} list omitted a valid total count")
if int(raw_total) != len(values):
raise RuntimeError(f"Harbor {label} list was truncated")
def _allowed(access: dict[str, Any], resource: str, action: str) -> bool:
"""Match one positive robot permission, accepting Harbor's default effect."""
return (
access.get("resource") == resource
and access.get("action") == action
and access.get("effect") in (None, "", "allow")
)
def _publisher_scope(robot: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]:
"""Return the existing bstein scope after validating its push boundary."""
permissions = robot.get("permissions")
if not isinstance(permissions, list) or not all(
isinstance(item, dict) for item in permissions
):
raise RuntimeError("Jenkins Harbor robot permissions have an invalid shape")
matches = [
item
for item in permissions
if item.get("kind") == "project" and item.get("namespace") == PROJECT
]
if len(matches) != 1:
raise RuntimeError("Jenkins Harbor robot must have one existing bstein scope")
access = matches[0].get("access")
if not isinstance(access, list) or not all(isinstance(item, dict) for item in access):
raise RuntimeError("Jenkins Harbor robot bstein access has an invalid shape")
for required in (("repository", "pull"), ("repository", "push")):
if not any(_allowed(item, *required) for item in access):
raise RuntimeError("Jenkins Harbor robot lacks its existing pull/push boundary")
immutable_access = [
item for item in access if item.get("resource") == "immutable-tag"
]
if immutable_access and not (
len(immutable_access) == 1
and _allowed(immutable_access[0], "immutable-tag", "list")
):
raise RuntimeError("Jenkins Harbor robot has broader immutable-tag access")
return matches[0], access
def _verified_publisher(robot: dict[str, Any]) -> bool:
"""Check the exact identity and least-privilege policy after persistence."""
if (
robot.get("name") != PUBLISH_ROBOT
or robot.get("level") != "system"
or robot.get("editable") is not True
or robot.get("disable") is not False
):
return False
try:
_scope, access = _publisher_scope(robot)
except RuntimeError:
return False
return any(_allowed(item, "immutable-tag", "list") for item in access)
def ensure_publisher_can_read_rule(client: HarborClient) -> int:
"""Grant only immutable-tag:list to the existing Jenkins push robot."""
status, body, headers = client.request("GET", "/robots?page=1&page_size=100")
if status in TRANSIENT_STATUSES:
raise HarborUnavailable(f"Harbor robot list returned HTTP {status}")
if status != 200:
raise RuntimeError(f"Harbor robot list returned HTTP {status}")
robots = _json_list(body, "robot")
_require_complete_page(robots, headers, "robot")
matches = [
item
for item in robots
if item.get("name") == PUBLISH_ROBOT
]
if len(matches) != 1:
raise RuntimeError("expected exactly one Jenkins Harbor publisher robot")
robot_id = matches[0].get("id")
if not isinstance(robot_id, int) or robot_id < 1:
raise RuntimeError("Jenkins Harbor robot omitted a valid ID")
status, body, _headers = client.request("GET", f"/robots/{robot_id}")
if status in TRANSIENT_STATUSES:
raise HarborUnavailable(f"Harbor robot read returned HTTP {status}")
if status != 200:
raise RuntimeError(f"Harbor robot read returned HTTP {status}")
try:
robot = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError("Harbor returned invalid robot JSON") from exc
if not isinstance(robot, dict):
raise RuntimeError("Harbor robot response has an invalid shape")
if (
robot.get("name") != PUBLISH_ROBOT
or robot.get("level") != "system"
or robot.get("editable") is not True
or robot.get("disable") is not False
):
raise RuntimeError("Jenkins Harbor publisher identity is not active and exact")
scope, access = _publisher_scope(robot)
if any(_allowed(item, "immutable-tag", "list") for item in access):
return robot_id
# Harbor updates permissions through PUT /robots/{id}; its separate PATCH
# endpoint is the only secret-rotation operation. Preserve every current
# scope and field while adding the one read-only action.
access.append({"resource": "immutable-tag", "action": "list"})
payload = {
"name": robot["name"],
"description": robot.get("description") or "",
"level": robot["level"],
"disable": robot["disable"],
"permissions": robot["permissions"],
}
duration = robot.get("duration")
if duration is not None:
if not isinstance(duration, int):
raise RuntimeError("Jenkins Harbor robot duration has an invalid shape")
payload["duration"] = duration
# Keep the validated object reference live in the preserved permission list.
if scope.get("access") is not access:
raise RuntimeError("Jenkins Harbor robot permission normalization drifted")
status, _body, _headers = client.request(
"PUT", f"/robots/{robot_id}", payload
)
if status in TRANSIENT_STATUSES:
raise HarborUnavailable(f"Harbor robot update returned HTTP {status}")
if status != 200:
raise RuntimeError(f"Harbor robot policy update returned HTTP {status}")
for attempt in range(1, 6):
status, body, _headers = client.request("GET", f"/robots/{robot_id}")
if status == 200:
try:
persisted = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
persisted = None
if isinstance(persisted, dict) and _verified_publisher(persisted):
return robot_id
if attempt < 5:
time.sleep(attempt)
raise RuntimeError("Jenkins Harbor robot policy did not verify exactly")
def ensure_rule(client: HarborClient) -> int:
"""Create once, or validate the one exact enabled rule already present."""
rules = list_rules(client)
matches = [rule for rule in rules if targets_hermes_builds(rule)]
if len(matches) > 1:
raise RuntimeError("multiple Hermes agent immutable rules exist")
if matches:
if normalized_rule(matches[0]) != EXPECTED_RULE:
raise RuntimeError("Hermes agent immutable rule exists but is not enabled/exact")
rule_id = matches[0].get("id")
if not isinstance(rule_id, int) or rule_id < 1:
raise RuntimeError("Harbor immutable rule omitted a valid ID")
return rule_id
path = f"/projects/{PROJECT}/immutabletagrules"
status, _body, headers = client.request("POST", path, EXPECTED_RULE)
if status in TRANSIENT_STATUSES:
raise HarborUnavailable(f"Harbor immutable rule create returned HTTP {status}")
if status != 201:
raise RuntimeError(f"Harbor immutable rule create returned HTTP {status}")
location = headers.get("Location") or headers.get("location") or ""
api_path = urllib.parse.urlsplit(client.origin).path.rstrip("/")
expected_prefix = f"{api_path}{path}/"
if not location.startswith(expected_prefix):
raise RuntimeError("Harbor immutable rule create omitted the exact Location")
suffix = location[len(expected_prefix) :]
if not suffix.isdecimal() or int(suffix) < 1:
raise RuntimeError("Harbor immutable rule Location has an invalid ID")
# Harbor persists synchronously, but bounded retries distinguish a stale
# read from accepting an unverified policy.
for attempt in range(1, 6):
matches = [rule for rule in list_rules(client) if targets_hermes_builds(rule)]
if len(matches) == 1 and normalized_rule(matches[0]) == EXPECTED_RULE:
rule_id = matches[0].get("id")
if rule_id == int(suffix):
return rule_id
if attempt < 5:
time.sleep(attempt)
raise RuntimeError("created Harbor immutable rule did not verify exactly")
def main() -> int:
"""Load the runtime-only admin credential and enforce the tracked policy."""
origin = os.environ.get("HARBOR_API_ORIGIN", "")
password_file = Path(os.environ.get("HARBOR_ADMIN_PASSWORD_FILE", ""))
password = password_file.read_text(encoding="utf-8").strip()
if not password:
raise RuntimeError("Harbor admin password is empty")
client = HarborClient(origin, "admin", password)
for attempt in range(1, 13):
try:
rule_id = ensure_rule(client)
robot_id = ensure_publisher_can_read_rule(client)
break
except HarborUnavailable:
if attempt == 12:
raise
time.sleep(min(attempt * 2, 15))
print(
"Hermes agent immutable build-tag rule is active "
f"(id={rule_id}); publisher preflight access is active (robot={robot_id})"
)
return 0
if __name__ == "__main__": # pragma: no cover - exercised through main()
raise SystemExit(main())

View File

@ -0,0 +1,5 @@
# services/hermes-observer-bindings/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- rolebindings.yaml

View File

@ -0,0 +1,204 @@
# services/hermes-observer-bindings/rolebindings.yaml
apiVersion: v1
kind: List
items:
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: ai}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: bstein-dev-home}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: cassandra}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: cert-manager}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: climate}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: comms}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: crypto}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: default}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: finance}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: flux-system}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: game-stream}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: gitea}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: harbor}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: health}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: hermes}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: hermes-chat}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: hermes-triage-demo}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: jellyfin}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: jenkins}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: kube-node-lease}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: kube-public}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: kube-system}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: logging}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: longhorn-system}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: mailu-mailserver}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: maintenance}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: metallb-system}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: monitoring}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: nextcloud}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: openclaw}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: outline}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: planka}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: postgres}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: quality}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: sso}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: sui-metrics}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: traefik}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: vault}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: vaultwarden}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: hermes-agent-observer, namespace: veles}
subjects: [{kind: ServiceAccount, name: hermes-agent, namespace: hermes}]
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: hermes-agent-namespaced-observer-v2}

View File

@ -0,0 +1,5 @@
# services/hermes-observer-rbac/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- rbac.yaml

View File

@ -0,0 +1,112 @@
# services/hermes-observer-rbac/rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: hermes-agent-cluster-observer-v2
labels:
app.kubernetes.io/name: hermes-agent
app.kubernetes.io/part-of: hermes
rules:
- apiGroups: [""]
resources:
- namespaces
- nodes
- persistentvolumes
verbs: ["get", "list", "watch"]
- apiGroups: ["networking.k8s.io"]
resources: ["ingressclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources:
- csidrivers
- csinodes
- storageclasses
- volumeattachments
verbs: ["get", "list", "watch"]
- apiGroups: ["metrics.k8s.io"]
resources: ["nodes"]
verbs: ["get", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: hermes-agent-namespaced-observer-v2
labels:
app.kubernetes.io/name: hermes-agent
app.kubernetes.io/part-of: hermes
rules:
- apiGroups: [""]
resources:
- configmaps
- endpoints
- events
- persistentvolumeclaims
- pods
- pods/log
- replicationcontrollers
- resourcequotas
- services
verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
resources:
- controllerrevisions
- daemonsets
- deployments
- replicasets
- statefulsets
verbs: ["get", "list", "watch"]
- apiGroups: ["batch"]
resources: ["cronjobs", "jobs"]
verbs: ["get", "list", "watch"]
- apiGroups: ["autoscaling"]
resources: ["horizontalpodautoscalers"]
verbs: ["get", "list", "watch"]
- apiGroups: ["networking.k8s.io"]
resources: ["ingresses", "networkpolicies"]
verbs: ["get", "list", "watch"]
- apiGroups: ["discovery.k8s.io"]
resources: ["endpointslices"]
verbs: ["get", "list", "watch"]
- apiGroups: ["policy"]
resources: ["poddisruptionbudgets"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["csistoragecapacities"]
verbs: ["get", "list", "watch"]
- apiGroups: ["metrics.k8s.io"]
resources: ["pods"]
verbs: ["get", "list"]
- apiGroups:
- helm.toolkit.fluxcd.io
- image.toolkit.fluxcd.io
- kustomize.toolkit.fluxcd.io
- notification.toolkit.fluxcd.io
- source.toolkit.fluxcd.io
resources: ["*"]
verbs: ["get", "list", "watch"]
- apiGroups: ["longhorn.io"]
resources:
- backingimages
- engines
- instancemanagers
- nodes
- replicas
- settings
- volumes
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: hermes-agent-cluster-observer-v2
labels:
app.kubernetes.io/name: hermes-agent
app.kubernetes.io/part-of: hermes
subjects:
- kind: ServiceAccount
name: hermes-agent
namespace: hermes
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: hermes-agent-cluster-observer-v2

View File

@ -0,0 +1,99 @@
# services/hermes-scm-broker/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: hermes-scm-broker
namespace: hermes-scm
labels:
app: hermes-scm-broker
spec:
replicas: 1
revisionHistoryLimit: 2
selector:
matchLabels:
app: hermes-scm-broker
template:
metadata:
labels:
app: hermes-scm-broker
annotations:
ai.bstein.dev/config-rev: scm-boundary-v2
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-scm-broker
vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea
vault.hashicorp.com/agent-inject-template-gitea-token: |
{{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
{{ .Data.data.token }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 10m
vault.hashicorp.com/agent-requests-mem: 32Mi
vault.hashicorp.com/agent-limits-cpu: 50m
vault.hashicorp.com/agent-limits-mem: 64Mi
spec:
serviceAccountName: hermes-scm-broker
automountServiceAccountToken: true
securityContext:
fsGroup: 10000
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/arch
operator: In
values: [arm64]
- key: node-role.kubernetes.io/worker
operator: In
values: ["true"]
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: hardware
operator: In
values: [rpi5]
containers:
- name: broker
image: registry.bstein.dev/bstein/hermes-agent
imagePullPolicy: IfNotPresent
command:
- /opt/hermes/.venv/bin/python
- /opt/broker/scm_broker.py
ports:
- {name: http, containerPort: 9081, protocol: TCP}
readinessProbe:
httpGet: {path: /healthz, port: http}
initialDelaySeconds: 3
periodSeconds: 10
livenessProbe:
httpGet: {path: /healthz, port: http}
initialDelaySeconds: 15
periodSeconds: 30
failureThreshold: 10
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
volumeMounts:
- {name: broker-code, mountPath: /opt/broker, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 50m, memory: 128Mi}
limits: {cpu: "1", memory: 768Mi}
volumes:
- name: broker-code
configMap:
name: hermes-scm-boundary-v2
defaultMode: 0555
- name: tmp
emptyDir:
sizeLimit: 1Gi

View File

@ -0,0 +1,11 @@
# services/hermes-scm-broker/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: hermes-scm
resources:
- service.yaml
- deployment.yaml
- networkpolicy.yaml
images:
- name: registry.bstein.dev/bstein/hermes-agent
digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7

View File

@ -0,0 +1,52 @@
# services/hermes-scm-broker/networkpolicy.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-scm-broker-isolation
namespace: hermes-scm
spec:
podSelector:
matchLabels:
app: hermes-scm-broker
policyTypes: [Ingress, Egress]
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: hermes
podSelector:
matchLabels:
app: hermes-agent
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: hermes
podSelector:
matchLabels:
app: hermes-execution-mediator
ports:
- {protocol: TCP, port: 9081}
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- {protocol: UDP, port: 53}
- {protocol: TCP, port: 53}
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: vault
podSelector:
matchLabels:
app: vault
ports:
- {protocol: TCP, port: 8200}
- to:
- ipBlock:
cidr: 192.168.22.9/32
ports:
- {protocol: TCP, port: 443}

View File

@ -0,0 +1,13 @@
# services/hermes-scm-broker/service.yaml
apiVersion: v1
kind: Service
metadata:
name: hermes-scm-broker
namespace: hermes-scm
spec:
selector:
app: hermes-scm-broker
ports:
- name: http
port: 9081
targetPort: http

View File

@ -0,0 +1,5 @@
# services/hermes-scm-namespace/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml

View File

@ -0,0 +1,14 @@
# services/hermes-scm-namespace/namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
name: hermes-scm
labels:
app.kubernetes.io/name: hermes-scm-broker
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: hermes-scm-broker
namespace: hermes-scm
automountServiceAccountToken: true

View File

@ -102,6 +102,110 @@ console tails and named artifact contents in its deterministic bundle. A report
must say `retained Ariadne evidence` when that fallback is used; it must not
pretend direct Jenkins access succeeded.
## Publishing an agent image after review
The `hermes-agent-image` Jenkins job is the only supported agent image builder.
It runs daemonless Kaniko v1.23.2 under the unbound `hermes-image-builder`
ServiceAccount without a service-account token, host socket,
privileged container, or writable Git credential. It accepts only an exact
40-character revision that is both the checked-out commit and current
`atlas/titan-iac` `main`, so a human must merge the source PR first.
The build pod requires an ARM64 worker and gives `hardware=rpi5` the maximum
scheduler preference. Healthy rpi4 workers remain valid fallback capacity when
the rpi5 pool cannot fit the full request. Cordoned, unhealthy, amd64, and
reserved last-resort nodes remain excluded by the required placement boundary.
Kaniko runs as UID 0 because it must unpack an image root filesystem and enter
that filesystem for Dockerfile `RUN` instructions. Its capability set is the
minimum proven by an exact ARM64 no-push build of this Dockerfile:
`CHOWN`, `FOWNER`, `DAC_OVERRIDE`, `SETGID`, and `SETUID`. `SETGID` is required
because pinned Kaniko applies the base image's supplementary group list before
each Dockerfile `RUN`; `SETUID` lets apt drop privileges to its `_apt` account
while downloading package indexes. It still has no privilege
escalation, service-account token, host path, daemon socket, or Docker/BuildKit
TCP endpoint and uses the runtime-default seccomp profile. This is residual
root-in-the-build-pod risk, bounded to disposable `emptyDir` storage and a
human-reviewed `main` revision; Dockerfile changes require the same scrutiny as
executable cluster code.
Pinned Kaniko v1.23.2 parses Dockerfile `RUN` heredocs but does not materialize
their inline files when executing a command. The Jenkins lane therefore enables
a bounded compatibility replay after copying the reviewed Dockerfile and runner
from the same checked-out commit. Before every replay, the runner inventories
the entire Dockerfile as logical instructions, including split opcodes,
operators, and either Dockerfile escape character. It rejects every heredoc
form outside the exact nine Node/Python blocks before launching an interpreter.
Each accepted body is then replayed immediately after its corresponding Kaniko
`RUN`, preserving source order before any dependent regression. Docker and
BuildKit keep their native behavior because the compatibility argument defaults
off. The final TypeScript build, Python compile, and source assertions remain
mandatory, so an omitted or drifted replay fails before any image can pass
release verification.
From agent.hermes, trigger that one fixed job with:
```sh
jenkins_image_build_trigger.py '<reviewed full main commit>'
```
The helper has no general Jenkins credential or caller-selectable job name. Its
Vault-projected token is bound by Jenkins only to `hermes-agent-image`. The job
also requires its fixed publish confirmation. Each run claims the unique tag
`git-<reviewed-sha>-build-<jenkins-build-number>` and refuses to overwrite an
existing tag. It cross-checks Kaniko's digest and tagged-image evidence, then
independently reads the pushed tag from Harbor before archiving a JSON record
and Flux digest patch. A successful post-condition then revalidates the exact
six-file evidence set and archives those six explicit paths with empty archives
forbidden. Apply that patch on a new branch and submit it for human review; the
build never changes Git, reconciles Flux, or deploys by itself.
The helper posts only to the fixed HTTPS Build Token Root endpoint and accepts
only its real queue responses: HTTP 201 or a non-followed HTTP 303 with an exact
same-origin `/queue/item/<number>/` location. An unauthenticated request is
denied, and the token cannot select, configure, read, or administer another
Jenkins job. The pipeline independently rejects any commit that is not the
current `origin/main`, preserving the human merge/review boundary.
Harbor independently enforces an enabled immutable-tag rule scoped to only the
`bstein/hermes-agent` repository and `git-*-build-*` tags. A second manifest PUT,
retag, or deletion is rejected by Harbor even if a caller bypasses the Jenkins
preflight. A revisioned, Flux-tracked policy Job creates or verifies that exact
rule with a runtime-only Vault credential; it refuses to alter a conflicting
rule and retries only explicit transport/readiness failures while Harbor
starts. The same Job grants the existing Jenkins publisher only
`immutable-tag:list` on `bstein`, preserving its other project scopes and never
calling Harbor's separate secret-rotation endpoint. The pipeline uses that
read-only permission to require the exact enabled rule before Kaniko starts.
That check also makes an already-running Jenkins controller fail closed during
rollout; Jenkins cannot become Ready on the new revision before the policy Job
succeeds.
The tracked rollout order is deliberate: the revisioned Vault role Job must
complete before the Vault Kustomization becomes Ready. A separate Flux
Kustomization then runs the seed Job under the dedicated
`hermes-jenkins-token-seed` identity, which can only create/read the exact
`kv/data/atlas/hermes/developer-jenkins` path and request random bytes. Jenkins
depends on both that seed and the Harbor policy Job; Hermes depends on Jenkins.
The seeder uses KV-v2 CAS create-only semantics. It never changes an existing
token, never fills a missing field in an existing secret, and fails closed when
it cannot distinguish absence from a read error. The recurring Vault
configuration job reconciles the same narrow role and policy.
Rotate this fixed-job token only as a coordinated operator action:
1. Stop new `hermes-agent-image` triggers and wait for its queue and executor to
drain.
2. Read the current KV-v2 metadata version for
`kv/atlas/hermes/developer-jenkins`.
3. Generate a fresh value from Vault and patch only `build_token` with
`vault kv patch -cas=<current-version>`. Keep the value in a mode-0600
temporary file or standard input, never a command argument or log.
4. Roll Jenkins first and wait until it is Ready, then roll `hermes-agent` and
wait until it is Ready. Do not resume triggers between those two consumers.
5. Run one reviewed-main canary. If rollback is required, repeat the CAS patch
with the previous value as another coordinated rotation; never delete the
secret to make the seeder recreate it.
## The actual supervised triage algorithm
1. Classify the request as test/build triage, service health, or alert tuning.
@ -257,6 +361,60 @@ Use this short explanation:
leading different real incidents, improving the skill after failures, and
teaching the architecture without prompts.
## Distributed execution-pool rollout boundary
- Do not reconcile the pool until the approved credential owner has provisioned
six distinct Vault fields: `execution_worker_{0,1,2}_claude_credentials_json`
and `execution_worker_{0,1,2}_codex_auth_json`. Each ordinal needs an
independent account or refresh-token lineage; copying one rotating refresh
token into multiple fields recreates the lost-update failure this design
prevents.
- Worker credential files live on separate ordinal-owned RWO `provider-access`
claims. Provider refresh updates these durable private copies across Pod
restarts; they are never synchronized back to Vault. Rotate one bootstrap
credential at a time through the reviewed Vault workflow and reinitialize
only that ordinal after human approval.
- Existing task worktrees remain on the single local owner lane. Only tasks
without `workspace_path` enter the distributed pool, where each ordinal owns
one RWO checkout. Repository and base-branch identity comes from the canonical
board registry rather than task-supplied metadata.
- Model Pods have no pool key, broker mount, broker egress, or Kubernetes token.
Ordinal mediator Deployments are separate network identities that co-mount the
matching workspace PVC; they alone authenticate exact-run results and reach the
PR14 SCM broker.
- The ordinal workspace claim is `ReadWriteMany` because the worker and its
mediator both mount it. With `ReadWriteOnce` a drain or preemption that moved
only one of them left the other holding the attachment on the old node, and
the ordinal deadlocked on Multi-Attach until an operator deleted a Pod.
Colocation is now a scheduling preference and both Pods run at `scavenger`
priority, so either can be evicted and rescheduled on its own. There is
deliberately no PodDisruptionBudget: this is a best-effort pool and a budget
would impede node drains. Claims only one Pod mounts stay `ReadWriteOnce`.
- `lease_failed` is a *retryable* durable state, not a terminal one. It means the
ordinal was released but Kanban has not confirmed the outcome, so every
maintenance pass retries it. Only `finalized` and `stale` are terminal, and
only terminal rows are garbage-collected, so no run is dropped before the pool
knows what happened to it. If the log keeps reporting a deferred lease park for
the same run, Kanban is refusing an exact-run park on a task it still reports
as running: inspect that task rather than deleting the row.
- The broker accepts branch creation only. A pool retry that adds commits
therefore publishes a new `<branch>-attempt-<n>` (or, failing that,
`<branch>-<head12>`) ref in the same reviewed namespace instead of updating the
ref a previous attempt published, and adopts an existing ref that already
points at the exact head rather than pushing again. Expect one draft per
published ref; each is still human-reviewed. A submission that cannot land
blocks the run with the reason and leaves the commits on the ordinal's
workspace, so prior work is recoverable.
- Provider CLIs install once per pinned version onto a durable per-ordinal
`tools` claim, re-verified against the real binaries so a pruned cache
reinstalls. The pool is intentionally absent from the `hermes` Kustomization's
`healthChecks`: gating that 10m window on a best-effort pool would stall
`hermes-chat` and `hermes-observer-bindings`, which `dependsOn: hermes`.
- A hashed execution-pool ConfigMap, protocol-version readiness checks, and
versioned SCM boundary name make code/config changes controlled rollouts. A
rollout is still a human-reviewed operation; this repository change does not
reconcile or deploy it.
## Your shortest path to fluency
1. Explain the request diagram without looking.

View File

@ -88,6 +88,14 @@ data:
auto_decompose: true
auto_decompose_per_tick: 2
dispatch_stale_timeout_seconds: 14400
# Steady-state quota-aware routing for the direct CLI lane: below this
# remaining-percent a provider stops receiving NEW cli-auto work while
# active work finishes; lane-observed capacity/auth failures exclude a
# provider until the cooldown elapses, its quota window resets, or a
# fresh success is recorded.
provider_quota_min_remaining_percent: 15
provider_capacity_cooldown_seconds: 300
provider_auth_cooldown_seconds: 3600
model_catalog:
enabled: true
@ -189,9 +197,9 @@ data:
not replace these coordinator-wide rules. Never call `kanban_show` without
a known, non-empty task ID. Ad-hoc inspection and acceptance checks do not
need a synthetic Kanban lookup, and must load a skill only when its workflow
materially applies. Atlas HTTPS Git authentication is already supplied by
the runtime-only `GIT_ASKPASS`; use it without reading or exposing the
credential. Coordinator guidance lives at
materially applies. Atlas Git access is supplied by the isolated SCM
broker; no repository token is present in this pod. Use only configured
broker remotes and clients. Coordinator guidance lives at
`/opt/data/workspace/AGENTS.md` when more detail is needed.
The Jetson classifier is mandatory for AUTO selection. Switchyard may use
@ -295,70 +303,75 @@ data:
objective's difficulty warrants it; otherwise synthesize at the original
effort.
This owner-only pod has cluster-admin access across Atlas, including logs,
Secrets, exec, port-forwarding, rollout operations, and Flux reconciliation.
Prefer the titan-iac Git/Flux workflow for every durable cluster change;
direct operations are available for explicit operator requests, incident
recovery, and verification, and must be followed by a matching source-of-
truth change when they alter desired state. Never expose credentials in
chat or logs. Triage belongs at triage.hermes.bstein.dev.
This owner-only pod has scoped read-only Kubernetes diagnostics across
Atlas, including resource status, events, logs, and Flux/Helm evidence. It
cannot read Secrets, exec or attach to pods, create service-account tokens,
mutate workloads or RBAC, or reconcile Flux. Put every durable cluster
change on a reviewed titan-iac branch. Never expose credentials in chat or
logs. Triage belongs at triage.hermes.bstein.dev.
## Atlas engineering access
The Atlas organization has private visibility. Repository visibility is
preserved per project and may be public or private; do not infer a
repository's visibility from the organization setting. Repositories are
canonical at `https://scm.bstein.dev/atlas/<repo>.git`. HTTPS Git
authentication is already supplied through `GIT_ASKPASS`. Verify the remote
canonical at `https://scm.bstein.dev/atlas/<repo>.git`; worker Git traffic
uses `http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081/git/atlas/<repo>.git`.
Verify the broker remote
and cleanly separate pre-existing changes, create a task branch, run the
repository's tests, use `git push --dry-run` when proving access, and push a
real branch only when the requested implementation is review-ready. Never
force-push.
`scm.bstein.dev` is Gitea, not GitHub. Never load or follow a GitHub/`gh`
skill for an Atlas remote, and do not interpret an unauthenticated Gitea
HTTP 404 as a missing private repository. Use authenticated Git for clone,
fetch, and push. For pull-request metadata, comments, and merges, call
`/opt/coordinator/gitea_api.py METHOD /api/v1/...`; it injects the runtime
Vault token without exposing it to the command line, environment, output,
or transcript. Before a consequential merge, independently verify the
exact base/head ancestry and diff, run the relevant tests, and confirm the
candidate Jenkins result. Prefer the internal endpoint in
`JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use the
existing cluster access to inspect the controller's job/build files and
logs rather than guessing a public hostname. The preferred read-only path
is `/opt/coordinator/jenkins_build_evidence.py JOB [--branch BRANCH]
[--commit SHA] --wait`; it distinguishes a genuinely terminal build from
nested Jenkins execution metadata and returns bounded JSON/log evidence.
With a branch, the helper also tries the conventional `JOB-branches`
multibranch name. For JSON Gitea mutations, prefer repeatable shell-safe
`--field KEY=VALUE` arguments over hand-quoted JSON.
After merging, observe the default-branch build to a terminal result and
report exact commit, build, and test evidence.
`scm.bstein.dev` is Forgejo/Gitea, not GitHub. Never load or follow a
GitHub/`gh` skill for an Atlas remote, and do not interpret an
unauthenticated Gitea HTTP 404 as a missing private repository. Use
brokered Git for clone, fetch, and creation of a new namespaced feature
branch. Existing-ref updates, protected refs, deletion, and force-push are
rejected. The broker inflates and scans every pushed object, so thin
packs are rejected; always push with `git push --no-thin` so the pack
is self-contained. For bounded
repository/pull-request evidence or to create a review-ready draft PR, load
`$manage-atlas-pull-requests` and use `/opt/scm/gitea_api.py`. The
client carries no repository credential; a separate least-authority
workload performs the allowed operation. It permits only Atlas-scoped
metadata reads and verified same-repository draft creation. Updates, merge,
approve, close, delete, comments, repository administration, and force-push
are unavailable; never bypass the client with raw HTTP. Leave every PR
unmerged for Brad's review. Independently
verify the exact base/head ancestry and diff, run the relevant tests, and
confirm the candidate Jenkins result before presenting a handoff. Prefer
the internal endpoint in
`JENKINS_BASE_URL`; when Jenkins API authorization prevents a read, use
Kubernetes pod logs and status evidence without exec or secret access.
Do not bypass the boundary to inspect controller files. If Brad separately
reports that he merged the PR, observe
the default-branch build to a terminal result and report exact commit,
build, and test evidence.
The workspace already has a non-secret Hermes Git author identity. Do not
use `git reset --hard`, even in a fresh clone; use a detached worktree or a
clean branch switch when comparing revisions, and preserve any unexpected
file as user state. Use the Gitea pull-request merge endpoint for an open
PR so both Git history and PR state remain auditable. If an authorized
manual merge already reached the base branch, reconcile the open PR with
`Do=manually-merged` and its exact merge commit instead of leaving stale
review state.
file as user state. PR publication ends at the verified open draft; Brad
owns review and merge authority.
The terminal PATH contains the pinned operator tools. Start cluster work
with `kubectl config current-context`, read-only status/events/logs, and the
relevant `titan-iac` manifests. Put durable desired-state changes on a
reviewable `titan-iac` branch, validate Kustomize and client dry-run, then
use Flux reconciliation after the tracked change is published. Direct
`kubectl` mutations are for explicit incident recovery or ephemeral
verification, not ordinary delivery.
relevant `titan-iac` manifests. The broker namespace is deliberately
excluded from agent RBAC. Kubernetes cannot express a deny on one namespace
inside an all-namespace list, so enumerate namespaces and run namespaced
diagnostics instead of relying on `kubectl ... --all-namespaces`. Put
durable desired-state changes on a
reviewable `titan-iac` branch and validate Kustomize and client dry-run.
Brad owns merge and Flux reconciliation. Direct `kubectl` mutation is not
available to the agent; hand an explicit incident mutation back to Brad.
Node SSH uses a dedicated audited Hermes identity: `ssh titan-04` (or any
current Kubernetes node name). Host keys are pinned and password fallback
is disabled. Use SSH only for host-level evidence or repairs that cannot be
performed through Kubernetes. Read first, identify the exact node and
impact, avoid fleet-wide destructive commands, and reflect persistent host
configuration in the appropriate tracked provisioning source.
Node SSH uses the dedicated locked-password `hermes-agent` OS account:
`ssh titan-04` (or any current Kubernetes node name). Host keys are pinned,
password fallback is disabled, and the account has no sudo, disk, runtime,
or Kubernetes-storage authority. Use it for unprivileged host evidence;
hand privileged host repair back to Brad and reflect persistent changes in
the tracked provisioning source.
Treat every credential, SSH identity, host-trust record, and
credential-bearing access-client state as runtime-only Vault data. Keep
@ -395,7 +408,7 @@ data:
<low|medium|high|xhigh> [model]` for a persistent override. The first native
Codex worker requires one device-code login; refreshed provider credentials
persist through Vault while client caches remain disposable. The owner
workspace includes cluster-admin Kubernetes
access plus `kubectl`, `flux`, `helm`, `kustomize`, `vault`, `sops`, `age`,
workspace includes scoped read-only Kubernetes diagnostics plus `kubectl`,
`flux`, `helm`, `kustomize`, `vault`, `sops`, `age`,
`terraform`, `k9s`, `jq`, `yq`, `gh`, Git, SSH, Python, Node, the
browser/computer tools, and the native provider CLIs.

View File

@ -25,7 +25,7 @@ spec:
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
ai.bstein.dev/config-rev: "20260817-ai-usage-exporter-split-v2"
ai.bstein.dev/config-rev: "20260818-ai-usage-exporter-split-v3"
prometheus.io/scrape: "true"
prometheus.io/path: /metrics
prometheus.io/port: "9010"
@ -46,15 +46,10 @@ spec:
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ .Data.data.codex_auth_json }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/developer-gitea
vault.hashicorp.com/agent-inject-template-gitea-token: |
{{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
{{ .Data.data.token }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-gitea-username: kv/data/atlas/hermes/developer-gitea
vault.hashicorp.com/agent-inject-template-gitea-username: |
{{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
{{ .Data.data.username }}
vault.hashicorp.com/agent-inject-secret-jenkins-image-build-token: kv/data/atlas/hermes/developer-jenkins
vault.hashicorp.com/agent-inject-template-jenkins-image-build-token: |
{{- with secret "kv/data/atlas/hermes/developer-jenkins" -}}
{{ .Data.data.build_token }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-node-ssh-private-key: kv/data/atlas/hermes/developer-ssh
vault.hashicorp.com/agent-inject-template-node-ssh-private-key: |
@ -186,9 +181,8 @@ spec:
chmod 0600 "${profile_env}"
chown 10000:10000 "${profile_env}"
done
upsert_env GIT_ASKPASS /opt/coordinator/gitea_askpass.sh
upsert_env GIT_TERMINAL_PROMPT 0
upsert_env GITEA_BASE_URL https://scm.bstein.dev
upsert_env HERMES_SCM_BROKER_URL http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081
upsert_env JENKINS_BASE_URL http://jenkins.jenkins.svc.cluster.local:8080
upsert_env ARIADNE_BASE_URL http://ariadne.maintenance.svc.cluster.local
upsert_env VICTORIA_METRICS_URL http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428
@ -663,6 +657,8 @@ spec:
- {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py}
- {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true}
- {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}
- {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true}
- {name: scm-boundary, mountPath: /opt/scm, readOnly: true}
- {name: routing-catalog, mountPath: /routing-catalog, readOnly: true}
- {name: tmp, mountPath: /tmp}
startupProbe:
@ -826,6 +822,8 @@ spec:
- {name: subprocess-secret-patch, mountPath: /opt/hermes/tools/process_registry.py, subPath: process_registry.py}
- {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true}
- {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}
- {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true}
- {name: scm-boundary, mountPath: /opt/scm, readOnly: true}
- {name: routing-catalog, mountPath: /routing-catalog, readOnly: true}
- {name: tmp, mountPath: /tmp}
- {name: ttyd-index, mountPath: /ttyd-index, readOnly: true}
@ -866,6 +864,8 @@ spec:
. /opt/data/.env
set +a
exec /opt/hermes/.venv/bin/python /opt/coordinator/cli_lane_runner.py
ports:
- {name: lane-metrics, containerPort: 9011, protocol: TCP}
env:
- {name: HERMES_HOME, value: /opt/data}
- {name: HERMES_AUTH_FILE, value: /runtime-access/hermes-auth.json}
@ -875,6 +875,9 @@ spec:
- {name: KUBECONFIG, value: /opt/data/home/.kube/config}
- {name: PYTHONPATH, value: /opt/hermes}
- {name: HERMES_CLI_LANE_CONCURRENCY, value: "2"}
- {name: HERMES_CLI_HEALTH_MAX_AGE_SECONDS, value: "60"}
- {name: HERMES_CLI_LANE_METRICS_PORT, value: "9011"}
- {name: HERMES_CLI_QUOTA_METRICS_URL, value: "http://127.0.0.1:9010/metrics"}
- {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
- {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
securityContext:
@ -894,7 +897,18 @@ spec:
- {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
- {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true}
- {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}
- {name: atlas-pr-skill, mountPath: /opt/data/workspace/skills/manage-atlas-pull-requests, readOnly: true}
- {name: scm-boundary, mountPath: /opt/scm, readOnly: true}
- {name: tmp, mountPath: /tmp}
readinessProbe:
exec:
command:
- /opt/hermes/.venv/bin/python
- /opt/coordinator/cli_lane_capabilities.py
initialDelaySeconds: 2
periodSeconds: 5
timeoutSeconds: 5
failureThreshold: 3
resources:
requests: {cpu: 100m, memory: 256Mi}
limits: {cpu: "2", memory: 6Gi}
@ -1188,6 +1202,16 @@ spec:
- {key: dashboard-api.py, path: dashboard/plugin_api.py}
- {key: dashboard-index.js, path: dashboard/dist/index.js}
- {key: dashboard-style.css, path: dashboard/dist/style.css}
- name: atlas-pr-skill
configMap:
name: hermes-atlas-pr-skill
items:
- {key: SKILL.md, path: SKILL.md}
- {key: openai.yaml, path: agents/openai.yaml}
- name: scm-boundary
configMap:
name: hermes-scm-boundary-v2
defaultMode: 0555
- name: image-policy
configMap:
name: hermes-image-policy

View File

@ -11,9 +11,9 @@ users:
user:
tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
contexts:
- name: atlas-owner
- name: atlas-observer
context:
cluster: atlas
user: hermes-agent
namespace: default
current-context: atlas-owner
current-context: atlas-observer

View File

@ -1,16 +0,0 @@
# services/hermes/agent-rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: hermes-agent-cluster-admin
labels:
app.kubernetes.io/name: hermes-agent
app.kubernetes.io/part-of: hermes
subjects:
- kind: ServiceAccount
name: hermes-agent
namespace: hermes
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin

View File

@ -0,0 +1,66 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: hermes-agent
namespace: hermes
spec:
template:
metadata:
annotations:
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
{{- end }}
spec:
containers:
- name: cli-lane-runner
env:
- {name: HERMES_CLI_LANE_OWNED_WORKSPACES_ONLY, value: "true"}
- {name: HERMES_CLI_LANE_CONCURRENCY, value: "1"}
- name: execution-pool-coordinator
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_coordinator.py]
env:
- {name: HERMES_HOME, value: /opt/data}
- {name: HOME, value: /opt/data/home}
- {name: PYTHONPATH, value: /opt/hermes}
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- {name: PATH, value: /opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: home, mountPath: /opt/data}
- {name: runtime-access, mountPath: /pool-access/execution-pool-key, subPath: execution-pool-key, readOnly: true}
- {name: execution-pool-code, mountPath: /opt/coordinator, readOnly: true}
- {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
- {name: tmp, mountPath: /tmp}
ports:
- {name: execution-pool, containerPort: 9007, protocol: TCP}
startupProbe:
httpGet: {path: /ready, port: execution-pool}
periodSeconds: 5
failureThreshold: 60
readinessProbe:
httpGet: {path: /ready, port: execution-pool}
periodSeconds: 10
livenessProbe:
httpGet: {path: /ready, port: execution-pool}
initialDelaySeconds: 30
periodSeconds: 30
resources:
requests: {cpu: 50m, memory: 128Mi}
limits: {cpu: 500m, memory: 512Mi}
volumes:
- name: execution-pool-code
configMap:
name: hermes-execution-pool
defaultMode: 0555

View File

@ -0,0 +1,476 @@
# Privileged HMAC and SCM mediation runs outside the model Pods. Each mediator is
# colocated with exactly one ordinal's RWO workspace but has a distinct network
# identity, private HMAC derivation, and durable integrity state.
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: hermes-execution-mediator-state-0, namespace: hermes}
spec:
accessModes: [ReadWriteOnce]
storageClassName: astreae
resources: {requests: {storage: 1Gi}}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: hermes-execution-mediator-state-1, namespace: hermes}
spec:
accessModes: [ReadWriteOnce]
storageClassName: astreae
resources: {requests: {storage: 1Gi}}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: hermes-execution-mediator-state-2, namespace: hermes}
spec:
accessModes: [ReadWriteOnce]
storageClassName: astreae
resources: {requests: {storage: 1Gi}}
---
apiVersion: v1
kind: Service
metadata: {name: hermes-execution-mediator-0, namespace: hermes}
spec:
selector: {app: hermes-execution-mediator, pool-ordinal: "0"}
ports: [{name: mediator, port: 9009, targetPort: mediator}]
---
apiVersion: v1
kind: Service
metadata: {name: hermes-execution-mediator-1, namespace: hermes}
spec:
selector: {app: hermes-execution-mediator, pool-ordinal: "1"}
ports: [{name: mediator, port: 9009, targetPort: mediator}]
---
apiVersion: v1
kind: Service
metadata: {name: hermes-execution-mediator-2, namespace: hermes}
spec:
selector: {app: hermes-execution-mediator, pool-ordinal: "2"}
ports: [{name: mediator, port: 9009, targetPort: mediator}]
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: hermes-execution-mediator-0
namespace: hermes
labels: {app: hermes-execution-mediator, pool-ordinal: "0"}
spec:
replicas: 1
strategy: {type: Recreate}
selector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
template:
metadata:
labels: {app: hermes-execution-mediator, pool-ordinal: "0"}
annotations:
ai.bstein.dev/config-rev: execution-pool-v2-mediated
ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-execution-worker
vault.hashicorp.com/agent-inject-containers: stage-mediator-access
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 2m
vault.hashicorp.com/agent-requests-mem: 16Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
spec:
serviceAccountName: hermes-execution-worker
automountServiceAccountToken: false
enableServiceLinks: false
# Priority parity with the worker: a pressured node evicts the pair
# symmetrically instead of moving only the lower-priority half, and the
# pool never preempts anything else in the cluster.
priorityClassName: scavenger
securityContext:
fsGroup: 10000
fsGroupChangePolicy: OnRootMismatch
seccompProfile: {type: RuntimeDefault}
affinity:
# Colocation with the worker is a throughput preference, not a
# requirement. The shared workspace claim is ReadWriteMany, so this
# mediator still schedules and serves when its worker is Pending or
# placed elsewhere -- which is what makes a drain self-healing.
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
podAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
statefulset.kubernetes.io/pod-name: hermes-execution-worker-0
topologyKey: kubernetes.io/hostname
initContainers:
- name: stage-mediator-access
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
env:
- {name: HERMES_WORKER_ORDINAL, value: "0"}
- {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
runAsGroup: 0
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: pool-access, mountPath: /pool-access}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
resources:
requests: {cpu: 2m, memory: 16Mi}
limits: {cpu: 100m, memory: 64Mi}
containers:
- name: execution-mediator
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
env:
- {name: HERMES_WORKER_ORDINAL, value: "0"}
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
startupProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 5
failureThreshold: 60
readinessProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 10
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: workspace, mountPath: /workspace}
- {name: scm-state, mountPath: /scm-state}
- {name: pool-access, mountPath: /pool-access, readOnly: true}
- {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 2m, memory: 64Mi}
limits: {cpu: 250m, memory: 256Mi}
volumes:
- name: workspace
persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-0}
- name: scm-state
persistentVolumeClaim: {claimName: hermes-execution-mediator-state-0}
- name: pool-access
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: scm-broker-client
configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
- name: coordinator
configMap: {name: hermes-execution-pool, defaultMode: 0555}
- name: tmp
emptyDir: {sizeLimit: 64Mi}
- name: vault-auth-token
projected:
defaultMode: 0600
sources:
- serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
- configMap:
name: kube-root-ca.crt
items: [{key: ca.crt, path: ca.crt}]
- downwardAPI:
items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: hermes-execution-mediator-1
namespace: hermes
labels: {app: hermes-execution-mediator, pool-ordinal: "1"}
spec:
replicas: 1
strategy: {type: Recreate}
selector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
template:
metadata:
labels: {app: hermes-execution-mediator, pool-ordinal: "1"}
annotations:
ai.bstein.dev/config-rev: execution-pool-v2-mediated
ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-execution-worker
vault.hashicorp.com/agent-inject-containers: stage-mediator-access
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 2m
vault.hashicorp.com/agent-requests-mem: 16Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
spec:
serviceAccountName: hermes-execution-worker
automountServiceAccountToken: false
enableServiceLinks: false
# Priority parity with the worker: a pressured node evicts the pair
# symmetrically instead of moving only the lower-priority half, and the
# pool never preempts anything else in the cluster.
priorityClassName: scavenger
securityContext:
fsGroup: 10000
fsGroupChangePolicy: OnRootMismatch
seccompProfile: {type: RuntimeDefault}
affinity:
# Colocation with the worker is a throughput preference, not a
# requirement. The shared workspace claim is ReadWriteMany, so this
# mediator still schedules and serves when its worker is Pending or
# placed elsewhere -- which is what makes a drain self-healing.
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
podAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
statefulset.kubernetes.io/pod-name: hermes-execution-worker-1
topologyKey: kubernetes.io/hostname
initContainers:
- name: stage-mediator-access
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
env:
- {name: HERMES_WORKER_ORDINAL, value: "1"}
- {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
runAsGroup: 0
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: pool-access, mountPath: /pool-access}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
resources:
requests: {cpu: 2m, memory: 16Mi}
limits: {cpu: 100m, memory: 64Mi}
containers:
- name: execution-mediator
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
env:
- {name: HERMES_WORKER_ORDINAL, value: "1"}
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
startupProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 5
failureThreshold: 60
readinessProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 10
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: workspace, mountPath: /workspace}
- {name: scm-state, mountPath: /scm-state}
- {name: pool-access, mountPath: /pool-access, readOnly: true}
- {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 2m, memory: 64Mi}
limits: {cpu: 250m, memory: 256Mi}
volumes:
- name: workspace
persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-1}
- name: scm-state
persistentVolumeClaim: {claimName: hermes-execution-mediator-state-1}
- name: pool-access
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: scm-broker-client
configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
- name: coordinator
configMap: {name: hermes-execution-pool, defaultMode: 0555}
- name: tmp
emptyDir: {sizeLimit: 64Mi}
- name: vault-auth-token
projected:
defaultMode: 0600
sources:
- serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
- configMap:
name: kube-root-ca.crt
items: [{key: ca.crt, path: ca.crt}]
- downwardAPI:
items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: hermes-execution-mediator-2
namespace: hermes
labels: {app: hermes-execution-mediator, pool-ordinal: "2"}
spec:
replicas: 1
strategy: {type: Recreate}
selector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
template:
metadata:
labels: {app: hermes-execution-mediator, pool-ordinal: "2"}
annotations:
ai.bstein.dev/config-rev: execution-pool-v2-mediated
ai.bstein.dev/security-boundary: model-pod-has-no-hmac-or-scm-network-authority
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-execution-worker
vault.hashicorp.com/agent-inject-containers: stage-mediator-access
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
vault.hashicorp.com/agent-inject-secret-execution-pool-key: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-execution-pool-key: "0600"
vault.hashicorp.com/agent-inject-template-execution-pool-key: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ printf "hermes-execution-pool-root-v2:%s" .Data.data.agent_api_key | sha256sum }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 2m
vault.hashicorp.com/agent-requests-mem: 16Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
spec:
serviceAccountName: hermes-execution-worker
automountServiceAccountToken: false
enableServiceLinks: false
# Priority parity with the worker: a pressured node evicts the pair
# symmetrically instead of moving only the lower-priority half, and the
# pool never preempts anything else in the cluster.
priorityClassName: scavenger
securityContext:
fsGroup: 10000
fsGroupChangePolicy: OnRootMismatch
seccompProfile: {type: RuntimeDefault}
affinity:
# Colocation with the worker is a throughput preference, not a
# requirement. The shared workspace claim is ReadWriteMany, so this
# mediator still schedules and serves when its worker is Pending or
# placed elsewhere -- which is what makes a drain self-healing.
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
podAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
statefulset.kubernetes.io/pod-name: hermes-execution-worker-2
topologyKey: kubernetes.io/hostname
initContainers:
- name: stage-mediator-access
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-mediator]
env:
- {name: HERMES_WORKER_ORDINAL, value: "2"}
- {name: HERMES_POOL_ACCESS_ROOT, value: /pool-access}
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
runAsGroup: 0
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: pool-access, mountPath: /pool-access}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
resources:
requests: {cpu: 2m, memory: 16Mi}
limits: {cpu: 100m, memory: 64Mi}
containers:
- name: execution-mediator
image: registry.bstein.dev/bstein/hermes-agent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_client.py]
env:
- {name: HERMES_WORKER_ORDINAL, value: "2"}
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HERMES_SCM_STATE_ROOT, value: /scm-state}
- {name: HERMES_EXECUTION_POOL_KEY_FILE, value: /pool-access/execution-pool-key}
- {name: PYTHONPATH, value: /opt/scm:/opt/coordinator:/opt/hermes}
ports: [{name: mediator, containerPort: 9009, protocol: TCP}]
startupProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 5
failureThreshold: 60
readinessProbe:
httpGet: {path: /ready, port: mediator}
periodSeconds: 10
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: workspace, mountPath: /workspace}
- {name: scm-state, mountPath: /scm-state}
- {name: pool-access, mountPath: /pool-access, readOnly: true}
- {name: scm-broker-client, mountPath: /opt/scm, readOnly: true}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 2m, memory: 64Mi}
limits: {cpu: 250m, memory: 256Mi}
volumes:
- name: workspace
persistentVolumeClaim: {claimName: workspace-hermes-execution-worker-2}
- name: scm-state
persistentVolumeClaim: {claimName: hermes-execution-mediator-state-2}
- name: pool-access
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: scm-broker-client
configMap: {name: hermes-scm-boundary-v2, defaultMode: 0555}
- name: coordinator
configMap: {name: hermes-execution-pool, defaultMode: 0555}
- name: tmp
emptyDir: {sizeLimit: 64Mi}
- name: vault-auth-token
projected:
defaultMode: 0600
sources:
- serviceAccountToken: {audience: vault, expirationSeconds: 3600, path: token}
- configMap:
name: kube-root-ca.crt
items: [{key: ca.crt, path: ca.crt}]
- downwardAPI:
items: [{path: namespace, fieldRef: {fieldPath: metadata.namespace}}]

View File

@ -0,0 +1,206 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-worker-isolation
namespace: hermes
spec:
podSelector:
matchLabels:
app: hermes-execution-worker
policyTypes: [Ingress, Egress]
ingress: []
egress:
- to:
- namespaceSelector:
matchLabels: {kubernetes.io/metadata.name: kube-system}
podSelector:
matchLabels: {k8s-app: kube-dns}
ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}]
- to:
- namespaceSelector:
matchLabels: {kubernetes.io/metadata.name: vault}
podSelector:
matchLabels: {app: vault}
ports: [{protocol: TCP, port: 8200}]
- to:
- podSelector:
matchLabels: {app: hermes-switchyard}
ports: [{protocol: TCP, port: 9005}]
- to:
- ipBlock:
cidr: 0.0.0.0/0
except:
- 10.0.0.0/8
- 100.64.0.0/10
- 127.0.0.0/8
- 169.254.0.0/16
- 172.16.0.0/12
- 192.168.0.0/16
- 224.0.0.0/4
ports: [{protocol: TCP, port: 443}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-worker-mediator-0
namespace: hermes
spec:
podSelector:
matchLabels:
app: hermes-execution-worker
apps.kubernetes.io/pod-index: "0"
policyTypes: [Egress]
egress:
- to:
- podSelector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
ports: [{protocol: TCP, port: 9009}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-worker-mediator-1
namespace: hermes
spec:
podSelector:
matchLabels:
app: hermes-execution-worker
apps.kubernetes.io/pod-index: "1"
policyTypes: [Egress]
egress:
- to:
- podSelector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
ports: [{protocol: TCP, port: 9009}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-worker-mediator-2
namespace: hermes
spec:
podSelector:
matchLabels:
app: hermes-execution-worker
apps.kubernetes.io/pod-index: "2"
policyTypes: [Egress]
egress:
- to:
- podSelector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
ports: [{protocol: TCP, port: 9009}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-pool-ingress
namespace: hermes
spec:
podSelector:
matchLabels: {app: hermes-agent}
policyTypes: [Ingress]
ingress:
- from:
- podSelector:
matchLabels: {app: hermes-execution-mediator}
ports: [{protocol: TCP, port: 9007}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-mediator-isolation
namespace: hermes
spec:
podSelector:
matchLabels: {app: hermes-execution-mediator}
policyTypes: [Ingress, Egress]
ingress: []
egress:
- to:
- namespaceSelector:
matchLabels: {kubernetes.io/metadata.name: kube-system}
podSelector:
matchLabels: {k8s-app: kube-dns}
ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}]
- to:
- namespaceSelector:
matchLabels: {kubernetes.io/metadata.name: vault}
podSelector:
matchLabels: {app: vault}
ports: [{protocol: TCP, port: 8200}]
- to:
- podSelector:
matchLabels: {app: hermes-agent}
ports: [{protocol: TCP, port: 9007}]
- to:
- namespaceSelector:
matchLabels: {kubernetes.io/metadata.name: hermes-scm}
podSelector:
matchLabels: {app: hermes-scm-broker}
ports: [{protocol: TCP, port: 9081}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-mediator-worker-0
namespace: hermes
spec:
podSelector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "0"}
policyTypes: [Ingress]
ingress:
- from:
- podSelector:
matchLabels:
app: hermes-execution-worker
apps.kubernetes.io/pod-index: "0"
ports: [{protocol: TCP, port: 9009}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-mediator-worker-1
namespace: hermes
spec:
podSelector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "1"}
policyTypes: [Ingress]
ingress:
- from:
- podSelector:
matchLabels:
app: hermes-execution-worker
apps.kubernetes.io/pod-index: "1"
ports: [{protocol: TCP, port: 9009}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-mediator-worker-2
namespace: hermes
spec:
podSelector:
matchLabels: {app: hermes-execution-mediator, pool-ordinal: "2"}
policyTypes: [Ingress]
ingress:
- from:
- podSelector:
matchLabels:
app: hermes-execution-worker
apps.kubernetes.io/pod-index: "2"
ports: [{protocol: TCP, port: 9009}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-execution-switchyard-ingress
namespace: hermes
spec:
podSelector:
matchLabels: {app: hermes-switchyard}
policyTypes: [Ingress]
ingress:
- from:
- podSelector:
matchLabels: {app: hermes-execution-worker}
ports: [{protocol: TCP, port: 9005}]

View File

@ -0,0 +1,12 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: hermes-execution-worker
namespace: hermes
labels:
app.kubernetes.io/name: hermes-execution-worker
app.kubernetes.io/part-of: hermes
automountServiceAccountToken: false
# Intentionally no RoleBinding or ClusterRoleBinding. The projected, bounded
# token is mounted only into Vault-facing containers and grants no Kubernetes
# API verbs to the model-facing execution worker.

View File

@ -0,0 +1,291 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: hermes-execution-worker
namespace: hermes
labels:
app: hermes-execution-worker
spec:
serviceName: hermes-execution-worker
replicas: 3
podManagementPolicy: Parallel
revisionHistoryLimit: 2
selector:
matchLabels:
app: hermes-execution-worker
updateStrategy:
type: RollingUpdate
rollingUpdate:
partition: 0
template:
metadata:
labels:
app: hermes-execution-worker
app.kubernetes.io/name: hermes-execution-worker
app.kubernetes.io/part-of: hermes
annotations:
ai.bstein.dev/role: fenced-execution-only
ai.bstein.dev/scm-boundary: mediated-pr14-broker-with-completion-gates
ai.bstein.dev/model-policy: Switchyard AUTO with cross-provider fallback
ai.bstein.dev/storage: separate durable RWO workspace and OAuth refresh ownership per ordinal
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: hermes-execution-worker
vault.hashicorp.com/agent-inject-containers: stage-worker-access
vault.hashicorp.com/agent-service-account-token-volume-name: vault-auth-token
vault.hashicorp.com/agent-inject-secret-claude-credentials-0: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-claude-credentials-0: "0600"
vault.hashicorp.com/agent-inject-template-claude-credentials-0: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ .Data.data.execution_worker_0_claude_credentials_json }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-claude-credentials-1: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-claude-credentials-1: "0600"
vault.hashicorp.com/agent-inject-template-claude-credentials-1: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ .Data.data.execution_worker_1_claude_credentials_json }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-claude-credentials-2: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-claude-credentials-2: "0600"
vault.hashicorp.com/agent-inject-template-claude-credentials-2: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ .Data.data.execution_worker_2_claude_credentials_json }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-codex-auth-0: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-codex-auth-0: "0600"
vault.hashicorp.com/agent-inject-template-codex-auth-0: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ .Data.data.execution_worker_0_codex_auth_json }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-codex-auth-1: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-codex-auth-1: "0600"
vault.hashicorp.com/agent-inject-template-codex-auth-1: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ .Data.data.execution_worker_1_codex_auth_json }}
{{- end }}
vault.hashicorp.com/agent-inject-secret-codex-auth-2: kv/data/atlas/hermes/agent-tokens
vault.hashicorp.com/agent-inject-perms-codex-auth-2: "0600"
vault.hashicorp.com/agent-inject-template-codex-auth-2: |
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
{{ .Data.data.execution_worker_2_codex_auth_json }}
{{- end }}
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-requests-cpu: 2m
vault.hashicorp.com/agent-requests-mem: 16Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
spec:
serviceAccountName: hermes-execution-worker
priorityClassName: scavenger
automountServiceAccountToken: false
enableServiceLinks: false
terminationGracePeriodSeconds: 30
securityContext:
fsGroup: 10000
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- {key: kubernetes.io/arch, operator: In, values: [arm64]}
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-13, titan-14, titan-17, titan-18, titan-19, titan-22, titan-24]}
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- {key: node-role.kubernetes.io/accelerator, operator: Exists}
- weight: 50
preference:
matchExpressions:
- {key: hardware, operator: In, values: [rpi5]}
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app: hermes-execution-worker
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app: hermes-execution-worker
initContainers:
- name: stage-worker-access
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/stage_runtime_access.py, execution-worker]
env:
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HERMES_PROVIDER_ACCESS_ROOT, value: /provider-access}
- name: HERMES_WORKER_ORDINAL
valueFrom:
fieldRef:
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
runAsGroup: 0
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: workspace, mountPath: /workspace}
- {name: provider-access, mountPath: /provider-access}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
resources:
requests: {cpu: 2m, memory: 32Mi}
limits: {cpu: 100m, memory: 64Mi}
- name: install-provider-clis
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command: [/bin/sh, -ec]
args:
# The tools volume is durable, so this install runs once per pinned
# version rather than on every Pod start. The version marker is only
# trusted when both binaries are actually present and executable, so
# a partial or pruned cache reinstalls instead of failing closed
# forever. The install itself is time-bounded so a stalled registry
# surfaces as a fast init failure with backoff instead of an
# open-ended wait that a Flux health timeout would have to absorb.
- |
tools=/worker-data/tools
marker="${tools}/.cli-versions-0.147.0-2.1.226"
mkdir -p "${tools}/bin"
if [ ! -f "${marker}" ] || [ ! -x "${tools}/bin/codex" ] \
|| [ ! -x "${tools}/bin/claude" ]; then
rm -f "${marker}"
timeout 900 npm install --global --omit=dev --no-audit --no-fund \
--fetch-timeout=120000 --fetch-retries=2 --prefix "${tools}" \
@openai/codex@0.147.0 @anthropic-ai/claude-code@2.1.226
touch "${marker}"
fi
test -x "${tools}/bin/codex"
test -x "${tools}/bin/claude"
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: tools, mountPath: /worker-data/tools}
resources:
requests: {cpu: 5m, memory: 64Mi}
limits: {cpu: "1", memory: 1Gi}
containers:
- name: execution-worker
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent
command: [/opt/hermes/.venv/bin/python, /opt/coordinator/execution_pool_worker.py]
env:
- {name: HERMES_HOME, value: /worker-data}
- {name: HERMES_WORKER_ROOT, value: /workspace}
- {name: HOME, value: /worker-data/home}
- {name: CODEX_HOME, value: /provider-access/codex}
- {name: CLAUDE_CONFIG_DIR, value: /provider-access/claude}
- {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
- {name: PYTHONPATH, value: /opt/hermes}
- {name: PATH, value: /worker-data/tools/bin:/opt/coordinator:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
- name: HERMES_WORKER_ORDINAL
valueFrom:
fieldRef:
fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
- name: HERMES_WORKER_NODE
valueFrom:
fieldRef:
fieldPath: spec.nodeName
startupProbe:
exec:
command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json && test -w /provider-access/claude/.credentials.json"]
periodSeconds: 5
failureThreshold: 60
readinessProbe:
exec:
command: [/bin/sh, -ec, "test -w /workspace && test -w /provider-access/codex/auth.json"]
periodSeconds: 10
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10000
runAsGroup: 10000
seccompProfile: {type: RuntimeDefault}
volumeMounts:
- {name: workspace, mountPath: /workspace}
- {name: worker-data, mountPath: /worker-data}
- {name: tools, mountPath: /worker-data/tools, readOnly: true}
- {name: provider-access, mountPath: /provider-access}
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 5m, memory: 128Mi, ephemeral-storage: 1Gi}
limits: {cpu: "2", memory: 4Gi, ephemeral-storage: 8Gi}
volumes:
- name: worker-data
emptyDir: {sizeLimit: 128Mi}
- name: coordinator
configMap:
name: hermes-execution-pool
defaultMode: 0555
- name: tmp
emptyDir: {sizeLimit: 2Gi}
- name: vault-auth-token
projected:
defaultMode: 0600
sources:
- serviceAccountToken:
audience: vault
expirationSeconds: 3600
path: token
- configMap:
name: kube-root-ca.crt
items:
- {key: ca.crt, path: ca.crt}
- downwardAPI:
items:
- {path: namespace, fieldRef: {fieldPath: metadata.namespace}}
volumeClaimTemplates:
# The workspace is shared with this ordinal's mediator, which performs every
# SCM operation on it. ReadWriteMany removes the cross-node Multi-Attach
# deadlock that a drain or preemption used to create between the two Pods,
# so each can be evicted and rescheduled independently. Longhorn already
# serves the hermes-chat tenant workspaces this way on the same class.
- metadata:
name: workspace
labels:
app: hermes-execution-worker
spec:
accessModes: [ReadWriteMany]
storageClassName: astreae
resources:
requests:
storage: 30Gi
# Provider CLIs are installed once per pinned version onto a durable volume
# instead of being re-downloaded into an emptyDir on every Pod start.
- metadata:
name: tools
labels:
app: hermes-execution-worker
spec:
accessModes: [ReadWriteOnce]
storageClassName: astreae
resources:
requests:
storage: 2Gi
# OAuth refresh ownership is single-mounter by design: keep it ReadWriteOnce.
- metadata:
name: provider-access
labels:
app: hermes-execution-worker
spec:
accessModes: [ReadWriteOnce]
storageClassName: astreae
resources:
requests:
storage: 1Gi

View File

@ -7,13 +7,13 @@ images:
digest: sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7
resources:
- namespace.yaml
- scm-common
- vault-serviceaccount.yaml
- configmap.yaml
- agent-configmap.yaml
- chat-configmap.yaml
- switchyard-configmap.yaml
- rbac.yaml
- agent-rbac.yaml
- node-ssh-access.yaml
- pvc.yaml
- switchyard-pvc.yaml
@ -41,6 +41,13 @@ resources:
- oauth2-proxy.yaml
- agent-certificate.yaml
- agent-ingress.yaml
- execution-worker-rbac.yaml
- execution-worker-statefulset.yaml
- execution-mediator.yaml
- execution-worker-networkpolicy.yaml
patches:
- path: execution-coordinator-patch.yaml
configMapGenerator:
- name: hermes-chat-oauth-templates
@ -55,6 +62,38 @@ configMapGenerator:
- OPERATOR-RUNBOOK.md=NOTES.md
options:
disableNameSuffixHash: true
- name: hermes-execution-pool
namespace: hermes
files:
- cli_lane_board.py=scripts/cli_lane_board.py
- cli_lane_capabilities.py=scripts/cli_lane_capabilities.py
- cli_lane_config.py=scripts/cli_lane_config.py
- cli_lane_dispatch.py=scripts/cli_lane_dispatch.py
- cli_lane_evidence.py=scripts/cli_lane_evidence.py
- cli_lane_execution.py=scripts/cli_lane_execution.py
- cli_lane_files.py=scripts/cli_lane_files.py
- cli_lane_finalization.py=scripts/cli_lane_finalization.py
- cli_lane_goal.py=scripts/cli_lane_goal.py
- cli_lane_prompt.py=scripts/cli_lane_prompt.py
- cli_lane_provider.py=scripts/cli_lane_provider.py
- cli_lane_quarantine.py=scripts/cli_lane_quarantine.py
- cli_lane_records.py=scripts/cli_lane_records.py
- cli_lane_recovery.py=scripts/cli_lane_recovery.py
- cli_lane_retention.py=scripts/cli_lane_retention.py
- cli_lane_routing.py=scripts/cli_lane_routing.py
- cli_lane_runner.py=scripts/cli_lane_runner.py
- execution_pool_protocol.py=scripts/execution_pool_protocol.py
- execution_pool_store.py=scripts/execution_pool_store.py
- execution_pool_project.py=scripts/execution_pool_project.py
- execution_pool_coordinator.py=scripts/execution_pool_coordinator.py
- execution_pool_maintenance.py=scripts/execution_pool_maintenance.py
- execution_pool_server.py=scripts/execution_pool_server.py
- execution_pool_client.py=scripts/execution_pool_client.py
- execution_pool_worker.py=scripts/execution_pool_worker.py
- execution_pool_scm.py=scripts/execution_pool_scm.py
- gitea_api_policy.py=scm-common/scripts/gitea_api_policy.py
- scm_broker_client.py=scm-common/scripts/scm_broker_client.py
- stage_runtime_access.py=scripts/stage_runtime_access.py
- name: hermes-coordinator
namespace: hermes
files:
@ -63,7 +102,22 @@ configMapGenerator:
- ai_usage_http.py=scripts/ai_usage_http.py
- claude=scripts/claude
- claude_command_policy.py=scripts/claude_command_policy.py
- cli_lane_board.py=scripts/cli_lane_board.py
- cli_lane_capabilities.py=scripts/cli_lane_capabilities.py
- cli_lane_config.py=scripts/cli_lane_config.py
- cli_lane_dispatch.py=scripts/cli_lane_dispatch.py
- cli_lane_evidence.py=scripts/cli_lane_evidence.py
- cli_lane_execution.py=scripts/cli_lane_execution.py
- cli_lane_files.py=scripts/cli_lane_files.py
- cli_lane_finalization.py=scripts/cli_lane_finalization.py
- cli_lane_goal.py=scripts/cli_lane_goal.py
- cli_lane_prompt.py=scripts/cli_lane_prompt.py
- cli_lane_provider.py=scripts/cli_lane_provider.py
- cli_lane_quarantine.py=scripts/cli_lane_quarantine.py
- cli_lane_records.py=scripts/cli_lane_records.py
- cli_lane_recovery.py=scripts/cli_lane_recovery.py
- cli_lane_retention.py=scripts/cli_lane_retention.py
- cli_lane_routing.py=scripts/cli_lane_routing.py
- cli_lane_runner.py=scripts/cli_lane_runner.py
- codex=scripts/codex
- configure_agent_clients.py=scripts/configure_agent_clients.py
@ -71,14 +125,13 @@ configMapGenerator:
- classifier_broker.py=scripts/classifier_broker.py
- claude_oauth_broker.py=scripts/claude_oauth_broker.py
- worker_route_broker.py=scripts/worker_route_broker.py
- gitea_api.py=scripts/gitea_api.py
- gitea_askpass.sh=scripts/gitea_askpass.sh
- hermes_coordinator.py=scripts/hermes_coordinator.py
- hermes_model_routing.py=scripts/hermes_model_routing.py
- hermes_stt_client.py=scripts/hermes_stt_client.py
- image_broker.py=scripts/image_broker.py
- install_agent_tools.sh=scripts/install_agent_tools.sh
- jenkins_build_evidence.py=scripts/jenkins_build_evidence.py
- jenkins_image_build_trigger.py=scripts/jenkins_image_build_trigger.py
- kanban_status_recovery.py=scripts/kanban_status_recovery.py
- migrate_herdr_state.py=scripts/migrate_herdr_state.py
- migrate_api_session_lineage.py=scripts/migrate_api_session_lineage.py
@ -106,6 +159,15 @@ configMapGenerator:
- config=agent-kubeconfig.yaml
options:
disableNameSuffixHash: true
- name: hermes-node-account-hardener
namespace: hermes
files:
- node_account_hardening.py=scripts/node_account_hardening.py
- node_account_audit.py=scripts/node_account_audit.py
- node_account_io.py=scripts/node_account_io.py
- node_polkit_audit.py=scripts/node_polkit_audit.py
options:
disableNameSuffixHash: true
- name: hermes-auto-router-plugin
namespace: hermes
files:
@ -160,3 +222,10 @@ configMapGenerator:
- alert-review.md=skills/tune-atlas-alerts/references/alert-review.md
options:
disableNameSuffixHash: true
- name: hermes-atlas-pr-skill
namespace: hermes
files:
- SKILL.md=skills/manage-atlas-pull-requests/SKILL.md
- openai.yaml=skills/manage-atlas-pull-requests/agents/openai.yaml
options:
disableNameSuffixHash: true

View File

@ -119,11 +119,66 @@ spec:
app: server
ports:
- {protocol: TCP, port: 9010}
# agent.hermes.bstein.dev is an owner-only engineering workstation. The
# browser boundary remains OAuth-protected, while its workers need to reach
# every cluster namespace, Atlas LAN service, and hosted provider endpoint.
- {protocol: TCP, port: 9011}
egress:
- {}
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- {protocol: UDP, port: 53}
- {protocol: TCP, port: 53}
- to:
- namespaceSelector:
matchExpressions:
- key: kubernetes.io/metadata.name
operator: NotIn
values: [gitea, hermes-scm]
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: hermes-scm
podSelector:
matchLabels:
app: hermes-scm-broker
ports:
- {protocol: TCP, port: 9081}
- to:
- ipBlock:
cidr: 10.43.0.1/32
ports:
- {protocol: TCP, port: 443}
- to:
- ipBlock:
cidr: 192.168.0.0/16
except:
- 192.168.22.9/32
- to:
- ipBlock:
cidr: 0.0.0.0/0
except:
- 10.0.0.0/8
- 100.64.0.0/10
- 127.0.0.0/8
- 169.254.0.0/16
- 172.16.0.0/12
- 192.168.0.0/16
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-node-ssh-access-isolation
namespace: hermes
spec:
podSelector:
matchLabels:
app: hermes-node-ssh-access
policyTypes: [Ingress, Egress]
ingress: []
egress: []
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy

View File

@ -45,32 +45,16 @@ spec:
- operator: Exists
containers:
- name: key-reconciler
image: busybox:1.37
# The pinned index supports the cluster's amd64 and arm64 nodes.
image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df
imagePullPolicy: IfNotPresent
command: [/bin/sh, -ec]
args:
- |
reconcile() {
key="$(cat /vault/secrets/node-ssh-public-key)"
found=0
for user in atlas oceanus; do
home="/host-home/${user}"
[ -d "${home}" ] || continue
found=1
identity="$(awk -F: -v name="${user}" '$1 == name {print $3 ":" $4; exit}' /host-etc/passwd)"
[ -n "${identity}" ] || identity="$(stat -c %u:%g "${home}")"
uid="${identity%%:*}"
gid="${identity##*:}"
install -d -m 0700 -o "${uid}" -g "${gid}" "${home}/.ssh"
touch "${home}/.ssh/authorized_keys"
grep -qxF "${key}" "${home}/.ssh/authorized_keys" || printf '%s\n' "${key}" >> "${home}/.ssh/authorized_keys"
chown "${uid}:${gid}" "${home}/.ssh/authorized_keys"
chmod 0600 "${home}/.ssh/authorized_keys"
done
[ "${found}" = 1 ] || { echo "no supported node SSH account found" >&2; return 1; }
}
while true; do
reconcile
/usr/local/bin/python \
/opt/node-hardener/node_account_hardening.py \
--public-key-file /vault/secrets/node-ssh-public-key
sleep 300
done
securityContext:
@ -83,8 +67,21 @@ spec:
volumeMounts:
- name: host-home
mountPath: /host-home
- name: host-passwd
mountPath: /host-etc/passwd
- name: host-etc
mountPath: /host-etc
- name: host-polkit-share
mountPath: /host-polkit-share
readOnly: true
- name: host-k3s
mountPath: /host-k3s
- name: host-kubelet
mountPath: /host-kubelet
- name: host-run-k3s
mountPath: /host-run-k3s
- name: host-run-containerd
mountPath: /host-run-containerd
- name: coordinator
mountPath: /opt/node-hardener
readOnly: true
- name: vault-secrets
mountPath: /vault/secrets
@ -94,19 +91,43 @@ spec:
resources:
requests:
cpu: 5m
memory: 8Mi
memory: 32Mi
limits:
cpu: 50m
memory: 32Mi
memory: 128Mi
volumes:
- name: host-home
hostPath:
path: /home
type: Directory
- name: host-passwd
- name: host-etc
hostPath:
path: /etc/passwd
type: File
path: /etc
type: Directory
- name: host-polkit-share
hostPath:
path: /usr/share/polkit-1
type: Directory
- name: host-k3s
hostPath:
path: /var/lib/rancher/k3s
type: Directory
- name: host-kubelet
hostPath:
path: /var/lib/kubelet
type: Directory
- name: host-run-k3s
hostPath:
path: /run/k3s
type: Directory
- name: host-run-containerd
hostPath:
path: /run/containerd
type: Directory
- name: coordinator
configMap:
name: hermes-node-account-hardener
defaultMode: 0555
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io

View File

@ -16,6 +16,9 @@ metadata:
spec:
selector:
app: hermes-agent
# The agent dashboard and terminal must stay reachable while the
# cli-lane-runner readiness probe holds the shared pod NotReady.
publishNotReadyAddresses: true
ports:
- {name: http, port: 80, targetPort: auth-http}
---

View File

@ -0,0 +1,17 @@
# services/hermes/scm-common/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
configMapGenerator:
- name: hermes-scm-boundary-v2
files:
- deadline_http.py=scripts/deadline_http.py
- git_pack_objects.py=scripts/git_pack_objects.py
- gitea_api.py=scripts/gitea_api.py
- gitea_api_policy.py=scripts/gitea_api_policy.py
- receive_pack_scan.py=scripts/receive_pack_scan.py
- scm_broker.py=scripts/scm_broker.py
- scm_broker_io.py=scripts/scm_broker_io.py
- scm_broker_server.py=scripts/scm_broker_server.py
- scm_broker_client.py=scripts/scm_broker_client.py
options:
disableNameSuffixHash: true

View File

@ -0,0 +1,344 @@
#!/usr/bin/env python3
"""Absolute wall-clock deadlines for every outbound Hermes HTTP exchange.
Small control exchanges run inside a killable helper process, so connect,
send, read, and DNS all share one hard bound that no blocking socket state
can outlive. Large streaming Git exchanges cannot cross a process boundary;
they instead register their connection with a watchdog that force-closes the
socket when the same kind of absolute deadline expires.
"""
from __future__ import annotations
import base64
import contextlib
import email.message
import http.client
import io
import json
import os
import signal
import socket
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
from dataclasses import dataclass
from gitea_api_policy import PolicyError
MAX_CONTROL_BYTES = 128 * 1024
MAX_PROTOCOL_BYTES = 4 * 1024 * 1024
MAX_STREAM_SECONDS = 900.0
READ_CHUNK = 64 * 1024
class _RejectRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
raise urllib.error.URLError("redirect rejected")
@dataclass(frozen=True)
class Result:
"""Minimal HTTP evidence returned across the process boundary."""
status: int
content_type: str
body: bytes
def _read_response(response, maximum: int, deadline: float) -> bytes:
chunks: list[bytes] = []
total = 0
while True:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise TimeoutError
raw = getattr(getattr(response, "fp", None), "raw", None)
sock = getattr(raw, "_sock", raw)
if hasattr(sock, "settimeout"):
sock.settimeout(remaining)
chunk = response.read(min(READ_CHUNK, maximum + 1 - total))
if not chunk:
break
chunks.append(chunk)
total += len(chunk)
if total > maximum:
raise ValueError("response too large")
return b"".join(chunks)
def _child_exchange(control: dict[str, object]) -> Result:
url = control.get("url")
method = control.get("method")
headers = control.get("headers")
encoded = control.get("body")
maximum = control.get("maximum")
timeout = control.get("timeout")
if (
not isinstance(url, str)
or not isinstance(method, str)
or not isinstance(headers, dict)
or not all(isinstance(key, str) and isinstance(value, str) for key, value in headers.items())
or not isinstance(encoded, str)
or not isinstance(maximum, int)
or not 0 <= maximum <= 2 * 1024 * 1024
or not isinstance(timeout, (int, float))
or not 0 < timeout <= 120
):
raise ValueError("invalid control")
body = base64.b64decode(encoded, validate=True) if encoded else None
if body is not None and len(body) > MAX_CONTROL_BYTES:
raise ValueError("request too large")
deadline = time.monotonic() + float(timeout)
request = urllib.request.Request(
url, data=body, method=method, headers=headers # type: ignore[arg-type]
)
opener = urllib.request.build_opener(_RejectRedirect())
try:
response = opener.open(request, timeout=max(deadline - time.monotonic(), 0.001))
except urllib.error.HTTPError as error:
response = error
with response:
status = getattr(response, "status", response.getcode())
content_type = response.headers.get_content_type()
value = _read_response(response, maximum, deadline)
return Result(int(status), str(content_type), value)
def _child_main() -> int:
try:
raw = sys.stdin.buffer.read(MAX_CONTROL_BYTES + 1)
if len(raw) > MAX_CONTROL_BYTES:
raise ValueError("control too large")
control = json.loads(raw)
if not isinstance(control, dict):
raise ValueError("invalid control")
result = _child_exchange(control)
output = {
"ok": True,
"status": result.status,
"content_type": result.content_type,
"body": base64.b64encode(result.body).decode("ascii"),
}
except Exception:
output = {"ok": False}
sys.stdout.write(json.dumps(output, separators=(",", ":")))
return 0 if output["ok"] else 1
def _terminate(process: subprocess.Popen[bytes]) -> None:
with contextlib.suppress(ProcessLookupError):
os.killpg(process.pid, signal.SIGKILL)
process.communicate()
def exchange(
request: urllib.request.Request,
*,
maximum: int,
timeout: float,
popen=subprocess.Popen,
) -> Result:
"""Execute a whole request under one monotonic deadline and size limit."""
if not 0 < timeout <= 120 or not 0 <= maximum <= 2 * 1024 * 1024:
raise PolicyError("HTTP control request bounds are invalid")
body = request.data or b""
if not isinstance(body, bytes) or len(body) > MAX_CONTROL_BYTES:
raise PolicyError("HTTP control request exceeds the safe size limit")
control = json.dumps(
{
"url": request.full_url,
"method": request.get_method(),
"headers": dict(request.header_items()),
"body": base64.b64encode(body).decode("ascii"),
"maximum": maximum,
"timeout": timeout,
},
separators=(",", ":"),
).encode()
deadline = time.monotonic() + timeout
process = popen(
[sys.executable, os.path.realpath(__file__), "--child"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
start_new_session=True,
env={"LANG": "C.UTF-8", "PATH": "/usr/bin:/bin"},
)
try:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise subprocess.TimeoutExpired(process.args, timeout)
output, _error = process.communicate(control, timeout=remaining)
except subprocess.TimeoutExpired as exc:
_terminate(process)
raise PolicyError("HTTP control request deadline exceeded") from exc
if process.returncode != 0 or len(output) > MAX_PROTOCOL_BYTES:
raise PolicyError("HTTP control request failed")
try:
result = json.loads(output)
if not isinstance(result, dict) or result.get("ok") is not True:
raise ValueError
status = result["status"]
content_type = result["content_type"]
encoded = result["body"]
if not isinstance(status, int) or not isinstance(content_type, str) or not isinstance(encoded, str):
raise ValueError
value = base64.b64decode(encoded, validate=True)
except (KeyError, ValueError, TypeError, json.JSONDecodeError) as exc:
raise PolicyError("HTTP control helper returned invalid evidence") from exc
if len(value) > maximum:
raise PolicyError("HTTP control response exceeds the safe size limit")
return Result(status, content_type, value)
class _AdaptedResponse:
"""Present one finished exchange through the urlopen reader interface."""
def __init__(self, result: Result):
self.status = result.status
self.headers = email.message.Message()
self.headers["Content-Type"] = result.content_type
self._stream = io.BytesIO(result.body)
def read(self, limit: int = -1) -> bytes:
return self._stream.read(limit)
def __enter__(self):
return self
def __exit__(self, *_excinfo: object) -> bool:
return False
def open_bounded(
request: urllib.request.Request,
*,
maximum: int,
timeout: float,
popen=subprocess.Popen,
) -> _AdaptedResponse:
"""Open one control exchange whose whole lifetime shares one deadline."""
return _AdaptedResponse(
exchange(request, maximum=maximum, timeout=timeout, popen=popen)
)
def _resolve_within(host: str, port: int, timeout: float) -> None:
"""Resolve one host under a hard timeout the socket layer never covers.
``getaddrinfo`` ignores socket timeouts and runs before any socket
exists, so a slow or hostile resolver would otherwise outlive the whole
stream deadline. Running it in a joinable worker bounds resolution by the
same wall clock; a stuck lookup leaks only one daemon thread.
"""
outcome: dict[str, object] = {}
def run() -> None:
try:
outcome["value"] = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
except OSError as exc:
outcome["error"] = exc
worker = threading.Thread(target=run, daemon=True)
worker.start()
worker.join(timeout)
if worker.is_alive():
raise PolicyError("HTTP stream DNS resolution deadline exceeded")
if "error" in outcome:
raise outcome["error"] # type: ignore[misc]
class StreamDeadline:
"""Force-close tracked connections once an absolute deadline passes."""
def __init__(self, timeout: float, *, timer=threading.Timer):
if not 0 < timeout <= MAX_STREAM_SECONDS:
raise PolicyError("HTTP stream deadline bounds are invalid")
self._deadline = time.monotonic() + timeout
self._lock = threading.Lock()
self._connections: list[http.client.HTTPConnection] = []
self.expired = False
self._timer = timer(timeout, self._expire)
self._timer.daemon = True
self._timer.start()
def remaining(self) -> float:
value = self._deadline - time.monotonic()
if value <= 0 or self.expired:
raise PolicyError("HTTP stream deadline exceeded")
return value
def _expire(self) -> None:
with self._lock:
self.expired = True
connections = list(self._connections)
for connection in connections:
# Shut the raw socket down first so a blocked send or recv wakes
# immediately; close alone leaves peers waiting on the old fd.
sock = getattr(connection, "sock", None)
if sock is not None:
with contextlib.suppress(OSError):
sock.shutdown(socket.SHUT_RDWR)
connection.close()
def _track(self, connection: http.client.HTTPConnection):
with self._lock:
expired = self.expired
if not expired:
self._connections.append(connection)
if expired:
connection.close()
raise PolicyError("HTTP stream deadline exceeded")
return connection
def _bounded_connection(self, connection: http.client.HTTPConnection):
"""Register one connection and bound its DNS phase by the deadline.
The watchdog can only close a live socket, but ``connect`` resolves
the host before any socket exists. Wrapping ``connect`` runs that
resolution under the same absolute deadline first, then hands off to
the real connect for the socket phases the watchdog already covers.
"""
original_connect = connection.connect
def connect() -> None:
_resolve_within(connection.host, connection.port, self.remaining())
original_connect()
connection.connect = connect # type: ignore[method-assign]
return self._track(connection)
def cancel(self) -> None:
self._timer.cancel()
def handlers(self) -> tuple[urllib.request.BaseHandler, ...]:
"""Build urllib handlers whose connections obey this deadline."""
deadline = self
class GuardedHTTPHandler(urllib.request.HTTPHandler):
def http_open(self, req):
return self.do_open(
lambda host, **kwargs: deadline._bounded_connection(
http.client.HTTPConnection(host, **kwargs)
),
req,
)
class GuardedHTTPSHandler(urllib.request.HTTPSHandler):
def https_open(self, req):
return self.do_open(
lambda host, **kwargs: deadline._bounded_connection(
http.client.HTTPSConnection(host, **kwargs)
),
req,
)
return (GuardedHTTPHandler(), GuardedHTTPSHandler())
if __name__ == "__main__":
raise SystemExit(_child_main() if sys.argv[1:] == ["--child"] else 2)

View File

@ -0,0 +1,308 @@
#!/usr/bin/env python3
"""Bounded Git pack inflation and delta resolution for quarantine scanning.
Pushed objects arrive zlib-compressed inside a packfile, so any content
screening must inflate the real payloads first. This module unpacks one
self-contained pack under explicit object, size, and checksum bounds and
resolves every delta against bases inside the same pack. Thin packs are
rejected outright: a delta against a base the broker cannot see would leave
part of the pushed content unscanned.
"""
from __future__ import annotations
import hashlib
import tempfile
import zlib
from typing import BinaryIO
from gitea_api_policy import PolicyError
OBJECT_TYPE_NAMES = {1: b"commit", 2: b"tree", 3: b"blob", 4: b"tag"}
OFS_DELTA = 6
REF_DELTA = 7
MAX_PACK_OBJECTS = 50_000
MAX_OBJECT_BYTES = 32 * 1024 * 1024
MAX_TOTAL_BYTES = 256 * 1024 * 1024
MAX_RESOLVE_PASSES = 64
SPOOL_MEMORY_LIMIT = 256 * 1024
READ_CHUNK = 64 * 1024
def _read_exact(stream: BinaryIO, size: int) -> bytes:
value = stream.read(size)
if len(value) != size:
raise PolicyError("Git pack data ended early")
return value
def _object_header(stream: BinaryIO) -> tuple[int, int]:
"""Parse one object's type and declared inflated size."""
byte = _read_exact(stream, 1)[0]
type_code = (byte >> 4) & 0x7
size = byte & 0x0F
shift = 4
while byte & 0x80:
if shift > 60:
raise PolicyError("Git object header is invalid")
byte = _read_exact(stream, 1)[0]
size |= (byte & 0x7F) << shift
shift += 7
return type_code, size
def _delta_distance(stream: BinaryIO) -> int:
"""Parse one ofs-delta base distance."""
byte = _read_exact(stream, 1)[0]
value = byte & 0x7F
while byte & 0x80:
byte = _read_exact(stream, 1)[0]
value = ((value + 1) << 7) | (byte & 0x7F)
if value > 1 << 48:
raise PolicyError("Git delta base distance is invalid")
return value
def _inflate(stream: BinaryIO, declared: int) -> BinaryIO:
"""Inflate one object into a quarantine spool, bounded by its header."""
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool
max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
)
decompressor = zlib.decompressobj()
produced = 0
try:
while not decompressor.eof:
data = stream.read(READ_CHUNK)
if not data:
raise PolicyError("Git pack object data ended early")
while True:
value = decompressor.decompress(data, READ_CHUNK)
produced += len(value)
if produced > declared:
raise PolicyError("Git object size does not match its header")
spool.write(value)
data = decompressor.unconsumed_tail
if decompressor.eof or not data:
break
if decompressor.unused_data:
stream.seek(-len(decompressor.unused_data), 1)
if produced != declared:
raise PolicyError("Git object size does not match its header")
spool.seek(0)
return spool
except zlib.error as exc:
spool.close()
raise PolicyError("Git pack object data is corrupt") from exc
except Exception:
spool.close()
raise
def _delta_size(delta: bytes, position: int) -> tuple[int, int]:
size = 0
shift = 0
while True:
if position >= len(delta) or shift > 60:
raise PolicyError("Git delta header is invalid")
byte = delta[position]
position += 1
size |= (byte & 0x7F) << shift
shift += 7
if not byte & 0x80:
return size, position
def _apply_delta(base: bytes, delta: bytes) -> bytes:
"""Apply one Git delta and require exact declared sizes throughout."""
source_size, position = _delta_size(delta, 0)
target_size, position = _delta_size(delta, position)
if source_size != len(base) or target_size > MAX_OBJECT_BYTES:
raise PolicyError("Git delta sizes are invalid")
parts: list[bytes] = []
produced = 0
while position < len(delta):
opcode = delta[position]
position += 1
if opcode & 0x80:
operands = [bit for bit in (1, 2, 4, 8, 16, 32, 64) if opcode & bit]
if position + len(operands) > len(delta):
raise PolicyError("Git delta copy is truncated")
offset = 0
size = 0
for shift, bit in enumerate((1, 2, 4, 8)):
if opcode & bit:
offset |= delta[position] << (8 * shift)
position += 1
for shift, bit in enumerate((16, 32, 64)):
if opcode & bit:
size |= delta[position] << (8 * shift)
position += 1
size = size or 0x10000
if offset + size > len(base):
raise PolicyError("Git delta copy is out of bounds")
parts.append(base[offset : offset + size])
produced += size
elif opcode:
if position + opcode > len(delta):
raise PolicyError("Git delta insert is truncated")
parts.append(delta[position : position + opcode])
position += opcode
produced += opcode
else:
raise PolicyError("Git delta instruction is invalid")
if produced > target_size:
raise PolicyError("Git delta result exceeds its declared size")
if produced != target_size:
raise PolicyError("Git delta result size is invalid")
return b"".join(parts)
def _payload_sha(type_code: int, payload: BinaryIO, size: int) -> bytes:
digest = hashlib.sha1(OBJECT_TYPE_NAMES[type_code] + b" %d\x00" % size)
payload.seek(0)
while True:
chunk = payload.read(READ_CHUNK)
if not chunk:
break
digest.update(chunk)
payload.seek(0)
return digest.digest()
def _verify_checksum(stream: BinaryIO, start: int, end: int) -> None:
trailer = _read_exact(stream, 20)
stream.seek(start)
digest = hashlib.sha1()
remaining = end - start
while remaining:
chunk = stream.read(min(READ_CHUNK, remaining))
digest.update(chunk)
remaining -= len(chunk)
if digest.digest() != trailer:
raise PolicyError("Git pack checksum is invalid")
stream.seek(end + 20)
class _Entry:
"""One pack entry: either resolved content or an unresolved delta."""
def __init__(self, type_code: int, payload: BinaryIO, size: int, base):
self.type_code = type_code
self.payload = payload
self.size = size
self.base = base
def _resolve(entries: dict[int, _Entry], total: list[int]) -> None:
"""Resolve every delta against in-pack bases only, to a fixed point."""
by_sha: dict[bytes, int] = {}
for offset, entry in entries.items():
if entry.base is None:
by_sha.setdefault(
_payload_sha(entry.type_code, entry.payload, entry.size), offset
)
for _ in range(MAX_RESOLVE_PASSES):
pending = [item for item in entries.items() if item[1].base is not None]
if not pending:
return
progressed = False
for offset, entry in pending:
base_key = entry.base
base_offset = base_key if isinstance(base_key, int) else by_sha.get(base_key)
base = entries.get(base_offset) if base_offset is not None else None
if base is None or base.base is not None:
if isinstance(base_key, int) and base is None:
raise PolicyError("Git delta base offset is invalid")
continue
base.payload.seek(0)
entry.payload.seek(0)
value = _apply_delta(base.payload.read(), entry.payload.read())
total[0] += len(value)
if total[0] > MAX_TOTAL_BYTES:
raise PolicyError("Git pack contents exceed the safe size limit")
entry.payload.close()
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - entry owns spool
max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
)
spool.write(value)
spool.seek(0)
entry.type_code = base.type_code
entry.payload = spool
entry.size = len(value)
entry.base = None
by_sha.setdefault(_payload_sha(entry.type_code, spool, entry.size), offset)
progressed = True
if not progressed:
raise PolicyError("Git delta base is outside the pack; push full packs")
raise PolicyError("Git delta chains exceed the safe resolution limit")
def unpack_objects(stream: BinaryIO) -> list[tuple[int, BinaryIO]]:
"""Inflate one self-contained pack and return every resolved payload."""
start = stream.tell()
header = _read_exact(stream, 12)
version = int.from_bytes(header[4:8], "big")
count = int.from_bytes(header[8:12], "big")
if header[:4] != b"PACK" or version not in {2, 3}:
raise PolicyError("Git pack header is invalid")
if count > MAX_PACK_OBJECTS:
raise PolicyError("Git pack contains too many objects")
entries: dict[int, _Entry] = {}
total = [0]
try:
for _ in range(count):
offset = stream.tell() - start
type_code, declared = _object_header(stream)
if declared > MAX_OBJECT_BYTES:
raise PolicyError("Git object exceeds the safe size limit")
total[0] += declared
if total[0] > MAX_TOTAL_BYTES:
raise PolicyError("Git pack contents exceed the safe size limit")
base: int | bytes | None = None
if type_code == OFS_DELTA:
distance = _delta_distance(stream)
base = offset - distance
if base < 0:
raise PolicyError("Git delta base offset is invalid")
elif type_code == REF_DELTA:
base = _read_exact(stream, 20)
elif type_code not in OBJECT_TYPE_NAMES:
raise PolicyError("Git pack object type is invalid")
entries[offset] = _Entry(type_code, _inflate(stream, declared), declared, base)
_verify_checksum(stream, start, stream.tell())
_resolve(entries, total)
except Exception:
for entry in entries.values():
entry.payload.close()
raise
for entry in entries.values():
entry.payload.seek(0)
return [
(entries[offset].type_code, entries[offset].payload)
for offset in sorted(entries)
]
def parse_loose_object(value: bytes) -> tuple[int, bytes]:
"""Inflate one quarantined loose object and return its typed payload."""
if len(value) > MAX_OBJECT_BYTES:
raise PolicyError("Git object exceeds the safe size limit")
decompressor = zlib.decompressobj()
try:
inflated = decompressor.decompress(value, MAX_OBJECT_BYTES + 1)
except zlib.error as exc:
raise PolicyError("Git loose object data is corrupt") from exc
if not decompressor.eof or decompressor.unused_data or len(inflated) > MAX_OBJECT_BYTES:
raise PolicyError("Git loose object framing is invalid")
kind, separator, remainder = inflated.partition(b"\x00")
name, space, size = kind.partition(b" ")
types = {label: code for code, label in OBJECT_TYPE_NAMES.items()}
if (
not separator
or not space
or name not in types
or not size.isdigit()
or int(size) != len(remainder)
):
raise PolicyError("Git loose object header is invalid")
return types[name], remainder

View File

@ -0,0 +1,492 @@
#!/usr/bin/env python3
"""Use the private Atlas Forgejo API through a least-authority PR boundary."""
from __future__ import annotations
import argparse
import base64
import json
import os
import re
import sys
import urllib.error
import urllib.parse
import urllib.request
from collections.abc import Callable
from pathlib import Path
import deadline_http
from gitea_api_policy import (
PolicyError,
_draft_title,
_reject_forbidden,
_validate_body,
_validate_pr_number,
_validate_pr_number_segment,
_validate_query,
_validate_ref,
_validate_ref_bounds,
_validate_repo,
_validate_sha,
)
CANONICAL_BASE_URL = "https://scm.bstein.dev"
ALLOWED_OWNER = "atlas"
DEFAULT_TOKEN_FILE = Path("/vault/secrets/gitea-token")
CREATE_PATH_RE = re.compile(r"/api/v1/repos/atlas/([^/]+)/pulls\Z")
MAX_ERROR_BYTES = 8192
MAX_RESPONSE_BYTES = 2 * 1024 * 1024
MAX_API_TARGET_LENGTH = 768
MAX_API_PATH_LENGTH = 512
RAW_API_TARGET_RE = re.compile(r"[A-Za-z0-9/_.?&=-]+\Z")
def _safe_urlopen(request: urllib.request.Request, timeout: int):
"""Open one exchange under a killable absolute wall-clock deadline.
The helper process rejects redirects before urllib can copy
authentication headers, and its whole connect/send/read lifetime
shares one hard deadline.
"""
return deadline_http.open_bounded(
request, maximum=MAX_RESPONSE_BYTES, timeout=timeout
)
def read_token(path: Path = DEFAULT_TOKEN_FILE) -> str:
"""Read the token from the pod-lifetime Vault projection."""
token = path.read_text(encoding="utf-8").strip()
if not token:
raise ValueError(f"runtime credential is empty: {path}")
return token
def configured_base_url() -> str:
"""Reject attempts to redirect credentials away from the canonical origin."""
configured = os.environ.get("GITEA_BASE_URL", CANONICAL_BASE_URL)
if configured != CANONICAL_BASE_URL:
raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service")
return configured
def _split_api_path(path: str) -> urllib.parse.SplitResult:
if not isinstance(path, str) or len(path) > MAX_API_TARGET_LENGTH:
raise PolicyError("API target exceeds the safe size limit")
if (
not path.isascii()
or any(ord(character) < 32 or ord(character) == 127 for character in path)
or "\\" in path
or not RAW_API_TARGET_RE.fullmatch(path)
):
raise PolicyError("API target contains non-canonical raw characters")
target = urllib.parse.urlsplit(path)
canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, ""))
if canonical != path:
raise PolicyError("API target is not in exact canonical form")
if target.scheme or target.netloc or target.fragment:
raise PolicyError("API path must be relative to the Atlas SCM origin")
if not target.path.startswith("/api/v1/"):
raise PolicyError("API path must start with /api/v1/")
if len(target.path) > MAX_API_PATH_LENGTH:
raise PolicyError("API path exceeds the safe size limit")
segments = target.path.split("/")
if "//" in target.path or any(segment in {".", ".."} for segment in segments):
raise PolicyError("encoded or non-canonical API paths are not allowed")
return target
def _authorize_read(
target: urllib.parse.SplitResult, *, forbidden: tuple[str, ...] = ()
) -> str:
"""Allow only repository, PR, branch, commit, and status metadata reads."""
prefix = "/api/v1/repos/atlas/"
remainder = target.path.removeprefix(prefix)
if remainder == target.path:
raise PolicyError("reads are limited to explicit Atlas repository metadata")
repo, separator, suffix = remainder.partition("/")
_reject_forbidden(repo, "repository", forbidden)
_validate_repo(repo)
if not separator:
_validate_query(target, set())
return "repository"
if suffix == "pulls":
_validate_query(target, {"page", "limit", "state"})
return "pull-list"
content_match = re.fullmatch(r"pulls/([^/]+)\.(?:patch|diff)", suffix)
if content_match:
_validate_pr_number_segment(content_match.group(1))
raise PolicyError("repository API route is outside the metadata read allowlist")
pull_match = re.fullmatch(r"pulls/([^/]+)", suffix)
if pull_match:
_validate_pr_number_segment(pull_match.group(1))
_validate_query(target, set())
return "pull"
evidence_match = re.fullmatch(r"pulls/([^/]+)/(?:commits|files)", suffix)
if evidence_match:
_validate_pr_number_segment(evidence_match.group(1))
_validate_query(target, {"page", "limit"})
return "pull-evidence"
if suffix == "branches":
_validate_query(target, {"page", "limit"})
return "branch-list"
branch_match = re.fullmatch(r"branches/([^/]+)", suffix)
if branch_match:
_validate_ref(branch_match.group(1), "branch")
_validate_query(target, set())
return "branch"
if suffix == "commits":
_validate_query(target, {"page", "limit"})
return "commit-list"
if re.fullmatch(r"git/commits/[0-9a-fA-F]{40}", suffix):
_validate_query(target, set())
return "commit"
if re.fullmatch(
r"(?:commits/[0-9a-fA-F]{40}/status|commits/[0-9a-fA-F]{40}/statuses|statuses/[0-9a-fA-F]{40})",
suffix,
):
_validate_query(target, {"page", "limit"})
return "status"
raise PolicyError("repository API route is outside the metadata read allowlist")
def api_url(base_url: str, path: str) -> str:
"""Return a canonical same-origin URL without forwarding credentials."""
if base_url != CANONICAL_BASE_URL:
raise PolicyError("Forgejo origin is fixed to the private Atlas SCM service")
target = _split_api_path(path)
return urllib.parse.urlunsplit(
("https", "scm.bstein.dev", target.path, target.query, "")
)
def authorize_request(
method: str,
path: str,
data: object | None,
*,
forbidden: tuple[str, ...] = (),
) -> str:
"""Validate one request and return its bounded operation name."""
normalized_method = method.upper()
target = _split_api_path(path)
if normalized_method == "GET":
if data is not None:
raise PolicyError("read operations cannot include a request body")
return _authorize_read(target, forbidden=forbidden)
if target.query:
raise PolicyError("mutating operations cannot include query parameters")
if normalized_method != "POST":
raise PolicyError("only read and create-draft operations are available")
match = CREATE_PATH_RE.fullmatch(target.path)
if not match:
raise PolicyError("POST is limited to creating an Atlas draft pull request")
_reject_forbidden(match.group(1), "repository", forbidden)
_validate_repo(match.group(1))
if not isinstance(data, dict) or set(data) != {"base", "body", "head", "title"}:
raise PolicyError("draft creation accepts only base, body, head, and title")
if data["title"] != _draft_title(data["title"], forbidden=forbidden):
raise PolicyError("new pull requests must use the Gitea draft-title prefix")
_validate_body(data["body"], forbidden=forbidden)
_validate_ref_bounds(data["base"], "base", forbidden=forbidden)
_validate_ref_bounds(data["head"], "head", forbidden=forbidden)
_validate_ref(data["base"], "base", forbidden=forbidden)
_validate_ref(data["head"], "head", forbidden=forbidden)
return "create-draft"
def build_request(
method: str,
path: str,
*,
base_url: str,
token: str,
data: object | None = None,
) -> urllib.request.Request:
"""Build an authorized request without putting the token in its URL or body."""
authorize_request(method, path, data, forbidden=(token,))
return _build_request(method, path, base_url=base_url, token=token, data=data)
def _build_request(
method: str,
path: str,
*,
base_url: str,
token: str,
data: object | None = None,
) -> urllib.request.Request:
"""Build one already-authorized upstream request."""
payload = None
if data is not None:
payload = json.dumps(data, separators=(",", ":")).encode("utf-8")
return urllib.request.Request(
api_url(base_url, path),
data=payload,
method=method.upper(),
headers={
"Accept": "application/json",
"Authorization": f"token {token}",
"Content-Type": "application/json",
"User-Agent": "hermes-atlas-pr-client/2",
},
)
def redact_bytes(value: bytes, token: str) -> bytes:
"""Remove exact runtime-token and authorization-header values from output."""
redacted = value.replace(token.encode("utf-8"), b"[REDACTED]")
return re.sub(
rb"(?i)(authorization\s*[:=]\s*)(?:token|bearer)?\s*[^\s,;\"}]+",
rb"\1[REDACTED]",
redacted,
)
def _reject_response_credential(value: bytes, token: str) -> None:
"""Fail closed if an upstream body reflects any ordinary token encoding."""
raw = token.encode("utf-8")
forms = {
raw,
base64.b64encode(raw),
base64.b64encode(b"hermes-automation:" + raw),
urllib.parse.quote(token, safe="").encode("ascii"),
}
if any(form and form in value for form in forms):
raise PolicyError("Forgejo response contains runtime credential material")
def _request(
method: str,
path: str,
data: object | None,
*,
token: str,
opener: Callable[..., object] = _safe_urlopen,
authorized: bool = False,
expected_status: int,
) -> bytes:
builder = _build_request if authorized else build_request
request = builder(method, path, base_url=configured_base_url(), token=token, data=data)
with opener(request, timeout=30) as response: # type: ignore[attr-defined]
status = getattr(response, "status", None)
if status is None and hasattr(response, "getcode"):
status = response.getcode() # type: ignore[attr-defined]
if status != expected_status:
raise PolicyError("Forgejo returned an unexpected HTTP status")
if response.headers.get_content_type() != "application/json": # type: ignore[attr-defined]
raise PolicyError("Forgejo returned an unexpected response type")
body = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined]
if len(body) > MAX_RESPONSE_BYTES:
raise PolicyError("Forgejo response exceeds the safe size limit")
_reject_response_credential(body, token)
return redact_bytes(body, token)
def read(
path: str, *, token: str, opener: Callable[..., object] = _safe_urlopen
) -> bytes:
"""Read one explicitly allowed Atlas repository metadata resource."""
return _request(
"GET", path, None, token=token, opener=opener, expected_status=200
)
def _nested(document: dict[str, object], *keys: str) -> object:
value: object = document
for key in keys:
if not isinstance(value, dict) or key not in value:
raise PolicyError("Forgejo omitted required pull-request evidence")
value = value[key]
return value
def _require_create_response(
value: bytes,
*,
repo: str,
base: str,
head: str,
head_sha: str,
title: str,
body: str,
) -> dict[str, object]:
"""Require exact server evidence for the intended open draft handoff."""
try:
document = json.loads(value)
except json.JSONDecodeError as exc:
raise PolicyError("Forgejo returned invalid pull-request metadata") from exc
if not isinstance(document, dict):
raise PolicyError("Forgejo returned invalid pull-request metadata")
number = _validate_pr_number(document.get("number"))
full_name = f"{ALLOWED_OWNER}/{repo}"
expected_html = f"{CANONICAL_BASE_URL}/{full_name}/pulls/{number}"
checks = (
document.get("state") == "open",
document.get("draft") is True,
document.get("merged") is False,
document.get("html_url") == expected_html,
document.get("url") == expected_html,
document.get("title") == title,
document.get("body") == body,
_nested(document, "base", "ref") == base,
_nested(document, "base", "repo", "full_name") == full_name,
_nested(document, "head", "ref") == head,
_nested(document, "head", "sha") == head_sha,
_nested(document, "head", "repo", "full_name") == full_name,
)
if not all(checks):
raise PolicyError("Forgejo did not confirm the exact draft pull-request state")
return document
def create_draft(
repo: str,
*,
base: str,
head: str,
head_sha: str,
title: str,
body: str,
token: str,
opener: Callable[..., object] = _safe_urlopen,
) -> bytes:
"""Create and verify a same-repository draft pull request for human review."""
_reject_forbidden(repo, "repository", (token,))
repo = _validate_repo(repo)
title = _draft_title(title, forbidden=(token,))
body = _validate_body(body, forbidden=(token,))
head_sha = _validate_sha(head_sha)
_validate_ref_bounds(base, "base", forbidden=(token,))
_validate_ref_bounds(head, "head", forbidden=(token,))
base = _validate_ref(base, "base", forbidden=(token,))
head = _validate_ref(head, "head", forbidden=(token,))
data = {"base": base, "body": body, "head": head, "title": title}
result = _request(
"POST",
f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls",
data,
token=token,
opener=opener,
authorized=True,
expected_status=201,
)
_require_create_response(
result,
repo=repo,
base=base,
head=head,
head_sha=head_sha,
title=title,
body=body,
)
return result
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
"""Parse the intentionally narrow read and create-draft operations."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--dry-run",
action="store_true",
help="validate and describe the request without reading a token or using the network",
)
subparsers = parser.add_subparsers(dest="operation", required=True)
read_parser = subparsers.add_parser("read", help="read allowed Atlas metadata")
read_parser.add_argument("path")
create_parser = subparsers.add_parser(
"create-draft", help="create a same-repository draft pull request"
)
create_parser.add_argument("repo")
create_parser.add_argument("--base", required=True)
create_parser.add_argument("--head", required=True)
create_parser.add_argument("--head-sha", required=True)
create_parser.add_argument("--title", required=True)
create_parser.add_argument("--body", default="")
return parser.parse_args(argv)
def _operation(
args: argparse.Namespace, *, validate_refs: bool = True
) -> tuple[str, str, object | None]:
if args.operation == "read":
return "GET", args.path, None
repo = _validate_repo(args.repo)
_validate_sha(args.head_sha)
body = _validate_body(args.body)
title = _draft_title(args.title)
_validate_ref_bounds(args.base, "base")
_validate_ref_bounds(args.head, "head")
base = _validate_ref(args.base, "base") if validate_refs else args.base
head = _validate_ref(args.head, "head") if validate_refs else args.head
data = {
"base": base,
"body": body,
"head": head,
"title": title,
}
return "POST", f"/api/v1/repos/{ALLOWED_OWNER}/{repo}/pulls", data
def _write_body(body: bytes) -> None:
if body:
sys.stdout.buffer.write(body)
if not body.endswith(b"\n"):
sys.stdout.buffer.write(b"\n")
def main(argv: list[str] | None = None) -> int:
"""Validate, optionally describe, and execute one safe Forgejo operation."""
try:
args = parse_args(argv)
if args.dry_run:
method, path, data = _operation(args, validate_refs=False)
operation = authorize_request(method, path, data)
print(
json.dumps(
{
"dry_run": True,
"host": "scm.bstein.dev",
"operation": operation,
"owner": ALLOWED_OWNER,
"path": path,
"request_fields": sorted(data)
if isinstance(data, dict)
else [],
},
sort_keys=True,
)
)
return 0
method, path, data = _operation(args, validate_refs=False)
from scm_broker_client import create_draft as broker_create_draft
from scm_broker_client import read as broker_read
if args.operation == "read":
body = broker_read(path)
else:
assert isinstance(data, dict)
body = broker_create_draft(
args.repo,
base=str(data["base"]),
head=str(data["head"]),
head_sha=args.head_sha,
title=str(data["title"]),
body=str(data["body"]),
)
_write_body(body)
return 0
except urllib.error.HTTPError as exc:
exc.read(MAX_ERROR_BYTES)
print(f"SCM broker request failed with HTTP {exc.code}", file=sys.stderr)
return 1
except (OSError, PolicyError, ValueError, json.JSONDecodeError):
print(
"Forgejo request rejected or unavailable; no credential was disclosed",
file=sys.stderr,
)
return 1
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,490 @@
"""Validation policy for Hermes' least-authority Atlas Forgejo client.
The screening catches structured credential assignments, known token formats,
and long high-entropy values. It is a fail-closed accident barrier, not proof
that text is secret-free: a short or multiword secret under an innocuous key is
not reliably distinguishable from prose and must never be supplied by callers.
JSON depth/nodes/documents and multiline gaps are deliberately bounded; large
or unusual configuration blobs are not a supported pull-request field format.
"""
# ruff: noqa: SIM905
from __future__ import annotations
import json
import re
import subprocess
import urllib.parse
from collections import Counter
from math import log2
REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}\Z")
SHA_RE = re.compile(r"[0-9a-fA-F]{40}\Z")
DRAFT_TITLE_PREFIX = "WIP: "
GIT_BIN = "/usr/bin/git"
MAX_QUERY_LENGTH = 128
MAX_QUERY_ITEMS = 3
MAX_QUERY_KEY_LENGTH = 16
MAX_QUERY_VALUE_LENGTH = 16
MAX_PAGE = 10_000
MAX_LIMIT = 50
MAX_PR_NUMBER = 2_147_483_647
MAX_PR_NUMBER_DIGITS = 10
MAX_REF_CHARACTERS = 200
MAX_REF_UTF8_BYTES = 255
MAX_TITLE_UTF8_BYTES = 512
MAX_BODY_UTF8_BYTES = 32_768
MAX_JSON_DOCUMENTS = 32
MAX_JSON_NODES = 2_048
MAX_JSON_DEPTH = 32
CANONICAL_QUERY_RE = re.compile(r"[A-Za-z0-9_=&-]*\Z")
ASSIGNMENT_RE = re.compile(
r"""(?mx)
(?<![A-Za-z0-9_.-])
(?P<key_quote>[\"']?)
(?P<key>\.?[A-Za-z][A-Za-z0-9_.-]{0,127})
(?P=key_quote)
[ \t]*(?P<separator>[:=])[ \t]*
(?P<value>
\"(?:\\.|[^\"\\\r\n])*\" |
'(?:\\.|[^'\\\r\n])*' |
[^\r\n,;}&]{0,2048}
)
"""
)
JSON_KEY_RE = re.compile(r'"(?P<key>[^"]{1,512})"(?P<spacing>[\x00-\x20\x7f]{0,256}):')
MULTILINE_ASSIGNMENT_RE = re.compile(
r"""(?mx)
(?<![A-Za-z0-9_.-])
(?P<key_quote>["']?)
(?P<key>\.?[A-Za-z][A-Za-z0-9_.-]{0,127})
(?P=key_quote)
[ \t]{0,64}(?:\r?\n[ \t]{0,64}){0,4}
(?P<separator>[:=])
[ \t]{0,64}(?:\r?\n[ \t]{0,64}){1,4}
(?P<value>[^\r\n,;}&]{1,2048})
"""
)
CAMEL_BOUNDARY_RE = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])")
CREDENTIAL_SCHEME_RE = re.compile(
r"(?i)\b(?:bearer|basic|token)\s+"
r"(?=[A-Za-z0-9+/_=.:-]{8,}\b)(?=[A-Za-z0-9+/_=.:-]*[0-9+/_=.-])"
r"[A-Za-z0-9+/_=.:-]{8,}"
)
STANDALONE_SECRET_PATTERNS = tuple(
re.compile(pattern, re.IGNORECASE)
for pattern in (
r"(?<![A-Za-z0-9])gh[pousr]_[A-Za-z0-9]{20,}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])glpat-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])xox[baprs]-[A-Za-z0-9-]{20,}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])sk-(?:ant-|proj-)?[A-Za-z0-9_-]{20,}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])[rs]k_(?:test|live)_[A-Za-z0-9]{20,}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])whsec_[A-Za-z0-9]{20,}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])npm_[A-Za-z0-9]{20,}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])pypi-[A-Za-z0-9_-]{30,}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])hf_[A-Za-z0-9]{20,}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])(?:gta|gto|gitea|forgejo)_[A-Za-z0-9_-]{20,}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])ya29\.[A-Za-z0-9_-]{20,}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])oy2[A-Za-z0-9]{40,}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{20,}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])SK[0-9a-f]{32}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])(?:AKIA|ASIA)[0-9A-Z]{16}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])AIza[0-9A-Za-z_-]{35}(?![A-Za-z0-9])",
r"(?<![A-Za-z0-9])eyJ[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{8,}(?![A-Za-z0-9])",
r"-----BEGIN\s+[A-Z0-9 ][A-Z0-9 -]{1,62}-----",
r"\bssh-(?:rsa|ed25519|dss|ecdsa-[A-Za-z0-9-]+)\s+[A-Za-z0-9+/]{16,}={0,3}",
r"https://hooks\.slack\.com/services/[A-Za-z0-9/_-]{20,}",
r"https://(?:discord(?:app)?\.com)/api/webhooks/[0-9]+/[A-Za-z0-9._-]{20,}",
r"https://[^\s/]*webhook\.office\.com/[^\s]{20,}",
)
)
HIGH_ENTROPY_TOKEN_RE = re.compile(
r"(?<![A-Za-z0-9+/_=-])[A-Za-z0-9+/_=-]{48,16384}(?![A-Za-z0-9+/_=-])"
)
PROSE_LEAD_WORDS = frozenset(
"add allow change check describe document ensure explain fix handle keep preserve "
"prevent reject remove review support test validate".split()
)
SENSITIVE_KEY_WORDS = frozenset(
"auth authentication authorization auths basic bearer credential credentials "
"passwd password pat private sas secret secrets sig signature signing token tokens "
"webhook".split()
)
KEY_MODIFIER_WORDS = frozenset(
"access account api auth client encryption identity private registry secret service "
"session signing ssh".split()
)
SENSITIVE_COMPACT_KEYS = frozenset(
"accessid accesskeyid accountkey clientemail clientid connectionstring "
"dockerconfigjson privatekeyid serviceaccountkey sharedaccesssignature".split()
)
SENSITIVE_COMPACT_SUFFIXES = tuple(
"accesskey accesskeyid accountkey apikey authkey clientemail clientid clientkey "
"connectionstring credential credentials encryptionkey identitykey password passwd "
"privatekeyid privatekey secret secretkey servicekey sessionkey signature signingkey "
"sshkey token webhook".split()
)
class PolicyError(ValueError):
"""Raised when a requested Forgejo operation exceeds the safe boundary."""
def _validate_repo(repo: str) -> str:
if not REPO_RE.fullmatch(repo) or repo in {".", ".."}:
raise PolicyError("repository name is outside the Atlas allowlist")
return repo
def _reject_forbidden(value: object, name: str, forbidden: tuple[str, ...]) -> None:
"""Reject exact runtime credentials before invoking helpers or the network."""
if not isinstance(value, str) or any(secret and secret in value for secret in forbidden):
raise PolicyError(f"{name} contains runtime credential material")
def _validate_ref_bounds(
value: object, name: str, *, forbidden: tuple[str, ...] = ()
) -> str:
"""Reject oversized or non-encodable refs before invoking Git."""
_reject_forbidden(value, name, forbidden)
if not value or len(value) > MAX_REF_CHARACTERS:
raise PolicyError(f"{name} exceeds the safe branch-name limit")
try:
encoded = value.encode("utf-8", errors="strict")
except UnicodeEncodeError as exc:
raise PolicyError(f"{name} is not valid UTF-8 text") from exc
if len(encoded) > MAX_REF_UTF8_BYTES:
raise PolicyError(f"{name} exceeds the safe UTF-8 branch-name limit")
return value
def _validate_ref(
value: object, name: str, *, forbidden: tuple[str, ...] = ()
) -> str:
"""Validate the complete Git ref grammar using Git itself."""
value = _validate_ref_bounds(value, name, forbidden=forbidden)
if value.startswith("-"):
raise PolicyError(f"{name} must be a same-repository branch name")
if any(ord(character) < 32 or ord(character) == 127 for character in value):
raise PolicyError(f"{name} must be a safe same-repository branch name")
result = subprocess.run(
[GIT_BIN, "check-ref-format", f"refs/heads/{value}"],
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False,
)
if result.returncode != 0:
raise PolicyError(f"{name} must be a safe same-repository branch name")
return value
def _validate_sha(value: object, name: str = "head SHA") -> str:
if not isinstance(value, str) or not SHA_RE.fullmatch(value):
raise PolicyError(f"{name} must be a full Git SHA-1")
return value.lower()
def _validate_pr_number_segment(value: object) -> int:
"""Validate a canonical bounded pull-request number from an API path."""
if (
not isinstance(value, str)
or len(value) > MAX_PR_NUMBER_DIGITS
or not re.fullmatch(r"[1-9][0-9]*", value)
):
raise PolicyError("pull-request number must use bounded canonical ASCII digits")
number = int(value)
if number > MAX_PR_NUMBER:
raise PolicyError("pull-request number is outside its allowed range")
return number
def _validate_pr_number(value: object) -> int:
"""Validate a bounded pull-request number returned by Forgejo."""
if (
not isinstance(value, int)
or isinstance(value, bool)
or not 1 <= value <= MAX_PR_NUMBER
):
raise PolicyError("Forgejo omitted a valid pull-request number")
return value
def _validate_text(
value: object,
name: str,
maximum: int,
maximum_bytes: int,
*,
required: bool,
) -> str:
if not isinstance(value, str):
raise PolicyError(f"{name} must be text")
if required and not value.strip():
raise PolicyError(f"{name} must not be empty")
try:
encoded = value.encode("utf-8", errors="strict")
except UnicodeEncodeError as exc:
raise PolicyError(f"{name} is not valid UTF-8 text") from exc
if len(value) > maximum or len(encoded) > maximum_bytes or "\x00" in value:
raise PolicyError(f"{name} exceeds the safe request limit")
return value
def _normalized_key(value: str) -> tuple[tuple[str, ...], str]:
"""Split camelCase and separator-based assignment keys into semantics."""
separated = CAMEL_BOUNDARY_RE.sub(" ", value.strip(".\"'"))
words = tuple(re.findall(r"[A-Za-z0-9]+", separated.lower()))
return words, "".join(words)
def _is_sensitive_key(value: str) -> bool:
words, compact = _normalized_key(value)
word_set = set(words)
if word_set & SENSITIVE_KEY_WORDS:
return True
if compact in SENSITIVE_COMPACT_KEYS:
return True
if compact.endswith(SENSITIVE_COMPACT_SUFFIXES):
return True
if "key" in word_set and word_set & KEY_MODIFIER_WORDS:
return True
if {"client", "email"} <= word_set or {"client", "id"} <= word_set:
return True
if {"access", "id"} <= word_set or {"connection", "string"} <= word_set:
return True
return compact.endswith("configjson")
def _strip_assignment_value(value: str) -> tuple[str, bool]:
stripped = value.strip()
quoted = (
len(stripped) >= 2 and stripped[0] in {'"', "'"} and stripped[-1] == stripped[0]
)
if quoted:
stripped = stripped[1:-1].strip()
return stripped, quoted
def _looks_like_prose(value: str) -> bool:
words = re.findall(r"[A-Za-z]+(?:[-'][A-Za-z]+)?", value)
if len(words) < 2:
return False
if re.search(r"[$`{}\[\]\\@/:=+_]", value):
return False
return words[0].lower() in PROSE_LEAD_WORDS
def _looks_sensitive_assignment_value(value: str) -> bool:
stripped, quoted = _strip_assignment_value(value)
if not stripped:
return False
if quoted or stripped.startswith(("{", "[", "|", ">", "!!", "&", "*")):
return True
if CREDENTIAL_SCHEME_RE.search(stripped):
return True
if any(pattern.search(stripped) for pattern in STANDALONE_SECRET_PATTERNS):
return True
if re.search(r"(?:https?://|[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+)", stripped):
return True
if re.search(r"(?:\$\{|\$\(|\\[nrt]|[A-Za-z0-9+/]{16,}={0,3}\Z)", stripped):
return True
return not _looks_like_prose(stripped)
def _is_structural_credential_assignment(key: str, value: str) -> bool:
words, compact_key = _normalized_key(key)
stripped, _quoted = _strip_assignment_value(value)
_value_words, compact_value = _normalized_key(stripped)
return (
compact_key == "type" and compact_value in {"serviceaccount", "credential"}
) or ("service" in words and "account" in words and bool(stripped))
def _decode_json_key(raw: str) -> tuple[str, bool]:
"""Decode one bounded JSON key without silently accepting bad escapes."""
try:
decoded = json.loads(f'"{raw}"')
except (json.JSONDecodeError, UnicodeError):
approximate = re.sub(r"[^A-Za-z0-9]+", "_", raw.replace("\\", ""))
return approximate, False
return decoded if isinstance(decoded, str) else "", True
def _json_value_has_sensitive_assignment(
value: object, *, depth: int = 0, nodes: list[int] | None = None
) -> bool:
"""Walk one decoded JSON value with explicit depth and node ceilings."""
if nodes is None:
nodes = [0]
nodes[0] += 1
if nodes[0] > MAX_JSON_NODES or depth > MAX_JSON_DEPTH:
raise PolicyError("structured pull-request text exceeds the scan limit")
if isinstance(value, dict):
for key, assigned in value.items():
if not isinstance(key, str):
continue
if _is_sensitive_key(key):
return True
assigned_text = assigned if isinstance(assigned, str) else ""
if _is_structural_credential_assignment(key, assigned_text):
return True
if _json_value_has_sensitive_assignment(
assigned, depth=depth + 1, nodes=nodes
):
return True
elif isinstance(value, list):
return any(
_json_value_has_sensitive_assignment(item, depth=depth + 1, nodes=nodes)
for item in value
)
return False
def _decoded_json_documents(value: str):
"""Yield bounded embedded JSON object or array fragments."""
decoder = json.JSONDecoder()
index = 0
attempts = 0
while attempts < MAX_JSON_DOCUMENTS:
positions = [
position for token in "{[" if (position := value.find(token, index)) >= 0
]
if not positions:
return
start = min(positions)
attempts += 1
try:
document, end = decoder.raw_decode(value, start)
except (json.JSONDecodeError, RecursionError, ValueError):
index = start + 1
continue
yield document
index = max(end, start + 1)
if "{" in value[index:] or "[" in value[index:]:
raise PolicyError("structured pull-request text exceeds the scan limit")
def _has_structured_sensitive_assignment(value: str) -> bool:
"""Detect bounded JSON and multiline YAML/env credential assignments."""
for candidate in JSON_KEY_RE.finditer(value):
decoded, valid = _decode_json_key(candidate.group("key"))
if _is_sensitive_key(decoded):
return True
if not valid and _is_sensitive_key(candidate.group("key").replace("\\", "")):
return True
for candidate in MULTILINE_ASSIGNMENT_RE.finditer(value):
key = candidate.group("key")
assigned = candidate.group("value")
if _is_structural_credential_assignment(key, assigned) or (
_is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned)
):
return True
return any(
_json_value_has_sensitive_assignment(item)
for item in _decoded_json_documents(value)
)
def _has_high_entropy_token(value: str) -> bool:
for match in HIGH_ENTROPY_TOKEN_RE.finditer(value):
token = match.group(0)
if len(token) > 256:
return True
groups = sum(
bool(re.search(pattern, token))
for pattern in (r"[a-z]", r"[A-Z]", r"[0-9]", r"[+/_=-]")
)
counts = Counter(token)
entropy = -sum(
(count / len(token)) * log2(count / len(token)) for count in counts.values()
)
if groups >= 3 and entropy >= 4.0:
return True
return False
def _reject_sensitive(value: str, name: str, forbidden: tuple[str, ...]) -> None:
if any(secret and secret in value for secret in forbidden):
raise PolicyError(f"pull-request {name} contains runtime credential material")
if CREDENTIAL_SCHEME_RE.search(value):
raise PolicyError(f"pull-request {name} resembles credential material")
if any(pattern.search(value) for pattern in STANDALONE_SECRET_PATTERNS):
raise PolicyError(f"pull-request {name} resembles credential material")
if _has_structured_sensitive_assignment(value):
raise PolicyError(f"pull-request {name} resembles credential material")
for candidate in ASSIGNMENT_RE.finditer(value):
key = candidate.group("key")
assigned = candidate.group("value")
if _is_structural_credential_assignment(key, assigned) or (
_is_sensitive_key(key) and _looks_sensitive_assignment_value(assigned)
):
raise PolicyError(f"pull-request {name} resembles credential material")
if _has_high_entropy_token(value):
raise PolicyError(f"pull-request {name} resembles credential material")
def _validate_body(value: object, *, forbidden: tuple[str, ...] = ()) -> str:
body = _validate_text(value, "body", 16384, MAX_BODY_UTF8_BYTES, required=False)
_reject_sensitive(body, "body", forbidden)
return body
def _draft_title(value: object, *, forbidden: tuple[str, ...] = ()) -> str:
"""Return a bounded, secret-screened Gitea draft title."""
title = _validate_text(
value, "title", 251, MAX_TITLE_UTF8_BYTES, required=True
).strip()
_reject_sensitive(title, "title", forbidden)
for prefix in ("WIP:", "[WIP]"):
if title.upper().startswith(prefix):
title = title[len(prefix) :].lstrip()
break
if not title:
raise PolicyError("title must contain text after the draft prefix")
return DRAFT_TITLE_PREFIX + title
def _validate_query(target: urllib.parse.SplitResult, allowed: set[str]) -> None:
"""Accept only short canonical ASCII query strings with bounded pagination."""
raw = target.query
if (
len(raw) > MAX_QUERY_LENGTH
or not raw.isascii()
or not CANONICAL_QUERY_RE.fullmatch(raw)
or "%" in raw
):
raise PolicyError("API query must use short canonical ASCII form")
try:
pairs = urllib.parse.parse_qsl(raw, keep_blank_values=True, strict_parsing=True)
except ValueError as exc:
raise PolicyError("invalid API query") from exc
if len(pairs) > MAX_QUERY_ITEMS:
raise PolicyError("too many API query parameters")
if len({key for key, _ in pairs}) != len(pairs):
raise PolicyError("duplicate API query parameters are not allowed")
if any(
not key
or len(key) > MAX_QUERY_KEY_LENGTH
or len(value) > MAX_QUERY_VALUE_LENGTH
for key, value in pairs
):
raise PolicyError("API query key or value exceeds its safe limit")
if any(key not in allowed for key, _ in pairs):
raise PolicyError("API query parameter is outside the read allowlist")
values = dict(pairs)
for name in ("page", "limit"):
if name not in values:
continue
if not re.fullmatch(r"[0-9]+", values[name]):
raise PolicyError(f"{name} must use ASCII decimal digits")
number = int(values[name])
if values[name] != str(number):
raise PolicyError(f"{name} must use canonical ASCII decimal form")
ceiling = MAX_PAGE if name == "page" else MAX_LIMIT
if not 1 <= number <= ceiling:
raise PolicyError(f"{name} is outside its allowed range")
if "state" in values and values["state"] not in {"open", "closed", "all"}:
raise PolicyError("pull-request state is invalid")

View File

@ -0,0 +1,143 @@
#!/usr/bin/env python3
"""Validate Git receive-pack requests and scan pushed object content.
Ref commands may only create new namespaced feature branches. Every pushed
object is then inflated in quarantine and its decompressed payload scanned,
so credentials cannot ride through the broker inside compressed pack data.
The screening is a fail-closed accident barrier for credential-shaped
content: exact runtime-token forms, private keys, SSH key material, and
known provider token formats. Entropy heuristics are deliberately excluded
so SOPS ciphertext, pinned digests, and lock files remain pushable; a short
secret disguised as prose is not reliably detectable and must never be
supplied by callers.
"""
from __future__ import annotations
import io
import re
from typing import BinaryIO
import git_pack_objects
from gitea_api_policy import (
STANDALONE_SECRET_PATTERNS,
PolicyError,
_validate_ref,
)
ZERO_SHA = b"0" * 40
MAX_PUSH_COMMANDS = 16
SCAN_CHUNK = 256 * 1024
SCAN_CARRY = 8 * 1024
FEATURE_REF_RE = re.compile(
r"refs/heads/(?:(?:feature|fix|hermes|handoff)/[A-Za-z0-9][A-Za-z0-9._/-]{0,190})\Z"
)
PKT_HEADER_RE = re.compile(rb"[0-9a-f]{4}\Z")
CONTENT_SECRET_PATTERNS = tuple(
re.compile(pattern.pattern.encode("ascii"), re.IGNORECASE)
for pattern in STANDALONE_SECRET_PATTERNS
) + (
re.compile(rb"(?i)(?<![A-Za-z0-9])AGE-SECRET-KEY-1[A-Z0-9]{20,}"),
re.compile(rb"(?i)PuTTY-User-Key-File"),
re.compile(
rb"(?im)^[ \t]*authorization[ \t]*:[ \t]*(?:basic|bearer|token)[ \t]+"
rb"(?=[A-Za-z0-9+/_=.:-]*[0-9+/=.-])[A-Za-z0-9+/_=.:-]{8,}"
),
)
def _read_pkt_line(stream: BinaryIO) -> bytes | None:
"""Read one pkt-line; None marks the flush packet before the pack."""
header = stream.read(4)
if not header:
raise PolicyError("Git receive-pack request omitted its command terminator")
if not PKT_HEADER_RE.fullmatch(header):
raise PolicyError("Git receive-pack command framing is invalid")
size = int(header, 16)
if size == 0:
return None
if size < 4:
raise PolicyError("Git receive-pack command framing is invalid")
value = stream.read(size - 4)
if len(value) != size - 4:
raise PolicyError("Git receive-pack command framing is invalid")
return value
def _validate_command(line: bytes, token: str, forbidden: tuple[bytes, ...]) -> None:
"""Permit only creation of one new, namespaced feature branch."""
if any(form in line for form in forbidden):
raise PolicyError("Git request contains runtime credential material")
command = line.rstrip(b"\n").split(b"\x00", 1)[0]
fields = command.split(b" ")
if len(fields) != 3 or not all(
re.fullmatch(rb"[0-9a-f]{40}", item) for item in fields[:2]
):
raise PolicyError("Git receive-pack ref command is invalid")
old_sha, new_sha, raw_ref = fields
if old_sha != ZERO_SHA or new_sha == ZERO_SHA:
raise PolicyError("Git broker permits only new feature-branch creation")
try:
ref = raw_ref.decode("ascii")
except UnicodeDecodeError as exc:
raise PolicyError("Git ref must be canonical ASCII") from exc
if not FEATURE_REF_RE.fullmatch(ref):
raise PolicyError("Git push is limited to namespaced feature branches")
_validate_ref(ref.removeprefix("refs/heads/"), "head", forbidden=(token,))
def _scan_payload(payload: BinaryIO, forbidden: tuple[bytes, ...]) -> None:
"""Scan one decompressed object payload across chunk boundaries."""
carry = b""
while True:
chunk = payload.read(SCAN_CHUNK)
if not chunk:
return
window = carry + chunk
if any(form in window for form in forbidden):
raise PolicyError("Git push contains runtime credential material")
if any(pattern.search(window) for pattern in CONTENT_SECRET_PATTERNS):
raise PolicyError("Git push contains credential-shaped content")
carry = window[-SCAN_CARRY:]
def _scan_pack(stream: BinaryIO, forbidden: tuple[bytes, ...]) -> None:
objects = git_pack_objects.unpack_objects(stream)
try:
if stream.read(1):
raise PolicyError("Git request has bytes after its pack")
for _type_code, payload in objects:
_scan_payload(payload, forbidden)
finally:
for _type_code, payload in objects:
payload.close()
def validate_receive_pack(
body: bytes | BinaryIO, token: str, forbidden: tuple[bytes, ...]
) -> None:
"""Validate one whole receive-pack request: commands, pack, and content."""
stream = io.BytesIO(body) if isinstance(body, bytes) else body
stream.seek(0)
try:
commands = 0
while True:
line = _read_pkt_line(stream)
if line is None:
break
_validate_command(line, token, forbidden)
commands += 1
if commands > MAX_PUSH_COMMANDS:
raise PolicyError("Git push contains too many ref commands")
if commands == 0:
raise PolicyError("Git receive-pack request has no ref command")
_scan_pack(stream, forbidden)
finally:
stream.seek(0)
def scan_loose_object(value: bytes, forbidden: tuple[bytes, ...]) -> int:
"""Scan one quarantined loose object and return its parsed type code."""
type_code, payload = git_pack_objects.parse_loose_object(value)
_scan_payload(io.BytesIO(payload), forbidden)
return type_code

View File

@ -0,0 +1,455 @@
#!/usr/bin/env python3
"""Serve the credential-isolated Atlas API and Git smart-HTTP boundary."""
from __future__ import annotations
import argparse
import base64
import json
import re
import socket
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request
from http.server import BaseHTTPRequestHandler
from typing import BinaryIO
import deadline_http
from gitea_api import (
CANONICAL_BASE_URL,
PolicyError,
create_draft,
read,
read_token,
)
from gitea_api_policy import _reject_forbidden, _validate_repo
from receive_pack_scan import validate_receive_pack
from scm_broker_io import RejectRedirect, response_status as _status, spool_response
from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer
BROKER_PORT = 9081
GIT_USER = "hermes-automation"
MAX_CONTROL_BODY = 64 * 1024
MAX_GIT_REQUEST = 128 * 1024 * 1024
MAX_GIT_RESPONSE = 128 * 1024 * 1024
MAX_HEADERS = 32
MAX_HEADER_BYTES = 16 * 1024
SPOOL_MEMORY_LIMIT = 1024 * 1024
STREAM_CHUNK = 64 * 1024
INBOUND_HEADER_TIMEOUT = 10.0
INBOUND_BODY_TIMEOUT = 120.0
UPSTREAM_DEADLINE_SECONDS = 300.0
GIT_PATH_RE = re.compile(
r"/git/atlas/(?P<repo>[A-Za-z0-9][A-Za-z0-9._-]{0,99})\.git/"
r"(?P<operation>info/refs|git-upload-pack|git-receive-pack)\Z"
)
def _read_bounded(
stream,
maximum: int,
length: int | None = None,
*,
deadline: float | None = None,
set_timeout=None,
) -> bytes:
if length is not None and not 0 <= length <= maximum:
raise PolicyError("SCM request exceeds the safe size limit")
wanted = maximum + 1 if length is None else length
chunks: list[bytes] = []
total = 0
while total < wanted:
if deadline is not None:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise PolicyError("SCM request body deadline exceeded")
if set_timeout is not None:
set_timeout(remaining)
chunk = stream.read(min(STREAM_CHUNK, wanted - total))
if not chunk:
break
chunks.append(chunk)
total += len(chunk)
value = b"".join(chunks)
if len(value) > maximum or (length is not None and len(value) != length):
raise PolicyError("SCM request exceeds the safe size limit")
return value
def _spool_bounded(
stream,
maximum: int,
length: int,
*,
token: str,
context: str,
deadline: float,
set_timeout=None,
) -> tuple[BinaryIO, int]:
"""Copy a fixed-length exchange through a bounded-memory disk spool."""
if not 0 <= length <= maximum:
raise PolicyError("SCM request exceeds the safe size limit")
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool
max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
)
remaining = length
forms = _credential_forms(token)
carry = b""
try:
while remaining:
remaining_time = deadline - time.monotonic()
if remaining_time <= 0:
raise PolicyError("SCM request body deadline exceeded")
if set_timeout is not None:
set_timeout(remaining_time)
chunk = stream.read(min(STREAM_CHUNK, remaining))
if not chunk:
raise PolicyError("SCM request body ended early")
candidate = carry + chunk
if any(form in candidate for form in forms):
raise PolicyError(f"{context} contains runtime credential material")
width = max(len(form) for form in forms) - 1
carry = candidate[-width:] if width else b""
spool.write(chunk)
remaining -= len(chunk)
spool.seek(0)
return spool, length
except Exception:
spool.close()
raise
def _spool_response(
response, maximum: int, token: str, deadline_seconds: float
) -> tuple[BinaryIO, int]:
return spool_response(
response,
maximum,
_credential_forms(token),
memory_limit=SPOOL_MEMORY_LIMIT,
chunk_size=STREAM_CHUNK,
deadline_seconds=deadline_seconds,
)
def _load_json(handler: BaseHTTPRequestHandler) -> dict[str, object]:
if handler.headers.get("Transfer-Encoding"):
raise PolicyError("chunked broker control requests are not allowed")
if handler.headers.get_content_type() != "application/json":
raise PolicyError("broker control request must be JSON")
length = _content_length(handler.headers, MAX_CONTROL_BODY)
deadline = time.monotonic() + INBOUND_BODY_TIMEOUT
value = json.loads(
_read_bounded(
handler.rfile,
MAX_CONTROL_BODY,
length,
deadline=deadline,
set_timeout=handler.connection.settimeout,
)
)
if not isinstance(value, dict):
raise PolicyError("broker control request must be an object")
return value
def _git_target(raw: str) -> tuple[str, str, str]:
if (
not raw.isascii()
or any(ord(character) < 32 or ord(character) == 127 for character in raw)
or "%" in raw
or "\\" in raw
or len(raw) > 512
):
raise PolicyError("Git target is not canonical ASCII")
target = urllib.parse.urlsplit(raw)
canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, ""))
if canonical != raw:
raise PolicyError("Git target is not in exact canonical form")
if target.scheme or target.netloc or target.fragment:
raise PolicyError("Git target must be relative")
match = GIT_PATH_RE.fullmatch(target.path)
if not match:
raise PolicyError("Git target is outside the Atlas allowlist")
repo = _validate_repo(match.group("repo"))
operation = match.group("operation")
query = target.query
if operation == "info/refs":
if query not in {"service=git-upload-pack", "service=git-receive-pack"}:
raise PolicyError("Git discovery service is outside the allowlist")
service = query.removeprefix("service=")
elif query:
raise PolicyError("Git RPC query parameters are not allowed")
else:
service = operation
return repo, operation, service
def _content_length(headers: object, maximum: int) -> int:
"""Parse one short canonical bounded Content-Length header."""
raw = headers.get("Content-Length", "") # type: ignore[attr-defined]
if (
not isinstance(raw, str)
or not raw.isascii()
or len(raw) > 10
or not raw.isdigit()
):
raise PolicyError("SCM request length is invalid")
length = int(raw)
if raw != str(length) or not 0 <= length <= maximum:
raise PolicyError("SCM request length is invalid")
return length
def _credential_forms(token: str) -> tuple[bytes, ...]:
basic = base64.b64encode(f"{GIT_USER}:{token}".encode())
return token.encode("utf-8"), basic, b"Basic " + basic
def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None:
"""Permit only vetted new-feature-branch pushes with scanned contents."""
validate_receive_pack(body, token, _credential_forms(token))
def _guarded_opener(guard: deadline_http.StreamDeadline):
"""Build an opener whose connection obeys one absolute stream deadline."""
return urllib.request.build_opener(RejectRedirect(), *guard.handlers()).open
def _upstream_git_request(
target: str,
*,
method: str,
body: bytes | BinaryIO | None,
content_type: str | None,
expected_type: str,
token: str,
body_length: int | None = None,
stream_result: bool = False,
opener=None,
) -> bytes | tuple[BinaryIO, int]:
credentials = base64.b64encode(f"{GIT_USER}:{token}".encode()).decode("ascii")
headers = {"Accept": expected_type, "Authorization": f"Basic {credentials}"}
if content_type:
headers["Content-Type"] = content_type
if body is not None:
if body_length is None:
if not isinstance(body, bytes):
raise PolicyError("Git upstream request length is missing")
body_length = len(body)
headers["Content-Length"] = str(body_length)
request = urllib.request.Request(
CANONICAL_BASE_URL + target,
data=body,
method=method,
headers=headers,
)
# One wall-clock budget covers connect, send, headers, and body read; the
# watchdog closes the upstream socket if any phase overstays it.
guard = deadline_http.StreamDeadline(UPSTREAM_DEADLINE_SECONDS)
try:
open_upstream = opener if opener is not None else _guarded_opener(guard)
with open_upstream(request, timeout=guard.remaining()) as response:
if _status(response) != 200:
raise PolicyError("Git upstream returned an unexpected HTTP status")
if response.headers.get_content_type() != expected_type:
raise PolicyError("Git upstream returned an unexpected response type")
result, result_length = _spool_response(
response, MAX_GIT_RESPONSE, token, guard.remaining()
)
finally:
guard.cancel()
if stream_result:
return result, result_length
try:
return result.read()
finally:
result.close()
class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
"""Expose only bounded metadata, draft creation, and smart-HTTP Git."""
server_version = "HermesSCMBroker/1"
sys_version = ""
header_deadline_seconds = INBOUND_HEADER_TIMEOUT
def _validate_headers(self) -> None:
items = list(self.headers.items())
if len(items) > MAX_HEADERS:
raise PolicyError("SCM request has too many headers")
total = 0
for name, value in items:
if not name.isascii() or not value.isascii():
raise PolicyError("SCM request headers must be ASCII")
total += len(name) + len(value) + 4
if any(ord(character) < 32 and character != "\t" for character in value):
raise PolicyError("SCM request header contains controls")
if total > MAX_HEADER_BYTES:
raise PolicyError("SCM request headers exceed the safe limit")
if len(self.headers.get_all("Content-Length", [])) > 1:
raise PolicyError("SCM request has duplicate Content-Length")
def _stream(
self, status: int, content_type: str, body: BinaryIO, length: int
) -> None:
self.send_response(status)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(length))
self.end_headers()
deadline = time.monotonic() + INBOUND_BODY_TIMEOUT
while True:
chunk = body.read(STREAM_CHUNK)
if not chunk:
break
remaining = deadline - time.monotonic()
if remaining <= 0:
raise PolicyError("SCM response write deadline exceeded")
self.connection.settimeout(remaining)
self.wfile.write(chunk)
def log_message(self, _format: str, *_args: object) -> None:
return
def _json(self, status: int, body: bytes) -> None:
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(body)
def _reject(self, status: int = 400) -> None:
self._json(status, b'{"error":"request rejected"}')
def do_GET(self) -> None:
try:
self._validate_headers()
if self.path == "/healthz":
self._json(200, b'{"status":"ok"}')
return
repo, operation, service = _git_target(self.path)
if operation != "info/refs":
raise PolicyError("Git RPC requires POST")
token = read_token()
_reject_forbidden(repo, "repository", (token,))
expected = f"application/x-{service}-advertisement"
streamed = _upstream_git_request(
f"/atlas/{repo}.git/info/refs?service={service}",
method="GET",
body=None,
content_type=None,
expected_type=expected,
token=token,
stream_result=True,
)
body, length = streamed
try:
self._stream(200, expected, body, length)
finally:
body.close()
except (
OSError,
PolicyError,
socket.timeout,
urllib.error.URLError,
ValueError,
):
self._reject()
def do_POST(self) -> None:
try:
self._validate_headers()
self.connection.settimeout(INBOUND_BODY_TIMEOUT)
if self.path in {"/v1/metadata", "/v1/drafts"}:
self._control()
else:
self._git_rpc()
except (
OSError,
PolicyError,
socket.timeout,
urllib.error.URLError,
ValueError,
json.JSONDecodeError,
):
self._reject()
def _control(self) -> None:
data = _load_json(self)
token = read_token()
if self.path == "/v1/metadata":
if set(data) != {"path"} or not isinstance(data["path"], str):
raise PolicyError("metadata request fields are invalid")
result = read(data["path"], token=token)
else:
expected = {"base", "body", "head", "head_sha", "repo", "title"}
if set(data) != expected or not all(
isinstance(data[key], str) for key in expected
):
raise PolicyError("draft request fields are invalid")
result = create_draft(token=token, **data) # type: ignore[arg-type]
if token.encode("utf-8") in result:
raise PolicyError("SCM upstream reflected credential material")
self._json(200, result)
def _git_rpc(self) -> None:
repo, operation, service = _git_target(self.path)
if (
operation not in {"git-upload-pack", "git-receive-pack"}
or service != operation
):
raise PolicyError("Git RPC operation is outside the allowlist")
expected_request = f"application/x-{service}-request"
if self.headers.get_content_type() != expected_request or self.headers.get(
"Transfer-Encoding"
):
raise PolicyError("Git RPC request type is invalid")
length = _content_length(self.headers, MAX_GIT_REQUEST)
token = read_token()
_reject_forbidden(repo, "repository", (token,))
body, body_length = _spool_bounded(
self.rfile,
MAX_GIT_REQUEST,
length,
token=token,
context="Git request",
deadline=time.monotonic() + INBOUND_BODY_TIMEOUT,
set_timeout=self.connection.settimeout,
)
try:
if service == "git-receive-pack":
_validate_receive_pack(body, token)
expected = f"application/x-{service}-result"
streamed = _upstream_git_request(
f"/atlas/{repo}.git/{service}",
method="POST",
body=body,
body_length=body_length,
content_type=expected_request,
expected_type=expected,
token=token,
stream_result=True,
)
result, result_length = streamed
try:
self._stream(200, expected, result, result_length)
finally:
result.close()
finally:
body.close()
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--listen", default="0.0.0.0")
parser.add_argument("--port", type=int, default=BROKER_PORT)
args = parser.parse_args()
BoundedThreadingHTTPServer((args.listen, args.port), BrokerHandler).serve_forever()
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,93 @@
#!/usr/bin/env python3
"""Call the credential-isolated Hermes SCM broker over its fixed cluster origin."""
from __future__ import annotations
import json
import urllib.request
from collections.abc import Callable
import deadline_http
from gitea_api_policy import PolicyError
BROKER_ORIGIN = "http://hermes-scm-broker.hermes-scm.svc.cluster.local:9081"
MAX_RESPONSE_BYTES = 2 * 1024 * 1024
def _open(request: urllib.request.Request, timeout: int):
"""Open one broker exchange under a killable absolute deadline.
The helper process keeps every request on its fixed in-cluster origin by
rejecting redirects before they are followed.
"""
return deadline_http.open_bounded(
request, maximum=MAX_RESPONSE_BYTES, timeout=timeout
)
def request(
endpoint: str,
payload: dict[str, object],
*,
opener: Callable[..., object] = _open,
) -> bytes:
"""Send one bounded broker operation without any repository credential."""
if endpoint not in {"/v1/metadata", "/v1/drafts"}:
raise PolicyError("SCM broker operation is outside the client allowlist")
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
if len(body) > 64 * 1024:
raise PolicyError("SCM broker request exceeds the safe size limit")
outgoing = urllib.request.Request(
BROKER_ORIGIN + endpoint,
data=body,
method="POST",
headers={
"Accept": "application/json",
"Content-Type": "application/json",
"User-Agent": "hermes-scm-broker-client/1",
},
)
with opener(outgoing, timeout=30) as response: # type: ignore[attr-defined]
status = getattr(response, "status", None)
if status is None and hasattr(response, "getcode"):
status = response.getcode() # type: ignore[attr-defined]
if status != 200:
raise PolicyError("SCM broker returned an unexpected HTTP status")
content_type = response.headers.get_content_type() # type: ignore[attr-defined]
if content_type != "application/json":
raise PolicyError("SCM broker returned an unexpected response type")
result = response.read(MAX_RESPONSE_BYTES + 1) # type: ignore[attr-defined]
if len(result) > MAX_RESPONSE_BYTES:
raise PolicyError("SCM broker response exceeds the safe size limit")
json.loads(result)
return result
def read(path: str, *, opener: Callable[..., object] = _open) -> bytes:
"""Read explicitly allowed Atlas metadata through the broker."""
return request("/v1/metadata", {"path": path}, opener=opener)
def create_draft(
repo: str,
*,
base: str,
head: str,
head_sha: str,
title: str,
body: str,
opener: Callable[..., object] = _open,
) -> bytes:
"""Create one verified draft through the broker for human review."""
return request(
"/v1/drafts",
{
"base": base,
"body": body,
"head": head,
"head_sha": head_sha,
"repo": repo,
"title": title,
},
opener=opener,
)

View File

@ -0,0 +1,72 @@
#!/usr/bin/env python3
"""Bounded-memory stream helpers for the Hermes SCM broker."""
from __future__ import annotations
import tempfile
import time
import urllib.request
from typing import BinaryIO
from gitea_api_policy import PolicyError
class RejectRedirect(urllib.request.HTTPRedirectHandler):
"""Reject every upstream redirect before credentials can be forwarded."""
def redirect_request(self, req, fp, code, msg, headers, newurl):
raise PolicyError("SCM upstream redirects are not allowed")
def response_status(response: object) -> int | None:
"""Read an HTTP status from either urllib response interface."""
value = getattr(response, "status", None)
if value is None and hasattr(response, "getcode"):
value = response.getcode() # type: ignore[attr-defined]
return value
def spool_response(
response,
maximum: int,
forbidden: tuple[bytes, ...],
*,
memory_limit: int,
chunk_size: int,
deadline_seconds: float,
) -> tuple[BinaryIO, int]:
"""Spool an upstream response to disk while scanning chunk boundaries."""
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool
max_size=memory_limit, dir="/tmp"
)
carry = b""
total = 0
deadline = time.monotonic() + deadline_seconds
try:
while True:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise PolicyError("SCM upstream response deadline exceeded")
raw = getattr(getattr(response, "fp", None), "raw", None)
sock = getattr(raw, "_sock", raw)
if hasattr(sock, "settimeout"):
sock.settimeout(remaining)
chunk = response.read(chunk_size)
if not chunk:
break
total += len(chunk)
if total > maximum:
raise PolicyError("SCM upstream response exceeds the safe size limit")
candidate = carry + chunk
if any(value in candidate for value in forbidden):
raise PolicyError(
"Git upstream response contains runtime credential material"
)
width = max(len(value) for value in forbidden) - 1
carry = candidate[-width:] if width else b""
spool.write(chunk)
spool.seek(0)
return spool, total
except Exception:
spool.close()
raise

View File

@ -0,0 +1,118 @@
#!/usr/bin/env python3
"""Concurrency-bounded threaded HTTP server for the Hermes SCM broker."""
from __future__ import annotations
import threading
import time
from http.server import ThreadingHTTPServer
MAX_CONCURRENT_REQUESTS = 3
class _AbsoluteDeadlineReader:
"""Read header lines against one wall-clock deadline, not idle timeouts."""
def __init__(self, stream, connection):
self._stream = stream
self._connection = connection
self._deadline: float | None = None
def begin(self, timeout: float) -> None:
self._deadline = time.monotonic() + timeout
def end(self) -> None:
self._deadline = None
def readline(self, limit: int = -1) -> bytes:
if self._deadline is None:
return self._stream.readline(limit)
value = bytearray()
while limit < 0 or len(value) < limit:
remaining = self._deadline - time.monotonic()
if remaining <= 0:
raise TimeoutError("absolute SCM header deadline exceeded")
self._connection.settimeout(remaining)
character = self._stream.read(1)
if not character:
break
value.extend(character)
if character == b"\n":
break
return bytes(value)
def __getattr__(self, name: str):
return getattr(self._stream, name)
class AbsoluteHeaderDeadlineMixin:
"""Apply an absolute request-line plus header deadline to HTTP handlers."""
header_deadline_seconds = 10.0
def setup(self) -> None:
super().setup()
self._header_reader = _AbsoluteDeadlineReader(self.rfile, self.connection)
self.rfile = self._header_reader
def handle_one_request(self) -> None:
try:
self._header_reader.begin(self.header_deadline_seconds)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ""
self.request_version = ""
self.command = ""
self.send_error(414)
return
if not self.raw_requestline:
self.close_connection = True
return
if not self.parse_request():
return
finally:
self._header_reader.end()
method_name = "do_" + self.command
if not hasattr(self, method_name):
self.send_error(501, "Unsupported method")
return
getattr(self, method_name)()
self.wfile.flush()
except TimeoutError:
self.close_connection = True
class BoundedThreadingHTTPServer(ThreadingHTTPServer):
"""Cap active handlers so large Git exchanges cannot exhaust memory."""
daemon_threads = True
block_on_close = True
request_queue_size = 16
def __init__(self, *args, **kwargs):
self._slots = threading.BoundedSemaphore(MAX_CONCURRENT_REQUESTS)
super().__init__(*args, **kwargs)
def process_request(self, request, client_address) -> None:
if not self._slots.acquire(blocking=False):
try:
request.sendall(
b"HTTP/1.1 503 Service Unavailable\r\n"
b"Connection: close\r\nContent-Length: 0\r\n\r\n"
)
finally:
self.shutdown_request(request)
return
try:
super().process_request(request, client_address)
except Exception:
self._slots.release()
self.shutdown_request(request)
raise
def process_request_thread(self, request, client_address) -> None:
try:
super().process_request_thread(request, client_address)
finally:
self._slots.release()

View File

@ -0,0 +1,82 @@
#!/usr/bin/env python3
"""Bounded Kanban access and workspace resolution for Hermes CLI workers."""
from __future__ import annotations
import json
import sqlite3
import sys
import time
from pathlib import Path
from typing import Any, Callable
from cli_lane_config import (
BOARD_CORRUPTION_ERRORS,
EXTERNAL_PREFIX,
KANBAN_STORAGE_ATTEMPTS,
WORKTREE_LOCK,
)
def _task_value(task: Any, name: str, default: Any = None) -> Any:
return getattr(task, name, default)
def _task_context(kanban_db: Any, conn: Any, task_id: str) -> str:
value = kanban_db.build_worker_context(conn, task_id)
if isinstance(value, str):
return value
return json.dumps(value, indent=2, default=str)
def _resolve_workspace(kanban_db: Any, conn: Any, task: Any, board: str) -> Path:
# External coding lanes always receive their own linked worktree. A task
# that does not resolve to a Git repository is blocked rather than sharing
# a mutable checkout with another unattended worker.
# Git worktree creation updates common-repository metadata. Serialize that
# short materialization step while allowing the provider workers themselves
# to run concurrently in independent worktrees.
with WORKTREE_LOCK:
value, branch_name = kanban_db._resolve_worktree_workspace(task, board=board)
kanban_db.set_branch_name(conn, str(_task_value(task, "id")), branch_name)
return Path(value).resolve()
def _record_board_access_error(board: str, error: Exception) -> None:
"""Report one board access failure once while allowing other lanes to run."""
failure_kind = "storage" if isinstance(error, (OSError, sqlite3.Error)) else "access"
detail = f"{failure_kind} {type(error).__name__}: {error}"
if BOARD_CORRUPTION_ERRORS.get(board) == detail:
return
print(
f"temporarily skipping Kanban board {board!r}: {detail}",
file=sys.stderr,
flush=True,
)
BOARD_CORRUPTION_ERRORS[board] = detail
def _board_call(
kanban_db: Any,
board: str,
operation: Callable[[Any], Any],
) -> Any:
"""Run one bounded Kanban operation on a fresh, promptly closed connection."""
last_storage_error: Exception | None = None
for attempt in range(KANBAN_STORAGE_ATTEMPTS):
conn = None
try:
with kanban_db.scoped_current_board(board):
conn = kanban_db.connect(board=board)
result = operation(conn)
BOARD_CORRUPTION_ERRORS.pop(board, None)
return result
except (OSError, sqlite3.Error) as error:
last_storage_error = error
_record_board_access_error(board, error)
if attempt + 1 < KANBAN_STORAGE_ATTEMPTS:
time.sleep(0.2 * (attempt + 1))
finally:
if conn is not None:
conn.close()
assert last_storage_error is not None
raise last_storage_error
def _external(task: Any) -> bool:
return str(_task_value(task, "assignee", "") or "").startswith(EXTERNAL_PREFIX)

View File

@ -0,0 +1,238 @@
#!/usr/bin/env python3
"""Detect and publish the Kanban API boundary required by CLI lanes.
During a rollout the coordinator container and the cli-lane-runner worker can
run different hermes-agent image generations against one shared Kanban DB.
The image APIs that differ on the decomposition/finalization path:
* ``complete_task``: both generations accept ``expected_run_id`` (active
exact-run completion); only the patched image accepts
``replay_ended_run_id`` (ended-run replay).
* ``reclaim_task``: only the patched image accepts ``expected_run_id``
(guarded orphan reclaim after a runner restart).
* ``specify_triage_task``/``decompose_triage_task``: only the patched image
accepts ``require_no_runs`` (refuses redefining tasks with run history).
Worker-side handling per combination (proven in
testing/tests/test_hermes_cli_capabilities.py):
* legacy worker: health/readiness stay deferred, no new claims, no unguarded
reclaim, replay-needing results stay journaled for a patched successor.
* patched worker: ready health, exact completion, guarded replay/reclaim; a
run voided by a legacy coordinator's unguarded mid-run decomposition
converges to stale/conflict evidence instead of completing.
* patched coordinator: mid-run decomposition is refused entirely, so the
active exact run completes on either worker generation.
"""
from __future__ import annotations
import inspect
import json
import os
import sys
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable
from cli_lane_config import STATE_ROOT, utc_now
from cli_lane_files import atomic_json, load_json
_CAPABILITIES: KanbanCapabilities | None = None
_CAPABILITY_SOURCE: tuple[int, int] | None = None
@dataclass(frozen=True)
class KanbanCapabilities:
"""Safety-relevant keyword support exposed by the image's Kanban DB."""
exact_run_completion: bool
ended_run_replay: bool
exact_run_reclaim: bool
@property
def ready(self) -> bool:
"""Return whether dispatch and every recovery transition are safe."""
return (
self.exact_run_completion
and self.ended_run_replay
and self.exact_run_reclaim
)
@property
def deferred_features(self) -> tuple[str, ...]:
"""List fixed, bounded capability names absent from the image API."""
missing = []
if not self.exact_run_completion:
missing.append("exact-run-completion")
if not self.ended_run_replay:
missing.append("ended-run-replay")
if not self.exact_run_reclaim:
missing.append("exact-run-reclaim")
return tuple(missing)
def _explicit_keyword(function: Callable[..., Any], keyword: str) -> bool:
"""Require an explicit keyword parameter rather than trusting ``**kwargs``."""
try:
parameter = inspect.signature(function).parameters.get(keyword)
except (TypeError, ValueError):
return False
return parameter is not None and parameter.kind in {
inspect.Parameter.POSITIONAL_OR_KEYWORD,
inspect.Parameter.KEYWORD_ONLY,
}
def detect_kanban_capabilities(kanban_db: Any) -> KanbanCapabilities:
"""Inspect the loaded image API without invoking a mutating DB operation."""
complete_task = getattr(kanban_db, "complete_task", None)
reclaim_task = getattr(kanban_db, "reclaim_task", None)
return KanbanCapabilities(
exact_run_completion=callable(complete_task)
and _explicit_keyword(complete_task, "expected_run_id"),
ended_run_replay=callable(complete_task)
and _explicit_keyword(complete_task, "replay_ended_run_id"),
exact_run_reclaim=callable(reclaim_task)
and _explicit_keyword(reclaim_task, "expected_run_id"),
)
def _source_identity(kanban_db: Any) -> tuple[int, int]:
"""Bind cached capability evidence to the two inspected callables."""
return (
id(getattr(kanban_db, "complete_task", None)),
id(getattr(kanban_db, "reclaim_task", None)),
)
def _health_document(capabilities: KanbanCapabilities) -> dict[str, Any]:
"""Return a bounded operator-readable compatibility health document."""
return {
"component": "cli-lane-runner",
"schema_version": 1,
"state": "ready" if capabilities.ready else "deferred",
"ready": capabilities.ready,
"active_run_completion_safe": capabilities.exact_run_completion,
"deferred_features": list(capabilities.deferred_features),
"capabilities": asdict(capabilities),
"observed_at": utc_now(),
}
def initialize_kanban_capabilities(
kanban_db: Any,
*,
health_path: Path | None = None,
) -> KanbanCapabilities:
"""Detect startup compatibility and durably expose ready/deferred health."""
global _CAPABILITIES, _CAPABILITY_SOURCE
capabilities = detect_kanban_capabilities(kanban_db)
_CAPABILITIES = capabilities
_CAPABILITY_SOURCE = _source_identity(kanban_db)
destination = health_path or STATE_ROOT / "runtime-health.json"
destination.parent.mkdir(parents=True, exist_ok=True)
atomic_json(destination, _health_document(capabilities), 0o600)
if not capabilities.ready:
print(
"CLI lane safety compatibility deferred; missing image APIs: "
+ ", ".join(capabilities.deferred_features),
file=sys.stderr,
flush=True,
)
return capabilities
def refresh_kanban_capabilities(
kanban_db: Any,
*,
health_path: Path | None = None,
) -> KanbanCapabilities:
"""Reinspect the live callables and refresh bounded readiness evidence.
ConfigMap script updates and image changes are deliberately not imported or
reloaded here. A running process observes only the already-loaded module,
which prevents a half-old/half-new Python module graph. Tests may replace
the two callables to model an image API transition at a loop boundary.
"""
return initialize_kanban_capabilities(kanban_db, health_path=health_path)
def kanban_capabilities(kanban_db: Any) -> KanbanCapabilities:
"""Return startup evidence, redetecting only when the loaded API changes."""
if _CAPABILITIES is None or _source_identity(kanban_db) != _CAPABILITY_SOURCE:
return initialize_kanban_capabilities(kanban_db)
return _CAPABILITIES
def runtime_health(path: Path | None = None) -> dict[str, Any]:
"""Read the bounded health document for probes and operator diagnostics."""
document = load_json(path or STATE_ROOT / "runtime-health.json")
return document if isinstance(document, dict) else {}
def readiness_issue(
path: Path | None = None,
*,
now: datetime | None = None,
maximum_age_seconds: float = 60.0,
) -> str | None:
"""Return a bounded readiness reason without exposing health contents."""
destination = path or STATE_ROOT / "runtime-health.json"
try:
document = json.loads(destination.read_text(encoding="utf-8"))
except FileNotFoundError:
return "runtime health is missing"
except (OSError, UnicodeError, json.JSONDecodeError):
return "runtime health is malformed"
if not isinstance(document, dict) or document.get("schema_version") != 1:
return "runtime health is malformed"
if document.get("state") == "deferred" and document.get("ready") is False:
return "runtime compatibility is deferred"
capabilities = document.get("capabilities")
if (
document.get("state") != "ready"
or document.get("ready") is not True
or not isinstance(capabilities, dict)
or any(
capabilities.get(name) is not True
for name in (
"exact_run_completion",
"ended_run_replay",
"exact_run_reclaim",
)
)
):
return "runtime health is malformed"
observed_at = document.get("observed_at")
if not isinstance(observed_at, str):
return "runtime health is malformed"
try:
observed = datetime.fromisoformat(observed_at.replace("Z", "+00:00"))
except ValueError:
return "runtime health is malformed"
if observed.tzinfo is None:
return "runtime health is malformed"
current = now or datetime.now(timezone.utc)
age = (
current.astimezone(timezone.utc) - observed.astimezone(timezone.utc)
).total_seconds()
if age < -5.0 or age > maximum_age_seconds:
return "runtime health is stale"
return None
def main() -> int:
"""Provide the non-mutating Kubernetes readiness command."""
maximum_age = float(os.environ.get("HERMES_CLI_HEALTH_MAX_AGE_SECONDS", "60"))
issue = readiness_issue(maximum_age_seconds=maximum_age)
if issue:
print(issue, file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,220 @@
#!/usr/bin/env python3
"""Shared immutable configuration and result types for Hermes CLI lanes."""
from __future__ import annotations
import os
import re
import threading
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import yaml
import cli_lane_goal
DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data"))
SWITCHYARD_URL = os.environ.get(
"HERMES_SWITCHYARD_URL",
"http://hermes-switchyard.hermes.svc.cluster.local:9005/v1/chat/completions",
)
STATE_ROOT = DATA_ROOT / "cli-lanes"
CODEX_BIN = DATA_ROOT / "tools/bin/codex"
CLAUDE_BIN = DATA_ROOT / "tools/bin/claude"
CLAUDE_SETTINGS = DATA_ROOT / "home/.claude/settings.json"
RESULT_SCHEMA_PATH = STATE_ROOT / "worker-result.schema.json"
EFFORTS = ("low", "medium", "high", "xhigh")
EXTERNAL_PREFIX = "cli-"
DEFAULT_CLAIM_TTL = 7 * 24 * 60 * 60
DEFAULT_MAX_RUNTIME = 12 * 60 * 60
HEARTBEAT_SECONDS = 20
PROVIDER_HEALTH_MAX_AGE_SECONDS = 5 * 60
PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS = 12 * 60 * 60
QUOTA_MIN_REMAINING_PERCENT_DEFAULT = 15.0
CAPACITY_COOLDOWN_SECONDS_DEFAULT = 300.0
AUTH_COOLDOWN_SECONDS_DEFAULT = 3600.0
QUOTA_METRICS_URL = os.environ.get(
"HERMES_CLI_QUOTA_METRICS_URL", "http://127.0.0.1:9010/metrics"
)
KANBAN_STORAGE_ATTEMPTS = 5
ARTIFACT_GC_INTERVAL_SECONDS = 5 * 60
ARTIFACT_RETENTION_AGE_SECONDS = 30 * 24 * 60 * 60
ARTIFACT_RETENTION_COUNT = 256
ARTIFACT_RETENTION_BYTES = 128 * 1024 * 1024
MAX_TERMINAL_RECORD_BYTES = 1024 * 1024
QUARANTINE_HASH_BYTES = 64 * 1024
SQLITE_RUN_ID_MAX = 2**63 - 1
PROVIDER_HEALTH_PATHS = {
"codex": DATA_ROOT / "provider-health/codex.json",
"claude": DATA_ROOT / "provider-health/claude.json",
}
WORKTREE_LOCK = threading.Lock()
BOARD_CORRUPTION_ERRORS: dict[str, str] = {}
LAST_ARTIFACT_GC = 0.0
CAPACITY_PATTERN = re.compile(
r"(?:rate.?limit|capacity|overload|usage.?limit|quota|credit|exhaust|429|529"
r"|authentication|unauthorized|forbidden|oauth|token.*expired|401|403)",
re.I,
)
NO_CLAUDE_SESSION = "No conversation found with session ID:"
CLAUDE_SESSION_COLLISION = "Session ID already in use"
NO_CODEX_THREAD = "no rollout found for thread id"
RESULT_SCHEMA: dict[str, Any] = {
"type": "object",
"additionalProperties": False,
"required": [
"status",
"summary",
"changed_files",
"tests_run",
"artifacts",
"findings",
"blockers",
],
"properties": {
"status": {
"type": "string",
"enum": sorted(cli_lane_goal.RESULT_STATUSES),
},
"summary": {"type": "string"},
"changed_files": {"type": "array", "items": {"type": "string"}},
"tests_run": {"type": "array", "items": {"type": "string"}},
"artifacts": {"type": "array", "items": {"type": "string"}},
"findings": {
"type": "array",
"items": {"type": "string"},
"description": (
"Defects, risks, or observations discovered by a review or diagnosis. "
"Findings do not prevent the assigned review or diagnosis from completing."
),
},
"blockers": {
"type": "array",
"items": {"type": "string"},
"description": (
"Concrete obstacles that prevent completion of the assigned task itself. "
"This must be empty when status is completed; review findings belong in findings."
),
},
},
}
@dataclass(frozen=True)
class Route:
"""One provider/model/effort decision for an observable worker boundary."""
provider: str
model: str
effort: str
profile: str
classifier: str
reason: str
latency_ms: int
fallback_chain: tuple[str, ...]
@dataclass
class ProcessResult:
"""Captured outcome from one provider CLI invocation."""
returncode: int
output: str
structured: dict[str, Any] | None
capacity_failure: bool
class TerminalFinalizationPending(RuntimeError):
"""An accepted worker result is durable but not committed to Kanban yet."""
def canonical_run_id(value: object) -> int | None:
"""Return one positive SQLite-safe run ID from an int or canonical decimal."""
if type(value) is int:
candidate = value
elif type(value) is str:
# Bound conversion before ``int`` so an attacker cannot hand Python an
# arbitrarily large decimal from an otherwise lexically valid name.
if not re.fullmatch(r"[1-9][0-9]{0,18}", value):
return None
candidate = int(value)
else:
return None
if not 1 <= candidate <= SQLITE_RUN_ID_MAX:
return None
return candidate
@dataclass(frozen=True)
class TerminalIdentity:
"""Replay authority derived only from a journal's directory and filename."""
board: str
task_id: str
run_id: int
state: str
def __post_init__(self) -> None:
"""Reject noncanonical identities before they can reach SQLite."""
if type(self.run_id) is not int or canonical_run_id(self.run_id) is None:
raise ValueError("terminal run identity is not a positive SQLite int64")
@dataclass
class TerminalSnapshot:
"""One inode-bound, bounded journal read held open across finalization."""
document: dict[str, Any]
file_stat: os.stat_result
descriptor: int
directory_descriptor: int
def close(self) -> None:
"""Release the pinned file and directory descriptors."""
os.close(self.descriptor)
os.close(self.directory_descriptor)
@dataclass
class TerminalRecoverySnapshot:
"""One raw directory entry pinned before recovery classifies its payload."""
document: dict[str, Any] | None
file_stat: os.stat_result
descriptor: int | None
directory_descriptor: int
prefix: bytes
invalid_reason: str | None
def close(self) -> None:
"""Release the descriptors retained across classification/quarantine."""
if self.descriptor is not None:
os.close(self.descriptor)
self.descriptor = None
if self.directory_descriptor >= 0:
os.close(self.directory_descriptor)
self.directory_descriptor = -1
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def kanban_setting(name: str, default: float) -> float:
"""Read one numeric routing setting from the deployed kanban config block."""
try:
document = yaml.safe_load(
(DATA_ROOT / "config.yaml").read_text(encoding="utf-8")
)
except (OSError, yaml.YAMLError):
return default
kanban = document.get("kanban") if isinstance(document, dict) else None
value = kanban.get(name) if isinstance(kanban, dict) else None
if isinstance(value, bool) or not isinstance(value, (int, float)):
return default
return float(value)

View File

@ -0,0 +1,203 @@
#!/usr/bin/env python3
"""Concurrent board recovery, claim, and dispatch loop for CLI workers."""
from __future__ import annotations
import concurrent.futures
import os
import sys
import time
from collections.abc import Callable
from typing import Any
from cli_lane_board import _external, _record_board_access_error, _task_value
from cli_lane_capabilities import (
initialize_kanban_capabilities,
kanban_capabilities,
refresh_kanban_capabilities,
)
from cli_lane_config import (
BOARD_CORRUPTION_ERRORS,
DEFAULT_CLAIM_TTL,
EXTERNAL_PREFIX,
RESULT_SCHEMA,
RESULT_SCHEMA_PATH,
)
from cli_lane_execution import execute_claim
from cli_lane_files import atomic_json
from cli_lane_metrics import start_metrics_server
from cli_lane_recovery import _has_pending_finalization, recover_pending_finalizations
from cli_lane_retention import maybe_gc_lane_artifacts
def _board_slug(board: Any) -> str:
if isinstance(board, dict):
return str(board.get("slug") or board.get("id") or "")
return str(getattr(board, "slug", None) or getattr(board, "id", None) or board)
def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None:
"""Open one board without letting localized storage faults stop other lanes."""
try:
return kanban_db.connect(board=board)
except Exception as error:
_record_board_access_error(board, error)
return None
def recover_orphans() -> None:
"""Return external running tasks to ready after a runner/pod restart."""
from hermes_cli import kanban_db
recover_pending_finalizations()
if not kanban_capabilities(kanban_db).exact_run_reclaim:
return
try:
boards = kanban_db.list_boards(include_archived=False)
except Exception as error:
_record_board_access_error("board-registry", error)
return
for raw_board in boards:
board = _board_slug(raw_board)
if not board:
continue
with kanban_db.scoped_current_board(board):
conn = _connect_healthy_board(kanban_db, board)
if conn is None:
continue
try:
for task in kanban_db.list_tasks(conn):
if (
_external(task)
and str(_task_value(task, "status", "")) == "running"
):
task_id = str(_task_value(task, "id"))
run_id = _task_value(task, "current_run_id", None)
if not isinstance(run_id, int):
continue
if _has_pending_finalization(board, task_id, run_id):
continue
kanban_db.reclaim_task(
conn,
task_id,
reason="direct CLI lane restarted; provider session will resume",
expected_run_id=run_id,
)
BOARD_CORRUPTION_ERRORS.pop(board, None)
except Exception as error:
_record_board_access_error(board, error)
finally:
conn.close()
def claim_ready(
active: set[tuple[str, str]],
limit: int,
eligible: Callable[[str, Any], bool] | None = None,
) -> list[tuple[str, str]]:
"""Atomically claim external ready tasks across all non-archived boards."""
from hermes_cli import kanban_db
claimed: list[tuple[str, str]] = []
if limit <= 0:
return claimed
for raw_board in kanban_db.list_boards(include_archived=False):
board = _board_slug(raw_board)
if not board:
continue
with kanban_db.scoped_current_board(board):
conn = _connect_healthy_board(kanban_db, board)
if conn is None:
continue
try:
kanban_db.recompute_ready(conn)
tasks = kanban_db.list_tasks(conn)
BOARD_CORRUPTION_ERRORS.pop(board, None)
for task in tasks:
task_id = str(_task_value(task, "id", ""))
assignee = str(_task_value(task, "assignee", "") or "")
if (
task_id
and not assignee
and str(_task_value(task, "status", "")) == "ready"
and kanban_db.assign_task(conn, task_id, "cli-auto")
):
task = kanban_db.get_task(conn, task_id)
assignee = "cli-auto"
if (
not task_id
or (board, task_id) in active
or not assignee.startswith(EXTERNAL_PREFIX)
or str(_task_value(task, "status", "")) != "ready"
or (eligible is not None and not eligible(board, task))
):
continue
try:
result = kanban_db.claim_task(
conn,
task_id,
ttl_seconds=DEFAULT_CLAIM_TTL,
claimer="direct-cli-lane",
)
except Exception:
continue
if result is not None:
claimed.append((board, task_id))
if len(claimed) >= limit:
return claimed
except Exception as error:
_record_board_access_error(board, error)
finally:
conn.close()
return claimed
def main() -> int:
"""Continuously bridge external Kanban lanes to provider CLIs.
The loop is exited only by process signals or unrecoverable exceptions;
every per-board and per-claim fault is bounded inside one pass.
"""
from hermes_cli import kanban_db
RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
atomic_json(RESULT_SCHEMA_PATH, RESULT_SCHEMA, 0o644)
start_metrics_server()
capabilities = initialize_kanban_capabilities(kanban_db)
recover_orphans()
workers = max(1, min(int(os.environ.get("HERMES_CLI_LANE_CONCURRENCY", "4")), 8))
futures: dict[concurrent.futures.Future[None], tuple[str, str]] = {}
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
while True:
previous_capabilities = capabilities
capabilities = refresh_kanban_capabilities(kanban_db)
if capabilities.ready and not previous_capabilities.ready:
recover_orphans()
for future in list(futures):
if future.done():
try:
future.result()
except Exception as error:
print(
f"worker future failed: {error}",
file=sys.stderr,
flush=True,
)
del futures[future]
recover_pending_finalizations()
maybe_gc_lane_artifacts()
active = set(futures.values())
try:
newly_claimed = (
claim_ready(active, workers - len(futures))
if capabilities.ready
else []
)
BOARD_CORRUPTION_ERRORS.pop("board-registry", None)
except Exception as error:
_record_board_access_error("board-registry", error)
newly_claimed = []
for board, task_id in newly_claimed:
future = pool.submit(execute_claim, board, task_id)
futures[future] = (board, task_id)
time.sleep(5)

View File

@ -0,0 +1,294 @@
#!/usr/bin/env python3
"""Immutable terminal evidence creation and identity-safe retirement."""
from __future__ import annotations
import ctypes
import contextlib
import errno
import hashlib
import json
import os
import stat
import uuid
from pathlib import Path
from typing import Any
from cli_lane_config import MAX_TERMINAL_RECORD_BYTES, TerminalIdentity, utc_now
from cli_lane_files import _terminal_evidence_identity, state_path
from cli_lane_records import _load_small_json, _terminal_record_valid
def _rename_noreplace(
source: str,
destination: str,
*,
source_dir: int,
destination_dir: int,
) -> None:
"""Atomically rename a directory entry without replacing a collision."""
renameat2 = getattr(ctypes.CDLL(None, use_errno=True), "renameat2", None)
if renameat2 is None:
raise OSError(errno.ENOSYS, "renameat2 is unavailable")
renameat2.argtypes = (
ctypes.c_int,
ctypes.c_char_p,
ctypes.c_int,
ctypes.c_char_p,
ctypes.c_uint,
)
renameat2.restype = ctypes.c_int
result = renameat2(
source_dir,
ctypes.c_char_p(os.fsencode(source)),
destination_dir,
ctypes.c_char_p(os.fsencode(destination)),
1, # RENAME_NOREPLACE
)
if result != 0:
error = ctypes.get_errno()
raise OSError(error, os.strerror(error), source, destination)
def _retire_terminal_entry(
path: Path,
source_stat: os.stat_result,
*,
board_descriptor: int,
quarantine_descriptor: int,
authority_name: str | None = None,
source_descriptor: int | None = None,
) -> str:
"""Retire only the inode previously inspected; preserve any replacement."""
pinned = os.fstat(source_descriptor) if source_descriptor is not None else source_stat
if pinned.st_dev != source_stat.st_dev or pinned.st_ino != source_stat.st_ino:
return "replacement"
try:
current = os.stat(
path.name,
dir_fd=board_descriptor,
follow_symlinks=False,
)
except FileNotFoundError:
return "missing"
if current.st_dev != pinned.st_dev or current.st_ino != pinned.st_ino:
return "replacement"
# A replacement can itself be staged repeatedly while recovery races a
# writer. Keep every generation bound to the original canonical pending
# name instead of hashing an intermediate .retire name and hiding it from
# the next recovery pass.
path_digest = hashlib.sha256(
(authority_name or path.name).encode("utf-8")
).hexdigest()[:16]
source_digest = hashlib.sha256(
f"{pinned.st_dev:x}\0{pinned.st_ino:x}".encode()
).hexdigest()[:16]
for sequence in range(32):
staging = f".retire.{path_digest}.{source_digest}.{sequence}"
try:
_rename_noreplace(
path.name,
staging,
source_dir=board_descriptor,
destination_dir=quarantine_descriptor,
)
except FileExistsError:
continue
break
else:
return "collision"
staged = os.stat(
staging,
dir_fd=quarantine_descriptor,
follow_symlinks=False,
)
if staged.st_dev == pinned.st_dev and staged.st_ino == pinned.st_ino:
os.unlink(staging, dir_fd=quarantine_descriptor)
os.fsync(quarantine_descriptor)
os.fsync(board_descriptor)
return "retired"
# The path changed after the pre-check but before the rename. Put the
# replacement back without overwriting an even newer entry.
try:
_rename_noreplace(
staging,
path.name,
source_dir=quarantine_descriptor,
destination_dir=board_descriptor,
)
except FileExistsError:
return "replacement-staged"
os.fsync(quarantine_descriptor)
os.fsync(board_descriptor)
return "replacement"
def _write_json_noreplace(path: Path, value: dict[str, Any]) -> bool:
"""Durably create one immutable JSON artifact without replacing a peer."""
payload = (json.dumps(value, indent=2, sort_keys=True) + "\n").encode("utf-8")
if len(payload) > MAX_TERMINAL_RECORD_BYTES + 4096:
raise ValueError("terminal evidence exceeds the bounded artifact limit")
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
os.chmod(path.parent, 0o700, follow_symlinks=False)
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
directory_flags |= getattr(os, "O_NOFOLLOW", 0)
create_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
create_flags |= getattr(os, "O_NOFOLLOW", 0)
directory = os.open(path.parent, directory_flags)
temporary = f".{path.name}.{uuid.uuid4().hex}.tmp"
temporary_stat = None
descriptor = None
cleanup_descriptor = None
try:
descriptor = os.open(temporary, create_flags, 0o600, dir_fd=directory)
temporary_stat = os.fstat(descriptor)
with os.fdopen(descriptor, "wb") as stream:
descriptor = None
stream.write(payload)
stream.flush()
os.fsync(stream.fileno())
written = os.fstat(stream.fileno())
if stat.S_IMODE(written.st_mode) != 0o600 or written.st_nlink != 1:
raise OSError("terminal evidence is not private and singly linked")
cleanup_descriptor = os.dup(stream.fileno())
try:
_rename_noreplace(
temporary,
path.name,
source_dir=directory,
destination_dir=directory,
)
except FileExistsError:
return False
os.fsync(directory)
return True
finally:
if descriptor is not None:
os.close(descriptor)
if temporary_stat is not None:
with contextlib.suppress(OSError):
_retire_terminal_entry(
Path(temporary),
temporary_stat,
board_descriptor=directory,
quarantine_descriptor=directory,
source_descriptor=cleanup_descriptor,
)
if cleanup_descriptor is not None:
os.close(cleanup_descriptor)
os.close(directory)
def _terminal_document_digest(document: dict[str, Any]) -> str:
"""Return the stable identity for one exact accepted result document."""
immutable = {
key: document.get(key)
for key in (
"board",
"task_id",
"expected_run_id",
"result",
"summary",
"metadata",
)
}
canonical = json.dumps(immutable, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:32]
def _terminal_evidence_path(
identity: TerminalIdentity,
state: str,
document: dict[str, Any],
) -> Path:
"""Return a deterministic immutable prepared or conflict evidence path."""
if state not in {"prepared", "conflict"}:
raise ValueError(f"invalid terminal evidence state: {state}")
base = state_path(identity.board, identity.task_id)
digest = _terminal_document_digest(document)
return base.with_name(
f"{base.stem}.run-{identity.run_id}.terminal.{state}-{digest}.json"
)
def _same_terminal_document(left: dict[str, Any], right: dict[str, Any]) -> bool:
"""Compare the immutable result-bearing fields of terminal evidence."""
keys = ("board", "task_id", "expected_run_id", "result", "summary", "metadata")
return all(left.get(key) == right.get(key) for key in keys)
def _terminal_evidence_valid(
document: dict[str, Any],
identity: TerminalIdentity,
state: str,
) -> bool:
"""Validate semantic terminal content plus its immutable evidence state."""
if document.get("kanban_state") != state:
return False
semantic = dict(document)
semantic_identity = identity
if state == "conflict":
semantic["kanban_state"] = "prepared"
semantic_identity = TerminalIdentity(
identity.board,
identity.task_id,
identity.run_id,
"prepared",
)
return _terminal_record_valid(semantic, semantic_identity)
def _persist_prepared_evidence(
identity: TerminalIdentity,
document: dict[str, Any],
) -> Path:
"""Fsync an immutable accepted result before entering the DB boundary."""
prepared = dict(document)
prepared["kanban_state"] = "prepared"
prepared["prepared_at"] = utc_now()
path = _terminal_evidence_path(identity, "prepared", document)
if _write_json_noreplace(path, prepared):
return path
if _terminal_evidence_identity(path) != TerminalIdentity(
identity.board, identity.task_id, identity.run_id, "prepared"
):
raise OSError("prepared terminal evidence identity is invalid")
existing = _load_small_json(path)
prepared_identity = TerminalIdentity(
identity.board,
identity.task_id,
identity.run_id,
"prepared",
)
if not _terminal_evidence_valid(
existing,
prepared_identity,
"prepared",
) or not _same_terminal_document(existing, prepared):
raise OSError("prepared terminal evidence path contains a conflicting result")
return path
def _persist_conflict_evidence(
identity: TerminalIdentity,
document: dict[str, Any],
reason: str,
) -> Path:
"""Preserve a losing valid result in full under deterministic retention."""
conflict = dict(document)
conflict["kanban_state"] = "conflict"
conflict["conflict_reason"] = reason
conflict["conflicted_at"] = utc_now()
path = _terminal_evidence_path(identity, "conflict", document)
if _write_json_noreplace(path, conflict):
return path
if _terminal_evidence_identity(path) != TerminalIdentity(
identity.board, identity.task_id, identity.run_id, "conflict"
):
raise OSError("terminal conflict evidence identity is invalid")
existing = _load_small_json(path)
conflict_identity = TerminalIdentity(
identity.board,
identity.task_id,
identity.run_id,
"conflict",
)
if not _terminal_evidence_valid(
existing,
conflict_identity,
"conflict",
) or not _same_terminal_document(existing, conflict):
raise OSError("terminal conflict evidence path contains a different result")
return path

View File

@ -0,0 +1,462 @@
#!/usr/bin/env python3
"""Single-claim provider execution and durable Kanban result handoff."""
from __future__ import annotations
import contextlib
import json
import sqlite3
import time
from pathlib import Path
import cli_lane_goal
from cli_lane_board import _board_call, _resolve_workspace, _task_context, _task_value
from cli_lane_config import (
DEFAULT_MAX_RUNTIME,
TerminalFinalizationPending,
)
from cli_lane_failover import _routed_or_blocked, capacity_failover
from cli_lane_files import (
_terminal_identity,
_terminal_path,
atomic_json,
load_json,
state_path,
)
from cli_lane_finalization import _finalize_terminal_record, _recover_exact_run
from cli_lane_health import record_provider_success
from cli_lane_metrics import record_route_decision
from cli_lane_prompt import build_prompt, git_handoff, workspace_artifacts
from cli_lane_provider import run_provider
from cli_lane_quota import selection_constraint
from cli_lane_records import _persist_candidate, _write_terminal_record
from cli_lane_recovery import _has_pending_finalization
from cli_lane_routing import fresh_unavailable_provider, select_route
def execute_claim(board: str, task_id: str) -> None:
"""Execute one already-claimed task and commit its outcome to Kanban."""
from hermes_cli import kanban_db
run_id = None
with kanban_db.scoped_current_board(board):
conn = kanban_db.connect(board=board)
try:
task = kanban_db.get_task(conn, task_id)
if task is None:
return
run_id = _task_value(task, "current_run_id", None)
assignee = str(_task_value(task, "assignee", "cli-auto") or "cli-auto")
state_file = state_path(board, task_id)
state = load_json(state_file)
state.update(
{
"board": board,
"task_id": task_id,
"run_id": run_id,
"assignee": assignee,
}
)
atomic_json(state_file, state)
log_path = Path(kanban_db.worker_log_path(task_id, board=board))
workspace = _resolve_workspace(kanban_db, conn, task, board)
kanban_db.set_workspace_path(conn, task_id, str(workspace))
context = _task_context(kanban_db, conn, task_id)
except Exception as error:
failure_reason = (
f"Direct CLI lane preparation failed: {type(error).__name__}: {error}"
)
conn.close()
_board_call(
kanban_db,
board,
lambda fresh: kanban_db.block_task(
fresh,
task_id,
reason=failure_reason,
kind="capability",
expected_run_id=run_id,
),
)
return
finally:
conn.close()
def heartbeat(note: str) -> bool:
try:
return bool(
_board_call(
kanban_db,
board,
lambda fresh: kanban_db.heartbeat_worker(
fresh,
task_id,
note=note,
expected_run_id=run_id,
),
)
)
except (OSError, sqlite3.Error):
# The claim TTL is deliberately long. Keep the expensive
# provider process alive during a transient volume stall and
# retry on the next heartbeat instead of losing its work.
return True
def comment(body: str) -> None:
with contextlib.suppress(OSError, sqlite3.Error):
_board_call(
kanban_db,
board,
lambda fresh: kanban_db.add_comment(
fresh,
task_id,
"cli-lane-runner",
body,
),
)
# Route state is also written to the durable lane-state file;
# a later heartbeat or terminal result remains authoritative.
try:
previous_route = state.get("current_route")
constraint = selection_constraint(assignee)
unavailable_provider = (
constraint.exclude_provider if constraint.source == "health" else None
)
route = _routed_or_blocked(
kanban_db,
board,
task_id,
run_id,
lambda: select_route(
context,
assignee,
exclude_provider=constraint.exclude_provider,
exclude_reason=constraint.exclude_reason,
),
)
if route is None:
return
for note in constraint.notes:
comment(note)
if constraint.exclude_provider:
comment(
f"Provider routing guard ({constraint.source}) excluded "
f"{constraint.exclude_provider} before automatic routing.",
)
comment(
f"CLI route: {route.provider}/{route.model} at {route.effort}; classifier={route.classifier}; {route.reason}",
)
resume_handoff = ""
if (
isinstance(previous_route, dict)
and previous_route.get("provider")
and previous_route.get("provider") != route.provider
):
try:
previous_output = log_path.read_text(encoding="utf-8")[-10000:]
except OSError:
previous_output = "Previous provider log was unavailable after restart."
resume_handoff = git_handoff(workspace, previous_output)
comment(
f"Restart-time provider change: {previous_route.get('provider')} -> {route.provider}; explicit workspace handoff attached.",
)
prompt = build_prompt(context, workspace, resume_handoff)
max_runtime = int(
_task_value(task, "max_runtime_seconds", 0) or DEFAULT_MAX_RUNTIME
)
goal_mode = bool(_task_value(task, "goal_mode", False))
goal_max_turns = max(1, int(_task_value(task, "goal_max_turns", 1) or 1))
goal_turn = max(1, int(state.get("goal_turn", 0) or 0) + 1)
deadline = time.monotonic() + max_runtime
while True:
state["goal_turn"] = goal_turn
atomic_json(state_file, state)
remaining = max(1, int(deadline - time.monotonic()))
result = run_provider(
route,
prompt,
workspace,
state,
state_file,
log_path,
heartbeat,
remaining,
)
candidate_file = None
if result.structured:
candidate_file = _persist_candidate(
state,
state_file,
dict(result.structured),
route=route,
returncode=result.returncode,
goal_turn=goal_turn,
)
if result.capacity_failure:
boundary = capacity_failover(
kanban_db,
board=board,
task_id=task_id,
run_id=run_id,
assignee=assignee,
comment=comment,
context=context,
workspace=workspace,
state=state,
state_file=state_file,
log_path=log_path,
heartbeat=heartbeat,
deadline=deadline,
route=route,
result=result,
candidate_file=candidate_file,
goal_turn=goal_turn,
)
route = boundary.route
result = boundary.result
candidate_file = boundary.candidate_file
if boundary.unavailable_provider is not None:
unavailable_provider = boundary.unavailable_provider
if boundary.blocked:
break
if result.returncode == 0:
record_provider_success(route.provider)
structured = result.structured
if structured:
structured = dict(structured)
structured["artifacts"] = workspace_artifacts(
workspace,
structured.get("artifacts"),
)
metadata = {
"executor": "direct-cli-lane",
"provider": route.provider,
"model": route.model,
"effort": route.effort,
"classifier": route.classifier,
"state_file": str(state_file),
"codex_thread_id": state.get("codex_thread_id"),
"claude_session_id": state.get("claude_session_id"),
"goal_mode": goal_mode,
"goal_turn": goal_turn,
}
if candidate_file is not None:
metadata["candidate_file"] = str(candidate_file)
if structured:
for key in (
"changed_files",
"tests_run",
"artifacts",
"findings",
"blockers",
):
value = structured.get(key)
metadata[key] = value if isinstance(value, list) else []
completion_problem = None
if structured and result.returncode == 0:
# Resolve the role from the card itself, before any
# controller evidence is appended, and hand it to the gate
# so a review is judged on its verdict rather than on prose
# about the artifact it reviewed.
task_role, role_source = cli_lane_goal.task_role(
context,
structured,
)
metadata["task_role"] = task_role
metadata["task_role_source"] = role_source
completion_problem = cli_lane_goal.unfinished_result_reason(
structured,
role=task_role,
)
if (
structured.get("status") == "completed"
and completion_problem is None
and goal_mode
):
heartbeat("local goal-completion judge active")
rejection_history = state.get("goal_rejections", [])
if not isinstance(rejection_history, list):
rejection_history = []
judge_context = context
if goal_turn > 1 or rejection_history:
judge_context += (
f"\n\n{cli_lane_goal.CONTROLLER_EVIDENCE_HEADING}\n"
f"current turn {goal_turn}/{goal_max_turns}; prior rejected "
f"reports: {json.dumps(rejection_history[-5:])}."
)
accepted, judge_reason = cli_lane_goal.judge_goal_completion(
judge_context,
structured,
)
metadata["goal_judge_reason"] = judge_reason
if not accepted:
completion_problem = f"local goal judge requested continuation: {judge_reason}"
if (
structured
and structured.get("status") == "completed"
and result.returncode == 0
and completion_problem is None
):
terminal_file = _terminal_path(state_file, run_id, "pending")
metadata["terminal_record"] = str(
_terminal_path(state_file, run_id, "committed")
)
try:
terminal_file, terminal_record = _write_terminal_record(
state_file,
board=board,
task_id=task_id,
run_id=run_id,
structured=structured,
summary=str(structured.get("summary") or "Completed"),
metadata=metadata,
)
outcome = _finalize_terminal_record(
kanban_db,
terminal_file,
terminal_record,
)
except Exception as error:
identity = _terminal_identity(terminal_file)
replayable = bool(
identity is not None
and _has_pending_finalization(
identity.board,
identity.task_id,
identity.run_id,
)
)
if not replayable:
_recover_exact_run(
kanban_db,
identity,
f"persistence raised {type(error).__name__}",
)
raise TerminalFinalizationPending(
"accepted worker result remains outside the generic block path; "
f"terminal replayable={replayable}; persistence/finalization "
f"raised {type(error).__name__}"
) from error
if outcome != "committed":
raise TerminalFinalizationPending(
"accepted worker result is durably journaled but "
f"Kanban finalization is {outcome}"
)
record_route_decision(
route.provider, route.effort, route.classifier, "completed"
)
break
can_continue = (
goal_mode
and completion_problem is not None
and goal_turn < goal_max_turns
and deadline - time.monotonic() > 30
)
if can_continue:
rejection_history = state.get("goal_rejections", [])
if not isinstance(rejection_history, list):
rejection_history = []
state["goal_rejections"] = [
*rejection_history[-4:],
completion_problem,
]
goal_turn += 1
comment(
f"Goal completion rejected; continuing turn {goal_turn}/{goal_max_turns}: {completion_problem}",
)
escalation_context = (
context
+ "\n\nThe previous worker turn missed its completion quality mark: "
+ completion_problem
+ "\nSelect a route that can finish and verify the remaining work."
)
excluded = unavailable_provider
if excluded is None and assignee == "cli-auto":
excluded = fresh_unavailable_provider()
record_route_decision(
route.provider, route.effort, route.classifier, "goal-continued"
)
next_route = _routed_or_blocked(
kanban_db,
board,
task_id,
run_id,
lambda boundary=escalation_context, blocked=excluded: select_route(
boundary,
assignee,
exclude_provider=blocked,
exclude_reason="is unavailable according to fresh native health"
if blocked
else None,
),
)
if next_route is None:
break
comment(
f"Goal route {goal_turn}/{goal_max_turns}: "
f"{next_route.provider}/{next_route.model} at {next_route.effort}; "
f"classifier={next_route.classifier}; {next_route.reason}",
)
handoff = (
git_handoff(workspace, result.output)
if next_route.provider != route.provider
else ""
)
route = next_route
prompt = build_prompt(
context,
workspace,
handoff
+ "\nGoal-loop continuation: the previous final report was rejected because "
+ completion_problem
+ ". Reinspect live state, finish the outstanding work, and return new final evidence.",
)
continue
reason = completion_problem
if reason is None and structured:
blockers = structured.get("blockers", [])
reason = "; ".join(str(item) for item in blockers)
reason = reason or str(structured.get("summary") or "")
if reason is None:
reason = result.output[-4000:]
failure_reason = reason or (
f"{route.provider} worker failed with exit {result.returncode}"
)
failure_kind = (
"transient" if result.capacity_failure else "capability"
)
record_route_decision(
route.provider, route.effort, route.classifier, f"blocked-{failure_kind}"
)
_board_call(
kanban_db,
board,
lambda fresh, failure_reason=failure_reason, failure_kind=failure_kind: kanban_db.block_task(
fresh,
task_id,
reason=failure_reason,
kind=failure_kind,
expected_run_id=run_id,
),
)
break
except TerminalFinalizationPending as error:
comment(str(error))
except Exception as error:
failure_reason = f"Direct CLI lane failed: {type(error).__name__}: {error}"
_board_call(
kanban_db,
board,
lambda fresh: kanban_db.block_task(
fresh,
task_id,
reason=failure_reason,
kind="capability",
expected_run_id=run_id,
),
)

View File

@ -0,0 +1,181 @@
#!/usr/bin/env python3
"""Bounded, fail-closed cross-provider failover for one capacity boundary."""
from __future__ import annotations
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable
from cli_lane_board import _board_call
from cli_lane_config import ProcessResult, Route
from cli_lane_health import classify_capacity_failure, record_provider_failure
from cli_lane_metrics import (
record_provider_fallback,
record_route_decision,
record_router_selection_failure,
)
from cli_lane_prompt import build_prompt, git_handoff
from cli_lane_provider import run_provider
from cli_lane_records import _persist_candidate
from cli_lane_routing import select_route
@dataclass
class FailoverOutcome:
"""Where one capacity boundary left the claim's route and result."""
route: Route
result: ProcessResult
candidate_file: Path | None
unavailable_provider: str | None
blocked: bool
def _block_transient(
kanban_db: Any, board: str, task_id: str, run_id: Any, reason: str
) -> None:
"""Park one card as retryable so it re-enters when providers recover."""
_board_call(
kanban_db,
board,
lambda fresh: kanban_db.block_task(
fresh,
task_id,
reason=reason,
kind="transient",
expected_run_id=run_id,
),
)
def _routed_or_blocked(
kanban_db: Any,
board: str,
task_id: str,
run_id: Any,
select: Callable[[], Route],
) -> Route | None:
"""Turn a Switchyard outage into a transient block, never a capability one."""
try:
return select()
except RuntimeError as error:
record_router_selection_failure()
_block_transient(
kanban_db,
board,
task_id,
run_id,
"Switchyard route selection is unavailable; the card retries "
f"when the router recovers: {error}",
)
return None
def capacity_failover(
kanban_db: Any,
*,
board: str,
task_id: str,
run_id: Any,
assignee: str,
comment: Callable[[str], None],
context: str,
workspace: Path,
state: dict[str, Any],
state_file: Path,
log_path: Path,
heartbeat: Callable[[str], bool],
deadline: float,
route: Route,
result: ProcessResult,
candidate_file: Path | None,
goal_turn: int,
) -> FailoverOutcome:
"""Apply the capacity policy once: record, fail closed, or fail over."""
failure_class = classify_capacity_failure(result.output)
record_provider_failure(route.provider, failure_class)
record_route_decision(
route.provider, route.effort, route.classifier, "capacity-failure"
)
if assignee != "cli-auto":
# Fail closed: an operator pinned this provider, so a capacity or
# auth failure must wait for that provider, never switch away.
_block_transient(
kanban_db,
board,
task_id,
run_id,
f"Manually pinned provider {route.provider} hit a "
f"{failure_class} failure; the explicit {assignee} lane never "
"switches providers. Retry after recovery or reassign to cli-auto.",
)
return FailoverOutcome(route, result, candidate_file, None, True)
retry_context = (
context
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. "
+ "Select the alternate hosted provider at an appropriate effort."
)
alternate = "claude" if route.provider == "codex" else "codex"
fallback = _routed_or_blocked(
kanban_db,
board,
task_id,
run_id,
lambda: select_route(retry_context, f"cli-{alternate}-{route.effort}"),
)
if fallback is None:
return FailoverOutcome(route, result, candidate_file, route.provider, True)
record_provider_fallback(route.provider, fallback.provider, failure_class)
comment(
f"Provider fallback: {route.provider} -> {fallback.provider} "
f"after a {failure_class} failure; Jetson reclassified the retry boundary.",
)
fallback_result = run_provider(
fallback,
build_prompt(
context,
workspace,
git_handoff(workspace, result.output),
),
workspace,
state,
state_file,
log_path,
heartbeat,
max(1, int(deadline - time.monotonic())),
)
if fallback_result.capacity_failure:
# Bounded failover depth: two hosted providers, never a third
# metered or local path.
fallback_class = classify_capacity_failure(fallback_result.output)
record_provider_failure(fallback.provider, fallback_class)
record_route_decision(
fallback.provider, fallback.effort, fallback.classifier, "capacity-failure"
)
_block_transient(
kanban_db,
board,
task_id,
run_id,
"Both hosted providers failed at this boundary "
f"({route.provider}: {failure_class}; {fallback.provider}: "
f"{fallback_class}); the lane never falls to a metered or "
"local path. Waiting for provider recovery.",
)
return FailoverOutcome(
fallback, fallback_result, candidate_file, route.provider, True
)
if fallback_result.structured:
candidate_file = _persist_candidate(
state,
state_file,
dict(fallback_result.structured),
route=fallback,
returncode=fallback_result.returncode,
goal_turn=goal_turn,
)
return FailoverOutcome(
fallback, fallback_result, candidate_file, route.provider, False
)

View File

@ -0,0 +1,173 @@
#!/usr/bin/env python3
"""Durable path and filesystem primitives for Hermes CLI lane journals."""
from __future__ import annotations
import contextlib
import json
import os
import re
import stat
import uuid
from pathlib import Path
from typing import Any
from cli_lane_config import STATE_ROOT, TerminalIdentity, canonical_run_id
def atomic_json(path: Path, value: dict[str, Any], mode: int = 0o600) -> None:
"""Durably replace a small state document without following temp symlinks."""
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
os.chmod(path.parent, 0o700, follow_symlinks=False)
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
flags |= getattr(os, "O_NOFOLLOW", 0)
descriptor = None
try:
descriptor = os.open(temporary, flags, mode)
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
descriptor = None
json.dump(value, stream, indent=2, sort_keys=True)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
os.chmod(temporary, mode, follow_symlinks=False)
os.replace(temporary, path)
_fsync_directory(path.parent)
finally:
if descriptor is not None:
os.close(descriptor)
with contextlib.suppress(FileNotFoundError):
temporary.unlink()
def _fsync_directory(directory: Path) -> None:
"""Persist directory-entry changes after an atomic rename or quarantine."""
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
flags |= getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(directory, flags)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
def load_json(path: Path) -> dict[str, Any]:
try:
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(path, flags)
with os.fdopen(descriptor, "r", encoding="utf-8") as stream:
value = json.load(stream)
except (OSError, UnicodeError, json.JSONDecodeError):
return {}
return value if isinstance(value, dict) else {}
def state_path(board: str, task_id: str) -> Path:
safe_board = re.sub(r"[^a-zA-Z0-9_.-]+", "-", board)
safe_task = re.sub(r"[^a-zA-Z0-9_.-]+", "-", task_id)
return STATE_ROOT / safe_board / f"{safe_task}.json"
def _result_path(state_file: Path, run_id: Any, sequence: int) -> Path:
"""Return a unique provider-result path; completed turns are never reused."""
safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown"))
return state_file.with_name(
f"{state_file.stem}.run-{safe_run}.provider-{sequence}.result.json"
)
def _candidate_path(state_file: Path, run_id: Any, sequence: int) -> Path:
"""Return the durable path for one exact structured worker response."""
safe_run = re.sub(r"[^a-zA-Z0-9_.-]+", "-", str(run_id or "unknown"))
return state_file.with_name(
f"{state_file.stem}.run-{safe_run}.candidate-{sequence}.json"
)
def _terminal_path(
state_file: Path,
run_id: Any,
state: str = "pending",
) -> Path:
"""Return the identity-bound replay path for an accepted terminal response."""
if state not in {"pending", "committed"}:
raise ValueError(f"invalid terminal journal state: {state}")
canonical = canonical_run_id(run_id)
if type(run_id) is not int or canonical is None:
raise ValueError("terminal run ID must be a positive SQLite int64")
return state_file.with_name(
f"{state_file.stem}.run-{canonical}.terminal.{state}.json"
)
_TERMINAL_NAME = re.compile(
r"^(?P<task>[a-zA-Z0-9_.-]+)\.run-(?P<run>[0-9]+)\."
r"terminal\.(?P<state>pending|committed)\.json$"
)
_TERMINAL_EVIDENCE_NAME = re.compile(
r"^(?P<task>[a-zA-Z0-9_.-]+)\.run-(?P<run>[0-9]+)\."
r"terminal\.(?P<state>prepared|conflict)-(?P<digest>[a-f0-9]{32})\.json$"
)
_RETIRE_NAME = re.compile(
r"^\.retire\.(?P<path_digest>[a-f0-9]{16})\."
r"(?P<source_digest>[a-f0-9]{16})\.(?P<sequence>[0-9]+)$"
)
_LEGACY_RETIRE_NAME = re.compile(r"^\.retire\.[a-f0-9]{32}\.[0-9]+$")
_SAFE_BOARD = re.compile(r"^[a-zA-Z0-9_.-]+$")
def _terminal_identity(path: Path) -> TerminalIdentity | None:
"""Parse replay authority lexically before opening a journal or board."""
try:
relative = path.relative_to(STATE_ROOT)
except ValueError:
return None
if len(relative.parts) != 2:
return None
board, filename = relative.parts
match = _TERMINAL_NAME.fullmatch(filename)
run_id = canonical_run_id(match.group("run")) if match else None
try:
board_stat = path.parent.stat(follow_symlinks=False)
except OSError:
return None
if (
not match
or run_id is None
or not stat.S_ISDIR(board_stat.st_mode)
or not _SAFE_BOARD.fullmatch(board)
or board in {".", ".."}
or match.group("task") in {".", ".."}
):
return None
return TerminalIdentity(
board=board,
task_id=match.group("task"),
run_id=run_id,
state=match.group("state"),
)
def _terminal_evidence_identity(path: Path) -> TerminalIdentity | None:
"""Parse immutable prepared/conflict evidence authority from its path."""
try:
relative = path.relative_to(STATE_ROOT)
except ValueError:
return None
if len(relative.parts) != 2:
return None
board, filename = relative.parts
match = _TERMINAL_EVIDENCE_NAME.fullmatch(filename)
run_id = canonical_run_id(match.group("run")) if match else None
try:
board_stat = path.parent.stat(follow_symlinks=False)
except OSError:
return None
if (
not match
or run_id is None
or not stat.S_ISDIR(board_stat.st_mode)
or not _SAFE_BOARD.fullmatch(board)
or board in {".", ".."}
or match.group("task") in {".", ".."}
):
return None
return TerminalIdentity(
board=board,
task_id=match.group("task"),
run_id=run_id,
state=match.group("state"),
)

Some files were not shown because too many files have changed in this diff Show More