From c0a9c92ee402729bfe30202e8932560977616a86 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 24 Aug 2026 13:54:27 -0300 Subject: [PATCH] hermes(worker): stage inert HUX foundation on the worker instance worker.bstein.dev (the hermes-agent Deployment) gains the same HUX shape as chat, staged and inert: a foundation-only hux sidecar on the reviewed WebUI image line (Flux setters bound, 5s probe budgets), an init that provisions the HMAC identity as slot-100 on the durable home subtree (create-once context key, O_EXCL subject binding, per-pod worker key; no relay/router/evidence keys so those trusts fail closed), and observe-only hook env in the agent container with the runtime plugin mounted but deliberately NOT enabled - activation is a reviewed one-line flip per docs/hux/WORKER-PLAN.md, which carries the rollout, verification gates, canary/rollback ladder and open questions. Cross-surface continuity remains unclaimed until the live gates pass. 7 new topology-adaptive delivery gates green. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf --- docs/hux/WORKER-PLAN.md | 238 +++++++++++++++ services/hermes/agent-configmap.yaml | 5 + services/hermes/agent-deployment.yaml | 220 ++++++++++++++ .../tests/test_hermes_worker_hux_delivery.py | 275 ++++++++++++++++++ 4 files changed, 738 insertions(+) create mode 100644 docs/hux/WORKER-PLAN.md create mode 100644 testing/tests/test_hermes_worker_hux_delivery.py diff --git a/docs/hux/WORKER-PLAN.md b/docs/hux/WORKER-PLAN.md new file mode 100644 index 00000000..22ded234 --- /dev/null +++ b/docs/hux/WORKER-PLAN.md @@ -0,0 +1,238 @@ +# Worker HUX integration plan (worker.bstein.dev) + +Status: PREPARED, working tree only — reviewed and applied at a controlled +moment by the coordinator. The hermes-agent pod is the live automation brain; +nothing here is pushed, merged, or reconciled by the preparing agent. + +## Which workload is the Worker + +`worker.bstein.dev` → Ingress `hermes-agent-dashboard` → Service +`oauth2-proxy-hermes-agent` (selector `app: hermes-agent`) → the +`oauth2-proxy` container inside the **`hermes-agent` Deployment** +(`services/hermes/agent-deployment.yaml`), which proxies loopback +`127.0.0.1:9119` (dashboard) and `127.0.0.1:7681` (terminal). The +`hermes` Deployment in `services/hermes/deployment.yaml` is the DASHBOARD +(triage) workload and is untouched. All edits land in +`agent-deployment.yaml` + `agent-configmap.yaml` only. + +## Design summary + +Mirrors the reviewed chat topology (`chat-statefulset.yaml`), adapted to a +single-replica `Recreate` Deployment: + +- **`hux` sidecar** on the SAME reviewed WebUI image line as chat + (`registry.bstein.dev/bstein/hermes-webui`, Flux marker + `hermes:hermes-webui-release`, plus `:tag`/`:digest` setters for + `HUX_IMAGE_TAG`/`HUX_IMAGE_DIGEST`). `python -m hux.server`, + `PYTHONPATH=/opt/hermes-hux`, binds `127.0.0.1:8790` only; port 8790 + appears in no Service, oauth2-proxy upstream, or NetworkPolicy. Probes are + loopback `/healthz` exec probes with the corrected 5s timeout budget (the + chat canary's 2s budget caused transient probe kills). +- **Flags**: `HUX_FLAGS=hux.foundation` only. `HUX_TOOL_ENFORCEMENT=0` + (observe-only). Card widening and enforcement are separate, gated commits. +- **Identity**: new `init-hux-runtime` init container provisions, exactly as + chat's (same HMAC scheme `usr_ + HMAC-SHA256(context-key, + "hux.subject.id.v1\0")`), a persistent 32-byte context key (0600), + an immutable `O_EXCL` subject binding (0440, verified on every restart), a + redaction canary, and a **worker transport key** in a `Memory` emptyDir + (0400, regenerated every pod replacement). Slot is pinned to **`slot-100`** + (the service's `SLOT_RE` requires `slot-[0-9]{1,3}`; 100 is disjoint from + the chat tenants' slot-0..3). Surface `worker`, trust `worker`. + **No relay key, no router key, no evidence key**: with those env/file + inputs absent, `hux/identity.py` fails those trusts closed — worker trust + is the only live caller class on this instance. +- **Storage**: the store is a subtree of the existing durable + `hermes-agent-home` PVC — `hux/` at the claim root, mounted into the + sidecar as `/var/lib/hux` via fixed `subPath: hux` (NOT `$(POD_NAME)`: + Deployment pod names churn, and identity must survive replacement). The + `hermes` container receives only read-only views: + `/run/hermes-hux-context` = `hux/context`, `/run/hermes-hux-subject` = + `hux/binding`. Chat's `hermes-chat-hux-data` claim is not referenced. +- **Agent hook wiring** (`hermes` gateway container): `HUX_BASE_URL`, + `HUX_RUNTIME_ENABLED=1`, `HUX_TOOL_ENFORCEMENT=0`, + `HUX_TENANT_SLOT=slot-100`, `HUX_WORKER_KEY_FILE`, `HUX_SUBJECT_FILE`, + `HUX_CONTEXT_KEY_FILE`, `HUX_TIMEOUT_SECONDS=3`, per + `dockerfiles/hermes-worker-hux/hux_hook/NOTES.md`. No key or subject is + ever a literal env value. The `hermes-hux-runtime-plugin` ConfigMap + (already generated by `kustomization.yaml` for chat) is mounted read-only + at `/opt/data/plugins/hux-runtime`, but `hux-runtime` is **deliberately + absent from `plugins.enabled`** in `agent-configmap.yaml`, so the whole + hook is inert until the activation commit flips that one line. + +### Storage justification (home-PVC subtree vs dedicated PVC) + +Chosen: `hermes-agent-home` subtree. Reasons: + +1. Self-contained: no new PVC resource. `pvc.yaml` and `kustomization.yaml` + stay untouched, and the pod can never be Pending on an unbound claim — + this pod is the live automation brain and must always schedule. +2. Single replica + `Recreate` strategy = single writer on the RWO claim; + chat needed RWX + `subPathExpr` because four pods share one claim. Here + one pod owns everything. +3. Durability is identical to the rest of the coordinator state (Kanban DB, + workspace) — one claim to back up, one failure domain. + +Honest caveat, stated for review: the `hermes` container runs as root and +mounts the whole home claim at `/opt/data`, so the store subtree is +*technically readable* at `/opt/data/hux` by the agent process. On chat the +dedicated claim makes the store physically unreachable; here the boundary is +"the agent only *uses* the loopback API" plus 0700 modes (which do not bind +root). This is acceptable for a single-operator, single-subject instance +whose pod is already one trust domain — but if harder isolation is wanted, +the follow-up in "Coordinator TODOs" moves the store to a dedicated +`hermes-agent-hux-data` claim (a two-line volume swap plus data copy). + +## Release rendering / Flux automation impact (coordinator, do not edit ci/) + +- Flux `ImageUpdateAutomation` for hermes uses `update: {strategy: Setters, + path: services/hermes}` — the new markers in `agent-deployment.yaml` are + inside that path, so Flux keeps the sidecar image + tag/digest bindings + current with zero renderer changes. +- `ci/scripts/hermes_webui_flux_release.py` rewrites only + `StatefulSet/hermes-chat-tenant` (1–3 consumers) and `Deployment/hermes` + (exactly 1); it never opens `agent-deployment.yaml`, so its + `expected_images` contracts are unaffected. `test_hermes_image_automation.py` + counts markers only in the chat and dashboard files — verified unaffected. +- OPTIONAL renderer extension (coordinator decision): add a third target + `(agent_manifest, "Deployment", "hermes-agent", "hermes-agent-deployment.yaml", + 1)` plus HUX build-metadata binding for the agent file, so the release + belt covers the Worker too. Requires matching updates to + `test_hermes_webui_release.py` expectations. Not required for correctness + (Flux setters are authoritative); listed for completeness. +- **Operational consequence**: once merged, every WebUI release digest bump + rewrites `agent-deployment.yaml` and therefore **rolls the hermes-agent pod + (Recreate ⇒ brief worker.bstein.dev outage, minutes)** on every WebUI + release. Chat already rolls on those releases; the Worker joining that + cadence must be an accepted trade. If not accepted, the alternative is + pinning the sidecar image without markers and bumping it manually — that + deviates from the "same reviewed image line" requirement and is NOT what + is prepared here. + +## Coordinator TODOs (files outside this prepare's ownership) + +1. **None required for kustomize/dry-run correctness.** All referenced + objects (`hermes-hux-runtime-plugin` ConfigMap, `hermes-agent-home` PVC) + already exist in the build. `testing/quality_contract.json` already globs + `testing/**/*.py`, so the new test needs no registration. +2. **At activation** (separate commit, after staged-state gates pass): + - add `- hux-runtime` to `plugins.enabled` in `agent-configmap.yaml`; + - update `testing/tests/test_hermes_agent_layout.py` + (`test_agent_uses_one_native_kanban_control_plane` asserts + `config["plugins"]["enabled"] == ["auto-router"]` — exact equality) to + the new list. The prepared delivery test intentionally pins the staged + state (`"hux-runtime" not in plugins.enabled`) and must be updated in + the same activation commit. +3. **Optional hardening**: dedicated `hermes-agent-hux-data` PVC (RWO, + astreae, 5Gi) in `pvc.yaml` + swap the sidecar's `home/subPath: hux` + volume for it; copy `hux/` content across during a quiet window. +4. **Optional**: renderer third target (above). +5. Pre-existing note: `test_hermes_ai_usage_exporter.py` pins + `ai.bstein.dev/config-rev == "20260823-dual-provider-quota-health"` while + the manifest carries `"20260823-image-release-automation"` — mismatched + before this work; the config-rev annotation was deliberately left + untouched here. + +## Rollout steps (exact) + +1. Review this diff; run the full gate locally + (`PYTHONPATH=. .venv/bin/pytest -q testing/tests/test_hermes_worker_hux_delivery.py` + plus the agent/webui suites) and `kustomize build services/hermes`. +2. Merge to `main` in a **quiet window** (no cli-lane tasks in flight: + check the Kanban board and `/api/status`; the Recreate roll kills live + worker sessions — durable CLI lanes resume, native children do not). +3. Let Flux reconcile. The pod recreates once with the sidecar. Staged + state: hux serving loopback foundation-only; hook completely inert + (plugin not enabled). +4. Verify the staged gates (below). Leave the staged state soaking for at + least one WebUI release cycle to observe the digest-roll behavior. +5. Activation commit (coordinator TODO 2) — the hook starts observing: + events/spend posted from real worker turns, zero enforcement. +6. Card widening (activity_timeline first), each with its lifecycle gate and + a delivery-test flag-pin update. `hux.autonomy` is BLOCKED on the + human-surface question below. Enforcement stays 0 throughout this plan. + +## Verification gates (all must pass before any wider claim) + +Staged (after step 3): + +- Pod `hermes-agent-*` all containers Ready, zero restart loops; public + `https://worker.bstein.dev` 200 behind SSO; terminal reachable. +- Sidecar loopback health: from the hux container, + `python -c "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8790/healthz').read())"` + → `status: ok`. +- Identity provisioning: `hux/binding/subject` exists (0440, `usr_` + + 64 hex), `hux/context/context-key` 32 bytes 0600; record + `sha256(context-key)` and the subject value. +- Capabilities matrix on the loopback: no headers → 401; worker key + wrong + slot → 401; worker key + `slot-100` + bound subject → 200 with ONLY + HUX-11 foundation routes; any card route → 404 `flag_off`. Confirm relay + trust 401s (no relay key exists). +- **Identity persistence across a pod replacement**: delete the pod (or ride + the next WebUI release roll); after the new pod is Ready, re-hash + context-key and re-read the subject — both MUST be identical. The + worker-key MUST differ (rotation proof). +- Store isolation spot check: no Service/NetworkPolicy exposes 8790; store + dirs 0700 uid 10000. + +Activated (after step 5): + +- Drive one real worker turn (dashboard chat or a Kanban dispatch) and read + `GET /hux/v1/conversations/…/events` via worker trust on the loopback: + the turn's events appear with `Idempotency-Key`-deduped rows and no raw + arguments/output in any summary (redaction canary from + `hux/context/redaction-canary` never appears in stored records). +- Gateway, terminal, cli-lane-runner all still Ready; a cli-lane task + completes end-to-end (the hook must never block at enforcement 0). + +Cross-surface continuity is NOT claimed until every gate above has passed on +the live Worker — and even then it is a separate, explicit follow-up (the +chat and worker instances have distinct subjects and stores by design; any +continuity feature is new work, not a side effect of this rollout). + +## Canary / rollback + +This is a single-replica Deployment: there is no partition canary. The +canary IS the staged inert state (steps 3–4) — sidecar live, hook off — held +for a soak period before activation. Roll it as a Flux-tracked change only. + +Rollback ladder (fastest first): + +1. **Activation rollback**: revert the activation commit (remove + `hux-runtime` from `plugins.enabled`). One pod roll; sidecar keeps + running; hook inert again. +2. **Full rollback**: revert the integration commit(s) touching + `agent-deployment.yaml`/`agent-configmap.yaml`. Flux recreates the pod + without sidecar/init/env. The `hux/` subtree on the home claim is + retained data, not referenced by anything — harmless, removable later. +3. **Emergency (cluster is broken, Flux unusable)**: Brad applies + `kubectl -n hermes rollout undo deployment/hermes-agent` — then + reconciles Git to match, since the agent itself must never mutate + workloads. + +What to watch during any roll: `kubectl -n hermes get pods -w` until ALL +containers Ready (13 app containers once rendered: the 12 in +`agent-deployment.yaml` including `hux`, plus `execution-pool-coordinator` +from `execution-coordinator-patch.yaml`); `https://worker.bstein.dev/api/status` 200; Kanban +dispatch resumes (`cli-lane-runner` readiness); Vault agent-inject succeeded +(init order unchanged); zero CrashLoopBackOff on `hux`. + +## Open questions + +1. **Approvals need a human surface.** This instance has only worker trust; + `hux.autonomy` decisions are rejected for worker callers by design (F1), + so enabling autonomy/enforcement on the Worker requires a human-surface + path first (candidates: a BFF on the worker dashboard asserting router + trust like chat's, or driving decisions from the chat instance — which + reopens cross-surface identity). Until decided, `hux.autonomy` and + `HUX_TOOL_ENFORCEMENT=1` are out of scope. +2. **Terminal and cli-lane-runner containers** run their own Hermes + processes but were intentionally NOT wired (env absent → plugin + `HUX_RUNTIME_ENABLED` gate keeps them clean even after activation). If + observation should cover TUI/lane turns too, mirror the `hermes` + container's env+mounts onto them in a follow-up. +3. **`HUX_PROJECT_SOURCE`** is `profile:default` for parity with chat; + a worker-specific source (e.g. `profile:worker`) would partition worker + activity into its own HUX project — decide before activation, changing it + later re-keys the derived project id. +4. Whether to take Coordinator TODO 3 (dedicated PVC) before or after + activation. diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index 40cfdf88..4fd66e76 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -118,6 +118,11 @@ data: plugins: enabled: - auto-router + # hux-runtime is STAGED, not enabled. The plugin ConfigMap is mounted + # at /opt/data/plugins/hux-runtime and the observe-only environment is + # wired in agent-deployment.yaml, but the Worker HUX hook stays inert + # until `- hux-runtime` is added here by the activation commit after + # the sidecar's live gates pass (docs/hux/WORKER-PLAN.md). skills: creation_nudge_interval: 15 diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 8998abf6..1bee1c0a 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -245,6 +245,123 @@ spec: resources: requests: {cpu: 25m, memory: 32Mi} limits: {cpu: 100m, memory: 64Mi} + # Worker HUX identity provisioning. Mirrors the chat tenants' + # init-hux-runtime, adapted to the single-replica Deployment: the + # subtree name is the fixed literal "hux" (Deployment pod names churn, + # so ${HOSTNAME} would break identity persistence across restarts) on + # the durable hermes-agent-home claim, and the slot is pinned to + # slot-100 — outside the chat tenants' slot-0..slot-3 range. Only a + # worker transport key is provisioned: the Worker has no relay or + # human browser surface, so relay trust stays fail-closed by absence. + - name: init-hux-runtime + image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 + imagePullPolicy: IfNotPresent + command: [/bin/sh, -ec] + args: + - | + umask 077 + worker_root="/hux-data/hux" + export HUX_INIT_ROOT="${worker_root}" + mkdir -p \ + "${worker_root}/binding" \ + "${worker_root}/context" \ + "${worker_root}/store" \ + /hux-worker + chown 10000:10000 \ + "${worker_root}" \ + "${worker_root}/binding" \ + "${worker_root}/context" \ + "${worker_root}/store" \ + /hux-worker + chmod 0700 \ + "${worker_root}" \ + "${worker_root}/binding" \ + "${worker_root}/context" \ + "${worker_root}/store" \ + /hux-worker + if [ ! -e "${worker_root}/context/context-key" ]; then + dd if=/dev/urandom of="${worker_root}/context/.context-key.tmp" bs=32 count=1 2>/dev/null + chown 10000:10000 "${worker_root}/context/.context-key.tmp" + chmod 0600 "${worker_root}/context/.context-key.tmp" + mv "${worker_root}/context/.context-key.tmp" "${worker_root}/context/context-key" + fi + test "$(wc -c < "${worker_root}/context/context-key")" -eq 32 + chown 10000:10000 "${worker_root}/context/context-key" + chmod 0600 "${worker_root}/context/context-key" + HUX_INIT_SLOT="slot-100" \ + /opt/hermes/.venv/bin/python - <<'PY' + import hashlib + import hmac + import os + import stat + from pathlib import Path + + root = Path(os.environ["HUX_INIT_ROOT"]) + key = (root / "context/context-key").read_bytes() + slot = os.environ["HUX_INIT_SLOT"] + subject = "usr_" + hmac.new( + key, + b"hux.subject.id.v1\0" + slot.encode("ascii"), + hashlib.sha256, + ).hexdigest() + target = root / "binding/subject" + expected = (subject + "\n").encode("ascii") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(target, flags, 0o440) + except FileExistsError: + info = target.lstat() + if ( + not stat.S_ISREG(info.st_mode) + or info.st_uid != 10000 + or stat.S_IMODE(info.st_mode) != 0o440 + or info.st_nlink != 1 + or target.read_bytes() != expected + ): + raise SystemExit("persistent HUX subject binding is unsafe") + else: + try: + os.write(descriptor, expected) + os.fchown(descriptor, 10000, 10000) + os.fchmod(descriptor, 0o440) + os.fsync(descriptor) + finally: + os.close(descriptor) + PY + if [ ! -e "${worker_root}/context/redaction-canary" ]; then + dd if=/dev/urandom bs=32 count=1 2>/dev/null \ + | sha256sum | cut -d ' ' -f 1 \ + > "${worker_root}/context/.redaction-canary.tmp" + chown 10000:10000 "${worker_root}/context/.redaction-canary.tmp" + chmod 0400 "${worker_root}/context/.redaction-canary.tmp" + mv "${worker_root}/context/.redaction-canary.tmp" "${worker_root}/context/redaction-canary" + fi + target=/hux-worker/worker-key + if [ ! -e "${target}" ]; then + dd if=/dev/urandom bs=32 count=1 2>/dev/null \ + | sha256sum | cut -d ' ' -f 1 > "${target}.tmp" + chown 10000:10000 "${target}.tmp" + chmod 0400 "${target}.tmp" + mv "${target}.tmp" "${target}" + fi + test "$(wc -c < "${target}")" -eq 65 + chown 10000:10000 "${target}" + chmod 0400 "${target}" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + add: [CHOWN, DAC_OVERRIDE, FOWNER] + runAsUser: 0 + runAsGroup: 0 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - {name: home, mountPath: /hux-data} + - {name: hux-worker-key, mountPath: /hux-worker} + resources: + requests: {cpu: 10m, memory: 16Mi} + limits: {cpu: 50m, memory: 32Mi} - name: stage-runtime-access image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 imagePullPolicy: IfNotPresent @@ -649,6 +766,20 @@ spec: - {name: HERMES_CODEX_HEALTH_PATH, value: /opt/data/provider-health/codex.json} - {name: HERMES_CLAUDE_HEALTH_PATH, value: /opt/data/provider-health/claude.json} - {name: HERMES_IMAGE_BROKER_KEY_FILE, value: /runtime-access/chat-relay-key} + # Worker HUX hook wiring (observe-only). The hux-runtime plugin is + # mounted but stays OUT of plugins.enabled until the activation + # commit (docs/hux/WORKER-PLAN.md), so this environment is inert + # until that deliberate flip. Enforcement is pinned to 0: the hook + # observes and emits, it never gates worker tools in this stage. + - {name: HUX_BASE_URL, value: 'http://127.0.0.1:8790'} + - {name: HUX_RUNTIME_ENABLED, value: "1"} + - {name: HUX_TOOL_ENFORCEMENT, value: "0"} + - {name: HUX_TENANT_SLOT, value: slot-100} + - {name: HUX_WORKER_KEY_FILE, value: /run/hermes-hux-worker/worker-key} + - {name: HUX_SUBJECT_FILE, value: /run/hermes-hux-subject/subject} + - {name: HUX_CONTEXT_KEY_FILE, value: /run/hermes-hux-context/context-key} + - {name: HUX_PROJECT_SOURCE, value: 'profile:default'} + - {name: HUX_TIMEOUT_SECONDS, value: "3"} volumeMounts: - {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true} - {name: home, mountPath: /opt/data} @@ -672,6 +803,10 @@ spec: - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: routing-catalog, mountPath: /routing-catalog, readOnly: true} - {name: tmp, mountPath: /tmp} + - {name: hux-runtime-plugin, mountPath: /opt/data/plugins/hux-runtime, readOnly: true} + - {name: hux-worker-key, mountPath: /run/hermes-hux-worker, readOnly: true} + - {name: home, mountPath: /run/hermes-hux-context, subPath: hux/context, readOnly: true} + - {name: home, mountPath: /run/hermes-hux-subject, subPath: hux/binding, readOnly: true} startupProbe: exec: command: [curl, -fsS, http://127.0.0.1:9119/api/status] @@ -1219,6 +1354,71 @@ spec: resources: requests: {cpu: 50m, memory: 128Mi} limits: {cpu: "3", memory: 3Gi} + # Worker HUX service: the same reviewed WebUI image line as the chat + # tenants (one artifact, one Flux image policy), loopback-only, with + # foundation as the only enabled card until the Worker lifecycle gates + # in docs/hux/WORKER-PLAN.md pass on this instance. + - name: hux + image: registry.bstein.dev/bstein/hermes-webui:git-043aa9ee89c13fc0f4fb270f489c09a0e2f18b12-build-23-release@sha256:b3b5830d08deb5b2f7fcc9e5c62ee9e71ad39a7e56df7bfbc583be424a9a87d6 # {"$imagepolicy": "hermes:hermes-webui-release"} + imagePullPolicy: IfNotPresent + command: [/opt/hermes/.venv/bin/python, -m, hux.server] + ports: + - {name: hux-loopback, containerPort: 8790, protocol: TCP} + env: + - {name: PYTHONPATH, value: /opt/hermes-hux} + - {name: PYTHONDONTWRITEBYTECODE, value: "1"} + - {name: HOME, value: /tmp} + - {name: HUX_BIND, value: 127.0.0.1} + - {name: HUX_PORT, value: "8790"} + - {name: HUX_TENANT_SLOT, value: slot-100} + - {name: HUX_DATA_ROOT, value: /var/lib/hux/store} + - {name: HUX_FLAGS, value: hux.foundation} + - {name: HUX_WORKER_KEY_FILE, value: /run/hermes-hux-worker/worker-key} + - {name: HUX_SUBJECT_BINDING_FILE, value: /var/lib/hux/binding/subject} + - {name: HUX_CONTEXT_KEY_FILE, value: /var/lib/hux/context/context-key} + - {name: HUX_CANARY_FILE, value: /var/lib/hux/context/redaction-canary} + - name: HUX_IMAGE_TAG + value: git-043aa9ee89c13fc0f4fb270f489c09a0e2f18b12-build-23-release # {"$imagepolicy": "hermes:hermes-webui-release:tag"} + - name: HUX_IMAGE_DIGEST + value: sha256:b3b5830d08deb5b2f7fcc9e5c62ee9e71ad39a7e56df7bfbc583be424a9a87d6 # {"$imagepolicy": "hermes:hermes-webui-release:digest"} + - {name: HUX_READS_PER_MINUTE, value: "600"} + - {name: HUX_WRITES_PER_MINUTE, value: "120"} + - {name: HUX_REQUEST_TIMEOUT_SECONDS, value: "10"} + volumeMounts: + - {name: home, mountPath: /var/lib/hux, subPath: hux} + - {name: hux-worker-key, mountPath: /run/hermes-hux-worker, readOnly: true} + - {name: hux-tmp, mountPath: /tmp} + readinessProbe: + exec: + command: + - /opt/hermes/.venv/bin/python + - -c + - "import json,urllib.request; body=json.load(urllib.request.urlopen('http://127.0.0.1:8790/healthz', timeout=2)); assert body['status']=='ok'" + initialDelaySeconds: 2 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 12 + livenessProbe: + exec: + command: + - /opt/hermes/.venv/bin/python + - -c + - "import json,urllib.request; body=json.load(urllib.request.urlopen('http://127.0.0.1:8790/healthz', timeout=2)); assert body['status']=='ok'" + initialDelaySeconds: 20 + periodSeconds: 20 + timeoutSeconds: 5 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + resources: + requests: {cpu: 25m, memory: 64Mi} + limits: {cpu: 250m, memory: 256Mi} volumes: - name: home persistentVolumeClaim: @@ -1300,3 +1500,23 @@ spec: - name: ttyd-index emptyDir: sizeLimit: 2Mi + - name: hux-worker-key + emptyDir: + medium: Memory + sizeLimit: 1Mi + - name: hux-tmp + emptyDir: + sizeLimit: 64Mi + - name: hux-runtime-plugin + configMap: + name: hermes-hux-runtime-plugin + items: + - {key: __init__.py, path: __init__.py} + - {key: context_ids.py, path: context_ids.py} + - {key: emitters.py, path: emitters.py} + - {key: runtime.py, path: runtime.py} + - {key: tool_policy.py, path: tool_policy.py} + - {key: plugin.yaml, path: plugin.yaml} + - {key: hux-hook-init.py, path: hux_hook/__init__.py} + - {key: hux-hook-client.py, path: hux_hook/client.py} + - {key: hux-hook-hooks.py, path: hux_hook/hooks.py} diff --git a/testing/tests/test_hermes_worker_hux_delivery.py b/testing/tests/test_hermes_worker_hux_delivery.py new file mode 100644 index 00000000..fc25eb76 --- /dev/null +++ b/testing/tests/test_hermes_worker_hux_delivery.py @@ -0,0 +1,275 @@ +"""Flux delivery gates for the Worker (hermes-agent) HUX service boundary. + +Topology-adaptive like the chat gates in ``test_hermes_hux_delivery.py``: +while the agent Deployment carries no ``hux`` sidecar the manifests must +contain zero partial Worker HUX wiring; once the sidecar exists, the full +Worker boundary below is enforced strictly. The Worker instance is the +``hermes-agent`` Deployment behind worker.bstein.dev (via the +``oauth2-proxy-hermes-agent`` Service), NOT the dashboard ``hermes`` +Deployment. See ``docs/hux/WORKER-PLAN.md``. +""" + +import re +from pathlib import Path + +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parents[2] +SERVICE = ROOT / "services/hermes" +AGENT = SERVICE / "agent-deployment.yaml" +AGENT_CONFIG = SERVICE / "agent-configmap.yaml" +CHAT = SERVICE / "chat-statefulset.yaml" +KUSTOMIZATION = SERVICE / "kustomization.yaml" +WEBUI_IMAGE = "registry.bstein.dev/bstein/hermes-webui" +WEBUI_MARKER = '"$imagepolicy": "hermes:hermes-webui-release"' +TAG_MARKER = '"$imagepolicy": "hermes:hermes-webui-release:tag"' +DIGEST_MARKER = '"$imagepolicy": "hermes:hermes-webui-release:digest"' +WORKER_SLOT = "slot-100" + + +def _deployment() -> dict: + return yaml.safe_load(AGENT.read_text(encoding="utf-8")) + + +def _pod() -> dict: + return _deployment()["spec"]["template"]["spec"] + + +def _hux_active() -> bool: + return any(item["name"] == "hux" for item in _pod()["containers"]) + + +def _require_activation() -> None: + if not _hux_active(): + pytest.skip("Worker HUX activation topology is not staged in this tree") + + +def _named(items: list[dict], name: str) -> dict: + return next(item for item in items if item["name"] == name) + + +def _env(container: dict) -> dict[str, str]: + return {item["name"]: item.get("value", "") for item in container.get("env", [])} + + +def _mounts(container: dict) -> list[dict]: + return container.get("volumeMounts", []) + + +def test_worker_hux_topology_is_all_or_nothing() -> None: + """A partial Worker HUX rollout (wiring without the sidecar) never ships.""" + pod = _pod() + if _hux_active(): + return + assert not any(item["name"].startswith("hux") for item in pod["containers"]) + assert not any( + item["name"] == "init-hux-runtime" for item in pod.get("initContainers", []) + ) + assert not any(item["name"].startswith("hux-") for item in pod["volumes"]) + for container in pod["containers"]: + assert not any(name.startswith("HUX_") for name in _env(container)) + assert WEBUI_IMAGE not in AGENT.read_text(encoding="utf-8") + + +def test_worker_hux_sidecar_is_loopback_only_on_the_reviewed_webui_image() -> None: + """One reviewed WebUI artifact serves chat and Worker; loopback is the only path.""" + _require_activation() + pod = _pod() + hux = _named(pod["containers"], "hux") + values = _env(hux) + raw = AGENT.read_text(encoding="utf-8") + + assert hux["image"].startswith(WEBUI_IMAGE + ":") + assert "@sha256:" in hux["image"] + # Exactly one WebUI consumer in this Deployment, on the Flux-managed line. + assert raw.count(WEBUI_MARKER) == 1 + marked_line = next(line for line in raw.splitlines() if WEBUI_MARKER in line) + assert hux["image"] in marked_line + # The same reviewed image line as the chat tenants: the repo-wide Flux + # Setters run (update.path services/hermes) rewrites both files together. + chat_pod = yaml.safe_load(CHAT.read_text(encoding="utf-8"))["spec"]["template"][ + "spec" + ] + assert _named(chat_pod["containers"], "webui")["image"] == hux["image"] + + assert hux["command"] == ["/opt/hermes/.venv/bin/python", "-m", "hux.server"] + assert values["PYTHONPATH"] == "/opt/hermes-hux" + assert values["PYTHONDONTWRITEBYTECODE"] == "1" + assert values["HUX_BIND"] == "127.0.0.1" + assert values["HUX_PORT"] == "8790" + assert values["HUX_TENANT_SLOT"] == WORKER_SLOT + assert values["HUX_DATA_ROOT"] == "/var/lib/hux/store" + # Foundation-only start: widening the Worker flag set is a deliberate, + # reviewed change gated on the WORKER-PLAN lifecycle checks, so this pin + # is updated in the same commit that turns each card on. + assert {flag for flag in values["HUX_FLAGS"].split(",") if flag} == { + "hux.foundation" + } + assert hux["securityContext"]["readOnlyRootFilesystem"] is True + assert hux["securityContext"]["capabilities"]["drop"] == ["ALL"] + assert hux["securityContext"]["runAsUser"] == 10000 + for probe in ("readinessProbe", "livenessProbe"): + assert "tcpSocket" not in hux[probe] + command = " ".join(hux[probe]["exec"]["command"]) + assert "127.0.0.1" in command + assert "8790" in command + assert "/healthz" in command + # The chat canary's transient exit-137s were probe kills under a 2s + # budget; the Worker ships with the corrected 5s budget from day one. + assert hux[probe]["timeoutSeconds"] >= 5 + # Loopback only: no Service, proxy upstream, or NetworkPolicy exposes 8790. + assert "8790" not in (SERVICE / "service.yaml").read_text(encoding="utf-8") + assert "8790" not in (SERVICE / "oauth2-proxy.yaml").read_text(encoding="utf-8") + assert "port: 8790" not in (SERVICE / "networkpolicy.yaml").read_text( + encoding="utf-8" + ) + + +def test_worker_hux_store_is_a_private_durable_subtree() -> None: + """The Worker owns its own store subtree and never shares chat's claim.""" + _require_activation() + pod = _pod() + hux = _named(pod["containers"], "hux") + hermes = _named(pod["containers"], "hermes") + + # The store lives on the durable hermes-agent-home claim under the fixed + # "hux" subtree (Deployment pod names churn, so no $(POD_NAME) scoping). + assert { + "name": "home", + "mountPath": "/var/lib/hux", + "subPath": "hux", + } in _mounts(hux) + assert not any("subPathExpr" in item for item in _mounts(hux)) + # The agent gets only its read-only key/context views of that subtree; + # ledger records are reachable from the agent only through loopback HTTP. + assert { + "name": "home", + "mountPath": "/run/hermes-hux-context", + "subPath": "hux/context", + "readOnly": True, + } in _mounts(hermes) + assert { + "name": "home", + "mountPath": "/run/hermes-hux-subject", + "subPath": "hux/binding", + "readOnly": True, + } in _mounts(hermes) + for container in pod["containers"]: + if container["name"] == "hux": + continue + assert not any( + item.get("mountPath") == "/var/lib/hux" for item in _mounts(container) + ) + if container["name"] != "hermes": + assert not any( + item["name"].startswith("hux-") for item in _mounts(container) + ) + volumes = {item["name"]: item for item in pod["volumes"]} + assert volumes["hux-worker-key"]["emptyDir"]["medium"] == "Memory" + # Single surface, single operator: no relay key and no evidence trust on + # the Worker instance — those trusts fail closed by absence. + assert "hux-relay-key" not in volumes + assert "hux-evidence-key" not in volumes + raw = AGENT.read_text(encoding="utf-8") + assert "hermes-chat-hux-data" not in raw + + +def test_worker_hux_init_provisions_durable_identity_and_rotating_key() -> None: + """Identity survives pod replacement; the transport key rotates with it.""" + _require_activation() + pod = _pod() + init = _named(pod["initContainers"], "init-hux-runtime") + script = init["args"][0] + + assert 'worker_root="/hux-data/hux"' in script + assert "${HOSTNAME}" not in script + assert "$(POD_NAME)" not in script + assert 'HUX_INIT_SLOT="slot-100"' in script + assert "chown -R" not in script + assert 'if [ ! -e "${worker_root}/context/context-key" ]' in script + assert "bs=32 count=1" in script + assert 'chmod 0600 "${worker_root}/context/context-key"' in script + assert 'b"hux.subject.id.v1\\0" + slot.encode("ascii")' in script + assert 'target = root / "binding/subject"' in script + assert "os.O_EXCL" in script + assert "or stat.S_IMODE(info.st_mode) != 0o440" in script + assert "target=/hux-worker/worker-key" in script + assert "chmod 0400" in script + assert "relay" not in script + assert init["securityContext"]["capabilities"] == { + "drop": ["ALL"], + "add": ["CHOWN", "DAC_OVERRIDE", "FOWNER"], + } + assert {"name": "home", "mountPath": "/hux-data"} in _mounts(init) + assert {"name": "hux-worker-key", "mountPath": "/hux-worker"} in _mounts(init) + + +def test_worker_hux_hook_env_is_file_backed_and_observe_only() -> None: + """The agent hook is wired for observation: file-backed keys, enforcement 0.""" + _require_activation() + pod = _pod() + hermes = _env(_named(pod["containers"], "hermes")) + + assert hermes["HUX_BASE_URL"] == "http://127.0.0.1:8790" + assert hermes["HUX_RUNTIME_ENABLED"] == "1" + # Observe-only pin: flipping enforcement on the Worker is its own gated + # change (approvals need a human surface this instance does not have yet). + assert hermes["HUX_TOOL_ENFORCEMENT"] == "0" + assert hermes["HUX_TENANT_SLOT"] == WORKER_SLOT + assert hermes["HUX_WORKER_KEY_FILE"] == "/run/hermes-hux-worker/worker-key" + assert hermes["HUX_SUBJECT_FILE"] == "/run/hermes-hux-subject/subject" + assert hermes["HUX_CONTEXT_KEY_FILE"] == "/run/hermes-hux-context/context-key" + assert hermes["HUX_TIMEOUT_SECONDS"] == "3" + forbidden = {"HUX_WORKER_KEY", "HUX_RELAY_KEY", "HUX_ROUTER_KEY", "HUX_SUBJECT"} + for container in pod["containers"]: + assert not forbidden & set(_env(container)) + + +def test_worker_hux_runtime_plugin_is_staged_but_inert() -> None: + """The vendored hook package is mounted, yet nothing activates it.""" + _require_activation() + pod = _pod() + hermes = _named(pod["containers"], "hermes") + plugin = _named(pod["volumes"], "hux-runtime-plugin")["configMap"] + items = {item["key"]: item["path"] for item in plugin["items"]} + + assert plugin["name"] == "hermes-hux-runtime-plugin" + assert { + "hux-hook-init.py": "hux_hook/__init__.py", + "hux-hook-client.py": "hux_hook/client.py", + "hux-hook-hooks.py": "hux_hook/hooks.py", + }.items() <= items.items() + assert { + "name": "hux-runtime-plugin", + "mountPath": "/opt/data/plugins/hux-runtime", + "readOnly": True, + } in _mounts(hermes) + rendered = KUSTOMIZATION.read_text(encoding="utf-8") + assert "hux-hook-client.py=plugins/hux-runtime/hux_hook/client.py" in rendered + # Staged, not active: the activation commit adds hux-runtime to + # plugins.enabled (docs/hux/WORKER-PLAN.md); until then the wiring is inert. + configmap = yaml.safe_load(AGENT_CONFIG.read_text(encoding="utf-8")) + config = yaml.safe_load(configmap["data"]["config.yaml"]) + assert "hux-runtime" not in config["plugins"]["enabled"] + + +def test_worker_hux_build_bindings_use_flux_setters() -> None: + """Sidecar build provenance is bound by the same Flux setters as chat.""" + _require_activation() + raw = AGENT.read_text(encoding="utf-8") + hux = _env(_named(_pod()["containers"], "hux")) + + assert raw.count(TAG_MARKER) == 1 + assert raw.count(DIGEST_MARKER) == 1 + release = re.fullmatch( + r"git-([0-9a-f]{40})-build-[1-9][0-9]*-release", hux["HUX_IMAGE_TAG"] + ) + assert release is not None + assert hux["HUX_IMAGE_DIGEST"].startswith("sha256:") + image = _named(_pod()["containers"], "hux")["image"] + assert image.split(":git-", 1)[1].split("@", 1)[0] == hux[ + "HUX_IMAGE_TAG" + ].removeprefix("git-") + assert image.endswith("@" + hux["HUX_IMAGE_DIGEST"])