From e40fc5ec5dfcbdd0b8195a19bb6329c9fd97b26f Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 24 Aug 2026 04:39:45 -0300 Subject: [PATCH] hermes(chat): stage the HUX-12 evidence producer sidecar Activation-layer staging, fail-closed until enablement: a per-tenant hux-evidence-producer sidecar on the exact reviewed WebUI image runs hux_producer.run_once on a 60s loop, inert until the Vault-staged evidence key (tolerant init, tmpfs, 0400, staged only for the hux service and producer containers - never hermes or webui), the policy ConfigMap, and the scope ConfigMap exist. Adds least-privilege read-only RBAC (pods+statefulset in hermes, the single named Flux Kustomization), tenant egress to the Kubernetes API ClusterIP and the traefik edge, the policy allowlist, hux_producer packaging in the WebUI image, and a third expected WebUI consumer in the Flux release renderer. Delivery and image-automation gates enforce the boundary. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf --- ci/scripts/hermes_webui_flux_release.py | 2 +- dockerfiles/Dockerfile.hermes-webui | 3 +- services/hermes/chat-statefulset.yaml | 100 ++++++++++++++++++ services/hermes/hux-evidence-policy.json | 13 +++ services/hermes/hux-evidence-rbac.yaml | 54 ++++++++++ services/hermes/kustomization.yaml | 7 ++ services/hermes/networkpolicy.yaml | 16 +++ testing/tests/test_hermes_hux_delivery.py | 41 +++++++ testing/tests/test_hermes_image_automation.py | 22 ++-- 9 files changed, 246 insertions(+), 12 deletions(-) create mode 100644 services/hermes/hux-evidence-policy.json create mode 100644 services/hermes/hux-evidence-rbac.yaml diff --git a/ci/scripts/hermes_webui_flux_release.py b/ci/scripts/hermes_webui_flux_release.py index a966cc8f..d8ad6234 100644 --- a/ci/scripts/hermes_webui_flux_release.py +++ b/ci/scripts/hermes_webui_flux_release.py @@ -132,7 +132,7 @@ def _targets(chat_manifest: Path, dashboard_manifest: Path): "StatefulSet", "hermes-chat-tenant", "hermes-chat-statefulset.yaml", - (1, 2), + (1, 2, 3), ), ( dashboard_manifest, diff --git a/dockerfiles/Dockerfile.hermes-webui b/dockerfiles/Dockerfile.hermes-webui index eae5fde8..f226ad41 100644 --- a/dockerfiles/Dockerfile.hermes-webui +++ b/dockerfiles/Dockerfile.hermes-webui @@ -12,6 +12,7 @@ USER root # while the gateway remains the only process that owns an agent conversation. COPY --from=webui /apptoo /opt/hermes-webui COPY dockerfiles/hermes-hux-foundation/hux /opt/hermes-hux/hux +COPY dockerfiles/hermes-hux-foundation/hux_producer /opt/hermes-hux/hux_producer COPY services/hermes/contracts/hux /opt/hermes-hux/contracts ENV HUX_CONTRACT_DIR=/opt/hermes-hux/contracts @@ -72,7 +73,7 @@ RUN /opt/hermes/.venv/bin/python /tmp/hermes-webui-base-patch.py \ RUN /opt/hermes/.venv/bin/python -c 'import cryptography, yaml' \ && test -f /opt/hermes-hux/contracts/flags.json \ && test -f /opt/hermes-hux/contracts/release-ledger.schema.json \ - && PYTHONPATH=/opt/hermes-hux /opt/hermes/.venv/bin/python -m compileall -q /opt/hermes-hux/hux \ + && PYTHONPATH=/opt/hermes-hux /opt/hermes/.venv/bin/python -m compileall -q /opt/hermes-hux/hux /opt/hermes-hux/hux_producer \ && PYTHONPATH=/opt/hermes-hux /opt/hermes/.venv/bin/python -c 'from pathlib import Path; from tempfile import TemporaryDirectory; from hux.server import build_router; root = TemporaryDirectory(); router = build_router(Path(root.name), {"HUX_FLAGS": ""}); assert router.routes; root.cleanup()' \ && grep -Fq 'VALID_REASONING_EFFORTS = ("minimal", "low", "medium", "high", "xhigh")' \ /opt/hermes-webui/api/config.py \ diff --git a/services/hermes/chat-statefulset.yaml b/services/hermes/chat-statefulset.yaml index 0eedd3d2..23bdc1e4 100644 --- a/services/hermes/chat-statefulset.yaml +++ b/services/hermes/chat-statefulset.yaml @@ -260,6 +260,37 @@ spec: resources: requests: {cpu: 10m, memory: 16Mi} limits: {cpu: 50m, memory: 32Mi} + - name: stage-hux-evidence + image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 + imagePullPolicy: IfNotPresent + command: [/bin/sh, -ec] + args: + - | + # HUX-12 enablement is fail-closed: until the Vault annotation + # projects hux-evidence-key, nothing is staged and the evidence + # capability stays off. The key is staged only for the hux + # service and producer containers, never for hermes or webui. + if [ -s /vault/secrets/hux-evidence-key ]; then + umask 077 + tr -d '\r\n' < /vault/secrets/hux-evidence-key > /hux-evidence/.evidence-key.tmp + chown 10000:10000 /hux-evidence/.evidence-key.tmp + chmod 0400 /hux-evidence/.evidence-key.tmp + mv /hux-evidence/.evidence-key.tmp /hux-evidence/evidence-key + fi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + add: [CHOWN, DAC_OVERRIDE, FOWNER] + runAsUser: 0 + runAsGroup: 0 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - {name: hux-evidence-key, mountPath: /hux-evidence} + resources: + requests: {cpu: 10m, memory: 16Mi} + limits: {cpu: 50m, memory: 32Mi} - name: stage-runtime-access image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 imagePullPolicy: IfNotPresent @@ -563,6 +594,8 @@ spec: - {name: HUX_IMAGE_TAG, value: 'git-91eb4f92b7cf46e65dc615106aedfde232c9c670-build-18-release'} # {"$imagepolicy": "hermes:hermes-webui-release:tag"} - {name: HUX_IMAGE_DIGEST, value: 'sha256:df91f8a3cdb54d685f8fea023a4539d89d1d0b3f89024cca4dce87b0b4a7df9c'} # {"$imagepolicy": "hermes:hermes-webui-release:digest"} - {name: HUX_SWITCHYARD_ROUTE_CATALOG, value: 'atlas/manual/codex/luna,atlas/manual/codex/terra,atlas/manual/codex/sol,atlas/manual/claude/haiku,atlas/manual/claude/fable,atlas/manual/claude/sonnet,atlas/manual/claude/opus,atlas/manual/local/qwen-14b'} + - {name: HUX_RELEASE_EVIDENCE_KEY_FILE, value: /run/hermes-hux-evidence/evidence-key} + - {name: HUX_RELEASE_EVIDENCE_POLICY_FILE, value: /etc/hux-evidence/policy.json} - {name: HUX_READS_PER_MINUTE, value: "600"} - {name: HUX_WRITES_PER_MINUTE, value: "120"} - {name: HUX_REQUEST_TIMEOUT_SECONDS, value: "10"} @@ -571,6 +604,8 @@ spec: - {name: hux-relay-key, mountPath: /run/hermes-webui-hux, readOnly: true} - {name: hux-worker-key, mountPath: /run/hermes-hux-worker, readOnly: true} - {name: hux-tmp, mountPath: /tmp} + - {name: hux-evidence-key, mountPath: /run/hermes-hux-evidence, readOnly: true} + - {name: hux-evidence-policy, mountPath: /etc/hux-evidence, readOnly: true} readinessProbe: exec: command: @@ -602,6 +637,63 @@ spec: resources: requests: {cpu: 25m, memory: 64Mi} limits: {cpu: 250m, memory: 256Mi} + - name: hux-evidence-producer + image: registry.bstein.dev/bstein/hermes-webui:git-91eb4f92b7cf46e65dc615106aedfde232c9c670-build-18-release@sha256:df91f8a3cdb54d685f8fea023a4539d89d1d0b3f89024cca4dce87b0b4a7df9c # {"$imagepolicy": "hermes:hermes-webui-release"} + imagePullPolicy: IfNotPresent + command: [/bin/sh, -ec] + args: + - | + ordinal="${HOSTNAME##*-}" + export HUX_TENANT_SLOT="slot-${ordinal}" + while true; do + if [ -s /run/hermes-hux-evidence/evidence-key ] \ + && [ -s /run/hermes-hux-subject/subject ] \ + && [ -n "${HUX_PRODUCER_PROJECT_ID:-}" ] \ + && [ -n "${HUX_PRODUCER_CONVERSATION_ID:-}" ]; then + HUX_PRODUCER_SUBJECT="$(tr -d '\r\n' < /run/hermes-hux-subject/subject)" \ + /opt/hermes/.venv/bin/python -c \ + 'from hux_producer import run_once; run_once()' || true + fi + sleep 60 + done + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - {name: PYTHONPATH, value: /opt/hermes-hux} + - {name: PYTHONDONTWRITEBYTECODE, value: "1"} + - {name: HOME, value: /tmp} + - {name: HUX_BASE_URL, value: 'http://127.0.0.1:8790'} + - {name: HUX_PRODUCER_WORKLOAD, value: hermes-webui} + - {name: HUX_PRODUCER_NAMESPACE, value: hermes} + - {name: HUX_PRODUCER_POD_SELECTOR, value: 'app=hermes-chat-tenant'} + - {name: HUX_PRODUCER_WORKLOAD_KIND, value: statefulset} + - {name: HUX_PRODUCER_WORKLOAD_NAME, value: hermes-chat-tenant} + - {name: HUX_PRODUCER_TIMEOUT_SECONDS, value: "10"} + - {name: HUX_RELEASE_EVIDENCE_KEY_FILE, value: /run/hermes-hux-evidence/evidence-key} + - {name: HUX_RELEASE_EVIDENCE_POLICY_FILE, value: /etc/hux-evidence/policy.json} + envFrom: + - configMapRef: + name: hermes-hux-evidence-scope + optional: true + volumeMounts: + - {name: hux-evidence-key, mountPath: /run/hermes-hux-evidence, readOnly: true} + - {name: hux-evidence-policy, mountPath: /etc/hux-evidence, readOnly: true} + - {name: hux-data, mountPath: /run/hermes-hux-subject, subPathExpr: $(POD_NAME)/binding, readOnly: true} + - {name: hux-tmp, mountPath: /tmp} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + resources: + requests: {cpu: 10m, memory: 48Mi} + limits: {cpu: 100m, memory: 128Mi} - name: telegram-media image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 imagePullPolicy: IfNotPresent @@ -692,6 +784,14 @@ spec: - name: hux-tmp emptyDir: sizeLimit: 64Mi + - name: hux-evidence-key + emptyDir: + medium: Memory + sizeLimit: 1Mi + - name: hux-evidence-policy + configMap: + name: hermes-hux-evidence-policy + defaultMode: 0444 - name: hux-data persistentVolumeClaim: claimName: hermes-chat-hux-data diff --git a/services/hermes/hux-evidence-policy.json b/services/hermes/hux-evidence-policy.json new file mode 100644 index 00000000..13a5d3e6 --- /dev/null +++ b/services/hermes/hux-evidence-policy.json @@ -0,0 +1,13 @@ +{ + "schema": "hux.release_evidence_policy.v1", + "max_evidence_age_seconds": 900, + "workloads": { + "hermes-webui": { + "review_url_prefix": "https://scm.bstein.dev/atlas/titan-iac/pulls/", + "jenkins_job_url": "https://jenkins.bstein.dev/job/hermes-webui-image", + "image_repository": "registry.bstein.dev/bstein/hermes-webui", + "flux_kustomization": "hermes", + "health_url": "https://chat.bstein.dev/health" + } + } +} diff --git a/services/hermes/hux-evidence-rbac.yaml b/services/hermes/hux-evidence-rbac.yaml new file mode 100644 index 00000000..ed80d4d1 --- /dev/null +++ b/services/hermes/hux-evidence-rbac.yaml @@ -0,0 +1,54 @@ +# services/hermes/hux-evidence-rbac.yaml +# Read-only facts for the HUX-12 evidence producer sidecar: Ready pod +# imageIDs and the desired StatefulSet image in this namespace, plus the +# single named Flux Kustomization's applied revision. The pod service +# account gains nothing writable and nothing secret. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: hermes-hux-evidence-read + namespace: hermes +rules: + - apiGroups: [""] + resources: [pods] + verbs: [get, list] + - apiGroups: [apps] + resources: [statefulsets] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: hermes-hux-evidence-read + namespace: hermes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: hermes-hux-evidence-read +subjects: + - kind: ServiceAccount + name: hermes-chat + namespace: hermes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hermes-hux-evidence-kustomization-read +rules: + - apiGroups: [kustomize.toolkit.fluxcd.io] + resources: [kustomizations] + resourceNames: [hermes] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hermes-hux-evidence-kustomization-read +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hermes-hux-evidence-kustomization-read +subjects: + - kind: ServiceAccount + name: hermes-chat + namespace: hermes diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index fbadda29..9378ca51 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -43,6 +43,7 @@ resources: - agent-certificate.yaml - agent-ingress.yaml - execution-worker-rbac.yaml + - hux-evidence-rbac.yaml - execution-worker-statefulset.yaml - execution-mediator.yaml - execution-worker-networkpolicy.yaml @@ -195,6 +196,12 @@ configMapGenerator: - dashboard-style.css=plugins/auto-router/dashboard/dist/style.css options: disableNameSuffixHash: true + - name: hermes-hux-evidence-policy + namespace: hermes + files: + - policy.json=hux-evidence-policy.json + options: + disableNameSuffixHash: true - name: hermes-hux-runtime-plugin namespace: hermes files: diff --git a/services/hermes/networkpolicy.yaml b/services/hermes/networkpolicy.yaml index fa6311f9..30f2e8d4 100644 --- a/services/hermes/networkpolicy.yaml +++ b/services/hermes/networkpolicy.yaml @@ -406,6 +406,22 @@ spec: ports: - {protocol: TCP, port: 9005} - {protocol: TCP, port: 9009} + # HUX-12 evidence producer: the Kubernetes API (ClusterIP) for pod and + # Flux facts, and the ingress edge for Jenkins/Harbor/health receipts. + - to: + - ipBlock: + cidr: 10.43.0.1/32 + ports: + - {protocol: TCP, port: 443} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + ports: + - {protocol: TCP, port: 443} - to: - podSelector: matchLabels: diff --git a/testing/tests/test_hermes_hux_delivery.py b/testing/tests/test_hermes_hux_delivery.py index cf078762..df32e030 100644 --- a/testing/tests/test_hermes_hux_delivery.py +++ b/testing/tests/test_hermes_hux_delivery.py @@ -5,6 +5,7 @@ manifests must contain no partial HUX wiring at all; once the ``hux`` sidecar exists, every boundary assertion below is enforced strictly. """ +import json from pathlib import Path import pytest @@ -213,6 +214,46 @@ def test_hux_identity_and_authentication_inputs_are_file_backed() -> None: assert not {"HUX_RELAY_KEY", "HUX_WORKER_KEY", "HUX_ROUTER_KEY"} & set(hux) +def test_hux_evidence_producer_is_isolated_and_fail_closed() -> None: + """The HUX-12 evidence key reaches only the service and the producer.""" + _require_activation() + stateful = _statefulset() + pod = stateful["spec"]["template"]["spec"] + producer = _named(pod["containers"], "hux-evidence-producer") + hux = _named(pod["containers"], "hux") + hermes = _named(pod["containers"], "hermes") + webui = _named(pod["containers"], "webui") + media = _named(pod["containers"], "telegram-media") + + assert producer["image"] == hux["image"] + values = _env(producer) + assert values["HUX_RELEASE_EVIDENCE_KEY_FILE"] == "/run/hermes-hux-evidence/evidence-key" + assert values["HUX_RELEASE_EVIDENCE_POLICY_FILE"] == "/etc/hux-evidence/policy.json" + assert values["HUX_PRODUCER_WORKLOAD"] == "hermes-webui" + assert _env(hux)["HUX_RELEASE_EVIDENCE_KEY_FILE"] == "/run/hermes-hux-evidence/evidence-key" + assert producer["securityContext"]["readOnlyRootFilesystem"] is True + assert producer["securityContext"]["capabilities"]["drop"] == ["ALL"] + for denied in (hermes, webui, media): + assert not any( + item["name"] == "hux-evidence-key" for item in denied.get("volumeMounts", []) + ) + staging = _named(pod["initContainers"], "stage-hux-evidence") + script = staging["args"][0] + assert "if [ -s /vault/secrets/hux-evidence-key ]" in script + assert "chmod 0400" in script + volumes = {item["name"]: item for item in pod["volumes"]} + assert volumes["hux-evidence-key"]["emptyDir"]["medium"] == "Memory" + assert volumes["hux-evidence-policy"]["configMap"]["name"] == "hermes-hux-evidence-policy" + policy = json.loads((SERVICE / "hux-evidence-policy.json").read_text(encoding="utf-8")) + assert policy["schema"] == "hux.release_evidence_policy.v1" + assert set(policy["workloads"]) == {"hermes-webui"} + rbac = (SERVICE / "hux-evidence-rbac.yaml").read_text(encoding="utf-8") + assert "verbs: [get, list]" in rbac and "secrets" not in rbac + rendered = KUSTOMIZATION.read_text(encoding="utf-8") + assert "hux-evidence-rbac.yaml" in rendered + assert "policy.json=hux-evidence-policy.json" in rendered + + def test_hux_runtime_plugin_renders_its_vendored_hook_package() -> None: """The mounted plugin must contain its Kustomize-local Python package tree.""" _require_activation() diff --git a/testing/tests/test_hermes_image_automation.py b/testing/tests/test_hermes_image_automation.py index 1b209d19..8a52b108 100644 --- a/testing/tests/test_hermes_image_automation.py +++ b/testing/tests/test_hermes_image_automation.py @@ -73,19 +73,19 @@ def test_flux_updates_only_the_reviewed_hermes_image_digests() -> None: assert agent.count('"$imagepolicy": "hermes:hermes-agent-release:digest"') == 1 webui_marker = '"$imagepolicy": "hermes:hermes-webui-release"' chat_object = yaml.safe_load(chat) - hux = next( - ( - item - for item in chat_object["spec"]["template"]["spec"]["containers"] - if item["name"] == "hux" - ), + containers = chat_object["spec"]["template"]["spec"]["containers"] + hux = next((item for item in containers if item["name"] == "hux"), None) + producer = next( + (item for item in containers if item["name"] == "hux-evidence-producer"), None, ) # Before activation the chat StatefulSet carries exactly one WebUI - # consumer and no HUX setters; after activation the HUX sidecar is the - # second consumer of the very same reviewed image line, with tag and - # digest setters binding its build metadata. - assert chat.count(webui_marker) == (2 if hux else 1) + # consumer and no HUX setters; after activation the HUX sidecar (and the + # HUX-12 evidence producer) consume the very same reviewed image line, + # with tag and digest setters binding the sidecar's build metadata. + assert producer is None or hux is not None + consumers = 1 + (1 if hux else 0) + (1 if producer else 0) + assert chat.count(webui_marker) == consumers assert dashboard.count(webui_marker) == 1 assert chat.count( '"$imagepolicy": "hermes:hermes-webui-release:tag"' @@ -106,6 +106,8 @@ def test_flux_updates_only_the_reviewed_hermes_image_digests() -> None: "HUX_IMAGE_TAG" ].removeprefix("git-") assert hux["image"].endswith("@" + hux_env["HUX_IMAGE_DIGEST"]) + if producer is not None: + assert producer["image"] == hux["image"] # A digest-only setter replaces the complete YAML scalar with ``sha256:...``. # Whole-image setters must retain the registry and repository in pod specs. for workload in (chat, dashboard):