hermes(chat): stage the HUX-12 evidence producer sidecar

Activation-layer staging, fail-closed until enablement: a per-tenant
hux-evidence-producer sidecar on the exact reviewed WebUI image runs
hux_producer.run_once on a 60s loop, inert until the Vault-staged
evidence key (tolerant init, tmpfs, 0400, staged only for the hux
service and producer containers - never hermes or webui), the policy
ConfigMap, and the scope ConfigMap exist. Adds least-privilege
read-only RBAC (pods+statefulset in hermes, the single named Flux
Kustomization), tenant egress to the Kubernetes API ClusterIP and the
traefik edge, the policy allowlist, hux_producer packaging in the WebUI
image, and a third expected WebUI consumer in the Flux release
renderer. Delivery and image-automation gates enforce the boundary.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf

ci/scripts/hermes_webui_flux_release.py

@@ -132,7 +132,7 @@ def _targets(chat_manifest: Path, dashboard_manifest: Path):
             "StatefulSet",
             "hermes-chat-tenant",
             "hermes-chat-statefulset.yaml",
-            (1, 2),
+            (1, 2, 3),
         ),
         (
             dashboard_manifest,

dockerfiles/Dockerfile.hermes-webui

@@ -12,6 +12,7 @@ USER root
 # while the gateway remains the only process that owns an agent conversation.
 COPY --from=webui /apptoo /opt/hermes-webui
 COPY dockerfiles/hermes-hux-foundation/hux /opt/hermes-hux/hux
+COPY dockerfiles/hermes-hux-foundation/hux_producer /opt/hermes-hux/hux_producer
 COPY services/hermes/contracts/hux /opt/hermes-hux/contracts
 ENV HUX_CONTRACT_DIR=/opt/hermes-hux/contracts
 
@@ -72,7 +73,7 @@ RUN /opt/hermes/.venv/bin/python /tmp/hermes-webui-base-patch.py \
 RUN /opt/hermes/.venv/bin/python -c 'import cryptography, yaml' \
     && test -f /opt/hermes-hux/contracts/flags.json \
     && test -f /opt/hermes-hux/contracts/release-ledger.schema.json \
-    && PYTHONPATH=/opt/hermes-hux /opt/hermes/.venv/bin/python -m compileall -q /opt/hermes-hux/hux \
+    && PYTHONPATH=/opt/hermes-hux /opt/hermes/.venv/bin/python -m compileall -q /opt/hermes-hux/hux /opt/hermes-hux/hux_producer \
     && PYTHONPATH=/opt/hermes-hux /opt/hermes/.venv/bin/python -c 'from pathlib import Path; from tempfile import TemporaryDirectory; from hux.server import build_router; root = TemporaryDirectory(); router = build_router(Path(root.name), {"HUX_FLAGS": ""}); assert router.routes; root.cleanup()' \
     && grep -Fq 'VALID_REASONING_EFFORTS = ("minimal", "low", "medium", "high", "xhigh")' \
         /opt/hermes-webui/api/config.py \

services/hermes/chat-statefulset.yaml

@@ -260,6 +260,37 @@ spec:
           resources:
             requests: {cpu: 10m, memory: 16Mi}
             limits: {cpu: 50m, memory: 32Mi}
+        - name: stage-hux-evidence
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
+          imagePullPolicy: IfNotPresent
+          command: [/bin/sh, -ec]
+          args:
+            - |
+              # HUX-12 enablement is fail-closed: until the Vault annotation
+              # projects hux-evidence-key, nothing is staged and the evidence
+              # capability stays off. The key is staged only for the hux
+              # service and producer containers, never for hermes or webui.
+              if [ -s /vault/secrets/hux-evidence-key ]; then
+                umask 077
+                tr -d '\r\n' < /vault/secrets/hux-evidence-key > /hux-evidence/.evidence-key.tmp
+                chown 10000:10000 /hux-evidence/.evidence-key.tmp
+                chmod 0400 /hux-evidence/.evidence-key.tmp
+                mv /hux-evidence/.evidence-key.tmp /hux-evidence/evidence-key
+              fi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+              add: [CHOWN, DAC_OVERRIDE, FOWNER]
+            runAsUser: 0
+            runAsGroup: 0
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - {name: hux-evidence-key, mountPath: /hux-evidence}
+          resources:
+            requests: {cpu: 10m, memory: 16Mi}
+            limits: {cpu: 50m, memory: 32Mi}
         - name: stage-runtime-access
           image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
           imagePullPolicy: IfNotPresent
@@ -563,6 +594,8 @@ spec:
             - {name: HUX_IMAGE_TAG, value: 'git-91eb4f92b7cf46e65dc615106aedfde232c9c670-build-18-release'} # {"$imagepolicy": "hermes:hermes-webui-release:tag"}
             - {name: HUX_IMAGE_DIGEST, value: 'sha256:df91f8a3cdb54d685f8fea023a4539d89d1d0b3f89024cca4dce87b0b4a7df9c'} # {"$imagepolicy": "hermes:hermes-webui-release:digest"}
             - {name: HUX_SWITCHYARD_ROUTE_CATALOG, value: 'atlas/manual/codex/luna,atlas/manual/codex/terra,atlas/manual/codex/sol,atlas/manual/claude/haiku,atlas/manual/claude/fable,atlas/manual/claude/sonnet,atlas/manual/claude/opus,atlas/manual/local/qwen-14b'}
+            - {name: HUX_RELEASE_EVIDENCE_KEY_FILE, value: /run/hermes-hux-evidence/evidence-key}
+            - {name: HUX_RELEASE_EVIDENCE_POLICY_FILE, value: /etc/hux-evidence/policy.json}
             - {name: HUX_READS_PER_MINUTE, value: "600"}
             - {name: HUX_WRITES_PER_MINUTE, value: "120"}
             - {name: HUX_REQUEST_TIMEOUT_SECONDS, value: "10"}
@@ -571,6 +604,8 @@ spec:
             - {name: hux-relay-key, mountPath: /run/hermes-webui-hux, readOnly: true}
             - {name: hux-worker-key, mountPath: /run/hermes-hux-worker, readOnly: true}
             - {name: hux-tmp, mountPath: /tmp}
+            - {name: hux-evidence-key, mountPath: /run/hermes-hux-evidence, readOnly: true}
+            - {name: hux-evidence-policy, mountPath: /etc/hux-evidence, readOnly: true}
           readinessProbe:
             exec:
               command:
@@ -602,6 +637,63 @@ spec:
           resources:
             requests: {cpu: 25m, memory: 64Mi}
             limits: {cpu: 250m, memory: 256Mi}
+        - name: hux-evidence-producer
+          image: registry.bstein.dev/bstein/hermes-webui:git-91eb4f92b7cf46e65dc615106aedfde232c9c670-build-18-release@sha256:df91f8a3cdb54d685f8fea023a4539d89d1d0b3f89024cca4dce87b0b4a7df9c # {"$imagepolicy": "hermes:hermes-webui-release"}
+          imagePullPolicy: IfNotPresent
+          command: [/bin/sh, -ec]
+          args:
+            - |
+              ordinal="${HOSTNAME##*-}"
+              export HUX_TENANT_SLOT="slot-${ordinal}"
+              while true; do
+                if [ -s /run/hermes-hux-evidence/evidence-key ] \
+                  && [ -s /run/hermes-hux-subject/subject ] \
+                  && [ -n "${HUX_PRODUCER_PROJECT_ID:-}" ] \
+                  && [ -n "${HUX_PRODUCER_CONVERSATION_ID:-}" ]; then
+                  HUX_PRODUCER_SUBJECT="$(tr -d '\r\n' < /run/hermes-hux-subject/subject)" \
+                    /opt/hermes/.venv/bin/python -c \
+                    'from hux_producer import run_once; run_once()' || true
+                fi
+                sleep 60
+              done
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - {name: PYTHONPATH, value: /opt/hermes-hux}
+            - {name: PYTHONDONTWRITEBYTECODE, value: "1"}
+            - {name: HOME, value: /tmp}
+            - {name: HUX_BASE_URL, value: 'http://127.0.0.1:8790'}
+            - {name: HUX_PRODUCER_WORKLOAD, value: hermes-webui}
+            - {name: HUX_PRODUCER_NAMESPACE, value: hermes}
+            - {name: HUX_PRODUCER_POD_SELECTOR, value: 'app=hermes-chat-tenant'}
+            - {name: HUX_PRODUCER_WORKLOAD_KIND, value: statefulset}
+            - {name: HUX_PRODUCER_WORKLOAD_NAME, value: hermes-chat-tenant}
+            - {name: HUX_PRODUCER_TIMEOUT_SECONDS, value: "10"}
+            - {name: HUX_RELEASE_EVIDENCE_KEY_FILE, value: /run/hermes-hux-evidence/evidence-key}
+            - {name: HUX_RELEASE_EVIDENCE_POLICY_FILE, value: /etc/hux-evidence/policy.json}
+          envFrom:
+            - configMapRef:
+                name: hermes-hux-evidence-scope
+                optional: true
+          volumeMounts:
+            - {name: hux-evidence-key, mountPath: /run/hermes-hux-evidence, readOnly: true}
+            - {name: hux-evidence-policy, mountPath: /etc/hux-evidence, readOnly: true}
+            - {name: hux-data, mountPath: /run/hermes-hux-subject, subPathExpr: $(POD_NAME)/binding, readOnly: true}
+            - {name: hux-tmp, mountPath: /tmp}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile:
+              type: RuntimeDefault
+          resources:
+            requests: {cpu: 10m, memory: 48Mi}
+            limits: {cpu: 100m, memory: 128Mi}
         - name: telegram-media
           image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
           imagePullPolicy: IfNotPresent
@@ -692,6 +784,14 @@ spec:
         - name: hux-tmp
           emptyDir:
             sizeLimit: 64Mi
+        - name: hux-evidence-key
+          emptyDir:
+            medium: Memory
+            sizeLimit: 1Mi
+        - name: hux-evidence-policy
+          configMap:
+            name: hermes-hux-evidence-policy
+            defaultMode: 0444
         - name: hux-data
           persistentVolumeClaim:
             claimName: hermes-chat-hux-data

services/hermes/hux-evidence-policy.json

@@ -0,0 +1,13 @@
+{
+  "schema": "hux.release_evidence_policy.v1",
+  "max_evidence_age_seconds": 900,
+  "workloads": {
+    "hermes-webui": {
+      "review_url_prefix": "https://scm.bstein.dev/atlas/titan-iac/pulls/",
+      "jenkins_job_url": "https://jenkins.bstein.dev/job/hermes-webui-image",
+      "image_repository": "registry.bstein.dev/bstein/hermes-webui",
+      "flux_kustomization": "hermes",
+      "health_url": "https://chat.bstein.dev/health"
+    }
+  }
+}

services/hermes/hux-evidence-rbac.yaml

@@ -0,0 +1,54 @@
+# services/hermes/hux-evidence-rbac.yaml
+# Read-only facts for the HUX-12 evidence producer sidecar: Ready pod
+# imageIDs and the desired StatefulSet image in this namespace, plus the
+# single named Flux Kustomization's applied revision. The pod service
+# account gains nothing writable and nothing secret.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: hermes-hux-evidence-read
+  namespace: hermes
+rules:
+  - apiGroups: [""]
+    resources: [pods]
+    verbs: [get, list]
+  - apiGroups: [apps]
+    resources: [statefulsets]
+    verbs: [get]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: hermes-hux-evidence-read
+  namespace: hermes
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: hermes-hux-evidence-read
+subjects:
+  - kind: ServiceAccount
+    name: hermes-chat
+    namespace: hermes
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: hermes-hux-evidence-kustomization-read
+rules:
+  - apiGroups: [kustomize.toolkit.fluxcd.io]
+    resources: [kustomizations]
+    resourceNames: [hermes]
+    verbs: [get]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: hermes-hux-evidence-kustomization-read
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: hermes-hux-evidence-kustomization-read
+subjects:
+  - kind: ServiceAccount
+    name: hermes-chat
+    namespace: hermes

services/hermes/kustomization.yaml

@@ -43,6 +43,7 @@ resources:
   - agent-certificate.yaml
   - agent-ingress.yaml
   - execution-worker-rbac.yaml
+  - hux-evidence-rbac.yaml
   - execution-worker-statefulset.yaml
   - execution-mediator.yaml
   - execution-worker-networkpolicy.yaml
@@ -195,6 +196,12 @@ configMapGenerator:
       - dashboard-style.css=plugins/auto-router/dashboard/dist/style.css
     options:
       disableNameSuffixHash: true
+  - name: hermes-hux-evidence-policy
+    namespace: hermes
+    files:
+      - policy.json=hux-evidence-policy.json
+    options:
+      disableNameSuffixHash: true
   - name: hermes-hux-runtime-plugin
     namespace: hermes
     files:

services/hermes/networkpolicy.yaml

@@ -406,6 +406,22 @@ spec:
       ports:
         - {protocol: TCP, port: 9005}
         - {protocol: TCP, port: 9009}
+    # HUX-12 evidence producer: the Kubernetes API (ClusterIP) for pod and
+    # Flux facts, and the ingress edge for Jenkins/Harbor/health receipts.
+    - to:
+        - ipBlock:
+            cidr: 10.43.0.1/32
+      ports:
+        - {protocol: TCP, port: 443}
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: traefik
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: traefik
+      ports:
+        - {protocol: TCP, port: 443}
     - to:
         - podSelector:
             matchLabels:

testing/tests/test_hermes_hux_delivery.py

@@ -5,6 +5,7 @@ manifests must contain no partial HUX wiring at all; once the ``hux``
 sidecar exists, every boundary assertion below is enforced strictly.
 """
 
+import json
 from pathlib import Path
 
 import pytest
@@ -213,6 +214,46 @@ def test_hux_identity_and_authentication_inputs_are_file_backed() -> None:
     assert not {"HUX_RELAY_KEY", "HUX_WORKER_KEY", "HUX_ROUTER_KEY"} & set(hux)
 
 
+def test_hux_evidence_producer_is_isolated_and_fail_closed() -> None:
+    """The HUX-12 evidence key reaches only the service and the producer."""
+    _require_activation()
+    stateful = _statefulset()
+    pod = stateful["spec"]["template"]["spec"]
+    producer = _named(pod["containers"], "hux-evidence-producer")
+    hux = _named(pod["containers"], "hux")
+    hermes = _named(pod["containers"], "hermes")
+    webui = _named(pod["containers"], "webui")
+    media = _named(pod["containers"], "telegram-media")
+
+    assert producer["image"] == hux["image"]
+    values = _env(producer)
+    assert values["HUX_RELEASE_EVIDENCE_KEY_FILE"] == "/run/hermes-hux-evidence/evidence-key"
+    assert values["HUX_RELEASE_EVIDENCE_POLICY_FILE"] == "/etc/hux-evidence/policy.json"
+    assert values["HUX_PRODUCER_WORKLOAD"] == "hermes-webui"
+    assert _env(hux)["HUX_RELEASE_EVIDENCE_KEY_FILE"] == "/run/hermes-hux-evidence/evidence-key"
+    assert producer["securityContext"]["readOnlyRootFilesystem"] is True
+    assert producer["securityContext"]["capabilities"]["drop"] == ["ALL"]
+    for denied in (hermes, webui, media):
+        assert not any(
+            item["name"] == "hux-evidence-key" for item in denied.get("volumeMounts", [])
+        )
+    staging = _named(pod["initContainers"], "stage-hux-evidence")
+    script = staging["args"][0]
+    assert "if [ -s /vault/secrets/hux-evidence-key ]" in script
+    assert "chmod 0400" in script
+    volumes = {item["name"]: item for item in pod["volumes"]}
+    assert volumes["hux-evidence-key"]["emptyDir"]["medium"] == "Memory"
+    assert volumes["hux-evidence-policy"]["configMap"]["name"] == "hermes-hux-evidence-policy"
+    policy = json.loads((SERVICE / "hux-evidence-policy.json").read_text(encoding="utf-8"))
+    assert policy["schema"] == "hux.release_evidence_policy.v1"
+    assert set(policy["workloads"]) == {"hermes-webui"}
+    rbac = (SERVICE / "hux-evidence-rbac.yaml").read_text(encoding="utf-8")
+    assert "verbs: [get, list]" in rbac and "secrets" not in rbac
+    rendered = KUSTOMIZATION.read_text(encoding="utf-8")
+    assert "hux-evidence-rbac.yaml" in rendered
+    assert "policy.json=hux-evidence-policy.json" in rendered
+
+
 def test_hux_runtime_plugin_renders_its_vendored_hook_package() -> None:
     """The mounted plugin must contain its Kustomize-local Python package tree."""
     _require_activation()

testing/tests/test_hermes_image_automation.py

@@ -73,19 +73,19 @@ def test_flux_updates_only_the_reviewed_hermes_image_digests() -> None:
     assert agent.count('"$imagepolicy": "hermes:hermes-agent-release:digest"') == 1
     webui_marker = '"$imagepolicy": "hermes:hermes-webui-release"'
     chat_object = yaml.safe_load(chat)
-    hux = next(
+    containers = chat_object["spec"]["template"]["spec"]["containers"]
-        (
+    hux = next((item for item in containers if item["name"] == "hux"), None)
-            item
+    producer = next(
-            for item in chat_object["spec"]["template"]["spec"]["containers"]
+        (item for item in containers if item["name"] == "hux-evidence-producer"),
-            if item["name"] == "hux"
-        ),
         None,
     )
     # Before activation the chat StatefulSet carries exactly one WebUI
-    # consumer and no HUX setters; after activation the HUX sidecar is the
+    # consumer and no HUX setters; after activation the HUX sidecar (and the
-    # second consumer of the very same reviewed image line, with tag and
+    # HUX-12 evidence producer) consume the very same reviewed image line,
-    # digest setters binding its build metadata.
+    # with tag and digest setters binding the sidecar's build metadata.
-    assert chat.count(webui_marker) == (2 if hux else 1)
+    assert producer is None or hux is not None
+    consumers = 1 + (1 if hux else 0) + (1 if producer else 0)
+    assert chat.count(webui_marker) == consumers
     assert dashboard.count(webui_marker) == 1
     assert chat.count(
         '"$imagepolicy": "hermes:hermes-webui-release:tag"'
@@ -106,6 +106,8 @@ def test_flux_updates_only_the_reviewed_hermes_image_digests() -> None:
             "HUX_IMAGE_TAG"
         ].removeprefix("git-")
         assert hux["image"].endswith("@" + hux_env["HUX_IMAGE_DIGEST"])
+        if producer is not None:
+            assert producer["image"] == hux["image"]
     # A digest-only setter replaces the complete YAML scalar with ``sha256:...``.
     # Whole-image setters must retain the registry and repository in pod specs.
     for workload in (chat, dashboard):