feat(hermes-triage-demo): isolated demo surface + Jenkins job + Hermes key seeding

- New hermes-triage-demo namespace with fixture PVC and RBAC scoped to
  Jenkins agent Job creation only
- JCasC pipelineJob hermes-triage-demo: SEED_FAILURE-parameterized fixture
  check running as a Kubernetes Job in the demo namespace, emitting the
  incident ID to pod stdout (Fluent Bit -> OpenSearch kube-*) and JUnit to
  Jenkins
- Hermes init container can seed API_SERVER_KEY in the persistent .env
  from an optional hermes-api-server-key Secret (no-op until it exists)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

clusters/atlas/flux-system/applications/hermes-triage-demo/kustomization.yaml

@@ -0,0 +1,16 @@
+# clusters/atlas/flux-system/applications/hermes-triage-demo/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: hermes-triage-demo
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/hermes-triage-demo
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: false
+  timeout: 5m

clusters/atlas/flux-system/applications/kustomization.yaml

@@ -29,6 +29,7 @@ resources:
   - openclaw/kustomization.yaml
   - hermes/kustomization.yaml
   - hermes-chat/kustomization.yaml
+  - hermes-triage-demo/kustomization.yaml
   - game-stream/kustomization.yaml
   - cassandra-auth/kustomization.yaml
   - cassandra/kustomization.yaml

services/hermes-triage-demo/fixture-pvc.yaml

@@ -0,0 +1,17 @@
+# services/hermes-triage-demo/fixture-pvc.yaml
+# Single-file demo fixture: holds one state value ("healthy"/"unhealthy").
+# The Jenkins test-runner reads it; the Ariadne repair Job writes it back to healthy.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: hermes-triage-demo-fixture
+  namespace: hermes-triage-demo
+  labels:
+    app.kubernetes.io/part-of: hermes-triage-demo
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 100Mi

services/hermes-triage-demo/kustomization.yaml

@@ -0,0 +1,7 @@
+# services/hermes-triage-demo/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - fixture-pvc.yaml
+  - rbac.yaml

services/hermes-triage-demo/namespace.yaml

@@ -0,0 +1,10 @@
+# services/hermes-triage-demo/namespace.yaml
+# Isolated namespace for the Hermes automated-triage demo loop.
+# Nothing outside this namespace may be touched by the demo repair action.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: hermes-triage-demo
+  labels:
+    app.kubernetes.io/part-of: hermes-triage-demo
+    atlas.bstein.dev/purpose: automated-triage-demo

services/hermes-triage-demo/rbac.yaml

@@ -0,0 +1,38 @@
+# services/hermes-triage-demo/rbac.yaml
+# Lets Jenkins agent pods (SA jenkins/jenkins) create and observe the demo
+# test-runner Job in this namespace only. The Ariadne repair path needs no
+# grant here: its ariadne-job-spawner ClusterRole already covers Job
+# create/watch, and the repair pod itself only mounts the fixture PVC.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: demo-test-runner
+  namespace: hermes-triage-demo
+  labels:
+    app.kubernetes.io/part-of: hermes-triage-demo
+rules:
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["create", "get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods/log"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: demo-test-runner-jenkins
+  namespace: hermes-triage-demo
+  labels:
+    app.kubernetes.io/part-of: hermes-triage-demo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: demo-test-runner
+subjects:
+  - kind: ServiceAccount
+    name: jenkins
+    namespace: jenkins

services/hermes/deployment.yaml

@@ -77,6 +77,17 @@ spec:
         - name: init-config
           image: busybox:1.37
           imagePullPolicy: IfNotPresent
+          env:
+            # When the Flux/Vault-managed shared key Secret exists, it becomes
+            # the API_SERVER_KEY in the persistent .env (which overrides pod
+            # env at runtime). Optional: absent Secret keeps the old behavior
+            # of generating a random key on first boot.
+            - name: API_SERVER_KEY_SEED
+              valueFrom:
+                secretKeyRef:
+                  name: hermes-api-server-key
+                  key: api-key
+                  optional: true
           command:
             - sh
             - -c
@@ -95,6 +106,11 @@ spec:
               cp /guide/OPERATOR-RUNBOOK.md /opt/data/HERMES-OPERATOR-RUNBOOK.md
               cp /config/ATLAS-TRIAGE-PROOFS.md /opt/data/ATLAS-TRIAGE-PROOFS.md
               touch /opt/data/.env
+              if [ -n "${API_SERVER_KEY_SEED:-}" ]; then
+                grep -v '^API_SERVER_KEY=' /opt/data/.env > /opt/data/.env.tmp || true
+                printf 'API_SERVER_KEY=%s\n' "${API_SERVER_KEY_SEED}" >> /opt/data/.env.tmp
+                mv /opt/data/.env.tmp /opt/data/.env
+              fi
               if ! grep -q '^API_SERVER_KEY=' /opt/data/.env; then
                 api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')"
                 printf '\nAPI_SERVER_KEY=%s\n' "${api_key}" >> /opt/data/.env

services/jenkins/configmap-jcasc.yaml

@@ -447,6 +447,142 @@ data:
               }
             }
           }
+          pipelineJob('hermes-triage-demo') {
+            disabled(false)
+            description('Hermes automated-triage demo: runs a deterministic fixture check as a Kubernetes Job in the hermes-triage-demo namespace. Build with SEED_FAILURE=true to arm the demo failure; the automated repair loop rebuilds with SEED_FAILURE=false.')
+            parameters {
+              booleanParam('SEED_FAILURE', false, 'Write "unhealthy" into the demo fixture before the check (arms the demo failure).')
+            }
+            definition {
+              cps {
+                sandbox(true)
+                script('''
+                  podTemplate(
+                    cloud: 'kubernetes',
+                    yaml: """
+                  apiVersion: v1
+                  kind: Pod
+                  spec:
+                    serviceAccountName: jenkins
+                    securityContext:
+                      runAsUser: 1001
+                      runAsGroup: 1001
+                    nodeSelector:
+                      kubernetes.io/arch: arm64
+                      node-role.kubernetes.io/worker: "true"
+                    containers:
+                      - name: kubectl
+                        image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
+                        command: ["sleep"]
+                        args: ["3600"]
+                        resources:
+                          requests:
+                            cpu: 50m
+                            memory: 64Mi
+                          limits:
+                            cpu: 250m
+                            memory: 128Mi
+                  """
+                  ) {
+                    node(POD_LABEL) {
+                      container('kubectl') {
+                        stage('Fixture check') {
+                          def buildNum = env.BUILD_NUMBER
+                          def incidentId = "hermes-triage-demo/${buildNum}"
+                          def jobName = "hermes-demo-test-${buildNum}"
+                          def seed = params.SEED_FAILURE ? "true" : "false"
+                          writeFile file: 'test-runner-job.yaml', text: """
+                  apiVersion: batch/v1
+                  kind: Job
+                  metadata:
+                    name: ${jobName}
+                    namespace: hermes-triage-demo
+                    labels:
+                      app.kubernetes.io/part-of: hermes-triage-demo
+                      atlas.bstein.dev/role: demo-test-runner
+                  spec:
+                    backoffLimit: 0
+                    ttlSecondsAfterFinished: 3600
+                    template:
+                      metadata:
+                        labels:
+                          app.kubernetes.io/part-of: hermes-triage-demo
+                          atlas.bstein.dev/role: demo-test-runner
+                      spec:
+                        restartPolicy: Never
+                        containers:
+                          - name: test-runner
+                            image: busybox:1.37
+                            env:
+                              - name: INCIDENT_ID
+                                value: "${incidentId}"
+                              - name: SEED_FAILURE
+                                value: "${seed}"
+                            command:
+                              - sh
+                              - -c
+                              - |
+                                STATE_FILE=/fixture/state
+                                if [ "\\$SEED_FAILURE" = "true" ]; then
+                                  echo unhealthy > "\\$STATE_FILE"
+                                fi
+                                if [ ! -f "\\$STATE_FILE" ]; then
+                                  echo healthy > "\\$STATE_FILE"
+                                fi
+                                STATE=\\$(cat "\\$STATE_FILE")
+                                if [ "\\$STATE" = "healthy" ]; then
+                                  echo "{\\\\"event\\\\":\\\\"hermes_demo_test_pass\\\\",\\\\"incident_id\\\\":\\\\"\\$INCIDENT_ID\\\\",\\\\"message\\\\":\\\\"fixture state healthy\\\\"}"
+                                  exit 0
+                                fi
+                                echo "{\\\\"event\\\\":\\\\"hermes_demo_test_failure\\\\",\\\\"incident_id\\\\":\\\\"\\$INCIDENT_ID\\\\",\\\\"classification_hint\\\\":\\\\"demo_fixture_unhealthy\\\\",\\\\"message\\\\":\\\\"expected fixture state healthy; found \\$STATE\\\\"}"
+                                exit 1
+                            volumeMounts:
+                              - name: fixture
+                                mountPath: /fixture
+                            resources:
+                              requests:
+                                cpu: 25m
+                                memory: 16Mi
+                              limits:
+                                cpu: 100m
+                                memory: 32Mi
+                        volumes:
+                          - name: fixture
+                            persistentVolumeClaim:
+                              claimName: hermes-triage-demo-fixture
+                  """
+                          sh "kubectl -n hermes-triage-demo create -f test-runner-job.yaml"
+                          def verdict = "unknown"
+                          timeout(time: 3, unit: 'MINUTES') {
+                            waitUntil(initialRecurrencePeriod: 5000) {
+                              def st = sh(script: "kubectl -n hermes-triage-demo get job ${jobName} -o jsonpath='{.status.succeeded}:{.status.failed}'", returnStdout: true).trim()
+                              if (st.startsWith("1")) { verdict = "pass"; return true }
+                              if (st.endsWith(":1")) { verdict = "fail"; return true }
+                              return false
+                            }
+                          }
+                          def runnerLog = sh(script: "kubectl -n hermes-triage-demo logs job/${jobName} --tail=20 || true", returnStdout: true).trim()
+                          echo "test-runner output:\\n${runnerLog}"
+                          def junitBody
+                          if (verdict == "pass") {
+                            junitBody = """<testsuite name="hermes-triage-demo" tests="1" failures="0"><testcase classname="hermes-triage-demo" name="fixture-state-check"/></testsuite>"""
+                          } else {
+                            def safeLog = runnerLog.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+                            junitBody = """<testsuite name="hermes-triage-demo" tests="1" failures="1"><testcase classname="hermes-triage-demo" name="fixture-state-check"><failure message="demo fixture unhealthy">${safeLog}</failure></testcase></testsuite>"""
+                          }
+                          writeFile file: 'demo-junit.xml', text: junitBody
+                          junit 'demo-junit.xml'
+                          if (verdict != "pass") {
+                            error("Demo fixture check failed for incident ${incidentId}")
+                          }
+                        }
+                      }
+                    }
+                  }
+                ''')
+              }
+            }
+          }
           multibranchPipelineJob('titan-iac-quality-gate') {
             branchSources {
               branchSource {