diff --git a/clusters/atlas/flux-system/applications/hermes-triage-demo/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-triage-demo/kustomization.yaml new file mode 100644 index 000000000..2f921d310 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-triage-demo/kustomization.yaml @@ -0,0 +1,16 @@ +# clusters/atlas/flux-system/applications/hermes-triage-demo/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-triage-demo + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-triage-demo + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: false + timeout: 5m diff --git a/clusters/atlas/flux-system/applications/kustomization.yaml b/clusters/atlas/flux-system/applications/kustomization.yaml index 671f019b8..eb8bc86ae 100644 --- a/clusters/atlas/flux-system/applications/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/kustomization.yaml @@ -29,6 +29,7 @@ resources: - openclaw/kustomization.yaml - hermes/kustomization.yaml - hermes-chat/kustomization.yaml + - hermes-triage-demo/kustomization.yaml - game-stream/kustomization.yaml - cassandra-auth/kustomization.yaml - cassandra/kustomization.yaml diff --git a/services/hermes-triage-demo/fixture-pvc.yaml b/services/hermes-triage-demo/fixture-pvc.yaml new file mode 100644 index 000000000..8827fd8b1 --- /dev/null +++ b/services/hermes-triage-demo/fixture-pvc.yaml @@ -0,0 +1,17 @@ +# services/hermes-triage-demo/fixture-pvc.yaml +# Single-file demo fixture: holds one state value ("healthy"/"unhealthy"). +# The Jenkins test-runner reads it; the Ariadne repair Job writes it back to healthy. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: hermes-triage-demo-fixture + namespace: hermes-triage-demo + labels: + app.kubernetes.io/part-of: hermes-triage-demo +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 100Mi diff --git a/services/hermes-triage-demo/kustomization.yaml b/services/hermes-triage-demo/kustomization.yaml new file mode 100644 index 000000000..40027dcec --- /dev/null +++ b/services/hermes-triage-demo/kustomization.yaml @@ -0,0 +1,7 @@ +# services/hermes-triage-demo/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - fixture-pvc.yaml + - rbac.yaml diff --git a/services/hermes-triage-demo/namespace.yaml b/services/hermes-triage-demo/namespace.yaml new file mode 100644 index 000000000..a501069b2 --- /dev/null +++ b/services/hermes-triage-demo/namespace.yaml @@ -0,0 +1,10 @@ +# services/hermes-triage-demo/namespace.yaml +# Isolated namespace for the Hermes automated-triage demo loop. +# Nothing outside this namespace may be touched by the demo repair action. +apiVersion: v1 +kind: Namespace +metadata: + name: hermes-triage-demo + labels: + app.kubernetes.io/part-of: hermes-triage-demo + atlas.bstein.dev/purpose: automated-triage-demo diff --git a/services/hermes-triage-demo/rbac.yaml b/services/hermes-triage-demo/rbac.yaml new file mode 100644 index 000000000..174f60923 --- /dev/null +++ b/services/hermes-triage-demo/rbac.yaml @@ -0,0 +1,38 @@ +# services/hermes-triage-demo/rbac.yaml +# Lets Jenkins agent pods (SA jenkins/jenkins) create and observe the demo +# test-runner Job in this namespace only. The Ariadne repair path needs no +# grant here: its ariadne-job-spawner ClusterRole already covers Job +# create/watch, and the repair pod itself only mounts the fixture PVC. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: demo-test-runner + namespace: hermes-triage-demo + labels: + app.kubernetes.io/part-of: hermes-triage-demo +rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: demo-test-runner-jenkins + namespace: hermes-triage-demo + labels: + app.kubernetes.io/part-of: hermes-triage-demo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: demo-test-runner +subjects: + - kind: ServiceAccount + name: jenkins + namespace: jenkins diff --git a/services/hermes/deployment.yaml b/services/hermes/deployment.yaml index 247d947cb..7a49900c2 100644 --- a/services/hermes/deployment.yaml +++ b/services/hermes/deployment.yaml @@ -77,6 +77,17 @@ spec: - name: init-config image: busybox:1.37 imagePullPolicy: IfNotPresent + env: + # When the Flux/Vault-managed shared key Secret exists, it becomes + # the API_SERVER_KEY in the persistent .env (which overrides pod + # env at runtime). Optional: absent Secret keeps the old behavior + # of generating a random key on first boot. + - name: API_SERVER_KEY_SEED + valueFrom: + secretKeyRef: + name: hermes-api-server-key + key: api-key + optional: true command: - sh - -c @@ -95,6 +106,11 @@ spec: cp /guide/OPERATOR-RUNBOOK.md /opt/data/HERMES-OPERATOR-RUNBOOK.md cp /config/ATLAS-TRIAGE-PROOFS.md /opt/data/ATLAS-TRIAGE-PROOFS.md touch /opt/data/.env + if [ -n "${API_SERVER_KEY_SEED:-}" ]; then + grep -v '^API_SERVER_KEY=' /opt/data/.env > /opt/data/.env.tmp || true + printf 'API_SERVER_KEY=%s\n' "${API_SERVER_KEY_SEED}" >> /opt/data/.env.tmp + mv /opt/data/.env.tmp /opt/data/.env + fi if ! grep -q '^API_SERVER_KEY=' /opt/data/.env; then api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')" printf '\nAPI_SERVER_KEY=%s\n' "${api_key}" >> /opt/data/.env diff --git a/services/jenkins/configmap-jcasc.yaml b/services/jenkins/configmap-jcasc.yaml index 0bee610d7..57a07cf11 100644 --- a/services/jenkins/configmap-jcasc.yaml +++ b/services/jenkins/configmap-jcasc.yaml @@ -447,6 +447,142 @@ data: } } } + pipelineJob('hermes-triage-demo') { + disabled(false) + description('Hermes automated-triage demo: runs a deterministic fixture check as a Kubernetes Job in the hermes-triage-demo namespace. Build with SEED_FAILURE=true to arm the demo failure; the automated repair loop rebuilds with SEED_FAILURE=false.') + parameters { + booleanParam('SEED_FAILURE', false, 'Write "unhealthy" into the demo fixture before the check (arms the demo failure).') + } + definition { + cps { + sandbox(true) + script(''' + podTemplate( + cloud: 'kubernetes', + yaml: """ + apiVersion: v1 + kind: Pod + spec: + serviceAccountName: jenkins + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + nodeSelector: + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + containers: + - name: kubectl + image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 + command: ["sleep"] + args: ["3600"] + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 250m + memory: 128Mi + """ + ) { + node(POD_LABEL) { + container('kubectl') { + stage('Fixture check') { + def buildNum = env.BUILD_NUMBER + def incidentId = "hermes-triage-demo/${buildNum}" + def jobName = "hermes-demo-test-${buildNum}" + def seed = params.SEED_FAILURE ? "true" : "false" + writeFile file: 'test-runner-job.yaml', text: """ + apiVersion: batch/v1 + kind: Job + metadata: + name: ${jobName} + namespace: hermes-triage-demo + labels: + app.kubernetes.io/part-of: hermes-triage-demo + atlas.bstein.dev/role: demo-test-runner + spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + app.kubernetes.io/part-of: hermes-triage-demo + atlas.bstein.dev/role: demo-test-runner + spec: + restartPolicy: Never + containers: + - name: test-runner + image: busybox:1.37 + env: + - name: INCIDENT_ID + value: "${incidentId}" + - name: SEED_FAILURE + value: "${seed}" + command: + - sh + - -c + - | + STATE_FILE=/fixture/state + if [ "\\$SEED_FAILURE" = "true" ]; then + echo unhealthy > "\\$STATE_FILE" + fi + if [ ! -f "\\$STATE_FILE" ]; then + echo healthy > "\\$STATE_FILE" + fi + STATE=\\$(cat "\\$STATE_FILE") + if [ "\\$STATE" = "healthy" ]; then + echo "{\\\\"event\\\\":\\\\"hermes_demo_test_pass\\\\",\\\\"incident_id\\\\":\\\\"\\$INCIDENT_ID\\\\",\\\\"message\\\\":\\\\"fixture state healthy\\\\"}" + exit 0 + fi + echo "{\\\\"event\\\\":\\\\"hermes_demo_test_failure\\\\",\\\\"incident_id\\\\":\\\\"\\$INCIDENT_ID\\\\",\\\\"classification_hint\\\\":\\\\"demo_fixture_unhealthy\\\\",\\\\"message\\\\":\\\\"expected fixture state healthy; found \\$STATE\\\\"}" + exit 1 + volumeMounts: + - name: fixture + mountPath: /fixture + resources: + requests: + cpu: 25m + memory: 16Mi + limits: + cpu: 100m + memory: 32Mi + volumes: + - name: fixture + persistentVolumeClaim: + claimName: hermes-triage-demo-fixture + """ + sh "kubectl -n hermes-triage-demo create -f test-runner-job.yaml" + def verdict = "unknown" + timeout(time: 3, unit: 'MINUTES') { + waitUntil(initialRecurrencePeriod: 5000) { + def st = sh(script: "kubectl -n hermes-triage-demo get job ${jobName} -o jsonpath='{.status.succeeded}:{.status.failed}'", returnStdout: true).trim() + if (st.startsWith("1")) { verdict = "pass"; return true } + if (st.endsWith(":1")) { verdict = "fail"; return true } + return false + } + } + def runnerLog = sh(script: "kubectl -n hermes-triage-demo logs job/${jobName} --tail=20 || true", returnStdout: true).trim() + echo "test-runner output:\\n${runnerLog}" + def junitBody + if (verdict == "pass") { + junitBody = """""" + } else { + def safeLog = runnerLog.replace("&", "&").replace("<", "<").replace(">", ">") + junitBody = """${safeLog}""" + } + writeFile file: 'demo-junit.xml', text: junitBody + junit 'demo-junit.xml' + if (verdict != "pass") { + error("Demo fixture check failed for incident ${incidentId}") + } + } + } + } + } + ''') + } + } + } multibranchPipelineJob('titan-iac-quality-gate') { branchSources { branchSource {