vault(consumption): sync secrets via CSI

infrastructure/vault-csi/secrets-store-csi-driver.yaml

@@ -16,5 +16,5 @@ spec:
         namespace: flux-system
   values:
     syncSecret:
-      enabled: false
+      enabled: true
     enableSecretRotation: false

services/comms/atlasbot-deployment.yaml

@@ -27,7 +27,8 @@ spec:
           command: ["/bin/sh","-c"]
           args:
             - |
-              python /app/bot.py
+              . /vault/scripts/comms_vault_env.sh
+              exec python /app/bot.py
           env:
             - name: MATRIX_BASE
               value: http://othrys-synapse-matrix-synapse:8008
@@ -39,16 +40,6 @@ spec:
               value: http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428
             - name: BOT_USER
               value: atlasbot
-            - name: BOT_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: atlasbot-credentials-runtime
-                  key: bot-password
-            - name: CHAT_API_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: chat-ai-keys-runtime
-                  key: matrix
             - name: OLLAMA_URL
               value: https://chat.ai.bstein.dev/
             - name: OLLAMA_MODEL
@@ -67,6 +58,12 @@ spec:
             - name: kb
               mountPath: /kb
               readOnly: true
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
       volumes:
         - name: code
           configMap:
@@ -85,3 +82,13 @@ spec:
                 path: catalog/runbooks.json
               - key: atlas-http.mmd
                 path: diagrams/atlas-http.mmd
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: comms-vault
+        - name: vault-scripts
+          configMap:
+            name: comms-vault-env
+            defaultMode: 0555

services/comms/bstein-force-leave-job.yaml

@@ -9,25 +9,26 @@ spec:
   template:
     spec:
       restartPolicy: Never
+      serviceAccountName: comms-vault
       volumes:
-        - name: mas-admin-client
+        - name: vault-secrets
-          secret:
+          csi:
-            secretName: mas-admin-client-runtime
+            driver: secrets-store.csi.k8s.io
-            items:
+            readOnly: true
-              - key: client_secret
+            volumeAttributes:
-                path: client_secret
+              secretProviderClass: comms-vault
       containers:
         - name: leave
           image: python:3.11-slim
           volumeMounts:
-            - name: mas-admin-client
+            - name: vault-secrets
-              mountPath: /etc/mas-admin-client
+              mountPath: /vault/secrets
               readOnly: true
           env:
             - name: MAS_ADMIN_CLIENT_ID
               value: 01KDXMVQBQ5JNY6SEJPZW6Z8BM
             - name: MAS_ADMIN_CLIENT_SECRET_FILE
-              value: /etc/mas-admin-client/client_secret
+              value: /vault/secrets/mas-admin-client-runtime__client_secret
             - name: MAS_TOKEN_URL
               value: http://matrix-authentication-service:8080/oauth2/token
             - name: MAS_ADMIN_API_BASE

services/comms/comms-secrets-ensure-job.yaml

@@ -20,73 +20,58 @@ spec:
               set -eu
               trap 'echo "comms-secrets-ensure failed"; sleep 300' ERR
               umask 077
+              apk add --no-cache curl jq >/dev/null
 
               safe_pass() {
                 head -c 32 /dev/urandom | base64 | tr -d '\n' | tr '+/' '-_' | tr -d '='
               }
 
-              get_secret_value() {
+              vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
-                ns="$1"
+              vault_role="${VAULT_ROLE:-comms-secrets}"
-                name="$2"
+              jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
-                key="$3"
+              login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
-                kubectl -n "${ns}" get secret "${name}" -o "jsonpath={.data.${key}}" 2>/dev/null | base64 -d 2>/dev/null || true
+              vault_token="$(curl -sS --request POST --data "${login_payload}" \
-              }
+                "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
-
+              if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
-              ensure_secret_key() {
+                echo "vault login failed" >&2
-                ns="$1"
+                exit 1
-                name="$2"
-                key="$3"
-                value="$4"
-                if ! kubectl -n "${ns}" get secret "${name}" >/dev/null 2>&1; then
-                  kubectl -n "${ns}" create secret generic "${name}" --from-literal="${key}=${value}" >/dev/null
-                  return
-                fi
-                existing="$(kubectl -n "${ns}" get secret "${name}" -o "jsonpath={.data.${key}}" 2>/dev/null || true)"
-                if [ -z "${existing}" ]; then
-                  b64="$(printf '%s' "${value}" | base64 | tr -d '\n')"
-                  payload="$(printf '{"data":{"%s":"%s"}}' "${key}" "${b64}")"
-                  kubectl -n "${ns}" patch secret "${name}" --type=merge -p "${payload}" >/dev/null
-                fi
-              }
-
-              ensure_chat_secret() {
-                ns="$1"
-                if ! kubectl -n "${ns}" get secret chat-ai-keys-runtime >/dev/null 2>&1; then
-                  kubectl -n "${ns}" create secret generic chat-ai-keys-runtime \
-                    --from-literal=matrix="${CHAT_KEY_MATRIX}" \
-                    --from-literal=homepage="${CHAT_KEY_HOMEPAGE}" >/dev/null
-                  return
-                fi
-                ensure_secret_key "${ns}" chat-ai-keys-runtime matrix "${CHAT_KEY_MATRIX}"
-                ensure_secret_key "${ns}" chat-ai-keys-runtime homepage "${CHAT_KEY_HOMEPAGE}"
-              }
-
-              CHAT_KEY_MATRIX="$(get_secret_value comms chat-ai-keys-runtime matrix)"
-              CHAT_KEY_HOMEPAGE="$(get_secret_value comms chat-ai-keys-runtime homepage)"
-              if [ -z "${CHAT_KEY_MATRIX}" ] || [ -z "${CHAT_KEY_HOMEPAGE}" ]; then
-                ALT_MATRIX="$(get_secret_value bstein-dev-home chat-ai-keys-runtime matrix)"
-                ALT_HOMEPAGE="$(get_secret_value bstein-dev-home chat-ai-keys-runtime homepage)"
-                [ -z "${CHAT_KEY_MATRIX}" ] && CHAT_KEY_MATRIX="${ALT_MATRIX}"
-                [ -z "${CHAT_KEY_HOMEPAGE}" ] && CHAT_KEY_HOMEPAGE="${ALT_HOMEPAGE}"
               fi
-              [ -z "${CHAT_KEY_MATRIX}" ] && CHAT_KEY_MATRIX="$(safe_pass)"
-              [ -z "${CHAT_KEY_HOMEPAGE}" ] && CHAT_KEY_HOMEPAGE="$(safe_pass)"
 
-              ensure_chat_secret comms
+              vault_read() {
-              ensure_chat_secret bstein-dev-home
+                path="$1"
+                key="$2"
+                curl -sS -H "X-Vault-Token: ${vault_token}" \
+                  "${vault_addr}/v1/kv/data/atlas/${path}" | jq -r --arg key "${key}" '.data.data[$key] // empty'
+              }
 
-              ensure_secret_key comms turn-shared-secret TURN_STATIC_AUTH_SECRET "$(safe_pass)"
+              vault_write() {
-              ensure_secret_key comms livekit-api primary "$(safe_pass)"
+                path="$1"
-              ensure_secret_key comms synapse-redis redis-password "$(safe_pass)"
+                key="$2"
-              ensure_secret_key comms synapse-macaroon macaroon_secret_key "$(safe_pass)"
+                value="$3"
-              ensure_secret_key comms atlasbot-credentials-runtime bot-password "$(safe_pass)"
+                payload="$(jq -nc --arg key "${key}" --arg value "${value}" '{data:{($key):$value}}')"
-              ensure_secret_key comms atlasbot-credentials-runtime seeder-password "$(safe_pass)"
+                curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
+                  -d "${payload}" "${vault_addr}/v1/kv/data/atlas/${path}" >/dev/null
+              }
 
-              SYN_PASS="$(get_secret_value comms synapse-db POSTGRES_PASSWORD)"
+              ensure_key() {
-              if [ -z "${SYN_PASS}" ]; then
+                path="$1"
-                SYN_PASS="$(safe_pass)"
+                key="$2"
-                kubectl -n comms create secret generic synapse-db --from-literal=POSTGRES_PASSWORD="${SYN_PASS}" >/dev/null
+                current="$(vault_read "${path}" "${key}")"
-              fi
+                if [ -z "${current}" ]; then
+                  current="$(safe_pass)"
+                  vault_write "${path}" "${key}" "${current}"
+                fi
+                printf '%s' "${current}"
+              }
+
+              ensure_key "comms/turn-shared-secret" "TURN_STATIC_AUTH_SECRET" >/dev/null
+              ensure_key "comms/livekit-api" "primary" >/dev/null
+              ensure_key "comms/synapse-redis" "redis-password" >/dev/null
+              ensure_key "comms/synapse-macaroon" "macaroon_secret_key" >/dev/null
+              ensure_key "comms/atlasbot-credentials-runtime" "bot-password" >/dev/null
+              ensure_key "comms/atlasbot-credentials-runtime" "seeder-password" >/dev/null
+
+              SYN_PASS="$(ensure_key "comms/synapse-db" "POSTGRES_PASSWORD")"
 
               POD_NAME="$(kubectl -n postgres get pods -l app=postgres -o jsonpath='{.items[0].metadata.name}')"
               if [ -z "${POD_NAME}" ]; then

services/comms/coturn.yaml

@@ -15,6 +15,7 @@ spec:
       labels:
         app: coturn
     spec:
+      serviceAccountName: comms-vault
       nodeSelector:
         hardware: rpi5
       affinity:
@@ -33,6 +34,7 @@ spec:
             - /bin/sh
             - -c
             - |
+              . /vault/scripts/comms_vault_env.sh
               exec /usr/bin/turnserver \
                 --no-cli \
                 --fingerprint \
@@ -57,11 +59,6 @@ spec:
                   fieldPath: status.podIP
             - name: TURN_PUBLIC_IP
               value: "38.28.125.112"
-            - name: TURN_STATIC_AUTH_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: turn-shared-secret
-                  key: TURN_STATIC_AUTH_SECRET
           ports:
             - name: turn-udp
               containerPort: 3478
@@ -76,6 +73,12 @@ spec:
             - name: tls
               mountPath: /etc/coturn/tls
               readOnly: true
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
           resources:
             requests:
               cpu: 200m
@@ -87,6 +90,16 @@ spec:
         - name: tls
           secret:
             secretName: turn-live-tls
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: comms-vault
+        - name: vault-scripts
+          configMap:
+            name: comms-vault-env
+            defaultMode: 0555
 ---
 apiVersion: v1
 kind: Service

services/comms/guest-name-job.yaml

@@ -16,19 +16,27 @@ spec:
       template:
         spec:
           restartPolicy: Never
+          serviceAccountName: comms-vault
           volumes:
-            - name: mas-admin-client
+            - name: vault-secrets
-              secret:
+              csi:
-                secretName: mas-admin-client-runtime
+                driver: secrets-store.csi.k8s.io
-                items:
+                readOnly: true
-                  - key: client_secret
+                volumeAttributes:
-                    path: client_secret
+                  secretProviderClass: comms-vault
+            - name: vault-scripts
+              configMap:
+                name: comms-vault-env
+                defaultMode: 0555
           containers:
             - name: rename
               image: python:3.11-slim
               volumeMounts:
-                - name: mas-admin-client
+                - name: vault-secrets
-                  mountPath: /etc/mas-admin-client
+                  mountPath: /vault/secrets
+                  readOnly: true
+                - name: vault-scripts
+                  mountPath: /vault/scripts
                   readOnly: true
               env:
                 - name: SYNAPSE_BASE
@@ -36,7 +44,7 @@ spec:
                 - name: MAS_ADMIN_CLIENT_ID
                   value: 01KDXMVQBQ5JNY6SEJPZW6Z8BM
                 - name: MAS_ADMIN_CLIENT_SECRET_FILE
-                  value: /etc/mas-admin-client/client_secret
+                  value: /vault/secrets/mas-admin-client-runtime__client_secret
                 - name: MAS_ADMIN_API_BASE
                   value: http://matrix-authentication-service:8081/api/admin/v1
                 - name: MAS_TOKEN_URL
@@ -51,16 +59,12 @@ spec:
                   value: synapse
                 - name: PGUSER
                   value: synapse
-                - name: PGPASSWORD
-                  valueFrom:
-                    secretKeyRef:
-                      name: synapse-db
-                      key: POSTGRES_PASSWORD
               command:
                 - /bin/sh
                 - -c
                 - |
                   set -euo pipefail
+                  . /vault/scripts/comms_vault_env.sh
                   pip install --no-cache-dir requests psycopg2-binary >/dev/null
                   python - <<'PY'
                   import base64

services/comms/guest-register-deployment.yaml

@@ -17,6 +17,7 @@ spec:
       labels:
         app.kubernetes.io/name: matrix-guest-register
     spec:
+      serviceAccountName: comms-vault
       securityContext:
         runAsNonRoot: true
         runAsUser: 10001
@@ -42,7 +43,7 @@ spec:
             - name: MAS_ADMIN_CLIENT_ID
               value: 01KDXMVQBQ5JNY6SEJPZW6Z8BM
             - name: MAS_ADMIN_CLIENT_SECRET_FILE
-              value: /etc/mas/admin-client/client_secret
+              value: /vault/secrets/mas-admin-client-runtime__client_secret
             - name: MAS_ADMIN_API_BASE
               value: http://matrix-authentication-service:8081/api/admin/v1
             - name: SYNAPSE_BASE
@@ -83,8 +84,8 @@ spec:
               mountPath: /app/server.py
               subPath: server.py
               readOnly: true
-            - name: mas-admin-client
+            - name: vault-secrets
-              mountPath: /etc/mas/admin-client
+              mountPath: /vault/secrets
               readOnly: true
           command:
             - python
@@ -96,9 +97,9 @@ spec:
             items:
               - key: server.py
                 path: server.py
-        - name: mas-admin-client
+        - name: vault-secrets
-          secret:
+          csi:
-            secretName: mas-admin-client-runtime
+            driver: secrets-store.csi.k8s.io
-            items:
+            readOnly: true
-              - key: client_secret
+            volumeAttributes:
-                path: client_secret
+              secretProviderClass: comms-vault

services/comms/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 namespace: comms
 resources:
   - namespace.yaml
+  - serviceaccount.yaml
+  - secretproviderclass.yaml
   - mas-configmap.yaml
   - helmrelease.yaml
   - livekit-config.yaml
@@ -18,6 +20,7 @@ resources:
   - comms-secrets-ensure-rbac.yaml
   - mas-db-ensure-rbac.yaml
   - synapse-signingkey-ensure-rbac.yaml
+  - vault-sync-deployment.yaml
   - mas-admin-client-secret-ensure-job.yaml
   - mas-db-ensure-job.yaml
   - comms-secrets-ensure-job.yaml
@@ -40,6 +43,11 @@ resources:
   - matrix-ingress.yaml
 
 configMapGenerator:
+  - name: comms-vault-env
+    files:
+      - comms_vault_env.sh=scripts/comms_vault_env.sh
+    options:
+      disableNameSuffixHash: true
   - name: matrix-guest-register
     files:
       - server.py=scripts/guest-register/server.py

services/comms/livekit-token-deployment.yaml

@@ -15,6 +15,7 @@ spec:
       labels:
         app: livekit-token-service
     spec:
+      serviceAccountName: comms-vault
       nodeSelector:
         hardware: rpi5
       affinity:
@@ -33,21 +34,29 @@ spec:
       containers:
         - name: token-service
           image: ghcr.io/element-hq/lk-jwt-service:0.3.0
+          command:
+            - /bin/sh
+            - -c
+            - |
+              . /vault/scripts/comms_vault_env.sh
+              exec /lk-jwt-service
           env:
             - name: LIVEKIT_URL
               value: wss://kit.live.bstein.dev/livekit/sfu
             - name: LIVEKIT_KEY
               value: primary
-            - name: LIVEKIT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: livekit-api
-                  key: primary
             - name: LIVEKIT_FULL_ACCESS_HOMESERVERS
               value: live.bstein.dev
           ports:
             - containerPort: 8080
               name: http
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
           resources:
             requests:
               cpu: 50m
@@ -55,6 +64,17 @@ spec:
             limits:
               cpu: 300m
               memory: 256Mi
+      volumes:
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: comms-vault
+        - name: vault-scripts
+          configMap:
+            name: comms-vault-env
+            defaultMode: 0555
 ---
 apiVersion: v1
 kind: Service

services/comms/livekit.yaml

@@ -17,6 +17,7 @@ spec:
       labels:
         app: livekit
     spec:
+      serviceAccountName: comms-vault
       enableServiceLinks: false
       nodeSelector:
         hardware: rpi5
@@ -36,16 +37,11 @@ spec:
           args:
             - |
               set -euo pipefail
+              . /vault/scripts/comms_vault_env.sh
               umask 077
               TURN_PASSWORD_ESCAPED="$(printf '%s' "${TURN_PASSWORD}" | sed 's/[\\/&]/\\&/g')"
               sed "s/@@TURN_PASSWORD@@/${TURN_PASSWORD_ESCAPED}/g" /etc/livekit-template/livekit.yaml > /etc/livekit/livekit.yaml
               chmod 0644 /etc/livekit/livekit.yaml
-          env:
-            - name: TURN_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: turn-shared-secret
-                  key: TURN_STATIC_AUTH_SECRET
           volumeMounts:
             - name: config-template
               mountPath: /etc/livekit-template
@@ -53,6 +49,12 @@ spec:
             - name: config
               mountPath: /etc/livekit
               readOnly: false
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
       containers:
         - name: livekit
           image: livekit/livekit-server:v1.9.0
@@ -61,6 +63,7 @@ spec:
             - -c
             - |
               set -euo pipefail
+              . /vault/scripts/comms_vault_env.sh
               umask 077
               printf "%s: %s\n" "${LIVEKIT_API_KEY_ID}" "${LIVEKIT_API_SECRET}" > /var/run/livekit/keys
               chmod 600 /var/run/livekit/keys
@@ -68,11 +71,6 @@ spec:
           env:
             - name: LIVEKIT_API_KEY_ID
               value: primary
-            - name: LIVEKIT_API_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: livekit-api
-                  key: primary
           ports:
             - containerPort: 7880
               name: http
@@ -92,6 +90,12 @@ spec:
               readOnly: true
             - name: runtime-keys
               mountPath: /var/run/livekit
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
           resources:
             requests:
               cpu: 500m
@@ -110,6 +114,16 @@ spec:
           emptyDir: {}
         - name: runtime-keys
           emptyDir: {}
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: comms-vault
+        - name: vault-scripts
+          configMap:
+            name: comms-vault-env
+            defaultMode: 0555
 ---
 apiVersion: v1
 kind: Service

services/comms/mas-admin-client-secret-ensure-job.yaml

@@ -67,18 +67,29 @@ spec:
           args:
             - |
               set -euo pipefail
-              if kubectl -n comms get secret mas-admin-client-runtime >/dev/null 2>&1; then
+              apk add --no-cache curl jq >/dev/null
-                if kubectl -n comms get secret mas-admin-client-runtime -o jsonpath='{.data.client_secret}' 2>/dev/null | grep -q .; then
+
-                  exit 0
+              vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
-                fi
+              vault_role="${VAULT_ROLE:-comms-secrets}"
-              else
+              jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
-                kubectl -n comms create secret generic mas-admin-client-runtime \
+              login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
-                  --from-file=client_secret=/work/client_secret >/dev/null
+              vault_token="$(curl -sS --request POST --data "${login_payload}" \
+                "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
+              if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
+                echo "vault login failed" >&2
+                exit 1
+              fi
+
+              current="$(curl -sS -H "X-Vault-Token: ${vault_token}" \
+                "${vault_addr}/v1/kv/data/atlas/comms/mas-admin-client-runtime" | jq -r '.data.data.client_secret // empty')"
+              if [ -n "${current}" ]; then
                 exit 0
               fi
-              secret_b64="$(base64 /work/client_secret | tr -d '\n')"
+
-              payload="$(printf '{"data":{"client_secret":"%s"}}' "${secret_b64}")"
+              value="$(cat /work/client_secret)"
-              kubectl -n comms patch secret mas-admin-client-runtime --type=merge -p "${payload}" >/dev/null
+              payload="$(jq -nc --arg value "${value}" '{data:{client_secret:$value}}')"
+              curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
+                -d "${payload}" "${vault_addr}/v1/kv/data/atlas/comms/mas-admin-client-runtime" >/dev/null
           volumeMounts:
             - name: work
               mountPath: /work

services/comms/mas-db-ensure-job.yaml

@@ -24,18 +24,35 @@ spec:
                 head -c 32 /dev/urandom | base64 | tr -d '\n' | tr '+/' '-_' | tr -d '='
               }
 
-              EXISTING_B64="$(kubectl -n comms get secret mas-db -o jsonpath='{.data.password}' 2>/dev/null || true)"
+              apk add --no-cache curl jq >/dev/null
-              if [ -n "${EXISTING_B64}" ]; then
+
-                MAS_PASS="$(printf '%s' "${EXISTING_B64}" | base64 -d)"
+              vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
-                if printf '%s' "${MAS_PASS}" | grep -Eq '[^A-Za-z0-9_-]'; then
+              vault_role="${VAULT_ROLE:-comms-secrets}"
-                  MAS_PASS="$(safe_pass)"
+              jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
-                  MAS_B64="$(printf '%s' "${MAS_PASS}" | base64 | tr -d '\n')"
+              login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
-                  payload="$(printf '{"data":{"password":"%s"}}' "${MAS_B64}")"
+              vault_token="$(curl -sS --request POST --data "${login_payload}" \
-                  kubectl -n comms patch secret mas-db --type=merge -p "${payload}" >/dev/null
+                "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
-                fi
+              if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
-              else
+                echo "vault login failed" >&2
+                exit 1
+              fi
+
+              vault_read() {
+                curl -sS -H "X-Vault-Token: ${vault_token}" \
+                  "${vault_addr}/v1/kv/data/atlas/comms/mas-db" | jq -r '.data.data.password // empty'
+              }
+
+              vault_write() {
+                value="$1"
+                payload="$(jq -nc --arg value "${value}" '{data:{password:$value}}')"
+                curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
+                  -d "${payload}" "${vault_addr}/v1/kv/data/atlas/comms/mas-db" >/dev/null
+              }
+
+              MAS_PASS="$(vault_read)"
+              if [ -z "${MAS_PASS}" ] || printf '%s' "${MAS_PASS}" | grep -Eq '[^A-Za-z0-9_-]'; then
                 MAS_PASS="$(safe_pass)"
-                kubectl -n comms create secret generic mas-db --from-literal=password="${MAS_PASS}" >/dev/null
+                vault_write "${MAS_PASS}"
               fi
 
               POD_NAME="$(kubectl -n postgres get pods -l app=postgres -o jsonpath='{.items[0].metadata.name}')"

services/comms/mas-deployment.yaml

@@ -18,6 +18,7 @@ spec:
         app: matrix-authentication-service
     spec:
       enableServiceLinks: false
+      serviceAccountName: comms-vault
       nodeSelector:
         hardware: rpi5
       affinity:
@@ -36,6 +37,7 @@ spec:
           args:
             - |
               set -euo pipefail
+              . /vault/scripts/comms_vault_env.sh
               umask 077
               DB_PASS_ESCAPED="$(printf '%s' "${MAS_DB_PASSWORD}" | sed 's/[\\/&]/\\&/g')"
               MATRIX_SECRET_ESCAPED="$(printf '%s' "${MATRIX_SHARED_SECRET}" | sed 's/[\\/&]/\\&/g')"
@@ -47,22 +49,6 @@ spec:
                 -e "s/@@KEYCLOAK_CLIENT_SECRET@@/${KC_SECRET_ESCAPED}/g" \
                 /etc/mas/config.yaml > /rendered/config.yaml
               chmod 0644 /rendered/config.yaml
-          env:
-            - name: MAS_DB_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: mas-db
-                  key: password
-            - name: MATRIX_SHARED_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: mas-secrets-runtime
-                  key: matrix_shared_secret
-            - name: KEYCLOAK_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: mas-secrets-runtime
-                  key: keycloak_client_secret
           volumeMounts:
             - name: config
               mountPath: /etc/mas/config.yaml
@@ -71,6 +57,12 @@ spec:
             - name: rendered
               mountPath: /rendered
               readOnly: false
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
       containers:
         - name: mas
           image: ghcr.io/element-hq/matrix-authentication-service:1.8.0
@@ -86,14 +78,25 @@ spec:
             - name: rendered
               mountPath: /rendered
               readOnly: true
-            - name: secrets
+            - name: vault-secrets
-              mountPath: /etc/mas/secrets
+              mountPath: /etc/mas/secrets/encryption
+              subPath: mas-secrets-runtime__encryption
               readOnly: true
-            - name: admin-client
+            - name: vault-secrets
-              mountPath: /etc/mas/admin-client
+              mountPath: /etc/mas/secrets/matrix_shared_secret
+              subPath: mas-secrets-runtime__matrix_shared_secret
               readOnly: true
-            - name: keys
+            - name: vault-secrets
-              mountPath: /etc/mas/keys
+              mountPath: /etc/mas/secrets/keycloak_client_secret
+              subPath: mas-secrets-runtime__keycloak_client_secret
+              readOnly: true
+            - name: vault-secrets
+              mountPath: /etc/mas/keys/rsa_key
+              subPath: mas-secrets-runtime__rsa_key
+              readOnly: true
+            - name: vault-secrets
+              mountPath: /etc/mas/admin-client/client_secret
+              subPath: mas-admin-client-runtime__client_secret
               readOnly: true
           resources:
             requests:
@@ -111,28 +114,16 @@ spec:
                 path: config.yaml
         - name: rendered
           emptyDir: {}
-        - name: secrets
+        - name: vault-secrets
-          secret:
+          csi:
-            secretName: mas-secrets-runtime
+            driver: secrets-store.csi.k8s.io
-            items:
+            readOnly: true
-              - key: encryption
+            volumeAttributes:
-                path: encryption
+              secretProviderClass: comms-vault
-              - key: matrix_shared_secret
+        - name: vault-scripts
-                path: matrix_shared_secret
+          configMap:
-              - key: keycloak_client_secret
+            name: comms-vault-env
-                path: keycloak_client_secret
+            defaultMode: 0555
-        - name: keys
-          secret:
-            secretName: mas-secrets-runtime
-            items:
-              - key: rsa_key
-                path: rsa_key
-        - name: admin-client
-          secret:
-            secretName: mas-admin-client-runtime
-            items:
-              - key: client_secret
-                path: client_secret
 ---
 apiVersion: v1
 kind: Service

services/comms/mas-local-users-ensure-job.yaml

@@ -10,48 +10,47 @@ spec:
   template:
     spec:
       restartPolicy: Never
+      serviceAccountName: comms-vault
       volumes:
-        - name: mas-admin-client
+        - name: vault-secrets
-          secret:
+          csi:
-            secretName: mas-admin-client-runtime
+            driver: secrets-store.csi.k8s.io
-            items:
+            readOnly: true
-              - key: client_secret
+            volumeAttributes:
-                path: client_secret
+              secretProviderClass: comms-vault
+        - name: vault-scripts
+          configMap:
+            name: comms-vault-env
+            defaultMode: 0555
       containers:
         - name: ensure
           image: python:3.11-slim
           volumeMounts:
-            - name: mas-admin-client
+            - name: vault-secrets
-              mountPath: /etc/mas-admin-client
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
               readOnly: true
           env:
             - name: MAS_ADMIN_CLIENT_ID
               value: 01KDXMVQBQ5JNY6SEJPZW6Z8BM
             - name: MAS_ADMIN_CLIENT_SECRET_FILE
-              value: /etc/mas-admin-client/client_secret
+              value: /vault/secrets/mas-admin-client-runtime__client_secret
             - name: MAS_TOKEN_URL
               value: http://matrix-authentication-service:8080/oauth2/token
             - name: MAS_ADMIN_API_BASE
               value: http://matrix-authentication-service:8081/api/admin/v1
             - name: SEEDER_USER
               value: othrys-seeder
-            - name: SEEDER_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: atlasbot-credentials-runtime
-                  key: seeder-password
             - name: BOT_USER
               value: atlasbot
-            - name: BOT_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: atlasbot-credentials-runtime
-                  key: bot-password
           command:
             - /bin/sh
             - -c
             - |
               set -euo pipefail
+              . /vault/scripts/comms_vault_env.sh
               pip install --no-cache-dir requests >/dev/null
               python - <<'PY'
               import base64

services/comms/othrys-kick-numeric-job.yaml

@@ -9,6 +9,7 @@ spec:
   template:
     spec:
       restartPolicy: Never
+      serviceAccountName: comms-vault
       containers:
         - name: kick
           image: python:3.11-slim
@@ -23,16 +24,12 @@ spec:
               value: "#othrys:live.bstein.dev"
             - name: SEEDER_USER
               value: othrys-seeder
-            - name: SEEDER_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: atlasbot-credentials-runtime
-                  key: seeder-password
           command:
             - /bin/sh
             - -c
             - |
               set -euo pipefail
+              . /vault/scripts/comms_vault_env.sh
               pip install --no-cache-dir requests >/dev/null
               python - <<'PY'
               import os
@@ -113,3 +110,21 @@ spec:
                   if is_numeric(user_id):
                       kick(token, room_id, user_id)
               PY
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
+      volumes:
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: comms-vault
+        - name: vault-scripts
+          configMap:
+            name: comms-vault-env
+            defaultMode: 0555

services/comms/pin-othrys-job.yaml

@@ -16,6 +16,7 @@ spec:
       template:
         spec:
           restartPolicy: Never
+          serviceAccountName: comms-vault
           containers:
             - name: pin
               image: python:3.11-slim
@@ -26,16 +27,12 @@ spec:
                   value: http://matrix-authentication-service:8080
                 - name: SEEDER_USER
                   value: othrys-seeder
-                - name: SEEDER_PASS
-                  valueFrom:
-                    secretKeyRef:
-                      name: atlasbot-credentials-runtime
-                      key: seeder-password
               command:
                 - /bin/sh
                 - -c
                 - |
                   set -euo pipefail
+                  . /vault/scripts/comms_vault_env.sh
                   pip install --no-cache-dir requests >/dev/null
                   python - <<'PY'
                   import os, requests, urllib.parse
@@ -121,3 +118,21 @@ spec:
                   eid = send(room_id, token, MESSAGE)
                   pin(room_id, token, eid)
                   PY
+              volumeMounts:
+                - name: vault-secrets
+                  mountPath: /vault/secrets
+                  readOnly: true
+                - name: vault-scripts
+                  mountPath: /vault/scripts
+                  readOnly: true
+          volumes:
+            - name: vault-secrets
+              csi:
+                driver: secrets-store.csi.k8s.io
+                readOnly: true
+                volumeAttributes:
+                  secretProviderClass: comms-vault
+            - name: vault-scripts
+              configMap:
+                name: comms-vault-env
+                defaultMode: 0555

services/comms/reset-othrys-room-job.yaml

@@ -16,6 +16,7 @@ spec:
       template:
         spec:
           restartPolicy: Never
+          serviceAccountName: comms-vault
           containers:
             - name: reset
               image: python:3.11-slim
@@ -34,11 +35,6 @@ spec:
                   value: "Invite guests: share https://live.bstein.dev/#/room/#othrys:live.bstein.dev?action=join and choose 'Continue' -> 'Join as guest'."
                 - name: SEEDER_USER
                   value: othrys-seeder
-                - name: SEEDER_PASS
-                  valueFrom:
-                    secretKeyRef:
-                      name: atlasbot-credentials-runtime
-                      key: seeder-password
                 - name: BOT_USER
                   value: atlasbot
               command:
@@ -46,6 +42,7 @@ spec:
                 - -c
                 - |
                   set -euo pipefail
+                  . /vault/scripts/comms_vault_env.sh
                   pip install --no-cache-dir requests >/dev/null
                   python - <<'PY'
                   import os
@@ -264,3 +261,21 @@ spec:
                   print(f"old_room_id={old_room_id}")
                   print(f"new_room_id={new_room_id}")
                   PY
+              volumeMounts:
+                - name: vault-secrets
+                  mountPath: /vault/secrets
+                  readOnly: true
+                - name: vault-scripts
+                  mountPath: /vault/scripts
+                  readOnly: true
+          volumes:
+            - name: vault-secrets
+              csi:
+                driver: secrets-store.csi.k8s.io
+                readOnly: true
+                volumeAttributes:
+                  secretProviderClass: comms-vault
+            - name: vault-scripts
+              configMap:
+                name: comms-vault-env
+                defaultMode: 0555

services/comms/scripts/comms_vault_env.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env sh
+set -eu
+
+vault_dir="/vault/secrets"
+
+read_secret() {
+  cat "${vault_dir}/$1"
+}
+
+export TURN_STATIC_AUTH_SECRET="$(read_secret turn-shared-secret__TURN_STATIC_AUTH_SECRET)"
+export TURN_PASSWORD="${TURN_STATIC_AUTH_SECRET}"
+
+export LIVEKIT_API_SECRET="$(read_secret livekit-api__primary)"
+export LIVEKIT_SECRET="${LIVEKIT_API_SECRET}"
+
+export BOT_PASS="$(read_secret atlasbot-credentials-runtime__bot-password)"
+export SEEDER_PASS="$(read_secret atlasbot-credentials-runtime__seeder-password)"
+
+export CHAT_API_KEY="$(read_secret chat-ai-keys-runtime__matrix)"
+export CHAT_API_HOMEPAGE="$(read_secret chat-ai-keys-runtime__homepage)"
+
+export MAS_ADMIN_CLIENT_SECRET_FILE="${vault_dir}/mas-admin-client-runtime__client_secret"
+export PGPASSWORD="$(read_secret synapse-db__POSTGRES_PASSWORD)"
+
+export MAS_DB_PASSWORD="$(read_secret mas-db__password)"
+export MATRIX_SHARED_SECRET="$(read_secret mas-secrets-runtime__matrix_shared_secret)"
+export KEYCLOAK_CLIENT_SECRET="$(read_secret mas-secrets-runtime__keycloak_client_secret)"

services/comms/secretproviderclass.yaml

@@ -0,0 +1,134 @@
+# services/comms/secretproviderclass.yaml
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: comms-vault
+  namespace: comms
+spec:
+  provider: vault
+  parameters:
+    vaultAddress: "http://vault.vault.svc.cluster.local:8200"
+    roleName: "comms"
+    objects: |
+      - objectName: "turn-shared-secret__TURN_STATIC_AUTH_SECRET"
+        secretPath: "kv/data/atlas/comms/turn-shared-secret"
+        secretKey: "TURN_STATIC_AUTH_SECRET"
+      - objectName: "livekit-api__primary"
+        secretPath: "kv/data/atlas/comms/livekit-api"
+        secretKey: "primary"
+      - objectName: "synapse-db__POSTGRES_PASSWORD"
+        secretPath: "kv/data/atlas/comms/synapse-db"
+        secretKey: "POSTGRES_PASSWORD"
+      - objectName: "synapse-redis__redis-password"
+        secretPath: "kv/data/atlas/comms/synapse-redis"
+        secretKey: "redis-password"
+      - objectName: "synapse-macaroon__macaroon_secret_key"
+        secretPath: "kv/data/atlas/comms/synapse-macaroon"
+        secretKey: "macaroon_secret_key"
+      - objectName: "atlasbot-credentials-runtime__bot-password"
+        secretPath: "kv/data/atlas/comms/atlasbot-credentials-runtime"
+        secretKey: "bot-password"
+      - objectName: "atlasbot-credentials-runtime__seeder-password"
+        secretPath: "kv/data/atlas/comms/atlasbot-credentials-runtime"
+        secretKey: "seeder-password"
+      - objectName: "chat-ai-keys-runtime__matrix"
+        secretPath: "kv/data/atlas/shared/chat-ai-keys-runtime"
+        secretKey: "matrix"
+      - objectName: "chat-ai-keys-runtime__homepage"
+        secretPath: "kv/data/atlas/shared/chat-ai-keys-runtime"
+        secretKey: "homepage"
+      - objectName: "mas-admin-client-runtime__client_secret"
+        secretPath: "kv/data/atlas/comms/mas-admin-client-runtime"
+        secretKey: "client_secret"
+      - objectName: "mas-db__password"
+        secretPath: "kv/data/atlas/comms/mas-db"
+        secretKey: "password"
+      - objectName: "mas-secrets-runtime__encryption"
+        secretPath: "kv/data/atlas/comms/mas-secrets-runtime"
+        secretKey: "encryption"
+      - objectName: "mas-secrets-runtime__matrix_shared_secret"
+        secretPath: "kv/data/atlas/comms/mas-secrets-runtime"
+        secretKey: "matrix_shared_secret"
+      - objectName: "mas-secrets-runtime__keycloak_client_secret"
+        secretPath: "kv/data/atlas/comms/mas-secrets-runtime"
+        secretKey: "keycloak_client_secret"
+      - objectName: "mas-secrets-runtime__rsa_key"
+        secretPath: "kv/data/atlas/comms/mas-secrets-runtime"
+        secretKey: "rsa_key"
+      - objectName: "othrys-synapse-signingkey__signing.key"
+        secretPath: "kv/data/atlas/comms/othrys-synapse-signingkey"
+        secretKey: "signing.key"
+      - objectName: "synapse-oidc__client-secret"
+        secretPath: "kv/data/atlas/comms/synapse-oidc"
+        secretKey: "client-secret"
+  secretObjects:
+    - secretName: turn-shared-secret
+      type: Opaque
+      data:
+        - objectName: turn-shared-secret__TURN_STATIC_AUTH_SECRET
+          key: TURN_STATIC_AUTH_SECRET
+    - secretName: livekit-api
+      type: Opaque
+      data:
+        - objectName: livekit-api__primary
+          key: primary
+    - secretName: synapse-db
+      type: Opaque
+      data:
+        - objectName: synapse-db__POSTGRES_PASSWORD
+          key: POSTGRES_PASSWORD
+    - secretName: synapse-redis
+      type: Opaque
+      data:
+        - objectName: synapse-redis__redis-password
+          key: redis-password
+    - secretName: synapse-macaroon
+      type: Opaque
+      data:
+        - objectName: synapse-macaroon__macaroon_secret_key
+          key: macaroon_secret_key
+    - secretName: atlasbot-credentials-runtime
+      type: Opaque
+      data:
+        - objectName: atlasbot-credentials-runtime__bot-password
+          key: bot-password
+        - objectName: atlasbot-credentials-runtime__seeder-password
+          key: seeder-password
+    - secretName: chat-ai-keys-runtime
+      type: Opaque
+      data:
+        - objectName: chat-ai-keys-runtime__matrix
+          key: matrix
+        - objectName: chat-ai-keys-runtime__homepage
+          key: homepage
+    - secretName: mas-admin-client-runtime
+      type: Opaque
+      data:
+        - objectName: mas-admin-client-runtime__client_secret
+          key: client_secret
+    - secretName: mas-db
+      type: Opaque
+      data:
+        - objectName: mas-db__password
+          key: password
+    - secretName: mas-secrets-runtime
+      type: Opaque
+      data:
+        - objectName: mas-secrets-runtime__encryption
+          key: encryption
+        - objectName: mas-secrets-runtime__matrix_shared_secret
+          key: matrix_shared_secret
+        - objectName: mas-secrets-runtime__keycloak_client_secret
+          key: keycloak_client_secret
+        - objectName: mas-secrets-runtime__rsa_key
+          key: rsa_key
+    - secretName: othrys-synapse-signingkey
+      type: Opaque
+      data:
+        - objectName: othrys-synapse-signingkey__signing.key
+          key: signing.key
+    - secretName: synapse-oidc
+      type: Opaque
+      data:
+        - objectName: synapse-oidc__client-secret
+          key: client-secret

services/comms/seed-othrys-room.yaml

@@ -14,6 +14,7 @@ spec:
       template:
         spec:
           restartPolicy: Never
+          serviceAccountName: comms-vault
           containers:
             - name: seed
               image: python:3.11-slim
@@ -24,23 +25,14 @@ spec:
                   value: http://matrix-authentication-service:8080
                 - name: SEEDER_USER
                   value: othrys-seeder
-                - name: SEEDER_PASS
-                  valueFrom:
-                    secretKeyRef:
-                      name: atlasbot-credentials-runtime
-                      key: seeder-password
                 - name: BOT_USER
                   value: atlasbot
-                - name: BOT_PASS
-                  valueFrom:
-                    secretKeyRef:
-                      name: atlasbot-credentials-runtime
-                      key: bot-password
               command:
                 - /bin/sh
                 - -c
                 - |
                   set -euo pipefail
+                  . /vault/scripts/comms_vault_env.sh
                   pip install --no-cache-dir requests pyyaml >/dev/null
                   python - <<'PY'
                   import os, requests, urllib.parse
@@ -140,7 +132,23 @@ spec:
                 - name: synapse-config
                   mountPath: /config
                   readOnly: true
+                - name: vault-secrets
+                  mountPath: /vault/secrets
+                  readOnly: true
+                - name: vault-scripts
+                  mountPath: /vault/scripts
+                  readOnly: true
           volumes:
             - name: synapse-config
               secret:
                 secretName: othrys-synapse-matrix-synapse
+            - name: vault-secrets
+              csi:
+                driver: secrets-store.csi.k8s.io
+                readOnly: true
+                volumeAttributes:
+                  secretProviderClass: comms-vault
+            - name: vault-scripts
+              configMap:
+                name: comms-vault-env
+                defaultMode: 0555

services/comms/serviceaccount.yaml

@@ -0,0 +1,6 @@
+# services/comms/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: comms-vault
+  namespace: comms

services/comms/synapse-seeder-admin-ensure-job.yaml

@@ -9,6 +9,7 @@ spec:
   template:
     spec:
       restartPolicy: OnFailure
+      serviceAccountName: comms-vault
       containers:
         - name: psql
           image: postgres:16-alpine
@@ -21,16 +22,30 @@ spec:
               value: synapse
             - name: PGUSER
               value: synapse
-            - name: PGPASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: synapse-db
-                  key: POSTGRES_PASSWORD
           command:
             - /bin/sh
             - -c
             - |
               set -euo pipefail
+              . /vault/scripts/comms_vault_env.sh
               psql -v ON_ERROR_STOP=1 <<'SQL'
               UPDATE users SET admin = 1 WHERE name = '@othrys-seeder:live.bstein.dev';
               SQL
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
+      volumes:
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: comms-vault
+        - name: vault-scripts
+          configMap:
+            name: comms-vault-env
+            defaultMode: 0555

services/comms/synapse-signingkey-ensure-job.yaml

@@ -37,15 +37,29 @@ spec:
           args:
             - |
               set -euo pipefail
-              set -x
+              apk add --no-cache curl jq >/dev/null
-              if kubectl -n comms get secret othrys-synapse-signingkey \
+
-                -o jsonpath='{.data.signing\.key}' 2>/tmp/get_err | grep -q .; then
+              vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
+              vault_role="${VAULT_ROLE:-comms-secrets}"
+              jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+              login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
+              vault_token="$(curl -sS --request POST --data "${login_payload}" \
+                "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
+              if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
+                echo "vault login failed" >&2
+                exit 1
+              fi
+
+              existing="$(curl -sS -H "X-Vault-Token: ${vault_token}" \
+                "${vault_addr}/v1/kv/data/atlas/comms/othrys-synapse-signingkey" | jq -r '.data.data["signing.key"] // empty')"
+              if [ -n "${existing}" ]; then
                 exit 0
               fi
-              cat /tmp/get_err >&2 || true
+
-              kubectl -n comms create secret generic othrys-synapse-signingkey \
+              value="$(cat /work/signing.key)"
-                --from-file=signing.key=/work/signing.key \
+              payload="$(jq -nc --arg value "${value}" '{data:{"signing.key":$value}}')"
-                --dry-run=client -o yaml | kubectl -n comms apply -f - >/dev/null
+              curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
+                -d "${payload}" "${vault_addr}/v1/kv/data/atlas/comms/othrys-synapse-signingkey" >/dev/null
           volumeMounts:
             - name: work
               mountPath: /work

services/comms/synapse-user-seed-job.yaml

@@ -10,6 +10,7 @@ spec:
   template:
     spec:
       restartPolicy: Never
+      serviceAccountName: comms-vault
       containers:
         - name: seed
           image: python:3.11-slim
@@ -22,30 +23,16 @@ spec:
               value: synapse
             - name: PGUSER
               value: synapse
-            - name: PGPASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: synapse-db
-                  key: POSTGRES_PASSWORD
             - name: SEEDER_USER
               value: othrys-seeder
-            - name: SEEDER_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: atlasbot-credentials-runtime
-                  key: seeder-password
             - name: BOT_USER
               value: atlasbot
-            - name: BOT_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: atlasbot-credentials-runtime
-                  key: bot-password
           command:
             - /bin/sh
             - -c
             - |
               set -euo pipefail
+              . /vault/scripts/comms_vault_env.sh
               pip install --no-cache-dir psycopg2-binary bcrypt >/dev/null
               python - <<'PY'
               import os
@@ -118,3 +105,21 @@ spec:
               finally:
                   conn.close()
               PY
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
+      volumes:
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: comms-vault
+        - name: vault-scripts
+          configMap:
+            name: comms-vault-env
+            defaultMode: 0555

services/comms/vault-sync-deployment.yaml

@@ -0,0 +1,34 @@
+# services/comms/vault-sync-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: comms-vault-sync
+  namespace: comms
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: comms-vault-sync
+  template:
+    metadata:
+      labels:
+        app: comms-vault-sync
+    spec:
+      serviceAccountName: comms-vault
+      containers:
+        - name: sync
+          image: alpine:3.20
+          command: ["/bin/sh", "-c"]
+          args:
+            - "sleep infinity"
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+      volumes:
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: comms-vault

services/harbor/kustomization.yaml

@@ -4,7 +4,10 @@ kind: Kustomization
 namespace: harbor
 resources:
   - namespace.yaml
+  - serviceaccount.yaml
+  - secretproviderclass.yaml
   - pvc.yaml
   - certificate.yaml
   - helmrelease.yaml
+  - vault-sync-deployment.yaml
   - image.yaml

services/harbor/secretproviderclass.yaml

@@ -0,0 +1,87 @@
+# services/harbor/secretproviderclass.yaml
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: harbor-vault
+  namespace: harbor
+spec:
+  provider: vault
+  parameters:
+    vaultAddress: "http://vault.vault.svc.cluster.local:8200"
+    roleName: "harbor"
+    objects: |
+      - objectName: "harbor-core__CSRF_KEY"
+        secretPath: "kv/data/atlas/harbor/harbor-core"
+        secretKey: "CSRF_KEY"
+      - objectName: "harbor-core__REGISTRY_CREDENTIAL_PASSWORD"
+        secretPath: "kv/data/atlas/harbor/harbor-core"
+        secretKey: "REGISTRY_CREDENTIAL_PASSWORD"
+      - objectName: "harbor-core__harbor_admin_password"
+        secretPath: "kv/data/atlas/harbor/harbor-core"
+        secretKey: "harbor_admin_password"
+      - objectName: "harbor-core__secret"
+        secretPath: "kv/data/atlas/harbor/harbor-core"
+        secretKey: "secret"
+      - objectName: "harbor-core__secretKey"
+        secretPath: "kv/data/atlas/harbor/harbor-core"
+        secretKey: "secretKey"
+      - objectName: "harbor-core__tls.crt"
+        secretPath: "kv/data/atlas/harbor/harbor-core"
+        secretKey: "tls.crt"
+      - objectName: "harbor-core__tls.key"
+        secretPath: "kv/data/atlas/harbor/harbor-core"
+        secretKey: "tls.key"
+      - objectName: "harbor-db__database"
+        secretPath: "kv/data/atlas/harbor/harbor-db"
+        secretKey: "database"
+      - objectName: "harbor-db__host"
+        secretPath: "kv/data/atlas/harbor/harbor-db"
+        secretKey: "host"
+      - objectName: "harbor-db__password"
+        secretPath: "kv/data/atlas/harbor/harbor-db"
+        secretKey: "password"
+      - objectName: "harbor-db__port"
+        secretPath: "kv/data/atlas/harbor/harbor-db"
+        secretKey: "port"
+      - objectName: "harbor-db__username"
+        secretPath: "kv/data/atlas/harbor/harbor-db"
+        secretKey: "username"
+      - objectName: "harbor-oidc__CONFIG_OVERWRITE_JSON"
+        secretPath: "kv/data/atlas/harbor/harbor-oidc"
+        secretKey: "CONFIG_OVERWRITE_JSON"
+  secretObjects:
+    - secretName: harbor-core
+      type: Opaque
+      data:
+        - objectName: harbor-core__CSRF_KEY
+          key: CSRF_KEY
+        - objectName: harbor-core__REGISTRY_CREDENTIAL_PASSWORD
+          key: REGISTRY_CREDENTIAL_PASSWORD
+        - objectName: harbor-core__harbor_admin_password
+          key: harbor_admin_password
+        - objectName: harbor-core__secret
+          key: secret
+        - objectName: harbor-core__secretKey
+          key: secretKey
+        - objectName: harbor-core__tls.crt
+          key: tls.crt
+        - objectName: harbor-core__tls.key
+          key: tls.key
+    - secretName: harbor-db
+      type: Opaque
+      data:
+        - objectName: harbor-db__database
+          key: database
+        - objectName: harbor-db__host
+          key: host
+        - objectName: harbor-db__password
+          key: password
+        - objectName: harbor-db__port
+          key: port
+        - objectName: harbor-db__username
+          key: username
+    - secretName: harbor-oidc
+      type: Opaque
+      data:
+        - objectName: harbor-oidc__CONFIG_OVERWRITE_JSON
+          key: CONFIG_OVERWRITE_JSON

services/harbor/serviceaccount.yaml

@@ -0,0 +1,6 @@
+# services/harbor/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: harbor-vault-sync
+  namespace: harbor

services/harbor/vault-sync-deployment.yaml

@@ -0,0 +1,34 @@
+# services/harbor/vault-sync-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: harbor-vault-sync
+  namespace: harbor
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: harbor-vault-sync
+  template:
+    metadata:
+      labels:
+        app: harbor-vault-sync
+    spec:
+      serviceAccountName: harbor-vault-sync
+      containers:
+        - name: sync
+          image: alpine:3.20
+          command: ["/bin/sh", "-c"]
+          args:
+            - "sleep infinity"
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+      volumes:
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: harbor-vault

services/keycloak/harbor-oidc-secret-ensure-job.yaml

@@ -16,6 +16,16 @@ spec:
           configMap:
             name: harbor-oidc-secret-ensure-script
             defaultMode: 0555
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: sso-vault
+        - name: vault-scripts
+          configMap:
+            name: sso-vault-env
+            defaultMode: 0555
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -30,18 +40,13 @@ spec:
         - name: apply
           image: alpine:3.20
           command: ["/scripts/harbor_oidc_secret_ensure.sh"]
-          env:
-            - name: KEYCLOAK_ADMIN
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: username
-            - name: KEYCLOAK_ADMIN_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: password
           volumeMounts:
             - name: harbor-oidc-secret-ensure-script
               mountPath: /scripts
               readOnly: true
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true

services/keycloak/ldap-federation-job.yaml

@@ -19,6 +19,7 @@ spec:
                   - key: node-role.kubernetes.io/worker
                     operator: Exists
       restartPolicy: OnFailure
+      serviceAccountName: sso-vault
       containers:
         - name: configure
           image: python:3.11-alpine
@@ -28,25 +29,10 @@ spec:
               value: http://keycloak.sso.svc.cluster.local
             - name: KEYCLOAK_REALM
               value: atlas
-            - name: KEYCLOAK_ADMIN_USER
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: username
-            - name: KEYCLOAK_ADMIN_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: password
             - name: LDAP_URL
               value: ldap://openldap.sso.svc.cluster.local:389
             - name: LDAP_BIND_DN
               value: cn=admin,dc=bstein,dc=dev
-            - name: LDAP_BIND_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: openldap-admin
-                  key: LDAP_ADMIN_PASSWORD
             - name: LDAP_USERS_DN
               value: ou=users,dc=bstein,dc=dev
             - name: LDAP_GROUPS_DN
@@ -55,6 +41,7 @@ spec:
           args:
             - |
               set -euo pipefail
+              . /vault/scripts/keycloak_vault_env.sh
               python - <<'PY'
               import json
               import os
@@ -360,3 +347,21 @@ spec:
               except Exception as e:
                   print(f"WARNING: LDAP cleanup failed (continuing): {e}")
               PY
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
+      volumes:
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: sso-vault
+        - name: vault-scripts
+          configMap:
+            name: sso-vault-env
+            defaultMode: 0555

services/keycloak/logs-oidc-secret-ensure-job.yaml

@@ -18,6 +18,7 @@ spec:
           args:
             - |
               set -euo pipefail
+              . /vault/scripts/keycloak_vault_env.sh
               apk add --no-cache curl jq kubectl openssl >/dev/null
 
               KC_URL="http://keycloak.sso.svc.cluster.local"
@@ -73,31 +74,56 @@ spec:
                 exit 1
               fi
 
-              if kubectl -n logging get secret oauth2-proxy-logs-oidc >/dev/null 2>&1; then
+              vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
-                current_cookie="$(kubectl -n logging get secret oauth2-proxy-logs-oidc -o jsonpath='{.data.cookie_secret}' 2>/dev/null || true)"
+              vault_role="${VAULT_ROLE:-sso-secrets}"
-                if [ -n "${current_cookie}" ]; then
+              jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
-                  decoded="$(printf '%s' "${current_cookie}" | base64 -d 2>/dev/null || true)"
+              login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
-                  length="$(printf '%s' "${decoded}" | wc -c | tr -d ' ')"
+              vault_token="$(curl -sS --request POST --data "${login_payload}" \
-                  if [ "${length}" = "16" ] || [ "${length}" = "24" ] || [ "${length}" = "32" ]; then
+                "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
-                    exit 0
+              if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
-                  fi
+                echo "vault login failed" >&2
-                fi
+                exit 1
               fi
 
-              COOKIE_SECRET="$(openssl rand -hex 16 | tr -d '\n')"
+              COOKIE_SECRET="$(curl -sS -H "X-Vault-Token: ${vault_token}" \
+                "${vault_addr}/v1/kv/data/atlas/logging/oauth2-proxy-logs-oidc" | jq -r '.data.data.cookie_secret // empty')"
+              if [ -n "${COOKIE_SECRET}" ]; then
+                length="$(printf '%s' "${COOKIE_SECRET}" | wc -c | tr -d ' ')"
+                if [ "${length}" != "16" ] && [ "${length}" != "24" ] && [ "${length}" != "32" ]; then
+                  COOKIE_SECRET=""
+                fi
+              fi
+              if [ -z "${COOKIE_SECRET}" ]; then
+                COOKIE_SECRET="$(openssl rand -hex 16 | tr -d '\n')"
+              fi
+
+              payload="$(jq -nc \
+                --arg client_id "logs" \
+                --arg client_secret "${CLIENT_SECRET}" \
+                --arg cookie_secret "${COOKIE_SECRET}" \
+                '{data:{client_id:$client_id,client_secret:$client_secret,cookie_secret:$cookie_secret}}')"
+              curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
+                -d "${payload}" "${vault_addr}/v1/kv/data/atlas/logging/oauth2-proxy-logs-oidc" >/dev/null
               kubectl -n logging create secret generic oauth2-proxy-logs-oidc \
                 --from-literal=client_id="logs" \
                 --from-literal=client_secret="${CLIENT_SECRET}" \
                 --from-literal=cookie_secret="${COOKIE_SECRET}" \
                 --dry-run=client -o yaml | kubectl -n logging apply -f - >/dev/null
-          env:
+          volumeMounts:
-            - name: KEYCLOAK_ADMIN
+            - name: vault-secrets
-              valueFrom:
+              mountPath: /vault/secrets
-                secretKeyRef:
+              readOnly: true
-                  name: keycloak-admin
+            - name: vault-scripts
-                  key: username
+              mountPath: /vault/scripts
-            - name: KEYCLOAK_ADMIN_PASSWORD
+              readOnly: true
-              valueFrom:
+      volumes:
-                secretKeyRef:
+        - name: vault-secrets
-                  name: keycloak-admin
+          csi:
-                  key: password
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: sso-vault
+        - name: vault-scripts
+          configMap:
+            name: sso-vault-env
+            defaultMode: 0555

services/keycloak/mas-secrets-ensure-job.yaml

@@ -20,6 +20,16 @@ spec:
       volumes:
         - name: work
           emptyDir: {}
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: sso-vault
+        - name: vault-scripts
+          configMap:
+            name: sso-vault-env
+            defaultMode: 0555
       initContainers:
         - name: generate
           image: alpine:3.20
@@ -27,6 +37,7 @@ spec:
           args:
             - |
               set -euo pipefail
+              . /vault/scripts/keycloak_vault_env.sh
               umask 077
               apk add --no-cache curl openssl jq >/dev/null
 
@@ -68,20 +79,15 @@ spec:
               openssl rand -hex 32 | tr -d '\n' > /work/matrix_shared_secret
               openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:4096 -out /work/rsa_key >/dev/null 2>&1
               chmod 0644 /work/*
-          env:
-            - name: KEYCLOAK_ADMIN
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: username
-            - name: KEYCLOAK_ADMIN_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: password
           volumeMounts:
             - name: work
               mountPath: /work
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
       containers:
         - name: apply
           image: registry.bstein.dev/bstein/kubectl:1.35.0
@@ -89,19 +95,36 @@ spec:
           args:
             - |
               set -euo pipefail
-              if kubectl -n comms get secret mas-secrets-runtime >/dev/null 2>&1; then
+              apk add --no-cache curl jq >/dev/null
-                kubectl -n comms get secret mas-secrets-runtime -o jsonpath='{.data.encryption}' | base64 -d 2>/dev/null > /tmp/encryption.current || true
+
-                current_len="$(wc -c < /tmp/encryption.current | tr -d ' ')"
+              vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
-                if [ "${current_len}" = "64" ] && grep -Eq '^[0-9a-fA-F]{64}$' /tmp/encryption.current; then
+              vault_role="${VAULT_ROLE:-sso-secrets}"
+              jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+              login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
+              vault_token="$(curl -sS --request POST --data "${login_payload}" \
+                "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
+              if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
+                echo "vault login failed" >&2
+                exit 1
+              fi
+
+              existing="$(curl -sS -H "X-Vault-Token: ${vault_token}" \
+                "${vault_addr}/v1/kv/data/atlas/comms/mas-secrets-runtime" | jq -r '.data.data.encryption // empty')"
+              if [ -n "${existing}" ]; then
+                current_len="$(printf '%s' "${existing}" | wc -c | tr -d ' ')"
+                if [ "${current_len}" = "64" ] && printf '%s' "${existing}" | grep -Eq '^[0-9a-fA-F]{64}$'; then
                   exit 0
                 fi
               fi
-              kubectl -n comms create secret generic mas-secrets-runtime \
+
-                --from-file=encryption=/work/encryption \
+              payload="$(jq -nc \
-                --from-file=matrix_shared_secret=/work/matrix_shared_secret \
+                --arg encryption "$(cat /work/encryption)" \
-                --from-file=keycloak_client_secret=/work/keycloak_client_secret \
+                --arg matrix_shared_secret "$(cat /work/matrix_shared_secret)" \
-                --from-file=rsa_key=/work/rsa_key \
+                --arg keycloak_client_secret "$(cat /work/keycloak_client_secret)" \
-                --dry-run=client -o yaml | kubectl -n comms apply -f - >/dev/null
+                --arg rsa_key "$(cat /work/rsa_key)" \
+                '{data:{encryption:$encryption, matrix_shared_secret:$matrix_shared_secret, keycloak_client_secret:$keycloak_client_secret, rsa_key:$rsa_key}}')"
+              curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
+                -d "${payload}" "${vault_addr}/v1/kv/data/atlas/comms/mas-secrets-runtime" >/dev/null
           volumeMounts:
             - name: work
               mountPath: /work

services/keycloak/portal-e2e-client-job.yaml

@@ -9,6 +9,7 @@ spec:
   template:
     spec:
       restartPolicy: Never
+      serviceAccountName: sso-vault
       containers:
         - name: configure
           image: python:3.11-alpine
@@ -17,30 +18,11 @@ spec:
               value: http://keycloak.sso.svc.cluster.local
             - name: KEYCLOAK_REALM
               value: atlas
-            - name: KEYCLOAK_ADMIN_USER
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: username
-            - name: KEYCLOAK_ADMIN_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: password
-            - name: PORTAL_E2E_CLIENT_ID
-              valueFrom:
-                secretKeyRef:
-                  name: portal-e2e-client
-                  key: client_id
-            - name: PORTAL_E2E_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: portal-e2e-client
-                  key: client_secret
           command: ["/bin/sh", "-c"]
           args:
             - |
               set -euo pipefail
+              . /vault/scripts/keycloak_vault_env.sh
               python - <<'PY'
               import json
               import os
@@ -245,3 +227,21 @@ spec:
                   if status not in (200, 204):
                       raise SystemExit(f"Role mapping update failed (status={status}) resp={resp}")
               PY
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
+      volumes:
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: sso-vault
+        - name: vault-scripts
+          configMap:
+            name: sso-vault-env
+            defaultMode: 0555

services/keycloak/portal-e2e-execute-actions-email-test-job.yaml

@@ -9,6 +9,7 @@ spec:
   template:
     spec:
       restartPolicy: Never
+      serviceAccountName: sso-vault
       containers:
         - name: test
           image: python:3.11-alpine
@@ -17,16 +18,6 @@ spec:
               value: http://keycloak.sso.svc.cluster.local
             - name: KEYCLOAK_REALM
               value: atlas
-            - name: PORTAL_E2E_CLIENT_ID
-              valueFrom:
-                secretKeyRef:
-                  name: portal-e2e-client
-                  key: client_id
-            - name: PORTAL_E2E_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: portal-e2e-client
-                  key: client_secret
             - name: E2E_PROBE_USERNAME
               value: e2e-smtp-probe
             - name: E2E_PROBE_EMAIL
@@ -39,13 +30,30 @@ spec:
           args:
             - |
               set -euo pipefail
+              . /vault/scripts/keycloak_vault_env.sh
               python /scripts/test_keycloak_execute_actions_email.py
           volumeMounts:
             - name: tests
               mountPath: /scripts
               readOnly: true
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
       volumes:
         - name: tests
           configMap:
             name: portal-e2e-tests
             defaultMode: 0555
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: sso-vault
+        - name: vault-scripts
+          configMap:
+            name: sso-vault-env
+            defaultMode: 0555

services/keycloak/portal-e2e-target-client-job.yaml

@@ -9,6 +9,7 @@ spec:
   template:
     spec:
       restartPolicy: Never
+      serviceAccountName: sso-vault
       containers:
         - name: configure
           image: python:3.11-alpine
@@ -17,22 +18,13 @@ spec:
               value: http://keycloak.sso.svc.cluster.local
             - name: KEYCLOAK_REALM
               value: atlas
-            - name: KEYCLOAK_ADMIN_USER
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: username
-            - name: KEYCLOAK_ADMIN_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: password
             - name: TARGET_CLIENT_ID
               value: bstein-dev-home
           command: ["/bin/sh", "-c"]
           args:
             - |
               set -euo pipefail
+              . /vault/scripts/keycloak_vault_env.sh
               python - <<'PY'
               import json
               import os
@@ -136,3 +128,21 @@ spec:
 
               print(f"OK: ensured token exchange enabled on client {target_client_id}")
               PY
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
+      volumes:
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: sso-vault
+        - name: vault-scripts
+          configMap:
+            name: sso-vault-env
+            defaultMode: 0555

services/keycloak/portal-e2e-token-exchange-permissions-job.yaml

@@ -9,6 +9,7 @@ spec:
   template:
     spec:
       restartPolicy: Never
+      serviceAccountName: sso-vault
       containers:
         - name: configure
           image: python:3.11-alpine
@@ -17,16 +18,6 @@ spec:
               value: http://keycloak.sso.svc.cluster.local
             - name: KEYCLOAK_REALM
               value: atlas
-            - name: KEYCLOAK_ADMIN_USER
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: username
-            - name: KEYCLOAK_ADMIN_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: password
             - name: PORTAL_E2E_CLIENT_ID
               value: test-portal-e2e
             - name: TARGET_CLIENT_ID
@@ -35,6 +26,7 @@ spec:
           args:
             - |
               set -euo pipefail
+              . /vault/scripts/keycloak_vault_env.sh
               python - <<'PY'
               import json
               import os
@@ -269,3 +261,21 @@ spec:
 
               print("OK: configured token exchange permissions for portal E2E client")
               PY
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
+      volumes:
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: sso-vault
+        - name: vault-scripts
+          configMap:
+            name: sso-vault-env
+            defaultMode: 0555

services/keycloak/portal-e2e-token-exchange-test-job.yaml

@@ -10,6 +10,7 @@ spec:
   template:
     spec:
       restartPolicy: Never
+      serviceAccountName: sso-vault
       containers:
         - name: test
           image: python:3.11-alpine
@@ -26,27 +27,34 @@ spec:
               value: "300"
             - name: RETRY_INTERVAL_SECONDS
               value: "5"
-            - name: PORTAL_E2E_CLIENT_ID
-              valueFrom:
-                secretKeyRef:
-                  name: portal-e2e-client
-                  key: client_id
-            - name: PORTAL_E2E_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: portal-e2e-client
-                  key: client_secret
           command: ["/bin/sh", "-c"]
           args:
             - |
               set -euo pipefail
+              . /vault/scripts/keycloak_vault_env.sh
               python /scripts/test_portal_token_exchange.py
           volumeMounts:
             - name: tests
               mountPath: /scripts
               readOnly: true
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
       volumes:
         - name: tests
           configMap:
             name: portal-e2e-tests
             defaultMode: 0555
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: sso-vault
+        - name: vault-scripts
+          configMap:
+            name: sso-vault-env
+            defaultMode: 0555

services/keycloak/realm-settings-job.yaml

@@ -19,6 +19,7 @@ spec:
                   - key: node-role.kubernetes.io/worker
                     operator: Exists
       restartPolicy: Never
+      serviceAccountName: sso-vault
       containers:
         - name: configure
           image: python:3.11-alpine
@@ -27,16 +28,6 @@ spec:
               value: http://keycloak.sso.svc.cluster.local
             - name: KEYCLOAK_REALM
               value: atlas
-            - name: KEYCLOAK_ADMIN_USER
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: username
-            - name: KEYCLOAK_ADMIN_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: password
             - name: KEYCLOAK_SMTP_HOST
               value: mailu-front.mailu-mailserver.svc.cluster.local
             - name: KEYCLOAK_SMTP_PORT
@@ -53,6 +44,7 @@ spec:
           args:
             - |
               set -euo pipefail
+              . /vault/scripts/keycloak_vault_env.sh
               python - <<'PY'
               import json
               import os
@@ -444,3 +436,21 @@ spec:
                               f"Unexpected execution update response for identity-provider-redirector: {status}"
                           )
               PY
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
+      volumes:
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: sso-vault
+        - name: vault-scripts
+          configMap:
+            name: sso-vault-env
+            defaultMode: 0555

services/keycloak/scripts/harbor_oidc_secret_ensure.sh

@@ -3,6 +3,8 @@ set -euo pipefail
 
 apk add --no-cache curl jq kubectl >/dev/null
 
+. /vault/scripts/keycloak_vault_env.sh
+
 KC_URL="http://keycloak.sso.svc.cluster.local"
 ACCESS_TOKEN=""
 for attempt in 1 2 3 4 5; do
@@ -99,6 +101,17 @@ CONFIG_OVERWRITE_JSON="$(jq -nc \
   --argjson oidc_logout true \
   '{auth_mode:$auth_mode,oidc_name:$oidc_name,oidc_client_id:$oidc_client_id,oidc_client_secret:$oidc_client_secret,oidc_endpoint:$oidc_endpoint,oidc_scope:$oidc_scope,oidc_user_claim:$oidc_user_claim,oidc_groups_claim:$oidc_groups_claim,oidc_admin_group:$oidc_admin_group,oidc_auto_onboard:$oidc_auto_onboard,oidc_verify_cert:$oidc_verify_cert,oidc_logout:$oidc_logout}')"
 
-kubectl -n harbor create secret generic harbor-oidc \
+vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
-  --from-literal=CONFIG_OVERWRITE_JSON="${CONFIG_OVERWRITE_JSON}" \
+vault_role="${VAULT_ROLE:-sso-secrets}"
-  --dry-run=client -o yaml | kubectl -n harbor apply -f - >/dev/null
+jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
+vault_token="$(curl -sS --request POST --data "${login_payload}" \
+  "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
+if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
+  echo "vault login failed" >&2
+  exit 1
+fi
+
+payload="$(jq -nc --arg value "${CONFIG_OVERWRITE_JSON}" '{data:{CONFIG_OVERWRITE_JSON:$value}}')"
+curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
+  -d "${payload}" "${vault_addr}/v1/kv/data/atlas/harbor/harbor-oidc" >/dev/null

services/keycloak/scripts/keycloak_vault_env.sh

@@ -23,3 +23,4 @@ export PORTAL_E2E_CLIENT_SECRET="$(read_secret portal-e2e-client__client_secret)
 
 export LDAP_ADMIN_PASSWORD="$(read_secret openldap-admin__LDAP_ADMIN_PASSWORD)"
 export LDAP_CONFIG_PASSWORD="$(read_secret openldap-admin__LDAP_CONFIG_PASSWORD)"
+export LDAP_BIND_PASSWORD="${LDAP_ADMIN_PASSWORD}"

services/keycloak/scripts/vault_oidc_secret_ensure.sh

@@ -3,6 +3,8 @@ set -euo pipefail
 
 apk add --no-cache curl jq kubectl >/dev/null
 
+. /vault/scripts/keycloak_vault_env.sh
+
 KC_URL="http://keycloak.sso.svc.cluster.local"
 ACCESS_TOKEN=""
 for attempt in 1 2 3 4 5; do
@@ -84,6 +86,37 @@ if [ -z "$CLIENT_SECRET" ] || [ "$CLIENT_SECRET" = "null" ]; then
   exit 1
 fi
 
+vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
+vault_role="${VAULT_ROLE:-sso-secrets}"
+jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
+vault_token="$(curl -sS --request POST --data "${login_payload}" \
+  "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
+if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
+  echo "vault login failed" >&2
+  exit 1
+fi
+
+payload="$(jq -nc \
+  --arg discovery_url "https://sso.bstein.dev/realms/atlas" \
+  --arg client_id "vault-oidc" \
+  --arg client_secret "${CLIENT_SECRET}" \
+  --arg default_role "admin" \
+  --arg scopes "openid profile email groups" \
+  --arg user_claim "preferred_username" \
+  --arg groups_claim "groups" \
+  --arg redirect_uris "https://secret.bstein.dev/ui/vault/auth/oidc/oidc/callback,http://localhost:8250/oidc/callback" \
+  --arg bound_audiences "vault-oidc" \
+  --arg admin_group "admin" \
+  --arg admin_policies "default,vault-admin" \
+  --arg dev_group "dev" \
+  --arg dev_policies "default,dev-kv" \
+  --arg user_group "dev" \
+  --arg user_policies "default,dev-kv" \
+  '{data:{discovery_url:$discovery_url,client_id:$client_id,client_secret:$client_secret,default_role:$default_role,scopes:$scopes,user_claim:$user_claim,groups_claim:$groups_claim,redirect_uris:$redirect_uris,bound_audiences:$bound_audiences,admin_group:$admin_group,admin_policies:$admin_policies,dev_group:$dev_group,dev_policies:$dev_policies,user_group:$user_group,user_policies:$user_policies}}')"
+curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
+  -d "${payload}" "${vault_addr}/v1/kv/data/atlas/vault/vault-oidc-config" >/dev/null
+
 kubectl -n vault create secret generic vault-oidc-config \
   --from-literal=discovery_url="https://sso.bstein.dev/realms/atlas" \
   --from-literal=client_id="vault-oidc" \

services/keycloak/synapse-oidc-secret-ensure-job.yaml

@@ -18,7 +18,8 @@ spec:
           args:
             - |
               set -euo pipefail
-              apk add --no-cache curl jq kubectl >/dev/null
+              . /vault/scripts/keycloak_vault_env.sh
+              apk add --no-cache curl jq >/dev/null
 
               KC_URL="http://keycloak.sso.svc.cluster.local"
               ACCESS_TOKEN=""
@@ -54,22 +55,35 @@ spec:
                 exit 1
               fi
 
-              existing="$(kubectl -n comms get secret synapse-oidc -o jsonpath='{.data.client-secret}' 2>/dev/null || true)"
+              vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
-              if [ -n "${existing}" ]; then
+              vault_role="${VAULT_ROLE:-sso-secrets}"
-                exit 0
+              jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+              login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
+              vault_token="$(curl -sS --request POST --data "${login_payload}" \
+                "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
+              if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
+                echo "vault login failed" >&2
+                exit 1
               fi
 
-              kubectl -n comms create secret generic synapse-oidc \
+              payload="$(jq -nc --arg value "${CLIENT_SECRET}" '{data:{"client-secret":$value}}')"
-                --from-literal=client-secret="${CLIENT_SECRET}" \
+              curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
-                --dry-run=client -o yaml | kubectl -n comms apply -f - >/dev/null
+                -d "${payload}" "${vault_addr}/v1/kv/data/atlas/comms/synapse-oidc" >/dev/null
-          env:
+          volumeMounts:
-            - name: KEYCLOAK_ADMIN
+            - name: vault-secrets
-              valueFrom:
+              mountPath: /vault/secrets
-                secretKeyRef:
+              readOnly: true
-                  name: keycloak-admin
+            - name: vault-scripts
-                  key: username
+              mountPath: /vault/scripts
-            - name: KEYCLOAK_ADMIN_PASSWORD
+              readOnly: true
-              valueFrom:
+      volumes:
-                secretKeyRef:
+        - name: vault-secrets
-                  name: keycloak-admin
+          csi:
-                  key: password
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: sso-vault
+        - name: vault-scripts
+          configMap:
+            name: sso-vault-env
+            defaultMode: 0555

services/keycloak/user-overrides-job.yaml

@@ -19,6 +19,7 @@ spec:
                   - key: node-role.kubernetes.io/worker
                     operator: Exists
       restartPolicy: Never
+      serviceAccountName: sso-vault
       containers:
         - name: configure
           image: python:3.11-alpine
@@ -27,16 +28,6 @@ spec:
               value: http://keycloak.sso.svc.cluster.local
             - name: KEYCLOAK_REALM
               value: atlas
-            - name: KEYCLOAK_ADMIN_USER
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: username
-            - name: KEYCLOAK_ADMIN_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: password
             - name: OVERRIDE_USERNAME
               value: bstein
             - name: OVERRIDE_MAILU_EMAIL
@@ -45,6 +36,7 @@ spec:
           args:
             - |
               set -euo pipefail
+              . /vault/scripts/keycloak_vault_env.sh
               python - <<'PY'
               import json
               import os
@@ -143,3 +135,21 @@ spec:
               if status not in (200, 204):
                   raise SystemExit(f"Unexpected user update response: {status}")
               PY
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
+      volumes:
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: sso-vault
+        - name: vault-scripts
+          configMap:
+            name: sso-vault-env
+            defaultMode: 0555

services/keycloak/vault-oidc-secret-ensure-job.yaml

@@ -16,6 +16,16 @@ spec:
           configMap:
             name: vault-oidc-secret-ensure-script
             defaultMode: 0555
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: sso-vault
+        - name: vault-scripts
+          configMap:
+            name: sso-vault-env
+            defaultMode: 0555
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -30,18 +40,13 @@ spec:
         - name: apply
           image: alpine:3.20
           command: ["/scripts/vault_oidc_secret_ensure.sh"]
-          env:
-            - name: KEYCLOAK_ADMIN
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: username
-            - name: KEYCLOAK_ADMIN_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: keycloak-admin
-                  key: password
           volumeMounts:
             - name: vault-oidc-secret-ensure-script
               mountPath: /scripts
               readOnly: true
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true

services/mailu/kustomization.yaml

@@ -4,7 +4,10 @@ kind: Kustomization
 namespace: mailu-mailserver
 resources:
   - namespace.yaml
+  - serviceaccount.yaml
+  - secretproviderclass.yaml
   - helmrelease.yaml
+  - vault-sync-deployment.yaml
   - certificate.yaml
   - vip-controller.yaml
   - unbound-configmap.yaml
@@ -16,6 +19,12 @@ resources:
   - front-lb.yaml
 
 configMapGenerator:
+  - name: mailu-vault-env
+    namespace: mailu-mailserver
+    files:
+      - mailu_vault_env.sh=scripts/mailu_vault_env.sh
+    options:
+      disableNameSuffixHash: true
   - name: mailu-sync-script
     namespace: mailu-mailserver
     files:

services/mailu/mailu-sync-cronjob.yaml

@@ -12,6 +12,7 @@ spec:
       template:
         spec:
           restartPolicy: OnFailure
+          serviceAccountName: mailu-vault-sync
           containers:
             - name: mailu-sync
               image: python:3.11-alpine
@@ -19,8 +20,10 @@ spec:
               command: ["/bin/sh", "-c"]
               args:
                 - |
+                  set -euo pipefail
+                  . /vault/scripts/mailu_vault_env.sh
                   pip install --no-cache-dir requests psycopg2-binary passlib >/tmp/pip.log \
-                  && python /app/sync.py
+                    && python /app/sync.py
               env:
                 - name: KEYCLOAK_BASE_URL
                   value: http://keycloak.sso.svc.cluster.local
@@ -34,35 +37,16 @@ spec:
                   value: postgres-service.postgres.svc.cluster.local
                 - name: MAILU_DB_PORT
                   value: "5432"
-                - name: MAILU_DB_NAME
-                  valueFrom:
-                    secretKeyRef:
-                      name: mailu-db-secret
-                      key: database
-                - name: MAILU_DB_USER
-                  valueFrom:
-                    secretKeyRef:
-                      name: mailu-db-secret
-                      key: username
-                - name: MAILU_DB_PASSWORD
-                  valueFrom:
-                    secretKeyRef:
-                      name: mailu-db-secret
-                      key: password
-                - name: KEYCLOAK_CLIENT_ID
-                  valueFrom:
-                    secretKeyRef:
-                      name: mailu-sync-credentials
-                      key: client-id
-                - name: KEYCLOAK_CLIENT_SECRET
-                  valueFrom:
-                    secretKeyRef:
-                      name: mailu-sync-credentials
-                      key: client-secret
               volumeMounts:
                 - name: sync-script
                   mountPath: /app/sync.py
                   subPath: sync.py
+                - name: vault-secrets
+                  mountPath: /vault/secrets
+                  readOnly: true
+                - name: vault-scripts
+                  mountPath: /vault/scripts
+                  readOnly: true
               resources:
                 requests:
                   cpu: 50m
@@ -75,3 +59,13 @@ spec:
               configMap:
                 name: mailu-sync-script
                 defaultMode: 0444
+            - name: vault-secrets
+              csi:
+                driver: secrets-store.csi.k8s.io
+                readOnly: true
+                volumeAttributes:
+                  secretProviderClass: mailu-vault
+            - name: vault-scripts
+              configMap:
+                name: mailu-vault-env
+                defaultMode: 0555

services/mailu/mailu-sync-job.yaml

@@ -8,6 +8,7 @@ spec:
   template:
     spec:
       restartPolicy: OnFailure
+      serviceAccountName: mailu-vault-sync
       containers:
         - name: mailu-sync
           image: python:3.11-alpine
@@ -15,8 +16,10 @@ spec:
           command: ["/bin/sh", "-c"]
           args:
             - |
+              set -euo pipefail
+              . /vault/scripts/mailu_vault_env.sh
               pip install --no-cache-dir requests psycopg2-binary passlib >/tmp/pip.log \
-              && python /app/sync.py
+                && python /app/sync.py
           env:
             - name: KEYCLOAK_BASE_URL
               value: http://keycloak.sso.svc.cluster.local
@@ -30,35 +33,16 @@ spec:
               value: postgres-service.postgres.svc.cluster.local
             - name: MAILU_DB_PORT
               value: "5432"
-            - name: MAILU_DB_NAME
-              valueFrom:
-                secretKeyRef:
-                  name: mailu-db-secret
-                  key: database
-            - name: MAILU_DB_USER
-              valueFrom:
-                secretKeyRef:
-                  name: mailu-db-secret
-                  key: username
-            - name: MAILU_DB_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: mailu-db-secret
-                  key: password
-            - name: KEYCLOAK_CLIENT_ID
-              valueFrom:
-                secretKeyRef:
-                  name: mailu-sync-credentials
-                  key: client-id
-            - name: KEYCLOAK_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: mailu-sync-credentials
-                  key: client-secret
           volumeMounts:
             - name: sync-script
               mountPath: /app/sync.py
               subPath: sync.py
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
           resources:
             requests:
               cpu: 50m
@@ -71,3 +55,13 @@ spec:
           configMap:
             name: mailu-sync-script
             defaultMode: 0444
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: mailu-vault
+        - name: vault-scripts
+          configMap:
+            name: mailu-vault-env
+            defaultMode: 0555

services/mailu/mailu-sync-listener.yaml

@@ -30,6 +30,7 @@ spec:
         app: mailu-sync-listener
     spec:
       restartPolicy: Always
+      serviceAccountName: mailu-vault-sync
       containers:
         - name: listener
           image: python:3.11-alpine
@@ -37,8 +38,10 @@ spec:
           command: ["/bin/sh", "-c"]
           args:
             - |
+              set -euo pipefail
+              . /vault/scripts/mailu_vault_env.sh
               pip install --no-cache-dir requests psycopg2-binary passlib >/tmp/pip.log \
-              && python /app/listener.py
+                && python /app/listener.py
           env:
             - name: KEYCLOAK_BASE_URL
               value: http://keycloak.sso.svc.cluster.local
@@ -52,31 +55,6 @@ spec:
               value: postgres-service.postgres.svc.cluster.local
             - name: MAILU_DB_PORT
               value: "5432"
-            - name: MAILU_DB_NAME
-              valueFrom:
-                secretKeyRef:
-                  name: mailu-db-secret
-                  key: database
-            - name: MAILU_DB_USER
-              valueFrom:
-                secretKeyRef:
-                  name: mailu-db-secret
-                  key: username
-            - name: MAILU_DB_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: mailu-db-secret
-                  key: password
-            - name: KEYCLOAK_CLIENT_ID
-              valueFrom:
-                secretKeyRef:
-                  name: mailu-sync-credentials
-                  key: client-id
-            - name: KEYCLOAK_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: mailu-sync-credentials
-                  key: client-secret
           volumeMounts:
             - name: sync-script
               mountPath: /app/sync.py
@@ -84,6 +62,12 @@ spec:
             - name: listener-script
               mountPath: /app/listener.py
               subPath: listener.py
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
           resources:
             requests:
               cpu: 50m
@@ -100,3 +84,13 @@ spec:
           configMap:
             name: mailu-sync-listener
             defaultMode: 0444
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: mailu-vault
+        - name: vault-scripts
+          configMap:
+            name: mailu-vault-env
+            defaultMode: 0555

services/mailu/scripts/mailu_vault_env.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env sh
+set -eu
+
+vault_dir="/vault/secrets"
+
+read_secret() {
+  cat "${vault_dir}/$1"
+}
+
+export MAILU_DB_NAME="$(read_secret mailu-db-secret__database)"
+export MAILU_DB_USER="$(read_secret mailu-db-secret__username)"
+export MAILU_DB_PASSWORD="$(read_secret mailu-db-secret__password)"
+export KEYCLOAK_CLIENT_ID="$(read_secret mailu-sync-credentials__client-id)"
+export KEYCLOAK_CLIENT_SECRET="$(read_secret mailu-sync-credentials__client-secret)"

services/mailu/secretproviderclass.yaml

@@ -0,0 +1,78 @@
+# services/mailu/secretproviderclass.yaml
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: mailu-vault
+  namespace: mailu-mailserver
+spec:
+  provider: vault
+  parameters:
+    vaultAddress: "http://vault.vault.svc.cluster.local:8200"
+    roleName: "mailu-mailserver"
+    objects: |
+      - objectName: "mailu-secret__secret-key"
+        secretPath: "kv/data/atlas/mailu/mailu-secret"
+        secretKey: "secret-key"
+      - objectName: "postmark-relay__relay-username"
+        secretPath: "kv/data/atlas/shared/postmark-relay"
+        secretKey: "relay-username"
+      - objectName: "postmark-relay__relay-password"
+        secretPath: "kv/data/atlas/shared/postmark-relay"
+        secretKey: "relay-password"
+      - objectName: "mailu-db-secret__database"
+        secretPath: "kv/data/atlas/mailu/mailu-db-secret"
+        secretKey: "database"
+      - objectName: "mailu-db-secret__username"
+        secretPath: "kv/data/atlas/mailu/mailu-db-secret"
+        secretKey: "username"
+      - objectName: "mailu-db-secret__password"
+        secretPath: "kv/data/atlas/mailu/mailu-db-secret"
+        secretKey: "password"
+      - objectName: "mailu-db-secret__url"
+        secretPath: "kv/data/atlas/mailu/mailu-db-secret"
+        secretKey: "url"
+      - objectName: "mailu-initial-account-secret__password"
+        secretPath: "kv/data/atlas/mailu/mailu-initial-account-secret"
+        secretKey: "password"
+      - objectName: "mailu-sync-credentials__client-id"
+        secretPath: "kv/data/atlas/mailu/mailu-sync-credentials"
+        secretKey: "client-id"
+      - objectName: "mailu-sync-credentials__client-secret"
+        secretPath: "kv/data/atlas/mailu/mailu-sync-credentials"
+        secretKey: "client-secret"
+  secretObjects:
+    - secretName: mailu-secret
+      type: Opaque
+      data:
+        - objectName: mailu-secret__secret-key
+          key: secret-key
+    - secretName: mailu-postmark-relay
+      type: Opaque
+      data:
+        - objectName: postmark-relay__relay-username
+          key: relay-username
+        - objectName: postmark-relay__relay-password
+          key: relay-password
+    - secretName: mailu-db-secret
+      type: Opaque
+      data:
+        - objectName: mailu-db-secret__database
+          key: database
+        - objectName: mailu-db-secret__username
+          key: username
+        - objectName: mailu-db-secret__password
+          key: password
+        - objectName: mailu-db-secret__url
+          key: url
+    - secretName: mailu-initial-account-secret
+      type: Opaque
+      data:
+        - objectName: mailu-initial-account-secret__password
+          key: password
+    - secretName: mailu-sync-credentials
+      type: Opaque
+      data:
+        - objectName: mailu-sync-credentials__client-id
+          key: client-id
+        - objectName: mailu-sync-credentials__client-secret
+          key: client-secret

services/mailu/serviceaccount.yaml

@@ -0,0 +1,6 @@
+# services/mailu/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mailu-vault-sync
+  namespace: mailu-mailserver

services/mailu/vault-sync-deployment.yaml

@@ -0,0 +1,34 @@
+# services/mailu/vault-sync-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mailu-vault-sync
+  namespace: mailu-mailserver
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mailu-vault-sync
+  template:
+    metadata:
+      labels:
+        app: mailu-vault-sync
+    spec:
+      serviceAccountName: mailu-vault-sync
+      containers:
+        - name: sync
+          image: alpine:3.20
+          command: ["/bin/sh", "-c"]
+          args:
+            - "sleep infinity"
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+      volumes:
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: mailu-vault

services/nextcloud-mail-sync/cronjob.yaml

@@ -17,47 +17,23 @@ spec:
           securityContext:
             runAsUser: 0
             runAsGroup: 0
+          serviceAccountName: nextcloud-vault
           containers:
             - name: mail-sync
               image: nextcloud:29-apache
               imagePullPolicy: IfNotPresent
               command:
-                - /bin/bash
+                - /bin/sh
-                - /sync/sync.sh
+                - -c
               env:
                 - name: KC_BASE
                   value: https://sso.bstein.dev
                 - name: KC_REALM
                   value: atlas
-                - name: KC_ADMIN_USER
-                  valueFrom:
-                    secretKeyRef:
-                      name: nextcloud-keycloak-admin
-                      key: username
-                - name: KC_ADMIN_PASS
-                  valueFrom:
-                    secretKeyRef:
-                      name: nextcloud-keycloak-admin
-                      key: password
                 - name: MAILU_DOMAIN
                   value: bstein.dev
                 - name: POSTGRES_HOST
                   value: postgres-service.postgres.svc.cluster.local
-                - name: POSTGRES_DB
-                  valueFrom:
-                    secretKeyRef:
-                      name: nextcloud-db
-                      key: database
-                - name: POSTGRES_USER
-                  valueFrom:
-                    secretKeyRef:
-                      name: nextcloud-db
-                      key: db-username
-                - name: POSTGRES_PASSWORD
-                  valueFrom:
-                    secretKeyRef:
-                      name: nextcloud-db
-                      key: db-password
               resources:
                 requests:
                   cpu: 100m
@@ -77,6 +53,17 @@ spec:
                 - name: sync-script
                   mountPath: /sync/sync.sh
                   subPath: sync.sh
+                - name: vault-secrets
+                  mountPath: /vault/secrets
+                  readOnly: true
+                - name: vault-scripts
+                  mountPath: /vault/scripts
+                  readOnly: true
+              args:
+                - |
+                  set -euo pipefail
+                  . /vault/scripts/nextcloud_vault_env.sh
+                  exec /sync/sync.sh
           volumes:
             - name: nextcloud-config-pvc
               persistentVolumeClaim:
@@ -94,3 +81,13 @@ spec:
               configMap:
                 name: nextcloud-mail-sync-script
                 defaultMode: 0755
+            - name: vault-secrets
+              csi:
+                driver: secrets-store.csi.k8s.io
+                readOnly: true
+                volumeAttributes:
+                  secretProviderClass: nextcloud-vault
+            - name: vault-scripts
+              configMap:
+                name: nextcloud-vault-env
+                defaultMode: 0555

services/nextcloud/deployment.yaml

@@ -22,6 +22,7 @@ spec:
         fsGroup: 33
         runAsUser: 33
         runAsGroup: 33
+      serviceAccountName: nextcloud-vault
       initContainers:
         - name: seed-nextcloud-web
           image: nextcloud:29-apache
@@ -80,6 +81,7 @@ spec:
           command: ["/bin/sh", "-c"]
           args:
             - |
+              . /vault/scripts/nextcloud_vault_env.sh
               installed="$(su -s /bin/sh www-data -c "php /var/www/html/occ status" 2>/dev/null | awk '/installed:/{print $3}' || true)"
               if [ ! -s /var/www/html/config/config.php ]; then
                 su -s /bin/sh www-data -c "php /var/www/html/occ maintenance:install --database pgsql --database-host \"${POSTGRES_HOST}\" --database-name \"${POSTGRES_DB}\" --database-user \"${POSTGRES_USER}\" --database-pass \"${POSTGRES_PASSWORD}\" --admin-user \"${NEXTCLOUD_ADMIN_USER}\" --admin-pass \"${NEXTCLOUD_ADMIN_PASSWORD}\" --data-dir /var/www/html/data"
@@ -150,41 +152,6 @@ spec:
           env:
             - name: POSTGRES_HOST
               value: postgres-service.postgres.svc.cluster.local
-            - name: POSTGRES_DB
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-db
-                  key: database
-            - name: POSTGRES_USER
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-db
-                  key: db-username
-            - name: POSTGRES_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-db
-                  key: db-password
-            - name: NEXTCLOUD_ADMIN_USER
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-admin
-                  key: admin-user
-            - name: NEXTCLOUD_ADMIN_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-admin
-                  key: admin-password
-            - name: OIDC_CLIENT_ID
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-oidc
-                  key: client-id
-            - name: OIDC_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-oidc
-                  key: client-secret
           volumeMounts:
             - name: nextcloud-web
               mountPath: /var/www/html
@@ -197,40 +164,26 @@ spec:
             - name: nextcloud-config-extra
               mountPath: /var/www/html/config/extra.config.php
               subPath: extra.config.php
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
       containers:
         - name: nextcloud
           image: nextcloud:29-apache
           imagePullPolicy: IfNotPresent
+          command: ["/bin/sh", "-c"]
+          args:
+            - >-
+              . /vault/scripts/nextcloud_vault_env.sh
+              && exec /entrypoint.sh apache2-foreground
           env:
             # DB (external secret required: nextcloud-db with keys username,password,database)
             - name: POSTGRES_HOST
               value: postgres-service.postgres.svc.cluster.local
-            - name: POSTGRES_DB
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-db
-                  key: database
-            - name: POSTGRES_USER
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-db
-                  key: db-username
-            - name: POSTGRES_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-db
-                  key: db-password
             # Admin bootstrap (external secret: nextcloud-admin with keys admin-user, admin-password)
-            - name: NEXTCLOUD_ADMIN_USER
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-admin
-                  key: admin-user
-            - name: NEXTCLOUD_ADMIN_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-admin
-                  key: admin-password
             - name: NEXTCLOUD_TRUSTED_DOMAINS
               value: cloud.bstein.dev
             - name: OVERWRITEHOST
@@ -246,31 +199,11 @@ spec:
               value: "587"
             - name: SMTP_SECURE
               value: tls
-            - name: SMTP_NAME
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-smtp
-                  key: smtp-username
-            - name: SMTP_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-smtp
-                  key: smtp-password
             - name: MAIL_FROM_ADDRESS
               value: no-reply
             - name: MAIL_DOMAIN
               value: bstein.dev
             # OIDC (external secret: nextcloud-oidc with keys client-id, client-secret)
-            - name: OIDC_CLIENT_ID
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-oidc
-                  key: client-id
-            - name: OIDC_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: nextcloud-oidc
-                  key: client-secret
             - name: NEXTCLOUD_UPDATE
               value: "1"
             - name: APP_INSTALL
@@ -290,6 +223,12 @@ spec:
             - name: nextcloud-config-extra
               mountPath: /var/www/html/config/extra.config.php
               subPath: extra.config.php
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+            - name: vault-scripts
+              mountPath: /vault/scripts
+              readOnly: true
           resources:
             requests:
               cpu: 250m
@@ -314,3 +253,13 @@ spec:
           configMap:
             name: nextcloud-config
             defaultMode: 0444
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: nextcloud-vault
+        - name: vault-scripts
+          configMap:
+            name: nextcloud-vault-env
+            defaultMode: 0555

services/nextcloud/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 namespace: nextcloud
 resources:
   - namespace.yaml
+  - serviceaccount.yaml
+  - secretproviderclass.yaml
   - configmap.yaml
   - pvc.yaml
   - deployment.yaml
@@ -13,6 +15,11 @@ resources:
   - service.yaml
   - ingress.yaml
 configMapGenerator:
+  - name: nextcloud-vault-env
+    files:
+      - nextcloud_vault_env.sh=scripts/nextcloud_vault_env.sh
+    options:
+      disableNameSuffixHash: true
   - name: nextcloud-maintenance-script
     files:
       - maintenance.sh=scripts/nextcloud-maintenance.sh

services/nextcloud/maintenance-cronjob.yaml

@@ -15,24 +15,20 @@ spec:
           securityContext:
             runAsUser: 0
             runAsGroup: 0
+          serviceAccountName: nextcloud-vault
           containers:
             - name: maintenance
               image: nextcloud:29-apache
               imagePullPolicy: IfNotPresent
-              command: ["/bin/bash", "/maintenance/maintenance.sh"]
+              command: ["/bin/sh", "-c"]
+              args:
+                - |
+                  set -euo pipefail
+                  . /vault/scripts/nextcloud_vault_env.sh
+                  exec /maintenance/maintenance.sh
               env:
                 - name: NC_URL
                   value: https://cloud.bstein.dev
-                - name: ADMIN_USER
-                  valueFrom:
-                    secretKeyRef:
-                      name: nextcloud-admin
-                      key: admin-user
-                - name: ADMIN_PASS
-                  valueFrom:
-                    secretKeyRef:
-                      name: nextcloud-admin
-                      key: admin-password
               volumeMounts:
                 - name: nextcloud-web
                   mountPath: /var/www/html
@@ -45,6 +41,12 @@ spec:
                 - name: maintenance-script
                   mountPath: /maintenance/maintenance.sh
                   subPath: maintenance.sh
+                - name: vault-secrets
+                  mountPath: /vault/secrets
+                  readOnly: true
+                - name: vault-scripts
+                  mountPath: /vault/scripts
+                  readOnly: true
               resources:
                 requests:
                   cpu: 100m
@@ -69,3 +71,13 @@ spec:
               configMap:
                 name: nextcloud-maintenance-script
                 defaultMode: 0755
+            - name: vault-secrets
+              csi:
+                driver: secrets-store.csi.k8s.io
+                readOnly: true
+                volumeAttributes:
+                  secretProviderClass: nextcloud-vault
+            - name: vault-scripts
+              configMap:
+                name: nextcloud-vault-env
+                defaultMode: 0555

services/nextcloud/scripts/nextcloud_vault_env.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env sh
+set -eu
+
+vault_dir="/vault/secrets"
+
+read_secret() {
+  cat "${vault_dir}/$1"
+}
+
+export POSTGRES_DB="$(read_secret nextcloud-db__database)"
+export POSTGRES_USER="$(read_secret nextcloud-db__db-username)"
+export POSTGRES_PASSWORD="$(read_secret nextcloud-db__db-password)"
+
+export NEXTCLOUD_ADMIN_USER="$(read_secret nextcloud-admin__admin-user)"
+export NEXTCLOUD_ADMIN_PASSWORD="$(read_secret nextcloud-admin__admin-password)"
+
+export ADMIN_USER="${NEXTCLOUD_ADMIN_USER}"
+export ADMIN_PASS="${NEXTCLOUD_ADMIN_PASSWORD}"
+
+export OIDC_CLIENT_ID="$(read_secret nextcloud-oidc__client-id)"
+export OIDC_CLIENT_SECRET="$(read_secret nextcloud-oidc__client-secret)"
+
+export SMTP_NAME="$(read_secret nextcloud-smtp__smtp-username)"
+export SMTP_PASSWORD="$(read_secret nextcloud-smtp__smtp-password)"
+
+export KC_ADMIN_USER="$(read_secret keycloak-admin__username)"
+export KC_ADMIN_PASS="$(read_secret keycloak-admin__password)"

services/nextcloud/secretproviderclass.yaml

@@ -0,0 +1,45 @@
+# services/nextcloud/secretproviderclass.yaml
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: nextcloud-vault
+  namespace: nextcloud
+spec:
+  provider: vault
+  parameters:
+    vaultAddress: "http://vault.vault.svc.cluster.local:8200"
+    roleName: "nextcloud"
+    objects: |
+      - objectName: "nextcloud-db__database"
+        secretPath: "kv/data/atlas/nextcloud/nextcloud-db"
+        secretKey: "database"
+      - objectName: "nextcloud-db__db-username"
+        secretPath: "kv/data/atlas/nextcloud/nextcloud-db"
+        secretKey: "db-username"
+      - objectName: "nextcloud-db__db-password"
+        secretPath: "kv/data/atlas/nextcloud/nextcloud-db"
+        secretKey: "db-password"
+      - objectName: "nextcloud-admin__admin-user"
+        secretPath: "kv/data/atlas/nextcloud/nextcloud-admin"
+        secretKey: "admin-user"
+      - objectName: "nextcloud-admin__admin-password"
+        secretPath: "kv/data/atlas/nextcloud/nextcloud-admin"
+        secretKey: "admin-password"
+      - objectName: "nextcloud-oidc__client-id"
+        secretPath: "kv/data/atlas/nextcloud/nextcloud-oidc"
+        secretKey: "client-id"
+      - objectName: "nextcloud-oidc__client-secret"
+        secretPath: "kv/data/atlas/nextcloud/nextcloud-oidc"
+        secretKey: "client-secret"
+      - objectName: "nextcloud-smtp__smtp-username"
+        secretPath: "kv/data/atlas/nextcloud/nextcloud-smtp"
+        secretKey: "smtp-username"
+      - objectName: "nextcloud-smtp__smtp-password"
+        secretPath: "kv/data/atlas/nextcloud/nextcloud-smtp"
+        secretKey: "smtp-password"
+      - objectName: "keycloak-admin__username"
+        secretPath: "kv/data/atlas/shared/keycloak-admin"
+        secretKey: "username"
+      - objectName: "keycloak-admin__password"
+        secretPath: "kv/data/atlas/shared/keycloak-admin"
+        secretKey: "password"

services/nextcloud/serviceaccount.yaml

@@ -0,0 +1,6 @@
+# services/nextcloud/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nextcloud-vault
+  namespace: nextcloud

services/vault/scripts/vault_k8s_auth_configure.sh

@@ -35,66 +35,71 @@ vault write auth/kubernetes/config \
   kubernetes_host="${k8s_host}" \
   kubernetes_ca_cert="${k8s_ca}"
 
-for namespace in outline planka bstein-dev-home gitea vaultwarden sso; do
+write_policy_and_role() {
-  policy_name="${namespace}"
+  role="$1"
-  service_account=""
+  namespace="$2"
-  shared_paths=""
+  service_accounts="$3"
+  read_paths="$4"
+  write_paths="$5"
 
-  case "${namespace}" in
+  policy_body=""
-    outline)
+  for path in ${read_paths}; do
-      service_account="outline-vault"
-      ;;
-    planka)
-      service_account="planka-vault"
-      ;;
-    bstein-dev-home)
-      service_account="bstein-dev-home"
-      shared_paths="shared/chat-ai-keys-runtime shared/portal-e2e-client"
-      ;;
-    gitea)
-      service_account="gitea-vault"
-      ;;
-    vaultwarden)
-      service_account="vaultwarden-vault"
-      ;;
-    sso)
-      service_account="sso-vault,mas-secrets-ensure"
-      shared_paths="shared/keycloak-admin shared/portal-e2e-client"
-      ;;
-    *)
-      log "unknown namespace ${namespace}"
-      exit 1
-      ;;
-  esac
-
-  policy_body="$(cat <<EOF
-path "kv/data/atlas/${namespace}/*" {
-  capabilities = ["read"]
-}
-path "kv/metadata/atlas/${namespace}/*" {
-  capabilities = ["list"]
-}
-EOF
-)"
-
-  for shared in ${shared_paths}; do
     policy_body="${policy_body}
-path \"kv/data/atlas/${shared}\" {
+path \"kv/data/atlas/${path}\" {
   capabilities = [\"read\"]
 }
-path \"kv/metadata/atlas/${shared}\" {
+path \"kv/metadata/atlas/${path}\" {
+  capabilities = [\"list\"]
+}
+"
+  done
+  for path in ${write_paths}; do
+    policy_body="${policy_body}
+path \"kv/data/atlas/${path}\" {
+  capabilities = [\"create\", \"update\", \"read\"]
+}
+path \"kv/metadata/atlas/${path}\" {
   capabilities = [\"list\"]
 }
 "
   done
 
-  log "writing policy ${policy_name}"
+  log "writing policy ${role}"
-  printf '%s\n' "${policy_body}" | vault policy write "${policy_name}" -
+  printf '%s\n' "${policy_body}" | vault policy write "${role}" -
 
-  log "writing role ${namespace}"
+  log "writing role ${role}"
-  vault write "auth/kubernetes/role/${namespace}" \
+  vault write "auth/kubernetes/role/${role}" \
-    bound_service_account_names="${service_account}" \
+    bound_service_account_names="${service_accounts}" \
     bound_service_account_namespaces="${namespace}" \
-    policies="${policy_name}" \
+    policies="${role}" \
     ttl="${role_ttl}"
-done
+}
+
+write_policy_and_role "outline" "outline" "outline-vault" \
+  "outline/*" ""
+write_policy_and_role "planka" "planka" "planka-vault" \
+  "planka/*" ""
+write_policy_and_role "bstein-dev-home" "bstein-dev-home" "bstein-dev-home" \
+  "bstein-dev-home/* shared/chat-ai-keys-runtime shared/portal-e2e-client" ""
+write_policy_and_role "gitea" "gitea" "gitea-vault" \
+  "gitea/*" ""
+write_policy_and_role "vaultwarden" "vaultwarden" "vaultwarden-vault" \
+  "vaultwarden/*" ""
+write_policy_and_role "sso" "sso" "sso-vault,mas-secrets-ensure" \
+  "sso/* shared/keycloak-admin shared/portal-e2e-client" ""
+write_policy_and_role "mailu-mailserver" "mailu-mailserver" "mailu-vault-sync" \
+  "mailu/* shared/postmark-relay" ""
+write_policy_and_role "harbor" "harbor" "harbor-vault-sync" \
+  "harbor/*" ""
+write_policy_and_role "nextcloud" "nextcloud" "nextcloud-vault" \
+  "nextcloud/* shared/keycloak-admin" ""
+write_policy_and_role "comms" "comms" "comms-vault,atlasbot" \
+  "comms/* shared/chat-ai-keys-runtime" ""
+
+write_policy_and_role "sso-secrets" "sso" "mas-secrets-ensure" \
+  "shared/keycloak-admin" \
+  "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc"
+write_policy_and_role "comms-secrets" "comms" \
+  "comms-secrets-ensure,mas-db-ensure,mas-admin-client-secret-writer,othrys-synapse-signingkey-job" \
+  "" \
+  "comms/turn-shared-secret comms/livekit-api comms/synapse-redis comms/synapse-macaroon comms/atlasbot-credentials-runtime comms/synapse-db comms/mas-db comms/mas-admin-client-runtime comms/mas-secrets-runtime comms/othrys-synapse-signingkey"