diff --git a/infrastructure/vault-csi/secrets-store-csi-driver.yaml b/infrastructure/vault-csi/secrets-store-csi-driver.yaml index fec4758..0b249fc 100644 --- a/infrastructure/vault-csi/secrets-store-csi-driver.yaml +++ b/infrastructure/vault-csi/secrets-store-csi-driver.yaml @@ -16,5 +16,5 @@ spec: namespace: flux-system values: syncSecret: - enabled: false + enabled: true enableSecretRotation: false diff --git a/services/comms/atlasbot-deployment.yaml b/services/comms/atlasbot-deployment.yaml index f9e1f79..0622d32 100644 --- a/services/comms/atlasbot-deployment.yaml +++ b/services/comms/atlasbot-deployment.yaml @@ -27,7 +27,8 @@ spec: command: ["/bin/sh","-c"] args: - | - python /app/bot.py + . /vault/scripts/comms_vault_env.sh + exec python /app/bot.py env: - name: MATRIX_BASE value: http://othrys-synapse-matrix-synapse:8008 @@ -39,16 +40,6 @@ spec: value: http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428 - name: BOT_USER value: atlasbot - - name: BOT_PASS - valueFrom: - secretKeyRef: - name: atlasbot-credentials-runtime - key: bot-password - - name: CHAT_API_KEY - valueFrom: - secretKeyRef: - name: chat-ai-keys-runtime - key: matrix - name: OLLAMA_URL value: https://chat.ai.bstein.dev/ - name: OLLAMA_MODEL @@ -67,6 +58,12 @@ spec: - name: kb mountPath: /kb readOnly: true + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true volumes: - name: code configMap: @@ -85,3 +82,13 @@ spec: path: catalog/runbooks.json - key: atlas-http.mmd path: diagrams/atlas-http.mmd + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault + - name: vault-scripts + configMap: + name: comms-vault-env + defaultMode: 0555 diff --git a/services/comms/bstein-force-leave-job.yaml b/services/comms/bstein-force-leave-job.yaml index 956330b..0c760a4 100644 --- a/services/comms/bstein-force-leave-job.yaml +++ b/services/comms/bstein-force-leave-job.yaml @@ -9,25 +9,26 @@ spec: template: spec: restartPolicy: Never + serviceAccountName: comms-vault volumes: - - name: mas-admin-client - secret: - secretName: mas-admin-client-runtime - items: - - key: client_secret - path: client_secret + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault containers: - name: leave image: python:3.11-slim volumeMounts: - - name: mas-admin-client - mountPath: /etc/mas-admin-client + - name: vault-secrets + mountPath: /vault/secrets readOnly: true env: - name: MAS_ADMIN_CLIENT_ID value: 01KDXMVQBQ5JNY6SEJPZW6Z8BM - name: MAS_ADMIN_CLIENT_SECRET_FILE - value: /etc/mas-admin-client/client_secret + value: /vault/secrets/mas-admin-client-runtime__client_secret - name: MAS_TOKEN_URL value: http://matrix-authentication-service:8080/oauth2/token - name: MAS_ADMIN_API_BASE diff --git a/services/comms/comms-secrets-ensure-job.yaml b/services/comms/comms-secrets-ensure-job.yaml index dffb222..cc8ee02 100644 --- a/services/comms/comms-secrets-ensure-job.yaml +++ b/services/comms/comms-secrets-ensure-job.yaml @@ -20,73 +20,58 @@ spec: set -eu trap 'echo "comms-secrets-ensure failed"; sleep 300' ERR umask 077 + apk add --no-cache curl jq >/dev/null safe_pass() { head -c 32 /dev/urandom | base64 | tr -d '\n' | tr '+/' '-_' | tr -d '=' } - get_secret_value() { - ns="$1" - name="$2" - key="$3" - kubectl -n "${ns}" get secret "${name}" -o "jsonpath={.data.${key}}" 2>/dev/null | base64 -d 2>/dev/null || true - } - - ensure_secret_key() { - ns="$1" - name="$2" - key="$3" - value="$4" - if ! kubectl -n "${ns}" get secret "${name}" >/dev/null 2>&1; then - kubectl -n "${ns}" create secret generic "${name}" --from-literal="${key}=${value}" >/dev/null - return - fi - existing="$(kubectl -n "${ns}" get secret "${name}" -o "jsonpath={.data.${key}}" 2>/dev/null || true)" - if [ -z "${existing}" ]; then - b64="$(printf '%s' "${value}" | base64 | tr -d '\n')" - payload="$(printf '{"data":{"%s":"%s"}}' "${key}" "${b64}")" - kubectl -n "${ns}" patch secret "${name}" --type=merge -p "${payload}" >/dev/null - fi - } - - ensure_chat_secret() { - ns="$1" - if ! kubectl -n "${ns}" get secret chat-ai-keys-runtime >/dev/null 2>&1; then - kubectl -n "${ns}" create secret generic chat-ai-keys-runtime \ - --from-literal=matrix="${CHAT_KEY_MATRIX}" \ - --from-literal=homepage="${CHAT_KEY_HOMEPAGE}" >/dev/null - return - fi - ensure_secret_key "${ns}" chat-ai-keys-runtime matrix "${CHAT_KEY_MATRIX}" - ensure_secret_key "${ns}" chat-ai-keys-runtime homepage "${CHAT_KEY_HOMEPAGE}" - } - - CHAT_KEY_MATRIX="$(get_secret_value comms chat-ai-keys-runtime matrix)" - CHAT_KEY_HOMEPAGE="$(get_secret_value comms chat-ai-keys-runtime homepage)" - if [ -z "${CHAT_KEY_MATRIX}" ] || [ -z "${CHAT_KEY_HOMEPAGE}" ]; then - ALT_MATRIX="$(get_secret_value bstein-dev-home chat-ai-keys-runtime matrix)" - ALT_HOMEPAGE="$(get_secret_value bstein-dev-home chat-ai-keys-runtime homepage)" - [ -z "${CHAT_KEY_MATRIX}" ] && CHAT_KEY_MATRIX="${ALT_MATRIX}" - [ -z "${CHAT_KEY_HOMEPAGE}" ] && CHAT_KEY_HOMEPAGE="${ALT_HOMEPAGE}" + vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" + vault_role="${VAULT_ROLE:-comms-secrets}" + jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" + login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')" + vault_token="$(curl -sS --request POST --data "${login_payload}" \ + "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" + if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then + echo "vault login failed" >&2 + exit 1 fi - [ -z "${CHAT_KEY_MATRIX}" ] && CHAT_KEY_MATRIX="$(safe_pass)" - [ -z "${CHAT_KEY_HOMEPAGE}" ] && CHAT_KEY_HOMEPAGE="$(safe_pass)" - ensure_chat_secret comms - ensure_chat_secret bstein-dev-home + vault_read() { + path="$1" + key="$2" + curl -sS -H "X-Vault-Token: ${vault_token}" \ + "${vault_addr}/v1/kv/data/atlas/${path}" | jq -r --arg key "${key}" '.data.data[$key] // empty' + } - ensure_secret_key comms turn-shared-secret TURN_STATIC_AUTH_SECRET "$(safe_pass)" - ensure_secret_key comms livekit-api primary "$(safe_pass)" - ensure_secret_key comms synapse-redis redis-password "$(safe_pass)" - ensure_secret_key comms synapse-macaroon macaroon_secret_key "$(safe_pass)" - ensure_secret_key comms atlasbot-credentials-runtime bot-password "$(safe_pass)" - ensure_secret_key comms atlasbot-credentials-runtime seeder-password "$(safe_pass)" + vault_write() { + path="$1" + key="$2" + value="$3" + payload="$(jq -nc --arg key "${key}" --arg value "${value}" '{data:{($key):$value}}')" + curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \ + -d "${payload}" "${vault_addr}/v1/kv/data/atlas/${path}" >/dev/null + } - SYN_PASS="$(get_secret_value comms synapse-db POSTGRES_PASSWORD)" - if [ -z "${SYN_PASS}" ]; then - SYN_PASS="$(safe_pass)" - kubectl -n comms create secret generic synapse-db --from-literal=POSTGRES_PASSWORD="${SYN_PASS}" >/dev/null - fi + ensure_key() { + path="$1" + key="$2" + current="$(vault_read "${path}" "${key}")" + if [ -z "${current}" ]; then + current="$(safe_pass)" + vault_write "${path}" "${key}" "${current}" + fi + printf '%s' "${current}" + } + + ensure_key "comms/turn-shared-secret" "TURN_STATIC_AUTH_SECRET" >/dev/null + ensure_key "comms/livekit-api" "primary" >/dev/null + ensure_key "comms/synapse-redis" "redis-password" >/dev/null + ensure_key "comms/synapse-macaroon" "macaroon_secret_key" >/dev/null + ensure_key "comms/atlasbot-credentials-runtime" "bot-password" >/dev/null + ensure_key "comms/atlasbot-credentials-runtime" "seeder-password" >/dev/null + + SYN_PASS="$(ensure_key "comms/synapse-db" "POSTGRES_PASSWORD")" POD_NAME="$(kubectl -n postgres get pods -l app=postgres -o jsonpath='{.items[0].metadata.name}')" if [ -z "${POD_NAME}" ]; then diff --git a/services/comms/coturn.yaml b/services/comms/coturn.yaml index 12fa78a..ac7e57b 100644 --- a/services/comms/coturn.yaml +++ b/services/comms/coturn.yaml @@ -15,6 +15,7 @@ spec: labels: app: coturn spec: + serviceAccountName: comms-vault nodeSelector: hardware: rpi5 affinity: @@ -33,6 +34,7 @@ spec: - /bin/sh - -c - | + . /vault/scripts/comms_vault_env.sh exec /usr/bin/turnserver \ --no-cli \ --fingerprint \ @@ -57,11 +59,6 @@ spec: fieldPath: status.podIP - name: TURN_PUBLIC_IP value: "38.28.125.112" - - name: TURN_STATIC_AUTH_SECRET - valueFrom: - secretKeyRef: - name: turn-shared-secret - key: TURN_STATIC_AUTH_SECRET ports: - name: turn-udp containerPort: 3478 @@ -76,6 +73,12 @@ spec: - name: tls mountPath: /etc/coturn/tls readOnly: true + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true resources: requests: cpu: 200m @@ -87,6 +90,16 @@ spec: - name: tls secret: secretName: turn-live-tls + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault + - name: vault-scripts + configMap: + name: comms-vault-env + defaultMode: 0555 --- apiVersion: v1 kind: Service diff --git a/services/comms/guest-name-job.yaml b/services/comms/guest-name-job.yaml index 156617d..1f9004e 100644 --- a/services/comms/guest-name-job.yaml +++ b/services/comms/guest-name-job.yaml @@ -16,19 +16,27 @@ spec: template: spec: restartPolicy: Never + serviceAccountName: comms-vault volumes: - - name: mas-admin-client - secret: - secretName: mas-admin-client-runtime - items: - - key: client_secret - path: client_secret + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault + - name: vault-scripts + configMap: + name: comms-vault-env + defaultMode: 0555 containers: - name: rename image: python:3.11-slim volumeMounts: - - name: mas-admin-client - mountPath: /etc/mas-admin-client + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts readOnly: true env: - name: SYNAPSE_BASE @@ -36,7 +44,7 @@ spec: - name: MAS_ADMIN_CLIENT_ID value: 01KDXMVQBQ5JNY6SEJPZW6Z8BM - name: MAS_ADMIN_CLIENT_SECRET_FILE - value: /etc/mas-admin-client/client_secret + value: /vault/secrets/mas-admin-client-runtime__client_secret - name: MAS_ADMIN_API_BASE value: http://matrix-authentication-service:8081/api/admin/v1 - name: MAS_TOKEN_URL @@ -51,16 +59,12 @@ spec: value: synapse - name: PGUSER value: synapse - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: synapse-db - key: POSTGRES_PASSWORD command: - /bin/sh - -c - | set -euo pipefail + . /vault/scripts/comms_vault_env.sh pip install --no-cache-dir requests psycopg2-binary >/dev/null python - <<'PY' import base64 diff --git a/services/comms/guest-register-deployment.yaml b/services/comms/guest-register-deployment.yaml index 284cc42..bdf5c37 100644 --- a/services/comms/guest-register-deployment.yaml +++ b/services/comms/guest-register-deployment.yaml @@ -17,6 +17,7 @@ spec: labels: app.kubernetes.io/name: matrix-guest-register spec: + serviceAccountName: comms-vault securityContext: runAsNonRoot: true runAsUser: 10001 @@ -42,7 +43,7 @@ spec: - name: MAS_ADMIN_CLIENT_ID value: 01KDXMVQBQ5JNY6SEJPZW6Z8BM - name: MAS_ADMIN_CLIENT_SECRET_FILE - value: /etc/mas/admin-client/client_secret + value: /vault/secrets/mas-admin-client-runtime__client_secret - name: MAS_ADMIN_API_BASE value: http://matrix-authentication-service:8081/api/admin/v1 - name: SYNAPSE_BASE @@ -83,8 +84,8 @@ spec: mountPath: /app/server.py subPath: server.py readOnly: true - - name: mas-admin-client - mountPath: /etc/mas/admin-client + - name: vault-secrets + mountPath: /vault/secrets readOnly: true command: - python @@ -96,9 +97,9 @@ spec: items: - key: server.py path: server.py - - name: mas-admin-client - secret: - secretName: mas-admin-client-runtime - items: - - key: client_secret - path: client_secret + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault diff --git a/services/comms/kustomization.yaml b/services/comms/kustomization.yaml index 5e50d0f..b0cc0da 100644 --- a/services/comms/kustomization.yaml +++ b/services/comms/kustomization.yaml @@ -4,6 +4,8 @@ kind: Kustomization namespace: comms resources: - namespace.yaml + - serviceaccount.yaml + - secretproviderclass.yaml - mas-configmap.yaml - helmrelease.yaml - livekit-config.yaml @@ -18,6 +20,7 @@ resources: - comms-secrets-ensure-rbac.yaml - mas-db-ensure-rbac.yaml - synapse-signingkey-ensure-rbac.yaml + - vault-sync-deployment.yaml - mas-admin-client-secret-ensure-job.yaml - mas-db-ensure-job.yaml - comms-secrets-ensure-job.yaml @@ -40,6 +43,11 @@ resources: - matrix-ingress.yaml configMapGenerator: + - name: comms-vault-env + files: + - comms_vault_env.sh=scripts/comms_vault_env.sh + options: + disableNameSuffixHash: true - name: matrix-guest-register files: - server.py=scripts/guest-register/server.py diff --git a/services/comms/livekit-token-deployment.yaml b/services/comms/livekit-token-deployment.yaml index 1b4cdca..750872c 100644 --- a/services/comms/livekit-token-deployment.yaml +++ b/services/comms/livekit-token-deployment.yaml @@ -15,6 +15,7 @@ spec: labels: app: livekit-token-service spec: + serviceAccountName: comms-vault nodeSelector: hardware: rpi5 affinity: @@ -33,21 +34,29 @@ spec: containers: - name: token-service image: ghcr.io/element-hq/lk-jwt-service:0.3.0 + command: + - /bin/sh + - -c + - | + . /vault/scripts/comms_vault_env.sh + exec /lk-jwt-service env: - name: LIVEKIT_URL value: wss://kit.live.bstein.dev/livekit/sfu - name: LIVEKIT_KEY value: primary - - name: LIVEKIT_SECRET - valueFrom: - secretKeyRef: - name: livekit-api - key: primary - name: LIVEKIT_FULL_ACCESS_HOMESERVERS value: live.bstein.dev ports: - containerPort: 8080 name: http + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true resources: requests: cpu: 50m @@ -55,6 +64,17 @@ spec: limits: cpu: 300m memory: 256Mi + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault + - name: vault-scripts + configMap: + name: comms-vault-env + defaultMode: 0555 --- apiVersion: v1 kind: Service diff --git a/services/comms/livekit.yaml b/services/comms/livekit.yaml index 46d57f8..adad92a 100644 --- a/services/comms/livekit.yaml +++ b/services/comms/livekit.yaml @@ -17,6 +17,7 @@ spec: labels: app: livekit spec: + serviceAccountName: comms-vault enableServiceLinks: false nodeSelector: hardware: rpi5 @@ -36,16 +37,11 @@ spec: args: - | set -euo pipefail + . /vault/scripts/comms_vault_env.sh umask 077 TURN_PASSWORD_ESCAPED="$(printf '%s' "${TURN_PASSWORD}" | sed 's/[\\/&]/\\&/g')" sed "s/@@TURN_PASSWORD@@/${TURN_PASSWORD_ESCAPED}/g" /etc/livekit-template/livekit.yaml > /etc/livekit/livekit.yaml chmod 0644 /etc/livekit/livekit.yaml - env: - - name: TURN_PASSWORD - valueFrom: - secretKeyRef: - name: turn-shared-secret - key: TURN_STATIC_AUTH_SECRET volumeMounts: - name: config-template mountPath: /etc/livekit-template @@ -53,6 +49,12 @@ spec: - name: config mountPath: /etc/livekit readOnly: false + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true containers: - name: livekit image: livekit/livekit-server:v1.9.0 @@ -61,6 +63,7 @@ spec: - -c - | set -euo pipefail + . /vault/scripts/comms_vault_env.sh umask 077 printf "%s: %s\n" "${LIVEKIT_API_KEY_ID}" "${LIVEKIT_API_SECRET}" > /var/run/livekit/keys chmod 600 /var/run/livekit/keys @@ -68,11 +71,6 @@ spec: env: - name: LIVEKIT_API_KEY_ID value: primary - - name: LIVEKIT_API_SECRET - valueFrom: - secretKeyRef: - name: livekit-api - key: primary ports: - containerPort: 7880 name: http @@ -92,6 +90,12 @@ spec: readOnly: true - name: runtime-keys mountPath: /var/run/livekit + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true resources: requests: cpu: 500m @@ -110,6 +114,16 @@ spec: emptyDir: {} - name: runtime-keys emptyDir: {} + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault + - name: vault-scripts + configMap: + name: comms-vault-env + defaultMode: 0555 --- apiVersion: v1 kind: Service diff --git a/services/comms/mas-admin-client-secret-ensure-job.yaml b/services/comms/mas-admin-client-secret-ensure-job.yaml index 9b76290..a84f68e 100644 --- a/services/comms/mas-admin-client-secret-ensure-job.yaml +++ b/services/comms/mas-admin-client-secret-ensure-job.yaml @@ -67,18 +67,29 @@ spec: args: - | set -euo pipefail - if kubectl -n comms get secret mas-admin-client-runtime >/dev/null 2>&1; then - if kubectl -n comms get secret mas-admin-client-runtime -o jsonpath='{.data.client_secret}' 2>/dev/null | grep -q .; then - exit 0 - fi - else - kubectl -n comms create secret generic mas-admin-client-runtime \ - --from-file=client_secret=/work/client_secret >/dev/null + apk add --no-cache curl jq >/dev/null + + vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" + vault_role="${VAULT_ROLE:-comms-secrets}" + jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" + login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')" + vault_token="$(curl -sS --request POST --data "${login_payload}" \ + "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" + if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then + echo "vault login failed" >&2 + exit 1 + fi + + current="$(curl -sS -H "X-Vault-Token: ${vault_token}" \ + "${vault_addr}/v1/kv/data/atlas/comms/mas-admin-client-runtime" | jq -r '.data.data.client_secret // empty')" + if [ -n "${current}" ]; then exit 0 fi - secret_b64="$(base64 /work/client_secret | tr -d '\n')" - payload="$(printf '{"data":{"client_secret":"%s"}}' "${secret_b64}")" - kubectl -n comms patch secret mas-admin-client-runtime --type=merge -p "${payload}" >/dev/null + + value="$(cat /work/client_secret)" + payload="$(jq -nc --arg value "${value}" '{data:{client_secret:$value}}')" + curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \ + -d "${payload}" "${vault_addr}/v1/kv/data/atlas/comms/mas-admin-client-runtime" >/dev/null volumeMounts: - name: work mountPath: /work diff --git a/services/comms/mas-db-ensure-job.yaml b/services/comms/mas-db-ensure-job.yaml index 1d1492e..28e7825 100644 --- a/services/comms/mas-db-ensure-job.yaml +++ b/services/comms/mas-db-ensure-job.yaml @@ -24,18 +24,35 @@ spec: head -c 32 /dev/urandom | base64 | tr -d '\n' | tr '+/' '-_' | tr -d '=' } - EXISTING_B64="$(kubectl -n comms get secret mas-db -o jsonpath='{.data.password}' 2>/dev/null || true)" - if [ -n "${EXISTING_B64}" ]; then - MAS_PASS="$(printf '%s' "${EXISTING_B64}" | base64 -d)" - if printf '%s' "${MAS_PASS}" | grep -Eq '[^A-Za-z0-9_-]'; then - MAS_PASS="$(safe_pass)" - MAS_B64="$(printf '%s' "${MAS_PASS}" | base64 | tr -d '\n')" - payload="$(printf '{"data":{"password":"%s"}}' "${MAS_B64}")" - kubectl -n comms patch secret mas-db --type=merge -p "${payload}" >/dev/null - fi - else + apk add --no-cache curl jq >/dev/null + + vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" + vault_role="${VAULT_ROLE:-comms-secrets}" + jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" + login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')" + vault_token="$(curl -sS --request POST --data "${login_payload}" \ + "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" + if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then + echo "vault login failed" >&2 + exit 1 + fi + + vault_read() { + curl -sS -H "X-Vault-Token: ${vault_token}" \ + "${vault_addr}/v1/kv/data/atlas/comms/mas-db" | jq -r '.data.data.password // empty' + } + + vault_write() { + value="$1" + payload="$(jq -nc --arg value "${value}" '{data:{password:$value}}')" + curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \ + -d "${payload}" "${vault_addr}/v1/kv/data/atlas/comms/mas-db" >/dev/null + } + + MAS_PASS="$(vault_read)" + if [ -z "${MAS_PASS}" ] || printf '%s' "${MAS_PASS}" | grep -Eq '[^A-Za-z0-9_-]'; then MAS_PASS="$(safe_pass)" - kubectl -n comms create secret generic mas-db --from-literal=password="${MAS_PASS}" >/dev/null + vault_write "${MAS_PASS}" fi POD_NAME="$(kubectl -n postgres get pods -l app=postgres -o jsonpath='{.items[0].metadata.name}')" diff --git a/services/comms/mas-deployment.yaml b/services/comms/mas-deployment.yaml index 2117c17..c7e6821 100644 --- a/services/comms/mas-deployment.yaml +++ b/services/comms/mas-deployment.yaml @@ -18,6 +18,7 @@ spec: app: matrix-authentication-service spec: enableServiceLinks: false + serviceAccountName: comms-vault nodeSelector: hardware: rpi5 affinity: @@ -36,6 +37,7 @@ spec: args: - | set -euo pipefail + . /vault/scripts/comms_vault_env.sh umask 077 DB_PASS_ESCAPED="$(printf '%s' "${MAS_DB_PASSWORD}" | sed 's/[\\/&]/\\&/g')" MATRIX_SECRET_ESCAPED="$(printf '%s' "${MATRIX_SHARED_SECRET}" | sed 's/[\\/&]/\\&/g')" @@ -47,22 +49,6 @@ spec: -e "s/@@KEYCLOAK_CLIENT_SECRET@@/${KC_SECRET_ESCAPED}/g" \ /etc/mas/config.yaml > /rendered/config.yaml chmod 0644 /rendered/config.yaml - env: - - name: MAS_DB_PASSWORD - valueFrom: - secretKeyRef: - name: mas-db - key: password - - name: MATRIX_SHARED_SECRET - valueFrom: - secretKeyRef: - name: mas-secrets-runtime - key: matrix_shared_secret - - name: KEYCLOAK_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: mas-secrets-runtime - key: keycloak_client_secret volumeMounts: - name: config mountPath: /etc/mas/config.yaml @@ -71,6 +57,12 @@ spec: - name: rendered mountPath: /rendered readOnly: false + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true containers: - name: mas image: ghcr.io/element-hq/matrix-authentication-service:1.8.0 @@ -86,14 +78,25 @@ spec: - name: rendered mountPath: /rendered readOnly: true - - name: secrets - mountPath: /etc/mas/secrets + - name: vault-secrets + mountPath: /etc/mas/secrets/encryption + subPath: mas-secrets-runtime__encryption readOnly: true - - name: admin-client - mountPath: /etc/mas/admin-client + - name: vault-secrets + mountPath: /etc/mas/secrets/matrix_shared_secret + subPath: mas-secrets-runtime__matrix_shared_secret readOnly: true - - name: keys - mountPath: /etc/mas/keys + - name: vault-secrets + mountPath: /etc/mas/secrets/keycloak_client_secret + subPath: mas-secrets-runtime__keycloak_client_secret + readOnly: true + - name: vault-secrets + mountPath: /etc/mas/keys/rsa_key + subPath: mas-secrets-runtime__rsa_key + readOnly: true + - name: vault-secrets + mountPath: /etc/mas/admin-client/client_secret + subPath: mas-admin-client-runtime__client_secret readOnly: true resources: requests: @@ -111,28 +114,16 @@ spec: path: config.yaml - name: rendered emptyDir: {} - - name: secrets - secret: - secretName: mas-secrets-runtime - items: - - key: encryption - path: encryption - - key: matrix_shared_secret - path: matrix_shared_secret - - key: keycloak_client_secret - path: keycloak_client_secret - - name: keys - secret: - secretName: mas-secrets-runtime - items: - - key: rsa_key - path: rsa_key - - name: admin-client - secret: - secretName: mas-admin-client-runtime - items: - - key: client_secret - path: client_secret + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault + - name: vault-scripts + configMap: + name: comms-vault-env + defaultMode: 0555 --- apiVersion: v1 kind: Service diff --git a/services/comms/mas-local-users-ensure-job.yaml b/services/comms/mas-local-users-ensure-job.yaml index 7853763..b81b94d 100644 --- a/services/comms/mas-local-users-ensure-job.yaml +++ b/services/comms/mas-local-users-ensure-job.yaml @@ -10,48 +10,47 @@ spec: template: spec: restartPolicy: Never + serviceAccountName: comms-vault volumes: - - name: mas-admin-client - secret: - secretName: mas-admin-client-runtime - items: - - key: client_secret - path: client_secret + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault + - name: vault-scripts + configMap: + name: comms-vault-env + defaultMode: 0555 containers: - name: ensure image: python:3.11-slim volumeMounts: - - name: mas-admin-client - mountPath: /etc/mas-admin-client + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts readOnly: true env: - name: MAS_ADMIN_CLIENT_ID value: 01KDXMVQBQ5JNY6SEJPZW6Z8BM - name: MAS_ADMIN_CLIENT_SECRET_FILE - value: /etc/mas-admin-client/client_secret + value: /vault/secrets/mas-admin-client-runtime__client_secret - name: MAS_TOKEN_URL value: http://matrix-authentication-service:8080/oauth2/token - name: MAS_ADMIN_API_BASE value: http://matrix-authentication-service:8081/api/admin/v1 - name: SEEDER_USER value: othrys-seeder - - name: SEEDER_PASS - valueFrom: - secretKeyRef: - name: atlasbot-credentials-runtime - key: seeder-password - name: BOT_USER value: atlasbot - - name: BOT_PASS - valueFrom: - secretKeyRef: - name: atlasbot-credentials-runtime - key: bot-password command: - /bin/sh - -c - | set -euo pipefail + . /vault/scripts/comms_vault_env.sh pip install --no-cache-dir requests >/dev/null python - <<'PY' import base64 diff --git a/services/comms/othrys-kick-numeric-job.yaml b/services/comms/othrys-kick-numeric-job.yaml index 8f02bbb..df96b9e 100644 --- a/services/comms/othrys-kick-numeric-job.yaml +++ b/services/comms/othrys-kick-numeric-job.yaml @@ -9,6 +9,7 @@ spec: template: spec: restartPolicy: Never + serviceAccountName: comms-vault containers: - name: kick image: python:3.11-slim @@ -23,16 +24,12 @@ spec: value: "#othrys:live.bstein.dev" - name: SEEDER_USER value: othrys-seeder - - name: SEEDER_PASS - valueFrom: - secretKeyRef: - name: atlasbot-credentials-runtime - key: seeder-password command: - /bin/sh - -c - | set -euo pipefail + . /vault/scripts/comms_vault_env.sh pip install --no-cache-dir requests >/dev/null python - <<'PY' import os @@ -113,3 +110,21 @@ spec: if is_numeric(user_id): kick(token, room_id, user_id) PY + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault + - name: vault-scripts + configMap: + name: comms-vault-env + defaultMode: 0555 diff --git a/services/comms/pin-othrys-job.yaml b/services/comms/pin-othrys-job.yaml index 3639194..babb6d1 100644 --- a/services/comms/pin-othrys-job.yaml +++ b/services/comms/pin-othrys-job.yaml @@ -16,6 +16,7 @@ spec: template: spec: restartPolicy: Never + serviceAccountName: comms-vault containers: - name: pin image: python:3.11-slim @@ -26,16 +27,12 @@ spec: value: http://matrix-authentication-service:8080 - name: SEEDER_USER value: othrys-seeder - - name: SEEDER_PASS - valueFrom: - secretKeyRef: - name: atlasbot-credentials-runtime - key: seeder-password command: - /bin/sh - -c - | set -euo pipefail + . /vault/scripts/comms_vault_env.sh pip install --no-cache-dir requests >/dev/null python - <<'PY' import os, requests, urllib.parse @@ -121,3 +118,21 @@ spec: eid = send(room_id, token, MESSAGE) pin(room_id, token, eid) PY + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault + - name: vault-scripts + configMap: + name: comms-vault-env + defaultMode: 0555 diff --git a/services/comms/reset-othrys-room-job.yaml b/services/comms/reset-othrys-room-job.yaml index dd056c3..6e20979 100644 --- a/services/comms/reset-othrys-room-job.yaml +++ b/services/comms/reset-othrys-room-job.yaml @@ -16,6 +16,7 @@ spec: template: spec: restartPolicy: Never + serviceAccountName: comms-vault containers: - name: reset image: python:3.11-slim @@ -34,11 +35,6 @@ spec: value: "Invite guests: share https://live.bstein.dev/#/room/#othrys:live.bstein.dev?action=join and choose 'Continue' -> 'Join as guest'." - name: SEEDER_USER value: othrys-seeder - - name: SEEDER_PASS - valueFrom: - secretKeyRef: - name: atlasbot-credentials-runtime - key: seeder-password - name: BOT_USER value: atlasbot command: @@ -46,6 +42,7 @@ spec: - -c - | set -euo pipefail + . /vault/scripts/comms_vault_env.sh pip install --no-cache-dir requests >/dev/null python - <<'PY' import os @@ -264,3 +261,21 @@ spec: print(f"old_room_id={old_room_id}") print(f"new_room_id={new_room_id}") PY + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault + - name: vault-scripts + configMap: + name: comms-vault-env + defaultMode: 0555 diff --git a/services/comms/scripts/comms_vault_env.sh b/services/comms/scripts/comms_vault_env.sh new file mode 100644 index 0000000..98b3fc4 --- /dev/null +++ b/services/comms/scripts/comms_vault_env.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env sh +set -eu + +vault_dir="/vault/secrets" + +read_secret() { + cat "${vault_dir}/$1" +} + +export TURN_STATIC_AUTH_SECRET="$(read_secret turn-shared-secret__TURN_STATIC_AUTH_SECRET)" +export TURN_PASSWORD="${TURN_STATIC_AUTH_SECRET}" + +export LIVEKIT_API_SECRET="$(read_secret livekit-api__primary)" +export LIVEKIT_SECRET="${LIVEKIT_API_SECRET}" + +export BOT_PASS="$(read_secret atlasbot-credentials-runtime__bot-password)" +export SEEDER_PASS="$(read_secret atlasbot-credentials-runtime__seeder-password)" + +export CHAT_API_KEY="$(read_secret chat-ai-keys-runtime__matrix)" +export CHAT_API_HOMEPAGE="$(read_secret chat-ai-keys-runtime__homepage)" + +export MAS_ADMIN_CLIENT_SECRET_FILE="${vault_dir}/mas-admin-client-runtime__client_secret" +export PGPASSWORD="$(read_secret synapse-db__POSTGRES_PASSWORD)" + +export MAS_DB_PASSWORD="$(read_secret mas-db__password)" +export MATRIX_SHARED_SECRET="$(read_secret mas-secrets-runtime__matrix_shared_secret)" +export KEYCLOAK_CLIENT_SECRET="$(read_secret mas-secrets-runtime__keycloak_client_secret)" diff --git a/services/comms/secretproviderclass.yaml b/services/comms/secretproviderclass.yaml new file mode 100644 index 0000000..971d408 --- /dev/null +++ b/services/comms/secretproviderclass.yaml @@ -0,0 +1,134 @@ +# services/comms/secretproviderclass.yaml +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: comms-vault + namespace: comms +spec: + provider: vault + parameters: + vaultAddress: "http://vault.vault.svc.cluster.local:8200" + roleName: "comms" + objects: | + - objectName: "turn-shared-secret__TURN_STATIC_AUTH_SECRET" + secretPath: "kv/data/atlas/comms/turn-shared-secret" + secretKey: "TURN_STATIC_AUTH_SECRET" + - objectName: "livekit-api__primary" + secretPath: "kv/data/atlas/comms/livekit-api" + secretKey: "primary" + - objectName: "synapse-db__POSTGRES_PASSWORD" + secretPath: "kv/data/atlas/comms/synapse-db" + secretKey: "POSTGRES_PASSWORD" + - objectName: "synapse-redis__redis-password" + secretPath: "kv/data/atlas/comms/synapse-redis" + secretKey: "redis-password" + - objectName: "synapse-macaroon__macaroon_secret_key" + secretPath: "kv/data/atlas/comms/synapse-macaroon" + secretKey: "macaroon_secret_key" + - objectName: "atlasbot-credentials-runtime__bot-password" + secretPath: "kv/data/atlas/comms/atlasbot-credentials-runtime" + secretKey: "bot-password" + - objectName: "atlasbot-credentials-runtime__seeder-password" + secretPath: "kv/data/atlas/comms/atlasbot-credentials-runtime" + secretKey: "seeder-password" + - objectName: "chat-ai-keys-runtime__matrix" + secretPath: "kv/data/atlas/shared/chat-ai-keys-runtime" + secretKey: "matrix" + - objectName: "chat-ai-keys-runtime__homepage" + secretPath: "kv/data/atlas/shared/chat-ai-keys-runtime" + secretKey: "homepage" + - objectName: "mas-admin-client-runtime__client_secret" + secretPath: "kv/data/atlas/comms/mas-admin-client-runtime" + secretKey: "client_secret" + - objectName: "mas-db__password" + secretPath: "kv/data/atlas/comms/mas-db" + secretKey: "password" + - objectName: "mas-secrets-runtime__encryption" + secretPath: "kv/data/atlas/comms/mas-secrets-runtime" + secretKey: "encryption" + - objectName: "mas-secrets-runtime__matrix_shared_secret" + secretPath: "kv/data/atlas/comms/mas-secrets-runtime" + secretKey: "matrix_shared_secret" + - objectName: "mas-secrets-runtime__keycloak_client_secret" + secretPath: "kv/data/atlas/comms/mas-secrets-runtime" + secretKey: "keycloak_client_secret" + - objectName: "mas-secrets-runtime__rsa_key" + secretPath: "kv/data/atlas/comms/mas-secrets-runtime" + secretKey: "rsa_key" + - objectName: "othrys-synapse-signingkey__signing.key" + secretPath: "kv/data/atlas/comms/othrys-synapse-signingkey" + secretKey: "signing.key" + - objectName: "synapse-oidc__client-secret" + secretPath: "kv/data/atlas/comms/synapse-oidc" + secretKey: "client-secret" + secretObjects: + - secretName: turn-shared-secret + type: Opaque + data: + - objectName: turn-shared-secret__TURN_STATIC_AUTH_SECRET + key: TURN_STATIC_AUTH_SECRET + - secretName: livekit-api + type: Opaque + data: + - objectName: livekit-api__primary + key: primary + - secretName: synapse-db + type: Opaque + data: + - objectName: synapse-db__POSTGRES_PASSWORD + key: POSTGRES_PASSWORD + - secretName: synapse-redis + type: Opaque + data: + - objectName: synapse-redis__redis-password + key: redis-password + - secretName: synapse-macaroon + type: Opaque + data: + - objectName: synapse-macaroon__macaroon_secret_key + key: macaroon_secret_key + - secretName: atlasbot-credentials-runtime + type: Opaque + data: + - objectName: atlasbot-credentials-runtime__bot-password + key: bot-password + - objectName: atlasbot-credentials-runtime__seeder-password + key: seeder-password + - secretName: chat-ai-keys-runtime + type: Opaque + data: + - objectName: chat-ai-keys-runtime__matrix + key: matrix + - objectName: chat-ai-keys-runtime__homepage + key: homepage + - secretName: mas-admin-client-runtime + type: Opaque + data: + - objectName: mas-admin-client-runtime__client_secret + key: client_secret + - secretName: mas-db + type: Opaque + data: + - objectName: mas-db__password + key: password + - secretName: mas-secrets-runtime + type: Opaque + data: + - objectName: mas-secrets-runtime__encryption + key: encryption + - objectName: mas-secrets-runtime__matrix_shared_secret + key: matrix_shared_secret + - objectName: mas-secrets-runtime__keycloak_client_secret + key: keycloak_client_secret + - objectName: mas-secrets-runtime__rsa_key + key: rsa_key + - secretName: othrys-synapse-signingkey + type: Opaque + data: + - objectName: othrys-synapse-signingkey__signing.key + key: signing.key + - secretName: synapse-oidc + type: Opaque + data: + - objectName: synapse-oidc__client-secret + key: client-secret diff --git a/services/comms/seed-othrys-room.yaml b/services/comms/seed-othrys-room.yaml index 901f14d..0508e0e 100644 --- a/services/comms/seed-othrys-room.yaml +++ b/services/comms/seed-othrys-room.yaml @@ -14,6 +14,7 @@ spec: template: spec: restartPolicy: Never + serviceAccountName: comms-vault containers: - name: seed image: python:3.11-slim @@ -24,23 +25,14 @@ spec: value: http://matrix-authentication-service:8080 - name: SEEDER_USER value: othrys-seeder - - name: SEEDER_PASS - valueFrom: - secretKeyRef: - name: atlasbot-credentials-runtime - key: seeder-password - name: BOT_USER value: atlasbot - - name: BOT_PASS - valueFrom: - secretKeyRef: - name: atlasbot-credentials-runtime - key: bot-password command: - /bin/sh - -c - | set -euo pipefail + . /vault/scripts/comms_vault_env.sh pip install --no-cache-dir requests pyyaml >/dev/null python - <<'PY' import os, requests, urllib.parse @@ -140,7 +132,23 @@ spec: - name: synapse-config mountPath: /config readOnly: true + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true volumes: - name: synapse-config secret: secretName: othrys-synapse-matrix-synapse + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault + - name: vault-scripts + configMap: + name: comms-vault-env + defaultMode: 0555 diff --git a/services/comms/serviceaccount.yaml b/services/comms/serviceaccount.yaml new file mode 100644 index 0000000..1b975b8 --- /dev/null +++ b/services/comms/serviceaccount.yaml @@ -0,0 +1,6 @@ +# services/comms/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: comms-vault + namespace: comms diff --git a/services/comms/synapse-seeder-admin-ensure-job.yaml b/services/comms/synapse-seeder-admin-ensure-job.yaml index 0885722..dbe5609 100644 --- a/services/comms/synapse-seeder-admin-ensure-job.yaml +++ b/services/comms/synapse-seeder-admin-ensure-job.yaml @@ -9,6 +9,7 @@ spec: template: spec: restartPolicy: OnFailure + serviceAccountName: comms-vault containers: - name: psql image: postgres:16-alpine @@ -21,16 +22,30 @@ spec: value: synapse - name: PGUSER value: synapse - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: synapse-db - key: POSTGRES_PASSWORD command: - /bin/sh - -c - | set -euo pipefail + . /vault/scripts/comms_vault_env.sh psql -v ON_ERROR_STOP=1 <<'SQL' UPDATE users SET admin = 1 WHERE name = '@othrys-seeder:live.bstein.dev'; SQL + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault + - name: vault-scripts + configMap: + name: comms-vault-env + defaultMode: 0555 diff --git a/services/comms/synapse-signingkey-ensure-job.yaml b/services/comms/synapse-signingkey-ensure-job.yaml index 81d95a7..ca83f52 100644 --- a/services/comms/synapse-signingkey-ensure-job.yaml +++ b/services/comms/synapse-signingkey-ensure-job.yaml @@ -37,15 +37,29 @@ spec: args: - | set -euo pipefail - set -x - if kubectl -n comms get secret othrys-synapse-signingkey \ - -o jsonpath='{.data.signing\.key}' 2>/tmp/get_err | grep -q .; then + apk add --no-cache curl jq >/dev/null + + vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" + vault_role="${VAULT_ROLE:-comms-secrets}" + jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" + login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')" + vault_token="$(curl -sS --request POST --data "${login_payload}" \ + "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" + if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then + echo "vault login failed" >&2 + exit 1 + fi + + existing="$(curl -sS -H "X-Vault-Token: ${vault_token}" \ + "${vault_addr}/v1/kv/data/atlas/comms/othrys-synapse-signingkey" | jq -r '.data.data["signing.key"] // empty')" + if [ -n "${existing}" ]; then exit 0 fi - cat /tmp/get_err >&2 || true - kubectl -n comms create secret generic othrys-synapse-signingkey \ - --from-file=signing.key=/work/signing.key \ - --dry-run=client -o yaml | kubectl -n comms apply -f - >/dev/null + + value="$(cat /work/signing.key)" + payload="$(jq -nc --arg value "${value}" '{data:{"signing.key":$value}}')" + curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \ + -d "${payload}" "${vault_addr}/v1/kv/data/atlas/comms/othrys-synapse-signingkey" >/dev/null volumeMounts: - name: work mountPath: /work diff --git a/services/comms/synapse-user-seed-job.yaml b/services/comms/synapse-user-seed-job.yaml index 083f72e..2285dad 100644 --- a/services/comms/synapse-user-seed-job.yaml +++ b/services/comms/synapse-user-seed-job.yaml @@ -10,6 +10,7 @@ spec: template: spec: restartPolicy: Never + serviceAccountName: comms-vault containers: - name: seed image: python:3.11-slim @@ -22,30 +23,16 @@ spec: value: synapse - name: PGUSER value: synapse - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: synapse-db - key: POSTGRES_PASSWORD - name: SEEDER_USER value: othrys-seeder - - name: SEEDER_PASS - valueFrom: - secretKeyRef: - name: atlasbot-credentials-runtime - key: seeder-password - name: BOT_USER value: atlasbot - - name: BOT_PASS - valueFrom: - secretKeyRef: - name: atlasbot-credentials-runtime - key: bot-password command: - /bin/sh - -c - | set -euo pipefail + . /vault/scripts/comms_vault_env.sh pip install --no-cache-dir psycopg2-binary bcrypt >/dev/null python - <<'PY' import os @@ -118,3 +105,21 @@ spec: finally: conn.close() PY + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault + - name: vault-scripts + configMap: + name: comms-vault-env + defaultMode: 0555 diff --git a/services/comms/vault-sync-deployment.yaml b/services/comms/vault-sync-deployment.yaml new file mode 100644 index 0000000..f5b5849 --- /dev/null +++ b/services/comms/vault-sync-deployment.yaml @@ -0,0 +1,34 @@ +# services/comms/vault-sync-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: comms-vault-sync + namespace: comms +spec: + replicas: 1 + selector: + matchLabels: + app: comms-vault-sync + template: + metadata: + labels: + app: comms-vault-sync + spec: + serviceAccountName: comms-vault + containers: + - name: sync + image: alpine:3.20 + command: ["/bin/sh", "-c"] + args: + - "sleep infinity" + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: comms-vault diff --git a/services/harbor/kustomization.yaml b/services/harbor/kustomization.yaml index 7da3d50..2a9cb9e 100644 --- a/services/harbor/kustomization.yaml +++ b/services/harbor/kustomization.yaml @@ -4,7 +4,10 @@ kind: Kustomization namespace: harbor resources: - namespace.yaml + - serviceaccount.yaml + - secretproviderclass.yaml - pvc.yaml - certificate.yaml - helmrelease.yaml + - vault-sync-deployment.yaml - image.yaml diff --git a/services/harbor/secretproviderclass.yaml b/services/harbor/secretproviderclass.yaml new file mode 100644 index 0000000..1e1a7f1 --- /dev/null +++ b/services/harbor/secretproviderclass.yaml @@ -0,0 +1,87 @@ +# services/harbor/secretproviderclass.yaml +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: harbor-vault + namespace: harbor +spec: + provider: vault + parameters: + vaultAddress: "http://vault.vault.svc.cluster.local:8200" + roleName: "harbor" + objects: | + - objectName: "harbor-core__CSRF_KEY" + secretPath: "kv/data/atlas/harbor/harbor-core" + secretKey: "CSRF_KEY" + - objectName: "harbor-core__REGISTRY_CREDENTIAL_PASSWORD" + secretPath: "kv/data/atlas/harbor/harbor-core" + secretKey: "REGISTRY_CREDENTIAL_PASSWORD" + - objectName: "harbor-core__harbor_admin_password" + secretPath: "kv/data/atlas/harbor/harbor-core" + secretKey: "harbor_admin_password" + - objectName: "harbor-core__secret" + secretPath: "kv/data/atlas/harbor/harbor-core" + secretKey: "secret" + - objectName: "harbor-core__secretKey" + secretPath: "kv/data/atlas/harbor/harbor-core" + secretKey: "secretKey" + - objectName: "harbor-core__tls.crt" + secretPath: "kv/data/atlas/harbor/harbor-core" + secretKey: "tls.crt" + - objectName: "harbor-core__tls.key" + secretPath: "kv/data/atlas/harbor/harbor-core" + secretKey: "tls.key" + - objectName: "harbor-db__database" + secretPath: "kv/data/atlas/harbor/harbor-db" + secretKey: "database" + - objectName: "harbor-db__host" + secretPath: "kv/data/atlas/harbor/harbor-db" + secretKey: "host" + - objectName: "harbor-db__password" + secretPath: "kv/data/atlas/harbor/harbor-db" + secretKey: "password" + - objectName: "harbor-db__port" + secretPath: "kv/data/atlas/harbor/harbor-db" + secretKey: "port" + - objectName: "harbor-db__username" + secretPath: "kv/data/atlas/harbor/harbor-db" + secretKey: "username" + - objectName: "harbor-oidc__CONFIG_OVERWRITE_JSON" + secretPath: "kv/data/atlas/harbor/harbor-oidc" + secretKey: "CONFIG_OVERWRITE_JSON" + secretObjects: + - secretName: harbor-core + type: Opaque + data: + - objectName: harbor-core__CSRF_KEY + key: CSRF_KEY + - objectName: harbor-core__REGISTRY_CREDENTIAL_PASSWORD + key: REGISTRY_CREDENTIAL_PASSWORD + - objectName: harbor-core__harbor_admin_password + key: harbor_admin_password + - objectName: harbor-core__secret + key: secret + - objectName: harbor-core__secretKey + key: secretKey + - objectName: harbor-core__tls.crt + key: tls.crt + - objectName: harbor-core__tls.key + key: tls.key + - secretName: harbor-db + type: Opaque + data: + - objectName: harbor-db__database + key: database + - objectName: harbor-db__host + key: host + - objectName: harbor-db__password + key: password + - objectName: harbor-db__port + key: port + - objectName: harbor-db__username + key: username + - secretName: harbor-oidc + type: Opaque + data: + - objectName: harbor-oidc__CONFIG_OVERWRITE_JSON + key: CONFIG_OVERWRITE_JSON diff --git a/services/harbor/serviceaccount.yaml b/services/harbor/serviceaccount.yaml new file mode 100644 index 0000000..46bb816 --- /dev/null +++ b/services/harbor/serviceaccount.yaml @@ -0,0 +1,6 @@ +# services/harbor/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: harbor-vault-sync + namespace: harbor diff --git a/services/harbor/vault-sync-deployment.yaml b/services/harbor/vault-sync-deployment.yaml new file mode 100644 index 0000000..11aae09 --- /dev/null +++ b/services/harbor/vault-sync-deployment.yaml @@ -0,0 +1,34 @@ +# services/harbor/vault-sync-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: harbor-vault-sync + namespace: harbor +spec: + replicas: 1 + selector: + matchLabels: + app: harbor-vault-sync + template: + metadata: + labels: + app: harbor-vault-sync + spec: + serviceAccountName: harbor-vault-sync + containers: + - name: sync + image: alpine:3.20 + command: ["/bin/sh", "-c"] + args: + - "sleep infinity" + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: harbor-vault diff --git a/services/keycloak/harbor-oidc-secret-ensure-job.yaml b/services/keycloak/harbor-oidc-secret-ensure-job.yaml index 21a7ff0..e4fbcee 100644 --- a/services/keycloak/harbor-oidc-secret-ensure-job.yaml +++ b/services/keycloak/harbor-oidc-secret-ensure-job.yaml @@ -16,6 +16,16 @@ spec: configMap: name: harbor-oidc-secret-ensure-script defaultMode: 0555 + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: sso-vault + - name: vault-scripts + configMap: + name: sso-vault-env + defaultMode: 0555 affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -30,18 +40,13 @@ spec: - name: apply image: alpine:3.20 command: ["/scripts/harbor_oidc_secret_ensure.sh"] - env: - - name: KEYCLOAK_ADMIN - valueFrom: - secretKeyRef: - name: keycloak-admin - key: username - - name: KEYCLOAK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-admin - key: password volumeMounts: - name: harbor-oidc-secret-ensure-script mountPath: /scripts readOnly: true + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true diff --git a/services/keycloak/ldap-federation-job.yaml b/services/keycloak/ldap-federation-job.yaml index 9650468..d9d650b 100644 --- a/services/keycloak/ldap-federation-job.yaml +++ b/services/keycloak/ldap-federation-job.yaml @@ -19,6 +19,7 @@ spec: - key: node-role.kubernetes.io/worker operator: Exists restartPolicy: OnFailure + serviceAccountName: sso-vault containers: - name: configure image: python:3.11-alpine @@ -28,25 +29,10 @@ spec: value: http://keycloak.sso.svc.cluster.local - name: KEYCLOAK_REALM value: atlas - - name: KEYCLOAK_ADMIN_USER - valueFrom: - secretKeyRef: - name: keycloak-admin - key: username - - name: KEYCLOAK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-admin - key: password - name: LDAP_URL value: ldap://openldap.sso.svc.cluster.local:389 - name: LDAP_BIND_DN value: cn=admin,dc=bstein,dc=dev - - name: LDAP_BIND_PASSWORD - valueFrom: - secretKeyRef: - name: openldap-admin - key: LDAP_ADMIN_PASSWORD - name: LDAP_USERS_DN value: ou=users,dc=bstein,dc=dev - name: LDAP_GROUPS_DN @@ -55,6 +41,7 @@ spec: args: - | set -euo pipefail + . /vault/scripts/keycloak_vault_env.sh python - <<'PY' import json import os @@ -360,3 +347,21 @@ spec: except Exception as e: print(f"WARNING: LDAP cleanup failed (continuing): {e}") PY + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: sso-vault + - name: vault-scripts + configMap: + name: sso-vault-env + defaultMode: 0555 diff --git a/services/keycloak/logs-oidc-secret-ensure-job.yaml b/services/keycloak/logs-oidc-secret-ensure-job.yaml index 11d48f9..df3d569 100644 --- a/services/keycloak/logs-oidc-secret-ensure-job.yaml +++ b/services/keycloak/logs-oidc-secret-ensure-job.yaml @@ -18,6 +18,7 @@ spec: args: - | set -euo pipefail + . /vault/scripts/keycloak_vault_env.sh apk add --no-cache curl jq kubectl openssl >/dev/null KC_URL="http://keycloak.sso.svc.cluster.local" @@ -73,31 +74,56 @@ spec: exit 1 fi - if kubectl -n logging get secret oauth2-proxy-logs-oidc >/dev/null 2>&1; then - current_cookie="$(kubectl -n logging get secret oauth2-proxy-logs-oidc -o jsonpath='{.data.cookie_secret}' 2>/dev/null || true)" - if [ -n "${current_cookie}" ]; then - decoded="$(printf '%s' "${current_cookie}" | base64 -d 2>/dev/null || true)" - length="$(printf '%s' "${decoded}" | wc -c | tr -d ' ')" - if [ "${length}" = "16" ] || [ "${length}" = "24" ] || [ "${length}" = "32" ]; then - exit 0 - fi - fi + vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" + vault_role="${VAULT_ROLE:-sso-secrets}" + jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" + login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')" + vault_token="$(curl -sS --request POST --data "${login_payload}" \ + "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" + if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then + echo "vault login failed" >&2 + exit 1 fi - COOKIE_SECRET="$(openssl rand -hex 16 | tr -d '\n')" + COOKIE_SECRET="$(curl -sS -H "X-Vault-Token: ${vault_token}" \ + "${vault_addr}/v1/kv/data/atlas/logging/oauth2-proxy-logs-oidc" | jq -r '.data.data.cookie_secret // empty')" + if [ -n "${COOKIE_SECRET}" ]; then + length="$(printf '%s' "${COOKIE_SECRET}" | wc -c | tr -d ' ')" + if [ "${length}" != "16" ] && [ "${length}" != "24" ] && [ "${length}" != "32" ]; then + COOKIE_SECRET="" + fi + fi + if [ -z "${COOKIE_SECRET}" ]; then + COOKIE_SECRET="$(openssl rand -hex 16 | tr -d '\n')" + fi + + payload="$(jq -nc \ + --arg client_id "logs" \ + --arg client_secret "${CLIENT_SECRET}" \ + --arg cookie_secret "${COOKIE_SECRET}" \ + '{data:{client_id:$client_id,client_secret:$client_secret,cookie_secret:$cookie_secret}}')" + curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \ + -d "${payload}" "${vault_addr}/v1/kv/data/atlas/logging/oauth2-proxy-logs-oidc" >/dev/null kubectl -n logging create secret generic oauth2-proxy-logs-oidc \ --from-literal=client_id="logs" \ --from-literal=client_secret="${CLIENT_SECRET}" \ --from-literal=cookie_secret="${COOKIE_SECRET}" \ --dry-run=client -o yaml | kubectl -n logging apply -f - >/dev/null - env: - - name: KEYCLOAK_ADMIN - valueFrom: - secretKeyRef: - name: keycloak-admin - key: username - - name: KEYCLOAK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-admin - key: password + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: sso-vault + - name: vault-scripts + configMap: + name: sso-vault-env + defaultMode: 0555 diff --git a/services/keycloak/mas-secrets-ensure-job.yaml b/services/keycloak/mas-secrets-ensure-job.yaml index 4d10aae..ec2d7a0 100644 --- a/services/keycloak/mas-secrets-ensure-job.yaml +++ b/services/keycloak/mas-secrets-ensure-job.yaml @@ -20,6 +20,16 @@ spec: volumes: - name: work emptyDir: {} + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: sso-vault + - name: vault-scripts + configMap: + name: sso-vault-env + defaultMode: 0555 initContainers: - name: generate image: alpine:3.20 @@ -27,6 +37,7 @@ spec: args: - | set -euo pipefail + . /vault/scripts/keycloak_vault_env.sh umask 077 apk add --no-cache curl openssl jq >/dev/null @@ -68,20 +79,15 @@ spec: openssl rand -hex 32 | tr -d '\n' > /work/matrix_shared_secret openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:4096 -out /work/rsa_key >/dev/null 2>&1 chmod 0644 /work/* - env: - - name: KEYCLOAK_ADMIN - valueFrom: - secretKeyRef: - name: keycloak-admin - key: username - - name: KEYCLOAK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-admin - key: password volumeMounts: - name: work mountPath: /work + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true containers: - name: apply image: registry.bstein.dev/bstein/kubectl:1.35.0 @@ -89,19 +95,36 @@ spec: args: - | set -euo pipefail - if kubectl -n comms get secret mas-secrets-runtime >/dev/null 2>&1; then - kubectl -n comms get secret mas-secrets-runtime -o jsonpath='{.data.encryption}' | base64 -d 2>/dev/null > /tmp/encryption.current || true - current_len="$(wc -c < /tmp/encryption.current | tr -d ' ')" - if [ "${current_len}" = "64" ] && grep -Eq '^[0-9a-fA-F]{64}$' /tmp/encryption.current; then + apk add --no-cache curl jq >/dev/null + + vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" + vault_role="${VAULT_ROLE:-sso-secrets}" + jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" + login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')" + vault_token="$(curl -sS --request POST --data "${login_payload}" \ + "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" + if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then + echo "vault login failed" >&2 + exit 1 + fi + + existing="$(curl -sS -H "X-Vault-Token: ${vault_token}" \ + "${vault_addr}/v1/kv/data/atlas/comms/mas-secrets-runtime" | jq -r '.data.data.encryption // empty')" + if [ -n "${existing}" ]; then + current_len="$(printf '%s' "${existing}" | wc -c | tr -d ' ')" + if [ "${current_len}" = "64" ] && printf '%s' "${existing}" | grep -Eq '^[0-9a-fA-F]{64}$'; then exit 0 fi fi - kubectl -n comms create secret generic mas-secrets-runtime \ - --from-file=encryption=/work/encryption \ - --from-file=matrix_shared_secret=/work/matrix_shared_secret \ - --from-file=keycloak_client_secret=/work/keycloak_client_secret \ - --from-file=rsa_key=/work/rsa_key \ - --dry-run=client -o yaml | kubectl -n comms apply -f - >/dev/null + + payload="$(jq -nc \ + --arg encryption "$(cat /work/encryption)" \ + --arg matrix_shared_secret "$(cat /work/matrix_shared_secret)" \ + --arg keycloak_client_secret "$(cat /work/keycloak_client_secret)" \ + --arg rsa_key "$(cat /work/rsa_key)" \ + '{data:{encryption:$encryption, matrix_shared_secret:$matrix_shared_secret, keycloak_client_secret:$keycloak_client_secret, rsa_key:$rsa_key}}')" + curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \ + -d "${payload}" "${vault_addr}/v1/kv/data/atlas/comms/mas-secrets-runtime" >/dev/null volumeMounts: - name: work mountPath: /work diff --git a/services/keycloak/portal-e2e-client-job.yaml b/services/keycloak/portal-e2e-client-job.yaml index 7f6c5dd..ea15178 100644 --- a/services/keycloak/portal-e2e-client-job.yaml +++ b/services/keycloak/portal-e2e-client-job.yaml @@ -9,6 +9,7 @@ spec: template: spec: restartPolicy: Never + serviceAccountName: sso-vault containers: - name: configure image: python:3.11-alpine @@ -17,30 +18,11 @@ spec: value: http://keycloak.sso.svc.cluster.local - name: KEYCLOAK_REALM value: atlas - - name: KEYCLOAK_ADMIN_USER - valueFrom: - secretKeyRef: - name: keycloak-admin - key: username - - name: KEYCLOAK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-admin - key: password - - name: PORTAL_E2E_CLIENT_ID - valueFrom: - secretKeyRef: - name: portal-e2e-client - key: client_id - - name: PORTAL_E2E_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: portal-e2e-client - key: client_secret command: ["/bin/sh", "-c"] args: - | set -euo pipefail + . /vault/scripts/keycloak_vault_env.sh python - <<'PY' import json import os @@ -245,3 +227,21 @@ spec: if status not in (200, 204): raise SystemExit(f"Role mapping update failed (status={status}) resp={resp}") PY + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: sso-vault + - name: vault-scripts + configMap: + name: sso-vault-env + defaultMode: 0555 diff --git a/services/keycloak/portal-e2e-execute-actions-email-test-job.yaml b/services/keycloak/portal-e2e-execute-actions-email-test-job.yaml index 877dd55..817c526 100644 --- a/services/keycloak/portal-e2e-execute-actions-email-test-job.yaml +++ b/services/keycloak/portal-e2e-execute-actions-email-test-job.yaml @@ -9,6 +9,7 @@ spec: template: spec: restartPolicy: Never + serviceAccountName: sso-vault containers: - name: test image: python:3.11-alpine @@ -17,16 +18,6 @@ spec: value: http://keycloak.sso.svc.cluster.local - name: KEYCLOAK_REALM value: atlas - - name: PORTAL_E2E_CLIENT_ID - valueFrom: - secretKeyRef: - name: portal-e2e-client - key: client_id - - name: PORTAL_E2E_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: portal-e2e-client - key: client_secret - name: E2E_PROBE_USERNAME value: e2e-smtp-probe - name: E2E_PROBE_EMAIL @@ -39,13 +30,30 @@ spec: args: - | set -euo pipefail + . /vault/scripts/keycloak_vault_env.sh python /scripts/test_keycloak_execute_actions_email.py volumeMounts: - name: tests mountPath: /scripts readOnly: true + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true volumes: - name: tests configMap: name: portal-e2e-tests defaultMode: 0555 + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: sso-vault + - name: vault-scripts + configMap: + name: sso-vault-env + defaultMode: 0555 diff --git a/services/keycloak/portal-e2e-target-client-job.yaml b/services/keycloak/portal-e2e-target-client-job.yaml index 45b3980..63a3ea9 100644 --- a/services/keycloak/portal-e2e-target-client-job.yaml +++ b/services/keycloak/portal-e2e-target-client-job.yaml @@ -9,6 +9,7 @@ spec: template: spec: restartPolicy: Never + serviceAccountName: sso-vault containers: - name: configure image: python:3.11-alpine @@ -17,22 +18,13 @@ spec: value: http://keycloak.sso.svc.cluster.local - name: KEYCLOAK_REALM value: atlas - - name: KEYCLOAK_ADMIN_USER - valueFrom: - secretKeyRef: - name: keycloak-admin - key: username - - name: KEYCLOAK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-admin - key: password - name: TARGET_CLIENT_ID value: bstein-dev-home command: ["/bin/sh", "-c"] args: - | set -euo pipefail + . /vault/scripts/keycloak_vault_env.sh python - <<'PY' import json import os @@ -136,3 +128,21 @@ spec: print(f"OK: ensured token exchange enabled on client {target_client_id}") PY + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: sso-vault + - name: vault-scripts + configMap: + name: sso-vault-env + defaultMode: 0555 diff --git a/services/keycloak/portal-e2e-token-exchange-permissions-job.yaml b/services/keycloak/portal-e2e-token-exchange-permissions-job.yaml index 104d6f0..c0ec397 100644 --- a/services/keycloak/portal-e2e-token-exchange-permissions-job.yaml +++ b/services/keycloak/portal-e2e-token-exchange-permissions-job.yaml @@ -9,6 +9,7 @@ spec: template: spec: restartPolicy: Never + serviceAccountName: sso-vault containers: - name: configure image: python:3.11-alpine @@ -17,16 +18,6 @@ spec: value: http://keycloak.sso.svc.cluster.local - name: KEYCLOAK_REALM value: atlas - - name: KEYCLOAK_ADMIN_USER - valueFrom: - secretKeyRef: - name: keycloak-admin - key: username - - name: KEYCLOAK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-admin - key: password - name: PORTAL_E2E_CLIENT_ID value: test-portal-e2e - name: TARGET_CLIENT_ID @@ -35,6 +26,7 @@ spec: args: - | set -euo pipefail + . /vault/scripts/keycloak_vault_env.sh python - <<'PY' import json import os @@ -269,3 +261,21 @@ spec: print("OK: configured token exchange permissions for portal E2E client") PY + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: sso-vault + - name: vault-scripts + configMap: + name: sso-vault-env + defaultMode: 0555 diff --git a/services/keycloak/portal-e2e-token-exchange-test-job.yaml b/services/keycloak/portal-e2e-token-exchange-test-job.yaml index ab43303..694a8ca 100644 --- a/services/keycloak/portal-e2e-token-exchange-test-job.yaml +++ b/services/keycloak/portal-e2e-token-exchange-test-job.yaml @@ -10,6 +10,7 @@ spec: template: spec: restartPolicy: Never + serviceAccountName: sso-vault containers: - name: test image: python:3.11-alpine @@ -26,27 +27,34 @@ spec: value: "300" - name: RETRY_INTERVAL_SECONDS value: "5" - - name: PORTAL_E2E_CLIENT_ID - valueFrom: - secretKeyRef: - name: portal-e2e-client - key: client_id - - name: PORTAL_E2E_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: portal-e2e-client - key: client_secret command: ["/bin/sh", "-c"] args: - | set -euo pipefail + . /vault/scripts/keycloak_vault_env.sh python /scripts/test_portal_token_exchange.py volumeMounts: - name: tests mountPath: /scripts readOnly: true + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true volumes: - name: tests configMap: name: portal-e2e-tests defaultMode: 0555 + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: sso-vault + - name: vault-scripts + configMap: + name: sso-vault-env + defaultMode: 0555 diff --git a/services/keycloak/realm-settings-job.yaml b/services/keycloak/realm-settings-job.yaml index bdc816d..0c5752f 100644 --- a/services/keycloak/realm-settings-job.yaml +++ b/services/keycloak/realm-settings-job.yaml @@ -19,6 +19,7 @@ spec: - key: node-role.kubernetes.io/worker operator: Exists restartPolicy: Never + serviceAccountName: sso-vault containers: - name: configure image: python:3.11-alpine @@ -27,16 +28,6 @@ spec: value: http://keycloak.sso.svc.cluster.local - name: KEYCLOAK_REALM value: atlas - - name: KEYCLOAK_ADMIN_USER - valueFrom: - secretKeyRef: - name: keycloak-admin - key: username - - name: KEYCLOAK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-admin - key: password - name: KEYCLOAK_SMTP_HOST value: mailu-front.mailu-mailserver.svc.cluster.local - name: KEYCLOAK_SMTP_PORT @@ -53,6 +44,7 @@ spec: args: - | set -euo pipefail + . /vault/scripts/keycloak_vault_env.sh python - <<'PY' import json import os @@ -444,3 +436,21 @@ spec: f"Unexpected execution update response for identity-provider-redirector: {status}" ) PY + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: sso-vault + - name: vault-scripts + configMap: + name: sso-vault-env + defaultMode: 0555 diff --git a/services/keycloak/scripts/harbor_oidc_secret_ensure.sh b/services/keycloak/scripts/harbor_oidc_secret_ensure.sh index 4767ef0..f2dafc6 100755 --- a/services/keycloak/scripts/harbor_oidc_secret_ensure.sh +++ b/services/keycloak/scripts/harbor_oidc_secret_ensure.sh @@ -3,6 +3,8 @@ set -euo pipefail apk add --no-cache curl jq kubectl >/dev/null +. /vault/scripts/keycloak_vault_env.sh + KC_URL="http://keycloak.sso.svc.cluster.local" ACCESS_TOKEN="" for attempt in 1 2 3 4 5; do @@ -99,6 +101,17 @@ CONFIG_OVERWRITE_JSON="$(jq -nc \ --argjson oidc_logout true \ '{auth_mode:$auth_mode,oidc_name:$oidc_name,oidc_client_id:$oidc_client_id,oidc_client_secret:$oidc_client_secret,oidc_endpoint:$oidc_endpoint,oidc_scope:$oidc_scope,oidc_user_claim:$oidc_user_claim,oidc_groups_claim:$oidc_groups_claim,oidc_admin_group:$oidc_admin_group,oidc_auto_onboard:$oidc_auto_onboard,oidc_verify_cert:$oidc_verify_cert,oidc_logout:$oidc_logout}')" -kubectl -n harbor create secret generic harbor-oidc \ - --from-literal=CONFIG_OVERWRITE_JSON="${CONFIG_OVERWRITE_JSON}" \ - --dry-run=client -o yaml | kubectl -n harbor apply -f - >/dev/null +vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" +vault_role="${VAULT_ROLE:-sso-secrets}" +jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" +login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')" +vault_token="$(curl -sS --request POST --data "${login_payload}" \ + "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" +if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then + echo "vault login failed" >&2 + exit 1 +fi + +payload="$(jq -nc --arg value "${CONFIG_OVERWRITE_JSON}" '{data:{CONFIG_OVERWRITE_JSON:$value}}')" +curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \ + -d "${payload}" "${vault_addr}/v1/kv/data/atlas/harbor/harbor-oidc" >/dev/null diff --git a/services/keycloak/scripts/keycloak_vault_env.sh b/services/keycloak/scripts/keycloak_vault_env.sh index a9cfdae..62f7f38 100644 --- a/services/keycloak/scripts/keycloak_vault_env.sh +++ b/services/keycloak/scripts/keycloak_vault_env.sh @@ -23,3 +23,4 @@ export PORTAL_E2E_CLIENT_SECRET="$(read_secret portal-e2e-client__client_secret) export LDAP_ADMIN_PASSWORD="$(read_secret openldap-admin__LDAP_ADMIN_PASSWORD)" export LDAP_CONFIG_PASSWORD="$(read_secret openldap-admin__LDAP_CONFIG_PASSWORD)" +export LDAP_BIND_PASSWORD="${LDAP_ADMIN_PASSWORD}" diff --git a/services/keycloak/scripts/vault_oidc_secret_ensure.sh b/services/keycloak/scripts/vault_oidc_secret_ensure.sh index f7b3261..680057f 100755 --- a/services/keycloak/scripts/vault_oidc_secret_ensure.sh +++ b/services/keycloak/scripts/vault_oidc_secret_ensure.sh @@ -3,6 +3,8 @@ set -euo pipefail apk add --no-cache curl jq kubectl >/dev/null +. /vault/scripts/keycloak_vault_env.sh + KC_URL="http://keycloak.sso.svc.cluster.local" ACCESS_TOKEN="" for attempt in 1 2 3 4 5; do @@ -84,6 +86,37 @@ if [ -z "$CLIENT_SECRET" ] || [ "$CLIENT_SECRET" = "null" ]; then exit 1 fi +vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" +vault_role="${VAULT_ROLE:-sso-secrets}" +jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" +login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')" +vault_token="$(curl -sS --request POST --data "${login_payload}" \ + "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" +if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then + echo "vault login failed" >&2 + exit 1 +fi + +payload="$(jq -nc \ + --arg discovery_url "https://sso.bstein.dev/realms/atlas" \ + --arg client_id "vault-oidc" \ + --arg client_secret "${CLIENT_SECRET}" \ + --arg default_role "admin" \ + --arg scopes "openid profile email groups" \ + --arg user_claim "preferred_username" \ + --arg groups_claim "groups" \ + --arg redirect_uris "https://secret.bstein.dev/ui/vault/auth/oidc/oidc/callback,http://localhost:8250/oidc/callback" \ + --arg bound_audiences "vault-oidc" \ + --arg admin_group "admin" \ + --arg admin_policies "default,vault-admin" \ + --arg dev_group "dev" \ + --arg dev_policies "default,dev-kv" \ + --arg user_group "dev" \ + --arg user_policies "default,dev-kv" \ + '{data:{discovery_url:$discovery_url,client_id:$client_id,client_secret:$client_secret,default_role:$default_role,scopes:$scopes,user_claim:$user_claim,groups_claim:$groups_claim,redirect_uris:$redirect_uris,bound_audiences:$bound_audiences,admin_group:$admin_group,admin_policies:$admin_policies,dev_group:$dev_group,dev_policies:$dev_policies,user_group:$user_group,user_policies:$user_policies}}')" +curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \ + -d "${payload}" "${vault_addr}/v1/kv/data/atlas/vault/vault-oidc-config" >/dev/null + kubectl -n vault create secret generic vault-oidc-config \ --from-literal=discovery_url="https://sso.bstein.dev/realms/atlas" \ --from-literal=client_id="vault-oidc" \ diff --git a/services/keycloak/synapse-oidc-secret-ensure-job.yaml b/services/keycloak/synapse-oidc-secret-ensure-job.yaml index 7486ced..38e6753 100644 --- a/services/keycloak/synapse-oidc-secret-ensure-job.yaml +++ b/services/keycloak/synapse-oidc-secret-ensure-job.yaml @@ -18,7 +18,8 @@ spec: args: - | set -euo pipefail - apk add --no-cache curl jq kubectl >/dev/null + . /vault/scripts/keycloak_vault_env.sh + apk add --no-cache curl jq >/dev/null KC_URL="http://keycloak.sso.svc.cluster.local" ACCESS_TOKEN="" @@ -54,22 +55,35 @@ spec: exit 1 fi - existing="$(kubectl -n comms get secret synapse-oidc -o jsonpath='{.data.client-secret}' 2>/dev/null || true)" - if [ -n "${existing}" ]; then - exit 0 + vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" + vault_role="${VAULT_ROLE:-sso-secrets}" + jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" + login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')" + vault_token="$(curl -sS --request POST --data "${login_payload}" \ + "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" + if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then + echo "vault login failed" >&2 + exit 1 fi - kubectl -n comms create secret generic synapse-oidc \ - --from-literal=client-secret="${CLIENT_SECRET}" \ - --dry-run=client -o yaml | kubectl -n comms apply -f - >/dev/null - env: - - name: KEYCLOAK_ADMIN - valueFrom: - secretKeyRef: - name: keycloak-admin - key: username - - name: KEYCLOAK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-admin - key: password + payload="$(jq -nc --arg value "${CLIENT_SECRET}" '{data:{"client-secret":$value}}')" + curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \ + -d "${payload}" "${vault_addr}/v1/kv/data/atlas/comms/synapse-oidc" >/dev/null + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: sso-vault + - name: vault-scripts + configMap: + name: sso-vault-env + defaultMode: 0555 diff --git a/services/keycloak/user-overrides-job.yaml b/services/keycloak/user-overrides-job.yaml index 43813ee..2f580a9 100644 --- a/services/keycloak/user-overrides-job.yaml +++ b/services/keycloak/user-overrides-job.yaml @@ -19,6 +19,7 @@ spec: - key: node-role.kubernetes.io/worker operator: Exists restartPolicy: Never + serviceAccountName: sso-vault containers: - name: configure image: python:3.11-alpine @@ -27,16 +28,6 @@ spec: value: http://keycloak.sso.svc.cluster.local - name: KEYCLOAK_REALM value: atlas - - name: KEYCLOAK_ADMIN_USER - valueFrom: - secretKeyRef: - name: keycloak-admin - key: username - - name: KEYCLOAK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-admin - key: password - name: OVERRIDE_USERNAME value: bstein - name: OVERRIDE_MAILU_EMAIL @@ -45,6 +36,7 @@ spec: args: - | set -euo pipefail + . /vault/scripts/keycloak_vault_env.sh python - <<'PY' import json import os @@ -143,3 +135,21 @@ spec: if status not in (200, 204): raise SystemExit(f"Unexpected user update response: {status}") PY + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: sso-vault + - name: vault-scripts + configMap: + name: sso-vault-env + defaultMode: 0555 diff --git a/services/keycloak/vault-oidc-secret-ensure-job.yaml b/services/keycloak/vault-oidc-secret-ensure-job.yaml index ce3a1f0..2a8c382 100644 --- a/services/keycloak/vault-oidc-secret-ensure-job.yaml +++ b/services/keycloak/vault-oidc-secret-ensure-job.yaml @@ -16,6 +16,16 @@ spec: configMap: name: vault-oidc-secret-ensure-script defaultMode: 0555 + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: sso-vault + - name: vault-scripts + configMap: + name: sso-vault-env + defaultMode: 0555 affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -30,18 +40,13 @@ spec: - name: apply image: alpine:3.20 command: ["/scripts/vault_oidc_secret_ensure.sh"] - env: - - name: KEYCLOAK_ADMIN - valueFrom: - secretKeyRef: - name: keycloak-admin - key: username - - name: KEYCLOAK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-admin - key: password volumeMounts: - name: vault-oidc-secret-ensure-script mountPath: /scripts readOnly: true + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true diff --git a/services/mailu/kustomization.yaml b/services/mailu/kustomization.yaml index af4b2b1..31b1cb9 100644 --- a/services/mailu/kustomization.yaml +++ b/services/mailu/kustomization.yaml @@ -4,7 +4,10 @@ kind: Kustomization namespace: mailu-mailserver resources: - namespace.yaml + - serviceaccount.yaml + - secretproviderclass.yaml - helmrelease.yaml + - vault-sync-deployment.yaml - certificate.yaml - vip-controller.yaml - unbound-configmap.yaml @@ -16,6 +19,12 @@ resources: - front-lb.yaml configMapGenerator: + - name: mailu-vault-env + namespace: mailu-mailserver + files: + - mailu_vault_env.sh=scripts/mailu_vault_env.sh + options: + disableNameSuffixHash: true - name: mailu-sync-script namespace: mailu-mailserver files: diff --git a/services/mailu/mailu-sync-cronjob.yaml b/services/mailu/mailu-sync-cronjob.yaml index 268680f..4d73afa 100644 --- a/services/mailu/mailu-sync-cronjob.yaml +++ b/services/mailu/mailu-sync-cronjob.yaml @@ -12,6 +12,7 @@ spec: template: spec: restartPolicy: OnFailure + serviceAccountName: mailu-vault-sync containers: - name: mailu-sync image: python:3.11-alpine @@ -19,8 +20,10 @@ spec: command: ["/bin/sh", "-c"] args: - | + set -euo pipefail + . /vault/scripts/mailu_vault_env.sh pip install --no-cache-dir requests psycopg2-binary passlib >/tmp/pip.log \ - && python /app/sync.py + && python /app/sync.py env: - name: KEYCLOAK_BASE_URL value: http://keycloak.sso.svc.cluster.local @@ -34,35 +37,16 @@ spec: value: postgres-service.postgres.svc.cluster.local - name: MAILU_DB_PORT value: "5432" - - name: MAILU_DB_NAME - valueFrom: - secretKeyRef: - name: mailu-db-secret - key: database - - name: MAILU_DB_USER - valueFrom: - secretKeyRef: - name: mailu-db-secret - key: username - - name: MAILU_DB_PASSWORD - valueFrom: - secretKeyRef: - name: mailu-db-secret - key: password - - name: KEYCLOAK_CLIENT_ID - valueFrom: - secretKeyRef: - name: mailu-sync-credentials - key: client-id - - name: KEYCLOAK_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: mailu-sync-credentials - key: client-secret volumeMounts: - name: sync-script mountPath: /app/sync.py subPath: sync.py + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true resources: requests: cpu: 50m @@ -75,3 +59,13 @@ spec: configMap: name: mailu-sync-script defaultMode: 0444 + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: mailu-vault + - name: vault-scripts + configMap: + name: mailu-vault-env + defaultMode: 0555 diff --git a/services/mailu/mailu-sync-job.yaml b/services/mailu/mailu-sync-job.yaml index 7230c1d..60d48cb 100644 --- a/services/mailu/mailu-sync-job.yaml +++ b/services/mailu/mailu-sync-job.yaml @@ -8,6 +8,7 @@ spec: template: spec: restartPolicy: OnFailure + serviceAccountName: mailu-vault-sync containers: - name: mailu-sync image: python:3.11-alpine @@ -15,8 +16,10 @@ spec: command: ["/bin/sh", "-c"] args: - | + set -euo pipefail + . /vault/scripts/mailu_vault_env.sh pip install --no-cache-dir requests psycopg2-binary passlib >/tmp/pip.log \ - && python /app/sync.py + && python /app/sync.py env: - name: KEYCLOAK_BASE_URL value: http://keycloak.sso.svc.cluster.local @@ -30,35 +33,16 @@ spec: value: postgres-service.postgres.svc.cluster.local - name: MAILU_DB_PORT value: "5432" - - name: MAILU_DB_NAME - valueFrom: - secretKeyRef: - name: mailu-db-secret - key: database - - name: MAILU_DB_USER - valueFrom: - secretKeyRef: - name: mailu-db-secret - key: username - - name: MAILU_DB_PASSWORD - valueFrom: - secretKeyRef: - name: mailu-db-secret - key: password - - name: KEYCLOAK_CLIENT_ID - valueFrom: - secretKeyRef: - name: mailu-sync-credentials - key: client-id - - name: KEYCLOAK_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: mailu-sync-credentials - key: client-secret volumeMounts: - name: sync-script mountPath: /app/sync.py subPath: sync.py + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true resources: requests: cpu: 50m @@ -71,3 +55,13 @@ spec: configMap: name: mailu-sync-script defaultMode: 0444 + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: mailu-vault + - name: vault-scripts + configMap: + name: mailu-vault-env + defaultMode: 0555 diff --git a/services/mailu/mailu-sync-listener.yaml b/services/mailu/mailu-sync-listener.yaml index 2127313..f90164c 100644 --- a/services/mailu/mailu-sync-listener.yaml +++ b/services/mailu/mailu-sync-listener.yaml @@ -30,6 +30,7 @@ spec: app: mailu-sync-listener spec: restartPolicy: Always + serviceAccountName: mailu-vault-sync containers: - name: listener image: python:3.11-alpine @@ -37,8 +38,10 @@ spec: command: ["/bin/sh", "-c"] args: - | + set -euo pipefail + . /vault/scripts/mailu_vault_env.sh pip install --no-cache-dir requests psycopg2-binary passlib >/tmp/pip.log \ - && python /app/listener.py + && python /app/listener.py env: - name: KEYCLOAK_BASE_URL value: http://keycloak.sso.svc.cluster.local @@ -52,31 +55,6 @@ spec: value: postgres-service.postgres.svc.cluster.local - name: MAILU_DB_PORT value: "5432" - - name: MAILU_DB_NAME - valueFrom: - secretKeyRef: - name: mailu-db-secret - key: database - - name: MAILU_DB_USER - valueFrom: - secretKeyRef: - name: mailu-db-secret - key: username - - name: MAILU_DB_PASSWORD - valueFrom: - secretKeyRef: - name: mailu-db-secret - key: password - - name: KEYCLOAK_CLIENT_ID - valueFrom: - secretKeyRef: - name: mailu-sync-credentials - key: client-id - - name: KEYCLOAK_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: mailu-sync-credentials - key: client-secret volumeMounts: - name: sync-script mountPath: /app/sync.py @@ -84,6 +62,12 @@ spec: - name: listener-script mountPath: /app/listener.py subPath: listener.py + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true resources: requests: cpu: 50m @@ -100,3 +84,13 @@ spec: configMap: name: mailu-sync-listener defaultMode: 0444 + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: mailu-vault + - name: vault-scripts + configMap: + name: mailu-vault-env + defaultMode: 0555 diff --git a/services/mailu/scripts/mailu_vault_env.sh b/services/mailu/scripts/mailu_vault_env.sh new file mode 100644 index 0000000..082a51a --- /dev/null +++ b/services/mailu/scripts/mailu_vault_env.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env sh +set -eu + +vault_dir="/vault/secrets" + +read_secret() { + cat "${vault_dir}/$1" +} + +export MAILU_DB_NAME="$(read_secret mailu-db-secret__database)" +export MAILU_DB_USER="$(read_secret mailu-db-secret__username)" +export MAILU_DB_PASSWORD="$(read_secret mailu-db-secret__password)" +export KEYCLOAK_CLIENT_ID="$(read_secret mailu-sync-credentials__client-id)" +export KEYCLOAK_CLIENT_SECRET="$(read_secret mailu-sync-credentials__client-secret)" diff --git a/services/mailu/secretproviderclass.yaml b/services/mailu/secretproviderclass.yaml new file mode 100644 index 0000000..0ed32ba --- /dev/null +++ b/services/mailu/secretproviderclass.yaml @@ -0,0 +1,78 @@ +# services/mailu/secretproviderclass.yaml +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: mailu-vault + namespace: mailu-mailserver +spec: + provider: vault + parameters: + vaultAddress: "http://vault.vault.svc.cluster.local:8200" + roleName: "mailu-mailserver" + objects: | + - objectName: "mailu-secret__secret-key" + secretPath: "kv/data/atlas/mailu/mailu-secret" + secretKey: "secret-key" + - objectName: "postmark-relay__relay-username" + secretPath: "kv/data/atlas/shared/postmark-relay" + secretKey: "relay-username" + - objectName: "postmark-relay__relay-password" + secretPath: "kv/data/atlas/shared/postmark-relay" + secretKey: "relay-password" + - objectName: "mailu-db-secret__database" + secretPath: "kv/data/atlas/mailu/mailu-db-secret" + secretKey: "database" + - objectName: "mailu-db-secret__username" + secretPath: "kv/data/atlas/mailu/mailu-db-secret" + secretKey: "username" + - objectName: "mailu-db-secret__password" + secretPath: "kv/data/atlas/mailu/mailu-db-secret" + secretKey: "password" + - objectName: "mailu-db-secret__url" + secretPath: "kv/data/atlas/mailu/mailu-db-secret" + secretKey: "url" + - objectName: "mailu-initial-account-secret__password" + secretPath: "kv/data/atlas/mailu/mailu-initial-account-secret" + secretKey: "password" + - objectName: "mailu-sync-credentials__client-id" + secretPath: "kv/data/atlas/mailu/mailu-sync-credentials" + secretKey: "client-id" + - objectName: "mailu-sync-credentials__client-secret" + secretPath: "kv/data/atlas/mailu/mailu-sync-credentials" + secretKey: "client-secret" + secretObjects: + - secretName: mailu-secret + type: Opaque + data: + - objectName: mailu-secret__secret-key + key: secret-key + - secretName: mailu-postmark-relay + type: Opaque + data: + - objectName: postmark-relay__relay-username + key: relay-username + - objectName: postmark-relay__relay-password + key: relay-password + - secretName: mailu-db-secret + type: Opaque + data: + - objectName: mailu-db-secret__database + key: database + - objectName: mailu-db-secret__username + key: username + - objectName: mailu-db-secret__password + key: password + - objectName: mailu-db-secret__url + key: url + - secretName: mailu-initial-account-secret + type: Opaque + data: + - objectName: mailu-initial-account-secret__password + key: password + - secretName: mailu-sync-credentials + type: Opaque + data: + - objectName: mailu-sync-credentials__client-id + key: client-id + - objectName: mailu-sync-credentials__client-secret + key: client-secret diff --git a/services/mailu/serviceaccount.yaml b/services/mailu/serviceaccount.yaml new file mode 100644 index 0000000..d95410b --- /dev/null +++ b/services/mailu/serviceaccount.yaml @@ -0,0 +1,6 @@ +# services/mailu/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mailu-vault-sync + namespace: mailu-mailserver diff --git a/services/mailu/vault-sync-deployment.yaml b/services/mailu/vault-sync-deployment.yaml new file mode 100644 index 0000000..966f22b --- /dev/null +++ b/services/mailu/vault-sync-deployment.yaml @@ -0,0 +1,34 @@ +# services/mailu/vault-sync-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mailu-vault-sync + namespace: mailu-mailserver +spec: + replicas: 1 + selector: + matchLabels: + app: mailu-vault-sync + template: + metadata: + labels: + app: mailu-vault-sync + spec: + serviceAccountName: mailu-vault-sync + containers: + - name: sync + image: alpine:3.20 + command: ["/bin/sh", "-c"] + args: + - "sleep infinity" + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + volumes: + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: mailu-vault diff --git a/services/nextcloud-mail-sync/cronjob.yaml b/services/nextcloud-mail-sync/cronjob.yaml index 9976d8e..129022b 100644 --- a/services/nextcloud-mail-sync/cronjob.yaml +++ b/services/nextcloud-mail-sync/cronjob.yaml @@ -17,47 +17,23 @@ spec: securityContext: runAsUser: 0 runAsGroup: 0 + serviceAccountName: nextcloud-vault containers: - name: mail-sync image: nextcloud:29-apache imagePullPolicy: IfNotPresent command: - - /bin/bash - - /sync/sync.sh + - /bin/sh + - -c env: - name: KC_BASE value: https://sso.bstein.dev - name: KC_REALM value: atlas - - name: KC_ADMIN_USER - valueFrom: - secretKeyRef: - name: nextcloud-keycloak-admin - key: username - - name: KC_ADMIN_PASS - valueFrom: - secretKeyRef: - name: nextcloud-keycloak-admin - key: password - name: MAILU_DOMAIN value: bstein.dev - name: POSTGRES_HOST value: postgres-service.postgres.svc.cluster.local - - name: POSTGRES_DB - valueFrom: - secretKeyRef: - name: nextcloud-db - key: database - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: nextcloud-db - key: db-username - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: nextcloud-db - key: db-password resources: requests: cpu: 100m @@ -77,6 +53,17 @@ spec: - name: sync-script mountPath: /sync/sync.sh subPath: sync.sh + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true + args: + - | + set -euo pipefail + . /vault/scripts/nextcloud_vault_env.sh + exec /sync/sync.sh volumes: - name: nextcloud-config-pvc persistentVolumeClaim: @@ -94,3 +81,13 @@ spec: configMap: name: nextcloud-mail-sync-script defaultMode: 0755 + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: nextcloud-vault + - name: vault-scripts + configMap: + name: nextcloud-vault-env + defaultMode: 0555 diff --git a/services/nextcloud/deployment.yaml b/services/nextcloud/deployment.yaml index 295435e..894484c 100644 --- a/services/nextcloud/deployment.yaml +++ b/services/nextcloud/deployment.yaml @@ -22,6 +22,7 @@ spec: fsGroup: 33 runAsUser: 33 runAsGroup: 33 + serviceAccountName: nextcloud-vault initContainers: - name: seed-nextcloud-web image: nextcloud:29-apache @@ -80,6 +81,7 @@ spec: command: ["/bin/sh", "-c"] args: - | + . /vault/scripts/nextcloud_vault_env.sh installed="$(su -s /bin/sh www-data -c "php /var/www/html/occ status" 2>/dev/null | awk '/installed:/{print $3}' || true)" if [ ! -s /var/www/html/config/config.php ]; then su -s /bin/sh www-data -c "php /var/www/html/occ maintenance:install --database pgsql --database-host \"${POSTGRES_HOST}\" --database-name \"${POSTGRES_DB}\" --database-user \"${POSTGRES_USER}\" --database-pass \"${POSTGRES_PASSWORD}\" --admin-user \"${NEXTCLOUD_ADMIN_USER}\" --admin-pass \"${NEXTCLOUD_ADMIN_PASSWORD}\" --data-dir /var/www/html/data" @@ -150,41 +152,6 @@ spec: env: - name: POSTGRES_HOST value: postgres-service.postgres.svc.cluster.local - - name: POSTGRES_DB - valueFrom: - secretKeyRef: - name: nextcloud-db - key: database - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: nextcloud-db - key: db-username - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: nextcloud-db - key: db-password - - name: NEXTCLOUD_ADMIN_USER - valueFrom: - secretKeyRef: - name: nextcloud-admin - key: admin-user - - name: NEXTCLOUD_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: nextcloud-admin - key: admin-password - - name: OIDC_CLIENT_ID - valueFrom: - secretKeyRef: - name: nextcloud-oidc - key: client-id - - name: OIDC_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: nextcloud-oidc - key: client-secret volumeMounts: - name: nextcloud-web mountPath: /var/www/html @@ -197,40 +164,26 @@ spec: - name: nextcloud-config-extra mountPath: /var/www/html/config/extra.config.php subPath: extra.config.php + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true containers: - name: nextcloud image: nextcloud:29-apache imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] + args: + - >- + . /vault/scripts/nextcloud_vault_env.sh + && exec /entrypoint.sh apache2-foreground env: # DB (external secret required: nextcloud-db with keys username,password,database) - name: POSTGRES_HOST value: postgres-service.postgres.svc.cluster.local - - name: POSTGRES_DB - valueFrom: - secretKeyRef: - name: nextcloud-db - key: database - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: nextcloud-db - key: db-username - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: nextcloud-db - key: db-password # Admin bootstrap (external secret: nextcloud-admin with keys admin-user, admin-password) - - name: NEXTCLOUD_ADMIN_USER - valueFrom: - secretKeyRef: - name: nextcloud-admin - key: admin-user - - name: NEXTCLOUD_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: nextcloud-admin - key: admin-password - name: NEXTCLOUD_TRUSTED_DOMAINS value: cloud.bstein.dev - name: OVERWRITEHOST @@ -246,31 +199,11 @@ spec: value: "587" - name: SMTP_SECURE value: tls - - name: SMTP_NAME - valueFrom: - secretKeyRef: - name: nextcloud-smtp - key: smtp-username - - name: SMTP_PASSWORD - valueFrom: - secretKeyRef: - name: nextcloud-smtp - key: smtp-password - name: MAIL_FROM_ADDRESS value: no-reply - name: MAIL_DOMAIN value: bstein.dev # OIDC (external secret: nextcloud-oidc with keys client-id, client-secret) - - name: OIDC_CLIENT_ID - valueFrom: - secretKeyRef: - name: nextcloud-oidc - key: client-id - - name: OIDC_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: nextcloud-oidc - key: client-secret - name: NEXTCLOUD_UPDATE value: "1" - name: APP_INSTALL @@ -290,6 +223,12 @@ spec: - name: nextcloud-config-extra mountPath: /var/www/html/config/extra.config.php subPath: extra.config.php + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true resources: requests: cpu: 250m @@ -314,3 +253,13 @@ spec: configMap: name: nextcloud-config defaultMode: 0444 + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: nextcloud-vault + - name: vault-scripts + configMap: + name: nextcloud-vault-env + defaultMode: 0555 diff --git a/services/nextcloud/kustomization.yaml b/services/nextcloud/kustomization.yaml index 14e0ec1..f16db47 100644 --- a/services/nextcloud/kustomization.yaml +++ b/services/nextcloud/kustomization.yaml @@ -4,6 +4,8 @@ kind: Kustomization namespace: nextcloud resources: - namespace.yaml + - serviceaccount.yaml + - secretproviderclass.yaml - configmap.yaml - pvc.yaml - deployment.yaml @@ -13,6 +15,11 @@ resources: - service.yaml - ingress.yaml configMapGenerator: + - name: nextcloud-vault-env + files: + - nextcloud_vault_env.sh=scripts/nextcloud_vault_env.sh + options: + disableNameSuffixHash: true - name: nextcloud-maintenance-script files: - maintenance.sh=scripts/nextcloud-maintenance.sh diff --git a/services/nextcloud/maintenance-cronjob.yaml b/services/nextcloud/maintenance-cronjob.yaml index 618f548..d76478e 100644 --- a/services/nextcloud/maintenance-cronjob.yaml +++ b/services/nextcloud/maintenance-cronjob.yaml @@ -15,24 +15,20 @@ spec: securityContext: runAsUser: 0 runAsGroup: 0 + serviceAccountName: nextcloud-vault containers: - name: maintenance image: nextcloud:29-apache imagePullPolicy: IfNotPresent - command: ["/bin/bash", "/maintenance/maintenance.sh"] + command: ["/bin/sh", "-c"] + args: + - | + set -euo pipefail + . /vault/scripts/nextcloud_vault_env.sh + exec /maintenance/maintenance.sh env: - name: NC_URL value: https://cloud.bstein.dev - - name: ADMIN_USER - valueFrom: - secretKeyRef: - name: nextcloud-admin - key: admin-user - - name: ADMIN_PASS - valueFrom: - secretKeyRef: - name: nextcloud-admin - key: admin-password volumeMounts: - name: nextcloud-web mountPath: /var/www/html @@ -45,6 +41,12 @@ spec: - name: maintenance-script mountPath: /maintenance/maintenance.sh subPath: maintenance.sh + - name: vault-secrets + mountPath: /vault/secrets + readOnly: true + - name: vault-scripts + mountPath: /vault/scripts + readOnly: true resources: requests: cpu: 100m @@ -69,3 +71,13 @@ spec: configMap: name: nextcloud-maintenance-script defaultMode: 0755 + - name: vault-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: nextcloud-vault + - name: vault-scripts + configMap: + name: nextcloud-vault-env + defaultMode: 0555 diff --git a/services/nextcloud/scripts/nextcloud_vault_env.sh b/services/nextcloud/scripts/nextcloud_vault_env.sh new file mode 100644 index 0000000..0f34c9f --- /dev/null +++ b/services/nextcloud/scripts/nextcloud_vault_env.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env sh +set -eu + +vault_dir="/vault/secrets" + +read_secret() { + cat "${vault_dir}/$1" +} + +export POSTGRES_DB="$(read_secret nextcloud-db__database)" +export POSTGRES_USER="$(read_secret nextcloud-db__db-username)" +export POSTGRES_PASSWORD="$(read_secret nextcloud-db__db-password)" + +export NEXTCLOUD_ADMIN_USER="$(read_secret nextcloud-admin__admin-user)" +export NEXTCLOUD_ADMIN_PASSWORD="$(read_secret nextcloud-admin__admin-password)" + +export ADMIN_USER="${NEXTCLOUD_ADMIN_USER}" +export ADMIN_PASS="${NEXTCLOUD_ADMIN_PASSWORD}" + +export OIDC_CLIENT_ID="$(read_secret nextcloud-oidc__client-id)" +export OIDC_CLIENT_SECRET="$(read_secret nextcloud-oidc__client-secret)" + +export SMTP_NAME="$(read_secret nextcloud-smtp__smtp-username)" +export SMTP_PASSWORD="$(read_secret nextcloud-smtp__smtp-password)" + +export KC_ADMIN_USER="$(read_secret keycloak-admin__username)" +export KC_ADMIN_PASS="$(read_secret keycloak-admin__password)" diff --git a/services/nextcloud/secretproviderclass.yaml b/services/nextcloud/secretproviderclass.yaml new file mode 100644 index 0000000..b5e6c37 --- /dev/null +++ b/services/nextcloud/secretproviderclass.yaml @@ -0,0 +1,45 @@ +# services/nextcloud/secretproviderclass.yaml +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: nextcloud-vault + namespace: nextcloud +spec: + provider: vault + parameters: + vaultAddress: "http://vault.vault.svc.cluster.local:8200" + roleName: "nextcloud" + objects: | + - objectName: "nextcloud-db__database" + secretPath: "kv/data/atlas/nextcloud/nextcloud-db" + secretKey: "database" + - objectName: "nextcloud-db__db-username" + secretPath: "kv/data/atlas/nextcloud/nextcloud-db" + secretKey: "db-username" + - objectName: "nextcloud-db__db-password" + secretPath: "kv/data/atlas/nextcloud/nextcloud-db" + secretKey: "db-password" + - objectName: "nextcloud-admin__admin-user" + secretPath: "kv/data/atlas/nextcloud/nextcloud-admin" + secretKey: "admin-user" + - objectName: "nextcloud-admin__admin-password" + secretPath: "kv/data/atlas/nextcloud/nextcloud-admin" + secretKey: "admin-password" + - objectName: "nextcloud-oidc__client-id" + secretPath: "kv/data/atlas/nextcloud/nextcloud-oidc" + secretKey: "client-id" + - objectName: "nextcloud-oidc__client-secret" + secretPath: "kv/data/atlas/nextcloud/nextcloud-oidc" + secretKey: "client-secret" + - objectName: "nextcloud-smtp__smtp-username" + secretPath: "kv/data/atlas/nextcloud/nextcloud-smtp" + secretKey: "smtp-username" + - objectName: "nextcloud-smtp__smtp-password" + secretPath: "kv/data/atlas/nextcloud/nextcloud-smtp" + secretKey: "smtp-password" + - objectName: "keycloak-admin__username" + secretPath: "kv/data/atlas/shared/keycloak-admin" + secretKey: "username" + - objectName: "keycloak-admin__password" + secretPath: "kv/data/atlas/shared/keycloak-admin" + secretKey: "password" diff --git a/services/nextcloud/serviceaccount.yaml b/services/nextcloud/serviceaccount.yaml new file mode 100644 index 0000000..c97cd5b --- /dev/null +++ b/services/nextcloud/serviceaccount.yaml @@ -0,0 +1,6 @@ +# services/nextcloud/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nextcloud-vault + namespace: nextcloud diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index fdffbea..39577ba 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -35,66 +35,71 @@ vault write auth/kubernetes/config \ kubernetes_host="${k8s_host}" \ kubernetes_ca_cert="${k8s_ca}" -for namespace in outline planka bstein-dev-home gitea vaultwarden sso; do - policy_name="${namespace}" - service_account="" - shared_paths="" +write_policy_and_role() { + role="$1" + namespace="$2" + service_accounts="$3" + read_paths="$4" + write_paths="$5" - case "${namespace}" in - outline) - service_account="outline-vault" - ;; - planka) - service_account="planka-vault" - ;; - bstein-dev-home) - service_account="bstein-dev-home" - shared_paths="shared/chat-ai-keys-runtime shared/portal-e2e-client" - ;; - gitea) - service_account="gitea-vault" - ;; - vaultwarden) - service_account="vaultwarden-vault" - ;; - sso) - service_account="sso-vault,mas-secrets-ensure" - shared_paths="shared/keycloak-admin shared/portal-e2e-client" - ;; - *) - log "unknown namespace ${namespace}" - exit 1 - ;; - esac - - policy_body="$(cat <