bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

deploy #10

Merged
bstein merged 271 commits from deploy into main 2026-01-19 19:04:01 +00:00
Conversation 0 Commits 271 Files Changed 335 +1527 -686
61 changed files with 1527 additions and 686 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit de3db3133b - Show all commits

2
infrastructure/vault-csi/secrets-store-csi-driver.yaml
View File

@ -16,5 +16,5 @@ spec:
namespace: flux-system
values:
syncSecret:
enabled: false
enabled: true
enableSecretRotation: false

29
services/comms/atlasbot-deployment.yaml
View File

@ -27,7 +27,8 @@ spec:
command: ["/bin/sh","-c"]
args:
- |
python /app/bot.py
. /vault/scripts/comms_vault_env.sh
exec python /app/bot.py
env:
- name: MATRIX_BASE
value: http://othrys-synapse-matrix-synapse:8008
@ -39,16 +40,6 @@ spec:
value: http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428
- name: BOT_USER
value: atlasbot
- name: BOT_PASS
valueFrom:
secretKeyRef:
name: atlasbot-credentials-runtime
key: bot-password
- name: CHAT_API_KEY
valueFrom:
secretKeyRef:
name: chat-ai-keys-runtime
key: matrix
- name: OLLAMA_URL
value: https://chat.ai.bstein.dev/
- name: OLLAMA_MODEL
@ -67,6 +58,12 @@ spec:
- name: kb
mountPath: /kb
readOnly: true
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: code
configMap:
@ -85,3 +82,13 @@ spec:
path: catalog/runbooks.json
- key: atlas-http.mmd
path: diagrams/atlas-http.mmd
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault
- name: vault-scripts
configMap:
name: comms-vault-env
defaultMode: 0555

19
services/comms/bstein-force-leave-job.yaml
View File

@ -9,25 +9,26 @@ spec:
template:
spec:
restartPolicy: Never
serviceAccountName: comms-vault
volumes:
- name: mas-admin-client
secret:
secretName: mas-admin-client-runtime
items:
- key: client_secret
path: client_secret
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault
containers:
- name: leave
image: python:3.11-slim
volumeMounts:
- name: mas-admin-client
mountPath: /etc/mas-admin-client
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
env:
- name: MAS_ADMIN_CLIENT_ID
value: 01KDXMVQBQ5JNY6SEJPZW6Z8BM
- name: MAS_ADMIN_CLIENT_SECRET_FILE
value: /etc/mas-admin-client/client_secret
value: /vault/secrets/mas-admin-client-runtime__client_secret
- name: MAS_TOKEN_URL
value: http://matrix-authentication-service:8080/oauth2/token
- name: MAS_ADMIN_API_BASE

93
services/comms/comms-secrets-ensure-job.yaml
View File

@ -20,73 +20,58 @@ spec:
set -eu
trap 'echo "comms-secrets-ensure failed"; sleep 300' ERR
umask 077
apk add --no-cache curl jq >/dev/null
safe_pass() {
head -c 32 /dev/urandom | base64 | tr -d '\n' | tr '+/' '-_' | tr -d '='
}
get_secret_value() {
ns="$1"
name="$2"
key="$3"
kubectl -n "${ns}" get secret "${name}" -o "jsonpath={.data.${key}}" 2>/dev/null | base64 -d 2>/dev/null || true
vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
vault_role="${VAULT_ROLE:-comms-secrets}"
jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
vault_token="$(curl -sS --request POST --data "${login_payload}" \
"${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
echo "vault login failed" >&2
exit 1
fi
vault_read() {
path="$1"
key="$2"
curl -sS -H "X-Vault-Token: ${vault_token}" \
"${vault_addr}/v1/kv/data/atlas/${path}" | jq -r --arg key "${key}" '.data.data[$key] // empty'
}
ensure_secret_key() {
ns="$1"
name="$2"
key="$3"
value="$4"
if ! kubectl -n "${ns}" get secret "${name}" >/dev/null 2>&1; then
kubectl -n "${ns}" create secret generic "${name}" --from-literal="${key}=${value}" >/dev/null
return
fi
existing="$(kubectl -n "${ns}" get secret "${name}" -o "jsonpath={.data.${key}}" 2>/dev/null || true)"
if [ -z "${existing}" ]; then
b64="$(printf '%s' "${value}" | base64 | tr -d '\n')"
payload="$(printf '{"data":{"%s":"%s"}}' "${key}" "${b64}")"
kubectl -n "${ns}" patch secret "${name}" --type=merge -p "${payload}" >/dev/null
fi
vault_write() {
path="$1"
key="$2"
value="$3"
payload="$(jq -nc --arg key "${key}" --arg value "${value}" '{data:{($key):$value}}')"
curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
-d "${payload}" "${vault_addr}/v1/kv/data/atlas/${path}" >/dev/null
}
ensure_chat_secret() {
ns="$1"
if ! kubectl -n "${ns}" get secret chat-ai-keys-runtime >/dev/null 2>&1; then
kubectl -n "${ns}" create secret generic chat-ai-keys-runtime \
--from-literal=matrix="${CHAT_KEY_MATRIX}" \
--from-literal=homepage="${CHAT_KEY_HOMEPAGE}" >/dev/null
return
ensure_key() {
path="$1"
key="$2"
current="$(vault_read "${path}" "${key}")"
if [ -z "${current}" ]; then
current="$(safe_pass)"
vault_write "${path}" "${key}" "${current}"
fi
ensure_secret_key "${ns}" chat-ai-keys-runtime matrix "${CHAT_KEY_MATRIX}"
ensure_secret_key "${ns}" chat-ai-keys-runtime homepage "${CHAT_KEY_HOMEPAGE}"
printf '%s' "${current}"
}
CHAT_KEY_MATRIX="$(get_secret_value comms chat-ai-keys-runtime matrix)"
CHAT_KEY_HOMEPAGE="$(get_secret_value comms chat-ai-keys-runtime homepage)"
if [ -z "${CHAT_KEY_MATRIX}" ] || [ -z "${CHAT_KEY_HOMEPAGE}" ]; then
ALT_MATRIX="$(get_secret_value bstein-dev-home chat-ai-keys-runtime matrix)"
ALT_HOMEPAGE="$(get_secret_value bstein-dev-home chat-ai-keys-runtime homepage)"
[ -z "${CHAT_KEY_MATRIX}" ] && CHAT_KEY_MATRIX="${ALT_MATRIX}"
[ -z "${CHAT_KEY_HOMEPAGE}" ] && CHAT_KEY_HOMEPAGE="${ALT_HOMEPAGE}"
fi
[ -z "${CHAT_KEY_MATRIX}" ] && CHAT_KEY_MATRIX="$(safe_pass)"
[ -z "${CHAT_KEY_HOMEPAGE}" ] && CHAT_KEY_HOMEPAGE="$(safe_pass)"
ensure_key "comms/turn-shared-secret" "TURN_STATIC_AUTH_SECRET" >/dev/null
ensure_key "comms/livekit-api" "primary" >/dev/null
ensure_key "comms/synapse-redis" "redis-password" >/dev/null
ensure_key "comms/synapse-macaroon" "macaroon_secret_key" >/dev/null
ensure_key "comms/atlasbot-credentials-runtime" "bot-password" >/dev/null
ensure_key "comms/atlasbot-credentials-runtime" "seeder-password" >/dev/null
ensure_chat_secret comms
ensure_chat_secret bstein-dev-home
ensure_secret_key comms turn-shared-secret TURN_STATIC_AUTH_SECRET "$(safe_pass)"
ensure_secret_key comms livekit-api primary "$(safe_pass)"
ensure_secret_key comms synapse-redis redis-password "$(safe_pass)"
ensure_secret_key comms synapse-macaroon macaroon_secret_key "$(safe_pass)"
ensure_secret_key comms atlasbot-credentials-runtime bot-password "$(safe_pass)"
ensure_secret_key comms atlasbot-credentials-runtime seeder-password "$(safe_pass)"
SYN_PASS="$(get_secret_value comms synapse-db POSTGRES_PASSWORD)"
if [ -z "${SYN_PASS}" ]; then
SYN_PASS="$(safe_pass)"
kubectl -n comms create secret generic synapse-db --from-literal=POSTGRES_PASSWORD="${SYN_PASS}" >/dev/null
fi
SYN_PASS="$(ensure_key "comms/synapse-db" "POSTGRES_PASSWORD")"
POD_NAME="$(kubectl -n postgres get pods -l app=postgres -o jsonpath='{.items[0].metadata.name}')"
if [ -z "${POD_NAME}" ]; then

23
services/comms/coturn.yaml
View File

@ -15,6 +15,7 @@ spec:
labels:
app: coturn
spec:
serviceAccountName: comms-vault
nodeSelector:
hardware: rpi5
affinity:
@ -33,6 +34,7 @@ spec:
- /bin/sh
- -c
- |
. /vault/scripts/comms_vault_env.sh
exec /usr/bin/turnserver \
--no-cli \
--fingerprint \
@ -57,11 +59,6 @@ spec:
fieldPath: status.podIP
- name: TURN_PUBLIC_IP
value: "38.28.125.112"
- name: TURN_STATIC_AUTH_SECRET
valueFrom:
secretKeyRef:
name: turn-shared-secret
key: TURN_STATIC_AUTH_SECRET
ports:
- name: turn-udp
containerPort: 3478
@ -76,6 +73,12 @@ spec:
- name: tls
mountPath: /etc/coturn/tls
readOnly: true
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
resources:
requests:
cpu: 200m
@ -87,6 +90,16 @@ spec:
- name: tls
secret:
secretName: turn-live-tls
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault
- name: vault-scripts
configMap:
name: comms-vault-env
defaultMode: 0555
---
apiVersion: v1
kind: Service

32
services/comms/guest-name-job.yaml
View File

@ -16,19 +16,27 @@ spec:
template:
spec:
restartPolicy: Never
serviceAccountName: comms-vault
volumes:
- name: mas-admin-client
secret:
secretName: mas-admin-client-runtime
items:
- key: client_secret
path: client_secret
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault
- name: vault-scripts
configMap:
name: comms-vault-env
defaultMode: 0555
containers:
- name: rename
image: python:3.11-slim
volumeMounts:
- name: mas-admin-client
mountPath: /etc/mas-admin-client
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
env:
- name: SYNAPSE_BASE
@ -36,7 +44,7 @@ spec:
- name: MAS_ADMIN_CLIENT_ID
value: 01KDXMVQBQ5JNY6SEJPZW6Z8BM
- name: MAS_ADMIN_CLIENT_SECRET_FILE
value: /etc/mas-admin-client/client_secret
value: /vault/secrets/mas-admin-client-runtime__client_secret
- name: MAS_ADMIN_API_BASE
value: http://matrix-authentication-service:8081/api/admin/v1
- name: MAS_TOKEN_URL
@ -51,16 +59,12 @@ spec:
value: synapse
- name: PGUSER
value: synapse
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: synapse-db
key: POSTGRES_PASSWORD
command:
- /bin/sh
- -c
- |
set -euo pipefail
. /vault/scripts/comms_vault_env.sh
pip install --no-cache-dir requests psycopg2-binary >/dev/null
python - <<'PY'
import base64

19
services/comms/guest-register-deployment.yaml
View File

@ -17,6 +17,7 @@ spec:
labels:
app.kubernetes.io/name: matrix-guest-register
spec:
serviceAccountName: comms-vault
securityContext:
runAsNonRoot: true
runAsUser: 10001
@ -42,7 +43,7 @@ spec:
- name: MAS_ADMIN_CLIENT_ID
value: 01KDXMVQBQ5JNY6SEJPZW6Z8BM
- name: MAS_ADMIN_CLIENT_SECRET_FILE
value: /etc/mas/admin-client/client_secret
value: /vault/secrets/mas-admin-client-runtime__client_secret
- name: MAS_ADMIN_API_BASE
value: http://matrix-authentication-service:8081/api/admin/v1
- name: SYNAPSE_BASE
@ -83,8 +84,8 @@ spec:
mountPath: /app/server.py
subPath: server.py
readOnly: true
- name: mas-admin-client
mountPath: /etc/mas/admin-client
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
command:
- python
@ -96,9 +97,9 @@ spec:
items:
- key: server.py
path: server.py
- name: mas-admin-client
secret:
secretName: mas-admin-client-runtime
items:
- key: client_secret
path: client_secret
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault

8
services/comms/kustomization.yaml
View File

@ -4,6 +4,8 @@ kind: Kustomization
namespace: comms
resources:
- namespace.yaml
- serviceaccount.yaml
- secretproviderclass.yaml
- mas-configmap.yaml
- helmrelease.yaml
- livekit-config.yaml
@ -18,6 +20,7 @@ resources:
- comms-secrets-ensure-rbac.yaml
- mas-db-ensure-rbac.yaml
- synapse-signingkey-ensure-rbac.yaml
- vault-sync-deployment.yaml
- mas-admin-client-secret-ensure-job.yaml
- mas-db-ensure-job.yaml
- comms-secrets-ensure-job.yaml
@ -40,6 +43,11 @@ resources:
- matrix-ingress.yaml
configMapGenerator:
- name: comms-vault-env
files:
- comms_vault_env.sh=scripts/comms_vault_env.sh
options:
disableNameSuffixHash: true
- name: matrix-guest-register
files:
- server.py=scripts/guest-register/server.py

30
services/comms/livekit-token-deployment.yaml
View File

@ -15,6 +15,7 @@ spec:
labels:
app: livekit-token-service
spec:
serviceAccountName: comms-vault
nodeSelector:
hardware: rpi5
affinity:
@ -33,21 +34,29 @@ spec:
containers:
- name: token-service
image: ghcr.io/element-hq/lk-jwt-service:0.3.0
command:
- /bin/sh
- -c
- |
. /vault/scripts/comms_vault_env.sh
exec /lk-jwt-service
env:
- name: LIVEKIT_URL
value: wss://kit.live.bstein.dev/livekit/sfu
- name: LIVEKIT_KEY
value: primary
- name: LIVEKIT_SECRET
valueFrom:
secretKeyRef:
name: livekit-api
key: primary
- name: LIVEKIT_FULL_ACCESS_HOMESERVERS
value: live.bstein.dev
ports:
- containerPort: 8080
name: http
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
resources:
requests:
cpu: 50m
@ -55,6 +64,17 @@ spec:
limits:
cpu: 300m
memory: 256Mi
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault
- name: vault-scripts
configMap:
name: comms-vault-env
defaultMode: 0555
---
apiVersion: v1
kind: Service

36
services/comms/livekit.yaml
View File

@ -17,6 +17,7 @@ spec:
labels:
app: livekit
spec:
serviceAccountName: comms-vault
enableServiceLinks: false
nodeSelector:
hardware: rpi5
@ -36,16 +37,11 @@ spec:
args:
- |
set -euo pipefail
. /vault/scripts/comms_vault_env.sh
umask 077
TURN_PASSWORD_ESCAPED="$(printf '%s' "${TURN_PASSWORD}" | sed 's/[\\/&]/\\&/g')"
sed "s/@@TURN_PASSWORD@@/${TURN_PASSWORD_ESCAPED}/g" /etc/livekit-template/livekit.yaml > /etc/livekit/livekit.yaml
chmod 0644 /etc/livekit/livekit.yaml
env:
- name: TURN_PASSWORD
valueFrom:
secretKeyRef:
name: turn-shared-secret
key: TURN_STATIC_AUTH_SECRET
volumeMounts:
- name: config-template
mountPath: /etc/livekit-template
@ -53,6 +49,12 @@ spec:
- name: config
mountPath: /etc/livekit
readOnly: false
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
containers:
- name: livekit
image: livekit/livekit-server:v1.9.0
@ -61,6 +63,7 @@ spec:
- -c
- |
set -euo pipefail
. /vault/scripts/comms_vault_env.sh
umask 077
printf "%s: %s\n" "${LIVEKIT_API_KEY_ID}" "${LIVEKIT_API_SECRET}" > /var/run/livekit/keys
chmod 600 /var/run/livekit/keys
@ -68,11 +71,6 @@ spec:
env:
- name: LIVEKIT_API_KEY_ID
value: primary
- name: LIVEKIT_API_SECRET
valueFrom:
secretKeyRef:
name: livekit-api
key: primary
ports:
- containerPort: 7880
name: http
@ -92,6 +90,12 @@ spec:
readOnly: true
- name: runtime-keys
mountPath: /var/run/livekit
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
resources:
requests:
cpu: 500m
@ -110,6 +114,16 @@ spec:
emptyDir: {}
- name: runtime-keys
emptyDir: {}
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault
- name: vault-scripts
configMap:
name: comms-vault-env
defaultMode: 0555
---
apiVersion: v1
kind: Service

31
services/comms/mas-admin-client-secret-ensure-job.yaml
View File

@ -67,18 +67,29 @@ spec:
args:
- |
set -euo pipefail
if kubectl -n comms get secret mas-admin-client-runtime >/dev/null 2>&1; then
if kubectl -n comms get secret mas-admin-client-runtime -o jsonpath='{.data.client_secret}' 2>/dev/null | grep -q .; then
apk add --no-cache curl jq >/dev/null
vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
vault_role="${VAULT_ROLE:-comms-secrets}"
jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
vault_token="$(curl -sS --request POST --data "${login_payload}" \
"${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
echo "vault login failed" >&2
exit 1
fi
current="$(curl -sS -H "X-Vault-Token: ${vault_token}" \
"${vault_addr}/v1/kv/data/atlas/comms/mas-admin-client-runtime" | jq -r '.data.data.client_secret // empty')"
if [ -n "${current}" ]; then
exit 0
fi
else
kubectl -n comms create secret generic mas-admin-client-runtime \
--from-file=client_secret=/work/client_secret >/dev/null
exit 0
fi
secret_b64="$(base64 /work/client_secret | tr -d '\n')"
payload="$(printf '{"data":{"client_secret":"%s"}}' "${secret_b64}")"
kubectl -n comms patch secret mas-admin-client-runtime --type=merge -p "${payload}" >/dev/null
value="$(cat /work/client_secret)"
payload="$(jq -nc --arg value "${value}" '{data:{client_secret:$value}}')"
curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
-d "${payload}" "${vault_addr}/v1/kv/data/atlas/comms/mas-admin-client-runtime" >/dev/null
volumeMounts:
- name: work
mountPath: /work

37
services/comms/mas-db-ensure-job.yaml
View File

@ -24,18 +24,35 @@ spec:
head -c 32 /dev/urandom | base64 | tr -d '\n' | tr '+/' '-_' | tr -d '='
}
EXISTING_B64="$(kubectl -n comms get secret mas-db -o jsonpath='{.data.password}' 2>/dev/null || true)"
if [ -n "${EXISTING_B64}" ]; then
MAS_PASS="$(printf '%s' "${EXISTING_B64}" | base64 -d)"
if printf '%s' "${MAS_PASS}" | grep -Eq '[^A-Za-z0-9_-]'; then
MAS_PASS="$(safe_pass)"
MAS_B64="$(printf '%s' "${MAS_PASS}" | base64 | tr -d '\n')"
payload="$(printf '{"data":{"password":"%s"}}' "${MAS_B64}")"
kubectl -n comms patch secret mas-db --type=merge -p "${payload}" >/dev/null
apk add --no-cache curl jq >/dev/null
vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
vault_role="${VAULT_ROLE:-comms-secrets}"
jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
vault_token="$(curl -sS --request POST --data "${login_payload}" \
"${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
echo "vault login failed" >&2
exit 1
fi
else
vault_read() {
curl -sS -H "X-Vault-Token: ${vault_token}" \
"${vault_addr}/v1/kv/data/atlas/comms/mas-db" | jq -r '.data.data.password // empty'
}
vault_write() {
value="$1"
payload="$(jq -nc --arg value "${value}" '{data:{password:$value}}')"
curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
-d "${payload}" "${vault_addr}/v1/kv/data/atlas/comms/mas-db" >/dev/null
}
MAS_PASS="$(vault_read)"
if [ -z "${MAS_PASS}" ] || printf '%s' "${MAS_PASS}" | grep -Eq '[^A-Za-z0-9_-]'; then
MAS_PASS="$(safe_pass)"
kubectl -n comms create secret generic mas-db --from-literal=password="${MAS_PASS}" >/dev/null
vault_write "${MAS_PASS}"
fi
POD_NAME="$(kubectl -n postgres get pods -l app=postgres -o jsonpath='{.items[0].metadata.name}')"

79
services/comms/mas-deployment.yaml
View File

@ -18,6 +18,7 @@ spec:
app: matrix-authentication-service
spec:
enableServiceLinks: false
serviceAccountName: comms-vault
nodeSelector:
hardware: rpi5
affinity:
@ -36,6 +37,7 @@ spec:
args:
- |
set -euo pipefail
. /vault/scripts/comms_vault_env.sh
umask 077
DB_PASS_ESCAPED="$(printf '%s' "${MAS_DB_PASSWORD}" | sed 's/[\\/&]/\\&/g')"
MATRIX_SECRET_ESCAPED="$(printf '%s' "${MATRIX_SHARED_SECRET}" | sed 's/[\\/&]/\\&/g')"
@ -47,22 +49,6 @@ spec:
-e "s/@@KEYCLOAK_CLIENT_SECRET@@/${KC_SECRET_ESCAPED}/g" \
/etc/mas/config.yaml > /rendered/config.yaml
chmod 0644 /rendered/config.yaml
env:
- name: MAS_DB_PASSWORD
valueFrom:
secretKeyRef:
name: mas-db
key: password
- name: MATRIX_SHARED_SECRET
valueFrom:
secretKeyRef:
name: mas-secrets-runtime
key: matrix_shared_secret
- name: KEYCLOAK_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: mas-secrets-runtime
key: keycloak_client_secret
volumeMounts:
- name: config
mountPath: /etc/mas/config.yaml
@ -71,6 +57,12 @@ spec:
- name: rendered
mountPath: /rendered
readOnly: false
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
containers:
- name: mas
image: ghcr.io/element-hq/matrix-authentication-service:1.8.0
@ -86,14 +78,25 @@ spec:
- name: rendered
mountPath: /rendered
readOnly: true
- name: secrets
mountPath: /etc/mas/secrets
- name: vault-secrets
mountPath: /etc/mas/secrets/encryption
subPath: mas-secrets-runtime__encryption
readOnly: true
- name: admin-client
mountPath: /etc/mas/admin-client
- name: vault-secrets
mountPath: /etc/mas/secrets/matrix_shared_secret
subPath: mas-secrets-runtime__matrix_shared_secret
readOnly: true
- name: keys
mountPath: /etc/mas/keys
- name: vault-secrets
mountPath: /etc/mas/secrets/keycloak_client_secret
subPath: mas-secrets-runtime__keycloak_client_secret
readOnly: true
- name: vault-secrets
mountPath: /etc/mas/keys/rsa_key
subPath: mas-secrets-runtime__rsa_key
readOnly: true
- name: vault-secrets
mountPath: /etc/mas/admin-client/client_secret
subPath: mas-admin-client-runtime__client_secret
readOnly: true
resources:
requests:
@ -111,28 +114,16 @@ spec:
path: config.yaml
- name: rendered
emptyDir: {}
- name: secrets
secret:
secretName: mas-secrets-runtime
items:
- key: encryption
path: encryption
- key: matrix_shared_secret
path: matrix_shared_secret
- key: keycloak_client_secret
path: keycloak_client_secret
- name: keys
secret:
secretName: mas-secrets-runtime
items:
- key: rsa_key
path: rsa_key
- name: admin-client
secret:
secretName: mas-admin-client-runtime
items:
- key: client_secret
path: client_secret
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault
- name: vault-scripts
configMap:
name: comms-vault-env
defaultMode: 0555
---
apiVersion: v1
kind: Service

37
services/comms/mas-local-users-ensure-job.yaml
View File

@ -10,48 +10,47 @@ spec:
template:
spec:
restartPolicy: Never
serviceAccountName: comms-vault
volumes:
- name: mas-admin-client
secret:
secretName: mas-admin-client-runtime
items:
- key: client_secret
path: client_secret
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault
- name: vault-scripts
configMap:
name: comms-vault-env
defaultMode: 0555
containers:
- name: ensure
image: python:3.11-slim
volumeMounts:
- name: mas-admin-client
mountPath: /etc/mas-admin-client
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
env:
- name: MAS_ADMIN_CLIENT_ID
value: 01KDXMVQBQ5JNY6SEJPZW6Z8BM
- name: MAS_ADMIN_CLIENT_SECRET_FILE
value: /etc/mas-admin-client/client_secret
value: /vault/secrets/mas-admin-client-runtime__client_secret
- name: MAS_TOKEN_URL
value: http://matrix-authentication-service:8080/oauth2/token
- name: MAS_ADMIN_API_BASE
value: http://matrix-authentication-service:8081/api/admin/v1
- name: SEEDER_USER
value: othrys-seeder
- name: SEEDER_PASS
valueFrom:
secretKeyRef:
name: atlasbot-credentials-runtime
key: seeder-password
- name: BOT_USER
value: atlasbot
- name: BOT_PASS
valueFrom:
secretKeyRef:
name: atlasbot-credentials-runtime
key: bot-password
command:
- /bin/sh
- -c
- |
set -euo pipefail
. /vault/scripts/comms_vault_env.sh
pip install --no-cache-dir requests >/dev/null
python - <<'PY'
import base64

25
services/comms/othrys-kick-numeric-job.yaml
View File

@ -9,6 +9,7 @@ spec:
template:
spec:
restartPolicy: Never
serviceAccountName: comms-vault
containers:
- name: kick
image: python:3.11-slim
@ -23,16 +24,12 @@ spec:
value: "#othrys:live.bstein.dev"
- name: SEEDER_USER
value: othrys-seeder
- name: SEEDER_PASS
valueFrom:
secretKeyRef:
name: atlasbot-credentials-runtime
key: seeder-password
command:
- /bin/sh
- -c
- |
set -euo pipefail
. /vault/scripts/comms_vault_env.sh
pip install --no-cache-dir requests >/dev/null
python - <<'PY'
import os
@ -113,3 +110,21 @@ spec:
if is_numeric(user_id):
kick(token, room_id, user_id)
PY
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault
- name: vault-scripts
configMap:
name: comms-vault-env
defaultMode: 0555

25
services/comms/pin-othrys-job.yaml
View File

@ -16,6 +16,7 @@ spec:
template:
spec:
restartPolicy: Never
serviceAccountName: comms-vault
containers:
- name: pin
image: python:3.11-slim
@ -26,16 +27,12 @@ spec:
value: http://matrix-authentication-service:8080
- name: SEEDER_USER
value: othrys-seeder
- name: SEEDER_PASS
valueFrom:
secretKeyRef:
name: atlasbot-credentials-runtime
key: seeder-password
command:
- /bin/sh
- -c
- |
set -euo pipefail
. /vault/scripts/comms_vault_env.sh
pip install --no-cache-dir requests >/dev/null
python - <<'PY'
import os, requests, urllib.parse
@ -121,3 +118,21 @@ spec:
eid = send(room_id, token, MESSAGE)
pin(room_id, token, eid)
PY
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault
- name: vault-scripts
configMap:
name: comms-vault-env
defaultMode: 0555

25
services/comms/reset-othrys-room-job.yaml
View File

@ -16,6 +16,7 @@ spec:
template:
spec:
restartPolicy: Never
serviceAccountName: comms-vault
containers:
- name: reset
image: python:3.11-slim
@ -34,11 +35,6 @@ spec:
value: "Invite guests: share https://live.bstein.dev/#/room/#othrys:live.bstein.dev?action=join and choose 'Continue' -> 'Join as guest'."
- name: SEEDER_USER
value: othrys-seeder
- name: SEEDER_PASS
valueFrom:
secretKeyRef:
name: atlasbot-credentials-runtime
key: seeder-password
- name: BOT_USER
value: atlasbot
command:
@ -46,6 +42,7 @@ spec:
- -c
- |
set -euo pipefail
. /vault/scripts/comms_vault_env.sh
pip install --no-cache-dir requests >/dev/null
python - <<'PY'
import os
@ -264,3 +261,21 @@ spec:
print(f"old_room_id={old_room_id}")
print(f"new_room_id={new_room_id}")
PY
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault
- name: vault-scripts
configMap:
name: comms-vault-env
defaultMode: 0555

27
services/comms/scripts/comms_vault_env.sh Normal file
View File

@ -0,0 +1,27 @@
#!/usr/bin/env sh
set -eu
vault_dir="/vault/secrets"
read_secret() {
cat "${vault_dir}/$1"
}
export TURN_STATIC_AUTH_SECRET="$(read_secret turn-shared-secret__TURN_STATIC_AUTH_SECRET)"
export TURN_PASSWORD="${TURN_STATIC_AUTH_SECRET}"
export LIVEKIT_API_SECRET="$(read_secret livekit-api__primary)"
export LIVEKIT_SECRET="${LIVEKIT_API_SECRET}"
export BOT_PASS="$(read_secret atlasbot-credentials-runtime__bot-password)"
export SEEDER_PASS="$(read_secret atlasbot-credentials-runtime__seeder-password)"
export CHAT_API_KEY="$(read_secret chat-ai-keys-runtime__matrix)"
export CHAT_API_HOMEPAGE="$(read_secret chat-ai-keys-runtime__homepage)"
export MAS_ADMIN_CLIENT_SECRET_FILE="${vault_dir}/mas-admin-client-runtime__client_secret"
export PGPASSWORD="$(read_secret synapse-db__POSTGRES_PASSWORD)"
export MAS_DB_PASSWORD="$(read_secret mas-db__password)"
export MATRIX_SHARED_SECRET="$(read_secret mas-secrets-runtime__matrix_shared_secret)"
export KEYCLOAK_CLIENT_SECRET="$(read_secret mas-secrets-runtime__keycloak_client_secret)"

134
services/comms/secretproviderclass.yaml Normal file
View File

@ -0,0 +1,134 @@
# services/comms/secretproviderclass.yaml
apiVersion: secrets-store.csi.x-k8s.io/v1
kind: SecretProviderClass
metadata:
name: comms-vault
namespace: comms
spec:
provider: vault
parameters:
vaultAddress: "http://vault.vault.svc.cluster.local:8200"
roleName: "comms"
objects: |
- objectName: "turn-shared-secret__TURN_STATIC_AUTH_SECRET"
secretPath: "kv/data/atlas/comms/turn-shared-secret"
secretKey: "TURN_STATIC_AUTH_SECRET"
- objectName: "livekit-api__primary"
secretPath: "kv/data/atlas/comms/livekit-api"
secretKey: "primary"
- objectName: "synapse-db__POSTGRES_PASSWORD"
secretPath: "kv/data/atlas/comms/synapse-db"
secretKey: "POSTGRES_PASSWORD"
- objectName: "synapse-redis__redis-password"
secretPath: "kv/data/atlas/comms/synapse-redis"
secretKey: "redis-password"
- objectName: "synapse-macaroon__macaroon_secret_key"
secretPath: "kv/data/atlas/comms/synapse-macaroon"
secretKey: "macaroon_secret_key"
- objectName: "atlasbot-credentials-runtime__bot-password"
secretPath: "kv/data/atlas/comms/atlasbot-credentials-runtime"
secretKey: "bot-password"
- objectName: "atlasbot-credentials-runtime__seeder-password"
secretPath: "kv/data/atlas/comms/atlasbot-credentials-runtime"
secretKey: "seeder-password"
- objectName: "chat-ai-keys-runtime__matrix"
secretPath: "kv/data/atlas/shared/chat-ai-keys-runtime"
secretKey: "matrix"
- objectName: "chat-ai-keys-runtime__homepage"
secretPath: "kv/data/atlas/shared/chat-ai-keys-runtime"
secretKey: "homepage"
- objectName: "mas-admin-client-runtime__client_secret"
secretPath: "kv/data/atlas/comms/mas-admin-client-runtime"
secretKey: "client_secret"
- objectName: "mas-db__password"
secretPath: "kv/data/atlas/comms/mas-db"
secretKey: "password"
- objectName: "mas-secrets-runtime__encryption"
secretPath: "kv/data/atlas/comms/mas-secrets-runtime"
secretKey: "encryption"
- objectName: "mas-secrets-runtime__matrix_shared_secret"
secretPath: "kv/data/atlas/comms/mas-secrets-runtime"
secretKey: "matrix_shared_secret"
- objectName: "mas-secrets-runtime__keycloak_client_secret"
secretPath: "kv/data/atlas/comms/mas-secrets-runtime"
secretKey: "keycloak_client_secret"
- objectName: "mas-secrets-runtime__rsa_key"
secretPath: "kv/data/atlas/comms/mas-secrets-runtime"
secretKey: "rsa_key"
- objectName: "othrys-synapse-signingkey__signing.key"
secretPath: "kv/data/atlas/comms/othrys-synapse-signingkey"
secretKey: "signing.key"
- objectName: "synapse-oidc__client-secret"
secretPath: "kv/data/atlas/comms/synapse-oidc"
secretKey: "client-secret"
secretObjects:
- secretName: turn-shared-secret
type: Opaque
data:
- objectName: turn-shared-secret__TURN_STATIC_AUTH_SECRET
key: TURN_STATIC_AUTH_SECRET
- secretName: livekit-api
type: Opaque
data:
- objectName: livekit-api__primary
key: primary
- secretName: synapse-db
type: Opaque
data:
- objectName: synapse-db__POSTGRES_PASSWORD
key: POSTGRES_PASSWORD
- secretName: synapse-redis
type: Opaque
data:
- objectName: synapse-redis__redis-password
key: redis-password
- secretName: synapse-macaroon
type: Opaque
data:
- objectName: synapse-macaroon__macaroon_secret_key
key: macaroon_secret_key
- secretName: atlasbot-credentials-runtime
type: Opaque
data:
- objectName: atlasbot-credentials-runtime__bot-password
key: bot-password
- objectName: atlasbot-credentials-runtime__seeder-password
key: seeder-password
- secretName: chat-ai-keys-runtime
type: Opaque
data:
- objectName: chat-ai-keys-runtime__matrix
key: matrix
- objectName: chat-ai-keys-runtime__homepage
key: homepage
- secretName: mas-admin-client-runtime
type: Opaque
data:
- objectName: mas-admin-client-runtime__client_secret
key: client_secret
- secretName: mas-db
type: Opaque
data:
- objectName: mas-db__password
key: password
- secretName: mas-secrets-runtime
type: Opaque
data:
- objectName: mas-secrets-runtime__encryption
key: encryption
- objectName: mas-secrets-runtime__matrix_shared_secret
key: matrix_shared_secret
- objectName: mas-secrets-runtime__keycloak_client_secret
key: keycloak_client_secret
- objectName: mas-secrets-runtime__rsa_key
key: rsa_key
- secretName: othrys-synapse-signingkey
type: Opaque
data:
- objectName: othrys-synapse-signingkey__signing.key
key: signing.key
- secretName: synapse-oidc
type: Opaque
data:
- objectName: synapse-oidc__client-secret
key: client-secret

28
services/comms/seed-othrys-room.yaml
View File

@ -14,6 +14,7 @@ spec:
template:
spec:
restartPolicy: Never
serviceAccountName: comms-vault
containers:
- name: seed
image: python:3.11-slim
@ -24,23 +25,14 @@ spec:
value: http://matrix-authentication-service:8080
- name: SEEDER_USER
value: othrys-seeder
- name: SEEDER_PASS
valueFrom:
secretKeyRef:
name: atlasbot-credentials-runtime
key: seeder-password
- name: BOT_USER
value: atlasbot
- name: BOT_PASS
valueFrom:
secretKeyRef:
name: atlasbot-credentials-runtime
key: bot-password
command:
- /bin/sh
- -c
- |
set -euo pipefail
. /vault/scripts/comms_vault_env.sh
pip install --no-cache-dir requests pyyaml >/dev/null
python - <<'PY'
import os, requests, urllib.parse
@ -140,7 +132,23 @@ spec:
- name: synapse-config
mountPath: /config
readOnly: true
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: synapse-config
secret:
secretName: othrys-synapse-matrix-synapse
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault
- name: vault-scripts
configMap:
name: comms-vault-env
defaultMode: 0555

6
services/comms/serviceaccount.yaml Normal file
View File

@ -0,0 +1,6 @@
# services/comms/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: comms-vault
namespace: comms

25
services/comms/synapse-seeder-admin-ensure-job.yaml
View File

@ -9,6 +9,7 @@ spec:
template:
spec:
restartPolicy: OnFailure
serviceAccountName: comms-vault
containers:
- name: psql
image: postgres:16-alpine
@ -21,16 +22,30 @@ spec:
value: synapse
- name: PGUSER
value: synapse
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: synapse-db
key: POSTGRES_PASSWORD
command:
- /bin/sh
- -c
- |
set -euo pipefail
. /vault/scripts/comms_vault_env.sh
psql -v ON_ERROR_STOP=1 <<'SQL'
UPDATE users SET admin = 1 WHERE name = '@othrys-seeder:live.bstein.dev';
SQL
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault
- name: vault-scripts
configMap:
name: comms-vault-env
defaultMode: 0555

28
services/comms/synapse-signingkey-ensure-job.yaml
View File

@ -37,15 +37,29 @@ spec:
args:
- |
set -euo pipefail
set -x
if kubectl -n comms get secret othrys-synapse-signingkey \
-o jsonpath='{.data.signing\.key}' 2>/tmp/get_err | grep -q .; then
apk add --no-cache curl jq >/dev/null
vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
vault_role="${VAULT_ROLE:-comms-secrets}"
jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
vault_token="$(curl -sS --request POST --data "${login_payload}" \
"${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
echo "vault login failed" >&2
exit 1
fi
existing="$(curl -sS -H "X-Vault-Token: ${vault_token}" \
"${vault_addr}/v1/kv/data/atlas/comms/othrys-synapse-signingkey" | jq -r '.data.data["signing.key"] // empty')"
if [ -n "${existing}" ]; then
exit 0
fi
cat /tmp/get_err >&2 || true
kubectl -n comms create secret generic othrys-synapse-signingkey \
--from-file=signing.key=/work/signing.key \
--dry-run=client -o yaml | kubectl -n comms apply -f - >/dev/null
value="$(cat /work/signing.key)"
payload="$(jq -nc --arg value "${value}" '{data:{"signing.key":$value}}')"
curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
-d "${payload}" "${vault_addr}/v1/kv/data/atlas/comms/othrys-synapse-signingkey" >/dev/null
volumeMounts:
- name: work
mountPath: /work

35
services/comms/synapse-user-seed-job.yaml
View File

@ -10,6 +10,7 @@ spec:
template:
spec:
restartPolicy: Never
serviceAccountName: comms-vault
containers:
- name: seed
image: python:3.11-slim
@ -22,30 +23,16 @@ spec:
value: synapse
- name: PGUSER
value: synapse
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: synapse-db
key: POSTGRES_PASSWORD
- name: SEEDER_USER
value: othrys-seeder
- name: SEEDER_PASS
valueFrom:
secretKeyRef:
name: atlasbot-credentials-runtime
key: seeder-password
- name: BOT_USER
value: atlasbot
- name: BOT_PASS
valueFrom:
secretKeyRef:
name: atlasbot-credentials-runtime
key: bot-password
command:
- /bin/sh
- -c
- |
set -euo pipefail
. /vault/scripts/comms_vault_env.sh
pip install --no-cache-dir psycopg2-binary bcrypt >/dev/null
python - <<'PY'
import os
@ -118,3 +105,21 @@ spec:
finally:
conn.close()
PY
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault
- name: vault-scripts
configMap:
name: comms-vault-env
defaultMode: 0555

34
services/comms/vault-sync-deployment.yaml Normal file
View File

@ -0,0 +1,34 @@
# services/comms/vault-sync-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: comms-vault-sync
namespace: comms
spec:
replicas: 1
selector:
matchLabels:
app: comms-vault-sync
template:
metadata:
labels:
app: comms-vault-sync
spec:
serviceAccountName: comms-vault
containers:
- name: sync
image: alpine:3.20
command: ["/bin/sh", "-c"]
args:
- "sleep infinity"
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: comms-vault

3
services/harbor/kustomization.yaml
View File

@ -4,7 +4,10 @@ kind: Kustomization
namespace: harbor
resources:
- namespace.yaml
- serviceaccount.yaml
- secretproviderclass.yaml
- pvc.yaml
- certificate.yaml
- helmrelease.yaml
- vault-sync-deployment.yaml
- image.yaml

87
services/harbor/secretproviderclass.yaml Normal file
View File

@ -0,0 +1,87 @@
# services/harbor/secretproviderclass.yaml
apiVersion: secrets-store.csi.x-k8s.io/v1
kind: SecretProviderClass
metadata:
name: harbor-vault
namespace: harbor
spec:
provider: vault
parameters:
vaultAddress: "http://vault.vault.svc.cluster.local:8200"
roleName: "harbor"
objects: |
- objectName: "harbor-core__CSRF_KEY"
secretPath: "kv/data/atlas/harbor/harbor-core"
secretKey: "CSRF_KEY"
- objectName: "harbor-core__REGISTRY_CREDENTIAL_PASSWORD"
secretPath: "kv/data/atlas/harbor/harbor-core"
secretKey: "REGISTRY_CREDENTIAL_PASSWORD"
- objectName: "harbor-core__harbor_admin_password"
secretPath: "kv/data/atlas/harbor/harbor-core"
secretKey: "harbor_admin_password"
- objectName: "harbor-core__secret"
secretPath: "kv/data/atlas/harbor/harbor-core"
secretKey: "secret"
- objectName: "harbor-core__secretKey"
secretPath: "kv/data/atlas/harbor/harbor-core"
secretKey: "secretKey"
- objectName: "harbor-core__tls.crt"
secretPath: "kv/data/atlas/harbor/harbor-core"
secretKey: "tls.crt"
- objectName: "harbor-core__tls.key"
secretPath: "kv/data/atlas/harbor/harbor-core"
secretKey: "tls.key"
- objectName: "harbor-db__database"
secretPath: "kv/data/atlas/harbor/harbor-db"
secretKey: "database"
- objectName: "harbor-db__host"
secretPath: "kv/data/atlas/harbor/harbor-db"
secretKey: "host"
- objectName: "harbor-db__password"
secretPath: "kv/data/atlas/harbor/harbor-db"
secretKey: "password"
- objectName: "harbor-db__port"
secretPath: "kv/data/atlas/harbor/harbor-db"
secretKey: "port"
- objectName: "harbor-db__username"
secretPath: "kv/data/atlas/harbor/harbor-db"
secretKey: "username"
- objectName: "harbor-oidc__CONFIG_OVERWRITE_JSON"
secretPath: "kv/data/atlas/harbor/harbor-oidc"
secretKey: "CONFIG_OVERWRITE_JSON"
secretObjects:
- secretName: harbor-core
type: Opaque
data:
- objectName: harbor-core__CSRF_KEY
key: CSRF_KEY
- objectName: harbor-core__REGISTRY_CREDENTIAL_PASSWORD
key: REGISTRY_CREDENTIAL_PASSWORD
- objectName: harbor-core__harbor_admin_password
key: harbor_admin_password
- objectName: harbor-core__secret
key: secret
- objectName: harbor-core__secretKey
key: secretKey
- objectName: harbor-core__tls.crt
key: tls.crt
- objectName: harbor-core__tls.key
key: tls.key
- secretName: harbor-db
type: Opaque
data:
- objectName: harbor-db__database
key: database
- objectName: harbor-db__host
key: host
- objectName: harbor-db__password
key: password
- objectName: harbor-db__port
key: port
- objectName: harbor-db__username
key: username
- secretName: harbor-oidc
type: Opaque
data:
- objectName: harbor-oidc__CONFIG_OVERWRITE_JSON
key: CONFIG_OVERWRITE_JSON

6
services/harbor/serviceaccount.yaml Normal file
View File

@ -0,0 +1,6 @@
# services/harbor/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: harbor-vault-sync
namespace: harbor

34
services/harbor/vault-sync-deployment.yaml Normal file
View File

@ -0,0 +1,34 @@
# services/harbor/vault-sync-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: harbor-vault-sync
namespace: harbor
spec:
replicas: 1
selector:
matchLabels:
app: harbor-vault-sync
template:
metadata:
labels:
app: harbor-vault-sync
spec:
serviceAccountName: harbor-vault-sync
containers:
- name: sync
image: alpine:3.20
command: ["/bin/sh", "-c"]
args:
- "sleep infinity"
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: harbor-vault

27
services/keycloak/harbor-oidc-secret-ensure-job.yaml
View File

@ -16,6 +16,16 @@ spec:
configMap:
name: harbor-oidc-secret-ensure-script
defaultMode: 0555
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: sso-vault
- name: vault-scripts
configMap:
name: sso-vault-env
defaultMode: 0555
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
@ -30,18 +40,13 @@ spec:
- name: apply
image: alpine:3.20
command: ["/scripts/harbor_oidc_secret_ensure.sh"]
env:
- name: KEYCLOAK_ADMIN
valueFrom:
secretKeyRef:
name: keycloak-admin
key: username
- name: KEYCLOAK_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: keycloak-admin
key: password
volumeMounts:
- name: harbor-oidc-secret-ensure-script
mountPath: /scripts
readOnly: true
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true

35
services/keycloak/ldap-federation-job.yaml
View File

@ -19,6 +19,7 @@ spec:
- key: node-role.kubernetes.io/worker
operator: Exists
restartPolicy: OnFailure
serviceAccountName: sso-vault
containers:
- name: configure
image: python:3.11-alpine
@ -28,25 +29,10 @@ spec:
value: http://keycloak.sso.svc.cluster.local
- name: KEYCLOAK_REALM
value: atlas
- name: KEYCLOAK_ADMIN_USER
valueFrom:
secretKeyRef:
name: keycloak-admin
key: username
- name: KEYCLOAK_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: keycloak-admin
key: password
- name: LDAP_URL
value: ldap://openldap.sso.svc.cluster.local:389
- name: LDAP_BIND_DN
value: cn=admin,dc=bstein,dc=dev
- name: LDAP_BIND_PASSWORD
valueFrom:
secretKeyRef:
name: openldap-admin
key: LDAP_ADMIN_PASSWORD
- name: LDAP_USERS_DN
value: ou=users,dc=bstein,dc=dev
- name: LDAP_GROUPS_DN
@ -55,6 +41,7 @@ spec:
args:
- |
set -euo pipefail
. /vault/scripts/keycloak_vault_env.sh
python - <<'PY'
import json
import os
@ -360,3 +347,21 @@ spec:
except Exception as e:
print(f"WARNING: LDAP cleanup failed (continuing): {e}")
PY
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: sso-vault
- name: vault-scripts
configMap:
name: sso-vault-env
defaultMode: 0555

66
services/keycloak/logs-oidc-secret-ensure-job.yaml
View File

@ -18,6 +18,7 @@ spec:
args:
- |
set -euo pipefail
. /vault/scripts/keycloak_vault_env.sh
apk add --no-cache curl jq kubectl openssl >/dev/null
KC_URL="http://keycloak.sso.svc.cluster.local"
@ -73,31 +74,56 @@ spec:
exit 1
fi
if kubectl -n logging get secret oauth2-proxy-logs-oidc >/dev/null 2>&1; then
current_cookie="$(kubectl -n logging get secret oauth2-proxy-logs-oidc -o jsonpath='{.data.cookie_secret}' 2>/dev/null || true)"
if [ -n "${current_cookie}" ]; then
decoded="$(printf '%s' "${current_cookie}" | base64 -d 2>/dev/null || true)"
length="$(printf '%s' "${decoded}" | wc -c | tr -d ' ')"
if [ "${length}" = "16" ] || [ "${length}" = "24" ] || [ "${length}" = "32" ]; then
exit 0
fi
fi
vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
vault_role="${VAULT_ROLE:-sso-secrets}"
jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
vault_token="$(curl -sS --request POST --data "${login_payload}" \
"${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
echo "vault login failed" >&2
exit 1
fi
COOKIE_SECRET="$(curl -sS -H "X-Vault-Token: ${vault_token}" \
"${vault_addr}/v1/kv/data/atlas/logging/oauth2-proxy-logs-oidc" | jq -r '.data.data.cookie_secret // empty')"
if [ -n "${COOKIE_SECRET}" ]; then
length="$(printf '%s' "${COOKIE_SECRET}" | wc -c | tr -d ' ')"
if [ "${length}" != "16" ] && [ "${length}" != "24" ] && [ "${length}" != "32" ]; then
COOKIE_SECRET=""
fi
fi
if [ -z "${COOKIE_SECRET}" ]; then
COOKIE_SECRET="$(openssl rand -hex 16 | tr -d '\n')"
fi
payload="$(jq -nc \
--arg client_id "logs" \
--arg client_secret "${CLIENT_SECRET}" \
--arg cookie_secret "${COOKIE_SECRET}" \
'{data:{client_id:$client_id,client_secret:$client_secret,cookie_secret:$cookie_secret}}')"
curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
-d "${payload}" "${vault_addr}/v1/kv/data/atlas/logging/oauth2-proxy-logs-oidc" >/dev/null
kubectl -n logging create secret generic oauth2-proxy-logs-oidc \
--from-literal=client_id="logs" \
--from-literal=client_secret="${CLIENT_SECRET}" \
--from-literal=cookie_secret="${COOKIE_SECRET}" \
--dry-run=client -o yaml | kubectl -n logging apply -f - >/dev/null
env:
- name: KEYCLOAK_ADMIN
valueFrom:
secretKeyRef:
name: keycloak-admin
key: username
- name: KEYCLOAK_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: keycloak-admin
key: password
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: sso-vault
- name: vault-scripts
configMap:
name: sso-vault-env
defaultMode: 0555

65
services/keycloak/mas-secrets-ensure-job.yaml
View File

@ -20,6 +20,16 @@ spec:
volumes:
- name: work
emptyDir: {}
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: sso-vault
- name: vault-scripts
configMap:
name: sso-vault-env
defaultMode: 0555
initContainers:
- name: generate
image: alpine:3.20
@ -27,6 +37,7 @@ spec:
args:
- |
set -euo pipefail
. /vault/scripts/keycloak_vault_env.sh
umask 077
apk add --no-cache curl openssl jq >/dev/null
@ -68,20 +79,15 @@ spec:
openssl rand -hex 32 | tr -d '\n' > /work/matrix_shared_secret
openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:4096 -out /work/rsa_key >/dev/null 2>&1
chmod 0644 /work/*
env:
- name: KEYCLOAK_ADMIN
valueFrom:
secretKeyRef:
name: keycloak-admin
key: username
- name: KEYCLOAK_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: keycloak-admin
key: password
volumeMounts:
- name: work
mountPath: /work
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
containers:
- name: apply
image: registry.bstein.dev/bstein/kubectl:1.35.0
@ -89,19 +95,36 @@ spec:
args:
- |
set -euo pipefail
if kubectl -n comms get secret mas-secrets-runtime >/dev/null 2>&1; then
kubectl -n comms get secret mas-secrets-runtime -o jsonpath='{.data.encryption}' | base64 -d 2>/dev/null > /tmp/encryption.current || true
current_len="$(wc -c < /tmp/encryption.current | tr -d ' ')"
if [ "${current_len}" = "64" ] && grep -Eq '^[0-9a-fA-F]{64}$' /tmp/encryption.current; then
apk add --no-cache curl jq >/dev/null
vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
vault_role="${VAULT_ROLE:-sso-secrets}"
jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
vault_token="$(curl -sS --request POST --data "${login_payload}" \
"${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
echo "vault login failed" >&2
exit 1
fi
existing="$(curl -sS -H "X-Vault-Token: ${vault_token}" \
"${vault_addr}/v1/kv/data/atlas/comms/mas-secrets-runtime" | jq -r '.data.data.encryption // empty')"
if [ -n "${existing}" ]; then
current_len="$(printf '%s' "${existing}" | wc -c | tr -d ' ')"
if [ "${current_len}" = "64" ] && printf '%s' "${existing}" | grep -Eq '^[0-9a-fA-F]{64}$'; then
exit 0
fi
fi
kubectl -n comms create secret generic mas-secrets-runtime \
--from-file=encryption=/work/encryption \
--from-file=matrix_shared_secret=/work/matrix_shared_secret \
--from-file=keycloak_client_secret=/work/keycloak_client_secret \
--from-file=rsa_key=/work/rsa_key \
--dry-run=client -o yaml | kubectl -n comms apply -f - >/dev/null
payload="$(jq -nc \
--arg encryption "$(cat /work/encryption)" \
--arg matrix_shared_secret "$(cat /work/matrix_shared_secret)" \
--arg keycloak_client_secret "$(cat /work/keycloak_client_secret)" \
--arg rsa_key "$(cat /work/rsa_key)" \
'{data:{encryption:$encryption, matrix_shared_secret:$matrix_shared_secret, keycloak_client_secret:$keycloak_client_secret, rsa_key:$rsa_key}}')"
curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
-d "${payload}" "${vault_addr}/v1/kv/data/atlas/comms/mas-secrets-runtime" >/dev/null
volumeMounts:
- name: work
mountPath: /work

40
services/keycloak/portal-e2e-client-job.yaml
View File

@ -9,6 +9,7 @@ spec:
template:
spec:
restartPolicy: Never
serviceAccountName: sso-vault
containers:
- name: configure
image: python:3.11-alpine
@ -17,30 +18,11 @@ spec:
value: http://keycloak.sso.svc.cluster.local
- name: KEYCLOAK_REALM
value: atlas
- name: KEYCLOAK_ADMIN_USER
valueFrom:
secretKeyRef:
name: keycloak-admin
key: username
- name: KEYCLOAK_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: keycloak-admin
key: password
- name: PORTAL_E2E_CLIENT_ID
valueFrom:
secretKeyRef:
name: portal-e2e-client
key: client_id
- name: PORTAL_E2E_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: portal-e2e-client
key: client_secret
command: ["/bin/sh", "-c"]
args:
- |
set -euo pipefail
. /vault/scripts/keycloak_vault_env.sh
python - <<'PY'
import json
import os
@ -245,3 +227,21 @@ spec:
if status not in (200, 204):
raise SystemExit(f"Role mapping update failed (status={status}) resp={resp}")
PY
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: sso-vault
- name: vault-scripts
configMap:
name: sso-vault-env
defaultMode: 0555

28
services/keycloak/portal-e2e-execute-actions-email-test-job.yaml
View File

@ -9,6 +9,7 @@ spec:
template:
spec:
restartPolicy: Never
serviceAccountName: sso-vault
containers:
- name: test
image: python:3.11-alpine
@ -17,16 +18,6 @@ spec:
value: http://keycloak.sso.svc.cluster.local
- name: KEYCLOAK_REALM
value: atlas
- name: PORTAL_E2E_CLIENT_ID
valueFrom:
secretKeyRef:
name: portal-e2e-client
key: client_id
- name: PORTAL_E2E_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: portal-e2e-client
key: client_secret
- name: E2E_PROBE_USERNAME
value: e2e-smtp-probe
- name: E2E_PROBE_EMAIL
@ -39,13 +30,30 @@ spec:
args:
- |
set -euo pipefail
. /vault/scripts/keycloak_vault_env.sh
python /scripts/test_keycloak_execute_actions_email.py
volumeMounts:
- name: tests
mountPath: /scripts
readOnly: true
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: tests
configMap:
name: portal-e2e-tests
defaultMode: 0555
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: sso-vault
- name: vault-scripts
configMap:
name: sso-vault-env
defaultMode: 0555

30
services/keycloak/portal-e2e-target-client-job.yaml
View File

@ -9,6 +9,7 @@ spec:
template:
spec:
restartPolicy: Never
serviceAccountName: sso-vault
containers:
- name: configure
image: python:3.11-alpine
@ -17,22 +18,13 @@ spec:
value: http://keycloak.sso.svc.cluster.local
- name: KEYCLOAK_REALM
value: atlas
- name: KEYCLOAK_ADMIN_USER
valueFrom:
secretKeyRef:
name: keycloak-admin
key: username
- name: KEYCLOAK_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: keycloak-admin
key: password
- name: TARGET_CLIENT_ID
value: bstein-dev-home
command: ["/bin/sh", "-c"]
args:
- |
set -euo pipefail
. /vault/scripts/keycloak_vault_env.sh
python - <<'PY'
import json
import os
@ -136,3 +128,21 @@ spec:
print(f"OK: ensured token exchange enabled on client {target_client_id}")
PY
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: sso-vault
- name: vault-scripts
configMap:
name: sso-vault-env
defaultMode: 0555

30
services/keycloak/portal-e2e-token-exchange-permissions-job.yaml
View File

@ -9,6 +9,7 @@ spec:
template:
spec:
restartPolicy: Never
serviceAccountName: sso-vault
containers:
- name: configure
image: python:3.11-alpine
@ -17,16 +18,6 @@ spec:
value: http://keycloak.sso.svc.cluster.local
- name: KEYCLOAK_REALM
value: atlas
- name: KEYCLOAK_ADMIN_USER
valueFrom:
secretKeyRef:
name: keycloak-admin
key: username
- name: KEYCLOAK_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: keycloak-admin
key: password
- name: PORTAL_E2E_CLIENT_ID
value: test-portal-e2e
- name: TARGET_CLIENT_ID
@ -35,6 +26,7 @@ spec:
args:
- |
set -euo pipefail
. /vault/scripts/keycloak_vault_env.sh
python - <<'PY'
import json
import os
@ -269,3 +261,21 @@ spec:
print("OK: configured token exchange permissions for portal E2E client")
PY
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: sso-vault
- name: vault-scripts
configMap:
name: sso-vault-env
defaultMode: 0555

28
services/keycloak/portal-e2e-token-exchange-test-job.yaml
View File

@ -10,6 +10,7 @@ spec:
template:
spec:
restartPolicy: Never
serviceAccountName: sso-vault
containers:
- name: test
image: python:3.11-alpine
@ -26,27 +27,34 @@ spec:
value: "300"
- name: RETRY_INTERVAL_SECONDS
value: "5"
- name: PORTAL_E2E_CLIENT_ID
valueFrom:
secretKeyRef:
name: portal-e2e-client
key: client_id
- name: PORTAL_E2E_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: portal-e2e-client
key: client_secret
command: ["/bin/sh", "-c"]
args:
- |
set -euo pipefail
. /vault/scripts/keycloak_vault_env.sh
python /scripts/test_portal_token_exchange.py
volumeMounts:
- name: tests
mountPath: /scripts
readOnly: true
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: tests
configMap:
name: portal-e2e-tests
defaultMode: 0555
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: sso-vault
- name: vault-scripts
configMap:
name: sso-vault-env
defaultMode: 0555

30
services/keycloak/realm-settings-job.yaml
View File

@ -19,6 +19,7 @@ spec:
- key: node-role.kubernetes.io/worker
operator: Exists
restartPolicy: Never
serviceAccountName: sso-vault
containers:
- name: configure
image: python:3.11-alpine
@ -27,16 +28,6 @@ spec:
value: http://keycloak.sso.svc.cluster.local
- name: KEYCLOAK_REALM
value: atlas
- name: KEYCLOAK_ADMIN_USER
valueFrom:
secretKeyRef:
name: keycloak-admin
key: username
- name: KEYCLOAK_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: keycloak-admin
key: password
- name: KEYCLOAK_SMTP_HOST
value: mailu-front.mailu-mailserver.svc.cluster.local
- name: KEYCLOAK_SMTP_PORT
@ -53,6 +44,7 @@ spec:
args:
- |
set -euo pipefail
. /vault/scripts/keycloak_vault_env.sh
python - <<'PY'
import json
import os
@ -444,3 +436,21 @@ spec:
f"Unexpected execution update response for identity-provider-redirector: {status}"
)
PY
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: sso-vault
- name: vault-scripts
configMap:
name: sso-vault-env
defaultMode: 0555

19
services/keycloak/scripts/harbor_oidc_secret_ensure.sh
View File

@ -3,6 +3,8 @@ set -euo pipefail
apk add --no-cache curl jq kubectl >/dev/null
. /vault/scripts/keycloak_vault_env.sh
KC_URL="http://keycloak.sso.svc.cluster.local"
ACCESS_TOKEN=""
for attempt in 1 2 3 4 5; do
@ -99,6 +101,17 @@ CONFIG_OVERWRITE_JSON="$(jq -nc \
--argjson oidc_logout true \
'{auth_mode:$auth_mode,oidc_name:$oidc_name,oidc_client_id:$oidc_client_id,oidc_client_secret:$oidc_client_secret,oidc_endpoint:$oidc_endpoint,oidc_scope:$oidc_scope,oidc_user_claim:$oidc_user_claim,oidc_groups_claim:$oidc_groups_claim,oidc_admin_group:$oidc_admin_group,oidc_auto_onboard:$oidc_auto_onboard,oidc_verify_cert:$oidc_verify_cert,oidc_logout:$oidc_logout}')"
kubectl -n harbor create secret generic harbor-oidc \
--from-literal=CONFIG_OVERWRITE_JSON="${CONFIG_OVERWRITE_JSON}" \
--dry-run=client -o yaml | kubectl -n harbor apply -f - >/dev/null
vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
vault_role="${VAULT_ROLE:-sso-secrets}"
jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
vault_token="$(curl -sS --request POST --data "${login_payload}" \
"${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
echo "vault login failed" >&2
exit 1
fi
payload="$(jq -nc --arg value "${CONFIG_OVERWRITE_JSON}" '{data:{CONFIG_OVERWRITE_JSON:$value}}')"
curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
-d "${payload}" "${vault_addr}/v1/kv/data/atlas/harbor/harbor-oidc" >/dev/null

1
services/keycloak/scripts/keycloak_vault_env.sh
View File

@ -23,3 +23,4 @@ export PORTAL_E2E_CLIENT_SECRET="$(read_secret portal-e2e-client__client_secret)
export LDAP_ADMIN_PASSWORD="$(read_secret openldap-admin__LDAP_ADMIN_PASSWORD)"
export LDAP_CONFIG_PASSWORD="$(read_secret openldap-admin__LDAP_CONFIG_PASSWORD)"
export LDAP_BIND_PASSWORD="${LDAP_ADMIN_PASSWORD}"

33
services/keycloak/scripts/vault_oidc_secret_ensure.sh
View File

@ -3,6 +3,8 @@ set -euo pipefail
apk add --no-cache curl jq kubectl >/dev/null
. /vault/scripts/keycloak_vault_env.sh
KC_URL="http://keycloak.sso.svc.cluster.local"
ACCESS_TOKEN=""
for attempt in 1 2 3 4 5; do
@ -84,6 +86,37 @@ if [ -z "$CLIENT_SECRET" ] || [ "$CLIENT_SECRET" = "null" ]; then
exit 1
fi
vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
vault_role="${VAULT_ROLE:-sso-secrets}"
jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
vault_token="$(curl -sS --request POST --data "${login_payload}" \
"${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
echo "vault login failed" >&2
exit 1
fi
payload="$(jq -nc \
--arg discovery_url "https://sso.bstein.dev/realms/atlas" \
--arg client_id "vault-oidc" \
--arg client_secret "${CLIENT_SECRET}" \
--arg default_role "admin" \
--arg scopes "openid profile email groups" \
--arg user_claim "preferred_username" \
--arg groups_claim "groups" \
--arg redirect_uris "https://secret.bstein.dev/ui/vault/auth/oidc/oidc/callback,http://localhost:8250/oidc/callback" \
--arg bound_audiences "vault-oidc" \
--arg admin_group "admin" \
--arg admin_policies "default,vault-admin" \
--arg dev_group "dev" \
--arg dev_policies "default,dev-kv" \
--arg user_group "dev" \
--arg user_policies "default,dev-kv" \
'{data:{discovery_url:$discovery_url,client_id:$client_id,client_secret:$client_secret,default_role:$default_role,scopes:$scopes,user_claim:$user_claim,groups_claim:$groups_claim,redirect_uris:$redirect_uris,bound_audiences:$bound_audiences,admin_group:$admin_group,admin_policies:$admin_policies,dev_group:$dev_group,dev_policies:$dev_policies,user_group:$user_group,user_policies:$user_policies}}')"
curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
-d "${payload}" "${vault_addr}/v1/kv/data/atlas/vault/vault-oidc-config" >/dev/null
kubectl -n vault create secret generic vault-oidc-config \
--from-literal=discovery_url="https://sso.bstein.dev/realms/atlas" \
--from-literal=client_id="vault-oidc" \

50
services/keycloak/synapse-oidc-secret-ensure-job.yaml
View File

@ -18,7 +18,8 @@ spec:
args:
- |
set -euo pipefail
apk add --no-cache curl jq kubectl >/dev/null
. /vault/scripts/keycloak_vault_env.sh
apk add --no-cache curl jq >/dev/null
KC_URL="http://keycloak.sso.svc.cluster.local"
ACCESS_TOKEN=""
@ -54,22 +55,35 @@ spec:
exit 1
fi
existing="$(kubectl -n comms get secret synapse-oidc -o jsonpath='{.data.client-secret}' 2>/dev/null || true)"
if [ -n "${existing}" ]; then
exit 0
vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
vault_role="${VAULT_ROLE:-sso-secrets}"
jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
vault_token="$(curl -sS --request POST --data "${login_payload}" \
"${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
echo "vault login failed" >&2
exit 1
fi
kubectl -n comms create secret generic synapse-oidc \
--from-literal=client-secret="${CLIENT_SECRET}" \
--dry-run=client -o yaml | kubectl -n comms apply -f - >/dev/null
env:
- name: KEYCLOAK_ADMIN
valueFrom:
secretKeyRef:
name: keycloak-admin
key: username
- name: KEYCLOAK_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: keycloak-admin
key: password
payload="$(jq -nc --arg value "${CLIENT_SECRET}" '{data:{"client-secret":$value}}')"
curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
-d "${payload}" "${vault_addr}/v1/kv/data/atlas/comms/synapse-oidc" >/dev/null
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: sso-vault
- name: vault-scripts
configMap:
name: sso-vault-env
defaultMode: 0555

30
services/keycloak/user-overrides-job.yaml
View File

@ -19,6 +19,7 @@ spec:
- key: node-role.kubernetes.io/worker
operator: Exists
restartPolicy: Never
serviceAccountName: sso-vault
containers:
- name: configure
image: python:3.11-alpine
@ -27,16 +28,6 @@ spec:
value: http://keycloak.sso.svc.cluster.local
- name: KEYCLOAK_REALM
value: atlas
- name: KEYCLOAK_ADMIN_USER
valueFrom:
secretKeyRef:
name: keycloak-admin
key: username
- name: KEYCLOAK_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: keycloak-admin
key: password
- name: OVERRIDE_USERNAME
value: bstein
- name: OVERRIDE_MAILU_EMAIL
@ -45,6 +36,7 @@ spec:
args:
- |
set -euo pipefail
. /vault/scripts/keycloak_vault_env.sh
python - <<'PY'
import json
import os
@ -143,3 +135,21 @@ spec:
if status not in (200, 204):
raise SystemExit(f"Unexpected user update response: {status}")
PY
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: sso-vault
- name: vault-scripts
configMap:
name: sso-vault-env
defaultMode: 0555

27
services/keycloak/vault-oidc-secret-ensure-job.yaml
View File

@ -16,6 +16,16 @@ spec:
configMap:
name: vault-oidc-secret-ensure-script
defaultMode: 0555
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: sso-vault
- name: vault-scripts
configMap:
name: sso-vault-env
defaultMode: 0555
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
@ -30,18 +40,13 @@ spec:
- name: apply
image: alpine:3.20
command: ["/scripts/vault_oidc_secret_ensure.sh"]
env:
- name: KEYCLOAK_ADMIN
valueFrom:
secretKeyRef:
name: keycloak-admin
key: username
- name: KEYCLOAK_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: keycloak-admin
key: password
volumeMounts:
- name: vault-oidc-secret-ensure-script
mountPath: /scripts
readOnly: true
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true

9
services/mailu/kustomization.yaml
View File

@ -4,7 +4,10 @@ kind: Kustomization
namespace: mailu-mailserver
resources:
- namespace.yaml
- serviceaccount.yaml
- secretproviderclass.yaml
- helmrelease.yaml
- vault-sync-deployment.yaml
- certificate.yaml
- vip-controller.yaml
- unbound-configmap.yaml
@ -16,6 +19,12 @@ resources:
- front-lb.yaml
configMapGenerator:
- name: mailu-vault-env
namespace: mailu-mailserver
files:
- mailu_vault_env.sh=scripts/mailu_vault_env.sh
options:
disableNameSuffixHash: true
- name: mailu-sync-script
namespace: mailu-mailserver
files:

44
services/mailu/mailu-sync-cronjob.yaml
View File

@ -12,6 +12,7 @@ spec:
template:
spec:
restartPolicy: OnFailure
serviceAccountName: mailu-vault-sync
containers:
- name: mailu-sync
image: python:3.11-alpine
@ -19,6 +20,8 @@ spec:
command: ["/bin/sh", "-c"]
args:
- |
set -euo pipefail
. /vault/scripts/mailu_vault_env.sh
pip install --no-cache-dir requests psycopg2-binary passlib >/tmp/pip.log \
&& python /app/sync.py
env:
@ -34,35 +37,16 @@ spec:
value: postgres-service.postgres.svc.cluster.local
- name: MAILU_DB_PORT
value: "5432"
- name: MAILU_DB_NAME
valueFrom:
secretKeyRef:
name: mailu-db-secret
key: database
- name: MAILU_DB_USER
valueFrom:
secretKeyRef:
name: mailu-db-secret
key: username
- name: MAILU_DB_PASSWORD
valueFrom:
secretKeyRef:
name: mailu-db-secret
key: password
- name: KEYCLOAK_CLIENT_ID
valueFrom:
secretKeyRef:
name: mailu-sync-credentials
key: client-id
- name: KEYCLOAK_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: mailu-sync-credentials
key: client-secret
volumeMounts:
- name: sync-script
mountPath: /app/sync.py
subPath: sync.py
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
resources:
requests:
cpu: 50m
@ -75,3 +59,13 @@ spec:
configMap:
name: mailu-sync-script
defaultMode: 0444
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: mailu-vault
- name: vault-scripts
configMap:
name: mailu-vault-env
defaultMode: 0555

44
services/mailu/mailu-sync-job.yaml
View File

@ -8,6 +8,7 @@ spec:
template:
spec:
restartPolicy: OnFailure
serviceAccountName: mailu-vault-sync
containers:
- name: mailu-sync
image: python:3.11-alpine
@ -15,6 +16,8 @@ spec:
command: ["/bin/sh", "-c"]
args:
- |
set -euo pipefail
. /vault/scripts/mailu_vault_env.sh
pip install --no-cache-dir requests psycopg2-binary passlib >/tmp/pip.log \
&& python /app/sync.py
env:
@ -30,35 +33,16 @@ spec:
value: postgres-service.postgres.svc.cluster.local
- name: MAILU_DB_PORT
value: "5432"
- name: MAILU_DB_NAME
valueFrom:
secretKeyRef:
name: mailu-db-secret
key: database
- name: MAILU_DB_USER
valueFrom:
secretKeyRef:
name: mailu-db-secret
key: username
- name: MAILU_DB_PASSWORD
valueFrom:
secretKeyRef:
name: mailu-db-secret
key: password
- name: KEYCLOAK_CLIENT_ID
valueFrom:
secretKeyRef:
name: mailu-sync-credentials
key: client-id
- name: KEYCLOAK_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: mailu-sync-credentials
key: client-secret
volumeMounts:
- name: sync-script
mountPath: /app/sync.py
subPath: sync.py
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
resources:
requests:
cpu: 50m
@ -71,3 +55,13 @@ spec:
configMap:
name: mailu-sync-script
defaultMode: 0444
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: mailu-vault
- name: vault-scripts
configMap:
name: mailu-vault-env
defaultMode: 0555

44
services/mailu/mailu-sync-listener.yaml
View File

@ -30,6 +30,7 @@ spec:
app: mailu-sync-listener
spec:
restartPolicy: Always
serviceAccountName: mailu-vault-sync
containers:
- name: listener
image: python:3.11-alpine
@ -37,6 +38,8 @@ spec:
command: ["/bin/sh", "-c"]
args:
- |
set -euo pipefail
. /vault/scripts/mailu_vault_env.sh
pip install --no-cache-dir requests psycopg2-binary passlib >/tmp/pip.log \
&& python /app/listener.py
env:
@ -52,31 +55,6 @@ spec:
value: postgres-service.postgres.svc.cluster.local
- name: MAILU_DB_PORT
value: "5432"
- name: MAILU_DB_NAME
valueFrom:
secretKeyRef:
name: mailu-db-secret
key: database
- name: MAILU_DB_USER
valueFrom:
secretKeyRef:
name: mailu-db-secret
key: username
- name: MAILU_DB_PASSWORD
valueFrom:
secretKeyRef:
name: mailu-db-secret
key: password
- name: KEYCLOAK_CLIENT_ID
valueFrom:
secretKeyRef:
name: mailu-sync-credentials
key: client-id
- name: KEYCLOAK_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: mailu-sync-credentials
key: client-secret
volumeMounts:
- name: sync-script
mountPath: /app/sync.py
@ -84,6 +62,12 @@ spec:
- name: listener-script
mountPath: /app/listener.py
subPath: listener.py
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
resources:
requests:
cpu: 50m
@ -100,3 +84,13 @@ spec:
configMap:
name: mailu-sync-listener
defaultMode: 0444
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: mailu-vault
- name: vault-scripts
configMap:
name: mailu-vault-env
defaultMode: 0555

14
services/mailu/scripts/mailu_vault_env.sh Normal file
View File

@ -0,0 +1,14 @@
#!/usr/bin/env sh
set -eu
vault_dir="/vault/secrets"
read_secret() {
cat "${vault_dir}/$1"
}
export MAILU_DB_NAME="$(read_secret mailu-db-secret__database)"
export MAILU_DB_USER="$(read_secret mailu-db-secret__username)"
export MAILU_DB_PASSWORD="$(read_secret mailu-db-secret__password)"
export KEYCLOAK_CLIENT_ID="$(read_secret mailu-sync-credentials__client-id)"
export KEYCLOAK_CLIENT_SECRET="$(read_secret mailu-sync-credentials__client-secret)"

78
services/mailu/secretproviderclass.yaml Normal file
View File

@ -0,0 +1,78 @@
# services/mailu/secretproviderclass.yaml
apiVersion: secrets-store.csi.x-k8s.io/v1
kind: SecretProviderClass
metadata:
name: mailu-vault
namespace: mailu-mailserver
spec:
provider: vault
parameters:
vaultAddress: "http://vault.vault.svc.cluster.local:8200"
roleName: "mailu-mailserver"
objects: |
- objectName: "mailu-secret__secret-key"
secretPath: "kv/data/atlas/mailu/mailu-secret"
secretKey: "secret-key"
- objectName: "postmark-relay__relay-username"
secretPath: "kv/data/atlas/shared/postmark-relay"
secretKey: "relay-username"
- objectName: "postmark-relay__relay-password"
secretPath: "kv/data/atlas/shared/postmark-relay"
secretKey: "relay-password"
- objectName: "mailu-db-secret__database"
secretPath: "kv/data/atlas/mailu/mailu-db-secret"
secretKey: "database"
- objectName: "mailu-db-secret__username"
secretPath: "kv/data/atlas/mailu/mailu-db-secret"
secretKey: "username"
- objectName: "mailu-db-secret__password"
secretPath: "kv/data/atlas/mailu/mailu-db-secret"
secretKey: "password"
- objectName: "mailu-db-secret__url"
secretPath: "kv/data/atlas/mailu/mailu-db-secret"
secretKey: "url"
- objectName: "mailu-initial-account-secret__password"
secretPath: "kv/data/atlas/mailu/mailu-initial-account-secret"
secretKey: "password"
- objectName: "mailu-sync-credentials__client-id"
secretPath: "kv/data/atlas/mailu/mailu-sync-credentials"
secretKey: "client-id"
- objectName: "mailu-sync-credentials__client-secret"
secretPath: "kv/data/atlas/mailu/mailu-sync-credentials"
secretKey: "client-secret"
secretObjects:
- secretName: mailu-secret
type: Opaque
data:
- objectName: mailu-secret__secret-key
key: secret-key
- secretName: mailu-postmark-relay
type: Opaque
data:
- objectName: postmark-relay__relay-username
key: relay-username
- objectName: postmark-relay__relay-password
key: relay-password
- secretName: mailu-db-secret
type: Opaque
data:
- objectName: mailu-db-secret__database
key: database
- objectName: mailu-db-secret__username
key: username
- objectName: mailu-db-secret__password
key: password
- objectName: mailu-db-secret__url
key: url
- secretName: mailu-initial-account-secret
type: Opaque
data:
- objectName: mailu-initial-account-secret__password
key: password
- secretName: mailu-sync-credentials
type: Opaque
data:
- objectName: mailu-sync-credentials__client-id
key: client-id
- objectName: mailu-sync-credentials__client-secret
key: client-secret

6
services/mailu/serviceaccount.yaml Normal file
View File

@ -0,0 +1,6 @@
# services/mailu/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: mailu-vault-sync
namespace: mailu-mailserver

34
services/mailu/vault-sync-deployment.yaml Normal file
View File

@ -0,0 +1,34 @@
# services/mailu/vault-sync-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: mailu-vault-sync
namespace: mailu-mailserver
spec:
replicas: 1
selector:
matchLabels:
app: mailu-vault-sync
template:
metadata:
labels:
app: mailu-vault-sync
spec:
serviceAccountName: mailu-vault-sync
containers:
- name: sync
image: alpine:3.20
command: ["/bin/sh", "-c"]
args:
- "sleep infinity"
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: mailu-vault

51
services/nextcloud-mail-sync/cronjob.yaml
View File

@ -17,47 +17,23 @@ spec:
securityContext:
runAsUser: 0
runAsGroup: 0
serviceAccountName: nextcloud-vault
containers:
- name: mail-sync
image: nextcloud:29-apache
imagePullPolicy: IfNotPresent
command:
- /bin/bash
- /sync/sync.sh
- /bin/sh
- -c
env:
- name: KC_BASE
value: https://sso.bstein.dev
- name: KC_REALM
value: atlas
- name: KC_ADMIN_USER
valueFrom:
secretKeyRef:
name: nextcloud-keycloak-admin
key: username
- name: KC_ADMIN_PASS
valueFrom:
secretKeyRef:
name: nextcloud-keycloak-admin
key: password
- name: MAILU_DOMAIN
value: bstein.dev
- name: POSTGRES_HOST
value: postgres-service.postgres.svc.cluster.local
- name: POSTGRES_DB
valueFrom:
secretKeyRef:
name: nextcloud-db
key: database
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: nextcloud-db
key: db-username
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: nextcloud-db
key: db-password
resources:
requests:
cpu: 100m
@ -77,6 +53,17 @@ spec:
- name: sync-script
mountPath: /sync/sync.sh
subPath: sync.sh
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
args:
- |
set -euo pipefail
. /vault/scripts/nextcloud_vault_env.sh
exec /sync/sync.sh
volumes:
- name: nextcloud-config-pvc
persistentVolumeClaim:
@ -94,3 +81,13 @@ spec:
configMap:
name: nextcloud-mail-sync-script
defaultMode: 0755
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: nextcloud-vault
- name: vault-scripts
configMap:
name: nextcloud-vault-env
defaultMode: 0555

109
services/nextcloud/deployment.yaml
View File

@ -22,6 +22,7 @@ spec:
fsGroup: 33
runAsUser: 33
runAsGroup: 33
serviceAccountName: nextcloud-vault
initContainers:
- name: seed-nextcloud-web
image: nextcloud:29-apache
@ -80,6 +81,7 @@ spec:
command: ["/bin/sh", "-c"]
args:
- |
. /vault/scripts/nextcloud_vault_env.sh
installed="$(su -s /bin/sh www-data -c "php /var/www/html/occ status" 2>/dev/null | awk '/installed:/{print $3}' || true)"
if [ ! -s /var/www/html/config/config.php ]; then
su -s /bin/sh www-data -c "php /var/www/html/occ maintenance:install --database pgsql --database-host \"${POSTGRES_HOST}\" --database-name \"${POSTGRES_DB}\" --database-user \"${POSTGRES_USER}\" --database-pass \"${POSTGRES_PASSWORD}\" --admin-user \"${NEXTCLOUD_ADMIN_USER}\" --admin-pass \"${NEXTCLOUD_ADMIN_PASSWORD}\" --data-dir /var/www/html/data"
@ -150,41 +152,6 @@ spec:
env:
- name: POSTGRES_HOST
value: postgres-service.postgres.svc.cluster.local
- name: POSTGRES_DB
valueFrom:
secretKeyRef:
name: nextcloud-db
key: database
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: nextcloud-db
key: db-username
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: nextcloud-db
key: db-password
- name: NEXTCLOUD_ADMIN_USER
valueFrom:
secretKeyRef:
name: nextcloud-admin
key: admin-user
- name: NEXTCLOUD_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: nextcloud-admin
key: admin-password
- name: OIDC_CLIENT_ID
valueFrom:
secretKeyRef:
name: nextcloud-oidc
key: client-id
- name: OIDC_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: nextcloud-oidc
key: client-secret
volumeMounts:
- name: nextcloud-web
mountPath: /var/www/html
@ -197,40 +164,26 @@ spec:
- name: nextcloud-config-extra
mountPath: /var/www/html/config/extra.config.php
subPath: extra.config.php
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
containers:
- name: nextcloud
image: nextcloud:29-apache
imagePullPolicy: IfNotPresent
command: ["/bin/sh", "-c"]
args:
- >-
. /vault/scripts/nextcloud_vault_env.sh
&& exec /entrypoint.sh apache2-foreground
env:
# DB (external secret required: nextcloud-db with keys username,password,database)
- name: POSTGRES_HOST
value: postgres-service.postgres.svc.cluster.local
- name: POSTGRES_DB
valueFrom:
secretKeyRef:
name: nextcloud-db
key: database
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: nextcloud-db
key: db-username
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: nextcloud-db
key: db-password
# Admin bootstrap (external secret: nextcloud-admin with keys admin-user, admin-password)
- name: NEXTCLOUD_ADMIN_USER
valueFrom:
secretKeyRef:
name: nextcloud-admin
key: admin-user
- name: NEXTCLOUD_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: nextcloud-admin
key: admin-password
- name: NEXTCLOUD_TRUSTED_DOMAINS
value: cloud.bstein.dev
- name: OVERWRITEHOST
@ -246,31 +199,11 @@ spec:
value: "587"
- name: SMTP_SECURE
value: tls
- name: SMTP_NAME
valueFrom:
secretKeyRef:
name: nextcloud-smtp
key: smtp-username
- name: SMTP_PASSWORD
valueFrom:
secretKeyRef:
name: nextcloud-smtp
key: smtp-password
- name: MAIL_FROM_ADDRESS
value: no-reply
- name: MAIL_DOMAIN
value: bstein.dev
# OIDC (external secret: nextcloud-oidc with keys client-id, client-secret)
- name: OIDC_CLIENT_ID
valueFrom:
secretKeyRef:
name: nextcloud-oidc
key: client-id
- name: OIDC_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: nextcloud-oidc
key: client-secret
- name: NEXTCLOUD_UPDATE
value: "1"
- name: APP_INSTALL
@ -290,6 +223,12 @@ spec:
- name: nextcloud-config-extra
mountPath: /var/www/html/config/extra.config.php
subPath: extra.config.php
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
resources:
requests:
cpu: 250m
@ -314,3 +253,13 @@ spec:
configMap:
name: nextcloud-config
defaultMode: 0444
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: nextcloud-vault
- name: vault-scripts
configMap:
name: nextcloud-vault-env
defaultMode: 0555

7
services/nextcloud/kustomization.yaml
View File

@ -4,6 +4,8 @@ kind: Kustomization
namespace: nextcloud
resources:
- namespace.yaml
- serviceaccount.yaml
- secretproviderclass.yaml
- configmap.yaml
- pvc.yaml
- deployment.yaml
@ -13,6 +15,11 @@ resources:
- service.yaml
- ingress.yaml
configMapGenerator:
- name: nextcloud-vault-env
files:
- nextcloud_vault_env.sh=scripts/nextcloud_vault_env.sh
options:
disableNameSuffixHash: true
- name: nextcloud-maintenance-script
files:
- maintenance.sh=scripts/nextcloud-maintenance.sh

34
services/nextcloud/maintenance-cronjob.yaml
View File

@ -15,24 +15,20 @@ spec:
securityContext:
runAsUser: 0
runAsGroup: 0
serviceAccountName: nextcloud-vault
containers:
- name: maintenance
image: nextcloud:29-apache
imagePullPolicy: IfNotPresent
command: ["/bin/bash", "/maintenance/maintenance.sh"]
command: ["/bin/sh", "-c"]
args:
- |
set -euo pipefail
. /vault/scripts/nextcloud_vault_env.sh
exec /maintenance/maintenance.sh
env:
- name: NC_URL
value: https://cloud.bstein.dev
- name: ADMIN_USER
valueFrom:
secretKeyRef:
name: nextcloud-admin
key: admin-user
- name: ADMIN_PASS
valueFrom:
secretKeyRef:
name: nextcloud-admin
key: admin-password
volumeMounts:
- name: nextcloud-web
mountPath: /var/www/html
@ -45,6 +41,12 @@ spec:
- name: maintenance-script
mountPath: /maintenance/maintenance.sh
subPath: maintenance.sh
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
- name: vault-scripts
mountPath: /vault/scripts
readOnly: true
resources:
requests:
cpu: 100m
@ -69,3 +71,13 @@ spec:
configMap:
name: nextcloud-maintenance-script
defaultMode: 0755
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: nextcloud-vault
- name: vault-scripts
configMap:
name: nextcloud-vault-env
defaultMode: 0555

27
services/nextcloud/scripts/nextcloud_vault_env.sh Normal file
View File

@ -0,0 +1,27 @@
#!/usr/bin/env sh
set -eu
vault_dir="/vault/secrets"
read_secret() {
cat "${vault_dir}/$1"
}
export POSTGRES_DB="$(read_secret nextcloud-db__database)"
export POSTGRES_USER="$(read_secret nextcloud-db__db-username)"
export POSTGRES_PASSWORD="$(read_secret nextcloud-db__db-password)"
export NEXTCLOUD_ADMIN_USER="$(read_secret nextcloud-admin__admin-user)"
export NEXTCLOUD_ADMIN_PASSWORD="$(read_secret nextcloud-admin__admin-password)"
export ADMIN_USER="${NEXTCLOUD_ADMIN_USER}"
export ADMIN_PASS="${NEXTCLOUD_ADMIN_PASSWORD}"
export OIDC_CLIENT_ID="$(read_secret nextcloud-oidc__client-id)"
export OIDC_CLIENT_SECRET="$(read_secret nextcloud-oidc__client-secret)"
export SMTP_NAME="$(read_secret nextcloud-smtp__smtp-username)"
export SMTP_PASSWORD="$(read_secret nextcloud-smtp__smtp-password)"
export KC_ADMIN_USER="$(read_secret keycloak-admin__username)"
export KC_ADMIN_PASS="$(read_secret keycloak-admin__password)"

45
services/nextcloud/secretproviderclass.yaml Normal file
View File

@ -0,0 +1,45 @@
# services/nextcloud/secretproviderclass.yaml
apiVersion: secrets-store.csi.x-k8s.io/v1
kind: SecretProviderClass
metadata:
name: nextcloud-vault
namespace: nextcloud
spec:
provider: vault
parameters:
vaultAddress: "http://vault.vault.svc.cluster.local:8200"
roleName: "nextcloud"
objects: |
- objectName: "nextcloud-db__database"
secretPath: "kv/data/atlas/nextcloud/nextcloud-db"
secretKey: "database"
- objectName: "nextcloud-db__db-username"
secretPath: "kv/data/atlas/nextcloud/nextcloud-db"
secretKey: "db-username"
- objectName: "nextcloud-db__db-password"
secretPath: "kv/data/atlas/nextcloud/nextcloud-db"
secretKey: "db-password"
- objectName: "nextcloud-admin__admin-user"
secretPath: "kv/data/atlas/nextcloud/nextcloud-admin"
secretKey: "admin-user"
- objectName: "nextcloud-admin__admin-password"
secretPath: "kv/data/atlas/nextcloud/nextcloud-admin"
secretKey: "admin-password"
- objectName: "nextcloud-oidc__client-id"
secretPath: "kv/data/atlas/nextcloud/nextcloud-oidc"
secretKey: "client-id"
- objectName: "nextcloud-oidc__client-secret"
secretPath: "kv/data/atlas/nextcloud/nextcloud-oidc"
secretKey: "client-secret"
- objectName: "nextcloud-smtp__smtp-username"
secretPath: "kv/data/atlas/nextcloud/nextcloud-smtp"
secretKey: "smtp-username"
- objectName: "nextcloud-smtp__smtp-password"
secretPath: "kv/data/atlas/nextcloud/nextcloud-smtp"
secretKey: "smtp-password"
- objectName: "keycloak-admin__username"
secretPath: "kv/data/atlas/shared/keycloak-admin"
secretKey: "username"
- objectName: "keycloak-admin__password"
secretPath: "kv/data/atlas/shared/keycloak-admin"
secretKey: "password"

6
services/nextcloud/serviceaccount.yaml Normal file
View File

@ -0,0 +1,6 @@
# services/nextcloud/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: nextcloud-vault
namespace: nextcloud

107
services/vault/scripts/vault_k8s_auth_configure.sh
View File

@ -35,66 +35,71 @@ vault write auth/kubernetes/config \
kubernetes_host="${k8s_host}" \
kubernetes_ca_cert="${k8s_ca}"
for namespace in outline planka bstein-dev-home gitea vaultwarden sso; do
policy_name="${namespace}"
service_account=""
shared_paths=""
write_policy_and_role() {
role="$1"
namespace="$2"
service_accounts="$3"
read_paths="$4"
write_paths="$5"
case "${namespace}" in
outline)
service_account="outline-vault"
;;
planka)
service_account="planka-vault"
;;
bstein-dev-home)
service_account="bstein-dev-home"
shared_paths="shared/chat-ai-keys-runtime shared/portal-e2e-client"
;;
gitea)
service_account="gitea-vault"
;;
vaultwarden)
service_account="vaultwarden-vault"
;;
sso)
service_account="sso-vault,mas-secrets-ensure"
shared_paths="shared/keycloak-admin shared/portal-e2e-client"
;;
*)
log "unknown namespace ${namespace}"
exit 1
;;
esac
policy_body="$(cat <<EOF
path "kv/data/atlas/${namespace}/*" {
capabilities = ["read"]
}
path "kv/metadata/atlas/${namespace}/*" {
capabilities = ["list"]
}
EOF
)"
for shared in ${shared_paths}; do
policy_body=""
for path in ${read_paths}; do
policy_body="${policy_body}
path \"kv/data/atlas/${shared}\" {
path \"kv/data/atlas/${path}\" {
capabilities = [\"read\"]
}
path \"kv/metadata/atlas/${shared}\" {
path \"kv/metadata/atlas/${path}\" {
capabilities = [\"list\"]
}
"
done
for path in ${write_paths}; do
policy_body="${policy_body}
path \"kv/data/atlas/${path}\" {
capabilities = [\"create\", \"update\", \"read\"]
}
path \"kv/metadata/atlas/${path}\" {
capabilities = [\"list\"]
}
"
done
log "writing policy ${policy_name}"
printf '%s\n' "${policy_body}" | vault policy write "${policy_name}" -
log "writing policy ${role}"
printf '%s\n' "${policy_body}" | vault policy write "${role}" -
log "writing role ${namespace}"
vault write "auth/kubernetes/role/${namespace}" \
bound_service_account_names="${service_account}" \
log "writing role ${role}"
vault write "auth/kubernetes/role/${role}" \
bound_service_account_names="${service_accounts}" \
bound_service_account_namespaces="${namespace}" \
policies="${policy_name}" \
policies="${role}" \
ttl="${role_ttl}"
done
}
write_policy_and_role "outline" "outline" "outline-vault" \
"outline/*" ""
write_policy_and_role "planka" "planka" "planka-vault" \
"planka/*" ""
write_policy_and_role "bstein-dev-home" "bstein-dev-home" "bstein-dev-home" \
"bstein-dev-home/* shared/chat-ai-keys-runtime shared/portal-e2e-client" ""
write_policy_and_role "gitea" "gitea" "gitea-vault" \
"gitea/*" ""
write_policy_and_role "vaultwarden" "vaultwarden" "vaultwarden-vault" \
"vaultwarden/*" ""
write_policy_and_role "sso" "sso" "sso-vault,mas-secrets-ensure" \
"sso/* shared/keycloak-admin shared/portal-e2e-client" ""
write_policy_and_role "mailu-mailserver" "mailu-mailserver" "mailu-vault-sync" \
"mailu/* shared/postmark-relay" ""
write_policy_and_role "harbor" "harbor" "harbor-vault-sync" \
"harbor/*" ""
write_policy_and_role "nextcloud" "nextcloud" "nextcloud-vault" \
"nextcloud/* shared/keycloak-admin" ""
write_policy_and_role "comms" "comms" "comms-vault,atlasbot" \
"comms/* shared/chat-ai-keys-runtime" ""
write_policy_and_role "sso-secrets" "sso" "mas-secrets-ensure" \
"shared/keycloak-admin" \
"harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc"
write_policy_and_role "comms-secrets" "comms" \
"comms-secrets-ensure,mas-db-ensure,mas-admin-client-secret-writer,othrys-synapse-signingkey-job" \
"" \
"comms/turn-shared-secret comms/livekit-api comms/synapse-redis comms/synapse-macaroon comms/atlasbot-credentials-runtime comms/synapse-db comms/mas-db comms/mas-admin-client-runtime comms/mas-secrets-runtime comms/othrys-synapse-signingkey"