sso: route metis through dedicated oauth2 proxy

services/keycloak/kustomization.yaml

@@ -22,6 +22,7 @@ resources:
   - oneoffs/mas-secrets-ensure-job.yaml
   - oneoffs/synapse-oidc-secret-ensure-job.yaml
   - oneoffs/logs-oidc-secret-ensure-job.yaml
+  - oneoffs/metis-oidc-secret-ensure-job.yaml
   - oneoffs/harbor-oidc-secret-ensure-job.yaml
   - oneoffs/vault-oidc-secret-ensure-job.yaml
   - oneoffs/actual-oidc-secret-ensure-job.yaml

services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml

@@ -0,0 +1,143 @@
+# services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml
+# One-off job for sso/metis-oidc-secret-ensure-1.
+# Purpose: ensure the Metis oauth2-proxy OIDC client and Vault secret exist.
+# Keep this completed Job around; bump the suffix if it ever needs to be rerun.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: metis-oidc-secret-ensure-1
+  namespace: sso
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      annotations:
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/role: "sso-secrets"
+        vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: "kv/data/atlas/shared/keycloak-admin"
+        vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: |
+          {{ with secret "kv/data/atlas/shared/keycloak-admin" }}
+          export KEYCLOAK_ADMIN="{{ .Data.data.username }}"
+          export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}"
+          export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}"
+          {{ end }}
+    spec:
+      serviceAccountName: mas-secrets-ensure
+      restartPolicy: Never
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/worker
+                    operator: Exists
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: ["arm64"]
+      containers:
+        - name: apply
+          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -euo pipefail
+              . /vault/secrets/keycloak-admin-env.sh
+              KC_URL="http://keycloak.sso.svc.cluster.local"
+              ACCESS_TOKEN=""
+              for attempt in 1 2 3 4 5; do
+                TOKEN_JSON="$(curl -sS -X POST "$KC_URL/realms/master/protocol/openid-connect/token" \
+                  -H 'Content-Type: application/x-www-form-urlencoded' \
+                  -d "grant_type=password" \
+                  -d "client_id=admin-cli" \
+                  -d "username=${KEYCLOAK_ADMIN}" \
+                  -d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)"
+                ACCESS_TOKEN="$(echo "$TOKEN_JSON" | jq -r '.access_token' 2>/dev/null || true)"
+                if [ -n "$ACCESS_TOKEN" ] && [ "$ACCESS_TOKEN" != "null" ]; then
+                  break
+                fi
+                echo "Keycloak token request failed (attempt ${attempt})" >&2
+                sleep $((attempt * 2))
+              done
+              if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then
+                echo "Failed to fetch Keycloak admin token" >&2
+                exit 1
+              fi
+
+              CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+                "$KC_URL/admin/realms/atlas/clients?clientId=metis" || true)"
+              CLIENT_ID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)"
+
+              if [ -z "$CLIENT_ID" ] || [ "$CLIENT_ID" = "null" ]; then
+                create_payload='{"clientId":"metis","enabled":true,"protocol":"openid-connect","publicClient":false,"standardFlowEnabled":true,"implicitFlowEnabled":false,"directAccessGrantsEnabled":false,"serviceAccountsEnabled":false,"redirectUris":["https://sentinel.bstein.dev/oauth2/callback"],"webOrigins":["https://sentinel.bstein.dev"],"rootUrl":"https://sentinel.bstein.dev","baseUrl":"/"}'
+                status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
+                  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+                  -H 'Content-Type: application/json' \
+                  -d "${create_payload}" \
+                  "$KC_URL/admin/realms/atlas/clients")"
+                if [ "$status" != "201" ] && [ "$status" != "204" ]; then
+                  echo "Keycloak client create failed (status ${status})" >&2
+                  exit 1
+                fi
+                CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+                  "$KC_URL/admin/realms/atlas/clients?clientId=metis" || true)"
+                CLIENT_ID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)"
+              fi
+
+              if [ -z "$CLIENT_ID" ] || [ "$CLIENT_ID" = "null" ]; then
+                echo "Keycloak client metis not found" >&2
+                exit 1
+              fi
+
+              update_payload='{"enabled":true,"clientId":"metis","protocol":"openid-connect","publicClient":false,"standardFlowEnabled":true,"implicitFlowEnabled":false,"directAccessGrantsEnabled":false,"serviceAccountsEnabled":false,"redirectUris":["https://sentinel.bstein.dev/oauth2/callback"],"webOrigins":["https://sentinel.bstein.dev"],"rootUrl":"https://sentinel.bstein.dev","baseUrl":"/"}'
+              status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \
+                -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+                -H 'Content-Type: application/json' \
+                -d "${update_payload}" \
+                "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}")"
+              if [ "$status" != "204" ]; then
+                echo "Keycloak client update failed (status ${status})" >&2
+                exit 1
+              fi
+
+              CLIENT_SECRET="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+                "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}/client-secret" | jq -r '.value' 2>/dev/null || true)"
+              if [ -z "$CLIENT_SECRET" ] || [ "$CLIENT_SECRET" = "null" ]; then
+                echo "Keycloak client secret not found" >&2
+                exit 1
+              fi
+
+              vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
+              vault_role="${VAULT_ROLE:-sso-secrets}"
+              jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+              login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
+              vault_token="$(curl -sS --request POST --data "${login_payload}" \
+                "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
+              if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
+                echo "vault login failed" >&2
+                exit 1
+              fi
+
+              COOKIE_SECRET="$(curl -sS -H "X-Vault-Token: ${vault_token}" \
+                "${vault_addr}/v1/kv/data/atlas/maintenance/metis-oidc" | jq -r '.data.data.cookie_secret // empty')"
+              if [ -n "${COOKIE_SECRET}" ]; then
+                length="$(printf '%s' "${COOKIE_SECRET}" | wc -c | tr -d ' ')"
+                if [ "${length}" != "16" ] && [ "${length}" != "24" ] && [ "${length}" != "32" ]; then
+                  COOKIE_SECRET=""
+                fi
+              fi
+              if [ -z "${COOKIE_SECRET}" ]; then
+                COOKIE_SECRET="$(openssl rand -hex 16 | tr -d '\n')"
+              fi
+
+              payload="$(jq -nc \
+                --arg client_id "metis" \
+                --arg client_secret "${CLIENT_SECRET}" \
+                --arg cookie_secret "${COOKIE_SECRET}" \
+                '{data:{client_id:$client_id,client_secret:$client_secret,cookie_secret:$cookie_secret}}')"
+              curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
+                -d "${payload}" "${vault_addr}/v1/kv/data/atlas/maintenance/metis-oidc" >/dev/null

services/maintenance/kustomization.yaml

@@ -35,6 +35,7 @@ resources:
   - node-image-sweeper-daemonset.yaml
   - image-sweeper-cronjob.yaml
   - metis-service.yaml
+  - oauth2-proxy-metis.yaml
   - metis-certificate.yaml
   - metis-ingress.yaml
 images:

services/maintenance/metis-ingress.yaml

@@ -8,7 +8,7 @@ metadata:
     kubernetes.io/ingress.class: traefik
     traefik.ingress.kubernetes.io/router.entrypoints: websecure
     traefik.ingress.kubernetes.io/router.tls: "true"
-    traefik.ingress.kubernetes.io/router.middlewares: sso-oauth2-proxy-errors@kubernetescrd,sso-oauth2-proxy-forward-auth@kubernetescrd
+    traefik.ingress.kubernetes.io/router.middlewares: ""
 spec:
   ingressClassName: traefik
   tls:
@@ -22,6 +22,6 @@ spec:
             pathType: Prefix
             backend:
               service:
-                name: metis
+                name: oauth2-proxy-metis
                 port:
                   number: 80

services/maintenance/oauth2-proxy-metis.yaml

@@ -0,0 +1,119 @@
+# services/maintenance/oauth2-proxy-metis.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: oauth2-proxy-metis
+  namespace: maintenance
+  labels:
+    app: oauth2-proxy-metis
+spec:
+  ports:
+    - name: http
+      port: 80
+      targetPort: 4180
+  selector:
+    app: oauth2-proxy-metis
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: oauth2-proxy-metis
+  namespace: maintenance
+  labels:
+    app: oauth2-proxy-metis
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: oauth2-proxy-metis
+  template:
+    metadata:
+      labels:
+        app: oauth2-proxy-metis
+      annotations:
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: "maintenance"
+        vault.hashicorp.com/agent-inject-secret-oidc-config: "kv/data/atlas/maintenance/metis-oidc"
+        vault.hashicorp.com/agent-inject-template-oidc-config: |
+          {{- with secret "kv/data/atlas/maintenance/metis-oidc" -}}
+          client_id = "{{ .Data.data.client_id }}"
+          client_secret = "{{ .Data.data.client_secret }}"
+          cookie_secret = "{{ .Data.data.cookie_secret }}"
+          {{- end -}}
+    spec:
+      serviceAccountName: maintenance-vault-sync
+      nodeSelector:
+        node-role.kubernetes.io/worker: "true"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: ["amd64","arm64"]
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: ["rpi5"]
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values: ["titan-13","titan-15","titan-17","titan-19"]
+      containers:
+        - name: oauth2-proxy
+          image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - --provider=oidc
+            - --config=/vault/secrets/oidc-config
+            - --redirect-url=https://sentinel.bstein.dev/oauth2/callback
+            - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas
+            - --scope=openid profile email groups
+            - --email-domain=*
+            - --allowed-group=admin
+            - --allowed-group=maintenance
+            - --allowed-group=maintainer
+            - --set-xauthrequest=true
+            - --pass-access-token=true
+            - --set-authorization-header=true
+            - --cookie-secure=true
+            - --cookie-samesite=lax
+            - --cookie-refresh=20m
+            - --cookie-expire=168h
+            - --insecure-oidc-allow-unverified-email=true
+            - --upstream=http://metis.maintenance.svc.cluster.local
+            - --http-address=0.0.0.0:4180
+            - --skip-provider-button=true
+            - --skip-jwt-bearer-tokens=true
+            - --oidc-groups-claim=groups
+            - --cookie-domain=sentinel.bstein.dev
+          ports:
+            - containerPort: 4180
+              name: http
+          readinessProbe:
+            httpGet:
+              path: /ping
+              port: 4180
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: 4180
+            initialDelaySeconds: 20
+            periodSeconds: 20
+          resources:
+            requests:
+              cpu: 25m
+              memory: 64Mi
+            limits:
+              cpu: 250m
+              memory: 256Mi

services/vault/scripts/vault_k8s_auth_configure.sh

@@ -231,7 +231,7 @@ write_policy_and_role "crypto" "crypto" "crypto-vault-sync" \
 write_policy_and_role "health" "health" "health-vault-sync" \
   "health/*" ""
 write_policy_and_role "maintenance" "maintenance" "ariadne,maintenance-vault-sync" \
-  "maintenance/ariadne-db portal/atlas-portal-db portal/bstein-dev-home-keycloak-admin mailu/mailu-db-secret mailu/mailu-initial-account-secret nextcloud/nextcloud-db nextcloud/nextcloud-admin health/wger-admin finance/firefly-secrets comms/mas-admin-client-runtime comms/atlasbot-credentials-runtime comms/synapse-db comms/synapse-admin vault/vault-oidc-config shared/harbor-pull" ""
+  "maintenance/ariadne-db maintenance/metis-oidc portal/atlas-portal-db portal/bstein-dev-home-keycloak-admin mailu/mailu-db-secret mailu/mailu-initial-account-secret nextcloud/nextcloud-db nextcloud/nextcloud-admin health/wger-admin finance/firefly-secrets comms/mas-admin-client-runtime comms/atlasbot-credentials-runtime comms/synapse-db comms/synapse-admin vault/vault-oidc-config shared/harbor-pull" ""
 write_policy_and_role "finance" "finance" "finance-vault" \
   "finance/* shared/postmark-relay" ""
 write_policy_and_role "finance-secrets" "finance" "finance-secrets-ensure" \
@@ -246,7 +246,7 @@ write_policy_and_role "vault" "vault" "vault" \
 
 write_policy_and_role "sso-secrets" "sso" "mas-secrets-ensure" \
   "shared/keycloak-admin" \
-  "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc"
+  "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc"
 write_policy_and_role "crypto-secrets" "crypto" "crypto-secrets-ensure" \
   "" \
   "crypto/wallet-monero-temp-rpc-auth"