agent(openclaw): expose oauth protected UI

infrastructure/core/coredns-custom.yaml

@@ -10,6 +10,7 @@ data:
       errors
       cache 30
       hosts {
+        192.168.22.9 agent.bstein.dev
         192.168.22.9 alerts.bstein.dev
         192.168.22.9 auth.bstein.dev
         192.168.22.9 bstein.dev

infrastructure/core/node-prefer-noschedule-cronjob.yaml

@@ -24,7 +24,7 @@ spec:
                 - bash
                 - -ceu
                 - |
-                  for node in titan-13 titan-15 titan-17 titan-19; do
+                  for node in titan-08 titan-13 titan-15 titan-17 titan-19; do
                     if kubectl get node "${node}" >/dev/null 2>&1; then
                       kubectl label node "${node}" atlas.bstein.dev/spillover=true --overwrite=true
                       kubectl taint node "${node}" longhorn=true:PreferNoSchedule --overwrite=true

services/keycloak/kustomization.yaml

@@ -25,6 +25,7 @@ resources:
   - oneoffs/metis-oidc-secret-ensure-job.yaml
   - oneoffs/soteria-oidc-secret-ensure-job.yaml
   - oneoffs/quality-oidc-secret-ensure-job.yaml
+  - oneoffs/agent-oidc-secret-ensure-job.yaml
   - oneoffs/metis-ssh-keys-secret-ensure-job.yaml
   - oneoffs/metis-node-passwords-secret-ensure-job.yaml
   - oneoffs/harbor-oidc-secret-ensure-job.yaml
@@ -48,3 +49,6 @@ configMapGenerator:
   - name: actual-oidc-secret-ensure-script
     files:
       - actual_oidc_secret_ensure.sh=scripts/actual_oidc_secret_ensure.sh
+  - name: agent-oidc-secret-ensure-script
+    files:
+      - agent_oidc_secret_ensure.sh=scripts/agent_oidc_secret_ensure.sh

services/keycloak/oneoffs/agent-oidc-secret-ensure-job.yaml

@@ -0,0 +1,50 @@
+# services/keycloak/oneoffs/agent-oidc-secret-ensure-job.yaml
+# One-off job for sso/agent-oidc-secret-ensure-1.
+# Purpose: ensure the agent UI oauth2-proxy OIDC client and Vault secret exist.
+# Keep this completed Job around; bump the suffix if it ever needs to be rerun.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: agent-oidc-secret-ensure-1
+  namespace: sso
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      annotations:
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/role: "sso-secrets"
+        vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: "kv/data/atlas/shared/keycloak-admin"
+        vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: |
+          {{ with secret "kv/data/atlas/shared/keycloak-admin" }}
+          export KEYCLOAK_ADMIN="{{ .Data.data.username }}"
+          export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}"
+          export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}"
+          {{ end }}
+    spec:
+      serviceAccountName: mas-secrets-ensure
+      restartPolicy: Never
+      volumes:
+        - name: agent-oidc-secret-ensure-script
+          configMap:
+            name: agent-oidc-secret-ensure-script
+            defaultMode: 0555
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: ["arm64"]
+                  - key: node-role.kubernetes.io/worker
+                    operator: Exists
+      containers:
+        - name: apply
+          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
+          command: ["/scripts/agent_oidc_secret_ensure.sh"]
+          volumeMounts:
+            - name: agent-oidc-secret-ensure-script
+              mountPath: /scripts
+              readOnly: true

services/keycloak/scripts/agent_oidc_secret_ensure.sh

@@ -0,0 +1,168 @@
+#!/usr/bin/env sh
+set -euo pipefail
+
+. /vault/secrets/keycloak-admin-env.sh
+
+CLIENT_NAME="agent"
+PUBLIC_URL="https://agent.bstein.dev"
+VAULT_SECRET_PATH="openclaw/agent-oidc"
+KC_URL="http://keycloak.sso.svc.cluster.local"
+
+ACCESS_TOKEN=""
+for attempt in 1 2 3 4 5 6 7 8 9 10; do
+  if curl -fsS "${KC_URL}/realms/master" >/dev/null 2>&1; then
+    break
+  fi
+  echo "Waiting for Keycloak to be reachable (attempt ${attempt})" >&2
+  sleep $((attempt * 2))
+done
+for attempt in 1 2 3 4 5; do
+  TOKEN_JSON="$(curl -sS -X POST "$KC_URL/realms/master/protocol/openid-connect/token" \
+    -H 'Content-Type: application/x-www-form-urlencoded' \
+    -d "grant_type=password" \
+    -d "client_id=admin-cli" \
+    -d "username=${KEYCLOAK_ADMIN}" \
+    -d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)"
+  ACCESS_TOKEN="$(echo "$TOKEN_JSON" | jq -r '.access_token' 2>/dev/null || true)"
+  if [ -n "$ACCESS_TOKEN" ] && [ "$ACCESS_TOKEN" != "null" ]; then
+    break
+  fi
+  echo "Keycloak token request failed (attempt ${attempt})" >&2
+  sleep $((attempt * 2))
+done
+if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then
+  echo "Failed to fetch Keycloak admin token" >&2
+  exit 1
+fi
+
+CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  "$KC_URL/admin/realms/atlas/clients?clientId=${CLIENT_NAME}" || true)"
+CLIENT_ID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)"
+client_payload="$(jq -nc \
+  --arg client_id "${CLIENT_NAME}" \
+  --arg redirect_uri "${PUBLIC_URL}/oauth2/callback" \
+  --arg web_origin "${PUBLIC_URL}" \
+  '{clientId:$client_id,enabled:true,protocol:"openid-connect",publicClient:false,standardFlowEnabled:true,implicitFlowEnabled:false,directAccessGrantsEnabled:false,serviceAccountsEnabled:false,redirectUris:[$redirect_uri],webOrigins:[$web_origin],rootUrl:$web_origin,baseUrl:"/"}')"
+
+if [ -z "$CLIENT_ID" ] || [ "$CLIENT_ID" = "null" ]; then
+  status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
+    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+    -H 'Content-Type: application/json' \
+    -d "${client_payload}" \
+    "$KC_URL/admin/realms/atlas/clients")"
+  if [ "$status" != "201" ] && [ "$status" != "204" ] && [ "$status" != "409" ]; then
+    echo "Keycloak client create failed (status ${status})" >&2
+    exit 1
+  fi
+  CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+    "$KC_URL/admin/realms/atlas/clients?clientId=${CLIENT_NAME}" || true)"
+  CLIENT_ID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)"
+fi
+
+if [ -z "$CLIENT_ID" ] || [ "$CLIENT_ID" = "null" ]; then
+  echo "Keycloak client ${CLIENT_NAME} not found" >&2
+  exit 1
+fi
+
+SCOPE_ID="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  "$KC_URL/admin/realms/atlas/client-scopes?search=groups" | jq -r '.[] | select(.name=="groups") | .id' 2>/dev/null | head -n1 || true)"
+if [ -z "$SCOPE_ID" ] || [ "$SCOPE_ID" = "null" ]; then
+  echo "Keycloak client scope groups not found" >&2
+  exit 1
+fi
+
+DEFAULT_SCOPES="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}/default-client-scopes" || true)"
+OPTIONAL_SCOPES="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}/optional-client-scopes" || true)"
+
+if ! echo "$DEFAULT_SCOPES" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1 \
+  && ! echo "$OPTIONAL_SCOPES" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1; then
+  status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \
+    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+    "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}/optional-client-scopes/${SCOPE_ID}")"
+  if [ "$status" != "200" ] && [ "$status" != "201" ] && [ "$status" != "204" ]; then
+    status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
+      -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+      "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}/optional-client-scopes/${SCOPE_ID}")"
+    if [ "$status" != "200" ] && [ "$status" != "201" ] && [ "$status" != "204" ]; then
+      echo "Failed to attach groups client scope to ${CLIENT_NAME} (status ${status})" >&2
+      exit 1
+    fi
+  fi
+fi
+
+status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \
+  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  -H 'Content-Type: application/json' \
+  -d "${client_payload}" \
+  "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}")"
+if [ "$status" != "204" ]; then
+  echo "Keycloak client update failed (status ${status})" >&2
+  exit 1
+fi
+
+CLIENT_SECRET="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}/client-secret" | jq -r '.value' 2>/dev/null || true)"
+if [ -z "$CLIENT_SECRET" ] || [ "$CLIENT_SECRET" = "null" ]; then
+  echo "Keycloak client secret not found" >&2
+  exit 1
+fi
+
+vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
+vault_role="${VAULT_ROLE:-sso-secrets}"
+jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
+vault_token="$(curl -sS --request POST --data "${login_payload}" \
+  "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
+if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
+  echo "vault login failed" >&2
+  exit 1
+fi
+
+read_status="$(curl -sS -o /tmp/agent-oidc-read.json -w "%{http_code}" \
+  -H "X-Vault-Token: ${vault_token}" \
+  "${vault_addr}/v1/kv/data/atlas/${VAULT_SECRET_PATH}" || true)"
+COOKIE_SECRET=""
+if [ "${read_status}" = "200" ]; then
+  COOKIE_SECRET="$(jq -r '.data.data.cookie_secret // empty' /tmp/agent-oidc-read.json)"
+elif [ "${read_status}" != "404" ]; then
+  echo "Vault read failed (status ${read_status})" >&2
+  cat /tmp/agent-oidc-read.json >&2 || true
+  exit 1
+fi
+if [ -n "${COOKIE_SECRET}" ]; then
+  length="$(printf '%s' "${COOKIE_SECRET}" | wc -c | tr -d ' ')"
+  if [ "${length}" != "16" ] && [ "${length}" != "24" ] && [ "${length}" != "32" ]; then
+    COOKIE_SECRET=""
+  fi
+fi
+if [ -z "${COOKIE_SECRET}" ]; then
+  COOKIE_SECRET="$(openssl rand -hex 16 | tr -d '\n')"
+fi
+
+payload="$(jq -nc \
+  --arg client_id "${CLIENT_NAME}" \
+  --arg client_secret "${CLIENT_SECRET}" \
+  --arg cookie_secret "${COOKIE_SECRET}" \
+  '{data:{client_id:$client_id,client_secret:$client_secret,cookie_secret:$cookie_secret}}')"
+write_status="$(curl -sS -o /tmp/agent-oidc-write.json -w "%{http_code}" -X POST \
+  -H "X-Vault-Token: ${vault_token}" \
+  -H 'Content-Type: application/json' \
+  -d "${payload}" "${vault_addr}/v1/kv/data/atlas/${VAULT_SECRET_PATH}")"
+if [ "${write_status}" != "200" ] && [ "${write_status}" != "204" ]; then
+  echo "Vault write failed (status ${write_status})" >&2
+  cat /tmp/agent-oidc-write.json >&2 || true
+  exit 1
+fi
+
+verify_status="$(curl -sS -o /tmp/agent-oidc-verify.json -w "%{http_code}" \
+  -H "X-Vault-Token: ${vault_token}" \
+  "${vault_addr}/v1/kv/data/atlas/${VAULT_SECRET_PATH}" || true)"
+if [ "${verify_status}" != "200" ]; then
+  echo "Vault verify failed (status ${verify_status})" >&2
+  cat /tmp/agent-oidc-verify.json >&2 || true
+  exit 1
+fi
+
+echo "Agent OIDC secret ready in Vault"

services/openclaw/agent-certificate.yaml

@@ -0,0 +1,13 @@
+# services/openclaw/agent-certificate.yaml
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: agent-tls
+  namespace: openclaw
+spec:
+  secretName: agent-tls
+  issuerRef:
+    kind: ClusterIssuer
+    name: letsencrypt
+  dnsNames:
+    - agent.bstein.dev

services/openclaw/agent-ingress.yaml

@@ -0,0 +1,26 @@
+# services/openclaw/agent-ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: agent
+  namespace: openclaw
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    traefik.ingress.kubernetes.io/router.tls: "true"
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts: ["agent.bstein.dev"]
+      secretName: agent-tls
+  rules:
+    - host: agent.bstein.dev
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: oauth2-proxy-agent
+                port:
+                  number: 80

services/openclaw/agent-vault-serviceaccount.yaml

@@ -0,0 +1,6 @@
+# services/openclaw/agent-vault-serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: agent-vault
+  namespace: openclaw

services/openclaw/kustomization.yaml

@@ -6,6 +6,10 @@ resources:
   - namespace.yaml
   - configmap.yaml
   - rbac.yaml
+  - agent-vault-serviceaccount.yaml
   - ollama-deployment.yaml
   - deployment.yaml
   - service.yaml
+  - oauth2-proxy-agent.yaml
+  - agent-certificate.yaml
+  - agent-ingress.yaml

services/openclaw/oauth2-proxy-agent.yaml

@@ -0,0 +1,133 @@
+# services/openclaw/oauth2-proxy-agent.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: oauth2-proxy-agent
+  namespace: openclaw
+  labels:
+    app: oauth2-proxy-agent
+spec:
+  ports:
+    - name: http
+      port: 80
+      targetPort: 4180
+  selector:
+    app: oauth2-proxy-agent
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: oauth2-proxy-agent
+  namespace: openclaw
+  labels:
+    app: oauth2-proxy-agent
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: oauth2-proxy-agent
+  template:
+    metadata:
+      labels:
+        app: oauth2-proxy-agent
+      annotations:
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: "openclaw"
+        vault.hashicorp.com/agent-inject-secret-oidc-config: "kv/data/atlas/openclaw/agent-oidc"
+        vault.hashicorp.com/agent-inject-template-oidc-config: |
+          {{- with secret "kv/data/atlas/openclaw/agent-oidc" -}}
+          client_id = "{{ .Data.data.client_id }}"
+          client_secret = "{{ .Data.data.client_secret }}"
+          cookie_secret = "{{ .Data.data.cookie_secret }}"
+          {{- end -}}
+    spec:
+      serviceAccountName: agent-vault
+      nodeSelector:
+        node-role.kubernetes.io/worker: "true"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: ["arm64"]
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: atlas.bstein.dev/spillover
+                    operator: DoesNotExist
+            - weight: 90
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: ["rpi5"]
+            - weight: 50
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: ["rpi4"]
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: ScheduleAnyway
+          labelSelector:
+            matchLabels:
+              app: oauth2-proxy-agent
+      containers:
+        - name: oauth2-proxy
+          image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - --provider=oidc
+            - --config=/vault/secrets/oidc-config
+            - --redirect-url=https://agent.bstein.dev/oauth2/callback
+            - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas
+            - --scope=openid profile email groups
+            - --email-domain=*
+            - --allowed-group=admin
+            - --allowed-group=/admin
+            - --allowed-group=dev
+            - --allowed-group=/dev
+            - --set-xauthrequest=true
+            - --pass-access-token=true
+            - --set-authorization-header=true
+            - --cookie-secure=true
+            - --cookie-samesite=lax
+            - --cookie-refresh=20m
+            - --cookie-expire=168h
+            - --insecure-oidc-allow-unverified-email=true
+            - --upstream=http://openclaw.openclaw.svc.cluster.local:18789
+            - --http-address=0.0.0.0:4180
+            - --skip-provider-button=true
+            - --approval-prompt=auto
+            - --skip-jwt-bearer-tokens=true
+            - --oidc-groups-claim=groups
+            - --cookie-domain=agent.bstein.dev
+          ports:
+            - containerPort: 4180
+              name: http
+          readinessProbe:
+            httpGet:
+              path: /ping
+              port: 4180
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: 4180
+            initialDelaySeconds: 20
+            periodSeconds: 20
+          resources:
+            requests:
+              cpu: 25m
+              memory: 64Mi
+            limits:
+              cpu: 250m
+              memory: 256Mi

services/quality/oauth2-proxy-sonarqube.yaml

@@ -64,6 +64,23 @@ spec:
                   - key: hardware
                     operator: In
                     values: ["rpi5"]
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                topologyKey: kubernetes.io/hostname
+                labelSelector:
+                  matchExpressions:
+                    - key: app
+                      operator: In
+                      values: ["sonarqube", "keycloak"]
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: ScheduleAnyway
+          labelSelector:
+            matchLabels:
+              app: oauth2-proxy-sonarqube
       containers:
         - name: oauth2-proxy
           image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0

services/quality/sonarqube-deployment.yaml

@@ -58,6 +58,24 @@ spec:
                   - key: kubernetes.io/hostname
                     operator: NotIn
                     values: ["titan-07", "titan-11"]
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                topologyKey: kubernetes.io/hostname
+                labelSelector:
+                  matchExpressions:
+                    - key: app
+                      operator: In
+                      values: ["keycloak"]
+            - weight: 80
+              podAffinityTerm:
+                topologyKey: kubernetes.io/hostname
+                labelSelector:
+                  matchExpressions:
+                    - key: app.kubernetes.io/name
+                      operator: In
+                      values: ["grafana"]
       initContainers:
         - name: prepare-volume-permissions
           image: busybox:1.36

services/vault/scripts/vault_k8s_auth_configure.sh

@@ -236,6 +236,8 @@ write_policy_and_role "crypto" "crypto" "crypto-vault-sync" \
   "crypto/* shared/harbor-pull" ""
 write_policy_and_role "health" "health" "health-vault-sync" \
   "health/*" ""
+write_policy_and_role "openclaw" "openclaw" "agent-vault" \
+  "openclaw/*" ""
 write_policy_and_role "maintenance" "maintenance" "ariadne,maintenance-vault-sync,metis" \
   "maintenance/ariadne-db maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys maintenance/metis-runtime portal/atlas-portal-db portal/bstein-dev-home-keycloak-admin mailu/mailu-db-secret mailu/mailu-initial-account-secret nextcloud/nextcloud-db nextcloud/nextcloud-admin health/wger-admin finance/firefly-secrets comms/mas-admin-client-runtime comms/atlasbot-credentials-runtime comms/synapse-db comms/synapse-admin vault/vault-oidc-config shared/harbor-pull shared/soteria-restic harbor/harbor-core" "" \
   '
@@ -263,7 +265,7 @@ write_policy_and_role "vault" "vault" "vault" \
 
 write_policy_and_role "sso-secrets" "sso" "mas-secrets-ensure" \
   "shared/keycloak-admin maintenance/metis-ssh-keys" \
-  "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys" \
+  "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys openclaw/agent-oidc" \
   '
 path "kv/data/atlas/nodes/*" {
   capabilities = ["create", "update", "read"]