chore(cassandra): federate sso and park artifact migration

services/cassandra-auth/bootstrap-jobs/cassandra-ldap-federation-job.yaml

@@ -0,0 +1,370 @@
+# services/cassandra-auth/bootstrap-jobs/cassandra-ldap-federation-job.yaml
+# One-off job for sso/cassandra-ldap-federation-1.
+# Purpose: attach the Cassandra Keycloak realm to the shared OpenLDAP directory.
+# Run unsuspended for migration, then suspend/replace with a new job name for future runs.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cassandra-ldap-federation-1
+  namespace: sso
+spec:
+  ttlSecondsAfterFinished: 3600
+  suspend: false
+  backoffLimit: 2
+  template:
+    metadata:
+      annotations:
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/role: "sso"
+        vault.hashicorp.com/agent-inject-secret-keycloak-env.sh: "kv/data/atlas/shared/keycloak-admin"
+        vault.hashicorp.com/agent-inject-template-keycloak-env.sh: |
+          {{ with secret "kv/data/atlas/shared/keycloak-admin" }}
+          export KEYCLOAK_ADMIN="{{ .Data.data.username }}"
+          export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}"
+          export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}"
+          {{ end }}
+          {{ with secret "kv/data/atlas/sso/openldap-admin" }}
+          export LDAP_ADMIN_PASSWORD="{{ .Data.data.LDAP_ADMIN_PASSWORD }}"
+          export LDAP_CONFIG_PASSWORD="{{ .Data.data.LDAP_CONFIG_PASSWORD }}"
+          export LDAP_BIND_PASSWORD="${LDAP_ADMIN_PASSWORD}"
+          {{ end }}
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: ["rpi5","rpi4"]
+                  - key: node-role.kubernetes.io/worker
+                    operator: Exists
+      restartPolicy: OnFailure
+      serviceAccountName: sso-vault
+      containers:
+        - name: configure
+          image: python:3.11-alpine
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: KEYCLOAK_SERVER
+              value: http://keycloak.sso.svc.cluster.local
+            - name: KEYCLOAK_REALM
+              value: cassandra
+            - name: LDAP_URL
+              value: ldap://openldap.sso.svc.cluster.local:389
+            - name: LDAP_BIND_DN
+              value: cn=admin,dc=bstein,dc=dev
+            - name: LDAP_USERS_DN
+              value: ou=users,dc=bstein,dc=dev
+            - name: LDAP_GROUPS_DN
+              value: ou=groups,dc=bstein,dc=dev
+            - name: VERIFY_USERS
+              value: veles-dev,cassandra-dev,daniel-test,viktor-test
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -euo pipefail
+              . /vault/secrets/keycloak-env.sh
+              python - <<'PY'
+              import json
+              import os
+              import time
+              import urllib.error
+              import urllib.parse
+              import urllib.request
+
+              base_url = os.environ["KEYCLOAK_SERVER"].rstrip("/")
+              realm = os.environ["KEYCLOAK_REALM"]
+              admin_user = os.environ["KEYCLOAK_ADMIN_USER"]
+              admin_password = os.environ["KEYCLOAK_ADMIN_PASSWORD"]
+              ldap_url = os.environ["LDAP_URL"]
+              ldap_bind_dn = os.environ["LDAP_BIND_DN"]
+              ldap_bind_password = os.environ["LDAP_BIND_PASSWORD"]
+              ldap_users_dn = os.environ["LDAP_USERS_DN"]
+              ldap_groups_dn = os.environ["LDAP_GROUPS_DN"]
+
+              def request(method, url, token=None, payload=None, timeout=30):
+                  data = None
+                  headers = {}
+                  if token:
+                      headers["Authorization"] = f"Bearer {token}"
+                  if payload is not None:
+                      data = json.dumps(payload).encode()
+                      headers["Content-Type"] = "application/json"
+                  req = urllib.request.Request(url, data=data, headers=headers, method=method)
+                  try:
+                      with urllib.request.urlopen(req, timeout=timeout) as resp:
+                          body = resp.read()
+                          if not body:
+                              return resp.status, None, dict(resp.headers)
+                          return resp.status, json.loads(body.decode()), dict(resp.headers)
+                  except urllib.error.HTTPError as exc:
+                      raw = exc.read()
+                      body = None
+                      if raw:
+                          try:
+                              body = json.loads(raw.decode())
+                          except Exception:
+                              body = {"raw": raw.decode(errors="replace")}
+                      return exc.code, body, dict(exc.headers)
+
+              def get_token():
+                  data = urllib.parse.urlencode(
+                      {
+                          "grant_type": "password",
+                          "client_id": "admin-cli",
+                          "username": admin_user,
+                          "password": admin_password,
+                      }
+                  ).encode()
+                  req = urllib.request.Request(
+                      f"{base_url}/realms/master/protocol/openid-connect/token",
+                      data=data,
+                      headers={"Content-Type": "application/x-www-form-urlencoded"},
+                      method="POST",
+                  )
+                  with urllib.request.urlopen(req, timeout=30) as resp:
+                      return json.loads(resp.read().decode())["access_token"]
+
+              token = None
+              for attempt in range(1, 31):
+                  try:
+                      token = get_token()
+                      break
+                  except Exception as exc:
+                      if attempt == 30:
+                          raise SystemExit(f"Keycloak token request failed: {exc}")
+                      time.sleep(2)
+              assert token
+
+              status, realm_rep, _ = request("GET", f"{base_url}/admin/realms/{realm}", token)
+              if status != 200 or not isinstance(realm_rep, dict) or not realm_rep.get("id"):
+                  raise SystemExit(f"Unable to resolve realm id for {realm}: status={status}")
+              realm_id = realm_rep["id"]
+
+              status, components, _ = request(
+                  "GET",
+                  f"{base_url}/admin/realms/{realm}/components",
+                  token,
+              )
+              if status != 200:
+                  raise SystemExit(f"Unable to list realm components: status={status}")
+              components = components or []
+
+              for component in components:
+                  if component.get("providerId") != "ldap":
+                      continue
+                  if component.get("providerType") != "org.keycloak.storage.UserStorageProvider":
+                      continue
+                  if component.get("parentId") == realm_id:
+                      continue
+                  cid = component.get("id")
+                  if not cid:
+                      continue
+                  status, payload, _ = request(
+                      "GET",
+                      f"{base_url}/admin/realms/{realm}/components/{cid}",
+                      token,
+                  )
+                  if status != 200 or not isinstance(payload, dict):
+                      raise SystemExit(f"Unable to fetch LDAP component {cid}: status={status}")
+                  payload["parentId"] = realm_id
+                  status, _, _ = request(
+                      "PUT",
+                      f"{base_url}/admin/realms/{realm}/components/{cid}",
+                      token,
+                      payload,
+                  )
+                  if status not in (200, 204):
+                      raise SystemExit(f"Unable to repair LDAP component {cid}: status={status}")
+
+              status, storage_components, _ = request(
+                  "GET",
+                  f"{base_url}/admin/realms/{realm}/components?type=org.keycloak.storage.UserStorageProvider",
+                  token,
+              )
+              if status != 200:
+                  raise SystemExit(f"Unable to list user-storage components: status={status}")
+              storage_components = storage_components or []
+
+              ldap_components = [
+                  c
+                  for c in storage_components
+                  if c.get("providerId") == "ldap" and c.get("id")
+              ]
+              candidates = []
+              for component in ldap_components:
+                  config = component.get("config") or {}
+                  if component.get("name") in ("openldap", "ldap") and (config.get("connectionUrl") or [None])[0] == ldap_url:
+                      candidates.append(component)
+              if not candidates:
+                  candidates = [c for c in ldap_components if c.get("name") in ("openldap", "ldap")]
+              candidates.sort(key=lambda item: item.get("id", ""))
+              ldap_component = candidates[0] if candidates else None
+              ldap_component_id = ldap_component.get("id") if ldap_component else None
+
+              desired = {
+                  "name": "openldap",
+                  "providerId": "ldap",
+                  "providerType": "org.keycloak.storage.UserStorageProvider",
+                  "parentId": realm_id,
+                  "config": {
+                      "enabled": ["true"],
+                      "priority": ["0"],
+                      "importEnabled": ["true"],
+                      "editMode": ["WRITABLE"],
+                      "syncRegistrations": ["true"],
+                      "vendor": ["other"],
+                      "connectionUrl": [ldap_url],
+                      "bindDn": [ldap_bind_dn],
+                      "bindCredential": [ldap_bind_password],
+                      "authType": ["simple"],
+                      "usersDn": [ldap_users_dn],
+                      "searchScope": ["1"],
+                      "pagination": ["true"],
+                      "usernameLDAPAttribute": ["uid"],
+                      "rdnLDAPAttribute": ["uid"],
+                      "uuidLDAPAttribute": ["entryUUID"],
+                      "userObjectClasses": ["inetOrgPerson, organizationalPerson, person, top"],
+                      "trustEmail": ["true"],
+                      "useTruststoreSpi": ["never"],
+                      "connectionPooling": ["true"],
+                      "cachePolicy": ["DEFAULT"],
+                      "useKerberosForPasswordAuthentication": ["false"],
+                      "allowKerberosAuthentication": ["false"],
+                  },
+              }
+
+              if ldap_component_id:
+                  desired["id"] = ldap_component_id
+                  print(f"Updating Cassandra LDAP federation provider: {ldap_component_id}")
+                  status, _, _ = request(
+                      "PUT",
+                      f"{base_url}/admin/realms/{realm}/components/{ldap_component_id}",
+                      token,
+                      desired,
+                  )
+                  if status not in (200, 204):
+                      raise SystemExit(f"LDAP provider update failed: status={status}")
+              else:
+                  print("Creating Cassandra LDAP federation provider")
+                  status, _, headers = request(
+                      "POST",
+                      f"{base_url}/admin/realms/{realm}/components",
+                      token,
+                      desired,
+                  )
+                  if status not in (201, 204):
+                      raise SystemExit(f"LDAP provider create failed: status={status}")
+                  location = headers.get("Location", "")
+                  ldap_component_id = location.rstrip("/").split("/")[-1] if location else None
+
+              if not ldap_component_id:
+                  raise SystemExit("Unable to determine Cassandra LDAP provider id")
+
+              status, mapper_components, _ = request(
+                  "GET",
+                  f"{base_url}/admin/realms/{realm}/components?type=org.keycloak.storage.ldap.mappers.LDAPStorageMapper",
+                  token,
+              )
+              if status != 200:
+                  raise SystemExit(f"Unable to list LDAP mappers: status={status}")
+              mapper_components = mapper_components or []
+
+              def upsert_mapper(name, provider_id, config):
+                  existing = next(
+                      (
+                          c
+                          for c in mapper_components
+                          if c.get("name") == name and c.get("parentId") == ldap_component_id
+                      ),
+                      None,
+                  )
+                  payload = {
+                      "name": name,
+                      "providerId": provider_id,
+                      "providerType": "org.keycloak.storage.ldap.mappers.LDAPStorageMapper",
+                      "parentId": ldap_component_id,
+                      "config": config,
+                  }
+                  if existing:
+                      payload["id"] = existing["id"]
+                      status, _, _ = request(
+                          "PUT",
+                          f"{base_url}/admin/realms/{realm}/components/{existing['id']}",
+                          token,
+                          payload,
+                      )
+                      action = "Updated"
+                  else:
+                      status, _, _ = request(
+                          "POST",
+                          f"{base_url}/admin/realms/{realm}/components",
+                          token,
+                          payload,
+                      )
+                      action = "Created"
+                  if status not in (200, 201, 204):
+                      raise SystemExit(f"Mapper {name} upsert failed: status={status}")
+                  print(f"{action} Cassandra LDAP mapper: {name}")
+
+              upsert_mapper(
+                  "openldap-groups",
+                  "group-ldap-mapper",
+                  {
+                      "groups.dn": [ldap_groups_dn],
+                      "group.name.ldap.attribute": ["cn"],
+                      "group.object.classes": ["groupOfNames"],
+                      "membership.ldap.attribute": ["member"],
+                      "membership.attribute.type": ["DN"],
+                      "mode": ["LDAP_ONLY"],
+                      "user.roles.retrieve.strategy": ["LOAD_GROUPS_BY_MEMBER_ATTRIBUTE"],
+                      "preserve.group.inheritance": ["true"],
+                  },
+              )
+              for name, ldap_attr, user_attr in (
+                  ("openldap-email", "mail", "email"),
+                  ("openldap-first-name", "givenName", "firstName"),
+                  ("openldap-last-name", "sn", "lastName"),
+              ):
+                  upsert_mapper(
+                      name,
+                      "user-attribute-ldap-mapper",
+                      {
+                          "ldap.attribute": [ldap_attr],
+                          "user.model.attribute": [user_attr],
+                          "read.only": ["false"],
+                          "always.read.value.from.ldap": ["false"],
+                          "is.mandatory.in.ldap": ["false"],
+                      },
+                  )
+
+              status, sync_body, _ = request(
+                  "POST",
+                  f"{base_url}/admin/realms/{realm}/user-storage/{ldap_component_id}/sync?action=triggerFullSync",
+                  token,
+                  timeout=120,
+              )
+              if status in (200, 201, 204):
+                  print(f"Cassandra LDAP full sync requested: {sync_body}")
+              else:
+                  print(f"WARNING: Cassandra LDAP full sync returned status={status} body={sync_body}")
+
+              for username in (item.strip() for item in os.environ.get("VERIFY_USERS", "").split(",")):
+                  if not username:
+                      continue
+                  encoded = urllib.parse.quote(username)
+                  status, users, _ = request(
+                      "GET",
+                      f"{base_url}/admin/realms/{realm}/users?username={encoded}&exact=true",
+                      token,
+                  )
+                  found = status == 200 and any(
+                      str(user.get("username") or "").casefold() == username.casefold()
+                      for user in users or []
+                  )
+                  print(f"Cassandra LDAP user check {username}: {'found' if found else 'missing'}")
+
+              print("Cassandra LDAP federation ready")
+              PY

services/cassandra-auth/bootstrap-jobs/cassandra-realm-ensure-job.yaml

@@ -5,7 +5,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: cassandra-realm-ensure-1
+  name: cassandra-realm-ensure-2
   namespace: sso
 spec:
   suspend: true
@@ -70,7 +70,7 @@ spec:
             - name: KEYCLOAK_SMTP_FROM_NAME
               value: Cassandra
             - name: CASSANDRA_PUBLIC_REPLAY_USERS
-              value: cassandra-dev,daniel-test,viktor-test
+              value: veles-dev,cassandra-dev,daniel-test,viktor-test
           command: ["/bin/sh", "-c"]
           args:
             - |

services/cassandra-auth/kustomization.yaml

@@ -3,6 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: sso
 resources:
+  - bootstrap-jobs/cassandra-ldap-federation-job.yaml
   - bootstrap-jobs/cassandra-realm-ensure-job.yaml
   - bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml
 configMapGenerator:

services/cassandra/backend-deployment.yaml

@@ -7,7 +7,7 @@ metadata:
   labels:
     app: cassandra-backend
 spec:
-  replicas: 0
+  replicas: 1
   revisionHistoryLimit: 2
   strategy:
     type: RollingUpdate

services/cassandra/migration-jobs/artifact-copy-veles-to-cassandra-job.yaml

@@ -6,7 +6,7 @@ metadata:
   name: cassandra-artifact-copy-from-veles-2
   namespace: cassandra
 spec:
-  suspend: false
+  suspend: true
   backoffLimit: 0
   activeDeadlineSeconds: 43200
   ttlSecondsAfterFinished: 86400

services/veles/migration-jobs/cassandra-artifact-export.yaml

@@ -20,7 +20,7 @@ metadata:
   name: veles-artifact-export-for-cassandra
   namespace: veles
 spec:
-  replicas: 1
+  replicas: 0
   selector:
     matchLabels:
       app: veles-artifact-export-for-cassandra