From 69cee91dda43b5ce948d92baddd793e45cee572a Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Thu, 15 Jan 2026 03:42:57 -0300 Subject: [PATCH] vault: fix data-prepper pipeline and portal admin secret job --- .../portal-onboarding-e2e-test-job.yaml | 4 +- services/keycloak/kustomization.yaml | 1 + ...portal-admin-client-secret-ensure-job.yaml | 138 ++++++++++++++++++ ...al-e2e-execute-actions-email-test-job.yaml | 4 +- .../logging/data-prepper-helmrelease.yaml | 4 +- 5 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 services/keycloak/portal-admin-client-secret-ensure-job.yaml diff --git a/services/bstein-dev-home/portal-onboarding-e2e-test-job.yaml b/services/bstein-dev-home/portal-onboarding-e2e-test-job.yaml index cfe35a1..17597df 100644 --- a/services/bstein-dev-home/portal-onboarding-e2e-test-job.yaml +++ b/services/bstein-dev-home/portal-onboarding-e2e-test-job.yaml @@ -60,7 +60,7 @@ spec: command: ["/bin/sh", "-c"] args: - | - set -euo pipefail + set -eu . /vault/secrets/portal-env.sh python /scripts/test_portal_onboarding_flow.py volumeMounts: @@ -71,4 +71,4 @@ spec: - name: tests configMap: name: portal-onboarding-e2e-tests - defaultMode: 0555 \ No newline at end of file + defaultMode: 0555 diff --git a/services/keycloak/kustomization.yaml b/services/keycloak/kustomization.yaml index e141467..316f447 100644 --- a/services/keycloak/kustomization.yaml +++ b/services/keycloak/kustomization.yaml @@ -11,6 +11,7 @@ resources: - vault-sync-deployment.yaml - deployment.yaml - realm-settings-job.yaml + - portal-admin-client-secret-ensure-job.yaml - portal-e2e-client-job.yaml - portal-e2e-target-client-job.yaml - portal-e2e-token-exchange-permissions-job.yaml diff --git a/services/keycloak/portal-admin-client-secret-ensure-job.yaml b/services/keycloak/portal-admin-client-secret-ensure-job.yaml new file mode 100644 index 0000000..350fc6e --- /dev/null +++ b/services/keycloak/portal-admin-client-secret-ensure-job.yaml @@ -0,0 +1,138 @@ +# services/keycloak/portal-admin-client-secret-ensure-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: keycloak-portal-admin-secret-ensure-1 + namespace: sso +spec: + backoffLimit: 0 + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/role: "sso" + vault.hashicorp.com/agent-inject-secret-keycloak-env.sh: "kv/data/atlas/shared/keycloak-admin" + vault.hashicorp.com/agent-inject-template-keycloak-env.sh: | + {{ with secret "kv/data/atlas/shared/keycloak-admin" }} + export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}" + export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}" + {{ end }} + {{ with secret "kv/data/atlas/portal/bstein-dev-home-keycloak-admin" }} + export PORTAL_ADMIN_CLIENT_SECRET="{{ .Data.data.client_secret }}" + {{ end }} + spec: + restartPolicy: Never + serviceAccountName: sso-vault + containers: + - name: configure + image: python:3.11-alpine + env: + - name: KEYCLOAK_SERVER + value: http://keycloak.sso.svc.cluster.local + - name: KEYCLOAK_REALM + value: atlas + - name: PORTAL_ADMIN_CLIENT_ID + value: bstein-dev-home-admin + command: ["/bin/sh", "-c"] + args: + - | + set -eu + . /vault/secrets/keycloak-env.sh + python - <<'PY' + import json + import os + import urllib.parse + import urllib.error + import urllib.request + + base_url = os.environ["KEYCLOAK_SERVER"].rstrip("/") + realm = os.environ["KEYCLOAK_REALM"] + admin_user = os.environ["KEYCLOAK_ADMIN_USER"] + admin_password = os.environ["KEYCLOAK_ADMIN_PASSWORD"] + client_id = os.environ["PORTAL_ADMIN_CLIENT_ID"] + client_secret = os.environ["PORTAL_ADMIN_CLIENT_SECRET"] + + def http_json(method: str, url: str, token: str, payload=None): + data = None + headers = {"Authorization": f"Bearer {token}"} + if payload is not None: + data = json.dumps(payload).encode() + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + body = resp.read() + if not body: + return resp.status, None + return resp.status, json.loads(body.decode()) + except urllib.error.HTTPError as exc: + raw = exc.read() + if not raw: + return exc.code, None + try: + return exc.code, json.loads(raw.decode()) + except Exception: + return exc.code, {"raw": raw.decode(errors="replace")} + + def get_admin_token() -> str: + token_data = urllib.parse.urlencode( + { + "grant_type": "password", + "client_id": "admin-cli", + "username": admin_user, + "password": admin_password, + } + ).encode() + req = urllib.request.Request( + f"{base_url}/realms/master/protocol/openid-connect/token", + data=token_data, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=15) as resp: + body = json.loads(resp.read().decode()) + except urllib.error.HTTPError as exc: + raw = exc.read().decode(errors="replace") + raise SystemExit(f"Token request failed: status={exc.code} body={raw}") + return body["access_token"] + + token = get_admin_token() + status, clients = http_json( + "GET", + f"{base_url}/admin/realms/{realm}/clients?clientId={urllib.parse.quote(client_id)}", + token, + ) + if status != 200 or not isinstance(clients, list) or not clients: + raise SystemExit(f"Unable to find client {client_id!r} (status={status})") + + client_uuid = None + for item in clients: + if isinstance(item, dict) and item.get("clientId") == client_id: + client_uuid = item.get("id") + break + if not client_uuid: + raise SystemExit(f"Client {client_id!r} has no id") + + status, client_rep = http_json( + "GET", + f"{base_url}/admin/realms/{realm}/clients/{client_uuid}", + token, + ) + if status != 200 or not isinstance(client_rep, dict): + raise SystemExit(f"Unable to fetch client representation (status={status})") + + if client_rep.get("secret") != client_secret: + client_rep["secret"] = client_secret + status, resp = http_json( + "PUT", + f"{base_url}/admin/realms/{realm}/clients/{client_uuid}", + token, + client_rep, + ) + if status not in (200, 204): + raise SystemExit(f"Client update failed (status={status}) resp={resp}") + + print(f"OK: ensured secret for {client_id}") + PY diff --git a/services/keycloak/portal-e2e-execute-actions-email-test-job.yaml b/services/keycloak/portal-e2e-execute-actions-email-test-job.yaml index 7ee4e20..70c0a01 100644 --- a/services/keycloak/portal-e2e-execute-actions-email-test-job.yaml +++ b/services/keycloak/portal-e2e-execute-actions-email-test-job.yaml @@ -59,7 +59,7 @@ spec: command: ["/bin/sh", "-c"] args: - | - set -euo pipefail + set -eu . /vault/secrets/keycloak-env.sh python /scripts/test_keycloak_execute_actions_email.py volumeMounts: @@ -70,4 +70,4 @@ spec: - name: tests configMap: name: portal-e2e-tests - defaultMode: 0555 \ No newline at end of file + defaultMode: 0555 diff --git a/services/logging/data-prepper-helmrelease.yaml b/services/logging/data-prepper-helmrelease.yaml index 9d932d5..66ae3c5 100644 --- a/services/logging/data-prepper-helmrelease.yaml +++ b/services/logging/data-prepper-helmrelease.yaml @@ -58,8 +58,8 @@ spec: name: data-prepper patch: |- - op: remove - path: /spec/template/spec/volumes/1/secret + path: /spec/template/spec/volumes/0/secret - op: add - path: /spec/template/spec/volumes/1/configMap + path: /spec/template/spec/volumes/0/configMap value: name: data-prepper-pipeline