From e22293db3eaecda1025ce98fa4288ce327b009c6 Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Fri, 12 Dec 2025 22:09:04 -0300 Subject: [PATCH] forcing 12-r3 over 12-r6 for redis --- scripts/mailu_sync.py | 204 ++++++++++++++++++++++++++ scripts/tests/test_mailu_sync.py | 83 +++++++++++ services/mailu/helmrelease.yaml | 55 +++++-- services/mailu/ingressroute.yaml | 19 +++ services/mailu/kustomization.yaml | 11 ++ services/mailu/mailu-sync-job.yaml | 73 +++++++++ services/mailu/serverstransport.yaml | 10 ++ services/mailu/unbound-configmap.yaml | 3 + 8 files changed, 449 insertions(+), 9 deletions(-) create mode 100644 scripts/mailu_sync.py create mode 100644 scripts/tests/test_mailu_sync.py create mode 100644 services/mailu/ingressroute.yaml create mode 100644 services/mailu/mailu-sync-job.yaml create mode 100644 services/mailu/serverstransport.yaml diff --git a/scripts/mailu_sync.py b/scripts/mailu_sync.py new file mode 100644 index 0000000..bb6e261 --- /dev/null +++ b/scripts/mailu_sync.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +""" +Sync Keycloak users to Mailu mailboxes. + - Generates/stores a mailu_app_password attribute in Keycloak (admin-only) + - Upserts the mailbox in Mailu Postgres using that password +""" + +import os +import sys +import json +import time +import secrets +import string +import datetime +import requests +import psycopg2 +from psycopg2.extras import RealDictCursor +from passlib.hash import bcrypt_sha256 + + +KC_BASE = os.environ["KEYCLOAK_BASE_URL"].rstrip("/") +KC_REALM = os.environ["KEYCLOAK_REALM"] +KC_CLIENT_ID = os.environ["KEYCLOAK_CLIENT_ID"] +KC_CLIENT_SECRET = os.environ["KEYCLOAK_CLIENT_SECRET"] + +MAILU_DOMAIN = os.environ["MAILU_DOMAIN"] +MAILU_DEFAULT_QUOTA = int(os.environ.get("MAILU_DEFAULT_QUOTA", "1000000000")) + +DB_CONFIG = { + "host": os.environ["MAILU_DB_HOST"], + "port": int(os.environ.get("MAILU_DB_PORT", "5432")), + "dbname": os.environ["MAILU_DB_NAME"], + "user": os.environ["MAILU_DB_USER"], + "password": os.environ["MAILU_DB_PASSWORD"], +} + +SESSION = requests.Session() + + +def log(msg): + sys.stdout.write(f"{msg}\n") + sys.stdout.flush() + + +def get_kc_token(): + resp = SESSION.post( + f"{KC_BASE}/realms/{KC_REALM}/protocol/openid-connect/token", + data={ + "grant_type": "client_credentials", + "client_id": KC_CLIENT_ID, + "client_secret": KC_CLIENT_SECRET, + }, + timeout=15, + ) + resp.raise_for_status() + return resp.json()["access_token"] + + +def kc_get_users(token): + users = [] + first = 0 + max_results = 200 + headers = {"Authorization": f"Bearer {token}"} + while True: + resp = SESSION.get( + f"{KC_BASE}/admin/realms/{KC_REALM}/users", + params={"first": first, "max": max_results, "enabled": "true"}, + headers=headers, + timeout=20, + ) + resp.raise_for_status() + batch = resp.json() + users.extend(batch) + if len(batch) < max_results: + break + first += max_results + return users + + +def kc_update_attributes(token, user, attributes): + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + payload = { + "firstName": user.get("firstName"), + "lastName": user.get("lastName"), + "email": user.get("email"), + "enabled": user.get("enabled", True), + "username": user["username"], + "emailVerified": user.get("emailVerified", False), + "attributes": attributes, + } + user_url = f"{KC_BASE}/admin/realms/{KC_REALM}/users/{user['id']}" + resp = SESSION.put(user_url, headers=headers, json=payload, timeout=20) + resp.raise_for_status() + verify = SESSION.get( + user_url, + headers={"Authorization": f"Bearer {token}"}, + params={"briefRepresentation": "false"}, + timeout=15, + ) + verify.raise_for_status() + attrs = verify.json().get("attributes") or {} + if not attrs.get("mailu_app_password"): + raise Exception(f"attribute not persisted for {user.get('email') or user['username']}") + + +def random_password(): + alphabet = string.ascii_letters + string.digits + return "".join(secrets.choice(alphabet) for _ in range(24)) + + +def ensure_mailu_user(cursor, email, password, display_name): + localpart, domain = email.split("@", 1) + if domain.lower() != MAILU_DOMAIN.lower(): + return + hashed = bcrypt_sha256.hash(password) + now = datetime.datetime.utcnow() + cursor.execute( + """ + INSERT INTO "user" ( + email, localpart, domain_name, password, + quota_bytes, quota_bytes_used, + global_admin, enabled, enable_imap, enable_pop, allow_spoofing, + forward_enabled, forward_destination, forward_keep, + reply_enabled, reply_subject, reply_body, reply_startdate, reply_enddate, + displayed_name, spam_enabled, spam_mark_as_read, spam_threshold, + change_pw_next_login, created_at, updated_at, comment + ) + VALUES ( + %(email)s, %(localpart)s, %(domain)s, %(password)s, + %(quota)s, 0, + false, true, true, true, false, + false, '', true, + false, NULL, NULL, DATE '1900-01-01', DATE '2999-12-31', + %(display)s, true, true, 80, + false, CURRENT_DATE, %(now)s, '' + ) + ON CONFLICT (email) DO UPDATE + SET password = EXCLUDED.password, + enabled = true, + updated_at = EXCLUDED.updated_at + """, + { + "email": email, + "localpart": localpart, + "domain": domain, + "password": hashed, + "quota": MAILU_DEFAULT_QUOTA, + "display": display_name or localpart, + "now": now, + }, + ) + + +def main(): + token = get_kc_token() + users = kc_get_users(token) + if not users: + log("No users found; exiting.") + return + + conn = psycopg2.connect(**DB_CONFIG) + conn.autocommit = True + cursor = conn.cursor(cursor_factory=RealDictCursor) + + for user in users: + attrs = user.get("attributes", {}) or {} + app_pw_value = attrs.get("mailu_app_password") + if isinstance(app_pw_value, list): + app_pw = app_pw_value[0] if app_pw_value else None + elif isinstance(app_pw_value, str): + app_pw = app_pw_value + else: + app_pw = None + + email = user.get("email") + if not email: + email = f"{user['username']}@{MAILU_DOMAIN}" + + if not app_pw: + app_pw = random_password() + attrs["mailu_app_password"] = app_pw + kc_update_attributes(token, user, attrs) + log(f"Set mailu_app_password for {email}") + + display_name = " ".join( + part for part in [user.get("firstName"), user.get("lastName")] if part + ).strip() + + ensure_mailu_user(cursor, email, app_pw, display_name) + log(f"Synced mailbox for {email}") + + cursor.close() + conn.close() + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + log(f"ERROR: {exc}") + sys.exit(1) diff --git a/scripts/tests/test_mailu_sync.py b/scripts/tests/test_mailu_sync.py new file mode 100644 index 0000000..f495c87 --- /dev/null +++ b/scripts/tests/test_mailu_sync.py @@ -0,0 +1,83 @@ +import importlib.util +import pathlib + +import pytest + + +def load_sync_module(monkeypatch): + # Minimal env required by module import + env = { + "KEYCLOAK_BASE_URL": "http://keycloak", + "KEYCLOAK_REALM": "atlas", + "KEYCLOAK_CLIENT_ID": "mailu-sync", + "KEYCLOAK_CLIENT_SECRET": "secret", + "MAILU_DOMAIN": "example.com", + "MAILU_DB_HOST": "localhost", + "MAILU_DB_PORT": "5432", + "MAILU_DB_NAME": "mailu", + "MAILU_DB_USER": "mailu", + "MAILU_DB_PASSWORD": "pw", + } + for k, v in env.items(): + monkeypatch.setenv(k, v) + module_path = pathlib.Path(__file__).resolve().parents[1] / "mailu_sync.py" + spec = importlib.util.spec_from_file_location("mailu_sync_testmod", module_path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_random_password_length_and_charset(monkeypatch): + sync = load_sync_module(monkeypatch) + pw = sync.random_password() + assert len(pw) == 24 + assert all(ch.isalnum() for ch in pw) + + +class _FakeResponse: + def __init__(self, json_data=None, status=200): + self._json_data = json_data or {} + self.status_code = status + + def raise_for_status(self): + if self.status_code >= 400: + raise AssertionError(f"status {self.status_code}") + + def json(self): + return self._json_data + + +class _FakeSession: + def __init__(self, put_resp, get_resp): + self.put_resp = put_resp + self.get_resp = get_resp + self.put_called = False + self.get_called = False + + def post(self, *args, **kwargs): + return _FakeResponse({"access_token": "dummy"}) + + def put(self, *args, **kwargs): + self.put_called = True + return self.put_resp + + def get(self, *args, **kwargs): + self.get_called = True + return self.get_resp + + +def test_kc_update_attributes_succeeds(monkeypatch): + sync = load_sync_module(monkeypatch) + ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}}) + sync.SESSION = _FakeSession(_FakeResponse({}), ok_resp) + sync.kc_update_attributes("token", {"id": "u1", "username": "u1"}, {"mailu_app_password": "abc"}) + assert sync.SESSION.put_called and sync.SESSION.get_called + + +def test_kc_update_attributes_raises_without_attribute(monkeypatch): + sync = load_sync_module(monkeypatch) + missing_attr_resp = _FakeResponse({"attributes": {}}, status=200) + sync.SESSION = _FakeSession(_FakeResponse({}), missing_attr_resp) + with pytest.raises(Exception): + sync.kc_update_attributes("token", {"id": "u1", "username": "u1"}, {"mailu_app_password": "abc"}) diff --git a/services/mailu/helmrelease.yaml b/services/mailu/helmrelease.yaml index bbbe2d3..7c1b608 100644 --- a/services/mailu/helmrelease.yaml +++ b/services/mailu/helmrelease.yaml @@ -16,9 +16,13 @@ spec: namespace: flux-system install: remediation: { retries: 3 } + timeout: 10m upgrade: - remediation: { retries: 3 } + remediation: + retries: 3 + remediateLastFailure: true cleanupOnFail: true + timeout: 10m values: mailuVersion: "2024.06" domain: bstein.dev @@ -59,13 +63,15 @@ spec: hostPort: enabled: false https: - enabled: true - external: true + enabled: false + external: false forceHttps: false externalService: enabled: true type: LoadBalancer externalTrafficPolicy: Cluster + ports: + submission: true nodePorts: pop3: 30010 pop3s: 30011 @@ -82,6 +88,20 @@ spec: logLevel: DEBUG nodeSelector: hardware: rpi4 + podLivenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + podReadinessProbe: + enabled: true + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 extraEnvVars: - name: FLASK_DEBUG value: "1" @@ -102,13 +122,28 @@ spec: - name: unbound-run emptyDir: {} extraVolumeMounts: - - name: unbound-config - mountPath: /etc/unbound - name: unbound-run mountPath: /var/lib/unbound extraContainers: - name: unbound - image: docker.io/mvance/unbound:1.22.0 + image: docker.io/alpine:3.20 + command: ["/bin/sh", "-c"] + args: + - | + while :; do + printf "nameserver 10.43.0.10\n" > /etc/resolv.conf + if apk add --no-cache unbound bind-tools; then + break + fi + echo "apk failed, retrying" >&2 + sleep 10 + done + cat >/etc/resolv.conf <<'EOF' + search mailu-mailserver.svc.cluster.local svc.cluster.local cluster.local + nameserver 127.0.0.1 + EOF + unbound-anchor -a /var/lib/unbound/root.key || true + exec unbound -d -c /opt/unbound/etc/unbound/unbound.conf ports: - containerPort: 53 protocol: UDP @@ -148,8 +183,8 @@ spec: architecture: standalone logLevel: DEBUG image: - repository: bitnami/redis - tag: 7.2.4-debian-12-r6 + repository: bitnamilegacy/redis + tag: 8.0.3-debian-12-r3 master: nodeSelector: hardware: rpi4 @@ -178,12 +213,14 @@ spec: nodeSelector: hardware: rpi4 ingress: - enabled: true + enabled: false ingressClassName: traefik tls: true existingSecret: mailu-certificates annotations: traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/service.serversscheme: https + traefik.ingress.kubernetes.io/service.serverstransport: mailu-transport@kubernetescrd extraRules: - host: mail.bstein.dev http: diff --git a/services/mailu/ingressroute.yaml b/services/mailu/ingressroute.yaml new file mode 100644 index 0000000..d4bc4f6 --- /dev/null +++ b/services/mailu/ingressroute.yaml @@ -0,0 +1,19 @@ +# services/mailu/ingressroute.yaml +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: mailu + namespace: mailu-mailserver +spec: + entryPoints: + - websecure + routes: + - match: Host(`mail.bstein.dev`) + kind: Rule + services: + - name: mailu-front + port: 443 + scheme: https + serversTransport: mailu-transport + tls: + secretName: mailu-certificates diff --git a/services/mailu/kustomization.yaml b/services/mailu/kustomization.yaml index 992ac25..b3248a9 100644 --- a/services/mailu/kustomization.yaml +++ b/services/mailu/kustomization.yaml @@ -8,3 +8,14 @@ resources: - certificate.yaml - vip-controller.yaml - unbound-configmap.yaml + - serverstransport.yaml + - ingressroute.yaml + - mailu-sync-job.yaml + +configMapGenerator: + - name: mailu-sync-script + namespace: mailu-mailserver + files: + - sync.py=../../scripts/mailu_sync.py + options: + disableNameSuffixHash: true diff --git a/services/mailu/mailu-sync-job.yaml b/services/mailu/mailu-sync-job.yaml new file mode 100644 index 0000000..14d9adb --- /dev/null +++ b/services/mailu/mailu-sync-job.yaml @@ -0,0 +1,73 @@ +# services/mailu/mailu-sync-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: mailu-sync + namespace: mailu-mailserver +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: mailu-sync + image: python:3.11-alpine + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] + args: + - | + pip install --no-cache-dir requests psycopg2-binary passlib >/tmp/pip.log \ + && python /app/sync.py + env: + - name: KEYCLOAK_BASE_URL + value: http://keycloak.sso.svc.cluster.local + - name: KEYCLOAK_REALM + value: atlas + - name: MAILU_DOMAIN + value: bstein.dev + - name: MAILU_DEFAULT_QUOTA + value: "1000000000" + - name: MAILU_DB_HOST + value: postgres-service.postgres.svc.cluster.local + - name: MAILU_DB_PORT + value: "5432" + - name: MAILU_DB_NAME + valueFrom: + secretKeyRef: + name: mailu-db-secret + key: database + - name: MAILU_DB_USER + valueFrom: + secretKeyRef: + name: mailu-db-secret + key: username + - name: MAILU_DB_PASSWORD + valueFrom: + secretKeyRef: + name: mailu-db-secret + key: password + - name: KEYCLOAK_CLIENT_ID + valueFrom: + secretKeyRef: + name: mailu-sync-credentials + key: client-id + - name: KEYCLOAK_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: mailu-sync-credentials + key: client-secret + volumeMounts: + - name: sync-script + mountPath: /app/sync.py + subPath: sync.py + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + volumes: + - name: sync-script + configMap: + name: mailu-sync-script + defaultMode: 0444 diff --git a/services/mailu/serverstransport.yaml b/services/mailu/serverstransport.yaml new file mode 100644 index 0000000..ace7b31 --- /dev/null +++ b/services/mailu/serverstransport.yaml @@ -0,0 +1,10 @@ +# services/mailu/serverstransport.yaml +apiVersion: traefik.io/v1alpha1 +kind: ServersTransport +metadata: + name: mailu-transport + namespace: mailu-mailserver +spec: + # Force SNI to mail.bstein.dev and skip backend cert verification (backend cert is for the host, not the pod IP). + serverName: mail.bstein.dev + insecureSkipVerify: true diff --git a/services/mailu/unbound-configmap.yaml b/services/mailu/unbound-configmap.yaml index 9a405dd..219cafb 100644 --- a/services/mailu/unbound-configmap.yaml +++ b/services/mailu/unbound-configmap.yaml @@ -18,6 +18,9 @@ data: qname-minimisation: yes harden-dnssec-stripped: yes val-clean-additional: yes + domain-insecure: "mailu-mailserver.svc.cluster.local." + domain-insecure: "svc.cluster.local." + domain-insecure: "cluster.local." cache-min-ttl: 120 cache-max-ttl: 86400 access-control: 0.0.0.0/0 allow