forcing 12-r3 over 12-r6 for redis

scripts/mailu_sync.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""
+Sync Keycloak users to Mailu mailboxes.
+ - Generates/stores a mailu_app_password attribute in Keycloak (admin-only)
+ - Upserts the mailbox in Mailu Postgres using that password
+"""
+
+import os
+import sys
+import json
+import time
+import secrets
+import string
+import datetime
+import requests
+import psycopg2
+from psycopg2.extras import RealDictCursor
+from passlib.hash import bcrypt_sha256
+
+
+KC_BASE = os.environ["KEYCLOAK_BASE_URL"].rstrip("/")
+KC_REALM = os.environ["KEYCLOAK_REALM"]
+KC_CLIENT_ID = os.environ["KEYCLOAK_CLIENT_ID"]
+KC_CLIENT_SECRET = os.environ["KEYCLOAK_CLIENT_SECRET"]
+
+MAILU_DOMAIN = os.environ["MAILU_DOMAIN"]
+MAILU_DEFAULT_QUOTA = int(os.environ.get("MAILU_DEFAULT_QUOTA", "1000000000"))
+
+DB_CONFIG = {
+    "host": os.environ["MAILU_DB_HOST"],
+    "port": int(os.environ.get("MAILU_DB_PORT", "5432")),
+    "dbname": os.environ["MAILU_DB_NAME"],
+    "user": os.environ["MAILU_DB_USER"],
+    "password": os.environ["MAILU_DB_PASSWORD"],
+}
+
+SESSION = requests.Session()
+
+
+def log(msg):
+    sys.stdout.write(f"{msg}\n")
+    sys.stdout.flush()
+
+
+def get_kc_token():
+    resp = SESSION.post(
+        f"{KC_BASE}/realms/{KC_REALM}/protocol/openid-connect/token",
+        data={
+            "grant_type": "client_credentials",
+            "client_id": KC_CLIENT_ID,
+            "client_secret": KC_CLIENT_SECRET,
+        },
+        timeout=15,
+    )
+    resp.raise_for_status()
+    return resp.json()["access_token"]
+
+
+def kc_get_users(token):
+    users = []
+    first = 0
+    max_results = 200
+    headers = {"Authorization": f"Bearer {token}"}
+    while True:
+        resp = SESSION.get(
+            f"{KC_BASE}/admin/realms/{KC_REALM}/users",
+            params={"first": first, "max": max_results, "enabled": "true"},
+            headers=headers,
+            timeout=20,
+        )
+        resp.raise_for_status()
+        batch = resp.json()
+        users.extend(batch)
+        if len(batch) < max_results:
+            break
+        first += max_results
+    return users
+
+
+def kc_update_attributes(token, user, attributes):
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Content-Type": "application/json",
+    }
+    payload = {
+        "firstName": user.get("firstName"),
+        "lastName": user.get("lastName"),
+        "email": user.get("email"),
+        "enabled": user.get("enabled", True),
+        "username": user["username"],
+        "emailVerified": user.get("emailVerified", False),
+        "attributes": attributes,
+    }
+    user_url = f"{KC_BASE}/admin/realms/{KC_REALM}/users/{user['id']}"
+    resp = SESSION.put(user_url, headers=headers, json=payload, timeout=20)
+    resp.raise_for_status()
+    verify = SESSION.get(
+        user_url,
+        headers={"Authorization": f"Bearer {token}"},
+        params={"briefRepresentation": "false"},
+        timeout=15,
+    )
+    verify.raise_for_status()
+    attrs = verify.json().get("attributes") or {}
+    if not attrs.get("mailu_app_password"):
+        raise Exception(f"attribute not persisted for {user.get('email') or user['username']}")
+
+
+def random_password():
+    alphabet = string.ascii_letters + string.digits
+    return "".join(secrets.choice(alphabet) for _ in range(24))
+
+
+def ensure_mailu_user(cursor, email, password, display_name):
+    localpart, domain = email.split("@", 1)
+    if domain.lower() != MAILU_DOMAIN.lower():
+        return
+    hashed = bcrypt_sha256.hash(password)
+    now = datetime.datetime.utcnow()
+    cursor.execute(
+        """
+        INSERT INTO "user" (
+          email, localpart, domain_name, password,
+          quota_bytes, quota_bytes_used,
+          global_admin, enabled, enable_imap, enable_pop, allow_spoofing,
+          forward_enabled, forward_destination, forward_keep,
+          reply_enabled, reply_subject, reply_body, reply_startdate, reply_enddate,
+          displayed_name, spam_enabled, spam_mark_as_read, spam_threshold,
+          change_pw_next_login, created_at, updated_at, comment
+        )
+        VALUES (
+          %(email)s, %(localpart)s, %(domain)s, %(password)s,
+          %(quota)s, 0,
+          false, true, true, true, false,
+          false, '', true,
+          false, NULL, NULL, DATE '1900-01-01', DATE '2999-12-31',
+          %(display)s, true, true, 80,
+          false, CURRENT_DATE, %(now)s, ''
+        )
+        ON CONFLICT (email) DO UPDATE
+        SET password = EXCLUDED.password,
+            enabled = true,
+            updated_at = EXCLUDED.updated_at
+        """,
+        {
+            "email": email,
+            "localpart": localpart,
+            "domain": domain,
+            "password": hashed,
+            "quota": MAILU_DEFAULT_QUOTA,
+            "display": display_name or localpart,
+            "now": now,
+        },
+    )
+
+
+def main():
+    token = get_kc_token()
+    users = kc_get_users(token)
+    if not users:
+        log("No users found; exiting.")
+        return
+
+    conn = psycopg2.connect(**DB_CONFIG)
+    conn.autocommit = True
+    cursor = conn.cursor(cursor_factory=RealDictCursor)
+
+    for user in users:
+        attrs = user.get("attributes", {}) or {}
+        app_pw_value = attrs.get("mailu_app_password")
+        if isinstance(app_pw_value, list):
+            app_pw = app_pw_value[0] if app_pw_value else None
+        elif isinstance(app_pw_value, str):
+            app_pw = app_pw_value
+        else:
+            app_pw = None
+
+        email = user.get("email")
+        if not email:
+            email = f"{user['username']}@{MAILU_DOMAIN}"
+
+        if not app_pw:
+            app_pw = random_password()
+            attrs["mailu_app_password"] = app_pw
+            kc_update_attributes(token, user, attrs)
+            log(f"Set mailu_app_password for {email}")
+
+        display_name = " ".join(
+            part for part in [user.get("firstName"), user.get("lastName")] if part
+        ).strip()
+
+        ensure_mailu_user(cursor, email, app_pw, display_name)
+        log(f"Synced mailbox for {email}")
+
+    cursor.close()
+    conn.close()
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as exc:
+        log(f"ERROR: {exc}")
+        sys.exit(1)

scripts/tests/test_mailu_sync.py

@@ -0,0 +1,83 @@
+import importlib.util
+import pathlib
+
+import pytest
+
+
+def load_sync_module(monkeypatch):
+    # Minimal env required by module import
+    env = {
+        "KEYCLOAK_BASE_URL": "http://keycloak",
+        "KEYCLOAK_REALM": "atlas",
+        "KEYCLOAK_CLIENT_ID": "mailu-sync",
+        "KEYCLOAK_CLIENT_SECRET": "secret",
+        "MAILU_DOMAIN": "example.com",
+        "MAILU_DB_HOST": "localhost",
+        "MAILU_DB_PORT": "5432",
+        "MAILU_DB_NAME": "mailu",
+        "MAILU_DB_USER": "mailu",
+        "MAILU_DB_PASSWORD": "pw",
+    }
+    for k, v in env.items():
+        monkeypatch.setenv(k, v)
+    module_path = pathlib.Path(__file__).resolve().parents[1] / "mailu_sync.py"
+    spec = importlib.util.spec_from_file_location("mailu_sync_testmod", module_path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_random_password_length_and_charset(monkeypatch):
+    sync = load_sync_module(monkeypatch)
+    pw = sync.random_password()
+    assert len(pw) == 24
+    assert all(ch.isalnum() for ch in pw)
+
+
+class _FakeResponse:
+    def __init__(self, json_data=None, status=200):
+        self._json_data = json_data or {}
+        self.status_code = status
+
+    def raise_for_status(self):
+        if self.status_code >= 400:
+            raise AssertionError(f"status {self.status_code}")
+
+    def json(self):
+        return self._json_data
+
+
+class _FakeSession:
+    def __init__(self, put_resp, get_resp):
+        self.put_resp = put_resp
+        self.get_resp = get_resp
+        self.put_called = False
+        self.get_called = False
+
+    def post(self, *args, **kwargs):
+        return _FakeResponse({"access_token": "dummy"})
+
+    def put(self, *args, **kwargs):
+        self.put_called = True
+        return self.put_resp
+
+    def get(self, *args, **kwargs):
+        self.get_called = True
+        return self.get_resp
+
+
+def test_kc_update_attributes_succeeds(monkeypatch):
+    sync = load_sync_module(monkeypatch)
+    ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}})
+    sync.SESSION = _FakeSession(_FakeResponse({}), ok_resp)
+    sync.kc_update_attributes("token", {"id": "u1", "username": "u1"}, {"mailu_app_password": "abc"})
+    assert sync.SESSION.put_called and sync.SESSION.get_called
+
+
+def test_kc_update_attributes_raises_without_attribute(monkeypatch):
+    sync = load_sync_module(monkeypatch)
+    missing_attr_resp = _FakeResponse({"attributes": {}}, status=200)
+    sync.SESSION = _FakeSession(_FakeResponse({}), missing_attr_resp)
+    with pytest.raises(Exception):
+        sync.kc_update_attributes("token", {"id": "u1", "username": "u1"}, {"mailu_app_password": "abc"})

services/mailu/helmrelease.yaml

@@ -16,9 +16,13 @@ spec:
         namespace: flux-system
   install:
     remediation: { retries: 3 }
+    timeout: 10m
   upgrade:
-    remediation: { retries: 3 }
+    remediation:
+      retries: 3
+      remediateLastFailure: true
     cleanupOnFail: true
+    timeout: 10m
   values:
     mailuVersion: "2024.06"
     domain: bstein.dev
@@ -59,13 +63,15 @@ spec:
       hostPort:
         enabled: false
       https:
-        enabled: true
+        enabled: false
-        external: true
+        external: false
         forceHttps: false
       externalService:
         enabled: true
         type: LoadBalancer
         externalTrafficPolicy: Cluster
+        ports:
+          submission: true
         nodePorts:
           pop3: 30010
           pop3s: 30011
@@ -82,6 +88,20 @@ spec:
       logLevel: DEBUG
       nodeSelector:
         hardware: rpi4
+      podLivenessProbe:
+        enabled: true
+        initialDelaySeconds: 30
+        periodSeconds: 10
+        timeoutSeconds: 5
+        failureThreshold: 6
+        successThreshold: 1
+      podReadinessProbe:
+        enabled: true
+        initialDelaySeconds: 20
+        periodSeconds: 10
+        timeoutSeconds: 5
+        failureThreshold: 6
+        successThreshold: 1
       extraEnvVars:
         - name: FLASK_DEBUG
           value: "1"
@@ -102,13 +122,28 @@ spec:
         - name: unbound-run
           emptyDir: {}
       extraVolumeMounts:
-        - name: unbound-config
-          mountPath: /etc/unbound
         - name: unbound-run
           mountPath: /var/lib/unbound
       extraContainers:
         - name: unbound
-          image: docker.io/mvance/unbound:1.22.0
+          image: docker.io/alpine:3.20
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              while :; do
+                printf "nameserver 10.43.0.10\n" > /etc/resolv.conf
+                if apk add --no-cache unbound bind-tools; then
+                  break
+                fi
+                echo "apk failed, retrying" >&2
+                sleep 10
+              done
+              cat >/etc/resolv.conf <<'EOF'
+              search mailu-mailserver.svc.cluster.local svc.cluster.local cluster.local
+              nameserver 127.0.0.1
+              EOF
+              unbound-anchor -a /var/lib/unbound/root.key || true
+              exec unbound -d -c /opt/unbound/etc/unbound/unbound.conf
           ports:
             - containerPort: 53
               protocol: UDP
@@ -148,8 +183,8 @@ spec:
       architecture: standalone
       logLevel: DEBUG
       image:
-        repository: bitnami/redis
+        repository: bitnamilegacy/redis
-        tag: 7.2.4-debian-12-r6
+        tag: 8.0.3-debian-12-r3
       master:
         nodeSelector:
           hardware: rpi4
@@ -178,12 +213,14 @@ spec:
       nodeSelector:
         hardware: rpi4
     ingress:
-      enabled: true
+      enabled: false
       ingressClassName: traefik
       tls: true
       existingSecret: mailu-certificates
       annotations:
         traefik.ingress.kubernetes.io/router.entrypoints: websecure
+        traefik.ingress.kubernetes.io/service.serversscheme: https
+        traefik.ingress.kubernetes.io/service.serverstransport: mailu-transport@kubernetescrd
       extraRules:
         - host: mail.bstein.dev
           http:

services/mailu/ingressroute.yaml

@@ -0,0 +1,19 @@
+# services/mailu/ingressroute.yaml
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: mailu
+  namespace: mailu-mailserver
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`mail.bstein.dev`)
+      kind: Rule
+      services:
+        - name: mailu-front
+          port: 443
+          scheme: https
+          serversTransport: mailu-transport
+  tls:
+    secretName: mailu-certificates

services/mailu/kustomization.yaml

@@ -8,3 +8,14 @@ resources:
   - certificate.yaml
   - vip-controller.yaml
   - unbound-configmap.yaml
+  - serverstransport.yaml
+  - ingressroute.yaml
+  - mailu-sync-job.yaml
+
+configMapGenerator:
+  - name: mailu-sync-script
+    namespace: mailu-mailserver
+    files:
+      - sync.py=../../scripts/mailu_sync.py
+    options:
+      disableNameSuffixHash: true

services/mailu/mailu-sync-job.yaml

@@ -0,0 +1,73 @@
+# services/mailu/mailu-sync-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mailu-sync
+  namespace: mailu-mailserver
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: mailu-sync
+          image: python:3.11-alpine
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              pip install --no-cache-dir requests psycopg2-binary passlib >/tmp/pip.log \
+              && python /app/sync.py
+          env:
+            - name: KEYCLOAK_BASE_URL
+              value: http://keycloak.sso.svc.cluster.local
+            - name: KEYCLOAK_REALM
+              value: atlas
+            - name: MAILU_DOMAIN
+              value: bstein.dev
+            - name: MAILU_DEFAULT_QUOTA
+              value: "1000000000"
+            - name: MAILU_DB_HOST
+              value: postgres-service.postgres.svc.cluster.local
+            - name: MAILU_DB_PORT
+              value: "5432"
+            - name: MAILU_DB_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: mailu-db-secret
+                  key: database
+            - name: MAILU_DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: mailu-db-secret
+                  key: username
+            - name: MAILU_DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mailu-db-secret
+                  key: password
+            - name: KEYCLOAK_CLIENT_ID
+              valueFrom:
+                secretKeyRef:
+                  name: mailu-sync-credentials
+                  key: client-id
+            - name: KEYCLOAK_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: mailu-sync-credentials
+                  key: client-secret
+          volumeMounts:
+            - name: sync-script
+              mountPath: /app/sync.py
+              subPath: sync.py
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              cpu: 200m
+              memory: 256Mi
+      volumes:
+        - name: sync-script
+          configMap:
+            name: mailu-sync-script
+            defaultMode: 0444

services/mailu/serverstransport.yaml

@@ -0,0 +1,10 @@
+# services/mailu/serverstransport.yaml
+apiVersion: traefik.io/v1alpha1
+kind: ServersTransport
+metadata:
+  name: mailu-transport
+  namespace: mailu-mailserver
+spec:
+  # Force SNI to mail.bstein.dev and skip backend cert verification (backend cert is for the host, not the pod IP).
+  serverName: mail.bstein.dev
+  insecureSkipVerify: true

services/mailu/unbound-configmap.yaml

@@ -18,6 +18,9 @@ data:
       qname-minimisation: yes
       harden-dnssec-stripped: yes
       val-clean-additional: yes
+      domain-insecure: "mailu-mailserver.svc.cluster.local."
+      domain-insecure: "svc.cluster.local."
+      domain-insecure: "cluster.local."
       cache-min-ttl: 120
       cache-max-ttl: 86400
       access-control: 0.0.0.0/0 allow