tests(portal): verify access requests via email

scripts/tests/test_portal_onboarding_flow.py

@@ -1,7 +1,11 @@
 #!/usr/bin/env python3
+import email
 import http.client
+import imaplib
 import json
 import os
+import re
+import ssl
 import sys
 import time
 import urllib.error
@@ -97,6 +101,89 @@ def _keycloak_get_user(keycloak_base: str, realm: str, token: str, user_id: str)
     return data
 
 
+def _extract_attr(attributes: object, key: str) -> str:
+    if not isinstance(attributes, dict):
+        return ""
+    value = attributes.get(key)
+    if isinstance(value, list) and value and isinstance(value[0], str):
+        return value[0]
+    if isinstance(value, str):
+        return value
+    return ""
+
+
+def _imap_wait_for_verify_token(
+    *,
+    host: str,
+    port: int,
+    username: str,
+    password: str,
+    request_code: str,
+    deadline_sec: int,
+) -> str:
+    ssl_context = ssl._create_unverified_context()
+    deadline_at = time.monotonic() + deadline_sec
+
+    with imaplib.IMAP4_SSL(host, port, ssl_context=ssl_context) as client:
+        client.login(username, password)
+        client.select("INBOX")
+
+        while time.monotonic() < deadline_at:
+            status, data = client.search(None, "TEXT", request_code)
+            if status == "OK" and data and data[0]:
+                ids = data[0].split()
+                msg_id = ids[-1]
+                fetch_status, msg_data = client.fetch(msg_id, "(RFC822)")
+                if fetch_status != "OK" or not msg_data:
+                    time.sleep(2)
+                    continue
+
+                raw = msg_data[0][1] if isinstance(msg_data[0], tuple) and len(msg_data[0]) > 1 else None
+                if not isinstance(raw, (bytes, bytearray)):
+                    time.sleep(2)
+                    continue
+
+                message = email.message_from_bytes(raw)
+                body = None
+                if message.is_multipart():
+                    for part in message.walk():
+                        if part.get_content_type() == "text/plain":
+                            payload = part.get_payload(decode=True)
+                            if isinstance(payload, (bytes, bytearray)):
+                                body = payload.decode(errors="replace")
+                                break
+                else:
+                    payload = message.get_payload(decode=True)
+                    if isinstance(payload, (bytes, bytearray)):
+                        body = payload.decode(errors="replace")
+
+                if not body:
+                    time.sleep(2)
+                    continue
+
+                url = None
+                for line in body.splitlines():
+                    candidate = line.strip()
+                    if "verify=" in candidate and candidate.startswith("http"):
+                        url = candidate
+                        break
+                if not url:
+                    match = re.search(r"https?://\\S+verify=\\S+", body)
+                    url = match.group(0) if match else None
+                if not url:
+                    time.sleep(2)
+                    continue
+
+                parsed = urllib.parse.urlparse(url)
+                query = urllib.parse.parse_qs(parsed.query)
+                token = query.get("verify", [""])[0]
+                if isinstance(token, str) and token:
+                    return token
+            time.sleep(2)
+
+    raise SystemExit("verification email not found before deadline")
+
+
 def main() -> int:
     portal_base = _env("PORTAL_BASE_URL").rstrip("/")
     db_url = _env("PORTAL_DATABASE_URL")
@@ -106,13 +193,38 @@ def main() -> int:
     kc_admin_client_id = _env("KEYCLOAK_ADMIN_CLIENT_ID")
     kc_admin_client_secret = _env("KEYCLOAK_ADMIN_CLIENT_SECRET")
 
+    contact_email = os.environ.get("E2E_CONTACT_EMAIL", "robotuser@bstein.dev").strip()
+    if not contact_email:
+        raise SystemExit("E2E_CONTACT_EMAIL must not be empty")
+
+    imap_host = os.environ.get("E2E_IMAP_HOST", "mailu-front.mailu-mailserver.svc.cluster.local").strip()
+    imap_port = int(os.environ.get("E2E_IMAP_PORT", "993"))
+    imap_keycloak_username = os.environ.get("E2E_IMAP_KEYCLOAK_USERNAME", "robotuser").strip()
+    imap_wait_sec = int(os.environ.get("E2E_IMAP_WAIT_SECONDS", "90"))
+
+    token = _keycloak_admin_token(keycloak_base, realm, kc_admin_client_id, kc_admin_client_secret)
+    mailbox_user = _keycloak_find_user(keycloak_base, realm, token, imap_keycloak_username)
+    if not mailbox_user:
+        raise SystemExit(f"unable to locate Keycloak mailbox user {imap_keycloak_username!r}")
+    mailbox_user_id = mailbox_user.get("id")
+    if not isinstance(mailbox_user_id, str) or not mailbox_user_id:
+        raise SystemExit("mailbox user missing id")
+
+    mailbox_full = _keycloak_get_user(keycloak_base, realm, token, mailbox_user_id)
+    mailbox_attrs = mailbox_full.get("attributes")
+    mailu_email = _extract_attr(mailbox_attrs, "mailu_email")
+    if not mailu_email:
+        mailu_email = contact_email
+    mailu_password = _extract_attr(mailbox_attrs, "mailu_app_password")
+    if not mailu_password:
+        raise SystemExit(f"Keycloak user {imap_keycloak_username!r} missing mailu_app_password attribute")
+
     username_prefix = os.environ.get("E2E_USERNAME_PREFIX", "e2e-user")
     now = int(time.time())
     username = f"{username_prefix}-{now}"
-    email = f"{username}@example.invalid"
 
     submit_url = f"{portal_base}/api/access/request"
-    submit_payload = {"username": username, "email": email, "note": "portal onboarding e2e"}
+    submit_payload = {"username": username, "email": contact_email, "note": "portal onboarding e2e"}
     submit = None
     for attempt in range(1, 6):
         try:
@@ -129,18 +241,23 @@ def main() -> int:
     if not isinstance(request_code, str) or not request_code:
         raise SystemExit(f"request submit did not return request_code: {submit}")
 
-    with psycopg.connect(db_url, autocommit=True) as conn:
-        # Bypass the emailed token by marking the request as verified and pending (same as /api/access/request/verify).
-        conn.execute(
-            """
-            UPDATE access_requests
-            SET status = 'pending',
-                email_verified_at = NOW(),
-                email_verification_token_hash = NULL
-            WHERE request_code = %s AND status = 'pending_email_verification'
-            """,
-            (request_code,),
+    verify_token = _imap_wait_for_verify_token(
+        host=imap_host,
+        port=imap_port,
+        username=mailu_email,
+        password=mailu_password,
+        request_code=request_code,
+        deadline_sec=imap_wait_sec,
     )
+    verify_resp = _post_json(
+        f"{portal_base}/api/access/request/verify",
+        {"request_code": request_code, "token": verify_token},
+        timeout_s=30,
+    )
+    if not isinstance(verify_resp, dict) or verify_resp.get("ok") is not True:
+        raise SystemExit(f"unexpected verify response: {verify_resp}")
+
+    with psycopg.connect(db_url, autocommit=True) as conn:
         # Simulate admin approval.
         conn.execute(
             """
@@ -183,7 +300,6 @@ def main() -> int:
             raise SystemExit(f"timed out waiting for provisioning to finish (last status={last_status}){suffix}")
         time.sleep(interval_s)
 
-    token = _keycloak_admin_token(keycloak_base, realm, kc_admin_client_id, kc_admin_client_secret)
     user = _keycloak_find_user(keycloak_base, realm, token, username)
     if not user:
         raise SystemExit("expected Keycloak user was not created")

services/bstein-dev-home/backend-deployment.yaml

@@ -80,6 +80,8 @@ spec:
               value: "120"
             - name: ACCESS_REQUEST_STATUS_RATE_WINDOW_SEC
               value: "60"
+            - name: ACCESS_REQUEST_INTERNAL_EMAIL_ALLOWLIST
+              value: robotuser@bstein.dev
           ports:
             - name: http
               containerPort: 8080

services/bstein-dev-home/portal-onboarding-e2e-test-job.yaml

@@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: portal-onboarding-e2e-test-4
+  name: portal-onboarding-e2e-test-5
   namespace: bstein-dev-home
 spec:
   backoffLimit: 0
@@ -33,6 +33,10 @@ spec:
                   key: client_secret
             - name: E2E_USERNAME_PREFIX
               value: e2e-portal
+            - name: E2E_CONTACT_EMAIL
+              value: robotuser@bstein.dev
+            - name: E2E_IMAP_KEYCLOAK_USERNAME
+              value: robotuser
             - name: E2E_DEADLINE_SECONDS
               value: "600"
             - name: E2E_POLL_SECONDS