ops: prepare vault-consumption branch

clusters/atlas/flux-system/gotk-sync.yaml

@@ -8,7 +8,7 @@ metadata:
 spec:
   interval: 1m0s
   ref:
-    branch: feature/sso-hardening
+    branch: feature/vault-consumption
   secretRef:
     name: flux-system-gitea
   url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git

scripts/test_atlas_user_cleanup.py

@@ -7,6 +7,8 @@ test accounts created via the bstein-dev-home onboarding portal.
 Targets (best-effort):
   - Keycloak users in realm "atlas"
   - Atlas portal Postgres rows (access_requests + dependent tables)
+  - Mailu mailboxes created for test users
+  - Nextcloud Mail accounts created for test users
   - Vaultwarden users/invites created by the portal
 
 Safety:
@@ -56,6 +58,19 @@ class VaultwardenUser:
     status: int
 
 
+@dataclass(frozen=True)
+class MailuUser:
+    email: str
+    localpart: str
+    domain: str
+
+
+@dataclass(frozen=True)
+class NextcloudMailAccount:
+    account_id: str
+    email: str
+
+
 def _run(cmd: list[str], *, input_bytes: bytes | None = None) -> str:
     proc = subprocess.run(
         cmd,
@@ -70,6 +85,19 @@ def _run(cmd: list[str], *, input_bytes: bytes | None = None) -> str:
     return proc.stdout.decode("utf-8", errors="replace")
 
 
+def _run_capture(cmd: list[str], *, input_bytes: bytes | None = None) -> tuple[int, str, str]:
+    proc = subprocess.run(
+        cmd,
+        input=input_bytes,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        check=False,
+    )
+    stdout = proc.stdout.decode("utf-8", errors="replace")
+    stderr = proc.stderr.decode("utf-8", errors="replace")
+    return proc.returncode, stdout, stderr
+
+
 def _kubectl_get_secret_value(namespace: str, name: str, key: str) -> str:
     raw_b64 = _run(
         [
@@ -110,6 +138,21 @@ def _kubectl_first_pod(namespace: str) -> str:
     return pod_name
 
 
+def _kubectl_exec(namespace: str, target: str, cmd: list[str]) -> tuple[int, str, str]:
+    return _run_capture(
+        [
+            "kubectl",
+            "-n",
+            namespace,
+            "exec",
+            "-i",
+            target,
+            "--",
+            *cmd,
+        ]
+    )
+
+
 def _validate_prefixes(prefixes: list[str]) -> list[str]:
     cleaned: list[str] = []
     for prefix in prefixes:
@@ -187,6 +230,62 @@ def _keycloak_delete_user(server: str, realm: str, token: str, user_id: str) ->
         raise
 
 
+def _sql_quote(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+def _psql_exec(db_name: str, sql: str, *, user: str = "postgres") -> str:
+    postgres_pod = _kubectl_first_pod("postgres")
+    return _run(
+        [
+            "kubectl",
+            "-n",
+            "postgres",
+            "exec",
+            "-i",
+            postgres_pod,
+            "--",
+            "psql",
+            "-U",
+            user,
+            "-d",
+            db_name,
+            "-c",
+            sql,
+        ]
+    )
+
+
+def _psql_tsv(db_name: str, sql: str, *, user: str = "postgres") -> list[list[str]]:
+    postgres_pod = _kubectl_first_pod("postgres")
+    out = _run(
+        [
+            "kubectl",
+            "-n",
+            "postgres",
+            "exec",
+            "-i",
+            postgres_pod,
+            "--",
+            "psql",
+            "-U",
+            user,
+            "-d",
+            db_name,
+            "-At",
+            "-F",
+            "\t",
+            "-c",
+            sql,
+        ]
+    )
+    rows: list[list[str]] = []
+    for line in out.splitlines():
+        parts = line.split("\t")
+        rows.append(parts)
+    return rows
+
+
 def _psql_json(portal_db_url: str, sql: str) -> list[dict[str, Any]]:
     postgres_pod = _kubectl_first_pod("postgres")
     out = _run(
@@ -256,6 +355,89 @@ def _portal_delete_requests(portal_db_url: str, prefixes: list[str]) -> int:
     return int(match.group(1)) if match else 0
 
 
+def _mailu_list_users(prefixes: list[str], domain: str, db_name: str, protected: set[str]) -> list[MailuUser]:
+    if not prefixes or not domain:
+        return []
+    clauses = " OR ".join([f"localpart LIKE '{p}%'" for p in prefixes])
+    sql = (
+        'SELECT email, localpart, domain_name '
+        'FROM "user" '
+        f"WHERE domain_name = {_sql_quote(domain)} AND ({clauses}) "
+        "ORDER BY email;"
+    )
+    rows = _psql_tsv(db_name, sql)
+    users: list[MailuUser] = []
+    for row in rows:
+        if len(row) < 3:
+            continue
+        email = row[0].strip()
+        if not email or email in protected:
+            continue
+        users.append(MailuUser(email=email, localpart=row[1].strip(), domain=row[2].strip()))
+    return users
+
+
+def _mailu_delete_users(db_name: str, emails: list[str]) -> int:
+    if not emails:
+        return 0
+    email_list = ",".join(_sql_quote(e) for e in emails)
+    sql = f'DELETE FROM "user" WHERE email IN ({email_list});'
+    out = _psql_exec(db_name, sql)
+    match = re.search(r"DELETE\\s+(\\d+)", out)
+    return int(match.group(1)) if match else 0
+
+
+_NEXTCLOUD_ACCOUNT_RE = re.compile(r"^Account\\s+(\\d+):")
+_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+")
+
+
+def _nextcloud_exec(cmd: list[str]) -> tuple[int, str, str]:
+    namespace = os.getenv("NEXTCLOUD_NAMESPACE", "nextcloud").strip() or "nextcloud"
+    target = os.getenv("NEXTCLOUD_EXEC_TARGET", "deploy/nextcloud").strip() or "deploy/nextcloud"
+    return _kubectl_exec(namespace, target, cmd)
+
+
+def _parse_nextcloud_mail_accounts(export_output: str) -> list[NextcloudMailAccount]:
+    accounts: list[NextcloudMailAccount] = []
+    current_id = ""
+    for line in export_output.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        match = _NEXTCLOUD_ACCOUNT_RE.match(line)
+        if match:
+            current_id = match.group(1)
+            continue
+        if not current_id or "@" not in line:
+            continue
+        email_match = _EMAIL_RE.search(line)
+        if not email_match:
+            continue
+        accounts.append(NextcloudMailAccount(account_id=current_id, email=email_match.group(0)))
+        current_id = ""
+    return accounts
+
+
+def _nextcloud_list_mail_accounts(username: str) -> list[NextcloudMailAccount]:
+    occ_path = os.getenv("NEXTCLOUD_OCC_PATH", "/var/www/html/occ").strip() or "/var/www/html/occ"
+    rc, out, err = _nextcloud_exec(["php", occ_path, "mail:account:export", username])
+    if rc != 0:
+        message = (err or out).strip()
+        lowered = message.lower()
+        if any(token in lowered for token in ("not found", "does not exist", "no such user", "unknown user")):
+            return []
+        raise RuntimeError(f"nextcloud mail export failed for {username}: {message}")
+    return _parse_nextcloud_mail_accounts(out)
+
+
+def _nextcloud_delete_mail_account(account_id: str) -> None:
+    occ_path = os.getenv("NEXTCLOUD_OCC_PATH", "/var/www/html/occ").strip() or "/var/www/html/occ"
+    rc, out, err = _nextcloud_exec(["php", occ_path, "mail:account:delete", "-q", account_id])
+    if rc != 0:
+        message = (err or out).strip()
+        raise RuntimeError(f"nextcloud mail delete failed for account {account_id}: {message}")
+
+
 def _vaultwarden_admin_cookie(admin_token: str, base_url: str) -> str:
     data = urllib.parse.urlencode({"token": admin_token}).encode("utf-8")
     req = urllib.request.Request(f"{base_url}/admin", data=data, method="POST")
@@ -356,6 +538,8 @@ def main() -> int:
         ),
     )
     parser.add_argument("--skip-keycloak", action="store_true", help="Skip Keycloak user deletion.")
+    parser.add_argument("--skip-mailu", action="store_true", help="Skip Mailu mailbox cleanup.")
+    parser.add_argument("--skip-nextcloud-mail", action="store_true", help="Skip Nextcloud Mail account cleanup.")
     parser.add_argument("--skip-portal-db", action="store_true", help="Skip portal DB cleanup.")
     parser.add_argument("--skip-vaultwarden", action="store_true", help="Skip Vaultwarden cleanup.")
     parser.add_argument(
@@ -364,6 +548,18 @@ def main() -> int:
         default=[],
         help="Keycloak usernames that must never be deleted (repeatable).",
     )
+    parser.add_argument(
+        "--protect-mailu-email",
+        action="append",
+        default=[],
+        help="Mailu emails that must never be deleted (repeatable).",
+    )
+    parser.add_argument(
+        "--protect-nextcloud-username",
+        action="append",
+        default=[],
+        help="Nextcloud usernames that must never be touched (repeatable).",
+    )
     parser.add_argument(
         "--protect-vaultwarden-email",
         action="append",
@@ -376,7 +572,11 @@ def main() -> int:
     apply = bool(args.apply)
     expected_confirm = ",".join(prefixes)
     protected_keycloak = {"bstein", "robotuser", *[u.strip() for u in args.protect_keycloak_username if u.strip()]}
+    protected_mailu = {e.strip() for e in args.protect_mailu_email if e.strip()}
+    protected_nextcloud = {u.strip() for u in args.protect_nextcloud_username if u.strip()}
     protected_vaultwarden = {e.strip() for e in args.protect_vaultwarden_email if e.strip()}
+    mailu_domain = os.getenv("MAILU_DOMAIN", "bstein.dev").strip() or "bstein.dev"
+    mailu_db_name = os.getenv("MAILU_DB_NAME", "mailu").strip() or "mailu"
 
     if apply and args.confirm != expected_confirm:
         raise SystemExit(
@@ -388,23 +588,29 @@ def main() -> int:
     print("mode:", "APPLY (destructive)" if apply else "DRY RUN (no changes)")
     if protected_keycloak:
         print("protected keycloak usernames:", ", ".join(sorted(protected_keycloak)))
+    if protected_mailu:
+        print("protected mailu emails:", ", ".join(sorted(protected_mailu)))
+    if protected_nextcloud:
+        print("protected nextcloud usernames:", ", ".join(sorted(protected_nextcloud)))
     if protected_vaultwarden:
         print("protected vaultwarden emails:", ", ".join(sorted(protected_vaultwarden)))
     print()
 
+    portal_requests: list[PortalRequestRow] = []
     if not args.skip_portal_db:
         portal_db_url = _kubectl_get_secret_value("bstein-dev-home", "atlas-portal-db", "PORTAL_DATABASE_URL")
-        requests = _portal_list_requests(portal_db_url, prefixes)
-        print(f"Portal DB: {len(requests)} access_requests matched")
-        for row in requests[:50]:
+        portal_requests = _portal_list_requests(portal_db_url, prefixes)
+        print(f"Portal DB: {len(portal_requests)} access_requests matched")
+        for row in portal_requests[:50]:
             print(f"  {row.request_code}\t{row.status}\t{row.username}")
-        if len(requests) > 50:
-            print(f"  ... and {len(requests) - 50} more")
-        if apply and requests:
+        if len(portal_requests) > 50:
+            print(f"  ... and {len(portal_requests) - 50} more")
+        if apply and portal_requests:
             deleted = _portal_delete_requests(portal_db_url, prefixes)
             print(f"Portal DB: deleted {deleted} access_requests (cascade removes tasks/steps/artifacts).")
         print()
 
+    keycloak_users: list[KeycloakUser] = []
     if not args.skip_keycloak:
         kc_server = os.getenv("KEYCLOAK_PUBLIC_URL", "https://sso.bstein.dev").rstrip("/")
         kc_realm = os.getenv("KEYCLOAK_REALM", "atlas")
@@ -421,18 +627,63 @@ def main() -> int:
                 if user.username in protected_keycloak:
                     continue
                 found[user.user_id] = user
-        users = list(found.values())
-        users.sort(key=lambda u: u.username)
-        print(f"Keycloak: {len(users)} users matched")
-        for user in users[:50]:
+        keycloak_users = list(found.values())
+        keycloak_users.sort(key=lambda u: u.username)
+        print(f"Keycloak: {len(keycloak_users)} users matched")
+        for user in keycloak_users[:50]:
             email = user.email or "-"
             print(f"  {user.username}\t{email}\t{user.user_id}")
-        if len(users) > 50:
-            print(f"  ... and {len(users) - 50} more")
-        if apply and users:
-            for user in users:
+        if len(keycloak_users) > 50:
+            print(f"  ... and {len(keycloak_users) - 50} more")
+        if apply and keycloak_users:
+            for user in keycloak_users:
                 _keycloak_delete_user(kc_server, kc_realm, token, user.user_id)
-            print(f"Keycloak: deleted {len(users)} users.")
+            print(f"Keycloak: deleted {len(keycloak_users)} users.")
+        print()
+
+    if not args.skip_mailu:
+        mailu_users = _mailu_list_users(prefixes, mailu_domain, mailu_db_name, protected_mailu)
+        print(f"Mailu: {len(mailu_users)} mailboxes matched (domain={mailu_domain})")
+        for user in mailu_users[:50]:
+            print(f"  {user.email}\t{user.localpart}\t{user.domain}")
+        if len(mailu_users) > 50:
+            print(f"  ... and {len(mailu_users) - 50} more")
+        if apply and mailu_users:
+            deleted = _mailu_delete_users(mailu_db_name, [u.email for u in mailu_users])
+            print(f"Mailu: deleted {deleted} mailboxes.")
+        print()
+
+    if not args.skip_nextcloud_mail:
+        nextcloud_usernames = {row.username for row in portal_requests if row.username}
+        nextcloud_usernames.update({u.username for u in keycloak_users if u.username})
+        nextcloud_usernames = {u for u in nextcloud_usernames if _starts_with_any(u, prefixes)}
+        nextcloud_usernames = {u for u in nextcloud_usernames if u not in protected_nextcloud}
+
+        matches: list[tuple[str, NextcloudMailAccount]] = []
+        for username in sorted(nextcloud_usernames):
+            accounts = _nextcloud_list_mail_accounts(username)
+            for account in accounts:
+                email = account.email.strip()
+                if not email:
+                    continue
+                if not email.lower().endswith(f"@{mailu_domain.lower()}"):
+                    continue
+                localpart = email.split("@", 1)[0]
+                if not _starts_with_any(localpart, prefixes):
+                    continue
+                if email in protected_mailu:
+                    continue
+                matches.append((username, account))
+
+        print(f"Nextcloud Mail: {len(matches)} accounts matched")
+        for username, account in matches[:50]:
+            print(f"  {username}\t{account.account_id}\t{account.email}")
+        if len(matches) > 50:
+            print(f"  ... and {len(matches) - 50} more")
+        if apply and matches:
+            for _, account in matches:
+                _nextcloud_delete_mail_account(account.account_id)
+            print(f"Nextcloud Mail: deleted {len(matches)} accounts.")
         print()
 
     if not args.skip_vaultwarden: