diff --git a/clusters/atlas/flux-system/gotk-sync.yaml b/clusters/atlas/flux-system/gotk-sync.yaml index 713e739..59cabae 100644 --- a/clusters/atlas/flux-system/gotk-sync.yaml +++ b/clusters/atlas/flux-system/gotk-sync.yaml @@ -8,7 +8,7 @@ metadata: spec: interval: 1m0s ref: - branch: feature/sso-hardening + branch: feature/vault-consumption secretRef: name: flux-system-gitea url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git diff --git a/scripts/test_atlas_user_cleanup.py b/scripts/test_atlas_user_cleanup.py index 41ba708..2acf8a7 100755 --- a/scripts/test_atlas_user_cleanup.py +++ b/scripts/test_atlas_user_cleanup.py @@ -7,6 +7,8 @@ test accounts created via the bstein-dev-home onboarding portal. Targets (best-effort): - Keycloak users in realm "atlas" - Atlas portal Postgres rows (access_requests + dependent tables) + - Mailu mailboxes created for test users + - Nextcloud Mail accounts created for test users - Vaultwarden users/invites created by the portal Safety: @@ -56,6 +58,19 @@ class VaultwardenUser: status: int +@dataclass(frozen=True) +class MailuUser: + email: str + localpart: str + domain: str + + +@dataclass(frozen=True) +class NextcloudMailAccount: + account_id: str + email: str + + def _run(cmd: list[str], *, input_bytes: bytes | None = None) -> str: proc = subprocess.run( cmd, @@ -70,6 +85,19 @@ def _run(cmd: list[str], *, input_bytes: bytes | None = None) -> str: return proc.stdout.decode("utf-8", errors="replace") +def _run_capture(cmd: list[str], *, input_bytes: bytes | None = None) -> tuple[int, str, str]: + proc = subprocess.run( + cmd, + input=input_bytes, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + stdout = proc.stdout.decode("utf-8", errors="replace") + stderr = proc.stderr.decode("utf-8", errors="replace") + return proc.returncode, stdout, stderr + + def _kubectl_get_secret_value(namespace: str, name: str, key: str) -> str: raw_b64 = _run( [ @@ -110,6 +138,21 @@ def _kubectl_first_pod(namespace: str) -> str: return pod_name +def _kubectl_exec(namespace: str, target: str, cmd: list[str]) -> tuple[int, str, str]: + return _run_capture( + [ + "kubectl", + "-n", + namespace, + "exec", + "-i", + target, + "--", + *cmd, + ] + ) + + def _validate_prefixes(prefixes: list[str]) -> list[str]: cleaned: list[str] = [] for prefix in prefixes: @@ -187,6 +230,62 @@ def _keycloak_delete_user(server: str, realm: str, token: str, user_id: str) -> raise +def _sql_quote(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def _psql_exec(db_name: str, sql: str, *, user: str = "postgres") -> str: + postgres_pod = _kubectl_first_pod("postgres") + return _run( + [ + "kubectl", + "-n", + "postgres", + "exec", + "-i", + postgres_pod, + "--", + "psql", + "-U", + user, + "-d", + db_name, + "-c", + sql, + ] + ) + + +def _psql_tsv(db_name: str, sql: str, *, user: str = "postgres") -> list[list[str]]: + postgres_pod = _kubectl_first_pod("postgres") + out = _run( + [ + "kubectl", + "-n", + "postgres", + "exec", + "-i", + postgres_pod, + "--", + "psql", + "-U", + user, + "-d", + db_name, + "-At", + "-F", + "\t", + "-c", + sql, + ] + ) + rows: list[list[str]] = [] + for line in out.splitlines(): + parts = line.split("\t") + rows.append(parts) + return rows + + def _psql_json(portal_db_url: str, sql: str) -> list[dict[str, Any]]: postgres_pod = _kubectl_first_pod("postgres") out = _run( @@ -256,6 +355,89 @@ def _portal_delete_requests(portal_db_url: str, prefixes: list[str]) -> int: return int(match.group(1)) if match else 0 +def _mailu_list_users(prefixes: list[str], domain: str, db_name: str, protected: set[str]) -> list[MailuUser]: + if not prefixes or not domain: + return [] + clauses = " OR ".join([f"localpart LIKE '{p}%'" for p in prefixes]) + sql = ( + 'SELECT email, localpart, domain_name ' + 'FROM "user" ' + f"WHERE domain_name = {_sql_quote(domain)} AND ({clauses}) " + "ORDER BY email;" + ) + rows = _psql_tsv(db_name, sql) + users: list[MailuUser] = [] + for row in rows: + if len(row) < 3: + continue + email = row[0].strip() + if not email or email in protected: + continue + users.append(MailuUser(email=email, localpart=row[1].strip(), domain=row[2].strip())) + return users + + +def _mailu_delete_users(db_name: str, emails: list[str]) -> int: + if not emails: + return 0 + email_list = ",".join(_sql_quote(e) for e in emails) + sql = f'DELETE FROM "user" WHERE email IN ({email_list});' + out = _psql_exec(db_name, sql) + match = re.search(r"DELETE\\s+(\\d+)", out) + return int(match.group(1)) if match else 0 + + +_NEXTCLOUD_ACCOUNT_RE = re.compile(r"^Account\\s+(\\d+):") +_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+") + + +def _nextcloud_exec(cmd: list[str]) -> tuple[int, str, str]: + namespace = os.getenv("NEXTCLOUD_NAMESPACE", "nextcloud").strip() or "nextcloud" + target = os.getenv("NEXTCLOUD_EXEC_TARGET", "deploy/nextcloud").strip() or "deploy/nextcloud" + return _kubectl_exec(namespace, target, cmd) + + +def _parse_nextcloud_mail_accounts(export_output: str) -> list[NextcloudMailAccount]: + accounts: list[NextcloudMailAccount] = [] + current_id = "" + for line in export_output.splitlines(): + line = line.strip() + if not line: + continue + match = _NEXTCLOUD_ACCOUNT_RE.match(line) + if match: + current_id = match.group(1) + continue + if not current_id or "@" not in line: + continue + email_match = _EMAIL_RE.search(line) + if not email_match: + continue + accounts.append(NextcloudMailAccount(account_id=current_id, email=email_match.group(0))) + current_id = "" + return accounts + + +def _nextcloud_list_mail_accounts(username: str) -> list[NextcloudMailAccount]: + occ_path = os.getenv("NEXTCLOUD_OCC_PATH", "/var/www/html/occ").strip() or "/var/www/html/occ" + rc, out, err = _nextcloud_exec(["php", occ_path, "mail:account:export", username]) + if rc != 0: + message = (err or out).strip() + lowered = message.lower() + if any(token in lowered for token in ("not found", "does not exist", "no such user", "unknown user")): + return [] + raise RuntimeError(f"nextcloud mail export failed for {username}: {message}") + return _parse_nextcloud_mail_accounts(out) + + +def _nextcloud_delete_mail_account(account_id: str) -> None: + occ_path = os.getenv("NEXTCLOUD_OCC_PATH", "/var/www/html/occ").strip() or "/var/www/html/occ" + rc, out, err = _nextcloud_exec(["php", occ_path, "mail:account:delete", "-q", account_id]) + if rc != 0: + message = (err or out).strip() + raise RuntimeError(f"nextcloud mail delete failed for account {account_id}: {message}") + + def _vaultwarden_admin_cookie(admin_token: str, base_url: str) -> str: data = urllib.parse.urlencode({"token": admin_token}).encode("utf-8") req = urllib.request.Request(f"{base_url}/admin", data=data, method="POST") @@ -356,6 +538,8 @@ def main() -> int: ), ) parser.add_argument("--skip-keycloak", action="store_true", help="Skip Keycloak user deletion.") + parser.add_argument("--skip-mailu", action="store_true", help="Skip Mailu mailbox cleanup.") + parser.add_argument("--skip-nextcloud-mail", action="store_true", help="Skip Nextcloud Mail account cleanup.") parser.add_argument("--skip-portal-db", action="store_true", help="Skip portal DB cleanup.") parser.add_argument("--skip-vaultwarden", action="store_true", help="Skip Vaultwarden cleanup.") parser.add_argument( @@ -364,6 +548,18 @@ def main() -> int: default=[], help="Keycloak usernames that must never be deleted (repeatable).", ) + parser.add_argument( + "--protect-mailu-email", + action="append", + default=[], + help="Mailu emails that must never be deleted (repeatable).", + ) + parser.add_argument( + "--protect-nextcloud-username", + action="append", + default=[], + help="Nextcloud usernames that must never be touched (repeatable).", + ) parser.add_argument( "--protect-vaultwarden-email", action="append", @@ -376,7 +572,11 @@ def main() -> int: apply = bool(args.apply) expected_confirm = ",".join(prefixes) protected_keycloak = {"bstein", "robotuser", *[u.strip() for u in args.protect_keycloak_username if u.strip()]} + protected_mailu = {e.strip() for e in args.protect_mailu_email if e.strip()} + protected_nextcloud = {u.strip() for u in args.protect_nextcloud_username if u.strip()} protected_vaultwarden = {e.strip() for e in args.protect_vaultwarden_email if e.strip()} + mailu_domain = os.getenv("MAILU_DOMAIN", "bstein.dev").strip() or "bstein.dev" + mailu_db_name = os.getenv("MAILU_DB_NAME", "mailu").strip() or "mailu" if apply and args.confirm != expected_confirm: raise SystemExit( @@ -388,23 +588,29 @@ def main() -> int: print("mode:", "APPLY (destructive)" if apply else "DRY RUN (no changes)") if protected_keycloak: print("protected keycloak usernames:", ", ".join(sorted(protected_keycloak))) + if protected_mailu: + print("protected mailu emails:", ", ".join(sorted(protected_mailu))) + if protected_nextcloud: + print("protected nextcloud usernames:", ", ".join(sorted(protected_nextcloud))) if protected_vaultwarden: print("protected vaultwarden emails:", ", ".join(sorted(protected_vaultwarden))) print() + portal_requests: list[PortalRequestRow] = [] if not args.skip_portal_db: portal_db_url = _kubectl_get_secret_value("bstein-dev-home", "atlas-portal-db", "PORTAL_DATABASE_URL") - requests = _portal_list_requests(portal_db_url, prefixes) - print(f"Portal DB: {len(requests)} access_requests matched") - for row in requests[:50]: + portal_requests = _portal_list_requests(portal_db_url, prefixes) + print(f"Portal DB: {len(portal_requests)} access_requests matched") + for row in portal_requests[:50]: print(f" {row.request_code}\t{row.status}\t{row.username}") - if len(requests) > 50: - print(f" ... and {len(requests) - 50} more") - if apply and requests: + if len(portal_requests) > 50: + print(f" ... and {len(portal_requests) - 50} more") + if apply and portal_requests: deleted = _portal_delete_requests(portal_db_url, prefixes) print(f"Portal DB: deleted {deleted} access_requests (cascade removes tasks/steps/artifacts).") print() + keycloak_users: list[KeycloakUser] = [] if not args.skip_keycloak: kc_server = os.getenv("KEYCLOAK_PUBLIC_URL", "https://sso.bstein.dev").rstrip("/") kc_realm = os.getenv("KEYCLOAK_REALM", "atlas") @@ -421,18 +627,63 @@ def main() -> int: if user.username in protected_keycloak: continue found[user.user_id] = user - users = list(found.values()) - users.sort(key=lambda u: u.username) - print(f"Keycloak: {len(users)} users matched") - for user in users[:50]: + keycloak_users = list(found.values()) + keycloak_users.sort(key=lambda u: u.username) + print(f"Keycloak: {len(keycloak_users)} users matched") + for user in keycloak_users[:50]: email = user.email or "-" print(f" {user.username}\t{email}\t{user.user_id}") - if len(users) > 50: - print(f" ... and {len(users) - 50} more") - if apply and users: - for user in users: + if len(keycloak_users) > 50: + print(f" ... and {len(keycloak_users) - 50} more") + if apply and keycloak_users: + for user in keycloak_users: _keycloak_delete_user(kc_server, kc_realm, token, user.user_id) - print(f"Keycloak: deleted {len(users)} users.") + print(f"Keycloak: deleted {len(keycloak_users)} users.") + print() + + if not args.skip_mailu: + mailu_users = _mailu_list_users(prefixes, mailu_domain, mailu_db_name, protected_mailu) + print(f"Mailu: {len(mailu_users)} mailboxes matched (domain={mailu_domain})") + for user in mailu_users[:50]: + print(f" {user.email}\t{user.localpart}\t{user.domain}") + if len(mailu_users) > 50: + print(f" ... and {len(mailu_users) - 50} more") + if apply and mailu_users: + deleted = _mailu_delete_users(mailu_db_name, [u.email for u in mailu_users]) + print(f"Mailu: deleted {deleted} mailboxes.") + print() + + if not args.skip_nextcloud_mail: + nextcloud_usernames = {row.username for row in portal_requests if row.username} + nextcloud_usernames.update({u.username for u in keycloak_users if u.username}) + nextcloud_usernames = {u for u in nextcloud_usernames if _starts_with_any(u, prefixes)} + nextcloud_usernames = {u for u in nextcloud_usernames if u not in protected_nextcloud} + + matches: list[tuple[str, NextcloudMailAccount]] = [] + for username in sorted(nextcloud_usernames): + accounts = _nextcloud_list_mail_accounts(username) + for account in accounts: + email = account.email.strip() + if not email: + continue + if not email.lower().endswith(f"@{mailu_domain.lower()}"): + continue + localpart = email.split("@", 1)[0] + if not _starts_with_any(localpart, prefixes): + continue + if email in protected_mailu: + continue + matches.append((username, account)) + + print(f"Nextcloud Mail: {len(matches)} accounts matched") + for username, account in matches[:50]: + print(f" {username}\t{account.account_id}\t{account.email}") + if len(matches) > 50: + print(f" ... and {len(matches) - 50} more") + if apply and matches: + for _, account in matches: + _nextcloud_delete_mail_account(account.account_id) + print(f"Nextcloud Mail: deleted {len(matches)} accounts.") print() if not args.skip_vaultwarden: