From 8b8d2c4aa88a9435bfb63602404a10a5c3c33ac7 Mon Sep 17 00:00:00 2001
From: Brad Stein <Brad.Stein@gmail.com>
Date: Sun, 18 Jan 2026 03:00:24 -0300
Subject: [PATCH] vaultwarden: add retry safeguards and db tuning

---
 .../scripts/vaultwarden_cred_sync.py          | 31 +++++++++++++++++++
 .../vaultwarden-cred-sync-cronjob.yaml        |  6 ++++
 services/vaultwarden/deployment.yaml          | 10 ++++++
 3 files changed, 47 insertions(+)

diff --git a/services/bstein-dev-home/scripts/vaultwarden_cred_sync.py b/services/bstein-dev-home/scripts/vaultwarden_cred_sync.py
index 9ee4eeb..cb4f9c8 100644
--- a/services/bstein-dev-home/scripts/vaultwarden_cred_sync.py
+++ b/services/bstein-dev-home/scripts/vaultwarden_cred_sync.py
@@ -2,8 +2,10 @@
 
 from __future__ import annotations
 
+import os
 import sys
 import time
+from datetime import datetime, timezone
 from typing import Any, Iterable
 
 import httpx
@@ -16,6 +18,8 @@ from atlas_portal.vaultwarden import invite_user
 VAULTWARDEN_EMAIL_ATTR = "vaultwarden_email"
 VAULTWARDEN_STATUS_ATTR = "vaultwarden_status"
 VAULTWARDEN_SYNCED_AT_ATTR = "vaultwarden_synced_at"
+VAULTWARDEN_RETRY_COOLDOWN_SEC = int(os.getenv("VAULTWARDEN_RETRY_COOLDOWN_SEC", "1800"))
+VAULTWARDEN_FAILURE_BAILOUT = int(os.getenv("VAULTWARDEN_FAILURE_BAILOUT", "2"))
 
 
 def _iter_keycloak_users(page_size: int = 200) -> Iterable[dict[str, Any]]:
@@ -82,6 +86,21 @@ def _extract_attr(attrs: Any, key: str) -> str:
     return ""
 
 
+def _parse_synced_at(value: str) -> float | None:
+    value = (value or "").strip()
+    if not value:
+        return None
+    for fmt in ("%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%dT%H:%M:%S%z"):
+        try:
+            parsed = datetime.strptime(value, fmt)
+            if parsed.tzinfo is None:
+                parsed = parsed.replace(tzinfo=timezone.utc)
+            return parsed.timestamp()
+        except ValueError:
+            continue
+    return None
+
+
 def _vaultwarden_email_for_user(user: dict[str, Any]) -> str:
     username = (user.get("username") if isinstance(user.get("username"), str) else "") or ""
     username = username.strip()
@@ -129,6 +148,7 @@ def main() -> int:
     created = 0
     skipped = 0
     failures = 0
+    consecutive_failures = 0
 
     for user in _iter_keycloak_users():
         username = (user.get("username") if isinstance(user.get("username"), str) else "") or ""
@@ -158,6 +178,11 @@ def main() -> int:
 
         current_status = _extract_attr(full_user.get("attributes"), VAULTWARDEN_STATUS_ATTR)
         current_synced_at = _extract_attr(full_user.get("attributes"), VAULTWARDEN_SYNCED_AT_ATTR)
+        current_synced_ts = _parse_synced_at(current_synced_at)
+        if current_status in {"rate_limited", "error"} and current_synced_ts:
+            if time.time() - current_synced_ts < VAULTWARDEN_RETRY_COOLDOWN_SEC:
+                skipped += 1
+                continue
         email = _vaultwarden_email_for_user(full_user)
         if not email:
             print(f"skip {username}: missing email", file=sys.stderr)
@@ -188,6 +213,7 @@ def main() -> int:
         result = invite_user(email)
         if result.ok:
             created += 1
+            consecutive_failures = 0
             print(f"ok {username}: {result.status}")
             try:
                 _set_user_attribute(username, VAULTWARDEN_STATUS_ATTR, result.status)
@@ -196,12 +222,17 @@ def main() -> int:
                 pass
         else:
             failures += 1
+            if result.status in {"rate_limited", "error"}:
+                consecutive_failures += 1
             print(f"err {username}: {result.status} {result.detail}", file=sys.stderr)
             try:
                 _set_user_attribute(username, VAULTWARDEN_STATUS_ATTR, result.status)
                 _set_user_attribute(username, VAULTWARDEN_SYNCED_AT_ATTR, time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
             except Exception:
                 pass
+            if consecutive_failures >= VAULTWARDEN_FAILURE_BAILOUT:
+                print("vaultwarden: too many consecutive failures; aborting run", file=sys.stderr)
+                break
 
     print(
         f"done processed={processed} created_or_present={created} skipped={skipped} failures={failures}",
diff --git a/services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml b/services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml
index 86eeaf1..29141fe 100644
--- a/services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml
+++ b/services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml
@@ -68,6 +68,12 @@ spec:
                   value: bstein-dev-home-admin
                 - name: HTTP_CHECK_TIMEOUT_SEC
                   value: "20"
+                - name: VAULTWARDEN_ADMIN_SESSION_TTL_SEC
+                  value: "900"
+                - name: VAULTWARDEN_RETRY_COOLDOWN_SEC
+                  value: "1800"
+                - name: VAULTWARDEN_FAILURE_BAILOUT
+                  value: "2"
               volumeMounts:
                 - name: vaultwarden-cred-sync-script
                   mountPath: /scripts
diff --git a/services/vaultwarden/deployment.yaml b/services/vaultwarden/deployment.yaml
index 6125ad8..219c402 100644
--- a/services/vaultwarden/deployment.yaml
+++ b/services/vaultwarden/deployment.yaml
@@ -50,6 +50,16 @@ spec:
               value: "true"
             - name: DOMAIN
               value: "https://vault.bstein.dev"
+            - name: DB_CONNECTION_RETRIES
+              value: "0"
+            - name: DATABASE_TIMEOUT
+              value: "60"
+            - name: DATABASE_MIN_CONNS
+              value: "2"
+            - name: DATABASE_MAX_CONNS
+              value: "20"
+            - name: DATABASE_IDLE_TIMEOUT
+              value: "600"
             - name: SMTP_HOST
               value: "mail.bstein.dev"
             - name: SMTP_PORT