portal: retry vaultwarden cred sync

services/bstein-dev-home/scripts/vaultwarden_cred_sync.py

@@ -26,14 +26,22 @@ def _iter_keycloak_users(page_size: int = 200) -> Iterable[dict[str, Any]]:
     url = f"{settings.KEYCLOAK_ADMIN_URL}/admin/realms/{settings.KEYCLOAK_REALM}/users"
     first = 0
     while True:
-        headers = client.headers()
+        headers = _headers_with_retry(client)
         # We need attributes for idempotency (vaultwarden_status/vaultwarden_email). Keycloak defaults to a
         # brief representation which may omit these.
         params = {"first": str(first), "max": str(page_size), "briefRepresentation": "false"}
-        with httpx.Client(timeout=settings.HTTP_CHECK_TIMEOUT_SEC) as http:
-            resp = http.get(url, params=params, headers=headers)
-            resp.raise_for_status()
-            payload = resp.json()
+        payload = None
+        for attempt in range(1, 6):
+            try:
+                with httpx.Client(timeout=settings.HTTP_CHECK_TIMEOUT_SEC) as http:
+                    resp = http.get(url, params=params, headers=headers)
+                    resp.raise_for_status()
+                    payload = resp.json()
+                break
+            except httpx.HTTPError as exc:
+                if attempt == 5:
+                    raise
+                time.sleep(attempt * 2)
 
         if not isinstance(payload, list) or not payload:
             return
@@ -47,6 +55,19 @@ def _iter_keycloak_users(page_size: int = 200) -> Iterable[dict[str, Any]]:
         first += page_size
 
 
+def _headers_with_retry(client, attempts: int = 6) -> dict[str, str]:
+    last_exc: Exception | None = None
+    for attempt in range(1, attempts + 1):
+        try:
+            return client.headers()
+        except Exception as exc:
+            last_exc = exc
+            time.sleep(attempt * 2)
+    if last_exc:
+        raise last_exc
+    raise RuntimeError("failed to fetch keycloak headers")
+
+
 def _extract_attr(attrs: Any, key: str) -> str:
     if not isinstance(attrs, dict):
         return ""