portal-e2e: add readiness checks

services/bstein-dev-home/scripts/test_portal_onboarding_flow.py

@@ -65,6 +65,23 @@ def _get_json(url: str, headers: dict[str, str] | None = None, timeout_s: int =
         raise SystemExit(f"HTTP {exc.code} from {url}: {raw}")
 
 
+def _wait_for_portal_ready(base_url: str, timeout_s: int = 60) -> None:
+    health_url = f"{base_url.rstrip('/')}/api/healthz"
+    deadline_at = time.monotonic() + timeout_s
+    last_error = None
+    while time.monotonic() < deadline_at:
+        try:
+            req = urllib.request.Request(health_url, method="GET")
+            with urllib.request.urlopen(req, timeout=10) as resp:
+                if resp.status == 200:
+                    return
+        except Exception as exc:
+            last_error = str(exc)
+        time.sleep(2)
+    suffix = f" (last_error={last_error})" if last_error else ""
+    raise SystemExit(f"portal health check timed out{suffix}")
+
+
 def _request_json(
     method: str,
     url: str,
@@ -235,6 +252,7 @@ def _imap_wait_for_verify_token(
 
 def main() -> int:
     portal_base = _env("PORTAL_BASE_URL").rstrip("/")
+    portal_ready_timeout = int(os.environ.get("E2E_PORTAL_READY_TIMEOUT_SECONDS", "60"))
 
     keycloak_base = _env("KEYCLOAK_ADMIN_URL").rstrip("/")
     realm = _env("KEYCLOAK_REALM", "atlas")
@@ -274,6 +292,8 @@ def main() -> int:
     if not mailu_password:
         raise SystemExit(f"Keycloak user {imap_keycloak_username!r} missing mailu_app_password attribute")
 
+    _wait_for_portal_ready(portal_base, timeout_s=portal_ready_timeout)
+
     username_prefix = os.environ.get("E2E_USERNAME_PREFIX", "e2e-user")
     now = int(time.time())
     username = f"{username_prefix}-{now}"
@@ -336,6 +356,8 @@ def main() -> int:
     except SystemExit as exc:
         raise SystemExit(f"failed to exchange token for portal approval as {portal_admin_username!r}: {exc}")
 
+    _wait_for_portal_ready(portal_base, timeout_s=portal_ready_timeout)
+
     approve_url = f"{portal_base}/api/admin/access/requests/{urllib.parse.quote(username, safe='')}/approve"
     approve_timeout_s = int(os.environ.get("E2E_APPROVE_TIMEOUT_SECONDS", "180"))
     approve_attempts = int(os.environ.get("E2E_APPROVE_ATTEMPTS", "3"))
@@ -348,6 +370,10 @@ def main() -> int:
             break
         except (http.client.RemoteDisconnected, TimeoutError, urllib.error.URLError) as exc:
             approve_error = str(exc)
+            try:
+                _wait_for_portal_ready(portal_base, timeout_s=min(30, portal_ready_timeout))
+            except SystemExit:
+                pass
             if attempt == approve_attempts:
                 break
             time.sleep(3)