test: stabilize portal onboarding e2e

scripts/tests/test_portal_onboarding_flow.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import http.client
 import json
 import os
 import sys
@@ -110,11 +111,19 @@ def main() -> int:
     username = f"{username_prefix}-{now}"
     email = f"{username}@example.invalid"
 
-    submit = _post_json(
-        f"{portal_base}/api/access/request",
-        {"username": username, "email": email, "note": "portal onboarding e2e"},
-        timeout_s=20,
-    )
+    submit_url = f"{portal_base}/api/access/request"
+    submit_payload = {"username": username, "email": email, "note": "portal onboarding e2e"}
+    submit = None
+    for attempt in range(1, 6):
+        try:
+            submit = _post_json(submit_url, submit_payload, timeout_s=20)
+            break
+        except (http.client.RemoteDisconnected, TimeoutError, urllib.error.URLError) as exc:
+            if attempt == 5:
+                raise SystemExit(f"portal submit failed after {attempt} attempts: {exc}")
+            time.sleep(2)
+    if not isinstance(submit, dict):
+        raise SystemExit("portal submit did not return json")
 
     request_code = submit.get("request_code")
     if not isinstance(request_code, str) or not request_code:
@@ -150,8 +159,17 @@ def main() -> int:
     deadline_at = time.monotonic() + deadline_s
 
     last_status = None
+    last_error = None
     while True:
-        status_payload = _post_json(status_url, {"request_code": request_code}, timeout_s=60)
+        try:
+            status_payload = _post_json(status_url, {"request_code": request_code}, timeout_s=60)
+            last_error = None
+        except (http.client.RemoteDisconnected, TimeoutError, urllib.error.URLError) as exc:
+            last_error = str(exc)
+            if time.monotonic() >= deadline_at:
+                raise SystemExit(f"timed out waiting for provisioning to finish (last error={last_error})")
+            time.sleep(interval_s)
+            continue
         status = status_payload.get("status")
         if isinstance(status, str):
             last_status = status
@@ -161,7 +179,8 @@ def main() -> int:
         if status in ("denied", "unknown"):
             raise SystemExit(f"request transitioned to unexpected terminal status: {status_payload}")
         if time.monotonic() >= deadline_at:
-            raise SystemExit(f"timed out waiting for provisioning to finish (last status={last_status})")
+            suffix = f" (last error={last_error})" if last_error else ""
+            raise SystemExit(f"timed out waiting for provisioning to finish (last status={last_status}){suffix}")
         time.sleep(interval_s)
 
     token = _keycloak_admin_token(keycloak_base, realm, kc_admin_client_id, kc_admin_client_secret)
@@ -188,4 +207,3 @@ def main() -> int:
 
 if __name__ == "__main__":
     sys.exit(main())
-

services/bstein-dev-home/portal-onboarding-e2e-test-job.yaml

@@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: portal-onboarding-e2e-test-2
+  name: portal-onboarding-e2e-test-3
   namespace: bstein-dev-home
 spec:
   backoffLimit: 0