keycloak: retry token exchange permissions job

services/keycloak/portal-e2e-token-exchange-permissions-job.yaml

@@ -2,10 +2,10 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: keycloak-portal-e2e-token-exchange-permissions-1
+  name: keycloak-portal-e2e-token-exchange-permissions-2
   namespace: sso
 spec:
-  backoffLimit: 0
+  backoffLimit: 6
   template:
     spec:
       restartPolicy: Never
@@ -39,6 +39,7 @@ spec:
               import json
               import os
               import re
+              import time
               import urllib.parse
               import urllib.error
               import urllib.request
@@ -79,6 +80,7 @@ spec:
                           return exc.code, {"raw": raw.decode(errors="replace")}
 
               def get_admin_token() -> str:
+                  last_error: str | None = None
                   token_data = urllib.parse.urlencode(
                       {
                           "grant_type": "password",
@@ -93,13 +95,24 @@ spec:
                       headers={"Content-Type": "application/x-www-form-urlencoded"},
                       method="POST",
                   )
-                  try:
-                      with urllib.request.urlopen(req, timeout=15) as resp:
-                          body = json.loads(resp.read().decode())
-                  except urllib.error.HTTPError as exc:
-                      raw = exc.read().decode(errors="replace")
-                      raise SystemExit(f"Token request failed: status={exc.code} body={raw}")
-                  return body["access_token"]
+                  for attempt in range(1, 61):
+                      try:
+                          with urllib.request.urlopen(req, timeout=15) as resp:
+                              body = json.loads(resp.read().decode())
+                          token = body.get("access_token")
+                          if isinstance(token, str) and token:
+                              return token
+                          last_error = "missing access_token"
+                      except urllib.error.HTTPError as exc:
+                          # Treat transient startup errors as retryable.
+                          if exc.code in (404, 429, 500, 502, 503, 504):
+                              last_error = f"http {exc.code}"
+                          else:
+                              raise SystemExit(f"Token request failed: status={exc.code}")
+                      except urllib.error.URLError as exc:
+                          last_error = str(exc.reason)
+                      time.sleep(2)
+                  raise SystemExit(f"Token request failed after retries: {last_error}")
 
               def find_client_uuid(token: str, client_id: str) -> str:
                   status, clients = http_json(