comms: retry mas local users and rerun

services/comms/mas-local-users-ensure-job.yaml

@@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: mas-local-users-ensure-13
+  name: mas-local-users-ensure-14
   namespace: comms
 spec:
   backoffLimit: 1
@@ -120,6 +120,16 @@ spec:
                           time.sleep(attempt * 2)
                   raise RuntimeError(f"MAS service not reachable: {last}")
 
+              def request_with_retry(method, url, attempts=6, **kwargs):
+                  last = None
+                  for attempt in range(1, attempts + 1):
+                      try:
+                          return requests.request(method, url, **kwargs)
+                      except requests.RequestException as exc:
+                          last = exc
+                          time.sleep(attempt * 2)
+                  raise RuntimeError(f"request failed for {url}: {last}")
+
               def admin_token():
                   with open(MAS_ADMIN_CLIENT_SECRET_FILE, "r", encoding="utf-8") as f:
                       secret = f.read().strip()
@@ -141,7 +151,8 @@ spec:
                   raise RuntimeError(f"MAS admin token request failed: {last}")
 
               def get_user(token, username):
-                  r = requests.get(
+                  r = request_with_retry(
+                      "GET",
                       f"{MAS_ADMIN_API_BASE}/users/by-username/{urllib.parse.quote(username)}",
                       headers={"Authorization": f"Bearer {token}"},
                       timeout=30,
@@ -165,7 +176,8 @@ spec:
                       {"username": username, "password": password},
                   ]
                   for payload in payloads:
-                      r = requests.post(
+                      r = request_with_retry(
+                          "POST",
                           f"{MAS_ADMIN_API_BASE}/users",
                           headers={"Authorization": f"Bearer {token}"},
                           json=payload,
@@ -178,7 +190,8 @@ spec:
                   return None
 
               def update_password(token, user_id, password):
-                  r = requests.post(
+                  r = request_with_retry(
+                      "POST",
                       f"{MAS_ADMIN_API_BASE}/users/{urllib.parse.quote(user_id)}/set-password",
                       headers={"Authorization": f"Bearer {token}"},
                       json={"password": password},
@@ -197,7 +210,8 @@ spec:
                   login_name = username
                   if not login_name.startswith("@"):
                       login_name = f"@{login_name}:{SERVER_NAME}"
-                  r = requests.post(
+                  r = request_with_retry(
+                      "POST",
                       f"{AUTH_BASE}/_matrix/client/v3/login",
                       json={
                           "type": "m.login.password",