portal: log and retry ariadne calls

backend/atlas_portal/ariadne_client.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import logging
+import time
 from typing import Any
 
 import httpx
@@ -7,6 +9,8 @@ from flask import jsonify, request
 
 from . import settings
 
+logger = logging.getLogger(__name__)
+
 
 class AriadneError(Exception):
     def __init__(self, message: str, status_code: int = 502) -> None:
@@ -39,17 +43,38 @@ def request_raw(
     if not enabled():
         raise AriadneError("ariadne not configured", 503)
 
+    url = _url(path)
+    attempts = max(1, settings.ARIADNE_RETRY_COUNT)
+    for attempt in range(1, attempts + 1):
         try:
             with httpx.Client(timeout=settings.ARIADNE_TIMEOUT_SEC) as client:
-            return client.request(
+                resp = client.request(
                     method,
-                _url(path),
+                    url,
                     headers=_auth_headers(),
                     json=payload,
                     params=params,
                 )
+            if resp.status_code >= 500:
+                logger.warning(
+                    "ariadne error response",
+                    extra={"method": method, "path": path, "status": resp.status_code},
+                )
+            return resp
         except httpx.RequestError as exc:
+            logger.warning(
+                "ariadne request failed",
+                extra={
+                    "method": method,
+                    "path": path,
+                    "attempt": attempt,
+                    "timeout_sec": settings.ARIADNE_TIMEOUT_SEC,
+                    "error": str(exc),
+                },
+            )
+            if attempt >= attempts:
                 raise AriadneError("ariadne unavailable", 502) from exc
+            time.sleep(settings.ARIADNE_RETRY_BACKOFF_SEC * attempt)
 
 
 def proxy(

backend/atlas_portal/settings.py

@@ -50,6 +50,8 @@ KEYCLOAK_ADMIN_REALM = os.getenv("KEYCLOAK_ADMIN_REALM", KEYCLOAK_REALM)
 
 ARIADNE_URL = os.getenv("ARIADNE_URL", "").strip()
 ARIADNE_TIMEOUT_SEC = float(os.getenv("ARIADNE_TIMEOUT_SEC", "10"))
+ARIADNE_RETRY_COUNT = int(os.getenv("ARIADNE_RETRY_COUNT", "2"))
+ARIADNE_RETRY_BACKOFF_SEC = float(os.getenv("ARIADNE_RETRY_BACKOFF_SEC", "0.2"))
 
 ACCOUNT_ALLOWED_GROUPS = [
     g.strip()