From 8f7b57419a72bc6e2726d7a21a4d23b9fb267774 Mon Sep 17 00:00:00 2001
From: jenkins <jenkins@bstein.dev>
Date: Sun, 16 Aug 2026 07:12:11 -0300
Subject: [PATCH] hermes: accept routed base64 vision

---
 services/hermes/agent-deployment.yaml     |  2 +-
 services/hermes/scripts/codex_broker.py   | 39 +++++++++++++++++++----
 testing/tests/test_hermes_chat_quality.py | 23 +++++++++++++
 3 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml
index 8eeaabb7..d3440491 100644
--- a/services/hermes/agent-deployment.yaml
+++ b/services/hermes/agent-deployment.yaml
@@ -25,7 +25,7 @@ spec:
         ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
         ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
         ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
-        ai.bstein.dev/config-rev: "20260816-routed-vision-v2"
+        ai.bstein.dev/config-rev: "20260816-routed-vision-v3"
         prometheus.io/scrape: "true"
         prometheus.io/path: /metrics
         prometheus.io/port: "9010"
diff --git a/services/hermes/scripts/codex_broker.py b/services/hermes/scripts/codex_broker.py
index 2320acb5..bc9c1f02 100644
--- a/services/hermes/scripts/codex_broker.py
+++ b/services/hermes/scripts/codex_broker.py
@@ -223,6 +223,34 @@ def _upstream_headers(token: str) -> dict[str, str]:
 
 def _normalize_input_images(response_input: list[Any]) -> None:
     """Normalize Chat-Completions image parts for the Codex Responses API."""
+
+    def image_url_value(part: dict[str, Any]) -> tuple[str | None, str | None]:
+        """Return a Responses URL from URL-object or base64-source forms."""
+        raw = part.get("image_url")
+        if isinstance(raw, str):
+            return raw, None
+        candidates = [raw, part.get("source")]
+        for candidate in candidates:
+            if not isinstance(candidate, dict):
+                continue
+            detail = candidate.get("detail")
+            url = candidate.get("url")
+            if isinstance(url, str) and url.strip():
+                return url, detail if isinstance(detail, str) else None
+            data = candidate.get("data")
+            media_type = candidate.get("media_type") or candidate.get("mime_type")
+            if (
+                isinstance(data, str)
+                and data
+                and isinstance(media_type, str)
+                and media_type.startswith("image/")
+                and all(character not in media_type for character in "\r\n;, ")
+            ):
+                return f"data:{media_type};base64,{data}", (
+                    detail if isinstance(detail, str) else None
+                )
+        return None, None
+
     for item in response_input:
         if not isinstance(item, dict):
             continue
@@ -231,20 +259,19 @@ def _normalize_input_images(response_input: list[Any]) -> None:
             continue
         for part in content:
             if not isinstance(part, dict) or part.get("type") not in {
+                "image",
                 "image_url",
                 "input_image",
             }:
                 continue
-            image_url = part.get("image_url")
-            if isinstance(image_url, dict):
-                detail = image_url.get("detail")
-                image_url = image_url.get("url")
-                if isinstance(detail, str) and detail and "detail" not in part:
-                    part["detail"] = detail
+            image_url, detail = image_url_value(part)
+            if isinstance(detail, str) and detail and "detail" not in part:
+                part["detail"] = detail
             if not isinstance(image_url, str) or not image_url.strip():
                 raise ValueError("non-empty Responses image URL required")
             part["type"] = "input_image"
             part["image_url"] = image_url
+            part.pop("source", None)
 
 
 def _validate_payload(payload: Any) -> dict[str, Any]:
diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py
index 1cfbf12d..78421c4b 100644
--- a/testing/tests/test_hermes_chat_quality.py
+++ b/testing/tests/test_hermes_chat_quality.py
@@ -801,6 +801,29 @@ def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch):
         "image_url": "data:image/png;base64,cHJpdmF0ZQ==",
         "detail": "auto",
     }
+    switchyard_base64_items = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "input_text", "text": "What color is this?"},
+                {
+                    "type": "image",
+                    "source": {
+                        "type": "base64",
+                        "media_type": "image/png",
+                        "data": "cHJpdmF0ZQ==",
+                    },
+                },
+            ],
+        }
+    ]
+    normalized_base64 = module._validate_payload(
+        {"model": "gpt-5.6-terra", "input": switchyard_base64_items}
+    )["input"][0]["content"][1]
+    assert normalized_base64 == {
+        "type": "input_image",
+        "image_url": "data:image/png;base64,cHJpdmF0ZQ==",
+    }
     with pytest.raises(ValueError, match="non-empty Responses image URL"):
         module._validate_payload(
             {