hermes: move chat tenants off the failing node runtime

services/hermes/chat-statefulset.yaml

@@ -65,10 +65,11 @@ spec:
                     values: ["true"]
                   - key: node-role.kubernetes.io/storage-backbone
                     operator: DoesNotExist
-                  # Verified chat workers while other Pi runtimes recover.
+                  # Healthy verified workers; titan-12 is the rpi4 fallback while
+                  # titan-08 is excluded after repeated containerd failures.
                   - key: kubernetes.io/hostname
                     operator: In
-                    values: [titan-08, titan-12]
+                    values: [titan-06, titan-07, titan-11, titan-12]
                   - key: kubernetes.io/hostname
                     operator: NotIn
                     # titan-04's kubelet stalled before handing the Hermes image pull to CRI.

testing/tests/test_hermes_chat_local_runtime.py

@@ -37,6 +37,45 @@ def test_titan20_serializes_classifier_and_local_chat_model_residency():
     assert models["persistentVolumeClaim"]["claimName"] == ("ollama-models-titan20")
 
 
+def test_chat_tenants_use_healthy_workers_with_rpi4_fallback():
+    """Keep chat placement off the failed titan-08 runtime while retaining fallback."""
+    statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
+    affinity = statefulset["spec"]["template"]["spec"]["affinity"]
+    required = affinity["nodeAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"]
+    expressions = required["nodeSelectorTerms"][0]["matchExpressions"]
+    allowed = next(
+        item
+        for item in expressions
+        if item["key"] == "kubernetes.io/hostname" and item["operator"] == "In"
+    )
+    assert allowed["values"] == [
+        "titan-06",
+        "titan-07",
+        "titan-11",
+        "titan-12",
+    ]
+    assert "titan-08" not in allowed["values"]
+    excluded = next(
+        item
+        for item in expressions
+        if item["key"] == "kubernetes.io/hostname" and item["operator"] == "NotIn"
+    )
+    assert {
+        "titan-04",
+        "titan-05",
+        "titan-13",
+        "titan-14",
+        "titan-17",
+        "titan-18",
+        "titan-19",
+    } == set(excluded["values"])
+    spread = affinity["podAntiAffinity"]["preferredDuringSchedulingIgnoredDuringExecution"]
+    assert spread[0]["podAffinityTerm"]["topologyKey"] == "kubernetes.io/hostname"
+    assert spread[0]["podAffinityTerm"]["labelSelector"]["matchLabels"] == {
+        "app": "hermes-chat-tenant"
+    }
+
+
 def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch):
     """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX."""
     source = ROOT / "dockerfiles" / "hermes-local-image-server.py"
@@ -121,7 +160,11 @@ def test_chat_auth_and_relay_are_pod_lifetime_only():
 
     for name in ("hermes", "webui"):
         container = next(item for item in containers if item["name"] == name)
-        env = {item["name"]: item["value"] for item in container["env"]}
+        env = {
+            item["name"]: item["value"]
+            for item in container["env"]
+            if "value" in item
+        }
         assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json"
         mount = next(
             item
@@ -134,6 +177,7 @@ def test_chat_auth_and_relay_are_pod_lifetime_only():
     hermes_env = {
         item["name"]: item["value"]
         for item in next(item for item in containers if item["name"] == "hermes")["env"]
+        if "value" in item
     }
     runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access")
     assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"}