diff --git a/services/hermes/chat-statefulset.yaml b/services/hermes/chat-statefulset.yaml index ca65c62d..c9010ea9 100644 --- a/services/hermes/chat-statefulset.yaml +++ b/services/hermes/chat-statefulset.yaml @@ -65,10 +65,11 @@ spec: values: ["true"] - key: node-role.kubernetes.io/storage-backbone operator: DoesNotExist - # Verified chat workers while other Pi runtimes recover. + # Healthy verified workers; titan-12 is the rpi4 fallback while + # titan-08 is excluded after repeated containerd failures. - key: kubernetes.io/hostname operator: In - values: [titan-08, titan-12] + values: [titan-06, titan-07, titan-11, titan-12] - key: kubernetes.io/hostname operator: NotIn # titan-04's kubelet stalled before handing the Hermes image pull to CRI. diff --git a/testing/tests/test_hermes_chat_local_runtime.py b/testing/tests/test_hermes_chat_local_runtime.py index ddb0ef40..be2fbcb3 100644 --- a/testing/tests/test_hermes_chat_local_runtime.py +++ b/testing/tests/test_hermes_chat_local_runtime.py @@ -37,6 +37,45 @@ def test_titan20_serializes_classifier_and_local_chat_model_residency(): assert models["persistentVolumeClaim"]["claimName"] == ("ollama-models-titan20") +def test_chat_tenants_use_healthy_workers_with_rpi4_fallback(): + """Keep chat placement off the failed titan-08 runtime while retaining fallback.""" + statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] + affinity = statefulset["spec"]["template"]["spec"]["affinity"] + required = affinity["nodeAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"] + expressions = required["nodeSelectorTerms"][0]["matchExpressions"] + allowed = next( + item + for item in expressions + if item["key"] == "kubernetes.io/hostname" and item["operator"] == "In" + ) + assert allowed["values"] == [ + "titan-06", + "titan-07", + "titan-11", + "titan-12", + ] + assert "titan-08" not in allowed["values"] + excluded = next( + item + for item in expressions + if item["key"] == "kubernetes.io/hostname" and item["operator"] == "NotIn" + ) + assert { + "titan-04", + "titan-05", + "titan-13", + "titan-14", + "titan-17", + "titan-18", + "titan-19", + } == set(excluded["values"]) + spread = affinity["podAntiAffinity"]["preferredDuringSchedulingIgnoredDuringExecution"] + assert spread[0]["podAffinityTerm"]["topologyKey"] == "kubernetes.io/hostname" + assert spread[0]["podAffinityTerm"]["labelSelector"]["matchLabels"] == { + "app": "hermes-chat-tenant" + } + + def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch): """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX.""" source = ROOT / "dockerfiles" / "hermes-local-image-server.py" @@ -121,7 +160,11 @@ def test_chat_auth_and_relay_are_pod_lifetime_only(): for name in ("hermes", "webui"): container = next(item for item in containers if item["name"] == name) - env = {item["name"]: item["value"] for item in container["env"]} + env = { + item["name"]: item["value"] + for item in container["env"] + if "value" in item + } assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json" mount = next( item @@ -134,6 +177,7 @@ def test_chat_auth_and_relay_are_pod_lifetime_only(): hermes_env = { item["name"]: item["value"] for item in next(item for item in containers if item["name"] == "hermes")["env"] + if "value" in item } runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access") assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"}