titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

hermes: move chat tenants off the failing node runtime

Browse Source
This commit is contained in:
jenkins 2026-09-13 16:23:55 -05:00
parent 13a7dbab46
commit 4d7087d394
2 changed files with 48 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
services/hermes/chat-statefulset.yaml
View File

@ -65,10 +65,11 @@ spec:
values: ["true"]
- key: node-role.kubernetes.io/storage-backbone
operator: DoesNotExist
# Verified chat workers while other Pi runtimes recover.
# Healthy verified workers; titan-12 is the rpi4 fallback while
# titan-08 is excluded after repeated containerd failures.
- key: kubernetes.io/hostname
operator: In
values: [titan-08, titan-12]
values: [titan-06, titan-07, titan-11, titan-12]
- key: kubernetes.io/hostname
operator: NotIn
# titan-04's kubelet stalled before handing the Hermes image pull to CRI.

46
testing/tests/test_hermes_chat_local_runtime.py
View File

@ -37,6 +37,45 @@ def test_titan20_serializes_classifier_and_local_chat_model_residency():
assert models["persistentVolumeClaim"]["claimName"] == ("ollama-models-titan20")
def test_chat_tenants_use_healthy_workers_with_rpi4_fallback():
"""Keep chat placement off the failed titan-08 runtime while retaining fallback."""
statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
affinity = statefulset["spec"]["template"]["spec"]["affinity"]
required = affinity["nodeAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"]
expressions = required["nodeSelectorTerms"][0]["matchExpressions"]
allowed = next(
item
for item in expressions
if item["key"] == "kubernetes.io/hostname" and item["operator"] == "In"
)
assert allowed["values"] == [
"titan-06",
"titan-07",
"titan-11",
"titan-12",
]
assert "titan-08" not in allowed["values"]
excluded = next(
item
for item in expressions
if item["key"] == "kubernetes.io/hostname" and item["operator"] == "NotIn"
)
assert {
"titan-04",
"titan-05",
"titan-13",
"titan-14",
"titan-17",
"titan-18",
"titan-19",
} == set(excluded["values"])
spread = affinity["podAntiAffinity"]["preferredDuringSchedulingIgnoredDuringExecution"]
assert spread[0]["podAffinityTerm"]["topologyKey"] == "kubernetes.io/hostname"
assert spread[0]["podAffinityTerm"]["labelSelector"]["matchLabels"] == {
"app": "hermes-chat-tenant"
}
def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch):
"""Lease-idle desktop spikes may coexist, but saturation still blocks FLUX."""
source = ROOT / "dockerfiles" / "hermes-local-image-server.py"
@ -121,7 +160,11 @@ def test_chat_auth_and_relay_are_pod_lifetime_only():
for name in ("hermes", "webui"):
container = next(item for item in containers if item["name"] == name)
env = {item["name"]: item["value"] for item in container["env"]}
env = {
item["name"]: item["value"]
for item in container["env"]
if "value" in item
}
assert env["HERMES_AUTH_FILE"] == "/runtime-access/hermes-auth.json"
mount = next(
item
@ -134,6 +177,7 @@ def test_chat_auth_and_relay_are_pod_lifetime_only():
hermes_env = {
item["name"]: item["value"]
for item in next(item for item in containers if item["name"] == "hermes")["env"]
if "value" in item
}
runtime = next(item for item in pod["volumes"] if item["name"] == "runtime-access")
assert runtime["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Mi"}