From dfa50b755bccbf3952ea0ec8b4da1acc6654c422 Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 25 Aug 2026 11:08:43 -0300 Subject: [PATCH] APPLY ONLY AFTER multi-arch image validated: allow hermes-agent on titan-22 Runtime affinity flip for the amd64 target node titan-22. Do NOT merge/apply until the multi-arch hermes-agent image has been built and validated (both arch leaves + promoted index), per docs/hermes_agent_multiarch.md step (d). - Add a second nodeSelectorTerm (OR'd) matching ONLY amd64 titan-22 (worker). The arm64 pi-fleet term is untouched, so the worker still runs on the pi fleet if titan-22 is unavailable. - Add a soft, equal-weight preference toward titan-22 so it is used as spare capacity, not forced. - Tolerate titan-22's atlas.bstein.dev/media-primary=true:PreferNoSchedule taint. This only lets hermes-agent also consider titan-22; it does not change jellyfin's scheduling or priority. Updates test_hermes_agent_layout.py to match the two-term topology, the added preference, and the toleration. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf --- services/hermes/agent-deployment.yaml | 33 +++++++++++++++++++- testing/tests/test_hermes_agent_layout.py | 38 +++++++++++++++++++++-- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 9fce2b2c..d38b4762 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -24,7 +24,7 @@ spec: ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback - ai.bstein.dev/placement: titan-08 rpi5; storage-backbone nodes excluded + ai.bstein.dev/placement: rpi5 fleet or amd64 titan-22 (spare); storage-backbone nodes excluded ai.bstein.dev/config-rev: "20260824-claude-fable-quota" prometheus.io/scrape: "true" prometheus.io/path: /metrics @@ -101,6 +101,9 @@ spec: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: + # Fallback set: the arm64 pi fleet (unchanged). nodeSelectorTerms + # are OR'd, so the worker can still run here if titan-22 is + # unavailable. - matchExpressions: - key: kubernetes.io/arch operator: In @@ -111,6 +114,17 @@ spec: - key: kubernetes.io/hostname operator: NotIn values: [titan-04, titan-06, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19] + # amd64 target enabled by the multi-arch image: titan-22 only. + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: [amd64] + - key: node-role.kubernetes.io/worker + operator: In + values: ["true"] + - key: kubernetes.io/hostname + operator: In + values: [titan-22] preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 preference: @@ -118,6 +132,23 @@ spec: - key: hardware operator: In values: [rpi5] + # Soft nudge toward titan-22 as amd64 spare capacity; the pi fleet + # stays equally preferred so titan-22 is used, not forced. + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: [titan-22] + tolerations: + # Permit (do not force) scheduling onto titan-22, tainted media-primary + # for jellyfin. A toleration only lets hermes-agent also consider + # titan-22 as spare; it does not change jellyfin's scheduling or + # priority. + - key: atlas.bstein.dev/media-primary + operator: Equal + value: "true" + effect: PreferNoSchedule initContainers: - name: init-config image: busybox:1.37 diff --git a/testing/tests/test_hermes_agent_layout.py b/testing/tests/test_hermes_agent_layout.py index 3c343e67..9ac4fdfa 100644 --- a/testing/tests/test_hermes_agent_layout.py +++ b/testing/tests/test_hermes_agent_layout.py @@ -154,12 +154,46 @@ def test_agent_avoids_unhealthy_nodes_and_stays_on_storage_workers(): assert hostnames["operator"] == "NotIn" assert set(hostnames["values"]) >= {"titan-04", "titan-19"} - assert len(terms) == 1 + # Multi-arch: the arm64 fleet term is preserved as a fallback (OR'd), plus a + # second term that permits ONLY amd64 titan-22 as spare capacity. + assert len(terms) == 2 + arch_values = { + expr["values"][0] + for term in terms + for expr in term["matchExpressions"] + if expr["key"] == "kubernetes.io/arch" + } + assert arch_values == {"arm64", "amd64"} + titan22_term = next( + term + for term in terms + if any( + expr["key"] == "kubernetes.io/hostname" and expr["operator"] == "In" + for expr in term["matchExpressions"] + ) + ) + titan22_hosts = next( + expr + for expr in titan22_term["matchExpressions"] + if expr["key"] == "kubernetes.io/hostname" + ) + assert titan22_hosts["values"] == ["titan-22"] preferences = pod["affinity"]["nodeAffinity"][ "preferredDuringSchedulingIgnoredDuringExecution" ] - assert [item["weight"] for item in preferences] == [100] + # rpi5 nudge preserved; titan-22 nudge added at equal weight (not forced). + assert [item["weight"] for item in preferences] == [100, 100] + + # hermes-agent may use titan-22 despite the media-primary taint, without + # changing jellyfin's own scheduling. + tolerations = pod.get("tolerations", []) + assert { + "key": "atlas.bstein.dev/media-primary", + "operator": "Equal", + "value": "true", + "effect": "PreferNoSchedule", + } in tolerations hermes = next(item for item in pod["containers"] if item["name"] == "hermes") assert hermes["resources"]["requests"] == {