From 391a7f2f1ff96fc47d5821f41c3271aee619ce61 Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 25 Aug 2026 20:17:28 -0300 Subject: [PATCH] hermes(agent): make titan-22 the strong primary home Now that both the agent image (a68d1c4d, via the kustomize images: override) and the hux sidecar (build-39) are multi-arch with amd64 leaves, move the worker onto the amd64 accelerator titan-22: - Add an OR'd nodeSelectorTerm for amd64 + node-role.kubernetes.io/accelerator + hostname titan-22, with NO worker=true requirement. Keep the arm64 pi-fleet term as an OR'd fallback so the worker is never stranded. - Strong primary preference: hostname=titan-22 at weight 100 (scheduler max), pi-fleet rpi5 nudge lowered to 50, so hermes actually lives on titan-22. - Tolerate node-role.kubernetes.io/accelerator=true:NoSchedule (harmless where absent) and the soft atlas.bstein.dev/media-primary:PreferNoSchedule that titan-22 currently carries, so the weight-100 preference is not offset and placement is deterministic. Completes the titan-22 effort the flip branches missed; the earlier branches never repointed to multi-arch images, which is why the worker never landed here. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf --- services/hermes/agent-deployment.yaml | 44 ++++++++++++++++++++++- testing/tests/test_hermes_agent_layout.py | 38 ++++++++++++++++++-- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index ddfcc0d8..d139d1f8 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -24,7 +24,7 @@ spec: ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback - ai.bstein.dev/placement: titan-08 rpi5; storage-backbone nodes excluded + ai.bstein.dev/placement: primary amd64 accelerator titan-22; arm64 rpi5 fleet fallback; storage-backbone nodes excluded ai.bstein.dev/config-rev: "20260825-claude-quota-expiry" prometheus.io/scrape: "true" prometheus.io/path: /metrics @@ -97,10 +97,16 @@ spec: securityContext: seccompProfile: type: RuntimeDefault + # titan-22 is the STRONG/primary home for hermes-agent now that both the + # agent image (a68d1c4d, via the kustomize images: override) and the hux + # sidecar (build-39) are multi-arch. The arm64 pi fleet stays as an OR'd + # fallback so the worker is never stranded if titan-22 is unavailable. affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: + # Fallback set: the arm64 pi fleet (unchanged). nodeSelectorTerms + # are OR'd, so hermes-agent still runs here if titan-22 is gone. - matchExpressions: - key: kubernetes.io/arch operator: In @@ -111,13 +117,49 @@ spec: - key: kubernetes.io/hostname operator: NotIn values: [titan-04, titan-06, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19] + # Primary amd64 home: accelerator titan-22. No worker=true here. + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: [amd64] + - key: node-role.kubernetes.io/accelerator + operator: In + values: ["true"] + - key: kubernetes.io/hostname + operator: In + values: [titan-22] preferredDuringSchedulingIgnoredDuringExecution: + # STRONG primary preference: titan-22 (weight 100, the scheduler + # maximum) outranks the pi-fleet nudge below, so hermes-agent + # actually lives on titan-22 whenever it is schedulable. - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: [titan-22] + # Weaker fallback nudge within the pi fleet. + - weight: 50 preference: matchExpressions: - key: hardware operator: In values: [rpi5] + tolerations: + # Permit (do not force) scheduling onto the hard-tainted accelerator + # titan-22. Harmless where the taint is absent; only lets hermes-agent + # consider titan-22. Does not change jellyfin's media-core priority. + - key: node-role.kubernetes.io/accelerator + operator: Equal + value: "true" + effect: NoSchedule + # titan-22 currently carries the soft media-primary taint (media node); + # tolerate it so the weight-100 titan-22 preference is not offset by the + # PreferNoSchedule penalty and placement on titan-22 is deterministic. + - key: atlas.bstein.dev/media-primary + operator: Equal + value: "true" + effect: PreferNoSchedule initContainers: - name: init-config image: busybox:1.37 diff --git a/testing/tests/test_hermes_agent_layout.py b/testing/tests/test_hermes_agent_layout.py index 3c343e67..4fe3edeb 100644 --- a/testing/tests/test_hermes_agent_layout.py +++ b/testing/tests/test_hermes_agent_layout.py @@ -154,12 +154,46 @@ def test_agent_avoids_unhealthy_nodes_and_stays_on_storage_workers(): assert hostnames["operator"] == "NotIn" assert set(hostnames["values"]) >= {"titan-04", "titan-19"} - assert len(terms) == 1 + # Two OR'd terms: the arm64 pi fleet (fallback) and the amd64 accelerator + # titan-22 (primary home once the multi-arch image exists). + assert len(terms) == 2 + pi_fleet = {item["key"]: item for item in terms[0]["matchExpressions"]} + assert pi_fleet["kubernetes.io/arch"]["values"] == ["arm64"] + assert pi_fleet["node-role.kubernetes.io/worker"]["values"] == ["true"] + + titan22 = {item["key"]: item for item in terms[1]["matchExpressions"]} + assert titan22["kubernetes.io/arch"]["values"] == ["amd64"] + assert titan22["kubernetes.io/hostname"]["values"] == ["titan-22"] + assert titan22["node-role.kubernetes.io/accelerator"]["values"] == ["true"] + # titan-22 access must NOT depend on the generic worker label. + assert "node-role.kubernetes.io/worker" not in titan22 preferences = pod["affinity"]["nodeAffinity"][ "preferredDuringSchedulingIgnoredDuringExecution" ] - assert [item["weight"] for item in preferences] == [100] + # titan-22 is the STRONG primary (max weight 100) and outranks the pi-fleet + # nudge (50), so hermes-agent actually lives on titan-22 when schedulable. + assert [item["weight"] for item in preferences] == [100, 50] + primary = preferences[0]["preference"]["matchExpressions"][0] + assert primary["key"] == "kubernetes.io/hostname" + assert primary["values"] == ["titan-22"] + + # The pod tolerates the accelerator hard taint so it can consider titan-22. + assert { + "key": "node-role.kubernetes.io/accelerator", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule", + } in pod["tolerations"] + # It also tolerates the soft media-primary taint that titan-22 currently + # carries, so the weight-100 titan-22 preference is not offset by the + # PreferNoSchedule penalty and placement there is deterministic. + assert { + "key": "atlas.bstein.dev/media-primary", + "operator": "Equal", + "value": "true", + "effect": "PreferNoSchedule", + } in pod["tolerations"] hermes = next(item for item in pod["containers"] if item["name"] == "hermes") assert hermes["resources"]["requests"] == {