From 8a8df5ee4fd7ad1fc42d7676c5e857068f5bf416 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sat, 22 Aug 2026 15:32:21 -0300 Subject: [PATCH] hermes: add stateful accelerator fallback --- .../core/node-prefer-noschedule-cronjob.yaml | 8 ++++ services/hermes/agent-deployment.yaml | 26 ++++++++++++- testing/tests/test_hermes_agent_layout.py | 38 ++++++++++++------- 3 files changed, 57 insertions(+), 15 deletions(-) diff --git a/infrastructure/core/node-prefer-noschedule-cronjob.yaml b/infrastructure/core/node-prefer-noschedule-cronjob.yaml index f3313243..6e82d7f8 100644 --- a/infrastructure/core/node-prefer-noschedule-cronjob.yaml +++ b/infrastructure/core/node-prefer-noschedule-cronjob.yaml @@ -81,6 +81,14 @@ spec: echo "skipping missing node titan-23" fi + if k get node titan-21 >/dev/null 2>&1; then + # Keep replicas off the Jetson, but allow Longhorn to attach + # stateful fallback workloads there. + k label node titan-21 longhorn-host=true --overwrite=true || true + else + echo "skipping missing node titan-21" + fi + for node in titan-13 titan-15 titan-17 titan-19; do if k get node "${node}" >/dev/null 2>&1; then k label node "${node}" \ diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index a263d355..e572b4cb 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -24,7 +24,7 @@ spec: ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback - ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available + ai.bstein.dev/placement: rpi5 preferred; rpi4 next; titan-21 is the stateful fallback ai.bstein.dev/config-rev: "20260818-ai-usage-exporter-split-v3" prometheus.io/scrape: "true" prometheus.io/path: /metrics @@ -107,6 +107,18 @@ spec: - key: kubernetes.io/hostname operator: NotIn values: [titan-04, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19] + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: [arm64] + - key: node-role.kubernetes.io/accelerator + operator: Exists + - key: kubernetes.io/hostname + operator: In + values: [titan-21] + - key: longhorn-host + operator: In + values: ["true"] preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 preference: @@ -114,6 +126,18 @@ spec: - key: hardware operator: In values: [rpi5] + - weight: 50 + preference: + matchExpressions: + - key: hardware + operator: In + values: [rpi4] + - weight: 25 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: [titan-21] initContainers: - name: init-config image: busybox:1.37 diff --git a/testing/tests/test_hermes_agent_layout.py b/testing/tests/test_hermes_agent_layout.py index 36cdff8e..1e945b95 100644 --- a/testing/tests/test_hermes_agent_layout.py +++ b/testing/tests/test_hermes_agent_layout.py @@ -134,34 +134,44 @@ def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth(): assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2" assert lane["resources"] == { - "requests": {"cpu": "100m", "memory": "256Mi"}, + "requests": {"cpu": "50m", "memory": "192Mi"}, "limits": {"cpu": "2", "memory": "6Gi"}, } -def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity(): - """Placement correction: keep the agent off nodes that cannot hold it. - - titan-04 is cordoned after repeated kernel undervoltage and kubelet - failure, and titan-19 was probe/Longhorn unstable under worker load, so - both must join the existing hard exclusions. That leaves titan-05 as the - healthy candidate, which is tight enough on requested CPU that the main - container has to give back 50m to schedule there. - """ +def test_agent_avoids_unhealthy_nodes_and_has_a_stateful_fallback(): + """The owner agent keeps a bounded fallback when Pi workers are full.""" pod = _agent_deployment()["spec"]["template"]["spec"] + terms = pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"] hostnames = next( item - for item in pod["affinity"]["nodeAffinity"][ - "requiredDuringSchedulingIgnoredDuringExecution" - ]["nodeSelectorTerms"][0]["matchExpressions"] + for item in terms[0]["matchExpressions"] if item["key"] == "kubernetes.io/hostname" ) assert hostnames["operator"] == "NotIn" assert set(hostnames["values"]) >= {"titan-04", "titan-19"} + fallback = {item["key"]: item for item in terms[1]["matchExpressions"]} + assert fallback["node-role.kubernetes.io/accelerator"]["operator"] == "Exists" + assert fallback["kubernetes.io/hostname"]["values"] == ["titan-21"] + assert fallback["longhorn-host"]["values"] == ["true"] + + preferences = pod["affinity"]["nodeAffinity"][ + "preferredDuringSchedulingIgnoredDuringExecution" + ] + assert [item["weight"] for item in preferences] == [100, 50, 25] + hermes = next(item for item in pod["containers"] if item["name"] == "hermes") - assert hermes["resources"]["requests"]["cpu"] == "300m" + assert hermes["resources"]["requests"]["cpu"] == "200m" + + node_labels = ( + Path(__file__).parents[2] + / "infrastructure/core/node-prefer-noschedule-cronjob.yaml" + ).read_text() + assert "k label node titan-21 longhorn-host=true" in node_labels def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path():