hermes: add stateful accelerator fallback

infrastructure/core/node-prefer-noschedule-cronjob.yaml

@@ -81,6 +81,14 @@ spec:
                     echo "skipping missing node titan-23"
                   fi
 
+                  if k get node titan-21 >/dev/null 2>&1; then
+                    # Keep replicas off the Jetson, but allow Longhorn to attach
+                    # stateful fallback workloads there.
+                    k label node titan-21 longhorn-host=true --overwrite=true || true
+                  else
+                    echo "skipping missing node titan-21"
+                  fi
+
                   for node in titan-13 titan-15 titan-17 titan-19; do
                     if k get node "${node}" >/dev/null 2>&1; then
                       k label node "${node}" \

services/hermes/agent-deployment.yaml

@@ -24,7 +24,7 @@ spec:
         ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive
         ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
         ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
-        ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
+        ai.bstein.dev/placement: rpi5 preferred; rpi4 next; titan-21 is the stateful fallback
         ai.bstein.dev/config-rev: "20260818-ai-usage-exporter-split-v3"
         prometheus.io/scrape: "true"
         prometheus.io/path: /metrics
@@ -107,6 +107,18 @@ spec:
                   - key: kubernetes.io/hostname
                     operator: NotIn
                     values: [titan-04, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19]
+              - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: [arm64]
+                  - key: node-role.kubernetes.io/accelerator
+                    operator: Exists
+                  - key: kubernetes.io/hostname
+                    operator: In
+                    values: [titan-21]
+                  - key: longhorn-host
+                    operator: In
+                    values: ["true"]
           preferredDuringSchedulingIgnoredDuringExecution:
             - weight: 100
               preference:
@@ -114,6 +126,18 @@ spec:
                   - key: hardware
                     operator: In
                     values: [rpi5]
+            - weight: 50
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: [rpi4]
+            - weight: 25
+              preference:
+                matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: In
+                    values: [titan-21]
       initContainers:
         - name: init-config
           image: busybox:1.37

testing/tests/test_hermes_agent_layout.py

@@ -134,34 +134,44 @@ def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth():
 
     assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2"
     assert lane["resources"] == {
-        "requests": {"cpu": "100m", "memory": "256Mi"},
+        "requests": {"cpu": "50m", "memory": "192Mi"},
         "limits": {"cpu": "2", "memory": "6Gi"},
     }
 
 
-def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity():
+def test_agent_avoids_unhealthy_nodes_and_has_a_stateful_fallback():
-    """Placement correction: keep the agent off nodes that cannot hold it.
+    """The owner agent keeps a bounded fallback when Pi workers are full."""
-
-    titan-04 is cordoned after repeated kernel undervoltage and kubelet
-    failure, and titan-19 was probe/Longhorn unstable under worker load, so
-    both must join the existing hard exclusions. That leaves titan-05 as the
-    healthy candidate, which is tight enough on requested CPU that the main
-    container has to give back 50m to schedule there.
-    """
     pod = _agent_deployment()["spec"]["template"]["spec"]
+    terms = pod["affinity"]["nodeAffinity"][
+        "requiredDuringSchedulingIgnoredDuringExecution"
+    ]["nodeSelectorTerms"]
     hostnames = next(
         item
-        for item in pod["affinity"]["nodeAffinity"][
+        for item in terms[0]["matchExpressions"]
-            "requiredDuringSchedulingIgnoredDuringExecution"
-        ]["nodeSelectorTerms"][0]["matchExpressions"]
         if item["key"] == "kubernetes.io/hostname"
     )
 
     assert hostnames["operator"] == "NotIn"
     assert set(hostnames["values"]) >= {"titan-04", "titan-19"}
 
+    fallback = {item["key"]: item for item in terms[1]["matchExpressions"]}
+    assert fallback["node-role.kubernetes.io/accelerator"]["operator"] == "Exists"
+    assert fallback["kubernetes.io/hostname"]["values"] == ["titan-21"]
+    assert fallback["longhorn-host"]["values"] == ["true"]
+
+    preferences = pod["affinity"]["nodeAffinity"][
+        "preferredDuringSchedulingIgnoredDuringExecution"
+    ]
+    assert [item["weight"] for item in preferences] == [100, 50, 25]
+
     hermes = next(item for item in pod["containers"] if item["name"] == "hermes")
-    assert hermes["resources"]["requests"]["cpu"] == "300m"
+    assert hermes["resources"]["requests"]["cpu"] == "200m"
+
+    node_labels = (
+        Path(__file__).parents[2]
+        / "infrastructure/core/node-prefer-noschedule-cronjob.yaml"
+    ).read_text()
+    assert "k label node titan-21 longhorn-host=true" in node_labels
 
 
 def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path():