From 48cbe13ee50ce3fcb07cea3fe8d088cfed349e0c Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 17 Aug 2026 05:44:55 -0300 Subject: [PATCH] hermes: run three lanes on accelerator headroom --- services/hermes/agent-deployment.yaml | 22 ++++++++-- testing/tests/test_hermes_cli_lanes.py | 58 +++++++++++++++++--------- 2 files changed, 57 insertions(+), 23 deletions(-) diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 5ed758cf..22d6b337 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -24,7 +24,7 @@ spec: ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback - ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available + ai.bstein.dev/placement: arm64 accelerator preferred for execution headroom; rpi5 fallback ai.bstein.dev/config-rev: "20260816-auth-health-quarantine-v6" prometheus.io/scrape: "true" prometheus.io/path: /metrics @@ -112,8 +112,22 @@ spec: - key: kubernetes.io/hostname operator: NotIn values: [titan-04, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19] + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: [arm64] + - key: node-role.kubernetes.io/accelerator + operator: Exists + - key: kubernetes.io/hostname + operator: In + values: [titan-20, titan-21] preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/accelerator + operator: Exists + - weight: 50 preference: matchExpressions: - key: hardware @@ -874,7 +888,7 @@ spec: - {name: CLAUDE_CONFIG_DIR, value: /runtime-access/claude} - {name: KUBECONFIG, value: /opt/data/home/.kube/config} - {name: PYTHONPATH, value: /opt/hermes} - - {name: HERMES_CLI_LANE_CONCURRENCY, value: "2"} + - {name: HERMES_CLI_LANE_CONCURRENCY, value: "3"} - {name: HERMES_AUTO_ROUTER_PROFILE, value: agent} - {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin} securityContext: @@ -896,8 +910,8 @@ spec: - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} - {name: tmp, mountPath: /tmp} resources: - requests: {cpu: 100m, memory: 256Mi} - limits: {cpu: "2", memory: 6Gi} + requests: {cpu: 500m, memory: 512Mi} + limits: {cpu: "3", memory: 6Gi} - name: model-steward image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107 imagePullPolicy: IfNotPresent diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index fe5272da..25286dba 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -1084,7 +1084,7 @@ def test_agent_uses_one_native_kanban_control_plane(): assert "herdr-dispatch" not in rendered -def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth(): +def test_cli_lane_reserves_capacity_for_three_concurrent_workers(): deployment = _agent_deployment() containers = { item["name"]: item @@ -1093,33 +1093,53 @@ def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth(): lane = containers["cli-lane-runner"] environment = {item["name"]: item["value"] for item in lane["env"]} - assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2" + assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "3" assert lane["resources"] == { - "requests": {"cpu": "100m", "memory": "256Mi"}, - "limits": {"cpu": "2", "memory": "6Gi"}, + "requests": {"cpu": "500m", "memory": "512Mi"}, + "limits": {"cpu": "3", "memory": "6Gi"}, } -def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity(): - """Placement correction: keep the agent off nodes that cannot hold it. - - titan-04 is cordoned after repeated kernel undervoltage and kubelet - failure, and titan-19 was probe/Longhorn unstable under worker load, so - both must join the existing hard exclusions. That leaves titan-05 as the - healthy candidate, which is tight enough on requested CPU that the main - container has to give back 50m to schedule there. - """ +def test_agent_avoids_unhealthy_nodes_and_prefers_accelerator_headroom(): + """Keep unsafe workers excluded and prefer the larger arm64 nodes.""" pod = _agent_deployment()["spec"]["template"]["spec"] - hostnames = next( + terms = pod["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"] + worker_hostnames = next( item - for item in pod["affinity"]["nodeAffinity"][ - "requiredDuringSchedulingIgnoredDuringExecution" - ]["nodeSelectorTerms"][0]["matchExpressions"] + for item in terms[0]["matchExpressions"] if item["key"] == "kubernetes.io/hostname" ) - assert hostnames["operator"] == "NotIn" - assert set(hostnames["values"]) >= {"titan-04", "titan-19"} + assert worker_hostnames["operator"] == "NotIn" + assert set(worker_hostnames["values"]) >= {"titan-04", "titan-19"} + + accelerator = {item["key"]: item for item in terms[1]["matchExpressions"]} + assert accelerator["kubernetes.io/arch"] == { + "key": "kubernetes.io/arch", + "operator": "In", + "values": ["arm64"], + } + assert ( + accelerator["node-role.kubernetes.io/accelerator"]["operator"] == "Exists" + ) + assert accelerator["kubernetes.io/hostname"] == { + "key": "kubernetes.io/hostname", + "operator": "In", + "values": ["titan-20", "titan-21"], + } + + preferences = pod["affinity"]["nodeAffinity"][ + "preferredDuringSchedulingIgnoredDuringExecution" + ] + assert preferences[0]["weight"] == 100 + assert preferences[0]["preference"]["matchExpressions"] == [ + { + "key": "node-role.kubernetes.io/accelerator", + "operator": "Exists", + } + ] hermes = next( item for item in pod["containers"] if item["name"] == "hermes"