From 396a9db9a89a4ca0efc5bf8ae750604b56a7906d Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 11 Aug 2026 06:10:39 -0300 Subject: [PATCH] gpu(titan-24): pin image guard to local exporter --- services/hermes/local-image-deployment.yaml | 2 +- .../monitoring/nvidia-process-exporter.yaml | 18 ++++++++++++++++++ testing/tests/test_hermes_chat_quality.py | 2 +- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/services/hermes/local-image-deployment.yaml b/services/hermes/local-image-deployment.yaml index f5c6c6c45..883009de8 100644 --- a/services/hermes/local-image-deployment.yaml +++ b/services/hermes/local-image-deployment.yaml @@ -64,7 +64,7 @@ spec: - name: LEASE_IMAGE_OWNER value: hermes-image - name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL - value: http://nvidia-process-exporter.monitoring.svc.cluster.local:9401/metrics + value: http://nvidia-process-exporter-local.monitoring.svc.cluster.local:9401/metrics - name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE value: titan-24 # The idle desktop stack and Wolf daemon use about 1.9 GiB and diff --git a/services/monitoring/nvidia-process-exporter.yaml b/services/monitoring/nvidia-process-exporter.yaml index 325a03794..16bdd7e4e 100644 --- a/services/monitoring/nvidia-process-exporter.yaml +++ b/services/monitoring/nvidia-process-exporter.yaml @@ -141,3 +141,21 @@ spec: - name: metrics port: 9401 targetPort: metrics +--- +apiVersion: v1 +kind: Service +metadata: + name: nvidia-process-exporter-local + namespace: monitoring + labels: + app: nvidia-process-exporter +spec: + # GPU arbitration must inspect the exporter on its own node. The regular + # service intentionally balances all exporters for cluster monitoring. + internalTrafficPolicy: Local + selector: + app: nvidia-process-exporter + ports: + - name: metrics + port: 9401 + targetPort: metrics diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index c6cd5f5d7..4c8574671 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -613,7 +613,7 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( "3221225472" ) - assert "nvidia-process-exporter.monitoring.svc.cluster.local" in model_env[ + assert "nvidia-process-exporter-local.monitoring.svc.cluster.local" in model_env[ "HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL" ] models_volume = next(item for item in pod["volumes"] if item["name"] == "models")