gpu(titan-24): pin image guard to local exporter
All checks were successful
Tests / Declarative: Post Actions passed: 247
All checks were successful
Tests / Declarative: Post Actions passed: 247
This commit is contained in:
parent
cec15568f9
commit
396a9db9a8
@ -64,7 +64,7 @@ spec:
|
|||||||
- name: LEASE_IMAGE_OWNER
|
- name: LEASE_IMAGE_OWNER
|
||||||
value: hermes-image
|
value: hermes-image
|
||||||
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL
|
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL
|
||||||
value: http://nvidia-process-exporter.monitoring.svc.cluster.local:9401/metrics
|
value: http://nvidia-process-exporter-local.monitoring.svc.cluster.local:9401/metrics
|
||||||
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE
|
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE
|
||||||
value: titan-24
|
value: titan-24
|
||||||
# The idle desktop stack and Wolf daemon use about 1.9 GiB and
|
# The idle desktop stack and Wolf daemon use about 1.9 GiB and
|
||||||
|
|||||||
@ -141,3 +141,21 @@ spec:
|
|||||||
- name: metrics
|
- name: metrics
|
||||||
port: 9401
|
port: 9401
|
||||||
targetPort: metrics
|
targetPort: metrics
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: nvidia-process-exporter-local
|
||||||
|
namespace: monitoring
|
||||||
|
labels:
|
||||||
|
app: nvidia-process-exporter
|
||||||
|
spec:
|
||||||
|
# GPU arbitration must inspect the exporter on its own node. The regular
|
||||||
|
# service intentionally balances all exporters for cluster monitoring.
|
||||||
|
internalTrafficPolicy: Local
|
||||||
|
selector:
|
||||||
|
app: nvidia-process-exporter
|
||||||
|
ports:
|
||||||
|
- name: metrics
|
||||||
|
port: 9401
|
||||||
|
targetPort: metrics
|
||||||
|
|||||||
@ -613,7 +613,7 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
|
|||||||
assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == (
|
assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == (
|
||||||
"3221225472"
|
"3221225472"
|
||||||
)
|
)
|
||||||
assert "nvidia-process-exporter.monitoring.svc.cluster.local" in model_env[
|
assert "nvidia-process-exporter-local.monitoring.svc.cluster.local" in model_env[
|
||||||
"HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL"
|
"HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL"
|
||||||
]
|
]
|
||||||
models_volume = next(item for item in pod["volumes"] if item["name"] == "models")
|
models_volume = next(item for item in pod["volumes"] if item["name"] == "models")
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user