gpu(titan-24): trust checkout lease over desktop spikes
All checks were successful
Tests / Declarative: Post Actions passed: 248

This commit is contained in:
jenkins 2026-08-11 15:19:35 -03:00
parent 8f67b979d9
commit 30aeab2bb9
2 changed files with 15 additions and 9 deletions

View File

@ -67,10 +67,11 @@ spec:
value: http://nvidia-process-exporter-local.monitoring.svc.cluster.local:9401/metrics value: http://nvidia-process-exporter-local.monitoring.svc.cluster.local:9401/metrics
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE - name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE
value: titan-24 value: titan-24
# The idle desktop stack and Wolf daemon use about 1.9 GiB and # The lease is authoritative for checkout state. Keep a high SM
# 3% SM. Active desktop/game sessions cross one of these bounds. # ceiling only as a fail-safe for unregistered GPU workloads;
# desktop compositors can briefly spike well above their idle use.
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT - name: HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT
value: "8" value: "80"
- name: HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES - name: HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES
value: "3221225472" value: "3221225472"
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLES - name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLES

View File

@ -621,7 +621,7 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
"e7b7dc27f91deacad38e78976d1f2b499d76a294" "e7b7dc27f91deacad38e78976d1f2b499d76a294"
) )
assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24"
assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "8" assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80"
assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == (
"3221225472" "3221225472"
) )
@ -692,15 +692,15 @@ def test_titan20_serializes_classifier_and_local_chat_model_residency():
) )
def test_local_image_gpu_guard_distinguishes_idle_and_active_desktop(monkeypatch): def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch):
"""Idle display shells may coexist, but active desktop/game work blocks FLUX.""" """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX."""
source = ROOT / "dockerfiles" / "hermes-local-image-server.py" source = ROOT / "dockerfiles" / "hermes-local-image-server.py"
spec = importlib.util.spec_from_file_location("hermes_local_image_server", source) spec = importlib.util.spec_from_file_location("hermes_local_image_server", source)
assert spec and spec.loader assert spec and spec.loader
module = importlib.util.module_from_spec(spec) module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module) spec.loader.exec_module(module)
monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24") monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24")
monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 8.0) monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0)
monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30) monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30)
idle = module._parse_gpu_activity( idle = module._parse_gpu_activity(
@ -717,11 +717,16 @@ def test_local_image_gpu_guard_distinguishes_idle_and_active_desktop(monkeypatch
assert idle["external_gpu_memory_bytes"] == 1900000000 assert idle["external_gpu_memory_bytes"] == 1900000000
assert idle["external_gpu_sm_percent"] == 3 assert idle["external_gpu_sm_percent"] == 3
background_spike = module._parse_gpu_activity(
'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n'
)
assert background_spike["interactive_active"] is False
active = module._parse_gpu_activity( active = module._parse_gpu_activity(
'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 41\n' 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n'
) )
assert active["interactive_active"] is True assert active["interactive_active"] is True
assert "41%" in active["gpu_guard_reason"] assert "91%" in active["gpu_guard_reason"]
def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch): def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch):