diff --git a/services/hermes/local-image-deployment.yaml b/services/hermes/local-image-deployment.yaml index b991ed809..b3a9f7263 100644 --- a/services/hermes/local-image-deployment.yaml +++ b/services/hermes/local-image-deployment.yaml @@ -67,10 +67,11 @@ spec: value: http://nvidia-process-exporter-local.monitoring.svc.cluster.local:9401/metrics - name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE value: titan-24 - # The idle desktop stack and Wolf daemon use about 1.9 GiB and - # 3% SM. Active desktop/game sessions cross one of these bounds. + # The lease is authoritative for checkout state. Keep a high SM + # ceiling only as a fail-safe for unregistered GPU workloads; + # desktop compositors can briefly spike well above their idle use. - name: HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT - value: "8" + value: "80" - name: HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES value: "3221225472" - name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLES diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 8d66faef3..fc290ea2a 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -621,7 +621,7 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): "e7b7dc27f91deacad38e78976d1f2b499d76a294" ) assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" - assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "8" + assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80" assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( "3221225472" ) @@ -692,15 +692,15 @@ def test_titan20_serializes_classifier_and_local_chat_model_residency(): ) -def test_local_image_gpu_guard_distinguishes_idle_and_active_desktop(monkeypatch): - """Idle display shells may coexist, but active desktop/game work blocks FLUX.""" +def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch): + """Lease-idle desktop spikes may coexist, but saturation still blocks FLUX.""" source = ROOT / "dockerfiles" / "hermes-local-image-server.py" spec = importlib.util.spec_from_file_location("hermes_local_image_server", source) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24") - monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 8.0) + monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0) monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30) idle = module._parse_gpu_activity( @@ -717,11 +717,16 @@ def test_local_image_gpu_guard_distinguishes_idle_and_active_desktop(monkeypatch assert idle["external_gpu_memory_bytes"] == 1900000000 assert idle["external_gpu_sm_percent"] == 3 + background_spike = module._parse_gpu_activity( + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n' + ) + assert background_spike["interactive_active"] is False + active = module._parse_gpu_activity( - 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 41\n' + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n' ) assert active["interactive_active"] is True - assert "41%" in active["gpu_guard_reason"] + assert "91%" in active["gpu_guard_reason"] def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch):