gpu(titan-24): trust checkout lease over desktop spikes
All checks were successful
Tests / Declarative: Post Actions passed: 248
All checks were successful
Tests / Declarative: Post Actions passed: 248
This commit is contained in:
parent
8f67b979d9
commit
30aeab2bb9
@ -67,10 +67,11 @@ spec:
|
|||||||
value: http://nvidia-process-exporter-local.monitoring.svc.cluster.local:9401/metrics
|
value: http://nvidia-process-exporter-local.monitoring.svc.cluster.local:9401/metrics
|
||||||
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE
|
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE
|
||||||
value: titan-24
|
value: titan-24
|
||||||
# The idle desktop stack and Wolf daemon use about 1.9 GiB and
|
# The lease is authoritative for checkout state. Keep a high SM
|
||||||
# 3% SM. Active desktop/game sessions cross one of these bounds.
|
# ceiling only as a fail-safe for unregistered GPU workloads;
|
||||||
|
# desktop compositors can briefly spike well above their idle use.
|
||||||
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT
|
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT
|
||||||
value: "8"
|
value: "80"
|
||||||
- name: HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES
|
- name: HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES
|
||||||
value: "3221225472"
|
value: "3221225472"
|
||||||
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLES
|
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLES
|
||||||
|
|||||||
@ -621,7 +621,7 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
|
|||||||
"e7b7dc27f91deacad38e78976d1f2b499d76a294"
|
"e7b7dc27f91deacad38e78976d1f2b499d76a294"
|
||||||
)
|
)
|
||||||
assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24"
|
assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24"
|
||||||
assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "8"
|
assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80"
|
||||||
assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == (
|
assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == (
|
||||||
"3221225472"
|
"3221225472"
|
||||||
)
|
)
|
||||||
@ -692,15 +692,15 @@ def test_titan20_serializes_classifier_and_local_chat_model_residency():
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_local_image_gpu_guard_distinguishes_idle_and_active_desktop(monkeypatch):
|
def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch):
|
||||||
"""Idle display shells may coexist, but active desktop/game work blocks FLUX."""
|
"""Lease-idle desktop spikes may coexist, but saturation still blocks FLUX."""
|
||||||
source = ROOT / "dockerfiles" / "hermes-local-image-server.py"
|
source = ROOT / "dockerfiles" / "hermes-local-image-server.py"
|
||||||
spec = importlib.util.spec_from_file_location("hermes_local_image_server", source)
|
spec = importlib.util.spec_from_file_location("hermes_local_image_server", source)
|
||||||
assert spec and spec.loader
|
assert spec and spec.loader
|
||||||
module = importlib.util.module_from_spec(spec)
|
module = importlib.util.module_from_spec(spec)
|
||||||
spec.loader.exec_module(module)
|
spec.loader.exec_module(module)
|
||||||
monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24")
|
monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24")
|
||||||
monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 8.0)
|
monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0)
|
||||||
monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30)
|
monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30)
|
||||||
|
|
||||||
idle = module._parse_gpu_activity(
|
idle = module._parse_gpu_activity(
|
||||||
@ -717,11 +717,16 @@ def test_local_image_gpu_guard_distinguishes_idle_and_active_desktop(monkeypatch
|
|||||||
assert idle["external_gpu_memory_bytes"] == 1900000000
|
assert idle["external_gpu_memory_bytes"] == 1900000000
|
||||||
assert idle["external_gpu_sm_percent"] == 3
|
assert idle["external_gpu_sm_percent"] == 3
|
||||||
|
|
||||||
|
background_spike = module._parse_gpu_activity(
|
||||||
|
'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n'
|
||||||
|
)
|
||||||
|
assert background_spike["interactive_active"] is False
|
||||||
|
|
||||||
active = module._parse_gpu_activity(
|
active = module._parse_gpu_activity(
|
||||||
'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 41\n'
|
'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n'
|
||||||
)
|
)
|
||||||
assert active["interactive_active"] is True
|
assert active["interactive_active"] is True
|
||||||
assert "41%" in active["gpu_guard_reason"]
|
assert "91%" in active["gpu_guard_reason"]
|
||||||
|
|
||||||
|
|
||||||
def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch):
|
def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch):
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user