bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

gpu(titan-24): trust checkout lease over desktop spikes
All checks were successful
Tests / Declarative: Post Actions passed: 248
Details

Browse Source
This commit is contained in:
jenkins 2026-08-11 15:19:35 -03:00
parent 8f67b979d9
commit 30aeab2bb9
2 changed files with 15 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
services/hermes/local-image-deployment.yaml
View File

@ -67,10 +67,11 @@ spec:
value: http://nvidia-process-exporter-local.monitoring.svc.cluster.local:9401/metrics
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE
value: titan-24
# The idle desktop stack and Wolf daemon use about 1.9 GiB and
# 3% SM. Active desktop/game sessions cross one of these bounds.
# The lease is authoritative for checkout state. Keep a high SM
# ceiling only as a fail-safe for unregistered GPU workloads;
# desktop compositors can briefly spike well above their idle use.
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT
value: "8"
value: "80"
- name: HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES
value: "3221225472"
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLES

17
testing/tests/test_hermes_chat_quality.py
View File

@ -621,7 +621,7 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
"e7b7dc27f91deacad38e78976d1f2b499d76a294"
)
assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24"
assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "8"
assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "80"
assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == (
"3221225472"
)
@ -692,15 +692,15 @@ def test_titan20_serializes_classifier_and_local_chat_model_residency():
)
def test_local_image_gpu_guard_distinguishes_idle_and_active_desktop(monkeypatch):
"""Idle display shells may coexist, but active desktop/game work blocks FLUX."""
def test_local_image_gpu_guard_distinguishes_background_and_saturated_gpu(monkeypatch):
"""Lease-idle desktop spikes may coexist, but saturation still blocks FLUX."""
source = ROOT / "dockerfiles" / "hermes-local-image-server.py"
spec = importlib.util.spec_from_file_location("hermes_local_image_server", source)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24")
monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 8.0)
monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 80.0)
monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30)
idle = module._parse_gpu_activity(
@ -717,11 +717,16 @@ def test_local_image_gpu_guard_distinguishes_idle_and_active_desktop(monkeypatch
assert idle["external_gpu_memory_bytes"] == 1900000000
assert idle["external_gpu_sm_percent"] == 3
background_spike = module._parse_gpu_activity(
'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="sway"} 41\n'
)
assert background_spike["interactive_active"] is False
active = module._parse_gpu_activity(
'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 41\n'
'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 91\n'
)
assert active["interactive_active"] is True
assert "41%" in active["gpu_guard_reason"]
assert "91%" in active["gpu_guard_reason"]
def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch):