From df310df1b5d357fe0666083faa8fa6a6e2955046 Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 11 Aug 2026 05:52:49 -0300 Subject: [PATCH] gpu(titan-24): protect desktop image sharing --- dockerfiles/hermes-local-image-server.py | 129 +++++++++++++++++++- services/hermes/local-image-deployment.yaml | 20 ++- testing/tests/test_hermes_chat_quality.py | 40 ++++++ 3 files changed, 185 insertions(+), 4 deletions(-) diff --git a/dockerfiles/hermes-local-image-server.py b/dockerfiles/hermes-local-image-server.py index b0c65a1ef..8a3fcb7e1 100644 --- a/dockerfiles/hermes-local-image-server.py +++ b/dockerfiles/hermes-local-image-server.py @@ -10,6 +10,7 @@ import json import os import ssl import threading +import time from datetime import UTC, datetime from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path @@ -35,6 +36,23 @@ LEASE_IDLE_OWNER = os.environ.get("LEASE_IDLE_OWNER", "hermes") LEASE_IMAGE_OWNER = os.environ.get("LEASE_IMAGE_OWNER", "hermes-image") MAX_BODY_BYTES = int(os.environ.get("HERMES_LOCAL_IMAGE_MAX_BODY", str(96 << 20))) QUEUE_TIMEOUT_SECONDS = float(os.environ.get("HERMES_LOCAL_IMAGE_QUEUE_TIMEOUT", "1200")) +GPU_ACTIVITY_URL = os.environ.get( + "HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL", + "http://nvidia-process-exporter.monitoring.svc.cluster.local:9401/metrics", +) +GPU_ACTIVITY_NODE = os.environ.get("HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE", "titan-24") +GPU_ACTIVE_SM_PERCENT = float( + os.environ.get("HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT", "8") +) +GPU_MAX_EXTERNAL_MEMORY_BYTES = int( + os.environ.get("HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES", str(3 << 30)) +) +GPU_ACTIVITY_SAMPLES = max( + 1, int(os.environ.get("HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLES", "3")) +) +GPU_ACTIVITY_SAMPLE_INTERVAL = max( + 0.0, float(os.environ.get("HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLE_INTERVAL", "1")) +) API_HOST = os.environ.get("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc") API_PORT = os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS", "443") TOKEN_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/token") @@ -56,9 +74,101 @@ _state: dict[str, Any] = { "phase": "idle", "last_error": None, "last_model": None, + "interactive_active": None, + "external_gpu_memory_bytes": None, + "external_gpu_sm_percent": None, + "gpu_guard_reason": None, } +def _prometheus_labels(raw: str) -> dict[str, str]: + """Parse the simple quoted labels emitted by nvidia-process-exporter.""" + labels: dict[str, str] = {} + for item in raw.split(","): + key, separator, value = item.partition("=") + if not separator: + continue + labels[key.strip()] = value.strip().strip('"') + return labels + + +def _parse_gpu_activity(metrics: str) -> dict[str, Any]: + """Summarize non-Hermes GPU processes on the configured image node.""" + memory_bytes = 0 + max_sm_percent = 0.0 + processes: set[str] = set() + for line in metrics.splitlines(): + if not line or line.startswith("#") or "{" not in line or "}" not in line: + continue + metric, remainder = line.split("{", 1) + raw_labels, separator, raw_value = remainder.partition("}") + if not separator or metric not in { + "nvidia_process_gpu_memory_used_bytes", + "nvidia_process_gpu_sm_util_percent", + }: + continue + labels = _prometheus_labels(raw_labels) + if labels.get("node") != GPU_ACTIVITY_NODE or labels.get("namespace") == "hermes": + continue + try: + value = float(raw_value.strip().split()[0]) + except (ValueError, IndexError): + continue + process = labels.get("process") or "unknown" + processes.add(f"{labels.get('namespace', 'unknown')}/{process}") + if metric == "nvidia_process_gpu_memory_used_bytes": + memory_bytes += max(0, int(value)) + else: + max_sm_percent = max(max_sm_percent, value) + active = ( + max_sm_percent >= GPU_ACTIVE_SM_PERCENT + or memory_bytes >= GPU_MAX_EXTERNAL_MEMORY_BYTES + ) + reasons = [] + if max_sm_percent >= GPU_ACTIVE_SM_PERCENT: + reasons.append(f"external SM utilization is {max_sm_percent:g}%") + if memory_bytes >= GPU_MAX_EXTERNAL_MEMORY_BYTES: + reasons.append(f"external GPU memory is {memory_bytes} bytes") + return { + "interactive_active": active, + "external_gpu_memory_bytes": memory_bytes, + "external_gpu_sm_percent": max_sm_percent, + "external_gpu_processes": sorted(processes), + "gpu_guard_reason": "; ".join(reasons) or None, + } + + +def _gpu_activity_snapshot() -> dict[str, Any]: + """Fetch live per-process GPU attribution, failing closed if unavailable.""" + request = Request(GPU_ACTIVITY_URL, headers={"Accept": "text/plain"}) + with urlopen(request, timeout=5) as response: + metrics = response.read(2 << 20).decode("utf-8", errors="replace") + return _parse_gpu_activity(metrics) + + +def _ensure_interactive_lane_idle() -> dict[str, Any]: + """Reject FLUX while desktop or Wolf activity indicates interactive use.""" + worst: dict[str, Any] | None = None + for sample in range(GPU_ACTIVITY_SAMPLES): + snapshot = _gpu_activity_snapshot() + if worst is None or ( + snapshot["external_gpu_sm_percent"], snapshot["external_gpu_memory_bytes"] + ) > (worst["external_gpu_sm_percent"], worst["external_gpu_memory_bytes"]): + worst = snapshot + if snapshot["interactive_active"]: + break + if sample + 1 < GPU_ACTIVITY_SAMPLES: + time.sleep(GPU_ACTIVITY_SAMPLE_INTERVAL) + assert worst is not None + _set_state(**{key: value for key, value in worst.items() if key != "external_gpu_processes"}) + if worst["interactive_active"]: + raise RuntimeError( + "titan-24 desktop/Wolf lane is active; reserve or release it from the Atlas GPU checkout" + f" ({worst['gpu_guard_reason']})" + ) + return worst + + def _set_state(**values: Any) -> None: """Update health state without exposing prompts or image content.""" with _state_lock: @@ -203,6 +313,11 @@ def _generate(payload: dict[str, Any]) -> dict[str, Any]: claimed = False _set_state(busy=True, phase="claiming-gpu", last_error=None) try: + if _lease_owner() != LEASE_IDLE_OWNER: + owner = _lease_owner() + raise RuntimeError(f"local image GPU unavailable while titan-24 owner is {owner}") + _set_state(phase="checking-interactive-lane") + _ensure_interactive_lane_idle() claimed = _change_owner(LEASE_IDLE_OWNER, LEASE_IMAGE_OWNER) if not claimed: owner = _lease_owner() @@ -252,11 +367,23 @@ class Handler(BaseHTTPRequestHandler): owner = _lease_owner() except Exception: owner = "unavailable" + try: + activity = _gpu_activity_snapshot() + _set_state(**{key: value for key, value in activity.items() if key != "external_gpu_processes"}) + state = _state_snapshot() + except Exception as exc: + activity = {"interactive_active": True, "gpu_guard_reason": f"GPU guard unavailable: {exc}"} + _set_state(**activity) + state = _state_snapshot() self._json( 200, { "success": True, - "available": owner == LEASE_IDLE_OWNER and not state["busy"], + "available": ( + owner == LEASE_IDLE_OWNER + and not state["busy"] + and not activity["interactive_active"] + ), "gpu_owner": owner, "model": "flux-2-klein-4b-local", **state, diff --git a/services/hermes/local-image-deployment.yaml b/services/hermes/local-image-deployment.yaml index a82d6bdeb..f5c6c6c45 100644 --- a/services/hermes/local-image-deployment.yaml +++ b/services/hermes/local-image-deployment.yaml @@ -20,8 +20,8 @@ spec: app: hermes-local-image annotations: ai.bstein.dev/model: black-forest-labs/FLUX.2-klein-4B - ai.bstein.dev/gpu: titan-24 lease-shared image and Wolf lane - ai.bstein.dev/config-rev: "20260811-flux2-klein-image-only" + ai.bstein.dev/gpu: titan-24 lease-shared image, desktop, and Wolf lane + ai.bstein.dev/config-rev: "20260811-flux2-klein-interactive-guard" spec: serviceAccountName: hermes-gpu-runtime runtimeClassName: nvidia @@ -43,7 +43,7 @@ spec: sizeLimit: 2Gi containers: - name: local-image - image: registry.bstein.dev/bstein/hermes-local-image@sha256:316d123cc48509a53fb32c26bfb384a3ed4d934cb334d73faa0840a62e047e65 + image: registry.bstein.dev/bstein/hermes-local-image@sha256:769a16f2b56b1de401f7dc4adafb522d39c8509d8de741668f69ee7a206502f7 imagePullPolicy: IfNotPresent ports: - name: local-image @@ -63,6 +63,20 @@ spec: value: hermes - name: LEASE_IMAGE_OWNER value: hermes-image + - name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL + value: http://nvidia-process-exporter.monitoring.svc.cluster.local:9401/metrics + - name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE + value: titan-24 + # The idle desktop stack and Wolf daemon use about 1.9 GiB and + # 3% SM. Active desktop/game sessions cross one of these bounds. + - name: HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT + value: "8" + - name: HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES + value: "3221225472" + - name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLES + value: "3" + - name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLE_INTERVAL + value: "1" - name: HF_HOME value: /models/huggingface - name: HOME diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 171efc878..c6cd5f5d7 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -608,6 +608,14 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == ( "e7b7dc27f91deacad38e78976d1f2b499d76a294" ) + assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24" + assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "8" + assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( + "3221225472" + ) + assert "nvidia-process-exporter.monitoring.svc.cluster.local" in model_env[ + "HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL" + ] models_volume = next(item for item in pod["volumes"] if item["name"] == "models") assert models_volume["persistentVolumeClaim"]["claimName"] == ( "hermes-image-models" @@ -672,6 +680,38 @@ def test_titan20_serializes_classifier_and_local_chat_model_residency(): ) +def test_local_image_gpu_guard_distinguishes_idle_and_active_desktop(monkeypatch): + """Idle display shells may coexist, but active desktop/game work blocks FLUX.""" + source = ROOT / "dockerfiles" / "hermes-local-image-server.py" + spec = importlib.util.spec_from_file_location("hermes_local_image_server", source) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24") + monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 8.0) + monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30) + + idle = module._parse_gpu_activity( + '\n'.join( + [ + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0', + 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000', + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3', + 'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000', + ] + ) + ) + assert idle["interactive_active"] is False + assert idle["external_gpu_memory_bytes"] == 1900000000 + assert idle["external_gpu_sm_percent"] == 3 + + active = module._parse_gpu_activity( + 'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 41\n' + ) + assert active["interactive_active"] is True + assert "41%" in active["gpu_guard_reason"] + + def test_chat_auth_file_mount_survives_atomic_provider_refresh(): statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] containers = statefulset["spec"]["template"]["spec"]["containers"]