gpu(titan-24): protect desktop image sharing

dockerfiles/hermes-local-image-server.py

@@ -10,6 +10,7 @@ import json
 import os
 import ssl
 import threading
+import time
 from datetime import UTC, datetime
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
@@ -35,6 +36,23 @@ LEASE_IDLE_OWNER = os.environ.get("LEASE_IDLE_OWNER", "hermes")
 LEASE_IMAGE_OWNER = os.environ.get("LEASE_IMAGE_OWNER", "hermes-image")
 MAX_BODY_BYTES = int(os.environ.get("HERMES_LOCAL_IMAGE_MAX_BODY", str(96 << 20)))
 QUEUE_TIMEOUT_SECONDS = float(os.environ.get("HERMES_LOCAL_IMAGE_QUEUE_TIMEOUT", "1200"))
+GPU_ACTIVITY_URL = os.environ.get(
+    "HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL",
+    "http://nvidia-process-exporter.monitoring.svc.cluster.local:9401/metrics",
+)
+GPU_ACTIVITY_NODE = os.environ.get("HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE", "titan-24")
+GPU_ACTIVE_SM_PERCENT = float(
+    os.environ.get("HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT", "8")
+)
+GPU_MAX_EXTERNAL_MEMORY_BYTES = int(
+    os.environ.get("HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES", str(3 << 30))
+)
+GPU_ACTIVITY_SAMPLES = max(
+    1, int(os.environ.get("HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLES", "3"))
+)
+GPU_ACTIVITY_SAMPLE_INTERVAL = max(
+    0.0, float(os.environ.get("HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLE_INTERVAL", "1"))
+)
 API_HOST = os.environ.get("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc")
 API_PORT = os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS", "443")
 TOKEN_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/token")
@@ -56,9 +74,101 @@ _state: dict[str, Any] = {
     "phase": "idle",
     "last_error": None,
     "last_model": None,
+    "interactive_active": None,
+    "external_gpu_memory_bytes": None,
+    "external_gpu_sm_percent": None,
+    "gpu_guard_reason": None,
 }
 
 
+def _prometheus_labels(raw: str) -> dict[str, str]:
+    """Parse the simple quoted labels emitted by nvidia-process-exporter."""
+    labels: dict[str, str] = {}
+    for item in raw.split(","):
+        key, separator, value = item.partition("=")
+        if not separator:
+            continue
+        labels[key.strip()] = value.strip().strip('"')
+    return labels
+
+
+def _parse_gpu_activity(metrics: str) -> dict[str, Any]:
+    """Summarize non-Hermes GPU processes on the configured image node."""
+    memory_bytes = 0
+    max_sm_percent = 0.0
+    processes: set[str] = set()
+    for line in metrics.splitlines():
+        if not line or line.startswith("#") or "{" not in line or "}" not in line:
+            continue
+        metric, remainder = line.split("{", 1)
+        raw_labels, separator, raw_value = remainder.partition("}")
+        if not separator or metric not in {
+            "nvidia_process_gpu_memory_used_bytes",
+            "nvidia_process_gpu_sm_util_percent",
+        }:
+            continue
+        labels = _prometheus_labels(raw_labels)
+        if labels.get("node") != GPU_ACTIVITY_NODE or labels.get("namespace") == "hermes":
+            continue
+        try:
+            value = float(raw_value.strip().split()[0])
+        except (ValueError, IndexError):
+            continue
+        process = labels.get("process") or "unknown"
+        processes.add(f"{labels.get('namespace', 'unknown')}/{process}")
+        if metric == "nvidia_process_gpu_memory_used_bytes":
+            memory_bytes += max(0, int(value))
+        else:
+            max_sm_percent = max(max_sm_percent, value)
+    active = (
+        max_sm_percent >= GPU_ACTIVE_SM_PERCENT
+        or memory_bytes >= GPU_MAX_EXTERNAL_MEMORY_BYTES
+    )
+    reasons = []
+    if max_sm_percent >= GPU_ACTIVE_SM_PERCENT:
+        reasons.append(f"external SM utilization is {max_sm_percent:g}%")
+    if memory_bytes >= GPU_MAX_EXTERNAL_MEMORY_BYTES:
+        reasons.append(f"external GPU memory is {memory_bytes} bytes")
+    return {
+        "interactive_active": active,
+        "external_gpu_memory_bytes": memory_bytes,
+        "external_gpu_sm_percent": max_sm_percent,
+        "external_gpu_processes": sorted(processes),
+        "gpu_guard_reason": "; ".join(reasons) or None,
+    }
+
+
+def _gpu_activity_snapshot() -> dict[str, Any]:
+    """Fetch live per-process GPU attribution, failing closed if unavailable."""
+    request = Request(GPU_ACTIVITY_URL, headers={"Accept": "text/plain"})
+    with urlopen(request, timeout=5) as response:
+        metrics = response.read(2 << 20).decode("utf-8", errors="replace")
+    return _parse_gpu_activity(metrics)
+
+
+def _ensure_interactive_lane_idle() -> dict[str, Any]:
+    """Reject FLUX while desktop or Wolf activity indicates interactive use."""
+    worst: dict[str, Any] | None = None
+    for sample in range(GPU_ACTIVITY_SAMPLES):
+        snapshot = _gpu_activity_snapshot()
+        if worst is None or (
+            snapshot["external_gpu_sm_percent"], snapshot["external_gpu_memory_bytes"]
+        ) > (worst["external_gpu_sm_percent"], worst["external_gpu_memory_bytes"]):
+            worst = snapshot
+        if snapshot["interactive_active"]:
+            break
+        if sample + 1 < GPU_ACTIVITY_SAMPLES:
+            time.sleep(GPU_ACTIVITY_SAMPLE_INTERVAL)
+    assert worst is not None
+    _set_state(**{key: value for key, value in worst.items() if key != "external_gpu_processes"})
+    if worst["interactive_active"]:
+        raise RuntimeError(
+            "titan-24 desktop/Wolf lane is active; reserve or release it from the Atlas GPU checkout"
+            f" ({worst['gpu_guard_reason']})"
+        )
+    return worst
+
+
 def _set_state(**values: Any) -> None:
     """Update health state without exposing prompts or image content."""
     with _state_lock:
@@ -203,6 +313,11 @@ def _generate(payload: dict[str, Any]) -> dict[str, Any]:
     claimed = False
     _set_state(busy=True, phase="claiming-gpu", last_error=None)
     try:
+        if _lease_owner() != LEASE_IDLE_OWNER:
+            owner = _lease_owner()
+            raise RuntimeError(f"local image GPU unavailable while titan-24 owner is {owner}")
+        _set_state(phase="checking-interactive-lane")
+        _ensure_interactive_lane_idle()
         claimed = _change_owner(LEASE_IDLE_OWNER, LEASE_IMAGE_OWNER)
         if not claimed:
             owner = _lease_owner()
@@ -252,11 +367,23 @@ class Handler(BaseHTTPRequestHandler):
             owner = _lease_owner()
         except Exception:
             owner = "unavailable"
+        try:
+            activity = _gpu_activity_snapshot()
+            _set_state(**{key: value for key, value in activity.items() if key != "external_gpu_processes"})
+            state = _state_snapshot()
+        except Exception as exc:
+            activity = {"interactive_active": True, "gpu_guard_reason": f"GPU guard unavailable: {exc}"}
+            _set_state(**activity)
+            state = _state_snapshot()
         self._json(
             200,
             {
                 "success": True,
-                "available": owner == LEASE_IDLE_OWNER and not state["busy"],
+                "available": (
+                    owner == LEASE_IDLE_OWNER
+                    and not state["busy"]
+                    and not activity["interactive_active"]
+                ),
                 "gpu_owner": owner,
                 "model": "flux-2-klein-4b-local",
                 **state,

services/hermes/local-image-deployment.yaml

@@ -20,8 +20,8 @@ spec:
         app: hermes-local-image
       annotations:
         ai.bstein.dev/model: black-forest-labs/FLUX.2-klein-4B
-        ai.bstein.dev/gpu: titan-24 lease-shared image and Wolf lane
+        ai.bstein.dev/gpu: titan-24 lease-shared image, desktop, and Wolf lane
-        ai.bstein.dev/config-rev: "20260811-flux2-klein-image-only"
+        ai.bstein.dev/config-rev: "20260811-flux2-klein-interactive-guard"
     spec:
       serviceAccountName: hermes-gpu-runtime
       runtimeClassName: nvidia
@@ -43,7 +43,7 @@ spec:
             sizeLimit: 2Gi
       containers:
         - name: local-image
-          image: registry.bstein.dev/bstein/hermes-local-image@sha256:316d123cc48509a53fb32c26bfb384a3ed4d934cb334d73faa0840a62e047e65
+          image: registry.bstein.dev/bstein/hermes-local-image@sha256:769a16f2b56b1de401f7dc4adafb522d39c8509d8de741668f69ee7a206502f7
           imagePullPolicy: IfNotPresent
           ports:
             - name: local-image
@@ -63,6 +63,20 @@ spec:
               value: hermes
             - name: LEASE_IMAGE_OWNER
               value: hermes-image
+            - name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL
+              value: http://nvidia-process-exporter.monitoring.svc.cluster.local:9401/metrics
+            - name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE
+              value: titan-24
+            # The idle desktop stack and Wolf daemon use about 1.9 GiB and
+            # 3% SM. Active desktop/game sessions cross one of these bounds.
+            - name: HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT
+              value: "8"
+            - name: HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES
+              value: "3221225472"
+            - name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLES
+              value: "3"
+            - name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLE_INTERVAL
+              value: "1"
             - name: HF_HOME
               value: /models/huggingface
             - name: HOME

testing/tests/test_hermes_chat_quality.py

@@ -608,6 +608,14 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
     assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == (
         "e7b7dc27f91deacad38e78976d1f2b499d76a294"
     )
+    assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24"
+    assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "8"
+    assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == (
+        "3221225472"
+    )
+    assert "nvidia-process-exporter.monitoring.svc.cluster.local" in model_env[
+        "HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL"
+    ]
     models_volume = next(item for item in pod["volumes"] if item["name"] == "models")
     assert models_volume["persistentVolumeClaim"]["claimName"] == (
         "hermes-image-models"
@@ -672,6 +680,38 @@ def test_titan20_serializes_classifier_and_local_chat_model_residency():
     )
 
 
+def test_local_image_gpu_guard_distinguishes_idle_and_active_desktop(monkeypatch):
+    """Idle display shells may coexist, but active desktop/game work blocks FLUX."""
+    source = ROOT / "dockerfiles" / "hermes-local-image-server.py"
+    spec = importlib.util.spec_from_file_location("hermes_local_image_server", source)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24")
+    monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 8.0)
+    monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30)
+
+    idle = module._parse_gpu_activity(
+        '\n'.join(
+            [
+                'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0',
+                'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000',
+                'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3',
+                'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000',
+            ]
+        )
+    )
+    assert idle["interactive_active"] is False
+    assert idle["external_gpu_memory_bytes"] == 1900000000
+    assert idle["external_gpu_sm_percent"] == 3
+
+    active = module._parse_gpu_activity(
+        'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 41\n'
+    )
+    assert active["interactive_active"] is True
+    assert "41%" in active["gpu_guard_reason"]
+
+
 def test_chat_auth_file_mount_survives_atomic_provider_refresh():
     statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
     containers = statefulset["spec"]["template"]["spec"]["containers"]