gpu(titan-24): protect desktop image sharing
This commit is contained in:
jenkins
parent
48aeb4873c
commit
df310df1b5
@ -10,6 +10,7 @@ import json
|
||||
import os
|
||||
import ssl
|
||||
import threading
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
@ -35,6 +36,23 @@ LEASE_IDLE_OWNER = os.environ.get("LEASE_IDLE_OWNER", "hermes")
|
||||
LEASE_IMAGE_OWNER = os.environ.get("LEASE_IMAGE_OWNER", "hermes-image")
|
||||
MAX_BODY_BYTES = int(os.environ.get("HERMES_LOCAL_IMAGE_MAX_BODY", str(96 << 20)))
|
||||
QUEUE_TIMEOUT_SECONDS = float(os.environ.get("HERMES_LOCAL_IMAGE_QUEUE_TIMEOUT", "1200"))
|
||||
GPU_ACTIVITY_URL = os.environ.get(
|
||||
"HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL",
|
||||
"http://nvidia-process-exporter.monitoring.svc.cluster.local:9401/metrics",
|
||||
)
|
||||
GPU_ACTIVITY_NODE = os.environ.get("HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE", "titan-24")
|
||||
GPU_ACTIVE_SM_PERCENT = float(
|
||||
os.environ.get("HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT", "8")
|
||||
)
|
||||
GPU_MAX_EXTERNAL_MEMORY_BYTES = int(
|
||||
os.environ.get("HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES", str(3 << 30))
|
||||
)
|
||||
GPU_ACTIVITY_SAMPLES = max(
|
||||
1, int(os.environ.get("HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLES", "3"))
|
||||
)
|
||||
GPU_ACTIVITY_SAMPLE_INTERVAL = max(
|
||||
0.0, float(os.environ.get("HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLE_INTERVAL", "1"))
|
||||
)
|
||||
API_HOST = os.environ.get("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc")
|
||||
API_PORT = os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS", "443")
|
||||
TOKEN_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/token")
|
||||
@ -56,9 +74,101 @@ _state: dict[str, Any] = {
|
||||
"phase": "idle",
|
||||
"last_error": None,
|
||||
"last_model": None,
|
||||
"interactive_active": None,
|
||||
"external_gpu_memory_bytes": None,
|
||||
"external_gpu_sm_percent": None,
|
||||
"gpu_guard_reason": None,
|
||||
}
|
||||
|
||||
|
||||
def _prometheus_labels(raw: str) -> dict[str, str]:
|
||||
"""Parse the simple quoted labels emitted by nvidia-process-exporter."""
|
||||
labels: dict[str, str] = {}
|
||||
for item in raw.split(","):
|
||||
key, separator, value = item.partition("=")
|
||||
if not separator:
|
||||
continue
|
||||
labels[key.strip()] = value.strip().strip('"')
|
||||
return labels
|
||||
|
||||
|
||||
def _parse_gpu_activity(metrics: str) -> dict[str, Any]:
|
||||
"""Summarize non-Hermes GPU processes on the configured image node."""
|
||||
memory_bytes = 0
|
||||
max_sm_percent = 0.0
|
||||
processes: set[str] = set()
|
||||
for line in metrics.splitlines():
|
||||
if not line or line.startswith("#") or "{" not in line or "}" not in line:
|
||||
continue
|
||||
metric, remainder = line.split("{", 1)
|
||||
raw_labels, separator, raw_value = remainder.partition("}")
|
||||
if not separator or metric not in {
|
||||
"nvidia_process_gpu_memory_used_bytes",
|
||||
"nvidia_process_gpu_sm_util_percent",
|
||||
}:
|
||||
continue
|
||||
labels = _prometheus_labels(raw_labels)
|
||||
if labels.get("node") != GPU_ACTIVITY_NODE or labels.get("namespace") == "hermes":
|
||||
continue
|
||||
try:
|
||||
value = float(raw_value.strip().split()[0])
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
process = labels.get("process") or "unknown"
|
||||
processes.add(f"{labels.get('namespace', 'unknown')}/{process}")
|
||||
if metric == "nvidia_process_gpu_memory_used_bytes":
|
||||
memory_bytes += max(0, int(value))
|
||||
else:
|
||||
max_sm_percent = max(max_sm_percent, value)
|
||||
active = (
|
||||
max_sm_percent >= GPU_ACTIVE_SM_PERCENT
|
||||
or memory_bytes >= GPU_MAX_EXTERNAL_MEMORY_BYTES
|
||||
)
|
||||
reasons = []
|
||||
if max_sm_percent >= GPU_ACTIVE_SM_PERCENT:
|
||||
reasons.append(f"external SM utilization is {max_sm_percent:g}%")
|
||||
if memory_bytes >= GPU_MAX_EXTERNAL_MEMORY_BYTES:
|
||||
reasons.append(f"external GPU memory is {memory_bytes} bytes")
|
||||
return {
|
||||
"interactive_active": active,
|
||||
"external_gpu_memory_bytes": memory_bytes,
|
||||
"external_gpu_sm_percent": max_sm_percent,
|
||||
"external_gpu_processes": sorted(processes),
|
||||
"gpu_guard_reason": "; ".join(reasons) or None,
|
||||
}
|
||||
|
||||
|
||||
def _gpu_activity_snapshot() -> dict[str, Any]:
|
||||
"""Fetch live per-process GPU attribution, failing closed if unavailable."""
|
||||
request = Request(GPU_ACTIVITY_URL, headers={"Accept": "text/plain"})
|
||||
with urlopen(request, timeout=5) as response:
|
||||
metrics = response.read(2 << 20).decode("utf-8", errors="replace")
|
||||
return _parse_gpu_activity(metrics)
|
||||
|
||||
|
||||
def _ensure_interactive_lane_idle() -> dict[str, Any]:
|
||||
"""Reject FLUX while desktop or Wolf activity indicates interactive use."""
|
||||
worst: dict[str, Any] | None = None
|
||||
for sample in range(GPU_ACTIVITY_SAMPLES):
|
||||
snapshot = _gpu_activity_snapshot()
|
||||
if worst is None or (
|
||||
snapshot["external_gpu_sm_percent"], snapshot["external_gpu_memory_bytes"]
|
||||
) > (worst["external_gpu_sm_percent"], worst["external_gpu_memory_bytes"]):
|
||||
worst = snapshot
|
||||
if snapshot["interactive_active"]:
|
||||
break
|
||||
if sample + 1 < GPU_ACTIVITY_SAMPLES:
|
||||
time.sleep(GPU_ACTIVITY_SAMPLE_INTERVAL)
|
||||
assert worst is not None
|
||||
_set_state(**{key: value for key, value in worst.items() if key != "external_gpu_processes"})
|
||||
if worst["interactive_active"]:
|
||||
raise RuntimeError(
|
||||
"titan-24 desktop/Wolf lane is active; reserve or release it from the Atlas GPU checkout"
|
||||
f" ({worst['gpu_guard_reason']})"
|
||||
)
|
||||
return worst
|
||||
|
||||
|
||||
def _set_state(**values: Any) -> None:
|
||||
"""Update health state without exposing prompts or image content."""
|
||||
with _state_lock:
|
||||
@ -203,6 +313,11 @@ def _generate(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
claimed = False
|
||||
_set_state(busy=True, phase="claiming-gpu", last_error=None)
|
||||
try:
|
||||
if _lease_owner() != LEASE_IDLE_OWNER:
|
||||
owner = _lease_owner()
|
||||
raise RuntimeError(f"local image GPU unavailable while titan-24 owner is {owner}")
|
||||
_set_state(phase="checking-interactive-lane")
|
||||
_ensure_interactive_lane_idle()
|
||||
claimed = _change_owner(LEASE_IDLE_OWNER, LEASE_IMAGE_OWNER)
|
||||
if not claimed:
|
||||
owner = _lease_owner()
|
||||
@ -252,11 +367,23 @@ class Handler(BaseHTTPRequestHandler):
|
||||
owner = _lease_owner()
|
||||
except Exception:
|
||||
owner = "unavailable"
|
||||
try:
|
||||
activity = _gpu_activity_snapshot()
|
||||
_set_state(**{key: value for key, value in activity.items() if key != "external_gpu_processes"})
|
||||
state = _state_snapshot()
|
||||
except Exception as exc:
|
||||
activity = {"interactive_active": True, "gpu_guard_reason": f"GPU guard unavailable: {exc}"}
|
||||
_set_state(**activity)
|
||||
state = _state_snapshot()
|
||||
self._json(
|
||||
200,
|
||||
{
|
||||
"success": True,
|
||||
"available": owner == LEASE_IDLE_OWNER and not state["busy"],
|
||||
"available": (
|
||||
owner == LEASE_IDLE_OWNER
|
||||
and not state["busy"]
|
||||
and not activity["interactive_active"]
|
||||
),
|
||||
"gpu_owner": owner,
|
||||
"model": "flux-2-klein-4b-local",
|
||||
**state,
|
||||
|
||||
@ -20,8 +20,8 @@ spec:
|
||||
app: hermes-local-image
|
||||
annotations:
|
||||
ai.bstein.dev/model: black-forest-labs/FLUX.2-klein-4B
|
||||
ai.bstein.dev/gpu: titan-24 lease-shared image and Wolf lane
|
||||
ai.bstein.dev/config-rev: "20260811-flux2-klein-image-only"
|
||||
ai.bstein.dev/gpu: titan-24 lease-shared image, desktop, and Wolf lane
|
||||
ai.bstein.dev/config-rev: "20260811-flux2-klein-interactive-guard"
|
||||
spec:
|
||||
serviceAccountName: hermes-gpu-runtime
|
||||
runtimeClassName: nvidia
|
||||
@ -43,7 +43,7 @@ spec:
|
||||
sizeLimit: 2Gi
|
||||
containers:
|
||||
- name: local-image
|
||||
image: registry.bstein.dev/bstein/hermes-local-image@sha256:316d123cc48509a53fb32c26bfb384a3ed4d934cb334d73faa0840a62e047e65
|
||||
image: registry.bstein.dev/bstein/hermes-local-image@sha256:769a16f2b56b1de401f7dc4adafb522d39c8509d8de741668f69ee7a206502f7
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: local-image
|
||||
@ -63,6 +63,20 @@ spec:
|
||||
value: hermes
|
||||
- name: LEASE_IMAGE_OWNER
|
||||
value: hermes-image
|
||||
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL
|
||||
value: http://nvidia-process-exporter.monitoring.svc.cluster.local:9401/metrics
|
||||
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE
|
||||
value: titan-24
|
||||
# The idle desktop stack and Wolf daemon use about 1.9 GiB and
|
||||
# 3% SM. Active desktop/game sessions cross one of these bounds.
|
||||
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT
|
||||
value: "8"
|
||||
- name: HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES
|
||||
value: "3221225472"
|
||||
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLES
|
||||
value: "3"
|
||||
- name: HERMES_LOCAL_IMAGE_GPU_ACTIVITY_SAMPLE_INTERVAL
|
||||
value: "1"
|
||||
- name: HF_HOME
|
||||
value: /models/huggingface
|
||||
- name: HOME
|
||||
|
||||
@ -608,6 +608,14 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
|
||||
assert model_env["HERMES_LOCAL_IMAGE_REVISION"] == (
|
||||
"e7b7dc27f91deacad38e78976d1f2b499d76a294"
|
||||
)
|
||||
assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVITY_NODE"] == "titan-24"
|
||||
assert model_env["HERMES_LOCAL_IMAGE_GPU_ACTIVE_SM_PERCENT"] == "8"
|
||||
assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == (
|
||||
"3221225472"
|
||||
)
|
||||
assert "nvidia-process-exporter.monitoring.svc.cluster.local" in model_env[
|
||||
"HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL"
|
||||
]
|
||||
models_volume = next(item for item in pod["volumes"] if item["name"] == "models")
|
||||
assert models_volume["persistentVolumeClaim"]["claimName"] == (
|
||||
"hermes-image-models"
|
||||
@ -672,6 +680,38 @@ def test_titan20_serializes_classifier_and_local_chat_model_residency():
|
||||
)
|
||||
|
||||
|
||||
def test_local_image_gpu_guard_distinguishes_idle_and_active_desktop(monkeypatch):
|
||||
"""Idle display shells may coexist, but active desktop/game work blocks FLUX."""
|
||||
source = ROOT / "dockerfiles" / "hermes-local-image-server.py"
|
||||
spec = importlib.util.spec_from_file_location("hermes_local_image_server", source)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
monkeypatch.setattr(module, "GPU_ACTIVITY_NODE", "titan-24")
|
||||
monkeypatch.setattr(module, "GPU_ACTIVE_SM_PERCENT", 8.0)
|
||||
monkeypatch.setattr(module, "GPU_MAX_EXTERNAL_MEMORY_BYTES", 3 << 30)
|
||||
|
||||
idle = module._parse_gpu_activity(
|
||||
'\n'.join(
|
||||
[
|
||||
'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="Xorg"} 0',
|
||||
'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="host",process="Xorg"} 1900000000',
|
||||
'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="game-stream",process="wolf"} 3',
|
||||
'nvidia_process_gpu_memory_used_bytes{node="titan-24",namespace="hermes",process="python"} 9000000000',
|
||||
]
|
||||
)
|
||||
)
|
||||
assert idle["interactive_active"] is False
|
||||
assert idle["external_gpu_memory_bytes"] == 1900000000
|
||||
assert idle["external_gpu_sm_percent"] == 3
|
||||
|
||||
active = module._parse_gpu_activity(
|
||||
'nvidia_process_gpu_sm_util_percent{node="titan-24",namespace="host",process="steam"} 41\n'
|
||||
)
|
||||
assert active["interactive_active"] is True
|
||||
assert "41%" in active["gpu_guard_reason"]
|
||||
|
||||
|
||||
def test_chat_auth_file_mount_survives_atomic_provider_refresh():
|
||||
statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
|
||||
containers = statefulset["spec"]["template"]["spec"]["containers"]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user