bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

gpu(titan-24): reduce FLUX peak memory

Browse Source
This commit is contained in:
jenkins 2026-08-11 15:26:08 -03:00
parent 30aeab2bb9
commit d5df6c9fd0
3 changed files with 28 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
dockerfiles/hermes-local-image-server.py
View File

@ -33,6 +33,7 @@ MODEL_REVISION = os.environ.get(
"HERMES_LOCAL_IMAGE_REVISION", "e7b7dc27f91deacad38e78976d1f2b499d76a294"
)
MODEL_CACHE = os.environ.get("HF_HOME", "/models/huggingface")
OFFLOAD_MODE = os.environ.get("HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential")
LEASE_NAMESPACE = os.environ.get("LEASE_NAMESPACE", "hermes")
LEASE_NAME = os.environ.get("LEASE_NAME", "titan-24-gpu-owner")
LEASE_IDLE_OWNER = os.environ.get("LEASE_IDLE_OWNER", "hermes")
@ -279,7 +280,17 @@ def _render_in_process(payload: dict[str, Any]) -> dict[str, Any]:
torch_dtype=torch.bfloat16,
cache_dir=MODEL_CACHE,
)
if OFFLOAD_MODE == "sequential":
# The shared 10 GiB card keeps the desktop and idle Wolf daemon
# resident. Layer-level offload preserves 1024px output quality
# without requiring those low-memory services to be torn down.
pipe.enable_sequential_cpu_offload()
elif OFFLOAD_MODE == "model":
pipe.enable_model_cpu_offload()
else:
raise ValueError("HERMES_LOCAL_IMAGE_OFFLOAD_MODE must be sequential or model")
pipe.enable_vae_slicing()
pipe.enable_vae_tiling()
_set_state(phase="rendering")
arguments: dict[str, Any] = {
"prompt": prompt,

6
services/hermes/local-image-deployment.yaml
View File

@ -21,7 +21,7 @@ spec:
annotations:
ai.bstein.dev/model: black-forest-labs/FLUX.2-klein-4B
ai.bstein.dev/gpu: titan-24 lease-shared image, desktop, and Wolf lane
ai.bstein.dev/config-rev: "20260811-flux2-klein-process-isolation"
ai.bstein.dev/config-rev: "20260811-flux2-klein-low-vram"
spec:
serviceAccountName: hermes-gpu-runtime
runtimeClassName: nvidia
@ -43,7 +43,7 @@ spec:
sizeLimit: 2Gi
containers:
- name: local-image
image: registry.bstein.dev/bstein/hermes-local-image@sha256:b158e33adab1305f93e3857786ca577b9cf3a4540d556c2a847ba9eb411de0dc
image: registry.bstein.dev/bstein/hermes-local-image@sha256:a937f02fe1e85df9d86c0edb1972de6c020fd8ef4f768809624cbebdc521bca1
imagePullPolicy: IfNotPresent
ports:
- name: local-image
@ -80,6 +80,8 @@ spec:
value: "1"
- name: HERMES_LOCAL_IMAGE_RENDER_TIMEOUT
value: "1200"
- name: HERMES_LOCAL_IMAGE_OFFLOAD_MODE
value: sequential
- name: PYTORCH_CUDA_ALLOC_CONF
value: expandable_segments:True
- name: HF_HOME

12
testing/tests/test_hermes_chat_quality.py
View File

@ -625,6 +625,7 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == (
"3221225472"
)
assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential"
assert "nvidia-process-exporter-local.monitoring.svc.cluster.local" in model_env[
"HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL"
]
@ -760,6 +761,17 @@ def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch):
assert options["check"] is False
def test_local_flux_uses_low_vram_offload_without_reducing_resolution():
"""The shared 3080 lane must trade time, not image size, for headroom."""
source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text()
assert 'OFFLOAD_MODE = os.environ.get(' in source
assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source
assert "pipe.enable_sequential_cpu_offload()" in source
assert "pipe.enable_vae_slicing()" in source
assert "pipe.enable_vae_tiling()" in source
assert '"square": (1024, 1024)' in source
def test_chat_auth_file_mount_survives_atomic_provider_refresh():
statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
containers = statefulset["spec"]["template"]["spec"]["containers"]