From d5df6c9fd0834f7c38ac1b0c2c457a0c9464c44e Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 11 Aug 2026 15:26:08 -0300 Subject: [PATCH] gpu(titan-24): reduce FLUX peak memory --- dockerfiles/hermes-local-image-server.py | 13 ++++++++++++- services/hermes/local-image-deployment.yaml | 6 ++++-- testing/tests/test_hermes_chat_quality.py | 12 ++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/dockerfiles/hermes-local-image-server.py b/dockerfiles/hermes-local-image-server.py index f361254d8..d9555ec24 100644 --- a/dockerfiles/hermes-local-image-server.py +++ b/dockerfiles/hermes-local-image-server.py @@ -33,6 +33,7 @@ MODEL_REVISION = os.environ.get( "HERMES_LOCAL_IMAGE_REVISION", "e7b7dc27f91deacad38e78976d1f2b499d76a294" ) MODEL_CACHE = os.environ.get("HF_HOME", "/models/huggingface") +OFFLOAD_MODE = os.environ.get("HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential") LEASE_NAMESPACE = os.environ.get("LEASE_NAMESPACE", "hermes") LEASE_NAME = os.environ.get("LEASE_NAME", "titan-24-gpu-owner") LEASE_IDLE_OWNER = os.environ.get("LEASE_IDLE_OWNER", "hermes") @@ -279,7 +280,17 @@ def _render_in_process(payload: dict[str, Any]) -> dict[str, Any]: torch_dtype=torch.bfloat16, cache_dir=MODEL_CACHE, ) - pipe.enable_model_cpu_offload() + if OFFLOAD_MODE == "sequential": + # The shared 10 GiB card keeps the desktop and idle Wolf daemon + # resident. Layer-level offload preserves 1024px output quality + # without requiring those low-memory services to be torn down. + pipe.enable_sequential_cpu_offload() + elif OFFLOAD_MODE == "model": + pipe.enable_model_cpu_offload() + else: + raise ValueError("HERMES_LOCAL_IMAGE_OFFLOAD_MODE must be sequential or model") + pipe.enable_vae_slicing() + pipe.enable_vae_tiling() _set_state(phase="rendering") arguments: dict[str, Any] = { "prompt": prompt, diff --git a/services/hermes/local-image-deployment.yaml b/services/hermes/local-image-deployment.yaml index b3a9f7263..1167eee95 100644 --- a/services/hermes/local-image-deployment.yaml +++ b/services/hermes/local-image-deployment.yaml @@ -21,7 +21,7 @@ spec: annotations: ai.bstein.dev/model: black-forest-labs/FLUX.2-klein-4B ai.bstein.dev/gpu: titan-24 lease-shared image, desktop, and Wolf lane - ai.bstein.dev/config-rev: "20260811-flux2-klein-process-isolation" + ai.bstein.dev/config-rev: "20260811-flux2-klein-low-vram" spec: serviceAccountName: hermes-gpu-runtime runtimeClassName: nvidia @@ -43,7 +43,7 @@ spec: sizeLimit: 2Gi containers: - name: local-image - image: registry.bstein.dev/bstein/hermes-local-image@sha256:b158e33adab1305f93e3857786ca577b9cf3a4540d556c2a847ba9eb411de0dc + image: registry.bstein.dev/bstein/hermes-local-image@sha256:a937f02fe1e85df9d86c0edb1972de6c020fd8ef4f768809624cbebdc521bca1 imagePullPolicy: IfNotPresent ports: - name: local-image @@ -80,6 +80,8 @@ spec: value: "1" - name: HERMES_LOCAL_IMAGE_RENDER_TIMEOUT value: "1200" + - name: HERMES_LOCAL_IMAGE_OFFLOAD_MODE + value: sequential - name: PYTORCH_CUDA_ALLOC_CONF value: expandable_segments:True - name: HF_HOME diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index fc290ea2a..3d1a7d885 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -625,6 +625,7 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): assert model_env["HERMES_LOCAL_IMAGE_GPU_MAX_EXTERNAL_MEMORY_BYTES"] == ( "3221225472" ) + assert model_env["HERMES_LOCAL_IMAGE_OFFLOAD_MODE"] == "sequential" assert "nvidia-process-exporter-local.monitoring.svc.cluster.local" in model_env[ "HERMES_LOCAL_IMAGE_GPU_ACTIVITY_URL" ] @@ -760,6 +761,17 @@ def test_local_flux_renderer_uses_a_disposable_cuda_worker(monkeypatch): assert options["check"] is False +def test_local_flux_uses_low_vram_offload_without_reducing_resolution(): + """The shared 3080 lane must trade time, not image size, for headroom.""" + source = (ROOT / "dockerfiles" / "hermes-local-image-server.py").read_text() + assert 'OFFLOAD_MODE = os.environ.get(' in source + assert '"HERMES_LOCAL_IMAGE_OFFLOAD_MODE", "sequential"' in source + assert "pipe.enable_sequential_cpu_offload()" in source + assert "pipe.enable_vae_slicing()" in source + assert "pipe.enable_vae_tiling()" in source + assert '"square": (1024, 1024)' in source + + def test_chat_auth_file_mount_survives_atomic_provider_refresh(): statefulset = _documents(HERMES / "chat-statefulset.yaml")[0] containers = statefulset["spec"]["template"]["spec"]["containers"]