ai: persist titan-20 model cache

services/ai-llm/deployment.yaml

@@ -36,8 +36,10 @@ spec:
       runtimeClassName: nvidia
       volumes:
         - name: models
-          # Accelerator nodes are not Longhorn attach targets; keep the model cache pod-local.
-          emptyDir: {}
+          # The pod is pinned to titan-20. A local-path claim preserves the
+          # 10 GiB model cache across rollouts without involving Longhorn.
+          persistentVolumeClaim:
+            claimName: ollama-models-titan20
       initContainers:
         - name: warm-model
           image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d
@@ -127,7 +129,10 @@ spec:
               pid="$!"
               trap 'kill -TERM "$pid"; wait "$pid"' TERM INT
               sleep 6
-              timeout 180s ollama run "${OLLAMA_FAST_MODEL}" --keepalive -1 "reply with just pong" >/tmp/ollama-fast-warm.log 2>&1
+              if ! timeout 180s ollama run "${OLLAMA_FAST_MODEL}" --keepalive=-1 "reply with just pong" >/tmp/ollama-fast-warm.log 2>&1; then
+                cat /tmp/ollama-fast-warm.log >&2
+                exit 1
+              fi
               touch /tmp/ollama-fast-ready
               wait "$pid"
           volumeMounts:

services/ai-llm/pvc.yaml

@@ -11,3 +11,16 @@ spec:
     requests:
       storage: 30Gi
   storageClassName: asteria
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ollama-models-titan20
+  namespace: ai
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 30Gi
+  storageClassName: local-path

testing/tests/test_hermes_chat_quality.py

@@ -663,6 +663,10 @@ def test_titan20_serializes_classifier_and_local_chat_model_residency():
     assert env["OLLAMA_NUM_PARALLEL"] == "1"
     assert env["OLLAMA_KEEP_ALIVE"] == "-1"
     assert env["OLLAMA_CONTEXT_LENGTH"] == "8192"
+    models = next(item for item in pod["volumes"] if item["name"] == "models")
+    assert models["persistentVolumeClaim"]["claimName"] == (
+        "ollama-models-titan20"
+    )
 
 
 def test_chat_auth_file_mount_survives_atomic_provider_refresh():